diff --git a/docs/security/payload-sanitization.md b/docs/security/payload-sanitization.md new file mode 100644 index 0000000000000..940e2a247946a --- /dev/null +++ b/docs/security/payload-sanitization.md @@ -0,0 +1,242 @@ +# Payload Sanitization Security Guide + +## Overview + +The payload sanitization system protects against CWE-532 vulnerabilities (Sensitive Data Exposure in Logs) by automatically detecting and redacting sensitive information before it's written to log files. + +## Configuration + +### Environment Variables + +| Variable | Options | Default | Description | +|----------|---------|---------|-------------| +| `OPENCLAW_ANTHROPIC_PAYLOAD_LOG` | `true`/`false` | `false` | Enable/disable payload logging | +| `PAYLOAD_LOG_SANITIZATION_LEVEL` | `none`/`minimal`/`standard`/`paranoid` | `standard` | Sanitization strictness | +| `PAYLOAD_LOG_MAX_CONTENT_LENGTH` | number | `10000` | Max content length before truncation | +| `PAYLOAD_LOG_INCLUDE_HINTS` | `true`/`false` | `true` | Include redaction type hints | +| `PAYLOAD_LOG_CUSTOM_FIELDS` | comma-separated | - | Additional sensitive field names | + +### Sanitization Levels + +#### `none` (⚠️ DANGEROUS) +- **No sanitization performed** +- Blocked in production environments +- Only for isolated debugging +- **WARNING**: Exposes sensitive data in plaintext + +#### `minimal` +- Detects obvious secrets (API keys, tokens) +- Suitable for controlled environments +- Fast processing + +#### `standard` (Recommended) +- Detects secrets + PII + sensitive fields +- Balances security and functionality +- Default for production + +#### `paranoid` +- Maximum protection +- Truncates system prompts aggressively +- May over-redact legitimate data + +## What Gets Sanitized + +### API Keys and Secrets +- Anthropic API keys (`sk-ant-...`) +- OpenAI API keys (`sk-...`, `sk-proj-...`) +- GitHub tokens (`ghp_...`, `github_pat_...`) +- AWS access keys (`AKIA...`) +- JWT tokens +- Bearer/Basic auth headers +- Private key blocks +- Generic API key patterns + +### Personal Information (PII) +- Social Security Numbers (`123-45-6789`) +- Credit card numbers (Visa, MC, Amex, Discover) +- US phone numbers +- Email addresses +- IP addresses (paranoid level only) + +### Sensitive Fields +Field names are matched case-insensitively with various naming conventions: +- `password`, `passwd`, `pwd` +- `secret`, `apiKey`, `api_key`, `token` +- `authorization`, `bearer`, `credentials` +- `sessionKey`, `session_token`, `cookie` +- `privateKey`, `private_key` +- And many more... + +## Security Features + +### Production Protection +- Forces minimum `STANDARD` level in production +- Prevents accidental exposure via environment override +- Displays clear warnings for risky configurations + +### Defense in Depth +1. **Field-level filtering**: Removes entire sensitive fields +2. **Pattern matching**: Regex-based secret detection +3. **Content truncation**: Limits payload size +4. **Safe error handling**: Fails securely if sanitization errors occur + +### Audit Trail +Each log entry includes sanitization metadata: +- `sanitizationLevel`: Applied protection level +- `redactionCount`: Number of items redacted +- `detectedSensitiveTypes`: Types of sensitive data found +- `sanitizationWarnings`: Any issues during processing + +## Example Usage + +### Basic Setup +```bash +# Enable logging with standard protection +export OPENCLAW_ANTHROPIC_PAYLOAD_LOG=true +export PAYLOAD_LOG_SANITIZATION_LEVEL=standard +``` + +### High-Security Environment +```bash +# Paranoid mode with custom sensitive fields +export OPENCLAW_ANTHROPIC_PAYLOAD_LOG=true +export PAYLOAD_LOG_SANITIZATION_LEVEL=paranoid +export PAYLOAD_LOG_CUSTOM_FIELDS="customer_id,internal_token,proprietary_key" +export PAYLOAD_LOG_MAX_CONTENT_LENGTH=5000 +``` + +### Development/Debug Mode +```bash +# Minimal sanitization for faster processing +export NODE_ENV=development +export OPENCLAW_ANTHROPIC_PAYLOAD_LOG=true +export PAYLOAD_LOG_SANITIZATION_LEVEL=minimal +``` + +## Log Output Example + +### Before Sanitization +```json +{ + "payload": { + "apiKey": "sk-ant-api03-abcdefghij1234567890", + "userQuery": "My SSN is 123-45-6789", + "systemPrompt": "Use API key sk-proj-xyz123 for calls" + } +} +``` + +### After Sanitization +```json +{ + "sanitizationLevel": "standard", + "redactionCount": 3, + "detectedSensitiveTypes": ["ANTHROPIC_API_KEY", "SSN", "OPENAI_PROJECT_KEY"], + "payload": { + "apiKey": "[REDACTED:SENSITIVE_FIELD]", + "userQuery": "My SSN is [REDACTED:SSN]", + "systemPrompt": "Use API key [REDACTED:OPENAI_PROJECT_KEY] for calls" + }, + "sanitizationWarnings": ["Redacted 3 instances of sensitive data"] +} +``` + +## Best Practices + +### For Production +1. Always use `STANDARD` or `PARANOID` level +2. Secure log file permissions (`600` or `640`) +3. Implement log rotation to prevent disk exhaustion +4. Restrict access to log files (audit who can read them) +5. Never transmit logs over insecure channels + +### For Development +1. Use `MINIMAL` level for faster iteration +2. Review logs periodically for false positives +3. Add custom patterns for domain-specific secrets +4. Test sanitization with realistic data + +### General Guidelines +1. **Defense in depth**: Don't rely solely on sanitization +2. **Regular audits**: Review what's being logged +3. **Incident response**: Know how to secure logs if compromised +4. **Documentation**: Keep security configurations documented + +## Limitations + +### What Sanitization Cannot Prevent +- Novel secret formats not covered by patterns +- Business logic secrets embedded in natural language +- Inference attacks on redacted data +- Accidental exposure if sanitization is disabled + +### Performance Considerations +- Regex matching adds processing overhead +- Large payloads take longer to sanitize +- `PARANOID` level has highest performance impact + +### False Positives/Negatives +- May redact legitimate UUIDs or tokens (false positives) +- May miss custom secret formats (false negatives) +- Pattern maintenance required for new secret types + +## Troubleshooting + +### Common Issues + +#### High Redaction Count +```bash +# Check what's being detected +grep "detectedSensitiveTypes" anthropic-payload.jsonl | head -5 +``` + +#### Performance Problems +```bash +# Reduce sanitization level or content length +export PAYLOAD_LOG_SANITIZATION_LEVEL=minimal +export PAYLOAD_LOG_MAX_CONTENT_LENGTH=5000 +``` + +#### Missing Custom Secrets +```bash +# Add custom patterns via environment +export PAYLOAD_LOG_CUSTOM_FIELDS="my_secret_field,internal_key" +``` + +#### Production Override Errors +Production environments automatically upgrade `none`/`minimal` to `standard`. This is intentional security behavior. + +### Emergency Procedures + +#### Suspected Data Exposure +1. Immediately disable logging: `OPENCLAW_ANTHROPIC_PAYLOAD_LOG=false` +2. Secure existing log files (move to restricted location) +3. Audit log contents for actual exposure +4. Rotate any potentially compromised secrets +5. Document incident for security review + +#### Log File Compromise +1. Assume all logged data is compromised +2. Rotate all API keys/tokens from the time period +3. Review access patterns for unauthorized activity +4. Implement additional monitoring + +## Contributing + +### Adding New Secret Patterns +1. Update `SECRET_PATTERNS` in `src/agents/payload-sanitizer.ts` +2. Include pattern name, regex, and minimum level +3. Add test cases in `src/agents/payload-sanitizer.test.ts` +4. Consider false positive rate + +### Testing Changes +```bash +# Run sanitization tests +npm test -- --testNamePattern="PayloadSanitizer" + +# Test with sample sensitive data +echo '{"key": "sk-ant-test123"}' | node -e " +const {sanitizePayload} = require('./dist/agents/payload-sanitizer.js'); +console.log(JSON.stringify(sanitizePayload(JSON.parse(require('fs').readFileSync(0))), null, 2)); +" +``` \ No newline at end of file diff --git a/src/agents/anthropic-payload-log.ts b/src/agents/anthropic-payload-log.ts index fbc0f254e726c..fbe2519618a78 100644 --- a/src/agents/anthropic-payload-log.ts +++ b/src/agents/anthropic-payload-log.ts @@ -1,3 +1,29 @@ +/** + * @fileoverview Anthropic API Payload Logging with Security Sanitization + * + * This module provides logging functionality for Anthropic API payloads, + * with comprehensive security sanitization to prevent CWE-532 vulnerabilities + * (Insertion of Sensitive Information into Log File). + * + * SECURITY FEATURES: + * - Automatic sanitization of API keys, secrets, and tokens + * - PII detection and redaction (SSN, credit cards, phone numbers) + * - Sensitive field filtering (passwords, credentials, session keys) + * - Content truncation to prevent log bloat + * - Production environment protection (forces minimum sanitization level) + * + * CONFIGURATION: + * - OPENCLAW_ANTHROPIC_PAYLOAD_LOG=true/false - Enable/disable logging + * - PAYLOAD_LOG_SANITIZATION_LEVEL=none|minimal|standard|paranoid + * - PAYLOAD_LOG_MAX_CONTENT_LENGTH=10000 - Max content length + * - NODE_ENV=production - Forces minimum STANDARD sanitization + * + * WARNING: Setting PAYLOAD_LOG_SANITIZATION_LEVEL=none in production is + * blocked by default. This module is designed to be secure by default. + * + * @see https://cwe.mitre.org/data/definitions/532.html + */ + import type { AgentMessage, StreamFn } from "@mariozechner/pi-agent-core"; import type { Api, Model } from "@mariozechner/pi-ai"; import crypto from "node:crypto"; @@ -7,6 +33,12 @@ import { resolveStateDir } from "../config/paths.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { resolveUserPath } from "../utils.js"; import { parseBooleanValue } from "../utils/boolean.js"; +import { + PayloadSanitizer, + SanitizationLevel, + type SanitizationResult, + buildConfigFromEnv +} from "./payload-sanitizer.js"; type PayloadLogStage = "request" | "usage"; @@ -15,20 +47,26 @@ type PayloadLogEvent = { stage: PayloadLogStage; runId?: string; sessionId?: string; - sessionKey?: string; + sessionKey?: string; // SECURITY NOTE: This field is now sanitized to remove auth tokens provider?: string; modelId?: string; modelApi?: string | null; workspaceDir?: string; - payload?: unknown; + payload?: unknown; // SECURITY NOTE: This field is now sanitized before logging usage?: Record; error?: string; payloadDigest?: string; + // New security fields + sanitizationLevel?: SanitizationLevel; + redactionCount?: number; + detectedSensitiveTypes?: string[]; + sanitizationWarnings?: string[]; }; type PayloadLogConfig = { enabled: boolean; filePath: string; + sanitizer: PayloadSanitizer; }; type PayloadLogWriter = { @@ -39,13 +77,65 @@ type PayloadLogWriter = { const writers = new Map(); const log = createSubsystemLogger("agent/anthropic-payload"); +/** + * Security warning banner displayed when payload logging is enabled. + */ +const SECURITY_WARNING = ` +================================================================================ +⚠️ ANTHROPIC PAYLOAD LOGGING IS ENABLED +================================================================================ +Payload logging is active. While this system sanitizes sensitive data, +logging API payloads can still pose security risks: + +1. Sanitization is not perfect - novel secret formats may not be detected +2. Business-sensitive information may still be logged +3. Large payloads can consume significant disk space +4. Log files should be secured and rotated appropriately + +Current sanitization level: %LEVEL% + +For production use, ensure: +- Log files are stored securely with appropriate permissions +- Log rotation is configured to prevent disk exhaustion +- Access to log files is restricted and audited +- Logs are not transmitted to insecure locations + +To disable logging: Set OPENCLAW_ANTHROPIC_PAYLOAD_LOG=false +================================================================================ +`; + function resolvePayloadLogConfig(env: NodeJS.ProcessEnv): PayloadLogConfig { const enabled = parseBooleanValue(env.OPENCLAW_ANTHROPIC_PAYLOAD_LOG) ?? false; const fileOverride = env.OPENCLAW_ANTHROPIC_PAYLOAD_LOG_FILE?.trim(); const filePath = fileOverride ? resolveUserPath(fileOverride) : path.join(resolveStateDir(env), "logs", "anthropic-payload.jsonl"); - return { enabled, filePath }; + + // Initialize sanitizer with environment-based configuration + const sanitizer = new PayloadSanitizer(buildConfigFromEnv()); + + // Display security warning when logging is enabled + if (enabled) { + const warning = SECURITY_WARNING.replace('%LEVEL%', sanitizer.getLevel()); + log.warn(warning); + + // Extra warning for risky sanitization levels + if (sanitizer.getLevel() === SanitizationLevel.NONE) { + log.error( + 'CRITICAL SECURITY WARNING: Sanitization is DISABLED! ' + + 'Sensitive data WILL be logged in plaintext. ' + + 'This is EXTREMELY DANGEROUS and should NEVER be used in production.' + ); + } else if (sanitizer.getLevel() === SanitizationLevel.MINIMAL) { + log.warn( + 'WARNING: Using MINIMAL sanitization level. ' + + 'PII and some sensitive data may still be logged. ' + + 'Consider using STANDARD or PARANOID level for better protection.' + ); + } + } + + return { enabled, filePath, sanitizer }; } function getWriter(filePath: string): PayloadLogWriter { @@ -110,6 +200,30 @@ function formatError(error: unknown): string | undefined { return undefined; } +/** + * Generate a safe session identifier for logging. + * + * SECURITY NOTE: This function extracts ONLY a session identifier, + * NOT authentication tokens or credentials. If the sessionKey contains + * sensitive auth data, it is sanitized/redacted. + */ +function getSafeSessionKey(sessionKey: string | undefined, sanitizer: PayloadSanitizer): string | undefined { + if (!sessionKey) { + return undefined; + } + + // Sanitize the session key to remove any embedded auth tokens + const result = sanitizer.sanitize(sessionKey); + const sanitized = result.sanitized as string; + + // If the sanitized result is different, it contained sensitive data + if (sanitized !== sessionKey && result.redactionCount > 0) { + log.warn(`Session key contained sensitive data and was sanitized (${result.detectedTypes.join(', ')})`); + } + + return sanitized; +} + function digest(value: unknown): string | undefined { const serialized = safeJsonStringify(value); if (!serialized) { @@ -155,10 +269,14 @@ export function createAnthropicPayloadLogger(params: { } const writer = getWriter(cfg.filePath); + + // SECURITY: Sanitize sessionKey to remove any embedded auth tokens + const safeSessionKey = getSafeSessionKey(params.sessionKey, cfg.sanitizer); + const base: Omit = { runId: params.runId, sessionId: params.sessionId, - sessionKey: params.sessionKey, + sessionKey: safeSessionKey, // Now sanitized provider: params.provider, modelId: params.modelId, modelApi: params.modelApi, @@ -179,12 +297,19 @@ export function createAnthropicPayloadLogger(params: { return streamFn(model, context, options); } const nextOnPayload = (payload: unknown) => { + // SECURITY: Sanitize payload before logging + const sanitizationResult = cfg.sanitizer.sanitize(payload); + record({ ...base, ts: new Date().toISOString(), stage: "request", - payload, - payloadDigest: digest(payload), + payload: sanitizationResult.sanitized, + payloadDigest: digest(sanitizationResult.sanitized), // Digest sanitized payload, not original + sanitizationLevel: cfg.sanitizer.getLevel(), + redactionCount: sanitizationResult.redactionCount, + detectedSensitiveTypes: sanitizationResult.detectedTypes, + sanitizationWarnings: sanitizationResult.warnings.length > 0 ? sanitizationResult.warnings : undefined, }); options?.onPayload?.(payload); }; @@ -199,28 +324,57 @@ export function createAnthropicPayloadLogger(params: { const recordUsage: AnthropicPayloadLogger["recordUsage"] = (messages, error) => { const usage = findLastAssistantUsage(messages); const errorMessage = formatError(error); - if (!usage) { - if (errorMessage) { - record({ - ...base, - ts: new Date().toISOString(), - stage: "usage", - error: errorMessage, - }); - } + + // SECURITY: Sanitize usage data and error message + let sanitizedUsage: unknown = undefined; + let usageSanitizationResult: SanitizationResult | undefined; + + if (usage) { + usageSanitizationResult = cfg.sanitizer.sanitize(usage); + sanitizedUsage = usageSanitizationResult.sanitized; + } + + let sanitizedError: string | undefined = errorMessage; + let errorSanitizationResult: SanitizationResult | undefined; + + if (errorMessage) { + errorSanitizationResult = cfg.sanitizer.sanitize(errorMessage); + sanitizedError = errorSanitizationResult.sanitized as string; + } + + if (!usage && !errorMessage) { return; } + + // Combine sanitization results + const totalRedactionCount = (usageSanitizationResult?.redactionCount || 0) + + (errorSanitizationResult?.redactionCount || 0); + const allDetectedTypes = [ + ...(usageSanitizationResult?.detectedTypes || []), + ...(errorSanitizationResult?.detectedTypes || []) + ]; + const allWarnings = [ + ...(usageSanitizationResult?.warnings || []), + ...(errorSanitizationResult?.warnings || []) + ]; + record({ ...base, ts: new Date().toISOString(), stage: "usage", - usage, - error: errorMessage, + usage: sanitizedUsage, + error: sanitizedError, + sanitizationLevel: cfg.sanitizer.getLevel(), + redactionCount: totalRedactionCount, + detectedSensitiveTypes: allDetectedTypes.length > 0 ? allDetectedTypes : undefined, + sanitizationWarnings: allWarnings.length > 0 ? allWarnings : undefined, }); + + // SECURITY: Also sanitize the separate info log log.info("anthropic usage", { runId: params.runId, sessionId: params.sessionId, - usage, + usage: sanitizedUsage, // Use sanitized usage data here too }); }; diff --git a/src/agents/payload-sanitizer.test.ts b/src/agents/payload-sanitizer.test.ts new file mode 100644 index 0000000000000..84c03ab02fe7d --- /dev/null +++ b/src/agents/payload-sanitizer.test.ts @@ -0,0 +1,313 @@ +/** + * @fileoverview Tests for PayloadSanitizer + * + * These tests verify that the sanitizer correctly identifies and redacts + * various types of sensitive data while preserving safe content. + */ + +import { + PayloadSanitizer, + SanitizationLevel, + sanitizePayload, + sanitizePayloadFull, + resetGlobalSanitizer, + SECRET_PATTERNS, + SENSITIVE_FIELD_NAMES, + buildConfigFromEnv, +} from './payload-sanitizer.js'; + +describe('PayloadSanitizer', () => { + beforeEach(() => { + // Reset environment and global state before each test + delete process.env.PAYLOAD_LOG_SANITIZATION_LEVEL; + delete process.env.NODE_ENV; + resetGlobalSanitizer(); + }); + + describe('API Key Detection', () => { + const sanitizer = new PayloadSanitizer({ level: SanitizationLevel.MINIMAL }); + + test('detects Anthropic API keys', () => { + const input = 'My key is sk-ant-api03-abcdefghijklmnopqrstuvwxyz'; + const result = sanitizer.sanitize(input); + expect(result.sanitized).not.toContain('sk-ant-'); + expect(result.detectedTypes).toContain('ANTHROPIC_API_KEY'); + expect(result.redactionCount).toBeGreaterThan(0); + }); + + test('detects OpenAI API keys', () => { + const input = 'Use this key: sk-proj-abcdefghijklmnopqrstuvwxyz12345'; + const result = sanitizer.sanitize(input); + expect(result.sanitized).not.toContain('sk-proj-'); + expect(result.redactionCount).toBeGreaterThan(0); + }); + + test('detects GitHub tokens', () => { + const inputs = [ + 'ghp_abcdefghijklmnopqrstuvwxyz1234567890', + 'github_pat_abcdefghijklmnopqrstuvwxyz', + 'gho_abcdefghijklmnopqrstuvwxyz1234567890', + ]; + + inputs.forEach(input => { + const result = sanitizer.sanitize(input); + expect(result.redactionCount).toBeGreaterThan(0); + }); + }); + + test('detects Bearer tokens', () => { + const input = 'Authorization: Bearer abc123xyz789token'; + const result = sanitizer.sanitize(input); + expect(result.sanitized).toContain('[REDACTED:BEARER_TOKEN]'); + }); + + test('detects JWT tokens', () => { + const jwt = 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U'; + const result = sanitizer.sanitize(jwt); + expect(result.sanitized).not.toContain('eyJ'); + expect(result.detectedTypes).toContain('JWT_TOKEN'); + }); + }); + + describe('PII Detection (STANDARD level)', () => { + const sanitizer = new PayloadSanitizer({ level: SanitizationLevel.STANDARD }); + + test('detects Social Security Numbers', () => { + const input = 'My SSN is 123-45-6789'; + const result = sanitizer.sanitize(input); + expect(result.sanitized).not.toContain('123-45-6789'); + expect(result.detectedTypes).toContain('SSN'); + }); + + test('detects credit card numbers', () => { + const inputs = [ + '4111111111111111', // Visa + '5500000000000004', // Mastercard + '340000000000009', // Amex + ]; + + inputs.forEach(input => { + const result = sanitizer.sanitize(`Card: ${input}`); + expect(result.redactionCount).toBeGreaterThan(0); + }); + }); + + test('detects email addresses', () => { + const input = 'Contact me at john.doe@example.com for details'; + const result = sanitizer.sanitize(input); + expect(result.sanitized).not.toContain('john.doe@example.com'); + expect(result.detectedTypes).toContain('EMAIL'); + }); + }); + + describe('Sensitive Field Filtering', () => { + const sanitizer = new PayloadSanitizer({ level: SanitizationLevel.STANDARD }); + + test('redacts password fields', () => { + const input = { + username: 'john', + password: 'supersecret123', + }; + const result = sanitizer.sanitize(input); + const sanitized = result.sanitized as Record; + + expect(sanitized.username).toBe('john'); + expect(String(sanitized.password)).toContain('[REDACTED'); + }); + + test('redacts API key fields with various naming conventions', () => { + const input = { + apiKey: 'key1', + api_key: 'key2', + 'api-key': 'key3', + APIKey: 'key4', + }; + const result = sanitizer.sanitize(input); + const sanitized = result.sanitized as Record; + + Object.values(sanitized).forEach(value => { + expect(String(value)).toContain('[REDACTED'); + }); + }); + + test('redacts nested sensitive fields', () => { + const input = { + user: { + profile: { + password: 'secret', + name: 'John', + }, + }, + }; + const result = sanitizer.sanitize(input); + const sanitized = result.sanitized as any; + + expect(String(sanitized.user.profile.password)).toContain('[REDACTED'); + expect(sanitized.user.profile.name).toBe('John'); + }); + + test('redacts sessionKey fields', () => { + const input = { + sessionKey: 'sensitive-session-data', + session_token: 'token123', + data: 'safe', + }; + const result = sanitizer.sanitize(input); + const sanitized = result.sanitized as any; + + expect(String(sanitized.sessionKey)).toContain('[REDACTED'); + expect(String(sanitized.session_token)).toContain('[REDACTED'); + expect(sanitized.data).toBe('safe'); + }); + }); + + describe('Content Truncation', () => { + test('truncates long strings', () => { + const sanitizer = new PayloadSanitizer({ + level: SanitizationLevel.STANDARD, + maxContentLength: 100, + }); + + const longString = 'a'.repeat(200); + const result = sanitizer.sanitize(longString); + + expect((result.sanitized as string).length).toBeLessThan(200); + expect(result.wasTruncated).toBe(true); + }); + }); + + describe('Sanitization Levels', () => { + test('NONE level does not sanitize (non-production)', () => { + // Force non-production for this test + const sanitizer = new PayloadSanitizer({ + level: SanitizationLevel.NONE, + isProduction: false, + }); + + const input = { password: 'secret', apiKey: 'sk-ant-test123456789012345' }; + const result = sanitizer.sanitize(input); + const sanitized = result.sanitized as any; + + expect(sanitized.password).toBe('secret'); + expect(sanitized.apiKey).toBe('sk-ant-test123456789012345'); + expect(result.warnings).toContain('Sanitization disabled - sensitive data may be exposed'); + }); + + test('STANDARD level catches secrets and PII', () => { + const sanitizer = new PayloadSanitizer({ level: SanitizationLevel.STANDARD }); + + const input = { + apiKey: 'sk-ant-test123456789012345', + ssn: '123-45-6789', + email: 'test@example.com', + }; + + const result = sanitizer.sanitize(input); + expect(result.redactionCount).toBeGreaterThanOrEqual(3); + }); + }); + + describe('Production Environment Protection', () => { + test('forces STANDARD level in production when NONE requested', () => { + process.env.NODE_ENV = 'production'; + process.env.PAYLOAD_LOG_SANITIZATION_LEVEL = 'none'; + + const config = buildConfigFromEnv(); + expect(config.level).toBe(SanitizationLevel.STANDARD); + }); + + test('forces STANDARD level in production when MINIMAL requested', () => { + process.env.NODE_ENV = 'production'; + process.env.PAYLOAD_LOG_SANITIZATION_LEVEL = 'minimal'; + + const config = buildConfigFromEnv(); + expect(config.level).toBe(SanitizationLevel.STANDARD); + }); + + test('allows PARANOID level in production', () => { + process.env.NODE_ENV = 'production'; + process.env.PAYLOAD_LOG_SANITIZATION_LEVEL = 'paranoid'; + + const config = buildConfigFromEnv(); + expect(config.level).toBe(SanitizationLevel.PARANOID); + }); + }); + + describe('Edge Cases', () => { + const sanitizer = new PayloadSanitizer({ level: SanitizationLevel.STANDARD }); + + test('handles null and undefined', () => { + expect(sanitizer.sanitize(null).sanitized).toBeNull(); + expect(sanitizer.sanitize(undefined).sanitized).toBeUndefined(); + }); + + test('handles empty objects and arrays', () => { + expect(sanitizer.sanitize({}).sanitized).toEqual({}); + expect(sanitizer.sanitize([]).sanitized).toEqual([]); + }); + + test('handles arrays with sensitive data', () => { + const input = [ + { password: 'secret1' }, + { password: 'secret2' }, + 'sk-ant-test123456789012345', + ]; + + const result = sanitizer.sanitize(input); + const sanitized = result.sanitized as any[]; + + expect(String(sanitized[0].password)).toContain('[REDACTED'); + expect(String(sanitized[1].password)).toContain('[REDACTED'); + expect(String(sanitized[2])).toContain('[REDACTED'); + }); + + test('handles circular references safely', () => { + const obj: any = { name: 'test' }; + obj.self = obj; + + const result = sanitizer.sanitize(obj); + // Should not throw an error + expect(result.sanitized).toBeDefined(); + }); + + test('preserves safe data', () => { + const input = { + username: 'john_doe', + age: 30, + preferences: ['dark_mode', 'notifications'], + metadata: { + created: '2023-01-01', + version: '1.0.0' + } + }; + + const result = sanitizer.sanitize(input); + const sanitized = result.sanitized as any; + + expect(sanitized.username).toBe('john_doe'); + expect(sanitized.age).toBe(30); + expect(sanitized.preferences).toEqual(['dark_mode', 'notifications']); + expect(sanitized.metadata.created).toBe('2023-01-01'); + expect(sanitized.metadata.version).toBe('1.0.0'); + expect(result.redactionCount).toBe(0); + }); + }); + + describe('Global Sanitizer Functions', () => { + test('sanitizePayload convenience function works', () => { + const input = { apiKey: 'sk-ant-test123456789012345', data: 'safe' }; + const result = sanitizePayload(input) as any; + + expect(String(result.apiKey)).toContain('[REDACTED'); + expect(result.data).toBe('safe'); + }); + + test('sanitizePayloadFull convenience function works', () => { + const input = { apiKey: 'sk-ant-test123456789012345', data: 'safe' }; + const result = sanitizePayloadFull(input); + + expect(result.redactionCount).toBeGreaterThan(0); + expect(result.detectedTypes.length).toBeGreaterThan(0); + }); + }); +}); \ No newline at end of file diff --git a/src/agents/payload-sanitizer.ts b/src/agents/payload-sanitizer.ts new file mode 100644 index 0000000000000..b89dcc315b1b6 --- /dev/null +++ b/src/agents/payload-sanitizer.ts @@ -0,0 +1,711 @@ +/** + * @fileoverview Payload Sanitization System for CWE-532 Mitigation + * + * This module provides comprehensive sanitization for API payloads before logging. + * It implements defense-in-depth with multiple layers of protection: + * + * 1. Field-level filtering - Remove entire sensitive fields + * 2. Secret detection - Regex-based detection of API keys, tokens, passwords + * 3. PII redaction - SSN, phone numbers, emails, credit cards + * 4. Content truncation - Limit payload size for debugging + * + * SECURITY NOTE: This is a security-critical module. Changes should be reviewed + * by security team and tested against the test cases in payload-sanitizer.test.ts + * + * @see https://cwe.mitre.org/data/definitions/532.html + */ + +import { createSubsystemLogger } from "../logging/subsystem.js"; + +const log = createSubsystemLogger("payload-sanitizer"); + +// ============================================================================= +// TYPES AND INTERFACES +// ============================================================================= + +/** + * Sanitization levels providing increasing levels of protection. + * + * - NONE: No sanitization (DANGEROUS - only for isolated debugging) + * - MINIMAL: Only obvious secrets like API keys + * - STANDARD: Secrets + PII + sensitive fields (recommended) + * - PARANOID: Maximum sanitization, truncates all content + */ +export enum SanitizationLevel { + NONE = 'none', + MINIMAL = 'minimal', + STANDARD = 'standard', + PARANOID = 'paranoid', +} + +export interface SanitizationConfig { + /** The sanitization level to apply */ + level: SanitizationLevel; + + /** Maximum length for string content (0 = unlimited) */ + maxContentLength: number; + + /** Maximum depth for nested object traversal */ + maxDepth: number; + + /** Additional custom patterns to detect */ + customPatterns: SecretPattern[]; + + /** Additional field names to filter */ + customSensitiveFields: string[]; + + /** Whether to log warnings about sanitization */ + logWarnings: boolean; + + /** Whether to include redaction type hints (e.g., [REDACTED:API_KEY]) */ + includeRedactionHints: boolean; + + /** Whether running in production (forces minimum STANDARD level) */ + isProduction: boolean; +} + +export interface SecretPattern { + /** Human-readable name for this pattern */ + name: string; + + /** Regex pattern to match */ + pattern: RegExp; + + /** Minimum sanitization level where this pattern applies */ + minLevel: SanitizationLevel; +} + +export interface SanitizationResult { + /** The sanitized data */ + sanitized: unknown; + + /** Count of redactions performed */ + redactionCount: number; + + /** Types of sensitive data detected */ + detectedTypes: string[]; + + /** Whether content was truncated */ + wasTruncated: boolean; + + /** Warnings generated during sanitization */ + warnings: string[]; +} + +export interface SanitizationStats { + totalRedactions: number; + byType: Record; + fieldsRemoved: string[]; + truncatedFields: string[]; +} + +// ============================================================================= +// PATTERN DEFINITIONS +// ============================================================================= + +/** + * Comprehensive list of regex patterns for detecting secrets. + * + * These patterns are ordered roughly by specificity - more specific patterns + * (like Anthropic API keys) come before generic patterns. + */ +export const SECRET_PATTERNS: SecretPattern[] = [ + // === API Keys (Specific Providers) === + { + name: 'ANTHROPIC_API_KEY', + pattern: /sk-ant-[a-zA-Z0-9_-]{20,}/gi, + minLevel: SanitizationLevel.MINIMAL, + }, + { + name: 'OPENAI_API_KEY', + pattern: /sk-[a-zA-Z0-9]{20,}/gi, + minLevel: SanitizationLevel.MINIMAL, + }, + { + name: 'OPENAI_PROJECT_KEY', + pattern: /sk-proj-[a-zA-Z0-9_-]{20,}/gi, + minLevel: SanitizationLevel.MINIMAL, + }, + { + name: 'AWS_ACCESS_KEY', + pattern: /AKIA[0-9A-Z]{16}/gi, + minLevel: SanitizationLevel.MINIMAL, + }, + { + name: 'GITHUB_TOKEN', + pattern: /gh[pousr]_[A-Za-z0-9_]{36,}/gi, + minLevel: SanitizationLevel.MINIMAL, + }, + { + name: 'GITHUB_FINE_GRAINED', + pattern: /github_pat_[A-Za-z0-9_]{22,}/gi, + minLevel: SanitizationLevel.MINIMAL, + }, + { + name: 'GOOGLE_API_KEY', + pattern: /AIza[0-9A-Za-z_-]{35}/gi, + minLevel: SanitizationLevel.MINIMAL, + }, + + // === Generic Secret Patterns === + { + name: 'BEARER_TOKEN', + pattern: /Bearer\s+[a-zA-Z0-9_\-.~+/]+=*/gi, + minLevel: SanitizationLevel.MINIMAL, + }, + { + name: 'BASIC_AUTH', + pattern: /Basic\s+[a-zA-Z0-9+/]+=*/gi, + minLevel: SanitizationLevel.MINIMAL, + }, + { + name: 'JWT_TOKEN', + pattern: /eyJ[a-zA-Z0-9_-]*\.eyJ[a-zA-Z0-9_-]*\.[a-zA-Z0-9_-]*/gi, + minLevel: SanitizationLevel.MINIMAL, + }, + { + name: 'PRIVATE_KEY_HEADER', + pattern: /-----BEGIN\s+(RSA\s+)?PRIVATE\s+KEY-----[\s\S]*?-----END\s+(RSA\s+)?PRIVATE\s+KEY-----/gi, + minLevel: SanitizationLevel.MINIMAL, + }, + { + name: 'GENERIC_API_KEY', + pattern: /(?:api[_-]?key|apikey|api[_-]?secret|secret[_-]?key)[\s]*[=:]\s*["']?([a-zA-Z0-9_\-]{16,})["']?/gi, + minLevel: SanitizationLevel.MINIMAL, + }, + { + name: 'PASSWORD_FIELD', + pattern: /(?:password|passwd|pwd)[\s]*[=:]\s*["']?([^\s"']{4,})["']?/gi, + minLevel: SanitizationLevel.MINIMAL, + }, + + // === PII Patterns (Standard level and above) === + { + name: 'SSN', + pattern: /\b\d{3}-\d{2}-\d{4}\b/g, + minLevel: SanitizationLevel.STANDARD, + }, + { + name: 'CREDIT_CARD', + pattern: /\b(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13}|6(?:011|5[0-9]{2})[0-9]{12})\b/g, + minLevel: SanitizationLevel.STANDARD, + }, + { + name: 'PHONE_US', + pattern: /\b(?:\+1[-.\s]?)?\(?[2-9]\d{2}\)?[-.\s]?\d{3}[-.\s]?\d{4}\b/g, + minLevel: SanitizationLevel.STANDARD, + }, + { + name: 'EMAIL', + pattern: /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, + minLevel: SanitizationLevel.STANDARD, + }, + { + name: 'IP_ADDRESS', + pattern: /\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b/g, + minLevel: SanitizationLevel.PARANOID, // Often needed for debugging + }, +]; + +/** + * Field names that should be entirely removed or redacted. + * These are checked case-insensitively with various naming conventions. + */ +export const SENSITIVE_FIELD_NAMES: Set = new Set([ + // Authentication + 'password', + 'passwd', + 'pwd', + 'secret', + 'apikey', + 'api_key', + 'apiKey', + 'api-key', + 'token', + 'accesstoken', + 'access_token', + 'accessToken', + 'access-token', + 'refreshtoken', + 'refresh_token', + 'refreshToken', + 'refresh-token', + 'bearer', + 'authorization', + 'auth', + 'credentials', + 'credential', + 'privatekey', + 'private_key', + 'privateKey', + 'private-key', + 'secretkey', + 'secret_key', + 'secretKey', + 'secret-key', + + // Session related + 'sessionkey', + 'session_key', + 'sessionKey', + 'session-key', + 'sessiontoken', + 'session_token', + 'sessionToken', + 'session-token', + 'cookie', + 'cookies', + 'set-cookie', + + // Personal Data + 'ssn', + 'socialsecuritynumber', + 'social_security_number', + 'taxid', + 'tax_id', + 'creditcard', + 'credit_card', + 'cardnumber', + 'card_number', + 'cvv', + 'cvc', + 'pin', +]); + +/** + * Fields that indicate the content might be a system prompt with embedded secrets. + */ +export const SYSTEM_PROMPT_FIELDS: Set = new Set([ + 'system', + 'systemprompt', + 'system_prompt', + 'systemPrompt', + 'instructions', + 'context', +]); + +// ============================================================================= +// CONFIGURATION +// ============================================================================= + +/** + * Default sanitization configuration. + * Secure by default - uses STANDARD level. + */ +export const DEFAULT_CONFIG: SanitizationConfig = { + level: SanitizationLevel.STANDARD, + maxContentLength: 10000, // 10KB max per field + maxDepth: 20, + customPatterns: [], + customSensitiveFields: [], + logWarnings: true, + includeRedactionHints: true, + isProduction: process.env.NODE_ENV === 'production', +}; + +/** + * Build sanitization configuration from environment variables. + */ +export function buildConfigFromEnv( + overrides: Partial = {} +): SanitizationConfig { + const envLevel = process.env.PAYLOAD_LOG_SANITIZATION_LEVEL?.toLowerCase(); + const envMaxLength = process.env.PAYLOAD_LOG_MAX_CONTENT_LENGTH; + const envIncludeHints = process.env.PAYLOAD_LOG_INCLUDE_HINTS; + const envCustomFields = process.env.PAYLOAD_LOG_CUSTOM_FIELDS; + const isProduction = process.env.NODE_ENV === 'production'; + + let level = DEFAULT_CONFIG.level; + if (envLevel && Object.values(SanitizationLevel).includes(envLevel as SanitizationLevel)) { + level = envLevel as SanitizationLevel; + } + + // CRITICAL: Production environment forces minimum STANDARD level + if (isProduction && levelToNumber(level) < levelToNumber(SanitizationLevel.STANDARD)) { + log.warn( + 'Production environment detected. ' + + `Overriding sanitization level from ${level} to ${SanitizationLevel.STANDARD}. ` + + 'Disable this protection only if you understand the security implications.' + ); + level = SanitizationLevel.STANDARD; + } + + const customSensitiveFields: string[] = envCustomFields + ? envCustomFields.split(',').map(f => f.trim()).filter(Boolean) + : []; + + return { + ...DEFAULT_CONFIG, + level, + maxContentLength: envMaxLength ? parseInt(envMaxLength, 10) : DEFAULT_CONFIG.maxContentLength, + includeRedactionHints: envIncludeHints === 'false' ? false : DEFAULT_CONFIG.includeRedactionHints, + customSensitiveFields, + isProduction, + ...overrides, + }; +} + +/** + * Convert sanitization level to numeric value for comparison. + */ +function levelToNumber(level: SanitizationLevel): number { + const levels: Record = { + [SanitizationLevel.NONE]: 0, + [SanitizationLevel.MINIMAL]: 1, + [SanitizationLevel.STANDARD]: 2, + [SanitizationLevel.PARANOID]: 3, + }; + return levels[level]; +} + +// ============================================================================= +// SANITIZER CLASS +// ============================================================================= + +/** + * Main sanitizer class providing payload sanitization functionality. + */ +export class PayloadSanitizer { + private readonly config: SanitizationConfig; + private readonly allPatterns: SecretPattern[]; + private readonly allSensitiveFields: Set; + + constructor(config: Partial = {}) { + this.config = buildConfigFromEnv(config); + + // Combine built-in patterns with custom patterns + this.allPatterns = [ + ...SECRET_PATTERNS, + ...this.config.customPatterns, + ].filter(p => levelToNumber(p.minLevel) <= levelToNumber(this.config.level)); + + // Combine built-in sensitive fields with custom fields + this.allSensitiveFields = new Set([ + ...SENSITIVE_FIELD_NAMES, + ...this.config.customSensitiveFields.map(f => f.toLowerCase()), + ]); + + // Log initialization in non-production + if (this.config.logWarnings && !this.config.isProduction) { + log.info( + `Payload sanitizer initialized: level=${this.config.level}, ` + + `patterns=${this.allPatterns.length}, sensitiveFields=${this.allSensitiveFields.size}` + ); + } + } + + /** + * Get the current sanitization level. + */ + getLevel(): SanitizationLevel { + return this.config.level; + } + + /** + * Main sanitization entry point. + */ + sanitize(data: unknown): SanitizationResult { + // Short-circuit for NONE level (with warning) + if (this.config.level === SanitizationLevel.NONE) { + if (this.config.logWarnings) { + log.warn( + 'Sanitization is DISABLED. ' + + 'Sensitive data may be logged in plaintext. ' + + 'This should NEVER be used in production.' + ); + } + return { + sanitized: data, + redactionCount: 0, + detectedTypes: [], + wasTruncated: false, + warnings: ['Sanitization disabled - sensitive data may be exposed'], + }; + } + + const stats: SanitizationStats = { + totalRedactions: 0, + byType: {}, + fieldsRemoved: [], + truncatedFields: [], + }; + + const warnings: string[] = []; + + try { + const sanitized = this.sanitizeValue(data, '', 0, stats); + + // Generate warnings for detected sensitive data + if (stats.totalRedactions > 0) { + warnings.push( + `Redacted ${stats.totalRedactions} instances of sensitive data` + ); + } + if (stats.fieldsRemoved.length > 0) { + warnings.push( + `Removed sensitive fields: ${stats.fieldsRemoved.slice(0, 5).join(', ')}` + + (stats.fieldsRemoved.length > 5 ? ` (+${stats.fieldsRemoved.length - 5} more)` : '') + ); + } + + return { + sanitized, + redactionCount: stats.totalRedactions, + detectedTypes: Object.keys(stats.byType), + wasTruncated: stats.truncatedFields.length > 0, + warnings, + }; + } catch (error) { + // On error, return a safe fallback + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + log.error(`Error during sanitization: ${errorMessage}`); + + return { + sanitized: '[SANITIZATION_ERROR: Data could not be safely processed]', + redactionCount: 0, + detectedTypes: [], + wasTruncated: false, + warnings: [`Sanitization error: ${errorMessage}`], + }; + } + } + + /** + * Recursively sanitize a value based on its type. + */ + private sanitizeValue( + value: unknown, + path: string, + depth: number, + stats: SanitizationStats + ): unknown { + // Depth limit protection + if (depth > this.config.maxDepth) { + return this.redact('MAX_DEPTH_EXCEEDED', stats); + } + + // Handle null/undefined + if (value === null || value === undefined) { + return value; + } + + // Handle primitives + if (typeof value === 'string') { + return this.sanitizeString(value, path, stats); + } + + if (typeof value === 'number' || typeof value === 'boolean') { + return value; + } + + // Handle arrays + if (Array.isArray(value)) { + return value.map((item, index) => + this.sanitizeValue(item, `${path}[${index}]`, depth + 1, stats) + ); + } + + // Handle objects + if (typeof value === 'object') { + return this.sanitizeObject(value as Record, path, depth, stats); + } + + // Handle functions and other types - redact + return this.redact('UNSUPPORTED_TYPE', stats); + } + + /** + * Sanitize an object, filtering sensitive fields and recursing into values. + */ + private sanitizeObject( + obj: Record, + path: string, + depth: number, + stats: SanitizationStats + ): Record { + const result: Record = {}; + + for (const [key, value] of Object.entries(obj)) { + const fullPath = path ? `${path}.${key}` : key; + const normalizedKey = key.toLowerCase().replace(/[-_]/g, ''); + + // Check if this is a sensitive field name + if (this.isSensitiveField(normalizedKey)) { + stats.fieldsRemoved.push(fullPath); + stats.totalRedactions++; + result[key] = this.redact('SENSITIVE_FIELD', stats, false); + continue; + } + + // Check if this is a system prompt field (needs extra scrutiny) + const isSystemPrompt = SYSTEM_PROMPT_FIELDS.has(normalizedKey); + + // Recursively sanitize the value + let sanitizedValue = this.sanitizeValue(value, fullPath, depth + 1, stats); + + // For system prompts in PARANOID mode, apply extra sanitization + if (isSystemPrompt && + this.config.level === SanitizationLevel.PARANOID && + typeof sanitizedValue === 'string') { + // Truncate system prompts more aggressively in paranoid mode + const maxPromptLength = Math.min(500, this.config.maxContentLength); + if (sanitizedValue.length > maxPromptLength) { + sanitizedValue = sanitizedValue.substring(0, maxPromptLength) + + this.redactSuffix('TRUNCATED_SYSTEM_PROMPT'); + stats.truncatedFields.push(fullPath); + } + } + + result[key] = sanitizedValue; + } + + return result; + } + + /** + * Sanitize a string value by applying all applicable patterns. + */ + private sanitizeString( + value: string, + path: string, + stats: SanitizationStats + ): string { + let result = value; + + // Apply all matching patterns + for (const pattern of this.allPatterns) { + // Reset regex lastIndex for global patterns + pattern.pattern.lastIndex = 0; + + if (pattern.pattern.test(result)) { + // Reset again for replacement + pattern.pattern.lastIndex = 0; + + const beforeLength = result.length; + result = result.replace(pattern.pattern, () => { + stats.totalRedactions++; + stats.byType[pattern.name] = (stats.byType[pattern.name] || 0) + 1; + return this.redactInline(pattern.name); + }); + + if (this.config.logWarnings && result.length !== beforeLength) { + // Only log in development and for significant detections + if (!this.config.isProduction) { + log.debug(`Detected ${pattern.name} at path: ${path || 'root'}`); + } + } + } + } + + // Apply length truncation + if (this.config.maxContentLength > 0 && result.length > this.config.maxContentLength) { + result = result.substring(0, this.config.maxContentLength) + + this.redactSuffix('TRUNCATED'); + stats.truncatedFields.push(path); + } + + return result; + } + + /** + * Check if a field name indicates sensitive data. + */ + private isSensitiveField(normalizedKey: string): boolean { + // Direct match + if (this.allSensitiveFields.has(normalizedKey)) { + return true; + } + + // Check for partial matches (e.g., "userPassword" contains "password") + for (const sensitiveField of this.allSensitiveFields) { + if (normalizedKey.includes(sensitiveField)) { + return true; + } + } + + return false; + } + + /** + * Generate a redaction placeholder. + */ + private redact(type: string, stats: SanitizationStats, incrementCount = true): string { + if (incrementCount) { + stats.totalRedactions++; + stats.byType[type] = (stats.byType[type] || 0) + 1; + } + + if (this.config.includeRedactionHints) { + return `[REDACTED:${type}]`; + } + return '[REDACTED]'; + } + + /** + * Generate an inline redaction (for pattern replacement). + */ + private redactInline(type: string): string { + if (this.config.includeRedactionHints) { + return `[REDACTED:${type}]`; + } + return '[REDACTED]'; + } + + /** + * Generate a redaction suffix (for truncation). + */ + private redactSuffix(type: string): string { + if (this.config.includeRedactionHints) { + return `...[${type}]`; + } + return '...[TRUNCATED]'; + } +} + +// ============================================================================= +// CONVENIENCE FUNCTIONS +// ============================================================================= + +/** + * Global sanitizer instance with default configuration. + */ +let globalSanitizer: PayloadSanitizer | null = null; + +/** + * Get or create the global sanitizer instance. + */ +export function getGlobalSanitizer(): PayloadSanitizer { + if (!globalSanitizer) { + globalSanitizer = new PayloadSanitizer(); + } + return globalSanitizer; +} + +/** + * Reset the global sanitizer (useful for testing). + */ +export function resetGlobalSanitizer(): void { + globalSanitizer = null; +} + +/** + * Convenience function to sanitize data using the global sanitizer. + */ +export function sanitizePayload(data: unknown): unknown { + return getGlobalSanitizer().sanitize(data).sanitized; +} + +/** + * Convenience function to sanitize data and get full result. + */ +export function sanitizePayloadFull(data: unknown): SanitizationResult { + return getGlobalSanitizer().sanitize(data); +} + +// Export for testing +export const __testing = { + levelToNumber, + SECRET_PATTERNS, + SENSITIVE_FIELD_NAMES, + SYSTEM_PROMPT_FIELDS, +}; \ No newline at end of file