diff --git a/junit.xml b/junit.xml index a2a40b7f..e29390d2 100644 --- a/junit.xml +++ b/junit.xml @@ -1,605 +1,901 @@ - - - + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + - + - + - + - + - - + + - + - + - + - + + + + + + + + + + + - + - + - + - + - + - + - + + + + + - - + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + + + + + - - + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + + + + + + + + + + + + + - - + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + + + - + - + - + - + - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - + - + - + - + - + - + - + - + - - + + + + - + - + - + + + + + - + - + - + + + + + - - + + + + - + - + - + - + - + - + - + - + - + - + + + - - + + - + - + - + - + - + - + - + - + - + - + - + - - + + + + - + - + - + - + + + - - + + + + + + - + - + - + - + - + - + - + + + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + - - + + + + + + + + + + - + - + - + - + - + - - + + - + - + - + + + - + - + - + - + - + - + - + - + - - - + + + + + + + - + - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + - + - + + + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + + + + - - + + \ No newline at end of file diff --git a/src/app.module.ts b/src/app.module.ts index 8ad8f3d0..a57d9b2d 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -49,6 +49,7 @@ import { AuditController } from './common/controllers/audit.controller'; // Middleware import { AuthRateLimitMiddleware } from './auth/middleware/auth.middleware'; +import { HeaderValidationMiddleware } from './security/middleware/header-validation.middleware'; import { ObservabilityModule } from './observability/observability.module'; @Module({ @@ -146,11 +147,11 @@ import { ObservabilityModule } from './observability/observability.module'; export class AppModule implements NestModule { configure(consumer: MiddlewareConsumer) { consumer + // Header validation for security + .apply(HeaderValidationMiddleware) + .forRoutes('*') // Auth rate limiting .apply(AuthRateLimitMiddleware) .forRoutes('/auth*'); - // Global security middleware - // .apply(SecurityMiddleware) // Uncomment when SecurityModule is properly integrated - // .forRoutes('*'); } } diff --git a/src/auth/auth.controller.ts b/src/auth/auth.controller.ts index b7b41632..6e76b5ef 100644 --- a/src/auth/auth.controller.ts +++ b/src/auth/auth.controller.ts @@ -163,7 +163,8 @@ export class AuthController { }) @ApiOperation({ summary: 'Refresh access token', - description: 'Exchanges refresh token for new access token. Implements token rotation. Rate limit: 10 requests per minute.', + description: + 'Exchanges refresh token for new access token. Implements token rotation. Rate limit: 10 requests per minute.', }) @ApiResponse({ status: 200, @@ -237,7 +238,8 @@ export class AuthController { }) @ApiOperation({ summary: 'Request password reset email', - description: 'Sends password reset link to user email. Returns generic message for security. Rate limit: 3 requests per 15 minutes.', + description: + 'Sends password reset link to user email. Returns generic message for security. Rate limit: 3 requests per 15 minutes.', }) @ApiResponse({ status: 200, @@ -274,7 +276,8 @@ export class AuthController { }) @ApiOperation({ summary: 'Reset password using reset token', - description: 'Sets new password using token from password reset email. Token valid for 1 hour. Rate limit: 5 requests per 15 minutes.', + description: + 'Sets new password using token from password reset email. Token valid for 1 hour. Rate limit: 5 requests per 15 minutes.', }) @ApiResponse({ status: 200, diff --git a/src/common/cache/cache-invalidation.service.ts b/src/common/cache/cache-invalidation.service.ts new file mode 100644 index 00000000..118a5d1a --- /dev/null +++ b/src/common/cache/cache-invalidation.service.ts @@ -0,0 +1,637 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { MultiLevelCacheService } from './multi-level-cache.service'; +import { RedisService } from '../services/redis.service'; + +export interface InvalidationRule { + id: string; + name: string; + description: string; + type: 'ttl' | 'tag' | 'pattern' | 'conditional' | 'dependency' | 'time-based'; + target: string | string[] | RegExp; + condition?: (value: any, metadata: CacheEntryMetadata) => boolean; + action: 'delete' | 'refresh' | 'cascade'; + priority: number; // 1-10, higher = executed first + enabled: boolean; + metadata?: { + createdAt: Date; + lastExecuted: Date | null; + executionCount: number; + }; +} + +export interface CacheEntryMetadata { + key: string; + createdAt: number; + lastAccessed: number; + accessCount: number; + tags: string[]; + ttl: number; + size: number; +} + +export interface InvalidationEvent { + ruleId: string; + key: string; + action: string; + timestamp: Date; + success: boolean; + error?: string; +} + +export interface InvalidationStats { + totalRules: number; + activeRules: number; + totalExecutions: number; + successfulInvalidations: number; + failedInvalidations: number; + events: InvalidationEvent[]; + lastCleanup: Date | null; +} + +@Injectable() +export class CacheInvalidationService { + private readonly logger = new Logger(CacheInvalidationService.name); + private rules: Map = new Map(); + private stats: InvalidationStats = { + totalRules: 0, + activeRules: 0, + totalExecutions: 0, + successfulInvalidations: 0, + failedInvalidations: 0, + events: [], + lastCleanup: null, + }; + private readonly maxEvents: number; + + constructor( + private readonly cacheService: MultiLevelCacheService, + private readonly redisService: RedisService, + private readonly configService: ConfigService, + ) { + this.maxEvents = this.configService.get('CACHE_INVALIDATION_MAX_EVENTS', 1000); + this.initializeDefaultRules(); + } + + /** + * Initialize default invalidation rules + */ + private initializeDefaultRules(): void { + // Rule: Invalidate stale user sessions + this.registerRule({ + id: 'rule-user-session-stale', + name: 'Stale User Session Cleanup', + description: 'Invalidate user sessions older than 24 hours', + type: 'time-based', + target: 'user:session:*', + action: 'delete', + priority: 5, + enabled: true, + condition: (_value, metadata) => { + const maxAge = 24 * 60 * 60 * 1000; // 24 hours + return Date.now() - metadata.createdAt > maxAge; + }, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }); + + // Rule: Cascade property changes + this.registerRule({ + id: 'rule-property-cascade', + name: 'Property Change Cascade', + description: 'Invalidate related caches when property data changes', + type: 'dependency', + target: 'property:*', + action: 'cascade', + priority: 9, + enabled: true, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }); + + // Rule: Refresh frequently accessed items + this.registerRule({ + id: 'rule-frequent-refresh', + name: 'Frequent Access Refresh', + description: 'Refresh cache entries with high access counts before they expire', + type: 'conditional', + target: '*', + action: 'refresh', + priority: 3, + enabled: true, + condition: (_value, metadata) => { + return metadata.accessCount > 100 && metadata.ttl < 300; // Less than 5 minutes remaining + }, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }); + + // Rule: Cleanup expired tags + this.registerRule({ + id: 'rule-expired-tags', + name: 'Expired Tag Cleanup', + description: 'Remove tag indexes for deleted entries', + type: 'tag', + target: 'tag:*', + action: 'delete', + priority: 2, + enabled: true, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }); + + // Rule: Large entry cleanup + this.registerRule({ + id: 'rule-large-entries', + name: 'Large Entry Cleanup', + description: 'Remove large cache entries that are rarely accessed', + type: 'conditional', + target: '*', + action: 'delete', + priority: 4, + enabled: true, + condition: (_value, metadata) => { + return metadata.size > 1024 * 1024 && metadata.accessCount < 5; // > 1MB and < 5 accesses + }, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }); + } + + /** + * Register a new invalidation rule + */ + registerRule(rule: InvalidationRule): void { + this.rules.set(rule.id, rule); + this.updateStats(); + this.logger.log(`Registered invalidation rule: ${rule.name} (${rule.id})`); + } + + /** + * Unregister an invalidation rule + */ + unregisterRule(ruleId: string): boolean { + const deleted = this.rules.delete(ruleId); + if (deleted) { + this.updateStats(); + this.logger.log(`Unregistered invalidation rule: ${ruleId}`); + } + return deleted; + } + + /** + * Enable/disable a rule + */ + setRuleEnabled(ruleId: string, enabled: boolean): boolean { + const rule = this.rules.get(ruleId); + if (rule) { + rule.enabled = enabled; + this.updateStats(); + this.logger.log(`Rule ${ruleId} ${enabled ? 'enabled' : 'disabled'}`); + return true; + } + return false; + } + + /** + * Execute a specific rule + */ + async executeRule(ruleId: string): Promise { + const rule = this.rules.get(ruleId); + if (!rule) { + this.logger.warn(`Rule not found: ${ruleId}`); + return 0; + } + + if (!rule.enabled) { + this.logger.log(`Rule ${ruleId} is disabled, skipping`); + return 0; + } + + this.logger.log(`Executing invalidation rule: ${rule.name}`); + let invalidatedCount = 0; + + try { + switch (rule.type) { + case 'pattern': + invalidatedCount = await this.executePatternRule(rule); + break; + case 'tag': + invalidatedCount = await this.executeTagRule(rule); + break; + case 'conditional': + invalidatedCount = await this.executeConditionalRule(rule); + break; + case 'dependency': + invalidatedCount = await this.executeDependencyRule(rule); + break; + case 'time-based': + invalidatedCount = await this.executeTimeBasedRule(rule); + break; + default: + this.logger.warn(`Unknown rule type: ${rule.type}`); + } + + // Update rule metadata + rule.metadata!.lastExecuted = new Date(); + rule.metadata!.executionCount++; + + // Record event + this.recordEvent({ + ruleId: rule.id, + key: rule.target as string, + action: rule.action, + timestamp: new Date(), + success: true, + }); + + this.stats.successfulInvalidations += invalidatedCount; + } catch (error) { + this.logger.error(`Failed to execute rule ${ruleId}: ${error.message}`); + + this.recordEvent({ + ruleId: rule.id, + key: rule.target as string, + action: rule.action, + timestamp: new Date(), + success: false, + error: error.message, + }); + + this.stats.failedInvalidations++; + } + + this.stats.totalExecutions++; + return invalidatedCount; + } + + /** + * Execute all enabled rules + */ + async executeAllRules(): Promise> { + const results = new Map(); + + // Sort rules by priority (highest first) + const sortedRules = Array.from(this.rules.values()) + .filter(r => r.enabled) + .sort((a, b) => b.priority - a.priority); + + for (const rule of sortedRules) { + const count = await this.executeRule(rule.id); + results.set(rule.id, count); + } + + return results; + } + + /** + * Execute pattern-based rule + */ + private async executePatternRule(rule: InvalidationRule): Promise { + const pattern = rule.target as string; + const keys = await this.cacheService.invalidateByPattern(pattern); + this.logger.log(`Pattern rule executed: ${keys} keys invalidated matching ${pattern}`); + return keys; + } + + /** + * Execute tag-based rule + */ + private async executeTagRule(rule: InvalidationRule): Promise { + const tagPattern = rule.target as string; + let totalInvalidated = 0; + + // Find all tags matching the pattern + const tagKeys = await this.redisService.keys(tagPattern); + + for (const tagKey of tagKeys) { + const tag = tagKey.replace('tag:', ''); + const count = await this.cacheService.invalidateByTag(tag); + totalInvalidated += count; + } + + this.logger.log(`Tag rule executed: ${totalInvalidated} keys invalidated`); + return totalInvalidated; + } + + /** + * Execute conditional rule + */ + private async executeConditionalRule(rule: InvalidationRule): Promise { + if (!rule.condition) { + return 0; + } + + const pattern = rule.target as string; + const keys = await this.redisService.keys(pattern); + let invalidatedCount = 0; + + for (const key of keys) { + try { + const value = await this.redisService.get(key); + if (!value) { + continue; + } + + const metadata = await this.getEntryMetadata(key, value); + + if (rule.condition(JSON.parse(value), metadata)) { + await this.cacheService.del(key); + invalidatedCount++; + } + } catch (error) { + this.logger.error(`Error checking condition for key ${key}: ${error.message}`); + } + } + + this.logger.log(`Conditional rule executed: ${invalidatedCount} keys invalidated`); + return invalidatedCount; + } + + /** + * Execute dependency rule (cascade invalidation) + */ + private async executeDependencyRule(rule: InvalidationRule): Promise { + const pattern = rule.target as string; + const keys = await this.redisService.keys(pattern); + let totalInvalidated = 0; + + for (const key of keys) { + await this.cacheService.invalidateWithCascade(key); + totalInvalidated++; + } + + this.logger.log(`Dependency rule executed: ${totalInvalidated} keys invalidated with cascade`); + return totalInvalidated; + } + + /** + * Execute time-based rule + */ + private async executeTimeBasedRule(rule: InvalidationRule): Promise { + if (!rule.condition) { + return 0; + } + + const pattern = rule.target as string; + const keys = await this.redisService.keys(pattern); + let invalidatedCount = 0; + + for (const key of keys) { + try { + const value = await this.redisService.get(key); + if (!value) { + continue; + } + + const metadata = await this.getEntryMetadata(key, value); + + if (rule.condition(null, metadata)) { + await this.cacheService.del(key); + invalidatedCount++; + } + } catch (error) { + this.logger.error(`Error checking time condition for key ${key}: ${error.message}`); + } + } + + this.logger.log(`Time-based rule executed: ${invalidatedCount} keys invalidated`); + return invalidatedCount; + } + + /** + * Get metadata for a cache entry + */ + private async getEntryMetadata(key: string, value: string): Promise { + const parsed = JSON.parse(value); + const ttl = await this.redisService.ttl(key); + + return { + key, + createdAt: parsed.timestamp || Date.now(), + lastAccessed: parsed.lastAccessed || parsed.timestamp || Date.now(), + accessCount: parsed.accessCount || 0, + tags: parsed.tags || [], + ttl, + size: value.length, + }; + } + + /** + * Invalidate by tags with policy enforcement + */ + async invalidateByTagsWithPolicy(tags: string[], policy?: { cascade?: boolean; refresh?: boolean }): Promise { + let totalInvalidated = 0; + + for (const tag of tags) { + const count = await this.cacheService.invalidateByTag(tag); + totalInvalidated += count; + + if (policy?.cascade) { + // Find and invalidate dependent keys + const dependentPatterns = this.getDependentPatterns(tag); + for (const pattern of dependentPatterns) { + const cascadeCount = await this.cacheService.invalidateByPattern(pattern); + totalInvalidated += cascadeCount; + } + } + } + + this.logger.log(`Invalidated ${totalInvalidated} entries by tags with policy`); + return totalInvalidated; + } + + /** + * Smart invalidation based on entity changes + */ + async smartInvalidate( + entityType: string, + entityId: string, + changeType: 'create' | 'update' | 'delete', + ): Promise { + this.logger.log(`Smart invalidation for ${entityType}:${entityId} (${changeType})`); + + // Define invalidation patterns based on entity type and change type + const invalidationMap: Record> = { + property: { + update: [`property:${entityId}`, `valuation:property:${entityId}:*`, `document:property:${entityId}:*`], + delete: [`property:${entityId}`, `property:*:list`, `valuation:*`], + create: ['property:*:list', 'property:recent:*'], + }, + user: { + update: [`user:${entityId}`, `user:${entityId}:permissions`, `user:${entityId}:roles`], + delete: [`user:${entityId}`, `user:*:list`, `session:${entityId}:*`], + create: ['user:*:list', 'user:active:*'], + }, + transaction: { + update: [`transaction:${entityId}`, `transaction:*:list`], + delete: [`transaction:${entityId}`, `transaction:*:list`, `balance:*`], + create: ['transaction:*:list', 'transaction:recent:*', 'balance:*'], + }, + }; + + const patterns = invalidationMap[entityType]?.[changeType] || []; + + for (const pattern of patterns) { + await this.cacheService.invalidateByPattern(pattern); + } + } + + /** + * Scheduled cleanup of expired entries + */ + @Cron(CronExpression.EVERY_HOUR) + async scheduledCleanup(): Promise { + this.logger.log('Running scheduled cache invalidation cleanup'); + + // Execute time-based rules + for (const rule of this.rules.values()) { + if (rule.enabled && rule.type === 'time-based') { + await this.executeRule(rule.id); + } + } + + // Clean up old events + this.cleanupOldEvents(); + + this.stats.lastCleanup = new Date(); + } + + /** + * Scheduled refresh of high-priority entries + */ + @Cron(CronExpression.EVERY_30_MINUTES) + async scheduledRefresh(): Promise { + this.logger.log('Running scheduled cache refresh'); + + // Execute conditional rules for refresh + for (const rule of this.rules.values()) { + if (rule.enabled && rule.type === 'conditional' && rule.action === 'refresh') { + await this.executeRule(rule.id); + } + } + } + + /** + * Get invalidation statistics + */ + getStats(): InvalidationStats { + return { + ...this.stats, + events: [...this.stats.events], + }; + } + + /** + * Get all rules + */ + getRules(): InvalidationRule[] { + return Array.from(this.rules.values()); + } + + /** + * Get a specific rule + */ + getRule(ruleId: string): InvalidationRule | undefined { + return this.rules.get(ruleId); + } + + /** + * Record an invalidation event + */ + private recordEvent(event: InvalidationEvent): void { + this.stats.events.push(event); + + // Keep only recent events + if (this.stats.events.length > this.maxEvents) { + this.stats.events.shift(); + } + } + + /** + * Clean up old events + */ + private cleanupOldEvents(): void { + const maxAge = 24 * 60 * 60 * 1000; // 24 hours + const cutoff = Date.now() - maxAge; + + this.stats.events = this.stats.events.filter(e => e.timestamp.getTime() > cutoff); + } + + /** + * Update statistics + */ + private updateStats(): void { + const allRules = Array.from(this.rules.values()); + this.stats.totalRules = allRules.length; + this.stats.activeRules = allRules.filter(r => r.enabled).length; + } + + /** + * Get dependent patterns for cascade invalidation + */ + private getDependentPatterns(tag: string): string[] { + const dependencies: Record = { + property: ['valuation:*', 'document:property:*'], + user: ['permissions:*', 'roles:*', 'session:*'], + transaction: ['balance:*', 'history:*'], + }; + + return dependencies[tag] || []; + } + + /** + * Batch invalidate multiple keys + */ + async batchInvalidate(keys: string[]): Promise<{ success: number; failed: number }> { + let success = 0; + let failed = 0; + + for (const key of keys) { + try { + await this.cacheService.del(key); + success++; + } catch (error) { + this.logger.error(`Failed to invalidate key ${key}: ${error.message}`); + failed++; + } + } + + return { success, failed }; + } + + /** + * Invalidate with callback for refresh + */ + async invalidateWithCallback(key: string, refreshCallback?: () => Promise): Promise { + // Delete the old value + await this.cacheService.del(key); + + // If refresh callback provided, execute it and cache the result + if (refreshCallback) { + try { + const newValue = await refreshCallback(); + await this.cacheService.set(key, newValue); + this.logger.log(`Refreshed cache key after invalidation: ${key}`); + } catch (error) { + this.logger.error(`Failed to refresh cache key ${key}: ${error.message}`); + } + } + } +} diff --git a/src/common/cache/cache-warming.service.ts b/src/common/cache/cache-warming.service.ts new file mode 100644 index 00000000..a1db8650 --- /dev/null +++ b/src/common/cache/cache-warming.service.ts @@ -0,0 +1,490 @@ +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { Cron, CronExpression, Interval } from '@nestjs/schedule'; +import { MultiLevelCacheService, MultiLevelCacheOptions } from './multi-level-cache.service'; + +export interface WarmupTask { + key: string; + factory: () => Promise; + options?: MultiLevelCacheOptions; + priority: number; // 1-10, higher = more important + condition?: () => boolean | Promise; + dependencies?: string[]; // Keys that must be warmed first +} + +export interface WarmupStrategy { + name: string; + description: string; + tasks: WarmupTask[]; + schedule?: CronExpression | string; + enabled: boolean; +} + +export interface CacheWarmingStats { + totalTasks: number; + completedTasks: number; + failedTasks: number; + skippedTasks: number; + lastRunTime: Date | null; + averageExecutionTime: number; + strategies: Map; +} + +@Injectable() +export class CacheWarmingService implements OnModuleInit { + private readonly logger = new Logger(CacheWarmingService.name); + private strategies: Map = new Map(); + private stats: CacheWarmingStats = { + totalTasks: 0, + completedTasks: 0, + failedTasks: 0, + skippedTasks: 0, + lastRunTime: null, + averageExecutionTime: 0, + strategies: new Map(), + }; + private executionTimes: number[] = []; + + constructor( + private readonly cacheService: MultiLevelCacheService, + private readonly configService: ConfigService, + ) {} + + async onModuleInit(): Promise { + // Initialize default strategies + this.initializeDefaultStrategies(); + this.logger.log('Cache warming service initialized'); + } + + /** + * Initialize default cache warming strategies + */ + private initializeDefaultStrategies(): void { + // Strategy: Warm frequently accessed user data + this.registerStrategy({ + name: 'user-data', + description: 'Warm cache with frequently accessed user data', + tasks: [ + { + key: 'user:active:list', + factory: async () => { + // This would typically fetch from database + return { users: [], timestamp: Date.now() }; + }, + priority: 8, + options: { l1Ttl: 300, l2Ttl: 1800, tags: ['user', 'active'] }, + }, + { + key: 'user:permissions:common', + factory: async () => { + return { permissions: ['read:property', 'read:valuation'] }; + }, + priority: 9, + options: { l1Ttl: 600, l2Ttl: 3600, tags: ['user', 'permissions'] }, + }, + ], + schedule: CronExpression.EVERY_10_MINUTES, + enabled: true, + }); + + // Strategy: Warm property data + this.registerStrategy({ + name: 'property-data', + description: 'Warm cache with popular property data', + tasks: [ + { + key: 'property:popular:list', + factory: async () => { + return { properties: [], count: 0 }; + }, + priority: 7, + options: { l1Ttl: 300, l2Ttl: 900, tags: ['property', 'popular'] }, + }, + { + key: 'property:recent:list', + factory: async () => { + return { properties: [], timestamp: Date.now() }; + }, + priority: 6, + options: { l1Ttl: 180, l2Ttl: 600, tags: ['property', 'recent'] }, + }, + ], + schedule: CronExpression.EVERY_5_MINUTES, + enabled: true, + }); + + // Strategy: Warm valuation data + this.registerStrategy({ + name: 'valuation-data', + description: 'Warm cache with valuation calculations', + tasks: [ + { + key: 'valuation:market:overview', + factory: async () => { + return { overview: {}, timestamp: Date.now() }; + }, + priority: 5, + options: { l1Ttl: 600, l2Ttl: 3600, tags: ['valuation', 'market'] }, + }, + ], + schedule: CronExpression.EVERY_HOUR, + enabled: true, + }); + } + + /** + * Register a new warming strategy + */ + registerStrategy(strategy: WarmupStrategy): void { + this.strategies.set(strategy.name, strategy); + this.logger.log(`Registered cache warming strategy: ${strategy.name}`); + } + + /** + * Unregister a warming strategy + */ + unregisterStrategy(name: string): boolean { + const deleted = this.strategies.delete(name); + if (deleted) { + this.logger.log(`Unregistered cache warming strategy: ${name}`); + } + return deleted; + } + + /** + * Enable/disable a strategy + */ + setStrategyEnabled(name: string, enabled: boolean): boolean { + const strategy = this.strategies.get(name); + if (strategy) { + strategy.enabled = enabled; + this.logger.log(`Strategy ${name} ${enabled ? 'enabled' : 'disabled'}`); + return true; + } + return false; + } + + /** + * Execute a specific warming strategy + */ + async executeStrategy(strategyName: string): Promise { + const strategy = this.strategies.get(strategyName); + if (!strategy) { + this.logger.warn(`Strategy not found: ${strategyName}`); + return; + } + + if (!strategy.enabled) { + this.logger.log(`Strategy ${strategyName} is disabled, skipping`); + return; + } + + const startTime = Date.now(); + this.logger.log(`Executing cache warming strategy: ${strategy.name}`); + + // Sort tasks by priority (highest first) and dependencies + const sortedTasks = this.sortTasksByPriorityAndDependencies(strategy.tasks); + + let completed = 0; + let failed = 0; + let skipped = 0; + + for (const task of sortedTasks) { + try { + // Check condition if provided + if (task.condition) { + const conditionResult = await task.condition(); + if (!conditionResult) { + this.logger.debug(`Skipping task ${task.key} - condition not met`); + skipped++; + continue; + } + } + + // Check if already cached + const existing = await this.cacheService.get(task.key); + if (existing !== undefined) { + this.logger.debug(`Task ${task.key} already cached, skipping`); + skipped++; + continue; + } + + // Execute the factory function + const value = await task.factory(); + + // Store in cache + await this.cacheService.set(task.key, value, task.options); + + this.logger.debug(`Successfully warmed cache for key: ${task.key}`); + completed++; + } catch (error) { + this.logger.error(`Failed to warm cache for key ${task.key}: ${error.message}`); + failed++; + } + } + + // Update stats + const executionTime = Date.now() - startTime; + this.executionTimes.push(executionTime); + if (this.executionTimes.length > 100) { + this.executionTimes.shift(); // Keep last 100 + } + + this.stats.totalTasks += sortedTasks.length; + this.stats.completedTasks += completed; + this.stats.failedTasks += failed; + this.stats.skippedTasks += skipped; + this.stats.lastRunTime = new Date(); + this.stats.averageExecutionTime = this.executionTimes.reduce((a, b) => a + b, 0) / this.executionTimes.length; + + this.stats.strategies.set(strategyName, { + success: (this.stats.strategies.get(strategyName)?.success || 0) + completed, + failed: (this.stats.strategies.get(strategyName)?.failed || 0) + failed, + }); + + this.logger.log( + `Strategy ${strategyName} completed: ${completed} completed, ${failed} failed, ${skipped} skipped (${executionTime}ms)`, + ); + } + + /** + * Execute all enabled strategies + */ + async executeAllStrategies(): Promise { + this.logger.log('Executing all enabled cache warming strategies'); + + for (const [name, strategy] of this.strategies) { + if (strategy.enabled) { + await this.executeStrategy(name); + } + } + } + + /** + * Execute a single warmup task + */ + async executeTask(task: WarmupTask): Promise { + try { + // Check condition if provided + if (task.condition) { + const conditionResult = await task.condition(); + if (!conditionResult) { + this.logger.debug(`Skipping task ${task.key} - condition not met`); + return false; + } + } + + // Check if already cached + const existing = await this.cacheService.get(task.key); + if (existing !== undefined) { + this.logger.debug(`Task ${task.key} already cached, skipping`); + return false; + } + + // Execute the factory function + const value = await task.factory(); + + // Store in cache + await this.cacheService.set(task.key, value, task.options); + + this.logger.debug(`Successfully warmed cache for key: ${task.key}`); + return true; + } catch (error) { + this.logger.error(`Failed to warm cache for key ${task.key}: ${error.message}`); + return false; + } + } + + /** + * Warm cache with custom tasks + */ + async warmCache(tasks: WarmupTask[]): Promise<{ completed: number; failed: number }> { + let completed = 0; + let failed = 0; + + // Sort by priority + const sortedTasks = [...tasks].sort((a, b) => b.priority - a.priority); + + for (const task of sortedTasks) { + const success = await this.executeTask(task); + if (success) { + completed++; + } else { + failed++; + } + } + + return { completed, failed }; + } + + /** + * Schedule-based warming for user data + */ + @Cron(CronExpression.EVERY_10_MINUTES) + async scheduledUserDataWarming(): Promise { + await this.executeStrategy('user-data'); + } + + /** + * Schedule-based warming for property data + */ + @Cron(CronExpression.EVERY_5_MINUTES) + async scheduledPropertyDataWarming(): Promise { + await this.executeStrategy('property-data'); + } + + /** + * Schedule-based warming for valuation data + */ + @Cron(CronExpression.EVERY_HOUR) + async scheduledValuationDataWarming(): Promise { + await this.executeStrategy('valuation-data'); + } + + /** + * Continuous warming for high-priority items + */ + @Interval(60000) // Every minute + async continuousWarming(): Promise { + // Find high-priority tasks that aren't cached + const highPriorityTasks: WarmupTask[] = []; + + for (const strategy of this.strategies.values()) { + if (!strategy.enabled) { + continue; + } + + for (const task of strategy.tasks) { + if (task.priority >= 8) { + highPriorityTasks.push(task); + } + } + } + + if (highPriorityTasks.length > 0) { + this.logger.debug(`Running continuous warming for ${highPriorityTasks.length} high-priority tasks`); + + for (const task of highPriorityTasks) { + const cached = await this.cacheService.get(task.key); + if (cached === undefined) { + await this.executeTask(task); + } + } + } + } + + /** + * Get warming statistics + */ + getStats(): CacheWarmingStats { + return { + ...this.stats, + strategies: new Map(this.stats.strategies), + }; + } + + /** + * Reset statistics + */ + resetStats(): void { + this.stats = { + totalTasks: 0, + completedTasks: 0, + failedTasks: 0, + skippedTasks: 0, + lastRunTime: null, + averageExecutionTime: 0, + strategies: new Map(), + }; + this.executionTimes = []; + } + + /** + * Get all registered strategies + */ + getStrategies(): WarmupStrategy[] { + return Array.from(this.strategies.values()); + } + + /** + * Get a specific strategy + */ + getStrategy(name: string): WarmupStrategy | undefined { + return this.strategies.get(name); + } + + /** + * Sort tasks by priority and handle dependencies + */ + private sortTasksByPriorityAndDependencies(tasks: WarmupTask[]): WarmupTask[] { + const taskMap = new Map(tasks.map(t => [t.key, t])); + const visited = new Set(); + const result: WarmupTask[] = []; + + const visit = (task: WarmupTask) => { + if (visited.has(task.key)) { + return; + } + visited.add(task.key); + + // Visit dependencies first + if (task.dependencies) { + for (const depKey of task.dependencies) { + const dep = taskMap.get(depKey); + if (dep) { + visit(dep); + } + } + } + + result.push(task); + }; + + // Sort by priority first + const sortedByPriority = [...tasks].sort((a, b) => b.priority - a.priority); + + for (const task of sortedByPriority) { + visit(task); + } + + return result; + } + + /** + * Pre-warm cache on startup + */ + async prewarmOnStartup(): Promise { + this.logger.log('Starting cache pre-warming on startup'); + + // Only execute critical strategies + const criticalStrategies = ['user-data', 'property-data']; + + for (const name of criticalStrategies) { + if (this.strategies.has(name)) { + await this.executeStrategy(name); + } + } + + this.logger.log('Cache pre-warming completed'); + } + + /** + * Warm cache based on access patterns + */ + async warmBasedOnAccessPatterns(accessPatterns: Array<{ key: string; frequency: number }>): Promise { + // Sort by frequency + const sortedPatterns = accessPatterns.sort((a, b) => b.frequency - a.frequency); + + // Warm top accessed items + const topPatterns = sortedPatterns.slice(0, 20); + + for (const pattern of topPatterns) { + const cached = await this.cacheService.get(pattern.key); + if (cached === undefined) { + this.logger.debug(`Warming frequently accessed key: ${pattern.key}`); + // Note: This would need the factory function to actually warm + // In practice, you'd store the factory with the pattern + } + } + } +} diff --git a/src/common/cache/cache.module.ts b/src/common/cache/cache.module.ts index c2a96d8b..10add4fe 100644 --- a/src/common/cache/cache.module.ts +++ b/src/common/cache/cache.module.ts @@ -1,9 +1,13 @@ import { Module, Global } from '@nestjs/common'; import { CacheModule as NestCacheModule } from '@nestjs/cache-manager'; import { ConfigService } from '@nestjs/config'; +import { ScheduleModule } from '@nestjs/schedule'; import * as redisStore from 'cache-manager-redis-store'; import { CacheService } from '../services/cache.service'; import { RedisService } from '../services/redis.service'; +import { MultiLevelCacheService } from './multi-level-cache.service'; +import { CacheWarmingService } from './cache-warming.service'; +import { CacheInvalidationService } from './cache-invalidation.service'; @Global() @Module({ @@ -20,8 +24,9 @@ import { RedisService } from '../services/redis.service'; }), inject: [ConfigService], }), + ScheduleModule.forRoot(), ], - providers: [CacheService, RedisService], - exports: [CacheService, NestCacheModule], + providers: [CacheService, RedisService, MultiLevelCacheService, CacheWarmingService, CacheInvalidationService], + exports: [CacheService, NestCacheModule, MultiLevelCacheService, CacheWarmingService, CacheInvalidationService], }) export class CacheModule {} diff --git a/src/common/cache/multi-level-cache.service.ts b/src/common/cache/multi-level-cache.service.ts new file mode 100644 index 00000000..8dede25a --- /dev/null +++ b/src/common/cache/multi-level-cache.service.ts @@ -0,0 +1,528 @@ +import { Injectable, Logger, OnModuleInit, OnModuleDestroy } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { RedisService } from '../services/redis.service'; + +export interface CacheEntry { + value: T; + expiresAt: number; + version: number; + tags: string[]; +} + +export interface MultiLevelCacheOptions { + l1Ttl?: number; // In-memory cache TTL (seconds) + l2Ttl?: number; // Redis cache TTL (seconds) + ttl?: number; // Generic TTL for backward compatibility + tags?: string[]; + version?: number; + staleWhileRevalidate?: boolean; +} + +export interface CacheStats { + l1Hits: number; + l1Misses: number; + l2Hits: number; + l2Misses: number; + totalRequests: number; + l1HitRate: number; + l2HitRate: number; + overallHitRate: number; + l1Size: number; + l2Size: number; +} + +export interface InvalidationPolicy { + type: 'ttl' | 'tag' | 'pattern' | 'conditional'; + value: string | number | ((entry: any) => boolean); + cascade?: boolean; +} + +@Injectable() +export class MultiLevelCacheService implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(MultiLevelCacheService.name); + private l1Cache: Map> = new Map(); + private stats: CacheStats = { + l1Hits: 0, + l1Misses: 0, + l2Hits: 0, + l2Misses: 0, + totalRequests: 0, + l1HitRate: 0, + l2HitRate: 0, + overallHitRate: 0, + l1Size: 0, + l2Size: 0, + }; + + private readonly l1MaxSize: number; + private readonly l1DefaultTtl: number; + private readonly l2DefaultTtl: number; + private cleanupInterval: NodeJS.Timeout | null = null; + private invalidationPolicies: Map = new Map(); + + constructor( + private readonly redisService: RedisService, + private readonly configService: ConfigService, + ) { + this.l1MaxSize = this.configService.get('CACHE_L1_MAX_SIZE', 1000); + this.l1DefaultTtl = this.configService.get('CACHE_L1_TTL', 300); // 5 minutes + this.l2DefaultTtl = this.configService.get('CACHE_L2_TTL', 3600); // 1 hour + } + + async onModuleInit(): Promise { + // Start cleanup interval for expired L1 entries + this.cleanupInterval = setInterval(() => { + this.cleanupExpiredL1Entries(); + }, 60000); // Run every minute + + // Initialize invalidation policies + this.initializeInvalidationPolicies(); + + this.logger.log('Multi-level cache service initialized'); + } + + async onModuleDestroy(): Promise { + if (this.cleanupInterval) { + clearInterval(this.cleanupInterval); + } + this.l1Cache.clear(); + this.logger.log('Multi-level cache service destroyed'); + } + + /** + * Initialize default invalidation policies + */ + private initializeInvalidationPolicies(): void { + // Property-related cache invalidation + this.invalidationPolicies.set('property', [ + { type: 'pattern', value: 'property:*', cascade: true }, + { type: 'pattern', value: 'valuation:property:*', cascade: true }, + ]); + + // User-related cache invalidation + this.invalidationPolicies.set('user', [ + { type: 'pattern', value: 'user:*', cascade: true }, + { type: 'pattern', value: 'permissions:user:*', cascade: true }, + ]); + + // Transaction-related cache invalidation + this.invalidationPolicies.set('transaction', [ + { type: 'pattern', value: 'transaction:*', cascade: true }, + { type: 'pattern', value: 'balance:*', cascade: true }, + ]); + } + + /** + * Get value from multi-level cache + */ + async get(key: string): Promise { + this.stats.totalRequests++; + + // Level 1: In-memory cache + const l1Entry = this.l1Cache.get(key); + if (l1Entry && l1Entry.expiresAt > Date.now()) { + this.stats.l1Hits++; + this.updateHitRates(); + this.logger.debug(`L1 cache HIT: ${key}`); + return l1Entry.value as T; + } + + // Remove expired L1 entry + if (l1Entry) { + this.l1Cache.delete(key); + } + + this.stats.l1Misses++; + + // Level 2: Redis cache + try { + const l2Value = await this.redisService.get(key); + if (l2Value) { + this.stats.l2Hits++; + const parsed = JSON.parse(l2Value); + + // Promote to L1 cache + this.setL1(key, parsed.value, { + l1Ttl: this.l1DefaultTtl, + tags: parsed.tags, + version: parsed.version, + }); + + this.updateHitRates(); + this.logger.debug(`L2 cache HIT: ${key}`); + return parsed.value as T; + } + } catch (error) { + this.logger.error(`L2 cache GET error for key ${key}: ${error.message}`); + } + + this.stats.l2Misses++; + this.updateHitRates(); + this.logger.debug(`Cache MISS: ${key}`); + return undefined; + } + + /** + * Set value in multi-level cache + */ + async set(key: string, value: T, options?: MultiLevelCacheOptions): Promise { + const l1Ttl = options?.l1Ttl ?? this.l1DefaultTtl; + const l2Ttl = options?.l2Ttl ?? this.l2DefaultTtl; + + // Set in L1 cache + this.setL1(key, value, options); + + // Set in L2 cache (Redis) + try { + const entry = { + value, + tags: options?.tags || [], + version: options?.version || 1, + timestamp: Date.now(), + }; + await this.redisService.setex(key, l2Ttl, JSON.stringify(entry)); + this.logger.debug(`Cache SET: ${key} (L1: ${l1Ttl}s, L2: ${l2Ttl}s)`); + } catch (error) { + this.logger.error(`L2 cache SET error for key ${key}: ${error.message}`); + } + + // Store tags for invalidation + if (options?.tags) { + for (const tag of options.tags) { + await this.addKeyToTag(tag, key); + } + } + } + + /** + * Set value in L1 (in-memory) cache only + */ + private setL1(key: string, value: T, options?: MultiLevelCacheOptions): void { + // Check if we need to evict entries + if (this.l1Cache.size >= this.l1MaxSize && !this.l1Cache.has(key)) { + this.evictL1Entries(); + } + + const ttl = (options?.l1Ttl ?? this.l1DefaultTtl) * 1000; // Convert to ms + const entry: CacheEntry = { + value, + expiresAt: Date.now() + ttl, + version: options?.version || 1, + tags: options?.tags || [], + }; + + this.l1Cache.set(key, entry); + this.stats.l1Size = this.l1Cache.size; + } + + /** + * Delete value from multi-level cache + */ + async del(key: string): Promise { + // Delete from L1 + this.l1Cache.delete(key); + this.stats.l1Size = this.l1Cache.size; + + // Delete from L2 + try { + await this.redisService.del(key); + this.logger.debug(`Cache DEL: ${key}`); + } catch (error) { + this.logger.error(`Cache DEL error for key ${key}: ${error.message}`); + } + } + + /** + * Get with automatic cache population + */ + async wrap(key: string, factory: () => Promise, options?: MultiLevelCacheOptions): Promise { + // Try to get from cache + const cached = await this.get(key); + if (cached !== undefined) { + return cached; + } + + // Generate fresh value + const fresh = await factory(); + + // Store in cache + await this.set(key, fresh, options); + + return fresh; + } + + /** + * Invalidate cache entries by tag + */ + async invalidateByTag(tag: string): Promise { + let count = 0; + + // Get all keys with this tag + const keys = await this.getKeysByTag(tag); + + for (const key of keys) { + await this.del(key); + count++; + } + + // Clear the tag index + await this.redisService.del(`tag:${tag}`); + + this.logger.log(`Invalidated ${count} cache entries by tag: ${tag}`); + return count; + } + + /** + * Invalidate cache entries by pattern + */ + async invalidateByPattern(pattern: string): Promise { + let count = 0; + + // Invalidate in L1 + for (const key of this.l1Cache.keys()) { + if (this.matchPattern(key, pattern)) { + this.l1Cache.delete(key); + count++; + } + } + + // Invalidate in L2 + try { + const keys = await this.redisService.keys(pattern); + for (const key of keys) { + await this.redisService.del(key); + count++; + } + } catch (error) { + this.logger.error(`Pattern invalidation error for ${pattern}: ${error.message}`); + } + + this.logger.log(`Invalidated ${count} cache entries by pattern: ${pattern}`); + return count; + } + + /** + * Invalidate with cascade based on policies + */ + async invalidateWithCascade(key: string): Promise { + const namespace = key.split(':')[0]; + const policies = this.invalidationPolicies.get(namespace) || []; + + // Apply invalidation policies + for (const policy of policies) { + if (policy.type === 'pattern' && typeof policy.value === 'string') { + if (this.matchPattern(key, policy.value)) { + await this.invalidateByPattern(policy.value); + + if (policy.cascade) { + // Cascade to dependent keys + const dependentPatterns = this.getDependentPatterns(namespace); + for (const depPattern of dependentPatterns) { + await this.invalidateByPattern(depPattern); + } + } + } + } + } + + // Invalidate the original key + await this.del(key); + } + + /** + * Register custom invalidation policy + */ + registerInvalidationPolicy(namespace: string, policy: InvalidationPolicy): void { + if (!this.invalidationPolicies.has(namespace)) { + this.invalidationPolicies.set(namespace, []); + } + this.invalidationPolicies.get(namespace)!.push(policy); + this.logger.log(`Registered invalidation policy for namespace: ${namespace}`); + } + + /** + * Get cache statistics + */ + getStats(): CacheStats { + return { ...this.stats }; + } + + /** + * Reset cache statistics + */ + resetStats(): void { + this.stats = { + l1Hits: 0, + l1Misses: 0, + l2Hits: 0, + l2Misses: 0, + totalRequests: 0, + l1HitRate: 0, + l2HitRate: 0, + overallHitRate: 0, + l1Size: this.l1Cache.size, + l2Size: 0, + }; + } + + /** + * Clear all caches + */ + async clear(): Promise { + // Clear L1 + this.l1Cache.clear(); + this.stats.l1Size = 0; + + // Clear L2 (Redis) + try { + await this.redisService.flushdb(); + this.logger.log('All caches cleared'); + } catch (error) { + this.logger.error(`Failed to clear L2 cache: ${error.message}`); + } + } + + /** + * Update hit rates + */ + private updateHitRates(): void { + const l1Total = this.stats.l1Hits + this.stats.l1Misses; + const l2Total = this.stats.l2Hits + this.stats.l2Misses; + + this.stats.l1HitRate = l1Total > 0 ? this.stats.l1Hits / l1Total : 0; + this.stats.l2HitRate = l2Total > 0 ? this.stats.l2Hits / l2Total : 0; + this.stats.overallHitRate = + this.stats.totalRequests > 0 ? (this.stats.l1Hits + this.stats.l2Hits) / this.stats.totalRequests : 0; + } + + /** + * Evict entries from L1 cache (LRU strategy) + */ + private evictL1Entries(): void { + const entriesToEvict = Math.ceil(this.l1MaxSize * 0.1); // Evict 10% + let evicted = 0; + + // Simple LRU: remove oldest entries first + const entries = Array.from(this.l1Cache.entries()); + entries.sort((a, b) => a[1].expiresAt - b[1].expiresAt); + + for (const [key] of entries) { + if (evicted >= entriesToEvict) { + break; + } + this.l1Cache.delete(key); + evicted++; + } + + this.stats.l1Size = this.l1Cache.size; + this.logger.debug(`Evicted ${evicted} entries from L1 cache`); + } + + /** + * Cleanup expired L1 entries + */ + private cleanupExpiredL1Entries(): void { + const now = Date.now(); + let cleaned = 0; + + for (const [key, entry] of this.l1Cache.entries()) { + if (entry.expiresAt <= now) { + this.l1Cache.delete(key); + cleaned++; + } + } + + if (cleaned > 0) { + this.stats.l1Size = this.l1Cache.size; + this.logger.debug(`Cleaned up ${cleaned} expired L1 cache entries`); + } + } + + /** + * Add key to tag index + */ + private async addKeyToTag(tag: string, key: string): Promise { + try { + await this.redisService.sadd(`tag:${tag}`, key); + } catch (error) { + this.logger.error(`Failed to add key to tag index: ${error.message}`); + } + } + + /** + * Get keys by tag + */ + private async getKeysByTag(tag: string): Promise { + try { + return await this.redisService.smembers(`tag:${tag}`); + } catch (error) { + this.logger.error(`Failed to get keys by tag: ${error.message}`); + return []; + } + } + + /** + * Match key against pattern + */ + private matchPattern(key: string, pattern: string): boolean { + const regex = new RegExp(`^${pattern.replace(/\*/g, '.*')}$`); + return regex.test(key); + } + + /** + * Get dependent patterns for cascade invalidation + */ + private getDependentPatterns(namespace: string): string[] { + const dependencies: Record = { + property: ['valuation:property:*', 'document:property:*'], + user: ['permissions:user:*', 'roles:user:*'], + transaction: ['balance:*', 'history:*'], + }; + return dependencies[namespace] || []; + } + + /** + * Get all keys in L1 cache + */ + getL1Keys(): string[] { + return Array.from(this.l1Cache.keys()); + } + + /** + * Get L2 (Redis) cache size + */ + async getL2Size(): Promise { + try { + const keys = await this.redisService.keys('*'); + return keys.length; + } catch (error) { + this.logger.error(`Failed to get L2 size: ${error.message}`); + return 0; + } + } + + /** + * Update cache entry version + */ + async incrementVersion(key: string): Promise { + const entry = this.l1Cache.get(key); + if (entry) { + entry.version++; + } + + try { + const l2Entry = await this.redisService.get(key); + if (l2Entry) { + const parsed = JSON.parse(l2Entry); + parsed.version++; + const ttl = await this.redisService.ttl(key); + await this.redisService.setex(key, ttl > 0 ? ttl : this.l2DefaultTtl, JSON.stringify(parsed)); + return parsed.version; + } + } catch (error) { + this.logger.error(`Failed to increment version: ${error.message}`); + } + + return entry?.version || 1; + } +} diff --git a/src/common/logging/logging.config.ts b/src/common/logging/logging.config.ts index 35dd5cff..cb5a9abf 100644 --- a/src/common/logging/logging.config.ts +++ b/src/common/logging/logging.config.ts @@ -66,10 +66,7 @@ const redactFormat = () => { // Mask database credentials in string messages if (typeof redactedInfo.message === 'string') { - redactedInfo.message = redactedInfo.message.replace( - /(postgres(?:ql)?|mysql):\/\/[^:]+:[^@]+@/g, - '$1://***:***@' - ); + redactedInfo.message = redactedInfo.message.replace(/(postgres(?:ql)?|mysql):\/\/[^:]+:[^@]+@/g, '$1://***:***@'); } // Redact nested data in 'meta' or 'data' fields diff --git a/src/common/services/redis.service.ts b/src/common/services/redis.service.ts index 9144909a..6a3eb629 100644 --- a/src/common/services/redis.service.ts +++ b/src/common/services/redis.service.ts @@ -75,4 +75,16 @@ export class RedisService { async flushdb(): Promise { return await this.redis.flushdb(); } + + async sadd(key: string, ...members: string[]): Promise { + return await this.redis.sadd(key, ...members); + } + + async smembers(key: string): Promise { + return await this.redis.smembers(key); + } + + async srem(key: string, ...members: string[]): Promise { + return await this.redis.srem(key, ...members); + } } diff --git a/src/database/prisma/prisma.service.ts b/src/database/prisma/prisma.service.ts index 1d5e7616..39ccb834 100644 --- a/src/database/prisma/prisma.service.ts +++ b/src/database/prisma/prisma.service.ts @@ -34,7 +34,7 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnModul const separator = databaseUrl.includes('?') ? '&' : '?'; databaseUrl += `${separator}connection_limit=5`; } - + if (!databaseUrl.includes('pool_timeout=')) { const separator = databaseUrl.includes('?') ? '&' : '?'; databaseUrl += `${separator}pool_timeout=10`; diff --git a/src/main.ts b/src/main.ts index 2c3c3751..bb1a60f2 100644 --- a/src/main.ts +++ b/src/main.ts @@ -7,6 +7,7 @@ import * as compression from 'compression'; import { AppModule } from './app.module'; import { StructuredLoggerService } from './common/logging/logger.service'; import { ErrorResponseDto } from './common/errors/error.dto'; +import { SecurityHeadersService } from './security/services/security-headers.service'; async function bootstrap() { const app = await NestFactory.create(AppModule, { @@ -21,15 +22,33 @@ async function bootstrap() { app.use(helmet()); app.use(compression()); - // Enhanced security headers - // const securityHeadersService = app.get(SecurityHeadersService); - // const securityHeaders = securityHeadersService.getSecurityHeaders(); - // Object.entries(securityHeaders).forEach(([key, value]) => { - // app.use((req, res, next) => { - // res.setHeader(key, value); - // next(); - // }); - // }); + // Enhanced security headers - CSP, HSTS, and other security headers + const securityHeadersService = app.get(SecurityHeadersService); + const isProduction = configService.get('NODE_ENV') === 'production'; + + // Get environment-specific security headers configuration + const securityConfig = isProduction + ? undefined // Use default production config + : securityHeadersService.getDevelopmentConfig(); + + // Validate configuration in production + if (isProduction) { + const configErrors = securityHeadersService.validateConfig(securityHeadersService['defaultConfig']); + if (configErrors.length > 0) { + logger.warn(`Security configuration warnings: ${configErrors.join(', ')}`); + } + } + + // Apply security headers middleware + const securityHeaders = securityHeadersService.getSecurityHeaders(securityConfig); + app.use((req: any, res: any, next: () => void) => { + Object.entries(securityHeaders).forEach(([key, value]) => { + res.setHeader(key, value); + }); + next(); + }); + + logger.log(`Security headers configured: ${Object.keys(securityHeaders).length} headers applied`); // CORS configuration app.enableCors({ diff --git a/src/security/guards/sensitive-endpoint-rate-limit.guard.ts b/src/security/guards/sensitive-endpoint-rate-limit.guard.ts index 7fcde636..de1aa8b5 100644 --- a/src/security/guards/sensitive-endpoint-rate-limit.guard.ts +++ b/src/security/guards/sensitive-endpoint-rate-limit.guard.ts @@ -27,10 +27,9 @@ export class SensitiveEndpointRateLimitGuard implements CanActivate { const request = context.switchToHttp().getRequest(); const response = context.switchToHttp().getResponse(); - const options = this.reflector.get( - 'sensitiveRateLimitOptions', - context.getHandler(), - ) || this.getDefaultOptions(); + const options = + this.reflector.get('sensitiveRateLimitOptions', context.getHandler()) || + this.getDefaultOptions(); const ip = this.getClientIp(request); @@ -104,7 +103,7 @@ export class SensitiveEndpointRateLimitGuard implements CanActivate { if (attempts > 0) { const delayMs = Math.min(attempts * 1000, 10000); this.logger.debug(`Applying progressive delay of ${delayMs}ms for key: ${key}`); - await new Promise((resolve) => setTimeout(resolve, delayMs)); + await new Promise(resolve => setTimeout(resolve, delayMs)); } } diff --git a/src/security/middleware/header-validation.middleware.ts b/src/security/middleware/header-validation.middleware.ts new file mode 100644 index 00000000..26bacd91 --- /dev/null +++ b/src/security/middleware/header-validation.middleware.ts @@ -0,0 +1,308 @@ +import { Injectable, NestMiddleware, Logger, BadRequestException } from '@nestjs/common'; +import { Request, Response, NextFunction } from 'express'; + +export interface HeaderValidationResult { + isValid: boolean; + errors: string[]; + warnings: string[]; + sanitizedHeaders: Record; +} + +export interface HeaderValidationConfig { + maxHeaderSize: number; + maxHeaderCount: number; + blockSuspiciousPatterns: boolean; + allowedHeaders?: string[]; + blockedHeaders?: string[]; +} + +@Injectable() +export class HeaderValidationMiddleware implements NestMiddleware { + private readonly logger = new Logger(HeaderValidationMiddleware.name); + private readonly config: HeaderValidationConfig; + + // Patterns that indicate potential attacks + private readonly suspiciousPatterns = [ + /)<[^<]*)*<\/script>/gi, // XSS scripts + /javascript:/gi, + /on\w+\s*=/gi, // Event handlers like onclick= + /data:\s*text\/html/gi, + /vbscript:/gi, + /expression\s*\(/gi, + /union\s+select/gi, // SQL injection + /or\s+1\s*=\s*1/gi, + /;\s*drop\s+table/gi, + /;\s*delete\s+from/gi, + /;\s*insert\s+into/gi, + /\.\.\//g, // Path traversal + /%2e%2e%2f/gi, // Encoded path traversal + /%252e%252e%252f/gi, // Double encoded path traversal + ]; + + // Headers that should be blocked for security reasons + private readonly defaultBlockedHeaders = [ + 'x-forwarded-host', // Can be spoofed + 'x-original-url', // Can be used for SSRF + 'x-rewrite-url', // Can be used for SSRF + ]; + + // Headers that are required for certain operations + private readonly requiredHeadersForApi = ['content-type']; + + constructor() { + this.config = { + maxHeaderSize: 8192, // 8KB max per header + maxHeaderCount: 50, + blockSuspiciousPatterns: true, + blockedHeaders: this.defaultBlockedHeaders, + }; + } + + use(req: Request, res: Response, next: NextFunction): void { + const result = this.validateHeaders(req); + + if (!result.isValid) { + this.logger.warn(`Header validation failed: ${result.errors.join(', ')}`); + throw new BadRequestException({ + statusCode: 400, + message: 'Invalid request headers', + errors: result.errors, + }); + } + + if (result.warnings.length > 0) { + this.logger.debug(`Header validation warnings: ${result.warnings.join(', ')}`); + } + + // Attach sanitized headers to request + (req as any).sanitizedHeaders = result.sanitizedHeaders; + + next(); + } + + /** + * Validate all request headers + */ + validateHeaders(req: Request): HeaderValidationResult { + const errors: string[] = []; + const warnings: string[] = []; + const sanitizedHeaders: Record = {}; + + const headers = req.headers; + + // Check header count + const headerCount = Object.keys(headers).length; + if (headerCount > this.config.maxHeaderCount) { + errors.push(`Too many headers: ${headerCount} (max: ${this.config.maxHeaderCount})`); + } + + // Validate each header + for (const [name, value] of Object.entries(headers)) { + const headerName = name.toLowerCase(); + + // Skip undefined values + if (value === undefined) { + continue; + } + + // Convert array values to string for validation + const headerValue = Array.isArray(value) ? value.join(', ') : String(value); + + // Check header size + if (headerValue.length > this.config.maxHeaderSize) { + errors.push(`Header '${headerName}' exceeds maximum size`); + continue; + } + + // Check for blocked headers + if (this.config.blockedHeaders?.includes(headerName)) { + errors.push(`Blocked header detected: '${headerName}'`); + continue; + } + + // Check for suspicious patterns + if (this.config.blockSuspiciousPatterns) { + const suspiciousCheck = this.detectSuspiciousPatterns(headerName, headerValue); + if (suspiciousCheck.detected) { + errors.push(`Suspicious pattern in header '${headerName}': ${suspiciousCheck.pattern}`); + continue; + } + } + + // Check for null bytes + if (headerValue.includes('\0') || headerName.includes('\0')) { + errors.push(`Null byte detected in header '${headerName}'`); + continue; + } + + // Check for control characters (except common ones) + if (this.containsControlCharacters(headerValue)) { + warnings.push(`Control characters detected in header '${headerName}'`); + } + + // Sanitize and store + sanitizedHeaders[headerName] = this.sanitizeHeaderValue(headerValue); + } + + // Validate Content-Type for POST/PUT/PATCH requests + if (['POST', 'PUT', 'PATCH'].includes(req.method)) { + const contentType = headers['content-type']; + if (!contentType) { + warnings.push('Missing Content-Type header for request with body'); + } else if (!this.isValidContentType(contentType)) { + errors.push(`Invalid Content-Type: ${contentType}`); + } + } + + return { + isValid: errors.length === 0, + errors, + warnings, + sanitizedHeaders, + }; + } + + /** + * Detect suspicious patterns in header value + */ + private detectSuspiciousPatterns(headerName: string, headerValue: string): { detected: boolean; pattern?: string } { + // Skip validation for certain headers that may contain legitimate complex values + const skipPatternCheck = ['authorization', 'cookie', 'x-api-key']; + if (skipPatternCheck.includes(headerName)) { + return { detected: false }; + } + + for (const pattern of this.suspiciousPatterns) { + if (pattern.test(headerValue)) { + return { detected: true, pattern: pattern.source }; + } + } + + return { detected: false }; + } + + /** + * Check if value contains control characters + */ + private containsControlCharacters(value: string): boolean { + // Check for control characters except tab, newline, carriage return + for (let i = 0; i < value.length; i++) { + const code = value.charCodeAt(i); + if (code < 32 && code !== 9 && code !== 10 && code !== 13) { + return true; + } + } + return false; + } + + /** + * Sanitize header value by removing dangerous characters + */ + private sanitizeHeaderValue(value: string): string { + return value + .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, '') // Remove control characters + .replace(/\r\n/g, ' ') // Replace CRLF with space (header injection prevention) + .trim(); + } + + /** + * Validate Content-Type header + */ + private isValidContentType(contentType: string): boolean { + const validContentTypes = [ + 'application/json', + 'application/x-www-form-urlencoded', + 'multipart/form-data', + 'text/plain', + 'text/html', + 'application/xml', + 'text/xml', + ]; + + const baseContentType = contentType.split(';')[0].trim().toLowerCase(); + return validContentTypes.includes(baseContentType) || baseContentType.startsWith('application/'); + } + + /** + * Validate specific header by name + */ + validateHeader(name: string, value: string): { isValid: boolean; error?: string } { + const headerName = name.toLowerCase(); + + if (this.config.blockedHeaders?.includes(headerName)) { + return { isValid: false, error: `Header '${name}' is blocked` }; + } + + if (value.length > this.config.maxHeaderSize) { + return { isValid: false, error: `Header '${name}' exceeds maximum size` }; + } + + const suspiciousCheck = this.detectSuspiciousPatterns(headerName, value); + if (suspiciousCheck.detected) { + return { isValid: false, error: `Suspicious pattern detected: ${suspiciousCheck.pattern}` }; + } + + return { isValid: true }; + } + + /** + * Get security report for headers + */ + getSecurityReport(headers: Record): { + score: number; + issues: string[]; + recommendations: string[]; + } { + const issues: string[] = []; + const recommendations: string[] = []; + let score = 100; + + // Check for missing security headers in response + const securityHeaders = { + 'content-security-policy': 20, + 'strict-transport-security': 15, + 'x-frame-options': 10, + 'x-content-type-options': 10, + 'x-xss-protection': 10, + 'referrer-policy': 5, + 'permissions-policy': 5, + }; + + for (const [header, points] of Object.entries(securityHeaders)) { + if (!headers[header]) { + issues.push(`Missing security header: ${header}`); + score -= points; + } + } + + // Check for information disclosure + if (headers['server'] && headers['server'].length > 20) { + issues.push('Server header reveals detailed information'); + recommendations.push('Configure server header to hide version information'); + score -= 5; + } + + if (headers['x-powered-by']) { + issues.push('X-Powered-By header exposes technology stack'); + recommendations.push('Remove X-Powered-By header'); + score -= 5; + } + + // Check HSTS configuration + if (headers['strict-transport-security']) { + const hsts = headers['strict-transport-security']; + if (!hsts.includes('includeSubDomains')) { + recommendations.push('Add includeSubDomains to HSTS header'); + } + if (!hsts.includes('preload')) { + recommendations.push('Consider adding preload to HSTS header'); + } + } + + return { + score: Math.max(0, score), + issues, + recommendations, + }; + } +} diff --git a/src/security/security.module.ts b/src/security/security.module.ts index cde68b55..3995db64 100644 --- a/src/security/security.module.ts +++ b/src/security/security.module.ts @@ -6,6 +6,7 @@ import { IpBlockingService } from './services/ip-blocking.service'; import { DdosProtectionService } from './services/ddos-protection.service'; import { ApiQuotaService } from './services/api-quota.service'; import { SecurityHeadersService } from './services/security-headers.service'; +import { HeaderValidationMiddleware } from './middleware/header-validation.middleware'; import { SecurityController } from './security.controller'; import { AdvancedRateLimitGuard } from './guards/advanced-rate-limit.guard'; import { SensitiveEndpointRateLimitGuard } from './guards/sensitive-endpoint-rate-limit.guard'; @@ -19,6 +20,7 @@ import { SensitiveEndpointRateLimitGuard } from './guards/sensitive-endpoint-rat DdosProtectionService, ApiQuotaService, SecurityHeadersService, + HeaderValidationMiddleware, AdvancedRateLimitGuard, SensitiveEndpointRateLimitGuard, ], @@ -28,6 +30,7 @@ import { SensitiveEndpointRateLimitGuard } from './guards/sensitive-endpoint-rat DdosProtectionService, ApiQuotaService, SecurityHeadersService, + HeaderValidationMiddleware, AdvancedRateLimitGuard, SensitiveEndpointRateLimitGuard, ], diff --git a/test/common/cache-invalidation.service.spec.ts b/test/common/cache-invalidation.service.spec.ts new file mode 100644 index 00000000..9e1285f2 --- /dev/null +++ b/test/common/cache-invalidation.service.spec.ts @@ -0,0 +1,618 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ConfigService } from '@nestjs/config'; +import { CacheInvalidationService, InvalidationRule } from '../../src/common/cache/cache-invalidation.service'; +import { MultiLevelCacheService } from '../../src/common/cache/multi-level-cache.service'; +import { RedisService } from '../../src/common/services/redis.service'; + +describe('CacheInvalidationService', () => { + let service: CacheInvalidationService; + let cacheService: jest.Mocked; + let redisService: jest.Mocked; + let configService: jest.Mocked; + + beforeEach(async () => { + const mockCacheService = { + get: jest.fn(), + set: jest.fn(), + del: jest.fn(), + invalidateByPattern: jest.fn(), + invalidateByTag: jest.fn(), + invalidateWithCascade: jest.fn(), + }; + + const mockRedisService = { + get: jest.fn(), + setex: jest.fn(), + del: jest.fn(), + keys: jest.fn(), + sadd: jest.fn(), + smembers: jest.fn(), + ttl: jest.fn(), + }; + + const mockConfigService = { + get: jest.fn().mockImplementation((key: string, defaultValue?: any) => defaultValue), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + CacheInvalidationService, + { provide: MultiLevelCacheService, useValue: mockCacheService }, + { provide: RedisService, useValue: mockRedisService }, + { provide: ConfigService, useValue: mockConfigService }, + ], + }).compile(); + + service = module.get(CacheInvalidationService); + cacheService = module.get(MultiLevelCacheService); + redisService = module.get(RedisService); + configService = module.get(ConfigService); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('initialization', () => { + it('should initialize with default rules', () => { + const rules = service.getRules(); + expect(rules.length).toBeGreaterThan(0); + }); + }); + + describe('registerRule', () => { + it('should register a new invalidation rule', () => { + const rule: InvalidationRule = { + id: 'test-rule', + name: 'Test Rule', + description: 'Test description', + type: 'pattern', + target: 'test:*', + action: 'delete', + priority: 5, + enabled: true, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }; + + service.registerRule(rule); + + expect(service.getRule('test-rule')).toEqual(rule); + }); + + it('should update stats when registering a rule', () => { + const initialStats = service.getStats(); + const initialCount = initialStats.totalRules; + + const rule: InvalidationRule = { + id: 'stats-rule', + name: 'Stats Rule', + description: 'Stats description', + type: 'pattern', + target: 'stats:*', + action: 'delete', + priority: 5, + enabled: true, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }; + + service.registerRule(rule); + + const newStats = service.getStats(); + expect(newStats.totalRules).toBe(initialCount + 1); + }); + }); + + describe('unregisterRule', () => { + it('should unregister a rule', () => { + const rule: InvalidationRule = { + id: 'temp-rule', + name: 'Temp Rule', + description: 'Temp description', + type: 'pattern', + target: 'temp:*', + action: 'delete', + priority: 5, + enabled: true, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }; + + service.registerRule(rule); + const result = service.unregisterRule('temp-rule'); + + expect(result).toBe(true); + expect(service.getRule('temp-rule')).toBeUndefined(); + }); + + it('should return false for non-existent rule', () => { + const result = service.unregisterRule('non-existent'); + expect(result).toBe(false); + }); + }); + + describe('setRuleEnabled', () => { + it('should enable/disable a rule', () => { + const rule: InvalidationRule = { + id: 'toggle-rule', + name: 'Toggle Rule', + description: 'Toggle description', + type: 'pattern', + target: 'toggle:*', + action: 'delete', + priority: 5, + enabled: true, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }; + + service.registerRule(rule); + + // Disable + let result = service.setRuleEnabled('toggle-rule', false); + expect(result).toBe(true); + expect(service.getRule('toggle-rule')?.enabled).toBe(false); + + // Enable + result = service.setRuleEnabled('toggle-rule', true); + expect(result).toBe(true); + expect(service.getRule('toggle-rule')?.enabled).toBe(true); + }); + + it('should return false for non-existent rule', () => { + const result = service.setRuleEnabled('non-existent', false); + expect(result).toBe(false); + }); + }); + + describe('executeRule', () => { + it('should execute a pattern rule', async () => { + cacheService.invalidateByPattern.mockResolvedValue(5); + + const rule: InvalidationRule = { + id: 'pattern-rule', + name: 'Pattern Rule', + description: 'Pattern description', + type: 'pattern', + target: 'pattern:*', + action: 'delete', + priority: 5, + enabled: true, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }; + + service.registerRule(rule); + const result = await service.executeRule('pattern-rule'); + + expect(result).toBe(5); + expect(cacheService.invalidateByPattern).toHaveBeenCalledWith('pattern:*'); + }); + + it('should execute a tag rule', async () => { + redisService.keys.mockResolvedValue(['tag:user', 'tag:property']); + cacheService.invalidateByTag.mockResolvedValue(3); + + const rule: InvalidationRule = { + id: 'tag-rule', + name: 'Tag Rule', + description: 'Tag description', + type: 'tag', + target: 'tag:*', + action: 'delete', + priority: 5, + enabled: true, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }; + + service.registerRule(rule); + const result = await service.executeRule('tag-rule'); + + expect(result).toBe(6); // 3 per tag + }); + + it('should skip disabled rules', async () => { + const rule: InvalidationRule = { + id: 'disabled-rule', + name: 'Disabled Rule', + description: 'Disabled description', + type: 'pattern', + target: 'disabled:*', + action: 'delete', + priority: 5, + enabled: false, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }; + + service.registerRule(rule); + const result = await service.executeRule('disabled-rule'); + + expect(result).toBe(0); + expect(cacheService.invalidateByPattern).not.toHaveBeenCalled(); + }); + + it('should return 0 for non-existent rule', async () => { + const result = await service.executeRule('non-existent'); + expect(result).toBe(0); + }); + + it('should execute conditional rule', async () => { + const condition = jest.fn().mockReturnValue(true); + redisService.keys.mockResolvedValue(['key1', 'key2']); + redisService.get.mockResolvedValue(JSON.stringify({ + value: 'test', + timestamp: Date.now(), + })); + redisService.ttl.mockResolvedValue(3600); + + const rule: InvalidationRule = { + id: 'conditional-rule', + name: 'Conditional Rule', + description: 'Conditional description', + type: 'conditional', + target: 'conditional:*', + action: 'delete', + priority: 5, + enabled: true, + condition, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }; + + service.registerRule(rule); + const result = await service.executeRule('conditional-rule'); + + expect(result).toBe(2); + expect(condition).toHaveBeenCalled(); + }); + + it('should execute dependency rule with cascade', async () => { + redisService.keys.mockResolvedValue(['key1', 'key2']); + + const rule: InvalidationRule = { + id: 'dependency-rule', + name: 'Dependency Rule', + description: 'Dependency description', + type: 'dependency', + target: 'dependency:*', + action: 'cascade', + priority: 5, + enabled: true, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }; + + service.registerRule(rule); + const result = await service.executeRule('dependency-rule'); + + expect(result).toBe(2); + expect(cacheService.invalidateWithCascade).toHaveBeenCalledTimes(2); + }); + + it('should handle rule execution errors', async () => { + cacheService.invalidateByPattern.mockRejectedValue(new Error('Execution error')); + + const rule: InvalidationRule = { + id: 'error-rule', + name: 'Error Rule', + description: 'Error description', + type: 'pattern', + target: 'error:*', + action: 'delete', + priority: 5, + enabled: true, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }; + + service.registerRule(rule); + const result = await service.executeRule('error-rule'); + + expect(result).toBe(0); + }); + }); + + describe('executeAllRules', () => { + it('should execute all enabled rules', async () => { + cacheService.invalidateByPattern.mockResolvedValue(1); + + const rule1: InvalidationRule = { + id: 'all-rule-1', + name: 'All Rule 1', + description: 'All Rule 1 description', + type: 'pattern', + target: 'all1:*', + action: 'delete', + priority: 10, + enabled: true, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }; + + const rule2: InvalidationRule = { + id: 'all-rule-2', + name: 'All Rule 2', + description: 'All Rule 2 description', + type: 'pattern', + target: 'all2:*', + action: 'delete', + priority: 5, + enabled: true, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }; + + const disabledRule: InvalidationRule = { + id: 'all-rule-disabled', + name: 'All Rule Disabled', + description: 'All Rule Disabled description', + type: 'pattern', + target: 'disabled:*', + action: 'delete', + priority: 1, + enabled: false, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }; + + service.registerRule(rule1); + service.registerRule(rule2); + service.registerRule(disabledRule); + + const results = await service.executeAllRules(); + + expect(results.get('all-rule-1')).toBe(1); + expect(results.get('all-rule-2')).toBe(1); + expect(results.has('all-rule-disabled')).toBe(false); + }); + }); + + describe('smartInvalidate', () => { + it('should invalidate based on entity type and change type', async () => { + cacheService.invalidateByPattern.mockResolvedValue(1); + + await service.smartInvalidate('property', '123', 'update'); + + expect(cacheService.invalidateByPattern).toHaveBeenCalled(); + }); + + it('should handle create change type', async () => { + cacheService.invalidateByPattern.mockResolvedValue(1); + + await service.smartInvalidate('user', '456', 'create'); + + expect(cacheService.invalidateByPattern).toHaveBeenCalledWith('user:*:list'); + expect(cacheService.invalidateByPattern).toHaveBeenCalledWith('user:active:*'); + }); + + it('should handle delete change type', async () => { + cacheService.invalidateByPattern.mockResolvedValue(1); + + await service.smartInvalidate('transaction', '789', 'delete'); + + expect(cacheService.invalidateByPattern).toHaveBeenCalledWith('transaction:789'); + expect(cacheService.invalidateByPattern).toHaveBeenCalledWith('transaction:*:list'); + expect(cacheService.invalidateByPattern).toHaveBeenCalledWith('balance:*'); + }); + }); + + describe('invalidateByTagsWithPolicy', () => { + it('should invalidate by tags with cascade', async () => { + cacheService.invalidateByTag.mockResolvedValue(3); + cacheService.invalidateByPattern.mockResolvedValue(2); + + const result = await service.invalidateByTagsWithPolicy(['property'], { cascade: true }); + + expect(result).toBeGreaterThanOrEqual(3); + expect(cacheService.invalidateByTag).toHaveBeenCalledWith('property'); + }); + + it('should invalidate by tags without cascade', async () => { + cacheService.invalidateByTag.mockResolvedValue(3); + + const result = await service.invalidateByTagsWithPolicy(['user'], { cascade: false }); + + expect(result).toBe(3); + expect(cacheService.invalidateByPattern).not.toHaveBeenCalled(); + }); + }); + + describe('batchInvalidate', () => { + it('should batch invalidate multiple keys', async () => { + const keys = ['key1', 'key2', 'key3']; + + const result = await service.batchInvalidate(keys); + + expect(result.success).toBe(3); + expect(result.failed).toBe(0); + expect(cacheService.del).toHaveBeenCalledTimes(3); + }); + + it('should handle partial failures', async () => { + const keys = ['key1', 'key2', 'key3']; + cacheService.del + .mockResolvedValueOnce(undefined) + .mockRejectedValueOnce(new Error('Delete error')) + .mockResolvedValueOnce(undefined); + + const result = await service.batchInvalidate(keys); + + expect(result.success).toBe(2); + expect(result.failed).toBe(1); + }); + }); + + describe('invalidateWithCallback', () => { + it('should invalidate and refresh with callback', async () => { + const refreshCallback = jest.fn().mockResolvedValue({ data: 'refreshed' }); + + await service.invalidateWithCallback('test:key', refreshCallback); + + expect(cacheService.del).toHaveBeenCalledWith('test:key'); + expect(refreshCallback).toHaveBeenCalled(); + expect(cacheService.set).toHaveBeenCalledWith('test:key', { data: 'refreshed' }); + }); + + it('should invalidate without callback', async () => { + await service.invalidateWithCallback('test:key'); + + expect(cacheService.del).toHaveBeenCalledWith('test:key'); + expect(cacheService.set).not.toHaveBeenCalled(); + }); + + it('should handle refresh callback errors', async () => { + const refreshCallback = jest.fn().mockRejectedValue(new Error('Refresh error')); + + await service.invalidateWithCallback('test:key', refreshCallback); + + expect(cacheService.del).toHaveBeenCalledWith('test:key'); + expect(refreshCallback).toHaveBeenCalled(); + expect(cacheService.set).not.toHaveBeenCalled(); + }); + }); + + describe('getStats', () => { + it('should return invalidation statistics', () => { + const stats = service.getStats(); + + expect(stats).toHaveProperty('totalRules'); + expect(stats).toHaveProperty('activeRules'); + expect(stats).toHaveProperty('totalExecutions'); + expect(stats).toHaveProperty('successfulInvalidations'); + expect(stats).toHaveProperty('failedInvalidations'); + expect(stats).toHaveProperty('events'); + }); + }); + + describe('getRules', () => { + it('should return all rules', () => { + const rules = service.getRules(); + + expect(Array.isArray(rules)).toBe(true); + expect(rules.length).toBeGreaterThan(0); + }); + }); + + describe('getRule', () => { + it('should return a specific rule', () => { + const rule = service.getRule('rule-user-session-stale'); + + expect(rule).toBeDefined(); + expect(rule?.id).toBe('rule-user-session-stale'); + }); + + it('should return undefined for non-existent rule', () => { + const rule = service.getRule('non-existent'); + + expect(rule).toBeUndefined(); + }); + }); + + describe('scheduledCleanup', () => { + it('should execute time-based rules', async () => { + const timeBasedRule: InvalidationRule = { + id: 'cleanup-rule', + name: 'Cleanup Rule', + description: 'Cleanup description', + type: 'time-based', + target: 'cleanup:*', + action: 'delete', + priority: 5, + enabled: true, + condition: () => true, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }; + + service.registerRule(timeBasedRule); + redisService.keys.mockResolvedValue(['key1']); + redisService.get.mockResolvedValue(JSON.stringify({ + value: 'test', + timestamp: Date.now(), + })); + redisService.ttl.mockResolvedValue(3600); + + await service.scheduledCleanup(); + + expect(redisService.keys).toHaveBeenCalledWith('cleanup:*'); + }); + }); + + describe('scheduledRefresh', () => { + it('should execute conditional refresh rules', async () => { + const refreshRule: InvalidationRule = { + id: 'refresh-rule', + name: 'Refresh Rule', + description: 'Refresh description', + type: 'conditional', + target: 'refresh:*', + action: 'refresh', + priority: 5, + enabled: true, + condition: () => true, + metadata: { + createdAt: new Date(), + lastExecuted: null, + executionCount: 0, + }, + }; + + service.registerRule(refreshRule); + redisService.keys.mockResolvedValue(['key1']); + redisService.get.mockResolvedValue(JSON.stringify({ + value: 'test', + timestamp: Date.now(), + })); + redisService.ttl.mockResolvedValue(3600); + + await service.scheduledRefresh(); + + expect(redisService.keys).toHaveBeenCalledWith('refresh:*'); + }); + }); +}); diff --git a/test/common/cache-warming.service.spec.ts b/test/common/cache-warming.service.spec.ts new file mode 100644 index 00000000..4035e974 --- /dev/null +++ b/test/common/cache-warming.service.spec.ts @@ -0,0 +1,516 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ConfigService } from '@nestjs/config'; +import { CacheWarmingService, WarmupTask, WarmupStrategy } from '../../src/common/cache/cache-warming.service'; +import { MultiLevelCacheService } from '../../src/common/cache/multi-level-cache.service'; + +describe('CacheWarmingService', () => { + let service: CacheWarmingService; + let cacheService: jest.Mocked; + let configService: jest.Mocked; + + beforeEach(async () => { + const mockCacheService = { + get: jest.fn(), + set: jest.fn(), + }; + + const mockConfigService = { + get: jest.fn().mockImplementation((key: string, defaultValue?: any) => defaultValue), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + CacheWarmingService, + { provide: MultiLevelCacheService, useValue: mockCacheService }, + { provide: ConfigService, useValue: mockConfigService }, + ], + }).compile(); + + service = module.get(CacheWarmingService); + cacheService = module.get(MultiLevelCacheService); + configService = module.get(ConfigService); + + await service.onModuleInit(); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + describe('onModuleInit', () => { + it('should initialize with default strategies', async () => { + const strategies = service.getStrategies(); + expect(strategies.length).toBeGreaterThan(0); + expect(strategies.some(s => s.name === 'user-data')).toBe(true); + expect(strategies.some(s => s.name === 'property-data')).toBe(true); + }); + }); + + describe('registerStrategy', () => { + it('should register a new strategy', () => { + const strategy: WarmupStrategy = { + name: 'test-strategy', + description: 'Test strategy', + tasks: [], + enabled: true, + }; + + service.registerStrategy(strategy); + + expect(service.getStrategy('test-strategy')).toEqual(strategy); + }); + }); + + describe('unregisterStrategy', () => { + it('should unregister a strategy', () => { + const strategy: WarmupStrategy = { + name: 'temp-strategy', + description: 'Temp strategy', + tasks: [], + enabled: true, + }; + + service.registerStrategy(strategy); + const result = service.unregisterStrategy('temp-strategy'); + + expect(result).toBe(true); + expect(service.getStrategy('temp-strategy')).toBeUndefined(); + }); + + it('should return false for non-existent strategy', () => { + const result = service.unregisterStrategy('non-existent'); + expect(result).toBe(false); + }); + }); + + describe('setStrategyEnabled', () => { + it('should enable/disable a strategy', () => { + const strategy: WarmupStrategy = { + name: 'toggle-strategy', + description: 'Toggle strategy', + tasks: [], + enabled: true, + }; + + service.registerStrategy(strategy); + + // Disable + let result = service.setStrategyEnabled('toggle-strategy', false); + expect(result).toBe(true); + expect(service.getStrategy('toggle-strategy')?.enabled).toBe(false); + + // Enable + result = service.setStrategyEnabled('toggle-strategy', true); + expect(result).toBe(true); + expect(service.getStrategy('toggle-strategy')?.enabled).toBe(true); + }); + + it('should return false for non-existent strategy', () => { + const result = service.setStrategyEnabled('non-existent', false); + expect(result).toBe(false); + }); + }); + + describe('executeStrategy', () => { + it('should execute a strategy and cache results', async () => { + const factory = jest.fn().mockResolvedValue({ data: 'test' }); + const strategy: WarmupStrategy = { + name: 'exec-strategy', + description: 'Exec strategy', + tasks: [ + { + key: 'test:key', + factory, + priority: 5, + }, + ], + enabled: true, + }; + + service.registerStrategy(strategy); + cacheService.get.mockResolvedValue(undefined); + + await service.executeStrategy('exec-strategy'); + + expect(factory).toHaveBeenCalled(); + expect(cacheService.set).toHaveBeenCalledWith( + 'test:key', + { data: 'test' }, + undefined, + ); + }); + + it('should skip already cached entries', async () => { + const factory = jest.fn().mockResolvedValue({ data: 'test' }); + const strategy: WarmupStrategy = { + name: 'skip-strategy', + description: 'Skip strategy', + tasks: [ + { + key: 'cached:key', + factory, + priority: 5, + }, + ], + enabled: true, + }; + + service.registerStrategy(strategy); + cacheService.get.mockResolvedValue({ data: 'existing' }); + + await service.executeStrategy('skip-strategy'); + + expect(factory).not.toHaveBeenCalled(); + expect(cacheService.set).not.toHaveBeenCalled(); + }); + + it('should skip disabled strategies', async () => { + const strategy: WarmupStrategy = { + name: 'disabled-strategy', + description: 'Disabled strategy', + tasks: [ + { + key: 'test:key', + factory: jest.fn(), + priority: 5, + }, + ], + enabled: false, + }; + + service.registerStrategy(strategy); + + await service.executeStrategy('disabled-strategy'); + + expect(cacheService.get).not.toHaveBeenCalled(); + }); + + it('should handle factory errors gracefully', async () => { + const factory = jest.fn().mockRejectedValue(new Error('Factory error')); + const strategy: WarmupStrategy = { + name: 'error-strategy', + description: 'Error strategy', + tasks: [ + { + key: 'error:key', + factory, + priority: 5, + }, + ], + enabled: true, + }; + + service.registerStrategy(strategy); + cacheService.get.mockResolvedValue(undefined); + + await service.executeStrategy('error-strategy'); + + expect(factory).toHaveBeenCalled(); + expect(cacheService.set).not.toHaveBeenCalled(); + }); + + it('should check condition before executing task', async () => { + const factory = jest.fn().mockResolvedValue({ data: 'test' }); + const condition = jest.fn().mockReturnValue(false); + const strategy: WarmupStrategy = { + name: 'condition-strategy', + description: 'Condition strategy', + tasks: [ + { + key: 'conditional:key', + factory, + priority: 5, + condition, + }, + ], + enabled: true, + }; + + service.registerStrategy(strategy); + + await service.executeStrategy('condition-strategy'); + + expect(condition).toHaveBeenCalled(); + expect(factory).not.toHaveBeenCalled(); + }); + + it('should handle async conditions', async () => { + const factory = jest.fn().mockResolvedValue({ data: 'test' }); + const condition = jest.fn().mockResolvedValue(true); + const strategy: WarmupStrategy = { + name: 'async-condition-strategy', + description: 'Async condition strategy', + tasks: [ + { + key: 'async:key', + factory, + priority: 5, + condition, + }, + ], + enabled: true, + }; + + service.registerStrategy(strategy); + cacheService.get.mockResolvedValue(undefined); + + await service.executeStrategy('async-condition-strategy'); + + expect(condition).toHaveBeenCalled(); + expect(factory).toHaveBeenCalled(); + }); + }); + + describe('executeAllStrategies', () => { + it('should execute all enabled strategies', async () => { + // Create fresh mock for this test + const mockSet = jest.fn(); + cacheService.set = mockSet; + + // First disable all existing strategies + for (const s of service.getStrategies()) { + service.setStrategyEnabled(s.name, false); + } + + const strategy1: WarmupStrategy = { + name: 'strategy1', + description: 'Strategy 1', + tasks: [ + { + key: 'key1', + factory: jest.fn().mockResolvedValue('value1'), + priority: 5, + }, + ], + enabled: true, + }; + + const strategy2: WarmupStrategy = { + name: 'strategy2', + description: 'Strategy 2', + tasks: [ + { + key: 'key2', + factory: jest.fn().mockResolvedValue('value2'), + priority: 5, + }, + ], + enabled: true, + }; + + const disabledStrategy: WarmupStrategy = { + name: 'disabled', + description: 'Disabled', + tasks: [ + { + key: 'key3', + factory: jest.fn(), + priority: 5, + }, + ], + enabled: false, + }; + + service.registerStrategy(strategy1); + service.registerStrategy(strategy2); + service.registerStrategy(disabledStrategy); + + cacheService.get.mockResolvedValue(undefined); + + await service.executeAllStrategies(); + + expect(mockSet).toHaveBeenCalledTimes(2); + }); + }); + + describe('executeTask', () => { + it('should execute a single task', async () => { + const task: WarmupTask = { + key: 'single:key', + factory: jest.fn().mockResolvedValue({ data: 'test' }), + priority: 5, + }; + + cacheService.get.mockResolvedValue(undefined); + + const result = await service.executeTask(task); + + expect(result).toBe(true); + expect(cacheService.set).toHaveBeenCalled(); + }); + + it('should return false if condition not met', async () => { + const task: WarmupTask = { + key: 'conditional:key', + factory: jest.fn(), + priority: 5, + condition: () => false, + }; + + const result = await service.executeTask(task); + + expect(result).toBe(false); + expect(cacheService.set).not.toHaveBeenCalled(); + }); + + it('should return false if already cached', async () => { + const task: WarmupTask = { + key: 'cached:key', + factory: jest.fn(), + priority: 5, + }; + + cacheService.get.mockResolvedValue({ data: 'existing' }); + + const result = await service.executeTask(task); + + expect(result).toBe(false); + expect(cacheService.set).not.toHaveBeenCalled(); + }); + + it('should return false on factory error', async () => { + const task: WarmupTask = { + key: 'error:key', + factory: jest.fn().mockRejectedValue(new Error('Factory error')), + priority: 5, + }; + + cacheService.get.mockResolvedValue(undefined); + + const result = await service.executeTask(task); + + expect(result).toBe(false); + }); + }); + + describe('warmCache', () => { + it('should warm cache with multiple tasks', async () => { + const tasks: WarmupTask[] = [ + { + key: 'key1', + factory: jest.fn().mockResolvedValue('value1'), + priority: 5, + }, + { + key: 'key2', + factory: jest.fn().mockResolvedValue('value2'), + priority: 3, + }, + ]; + + cacheService.get.mockResolvedValue(undefined); + + const result = await service.warmCache(tasks); + + expect(result.completed).toBe(2); + expect(result.failed).toBe(0); + }); + + it('should sort tasks by priority', async () => { + const executionOrder: string[] = []; + + const tasks: WarmupTask[] = [ + { + key: 'low', + factory: jest.fn().mockImplementation(async () => { + executionOrder.push('low'); + return 'value'; + }), + priority: 1, + }, + { + key: 'high', + factory: jest.fn().mockImplementation(async () => { + executionOrder.push('high'); + return 'value'; + }), + priority: 10, + }, + ]; + + cacheService.get.mockResolvedValue(undefined); + + await service.warmCache(tasks); + + expect(executionOrder[0]).toBe('high'); + expect(executionOrder[1]).toBe('low'); + }); + }); + + describe('getStats', () => { + it('should return warming statistics', () => { + const stats = service.getStats(); + + expect(stats).toHaveProperty('totalTasks'); + expect(stats).toHaveProperty('completedTasks'); + expect(stats).toHaveProperty('failedTasks'); + expect(stats).toHaveProperty('skippedTasks'); + expect(stats).toHaveProperty('averageExecutionTime'); + }); + }); + + describe('resetStats', () => { + it('should reset all statistics', async () => { + // Execute a strategy to generate stats + const strategy: WarmupStrategy = { + name: 'stats-strategy', + description: 'Stats strategy', + tasks: [ + { + key: 'key1', + factory: jest.fn().mockResolvedValue('value'), + priority: 5, + }, + ], + enabled: true, + }; + + service.registerStrategy(strategy); + cacheService.get.mockResolvedValue(undefined); + + await service.executeStrategy('stats-strategy'); + + // Reset stats + service.resetStats(); + + const stats = service.getStats(); + expect(stats.totalTasks).toBe(0); + expect(stats.completedTasks).toBe(0); + }); + }); + + describe('getStrategies', () => { + it('should return all registered strategies', () => { + const strategies = service.getStrategies(); + + expect(Array.isArray(strategies)).toBe(true); + expect(strategies.length).toBeGreaterThan(0); + }); + }); + + describe('getStrategy', () => { + it('should return a specific strategy', () => { + const strategy = service.getStrategy('user-data'); + + expect(strategy).toBeDefined(); + expect(strategy?.name).toBe('user-data'); + }); + + it('should return undefined for non-existent strategy', () => { + const strategy = service.getStrategy('non-existent'); + + expect(strategy).toBeUndefined(); + }); + }); + + describe('prewarmOnStartup', () => { + it('should prewarm critical strategies', async () => { + cacheService.get.mockResolvedValue(undefined); + + await service.prewarmOnStartup(); + + // Should have attempted to warm critical strategies + expect(cacheService.get).toHaveBeenCalled(); + }); + }); +}); diff --git a/test/common/multi-level-cache.service.spec.ts b/test/common/multi-level-cache.service.spec.ts new file mode 100644 index 00000000..09b3cd68 --- /dev/null +++ b/test/common/multi-level-cache.service.spec.ts @@ -0,0 +1,424 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { ConfigService } from '@nestjs/config'; +import { MultiLevelCacheService, MultiLevelCacheOptions } from '../../src/common/cache/multi-level-cache.service'; +import { RedisService } from '../../src/common/services/redis.service'; + +describe('MultiLevelCacheService', () => { + let service: MultiLevelCacheService; + let redisService: jest.Mocked; + let configService: jest.Mocked; + + beforeEach(async () => { + const mockRedisService = { + get: jest.fn(), + setex: jest.fn(), + del: jest.fn(), + keys: jest.fn(), + sadd: jest.fn(), + smembers: jest.fn(), + flushdb: jest.fn(), + ttl: jest.fn(), + }; + + const mockConfigService = { + get: jest.fn((key: string, defaultValue?: any) => { + const config: Record = { + CACHE_L1_MAX_SIZE: 100, + CACHE_L1_TTL: 300, + CACHE_L2_TTL: 3600, + }; + return config[key] ?? defaultValue; + }), + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + MultiLevelCacheService, + { provide: RedisService, useValue: mockRedisService }, + { provide: ConfigService, useValue: mockConfigService }, + ], + }).compile(); + + service = module.get(MultiLevelCacheService); + redisService = module.get(RedisService); + configService = module.get(ConfigService); + + // Initialize the service + await service.onModuleInit(); + }); + + afterEach(async () => { + await service.onModuleDestroy(); + jest.clearAllMocks(); + }); + + describe('get', () => { + it('should return value from L1 cache if present and not expired', async () => { + const key = 'test:key'; + const value = { data: 'test' }; + + // First set the value + await service.set(key, value); + + // Get should return from L1 + const result = await service.get(key); + + expect(result).toEqual(value); + expect(redisService.get).not.toHaveBeenCalled(); + }); + + it('should fetch from L2 cache if not in L1', async () => { + const key = 'test:key'; + const value = { data: 'test' }; + + redisService.get.mockResolvedValue(JSON.stringify({ + value, + tags: [], + version: 1, + timestamp: Date.now(), + })); + + const result = await service.get(key); + + expect(result).toEqual(value); + expect(redisService.get).toHaveBeenCalledWith(key); + }); + + it('should return undefined if not in any cache level', async () => { + const key = 'test:key'; + + redisService.get.mockResolvedValue(null); + + const result = await service.get(key); + + expect(result).toBeUndefined(); + }); + + it('should handle L2 cache errors gracefully', async () => { + const key = 'test:key'; + + redisService.get.mockRejectedValue(new Error('Redis error')); + + const result = await service.get(key); + + expect(result).toBeUndefined(); + }); + }); + + describe('set', () => { + it('should set value in both L1 and L2 cache', async () => { + const key = 'test:key'; + const value = { data: 'test' }; + const options: MultiLevelCacheOptions = { l1Ttl: 100, l2Ttl: 200 }; + + await service.set(key, value, options); + + // Should be in L1 + const l1Result = await service.get(key); + expect(l1Result).toEqual(value); + + // Should be set in L2 + expect(redisService.setex).toHaveBeenCalledWith( + key, + 200, + expect.any(String), + ); + }); + + it('should use default TTLs when not specified', async () => { + const key = 'test:key'; + const value = { data: 'test' }; + + await service.set(key, value); + + expect(redisService.setex).toHaveBeenCalledWith( + key, + 3600, + expect.any(String), + ); + }); + + it('should store tags for invalidation', async () => { + const key = 'test:key'; + const value = { data: 'test' }; + const options: MultiLevelCacheOptions = { tags: ['tag1', 'tag2'] }; + + await service.set(key, value, options); + + expect(redisService.sadd).toHaveBeenCalledWith('tag:tag1', key); + expect(redisService.sadd).toHaveBeenCalledWith('tag:tag2', key); + }); + }); + + describe('del', () => { + it('should delete from both L1 and L2 cache', async () => { + const key = 'test:key'; + + // First set the value + await service.set(key, { data: 'test' }); + + // Then delete it + await service.del(key); + + // Should be deleted from L1 + const l1Result = await service.get(key); + expect(l1Result).toBeUndefined(); + + // Should be deleted from L2 + expect(redisService.del).toHaveBeenCalledWith(key); + }); + }); + + describe('wrap', () => { + it('should return cached value if present', async () => { + const key = 'test:key'; + const value = { data: 'cached' }; + const factory = jest.fn().mockResolvedValue({ data: 'fresh' }); + + await service.set(key, value); + + const result = await service.wrap(key, factory); + + expect(result).toEqual(value); + expect(factory).not.toHaveBeenCalled(); + }); + + it('should call factory and cache result if not present', async () => { + const key = 'test:key'; + const value = { data: 'fresh' }; + const factory = jest.fn().mockResolvedValue(value); + + redisService.get.mockResolvedValue(null); + + const result = await service.wrap(key, factory); + + expect(result).toEqual(value); + expect(factory).toHaveBeenCalled(); + expect(redisService.setex).toHaveBeenCalled(); + }); + }); + + describe('invalidateByTag', () => { + it('should invalidate all entries with a given tag', async () => { + const tag = 'user'; + const keys = ['user:1', 'user:2']; + + redisService.smembers.mockResolvedValue(keys); + + const result = await service.invalidateByTag(tag); + + expect(result).toBe(2); + expect(redisService.del).toHaveBeenCalledWith('user:1'); + expect(redisService.del).toHaveBeenCalledWith('user:2'); + expect(redisService.del).toHaveBeenCalledWith(`tag:${tag}`); + }); + + it('should return 0 if no keys found for tag', async () => { + const tag = 'nonexistent'; + + redisService.smembers.mockResolvedValue([]); + + const result = await service.invalidateByTag(tag); + + expect(result).toBe(0); + }); + }); + + describe('invalidateByPattern', () => { + it('should invalidate entries matching pattern', async () => { + const pattern = 'user:*'; + const keys = ['user:1', 'user:2', 'user:3']; + + redisService.keys.mockResolvedValue(keys); + + const result = await service.invalidateByPattern(pattern); + + expect(result).toBe(3); + expect(redisService.keys).toHaveBeenCalledWith(pattern); + expect(redisService.del).toHaveBeenCalledTimes(3); + }); + + it('should handle L1 cache entries matching pattern', async () => { + const pattern = 'test:*'; + + // Set some L1 cache entries + await service.set('test:1', { data: 1 }); + await service.set('test:2', { data: 2 }); + await service.set('other:1', { data: 3 }); + + redisService.keys.mockResolvedValue([]); + + await service.invalidateByPattern(pattern); + + // L1 entries matching pattern should be deleted + expect(await service.get('test:1')).toBeUndefined(); + expect(await service.get('test:2')).toBeUndefined(); + // Non-matching entry should remain + expect(await service.get('other:1')).toEqual({ data: 3 }); + }); + }); + + describe('invalidateWithCascade', () => { + it('should invalidate with cascade for property namespace', async () => { + const key = 'property:123'; + + redisService.keys.mockResolvedValue([]); + + await service.invalidateWithCascade(key); + + expect(redisService.del).toHaveBeenCalledWith(key); + }); + }); + + describe('getStats', () => { + it('should return cache statistics', () => { + const stats = service.getStats(); + + expect(stats).toHaveProperty('l1Hits'); + expect(stats).toHaveProperty('l1Misses'); + expect(stats).toHaveProperty('l2Hits'); + expect(stats).toHaveProperty('l2Misses'); + expect(stats).toHaveProperty('l1HitRate'); + expect(stats).toHaveProperty('l2HitRate'); + expect(stats).toHaveProperty('overallHitRate'); + }); + + it('should track hits and misses correctly', async () => { + const key = 'test:key'; + + // Reset stats + service.resetStats(); + + // First access - miss + redisService.get.mockResolvedValue(null); + await service.get(key); + + // Set value + await service.set(key, { data: 'test' }); + + // Second access - L1 hit + await service.get(key); + + const stats = service.getStats(); + expect(stats.l1Hits).toBe(1); + expect(stats.l1Misses).toBe(1); + }); + }); + + describe('resetStats', () => { + it('should reset all statistics', async () => { + // Generate some stats + await service.set('key1', 'value1'); + await service.get('key1'); + + service.resetStats(); + + const stats = service.getStats(); + expect(stats.l1Hits).toBe(0); + expect(stats.l1Misses).toBe(0); + expect(stats.l2Hits).toBe(0); + expect(stats.l2Misses).toBe(0); + }); + }); + + describe('clear', () => { + it('should clear both L1 and L2 cache', async () => { + // Set some values + await service.set('key1', 'value1'); + await service.set('key2', 'value2'); + + await service.clear(); + + // L1 should be empty + expect(service.getL1Keys()).toHaveLength(0); + + // L2 should be flushed + expect(redisService.flushdb).toHaveBeenCalled(); + }); + }); + + describe('registerInvalidationPolicy', () => { + it('should register custom invalidation policy', () => { + const policy = { + type: 'pattern' as const, + value: 'custom:*', + cascade: true, + }; + + service.registerInvalidationPolicy('custom', policy); + + // Policy should be registered (no error thrown) + expect(() => service.registerInvalidationPolicy('custom', policy)).not.toThrow(); + }); + }); + + describe('incrementVersion', () => { + it('should increment version for existing entry', async () => { + const key = 'test:key'; + const value = { data: 'test' }; + + await service.set(key, value, { version: 1 }); + + redisService.get.mockResolvedValue(JSON.stringify({ + value, + tags: [], + version: 1, + timestamp: Date.now(), + })); + redisService.ttl.mockResolvedValue(3600); + + const newVersion = await service.incrementVersion(key); + + expect(newVersion).toBe(2); + }); + + it('should return 1 for non-existing entry', async () => { + const key = 'nonexistent:key'; + + redisService.get.mockResolvedValue(null); + + const version = await service.incrementVersion(key); + + expect(version).toBe(1); + }); + }); + + describe('L1 cache eviction', () => { + it('should evict entries when L1 cache is full', async () => { + // Set config to small size for testing + jest.spyOn(configService, 'get').mockImplementation((key: string, defaultValue?: any) => { + if (key === 'CACHE_L1_MAX_SIZE') return 2; + return defaultValue; + }); + + // Create new service with small cache size + const smallCacheService = new MultiLevelCacheService(redisService, configService); + await smallCacheService.onModuleInit(); + + // Add entries up to and beyond limit + await smallCacheService.set('key1', 'value1'); + await smallCacheService.set('key2', 'value2'); + await smallCacheService.set('key3', 'value3'); + + // Some entries should have been evicted + const l1Keys = smallCacheService.getL1Keys(); + expect(l1Keys.length).toBeLessThanOrEqual(2); + + await smallCacheService.onModuleDestroy(); + }); + }); + + describe('cleanup', () => { + it('should clean up expired L1 entries', async () => { + // Set a value with very short TTL + await service.set('key1', 'value1', { l1Ttl: 0 }); + + // Wait a bit + await new Promise(resolve => setTimeout(resolve, 10)); + + // The entry should eventually be cleaned up by the interval + // For testing, we can check that the cleanup doesn't throw + expect(service.getL1Keys()).not.toThrow; + }); + }); +}); diff --git a/test/security/header-validation.middleware.spec.ts b/test/security/header-validation.middleware.spec.ts new file mode 100644 index 00000000..7e4b9408 --- /dev/null +++ b/test/security/header-validation.middleware.spec.ts @@ -0,0 +1,314 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { BadRequestException } from '@nestjs/common'; +import { HeaderValidationMiddleware } from '../../src/security/middleware/header-validation.middleware'; + +describe('HeaderValidationMiddleware', () => { + let middleware: HeaderValidationMiddleware; + + const mockRequest = (headers: Record = {}, method = 'GET') => + ({ + headers, + method, + }) as any; + + const mockResponse = () => ({} as any); + + const mockNext = jest.fn(); + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [HeaderValidationMiddleware], + }).compile(); + + middleware = module.get(HeaderValidationMiddleware); + jest.clearAllMocks(); + }); + + it('should be defined', () => { + expect(middleware).toBeDefined(); + }); + + describe('use', () => { + it('should pass valid headers', () => { + const req = mockRequest({ + 'content-type': 'application/json', + authorization: 'Bearer token', + }); + const res = mockResponse(); + + middleware.use(req, res, mockNext); + + expect(mockNext).toHaveBeenCalled(); + expect(req.sanitizedHeaders).toBeDefined(); + }); + + it('should throw BadRequestException for blocked headers', () => { + const req = mockRequest({ + 'x-forwarded-host': 'malicious.com', + }); + const res = mockResponse(); + + expect(() => middleware.use(req, res, mockNext)).toThrow(BadRequestException); + }); + + it('should throw BadRequestException for XSS patterns', () => { + const req = mockRequest({ + 'x-custom-header': '', + }); + const res = mockResponse(); + + expect(() => middleware.use(req, res, mockNext)).toThrow(BadRequestException); + }); + + it('should throw BadRequestException for SQL injection patterns', () => { + const req = mockRequest({ + 'x-custom-header': "'; DROP TABLE users; --", + }); + const res = mockResponse(); + + expect(() => middleware.use(req, res, mockNext)).toThrow(BadRequestException); + }); + + it('should throw BadRequestException for null bytes', () => { + const req = mockRequest({ + 'x-custom-header': 'value\x00injection', + }); + const res = mockResponse(); + + expect(() => middleware.use(req, res, mockNext)).toThrow(BadRequestException); + }); + + it('should pass headers with authorization', () => { + const req = mockRequest({ + authorization: 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.test', + }); + const res = mockResponse(); + + middleware.use(req, res, mockNext); + + expect(mockNext).toHaveBeenCalled(); + }); + + it('should pass headers with API key', () => { + const req = mockRequest({ + 'x-api-key': 'propchain_live_abc123xyz', + }); + const res = mockResponse(); + + middleware.use(req, res, mockNext); + + expect(mockNext).toHaveBeenCalled(); + }); + }); + + describe('validateHeaders', () => { + it('should return valid for empty headers', () => { + const req = mockRequest({}); + const result = middleware.validateHeaders(req); + + expect(result.isValid).toBe(true); + expect(result.errors).toHaveLength(0); + }); + + it('should return invalid for oversized header', () => { + const longValue = 'a'.repeat(10000); + const req = mockRequest({ 'x-long-header': longValue }); + const result = middleware.validateHeaders(req); + + expect(result.isValid).toBe(false); + expect(result.errors[0]).toContain('exceeds maximum size'); + }); + + it('should return invalid for too many headers', () => { + const headers: Record = {}; + for (let i = 0; i < 60; i++) { + headers[`x-header-${i}`] = `value-${i}`; + } + const req = mockRequest(headers); + const result = middleware.validateHeaders(req); + + expect(result.isValid).toBe(false); + expect(result.errors[0]).toContain('Too many headers'); + }); + + it('should return warning for control characters', () => { + const req = mockRequest({ 'x-custom': 'value\x01test' }); + const result = middleware.validateHeaders(req); + + expect(result.warnings.length).toBeGreaterThan(0); + }); + + it('should return warning for missing Content-Type on POST', () => { + const req = mockRequest({}, 'POST'); + const result = middleware.validateHeaders(req); + + expect(result.warnings).toContain('Missing Content-Type header for request with body'); + }); + + it('should return invalid for invalid Content-Type', () => { + const req = mockRequest({ 'content-type': 'invalid/type' }, 'POST'); + const result = middleware.validateHeaders(req); + + expect(result.isValid).toBe(false); + expect(result.errors[0]).toContain('Invalid Content-Type'); + }); + + it('should sanitize headers', () => { + const req = mockRequest({ 'x-custom': ' value ' }); + const result = middleware.validateHeaders(req); + + expect(result.sanitizedHeaders['x-custom']).toBe('value'); + }); + }); + + describe('validateHeader', () => { + it('should return valid for normal header', () => { + const result = middleware.validateHeader('x-custom', 'value'); + + expect(result.isValid).toBe(true); + }); + + it('should return invalid for blocked header', () => { + const result = middleware.validateHeader('x-forwarded-host', 'malicious.com'); + + expect(result.isValid).toBe(false); + expect(result.error).toContain('blocked'); + }); + + it('should return invalid for oversized header', () => { + const longValue = 'a'.repeat(10000); + const result = middleware.validateHeader('x-custom', longValue); + + expect(result.isValid).toBe(false); + expect(result.error).toContain('exceeds maximum size'); + }); + + it('should return invalid for path traversal', () => { + const result = middleware.validateHeader('x-custom', '../../../etc/passwd'); + + expect(result.isValid).toBe(false); + }); + }); + + describe('getSecurityReport', () => { + it('should return high score for complete security headers', () => { + const headers = { + 'content-security-policy': "default-src 'self'", + 'strict-transport-security': 'max-age=31536000; includeSubDomains; preload', + 'x-frame-options': 'DENY', + 'x-content-type-options': 'nosniff', + 'x-xss-protection': '1; mode=block', + 'referrer-policy': 'strict-origin-when-cross-origin', + 'permissions-policy': 'geolocation=()', + }; + + const report = middleware.getSecurityReport(headers); + + expect(report.score).toBe(100); + expect(report.issues).toHaveLength(0); + }); + + it('should return low score for missing security headers', () => { + const headers = { + server: 'Apache/2.4.41 (Ubuntu) PHP/7.4.3', + 'x-powered-by': 'Express', + }; + + const report = middleware.getSecurityReport(headers); + + expect(report.score).toBeLessThan(50); + expect(report.issues.length).toBeGreaterThan(0); + expect(report.recommendations.length).toBeGreaterThan(0); + }); + + it('should detect information disclosure', () => { + const headers = { + server: 'Apache/2.4.41 (Ubuntu) with detailed version info', + 'x-powered-by': 'Express', + }; + + const report = middleware.getSecurityReport(headers); + + expect(report.issues).toContain('Server header reveals detailed information'); + expect(report.issues).toContain('X-Powered-By header exposes technology stack'); + }); + + it('should recommend HSTS improvements', () => { + const headers = { + 'strict-transport-security': 'max-age=31536000', + }; + + const report = middleware.getSecurityReport(headers); + + expect(report.recommendations).toContain('Add includeSubDomains to HSTS header'); + expect(report.recommendations).toContain('Consider adding preload to HSTS header'); + }); + }); + + describe('detectSuspiciousPatterns', () => { + it('should detect XSS patterns', () => { + const result = (middleware as any).detectSuspiciousPatterns( + 'x-custom', + '', + ); + + expect(result.detected).toBe(true); + }); + + it('should detect javascript: protocol', () => { + const result = (middleware as any).detectSuspiciousPatterns( + 'x-custom', + 'javascript:alert(1)', + ); + + expect(result.detected).toBe(true); + }); + + it('should detect SQL injection patterns', () => { + const result = (middleware as any).detectSuspiciousPatterns( + 'x-custom', + "' OR 1=1 --", + ); + + expect(result.detected).toBe(true); + }); + + it('should skip validation for authorization header', () => { + const result = (middleware as any).detectSuspiciousPatterns( + 'authorization', + 'Bearer ', + ); + + expect(result.detected).toBe(false); + }); + + it('should skip validation for cookie header', () => { + const result = (middleware as any).detectSuspiciousPatterns( + 'cookie', + 'session=', + ); + + expect(result.detected).toBe(false); + }); + }); + + describe('isValidContentType', () => { + it('should return true for application/json', () => { + const result = (middleware as any).isValidContentType('application/json'); + + expect(result).toBe(true); + }); + + it('should return true for multipart/form-data', () => { + const result = (middleware as any).isValidContentType('multipart/form-data; boundary=----'); + + expect(result).toBe(true); + }); + + it('should return false for invalid content type', () => { + const result = (middleware as any).isValidContentType('invalid/type'); + + expect(result).toBe(false); + }); + }); +});