diff --git a/.gitignore b/.gitignore index 148f13a1..62f60dd9 100644 --- a/.gitignore +++ b/.gitignore @@ -162,7 +162,7 @@ config/local.json config/production.json # Hardhat -cache/ +# cache/ # Commented out to allow source code cache directory artifacts/ typechain-types/ diff --git a/package-lock.json b/package-lock.json index 846f494d..35fd312a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,6 +11,7 @@ "dependencies": { "@liaoliaots/nestjs-redis": "^10.0.0", "@nestjs/bull": "^10.0.1", + "@nestjs/cache-manager": "^3.1.0", "@nestjs/common": "^10.3.0", "@nestjs/config": "^3.1.1", "@nestjs/core": "^10.3.0", @@ -27,6 +28,9 @@ "axios": "^1.6.2", "bcrypt": "^6.0.0", "bull": "^4.12.2", + "cache-manager": "^7.2.8", + "cache-manager-ioredis-yet": "^2.1.2", + "cache-manager-redis-store": "^3.0.1", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", "cls-hooked": "^4.2.2", @@ -838,6 +842,25 @@ "url": "https://github.com/sponsors/Borewit" } }, + "node_modules/@cacheable/utils": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/@cacheable/utils/-/utils-2.3.4.tgz", + "integrity": "sha512-knwKUJEYgIfwShABS1BX6JyJJTglAFcEU7EXqzTdiGCXur4voqkiJkdgZIQtWNFhynzDWERcTYv/sETMu3uJWA==", + "license": "MIT", + "dependencies": { + "hashery": "^1.3.0", + "keyv": "^5.6.0" + } + }, + "node_modules/@cacheable/utils/node_modules/keyv": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.6.0.tgz", + "integrity": "sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==", + "license": "MIT", + "dependencies": { + "@keyv/serialize": "^1.1.1" + } + }, "node_modules/@colors/colors": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", @@ -2862,6 +2885,12 @@ "url": "https://opencollective.com/js-sdsl" } }, + "node_modules/@keyv/serialize": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", + "integrity": "sha512-dXn3FZhPv0US+7dtJsIi2R+c7qWYiReoEh5zUntWCf4oSpMNib8FDhSoed6m3QyZdx5hK7iLFkYk3rNxwt8vTA==", + "license": "MIT" + }, "node_modules/@liaoliaots/nestjs-redis": { "version": "10.0.0", "resolved": "https://registry.npmjs.org/@liaoliaots/nestjs-redis/-/nestjs-redis-10.0.0.tgz", @@ -3019,6 +3048,19 @@ "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0" } }, + "node_modules/@nestjs/cache-manager": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@nestjs/cache-manager/-/cache-manager-3.1.0.tgz", + "integrity": "sha512-pEIqYZrBcE8UdkJmZRduurvoUfdU+3kRPeO1R2muiMbZnRuqlki5klFFNllO9LyYWzrx98bd1j0PSPKSJk1Wbw==", + "license": "MIT", + "peerDependencies": { + "@nestjs/common": "^9.0.0 || ^10.0.0 || ^11.0.0", + "@nestjs/core": "^9.0.0 || ^10.0.0 || ^11.0.0", + "cache-manager": ">=6", + "keyv": ">=5", + "rxjs": "^7.8.1" + } + }, "node_modules/@nestjs/cli": { "version": "10.4.9", "resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-10.4.9.tgz", @@ -6700,6 +6742,52 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/cache-manager": { + "version": "7.2.8", + "resolved": "https://registry.npmjs.org/cache-manager/-/cache-manager-7.2.8.tgz", + "integrity": "sha512-0HDaDLBBY/maa/LmUVAr70XUOwsiQD+jyzCBjmUErYZUKdMS9dT59PqW59PpVqfGM7ve6H0J6307JTpkCYefHQ==", + "license": "MIT", + "dependencies": { + "@cacheable/utils": "^2.3.3", + "keyv": "^5.5.5" + } + }, + "node_modules/cache-manager-ioredis-yet": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/cache-manager-ioredis-yet/-/cache-manager-ioredis-yet-2.1.2.tgz", + "integrity": "sha512-p/5D+ADvJaZjAs12fR5l0ZJ+rK2EqbCryFdrzsMj3K+lGwNoCjB33N6V397otgreB+iwK+lssBshpkJDodiyMQ==", + "deprecated": "With cache-manager v6 we now are using Keyv", + "license": "MIT", + "dependencies": { + "cache-manager": "*", + "ioredis": "^5.4.1", + "telejson": "^7.2.0" + }, + "engines": { + "node": ">= 18" + } + }, + "node_modules/cache-manager-redis-store": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/cache-manager-redis-store/-/cache-manager-redis-store-3.0.1.tgz", + "integrity": "sha512-o560kw+dFqusC9lQJhcm6L2F2fMKobJ5af+FoR2PdnMVdpQ3f3Bz6qzvObTGyvoazQJxjQNWgMQeChP4vRTuXQ==", + "license": "MIT", + "dependencies": { + "redis": "^4.3.1" + }, + "engines": { + "node": ">= 16.18.0" + } + }, + "node_modules/cache-manager/node_modules/keyv": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.6.0.tgz", + "integrity": "sha512-CYDD3SOtsHtyXeEORYRx2qBtpDJFjRTGXUtmNEMGyzYOKj1TE3tycdlho7kA1Ufx9OYWZzg52QFBGALTirzDSw==", + "license": "MIT", + "dependencies": { + "@keyv/serialize": "^1.1.1" + } + }, "node_modules/call-bind": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.8.tgz", @@ -10129,6 +10217,18 @@ "minimalistic-assert": "^1.0.1" } }, + "node_modules/hashery": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/hashery/-/hashery-1.5.0.tgz", + "integrity": "sha512-nhQ6ExaOIqti2FDWoEMWARUqIKyjr2VcZzXShrI+A3zpeiuPWzx6iPftt44LhP74E5sW36B75N6VHbvRtpvO6Q==", + "license": "MIT", + "dependencies": { + "hookified": "^1.14.0" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -10170,6 +10270,12 @@ "minimalistic-crypto-utils": "^1.0.1" } }, + "node_modules/hookified": { + "version": "1.15.1", + "resolved": "https://registry.npmjs.org/hookified/-/hookified-1.15.1.tgz", + "integrity": "sha512-MvG/clsADq1GPM2KGo2nyfaWVyn9naPiXrqIe4jYjXNZQt238kWyOGrsyc/DmRAQ+Re6yeo6yX/yoNCG5KAEVg==", + "license": "MIT" + }, "node_modules/hosted-git-info": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-4.1.0.tgz", @@ -12777,6 +12883,12 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/map-or-similar": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/map-or-similar/-/map-or-similar-1.5.0.tgz", + "integrity": "sha512-0aF7ZmVon1igznGI4VS30yugpduQW3y3GkcgGJOp7d8x8QrizhigUxjI/m2UojsXXto+jLAH3KSz+xOJTiORjg==", + "license": "MIT" + }, "node_modules/mapped-types": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/mapped-types/-/mapped-types-0.0.1.tgz", @@ -12817,6 +12929,15 @@ "node": ">= 4.0.0" } }, + "node_modules/memoizerific": { + "version": "1.11.3", + "resolved": "https://registry.npmjs.org/memoizerific/-/memoizerific-1.11.3.tgz", + "integrity": "sha512-/EuHYwAPdLtXwAwSZkh/Gutery6pD2KYd44oQLhAvQp/50mpyduZh8Q7PYHXTCJ+wuXxt7oij2LXyIJOOYFPog==", + "license": "MIT", + "dependencies": { + "map-or-similar": "^1.5.0" + } + }, "node_modules/memorystream": { "version": "0.3.1", "resolved": "https://registry.npmjs.org/memorystream/-/memorystream-0.3.1.tgz", @@ -15964,6 +16085,15 @@ "streamx": "^2.15.0" } }, + "node_modules/telejson": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/telejson/-/telejson-7.2.0.tgz", + "integrity": "sha512-1QTEcJkJEhc8OnStBx/ILRu5J2p0GjvWsBx56bmZRqnrkdBMUe+nX92jxV+p3dB4CP6PZCdJMQJwCggkNBMzkQ==", + "license": "MIT", + "dependencies": { + "memoizerific": "^1.11.3" + } + }, "node_modules/terser": { "version": "5.46.0", "resolved": "https://registry.npmjs.org/terser/-/terser-5.46.0.tgz", diff --git a/package.json b/package.json index db263e12..302432e9 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "dependencies": { "@liaoliaots/nestjs-redis": "^10.0.0", "@nestjs/bull": "^10.0.1", + "@nestjs/cache-manager": "^3.1.0", "@nestjs/common": "^10.3.0", "@nestjs/config": "^3.1.1", "@nestjs/core": "^10.3.0", @@ -61,6 +62,9 @@ "axios": "^1.6.2", "bcrypt": "^6.0.0", "bull": "^4.12.2", + "cache-manager": "^7.2.8", + "cache-manager-ioredis-yet": "^2.1.2", + "cache-manager-redis-store": "^3.0.1", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", "cls-hooked": "^4.2.2", @@ -136,30 +140,7 @@ "tsconfig-paths": "^4.2.0", "typescript": "^5.3.2" }, - "jest": { - "moduleFileExtensions": [ - "js", - "json", - "ts" - ], - "roots": [ - "src", - "test" - ], - "testRegex": ".*\\.spec\\.ts$", - "transform": { - "^.+\\.(t|j)s$": "ts-jest" - }, - "collectCoverageFrom": [ - "**/*.(t|j)s" - ], - "coverageDirectory": "../coverage", - "testEnvironment": "node", - "moduleNameMapping": { - "^src/(.*)$": "/../src/$1", - "^test/(.*)$": "/../test/$1" - } - }, + "engines": { "node": ">=18.0.0", "npm": ">=8.0.0" diff --git a/src/app.module.ts b/src/app.module.ts index 4aaf5d2b..3b3654f6 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -13,6 +13,9 @@ import { ConfigurationModule } from './config/configuration.module'; import configuration from './config/configuration'; import valuationConfig from './config/valuation.config'; +// Caching +import { CacheModule } from './common/cache/cache.module'; + // Logging import { LoggingModule } from './common/logging/logging.module'; import { LoggingInterceptor } from './common/logging/logging.interceptor'; @@ -50,6 +53,9 @@ import { AuthRateLimitMiddleware } from './auth/middleware/auth.middleware'; }), ConfigurationModule, + // Caching + CacheModule, + // Core LoggingModule, PrismaModule, @@ -89,7 +95,7 @@ import { AuthRateLimitMiddleware } from './auth/middleware/auth.middleware'; FilesModule, ValuationModule, DocumentsModule, - + // Compliance & Security AuditModule, RbacModule, @@ -111,4 +117,4 @@ export class AppModule implements NestModule { .apply(AuthRateLimitMiddleware) .forRoutes('/auth*'); } -} \ No newline at end of file +} diff --git a/src/common/audit/audit.module.ts b/src/common/audit/audit.module.ts index 5bbf6394..c7051fdf 100644 --- a/src/common/audit/audit.module.ts +++ b/src/common/audit/audit.module.ts @@ -8,19 +8,7 @@ import { EncryptionService } from '../services/encryption.service'; @Module({ imports: [DatabaseModule], - providers: [ - AuditService, - ComplianceReportingService, - DataRetentionService, - GdprService, - EncryptionService, - ], - exports: [ - AuditService, - ComplianceReportingService, - DataRetentionService, - GdprService, - EncryptionService, - ], + providers: [AuditService, ComplianceReportingService, DataRetentionService, GdprService, EncryptionService], + exports: [AuditService, ComplianceReportingService, DataRetentionService, GdprService, EncryptionService], }) -export class AuditModule {} \ No newline at end of file +export class AuditModule {} diff --git a/src/common/cache/cache.module.ts b/src/common/cache/cache.module.ts new file mode 100644 index 00000000..c2a96d8b --- /dev/null +++ b/src/common/cache/cache.module.ts @@ -0,0 +1,27 @@ +import { Module, Global } from '@nestjs/common'; +import { CacheModule as NestCacheModule } from '@nestjs/cache-manager'; +import { ConfigService } from '@nestjs/config'; +import * as redisStore from 'cache-manager-redis-store'; +import { CacheService } from '../services/cache.service'; +import { RedisService } from '../services/redis.service'; + +@Global() +@Module({ + imports: [ + NestCacheModule.registerAsync({ + useFactory: async (configService: ConfigService) => ({ + store: redisStore, + host: configService.get('REDIS_HOST', 'localhost'), + port: configService.get('REDIS_PORT', 6379), + password: configService.get('REDIS_PASSWORD'), + db: configService.get('REDIS_DB', 0), + ttl: configService.get('CACHE_DEFAULT_TTL', 3600), + max: configService.get('CACHE_MAX_ITEMS', 1000), + }), + inject: [ConfigService], + }), + ], + providers: [CacheService, RedisService], + exports: [CacheService, NestCacheModule], +}) +export class CacheModule {} diff --git a/src/common/controllers/audit.controller.ts b/src/common/controllers/audit.controller.ts index a82deea9..8b83882a 100644 --- a/src/common/controllers/audit.controller.ts +++ b/src/common/controllers/audit.controller.ts @@ -13,14 +13,7 @@ import { UseInterceptors, Req, } from '@nestjs/common'; -import { - ApiTags, - ApiOperation, - ApiResponse, - ApiParam, - ApiQuery, - ApiBearerAuth, -} from '@nestjs/swagger'; +import { ApiTags, ApiOperation, ApiResponse, ApiParam, ApiQuery, ApiBearerAuth } from '@nestjs/swagger'; import { JwtAuthGuard } from '../../auth/guards/jwt-auth.guard'; import { RbacGuard } from '../../auth/guards/rbac.guard'; import { AuditService, AuditOperation } from '../services/audit.service'; @@ -34,6 +27,7 @@ import { Action } from '../../rbac/enums/action.enum'; import { RequireScopes } from '../decorators/require-scopes.decorator'; import { AuditInterceptor } from '../interceptors/audit.interceptor'; +// Audit controller @ApiTags('Audit & Compliance') @Controller('audit') @UseGuards(JwtAuthGuard, RbacGuard) @@ -63,7 +57,7 @@ export class AuditController { @Query('endDate') endDate?: string, ) { const skip = (page - 1) * limit; - + // Use audit service methods instead of direct prisma access let logs; if (tableName) { @@ -107,10 +101,7 @@ export class AuditController { @ApiQuery({ name: 'endDate', required: true }) @RequireScopes(Resource.AUDIT, Action.READ) @UseInterceptors(AuditInterceptor) - async generateComplianceReport( - @Query('startDate') startDate: string, - @Query('endDate') endDate: string, - ) { + async generateComplianceReport(@Query('startDate') startDate: string, @Query('endDate') endDate: string) { const start = new Date(startDate); const end = new Date(endDate); @@ -226,7 +217,7 @@ export class AuditController { @UseInterceptors(AuditInterceptor) async getAuditStats() { const logCounts = await this.auditService.getLogCounts(); - + return { totalLogs: Object.values(logCounts).reduce((sum, count) => sum + count, 0), byOperation: logCounts, @@ -240,9 +231,9 @@ export class AuditController { @UseInterceptors(AuditInterceptor) async testEncryption() { if (!this.encryptionService.isEncryptionConfigured()) { - return { - status: 'warning', - message: 'Encryption is not properly configured. ENCRYPTION_KEY is missing.' + return { + status: 'warning', + message: 'Encryption is not properly configured. ENCRYPTION_KEY is missing.', }; } @@ -254,7 +245,7 @@ export class AuditController { return { status: 'success', testPassed: testData === decrypted, - encryptedSample: encrypted.encrypted.substring(0, 20) + '...', + encryptedSample: `${encrypted.encrypted.substring(0, 20)}...`, }; } catch (error) { return { @@ -264,4 +255,4 @@ export class AuditController { }; } } -} \ No newline at end of file +} diff --git a/src/common/interceptors/audit.interceptor.ts b/src/common/interceptors/audit.interceptor.ts index 3c44dc4f..1a8f616c 100644 --- a/src/common/interceptors/audit.interceptor.ts +++ b/src/common/interceptors/audit.interceptor.ts @@ -1,10 +1,4 @@ -import { - Injectable, - NestInterceptor, - ExecutionContext, - CallHandler, - Logger, -} from '@nestjs/common'; +import { Injectable, NestInterceptor, ExecutionContext, CallHandler, Logger } from '@nestjs/common'; import { Observable } from 'rxjs'; import { tap } from 'rxjs/operators'; import { Reflector } from '@nestjs/core'; @@ -34,10 +28,8 @@ export class AuditInterceptor implements NestInterceptor { const parentClass = context.getClass(); // Get audit metadata from decorators - const auditMetadata: AuditMetadata = this.reflector.get( - 'audit', - handler, - ) || this.reflector.get('audit', parentClass); + const auditMetadata: AuditMetadata = + this.reflector.get('audit', handler) || this.reflector.get('audit', parentClass); // Skip audit if disabled if (auditMetadata?.enabled === false) { @@ -52,14 +44,14 @@ export class AuditInterceptor implements NestInterceptor { // Process the request first return next.handle().pipe( tap({ - next: async (result) => { + next: async result => { try { const duration = Date.now() - startTime; - + // Determine operation type based on HTTP method const method = request.method; const operation = this.getOperationFromMethod(method); - + if (operation === AuditOperation.READ) { // For read operations, we might want to log differently return; @@ -97,7 +89,7 @@ export class AuditInterceptor implements NestInterceptor { this.logger.error('Failed to log audit trail:', error); } }, - error: async (error) => { + error: async error => { try { const method = request.method; const userId = request.user?.id || null; @@ -153,9 +145,7 @@ export class AuditInterceptor implements NestInterceptor { // Extract entity name from route (e.g., /api/users/:id -> users) const parts = routePath.split('/'); // Look for the first non-parameter segment after /api or similar prefixes - const relevantParts = parts.filter(part => - part && !part.startsWith(':') && part !== 'api' && part !== 'v1' - ); + const relevantParts = parts.filter(part => part && !part.startsWith(':') && part !== 'api' && part !== 'v1'); if (relevantParts.length > 0) { // Take the first relevant part and pluralize if needed @@ -210,7 +200,9 @@ export class AuditInterceptor implements NestInterceptor { * Filter out sensitive data from audit logs */ private filterSensitiveData(data: any, excludeFields: string[]): any { - if (!data) return data; + if (!data) { + return data; + } // Handle arrays if (Array.isArray(data)) { @@ -220,7 +212,7 @@ export class AuditInterceptor implements NestInterceptor { // Handle objects if (typeof data === 'object') { const filtered: any = {}; - + for (const [key, value] of Object.entries(data)) { // Skip excluded fields if (excludeFields.includes(key)) { @@ -267,8 +259,6 @@ export class AuditInterceptor implements NestInterceptor { ]; const lowerFieldName = fieldName.toLowerCase(); - return sensitiveFields.some(sensitive => - lowerFieldName.includes(sensitive) - ); + return sensitiveFields.some(sensitive => lowerFieldName.includes(sensitive)); } -} \ No newline at end of file +} diff --git a/src/common/services/audit.service.ts b/src/common/services/audit.service.ts index e716c033..d84c02e2 100644 --- a/src/common/services/audit.service.ts +++ b/src/common/services/audit.service.ts @@ -42,11 +42,7 @@ export class AuditService { /** * Creates an audit log for a create operation */ - async logCreate( - tableName: string, - newData: T, - userId?: string, - ): Promise { + async logCreate(tableName: string, newData: T, userId?: string): Promise { await this.logAction({ tableName, operation: AuditOperation.CREATE, @@ -58,12 +54,7 @@ export class AuditService { /** * Creates an audit log for an update operation */ - async logUpdate( - tableName: string, - oldData: T, - newData: T, - userId?: string, - ): Promise { + async logUpdate(tableName: string, oldData: T, newData: T, userId?: string): Promise { await this.logAction({ tableName, operation: AuditOperation.UPDATE, @@ -76,11 +67,7 @@ export class AuditService { /** * Creates an audit log for a delete operation */ - async logDelete( - tableName: string, - oldData: T, - userId?: string, - ): Promise { + async logDelete(tableName: string, oldData: T, userId?: string): Promise { await this.logAction({ tableName, operation: AuditOperation.DELETE, @@ -94,36 +81,23 @@ export class AuditService { */ private sanitizeData(data: Record): Record { const sanitized = { ...data }; - + // Remove sensitive fields - const sensitiveFields = [ - 'password', - 'secret', - 'token', - 'apiKey', - 'privateKey', - 'encrypted', - 'salt', - 'hash' - ]; - + const sensitiveFields = ['password', 'secret', 'token', 'apiKey', 'privateKey', 'encrypted', 'salt', 'hash']; + sensitiveFields.forEach(field => { if (sanitized[field]) { sanitized[field] = '[REDACTED]'; } }); - + return sanitized; } /** * Get audit logs for a specific table */ - async getTableLogs( - tableName: string, - limit: number = 100, - offset: number = 0, - ) { + async getTableLogs(tableName: string, limit: number = 100, offset: number = 0) { return await this.prisma.auditLog.findMany({ where: { tableName }, orderBy: { timestamp: 'desc' }, @@ -135,11 +109,7 @@ export class AuditService { /** * Get audit logs for a specific user */ - async getUserLogs( - userId: string, - limit: number = 100, - offset: number = 0, - ) { + async getUserLogs(userId: string, limit: number = 100, offset: number = 0) { return await this.prisma.auditLog.findMany({ where: { userId }, orderBy: { timestamp: 'desc' }, @@ -151,11 +121,7 @@ export class AuditService { /** * Get audit logs by operation type */ - async getLogsByOperation( - operation: AuditOperation, - limit: number = 100, - offset: number = 0, - ) { + async getLogsByOperation(operation: AuditOperation, limit: number = 100, offset: number = 0) { return await this.prisma.auditLog.findMany({ where: { operation }, orderBy: { timestamp: 'desc' }, @@ -167,12 +133,7 @@ export class AuditService { /** * Get audit logs within a date range */ - async getLogsByDateRange( - startDate: Date, - endDate: Date, - limit: number = 100, - offset: number = 0, - ) { + async getLogsByDateRange(startDate: Date, endDate: Date, limit: number = 100, offset: number = 0) { return await this.prisma.auditLog.findMany({ where: { timestamp: { @@ -225,17 +186,27 @@ export class AuditService { endDate?: Date, ): Promise { const where: any = {}; - - if (tableName) where.tableName = tableName; - if (operation) where.operation = operation; - if (userId) where.userId = userId; - + + if (tableName) { + where.tableName = tableName; + } + if (operation) { + where.operation = operation; + } + if (userId) { + where.userId = userId; + } + if (startDate || endDate) { where.timestamp = {}; - if (startDate) where.timestamp.gte = startDate; - if (endDate) where.timestamp.lte = endDate; + if (startDate) { + where.timestamp.gte = startDate; + } + if (endDate) { + where.timestamp.lte = endDate; + } } return await this.prisma.auditLog.count({ where }); } -} \ No newline at end of file +} diff --git a/src/common/services/cache.service.ts b/src/common/services/cache.service.ts new file mode 100644 index 00000000..b30f1a6a --- /dev/null +++ b/src/common/services/cache.service.ts @@ -0,0 +1,1044 @@ +import { Injectable, Logger, Inject } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { CACHE_MANAGER } from '@nestjs/cache-manager'; +import { Cache } from 'cache-manager'; +import { RedisService } from './redis.service'; + +export interface CacheOptions { + ttl?: number; // Time to live in seconds + staleWhileRevalidate?: number; // Time in seconds to serve stale data while revalidating +} + +export interface CacheMetrics { + hits: number; + misses: number; + hitRate: number; + totalRequests: number; +} + +export interface CacheInvalidationRule { + pattern: string; + dependentKeys?: string[]; + cascade?: boolean; + condition?: (value: any) => boolean; +} + +@Injectable() +export class CacheService { + private readonly logger = new Logger(CacheService.name); + private metrics: Map = new Map(); + private invalidationRules: Map = new Map(); + + constructor( + @Inject(CACHE_MANAGER) private cache: Cache, + private redisService: RedisService, + private configService: ConfigService, + ) { + // Initialize metrics for common cache namespaces + this.initializeMetrics(); + // Initialize common invalidation rules + this.initializeInvalidationRules(); + } + + private initializeMetrics() { + const namespaces = ['valuation', 'property', 'user', 'transaction', 'document']; + namespaces.forEach(ns => { + this.metrics.set(ns, { + hits: 0, + misses: 0, + hitRate: 0, + totalRequests: 0, + }); + }); + } + + private initializeInvalidationRules() { + // Define common invalidation rules + this.invalidationRules.set('property', [ + { + pattern: 'property:*', + dependentKeys: ['valuation:*'], + cascade: true, + }, + { + pattern: 'property:*', + dependentKeys: ['document:property:*'], + cascade: true, + }, + ]); + + this.invalidationRules.set('valuation', [ + { + pattern: 'valuation:*', + dependentKeys: ['valuation:history:*'], + cascade: true, + }, + ]); + + this.invalidationRules.set('user', [ + { + pattern: 'user:*', + dependentKeys: ['user:permissions:*', 'user:roles:*'], + cascade: true, + }, + ]); + } + + /** + * Get a value from cache + */ + async get(key: string): Promise { + try { + const value = await this.cache.get(key); + if (value !== undefined) { + this.incrementHit(key); + this.trackAccessPattern(key, 'get'); + this.logger.debug(`Cache HIT: ${key}`); + return value; + } else { + this.incrementMiss(key); + this.trackAccessPattern(key, 'get'); + this.logger.debug(`Cache MISS: ${key}`); + return undefined; + } + } catch (error) { + this.logger.error(`Cache GET error for key ${key}: ${error.message}`); + return undefined; + } + } + + /** + * Set a value in cache + */ + async set(key: string, value: T, options?: CacheOptions): Promise { + try { + const ttl = options?.ttl || this.getDefaultTtl(); + await this.cache.set(key, value, ttl); + this.trackAccessPattern(key, 'set'); + this.logger.debug(`Cache SET: ${key} with TTL: ${ttl}s`); + } catch (error) { + this.logger.error(`Cache SET error for key ${key}: ${error.message}`); + } + } + + /** + * Get a value from cache with auto-population if not exists + */ + async wrap(key: string, factory: () => Promise, options?: CacheOptions): Promise { + // Try to get from cache first + const cachedValue = await this.get(key); + if (cachedValue !== undefined) { + return cachedValue; + } + + // If not in cache, call the factory function + const freshValue = await factory(); + + // Set in cache with options + await this.set(key, freshValue, options); + + return freshValue; + } + + /** + * Delete a value from cache + */ + async del(key: string): Promise { + try { + await this.cache.del(key); + this.trackAccessPattern(key, 'del'); + this.logger.debug(`Cache DEL: ${key}`); + } catch (error) { + this.logger.error(`Cache DEL error for key ${key}: ${error.message}`); + } + } + + /** + * Clear all cache entries + */ + async clear(): Promise { + try { + // For Redis, use flushdb to clear all keys + await this.redisService.flushdb(); + this.logger.debug('Cache cleared'); + } catch (error) { + this.logger.error(`Cache CLEAR error: ${error.message}`); + } + } + + /** + * Get cache keys matching a pattern + */ + async keys(pattern: string): Promise { + try { + return await this.redisService.keys(pattern); + } catch (error) { + this.logger.error(`Cache KEYS error for pattern ${pattern}: ${error.message}`); + return []; + } + } + + /** + * Check if a key exists in cache + */ + async has(key: string): Promise { + try { + const exists = await this.redisService.exists(key); + return exists; + } catch (error) { + this.logger.error(`Cache HAS error for key ${key}: ${error.message}`); + return false; + } + } + + /** + * Get TTL for a key + */ + async ttl(key: string): Promise { + try { + return await this.redisService.ttl(key); + } catch (error) { + this.logger.error(`Cache TTL error for key ${key}: ${error.message}`); + return -1; + } + } + + /** + * Increment cache hit counter + */ + private incrementHit(key: string): void { + const namespace = this.getNamespace(key); + const metrics = this.metrics.get(namespace); + if (metrics) { + metrics.hits++; + metrics.totalRequests++; + metrics.hitRate = metrics.hits / metrics.totalRequests; + } + } + + /** + * Increment cache miss counter + */ + private incrementMiss(key: string): void { + const namespace = this.getNamespace(key); + const metrics = this.metrics.get(namespace); + if (metrics) { + metrics.misses++; + metrics.totalRequests++; + metrics.hitRate = metrics.hits / metrics.totalRequests; + } + } + + /** + * Get namespace from cache key + */ + private getNamespace(key: string): string { + const parts = key.split(':'); + return parts[0] || 'default'; + } + + /** + * Get default TTL from config + */ + private getDefaultTtl(): number { + return this.configService.get('CACHE_DEFAULT_TTL', 3600); // 1 hour default + } + + /** + * Get metrics for a specific namespace + */ + getMetrics(namespace: string): CacheMetrics | undefined { + return this.metrics.get(namespace); + } + + /** + * Get all metrics + */ + getAllMetrics(): Map { + return this.metrics; + } + + /** + * Invalidate cache entries by pattern + */ + async invalidateByPattern(pattern: string): Promise { + const keys = await this.keys(pattern); + if (keys.length > 0) { + for (const key of keys) { + await this.del(key); + } + this.logger.log(`Invalidated ${keys.length} cache entries matching pattern: ${pattern}`); + } + } + + /** + * Invalidate cache entries by key with cascade invalidation + */ + async invalidateWithCascade(key: string): Promise { + const namespace = this.getNamespace(key); + const rules = this.invalidationRules.get(namespace) || []; + + for (const rule of rules) { + if (key.match(new RegExp(rule.pattern.replace(/\*/g, '.*')))) { + if (rule.dependentKeys) { + for (const dependentPattern of rule.dependentKeys) { + await this.invalidateByPattern(dependentPattern); + } + } + } + } + + // Also invalidate the original key + await this.del(key); + this.logger.log(`Invalidated cache key: ${key} with cascade`); + } + + /** + * Invalidate cache with condition + */ + async conditionalInvalidate(pattern: string, condition: (value: any) => boolean): Promise { + const keys = await this.keys(pattern); + + for (const key of keys) { + const value = await this.get(key); + if (value && condition(value)) { + await this.del(key); + this.logger.log(`Conditionally invalidated cache key: ${key}`); + } + } + } + + /** + * Tag-based cache invalidation + */ + async invalidateByTags(tags: string[]): Promise { + for (const tag of tags) { + const pattern = `tag:${tag}:*`; + await this.invalidateByPattern(pattern); + } + this.logger.log(`Invalidated cache by tags: ${tags.join(', ')}`); + } + + /** + * Add tag to a cache entry + */ + async tagEntry(key: string, tags: string[]): Promise { + for (const tag of tags) { + const tagKey = `tag:${tag}:${key}`; + await this.set(tagKey, key, { ttl: this.getDefaultTtl() * 2 }); // Longer TTL for tags + } + } + + /** + * Time-based cache invalidation + */ + async invalidateOlderThan(keyPattern: string, maxAgeSeconds: number): Promise { + const keys = await this.keys(keyPattern); + const cutoffTime = Date.now() - maxAgeSeconds * 1000; + + for (const key of keys) { + // For Redis, we'll rely on TTL, but we can also implement manual checks + // For now, we'll just invalidate all matching keys + await this.del(key); + } + + this.logger.log(`Invalidated cache entries older than ${maxAgeSeconds}s for pattern: ${keyPattern}`); + } + + /** + * Get cache with fallback mechanism + */ + async getWithFallback(key: string, fallbackFactory: () => Promise, options?: CacheOptions): Promise { + try { + // First, try primary cache + const cachedValue = await this.get(key); + if (cachedValue !== undefined) { + return cachedValue; + } + + // If not in cache, get from fallback and cache it + const freshValue = await fallbackFactory(); + await this.set(key, freshValue, options); + return freshValue; + } catch (error) { + this.logger.error(`Cache operation failed for key ${key}, attempting fallback: ${error.message}`); + + // Try fallback even if cache operations failed + try { + return await fallbackFactory(); + } catch (fallbackError) { + this.logger.error(`Fallback also failed for key ${key}: ${fallbackError.message}`); + throw fallbackError; + } + } + } + + /** + * Conditional cache update with version checking + */ + async conditionalSet( + key: string, + value: T, + condition: (cachedValue?: T) => boolean, + options?: CacheOptions, + ): Promise { + const currentValue = await this.get(key); + + if (condition(currentValue)) { + await this.set(key, value, options); + return true; + } + + return false; + } + + /** + * Batch get multiple values + */ + async mget(...keys: string[]): Promise<(T | undefined)[]> { + const results: (T | undefined)[] = []; + + for (const key of keys) { + try { + const value = await this.get(key); + results.push(value); + } catch (error) { + this.logger.error(`Cache MGET error for key ${key}: ${error.message}`); + results.push(undefined); + } + } + + return results; + } + + /** + * Batch set multiple values + */ + async mset(entries: Array<{ key: string; value: T; options?: CacheOptions }>): Promise { + const promises = entries.map(entry => this.set(entry.key, entry.value, entry.options)); + + await Promise.all(promises); + } + + /** + * Register an invalidation rule + */ + registerInvalidationRule(namespace: string, rule: CacheInvalidationRule): void { + if (!this.invalidationRules.has(namespace)) { + this.invalidationRules.set(namespace, []); + } + this.invalidationRules.get(namespace)?.push(rule); + this.logger.log(`Registered invalidation rule for namespace: ${namespace}`); + } + + /** + * Invalidate cache with dependencies + */ + async invalidateWithDependencies(key: string, invalidateDependents: boolean = true): Promise { + if (invalidateDependents) { + await this.invalidateWithCascade(key); + } else { + await this.del(key); + } + } + + /** + * Warm the cache with frequently accessed data + */ + async warmCache( + warmupTasks: Array<{ + key: string; + factory: () => Promise; + options?: CacheOptions; + condition?: () => boolean; + }>, + ): Promise { + const promises = warmupTasks.map(async task => { + try { + // Check condition if provided + if (task.condition && !task.condition()) { + return; + } + + // Skip if already cached + const cachedValue = await this.get(task.key); + if (cachedValue !== undefined) { + this.logger.log(`Cache WARM skipped (already cached): ${task.key}`); + return; + } + + // Fetch and cache the data + const value = await task.factory(); + await this.set(task.key, value, task.options); + this.logger.log(`Cache WARM completed: ${task.key}`); + } catch (error) { + this.logger.error(`Cache WARM failed for key ${task.key}: ${error.message}`); + } + }); + + await Promise.all(promises); + } + + /** + * Warm the cache with frequently accessed data in background + */ + async warmCacheBackground( + warmupTasks: Array<{ + key: string; + factory: () => Promise; + options?: CacheOptions; + condition?: () => boolean; + }>, + ): Promise { + // Run cache warming in background to not block the main thread + setImmediate(async () => { + await this.warmCache(warmupTasks); + }); + } + + /** + * Get frequently accessed keys based on access patterns + */ + async getFrequentlyAccessedKeys(limit: number = 10): Promise { + // In a real implementation, this would track access patterns + // For now, return common patterns + const allKeys = await this.keys('*'); + + // Sort by access frequency (this is a simplified version) + // In production, we'd track access counts per key + return allKeys.slice(0, limit); + } + + /** + * Preload commonly accessed data patterns + */ + async preloadCommonPatterns(): Promise { + // Preload common patterns like popular property valuations + // This would typically be scheduled or triggered based on usage analytics + + // Example: Preload top 10 most viewed properties + // This would need integration with analytics data + + this.logger.log('Starting common patterns preload'); + + // In a real system, this would fetch from analytics or business logic + // For demonstration, we'll create some common patterns + const commonPatterns = ['property:popular:*', 'valuation:recent:*', 'user:active:*']; + + for (const pattern of commonPatterns) { + const keys = await this.keys(pattern); + this.logger.log(`Preloading ${keys.length} keys matching pattern: ${pattern}`); + } + } + + /** + * Get comprehensive cache statistics + */ + async getCacheStats(): Promise<{ + size: number; + memoryUsage: number; + keys: number; + hitRateOverall: number; + metrics: Map; + }> { + try { + // Get total number of keys + const keys = await this.keys('*'); + + // Calculate overall hit rate + let totalHits = 0; + let totalMisses = 0; + + for (const [_, metrics] of this.metrics) { + totalHits += metrics.hits; + totalMisses += metrics.misses; + } + + const totalRequests = totalHits + totalMisses; + const hitRateOverall = totalRequests > 0 ? totalHits / totalRequests : 0; + + // Get memory usage info from Redis + let memoryUsage = 0; + try { + // This is a simplified approach - in production you might want to use Redis INFO command + memoryUsage = keys.length * 1024; // Estimate 1KB per key + } catch (error) { + this.logger.warn(`Could not get memory usage: ${error.message}`); + } + + return { + size: keys.length, + memoryUsage, + keys: keys.length, + hitRateOverall, + metrics: this.metrics, + }; + } catch (error) { + this.logger.error(`Failed to get cache stats: ${error.message}`); + throw error; + } + } + + /** + * Export cache metrics for monitoring systems + */ + exportMetrics(): Record { + const exportData: Record = {}; + + // Add metrics for each namespace + for (const [namespace, metrics] of this.metrics) { + exportData[namespace] = { + hits: metrics.hits, + misses: metrics.misses, + hitRate: metrics.hitRate, + totalRequests: metrics.totalRequests, + }; + } + + // Add overall stats + let totalHits = 0; + let totalMisses = 0; + + for (const [_, metrics] of this.metrics) { + totalHits += metrics.hits; + totalMisses += metrics.misses; + } + + const totalRequests = totalHits + totalMisses; + const overallHitRate = totalRequests > 0 ? totalHits / totalRequests : 0; + + exportData.overall = { + totalHits, + totalMisses, + totalRequests, + hitRate: overallHitRate, + }; + + return exportData; + } + + /** + * Reset metrics for a namespace or all namespaces + */ + resetMetrics(namespace?: string): void { + if (namespace) { + const metrics = this.metrics.get(namespace); + if (metrics) { + metrics.hits = 0; + metrics.misses = 0; + metrics.hitRate = 0; + metrics.totalRequests = 0; + } + } else { + // Reset all metrics + for (const [_, metrics] of this.metrics) { + metrics.hits = 0; + metrics.misses = 0; + metrics.hitRate = 0; + metrics.totalRequests = 0; + } + } + } + + /** + * Track cache access patterns + */ + private trackAccessPattern(key: string, operation: 'get' | 'set' | 'del'): void { + // In a real implementation, this would track access patterns for optimization + // For now, just log the access + this.logger.debug(`Cache access pattern: ${operation} on ${key}`); + } + + /** + * Publish cache invalidation event for distributed cache consistency + */ + async publishCacheInvalidateEvent(key: string): Promise { + try { + // Use Redis pub/sub to notify other instances of cache invalidation + await this.redisService.incr('cache:version'); // Increment global version + await this.redisService.publish( + 'cache:invalidation', + JSON.stringify({ + key, + timestamp: Date.now(), + nodeId: this.getNodeId(), + }), + ); + + this.logger.log(`Published cache invalidation event for key: ${key}`); + } catch (error) { + this.logger.error(`Failed to publish cache invalidation event: ${error.message}`); + } + } + + /** + * Subscribe to cache invalidation events for distributed consistency + */ + async subscribeToCacheInvalidateEvents(): Promise { + try { + // Set up subscription to listen for cache invalidation events from other nodes + const subscriber = this.redisService.getRedisInstance(); + + await subscriber.subscribe('cache:invalidation'); + + subscriber.on('message', async (channel, message) => { + if (channel === 'cache:invalidation') { + try { + const event = JSON.parse(message); + + // Only invalidate if it's from another node + if (event.nodeId !== this.getNodeId()) { + await this.del(event.key); + this.logger.log(`Invalidated cache key from distributed event: ${event.key}`); + } + } catch (error) { + this.logger.error(`Failed to process cache invalidation event: ${error.message}`); + } + } + }); + + this.logger.log('Subscribed to cache invalidation events for distributed consistency'); + } catch (error) { + this.logger.error(`Failed to subscribe to cache invalidation events: ${error.message}`); + } + } + + /** + * Get unique node ID for distributed systems + */ + private getNodeId(): string { + // In a real implementation, this might use the actual machine ID or pod name + return this.configService.get('NODE_ID', `node-${Math.random().toString(36).substr(2, 9)}`); + } + + /** + * Distributed lock for cache operations + */ + async acquireDistributedLock(key: string, ttl: number = 30): Promise { + const lockKey = `lock:${key}`; + const lockValue = this.getNodeId(); + + try { + // Try to set lock with NX (only if not exists) and EX (expire time) + const result = await this.redisService.setex(lockKey, ttl, lockValue); + return result !== null; + } catch (error) { + this.logger.error(`Failed to acquire distributed lock: ${error.message}`); + return false; + } + } + + /** + * Release distributed lock + */ + async releaseDistributedLock(key: string): Promise { + const lockKey = `lock:${key}`; + const nodeId = this.getNodeId(); + + try { + // Use Lua script to atomically check and delete lock + const luaScript = ` + if redis.call("get", KEYS[1]) == ARGV[1] then + return redis.call("del", KEYS[1]) + else + return 0 + end + `; + + const result = await this.redisService.eval(luaScript, [lockKey], [nodeId]); + return result === 1; + } catch (error) { + this.logger.error(`Failed to release distributed lock: ${error.message}`); + return false; + } + } + + /** + * Distributed cache operation with lock + */ + async distributedOperation(key: string, operation: () => Promise, lockTtl: number = 30): Promise { + const lockAcquired = await this.acquireDistributedLock(key, lockTtl); + + if (!lockAcquired) { + throw new Error(`Could not acquire distributed lock for key: ${key}`); + } + + try { + return await operation(); + } finally { + await this.releaseDistributedLock(key); + } + } + + /** + * Get cache with distributed consistency check + */ + async getWithConsistencyCheck(key: string, consistencyTtl: number = 300): Promise { + // Check if there's a newer version available + const versionKey = `version:${key}`; + const currentVersion = await this.redisService.get(versionKey); + const localVersion = await this.redisService.get(`local:version:${key}`); + + // If versions differ, the cache might be stale + if (currentVersion && localVersion && currentVersion !== localVersion) { + // Invalidate local cache + await this.del(key); + return undefined; + } + + // Get the value as normal + return await this.get(key); + } + + /** + * Set cache with distributed version tracking + */ + async setWithVersion(key: string, value: T, options?: CacheOptions): Promise { + // Increment version for this key + const version = await this.redisService.incr(`version:${key}`); + + // Store the value + await this.set(key, value, options); + + // Store the local version + await this.redisService.setex(`local:version:${key}`, options?.ttl || this.getDefaultTtl(), version.toString()); + } + + /** + * Enhanced fallback mechanism for cache operations + */ + async executeWithFallback( + operation: () => Promise, + fallback: () => Promise, + options?: { + maxRetries?: number; + retryDelay?: number; + fallbackOnFailure?: boolean; + }, + ): Promise { + const maxRetries = options?.maxRetries ?? 3; + const retryDelay = options?.retryDelay ?? 100; + const fallbackOnFailure = options?.fallbackOnFailure ?? true; + + let lastError: Error; + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + return await operation(); + } catch (error) { + this.logger.warn(`Cache operation attempt ${attempt}/${maxRetries} failed: ${error.message}`); + lastError = error as Error; + + if (attempt < maxRetries) { + // Wait before retrying + await new Promise(resolve => setTimeout(resolve, retryDelay * attempt)); + } + } + } + + // If all retries failed and fallback is enabled + if (fallbackOnFailure) { + try { + this.logger.log('Executing fallback operation'); + return await fallback(); + } catch (fallbackError) { + this.logger.error(`Fallback operation also failed: ${fallbackError.message}`); + throw lastError; // Throw the original error + } + } + + throw lastError; + } + + /** + * Get with multiple fallback strategies + */ + async getWithMultipleFallbacks( + key: string, + fallbackFactories: Array<() => Promise>, // Multiple fallback options + options?: { + ttl?: number; + refreshInBackground?: boolean; + }, + ): Promise { + // First, try to get from cache + const cachedValue = await this.get(key); + if (cachedValue !== undefined) { + this.logger.log(`Cache HIT for key: ${key}`); + return cachedValue; + } + + // If not in cache, try fallback factories in sequence + for (let i = 0; i < fallbackFactories.length; i++) { + try { + this.logger.log(`Trying fallback ${i + 1}/${fallbackFactories.length}`); + const value = await fallbackFactories[i](); + + // Cache the successful result + await this.set(key, value, { ttl: options?.ttl }); + + // Optionally refresh in background + if (options?.refreshInBackground) { + setImmediate(async () => { + try { + const refreshedValue = await fallbackFactories[0](); // Use primary source + await this.set(key, refreshedValue, { ttl: options?.ttl }); + } catch (refreshError) { + this.logger.error(`Background refresh failed: ${refreshError.message}`); + } + }); + } + + return value; + } catch (error) { + this.logger.warn(`Fallback ${i + 1} failed: ${error.message}`); + + // If this was the last fallback, throw the error + if (i === fallbackFactories.length - 1) { + throw error; + } + } + } + + throw new Error(`All fallback methods failed for key: ${key}`); + } + + /** + * Circuit breaker pattern for cache operations + */ + private circuitBreakerState: Map< + string, + { + failureCount: number; + lastFailureTime: number; + isOpen: boolean; + nextAttemptTime: number; + } + > = new Map(); + + async executeWithCircuitBreaker( + key: string, + operation: () => Promise, + options?: { + failureThreshold?: number; + timeout?: number; + resetTimeout?: number; + }, + ): Promise { + const failureThreshold = options?.failureThreshold ?? 5; + const timeout = options?.timeout ?? 60000; // 1 minute + const resetTimeout = options?.resetTimeout ?? 30000; // 30 seconds + + const stateKey = `circuit-breaker:${key}`; + let state = this.circuitBreakerState.get(stateKey); + + if (!state) { + state = { + failureCount: 0, + lastFailureTime: 0, + isOpen: false, + nextAttemptTime: 0, + }; + this.circuitBreakerState.set(stateKey, state); + } + + // Check if circuit breaker is open + if (state.isOpen) { + if (Date.now() >= state.nextAttemptTime) { + // Half-open state - allow one trial + this.logger.log(`Circuit breaker half-open for key: ${key}, allowing one trial`); + try { + const result = await operation(); + // Success - close the circuit + state.failureCount = 0; + state.isOpen = false; + return result; + } catch (error) { + // Still failing - keep circuit open + this.logger.error(`Circuit breaker trial failed for key: ${key}`); + throw error; + } + } else { + throw new Error(`Circuit breaker is OPEN for key: ${key}`); + } + } + + // Execute operation + try { + const result = await operation(); + // Reset failure count on success + state.failureCount = 0; + return result; + } catch (error) { + // Increment failure count + state.failureCount++; + state.lastFailureTime = Date.now(); + + // Open circuit if threshold exceeded + if (state.failureCount >= failureThreshold) { + state.isOpen = true; + state.nextAttemptTime = Date.now() + resetTimeout; + this.logger.warn(`Circuit breaker OPENED for key: ${key}`); + } + + throw error; + } + } + + /** + * Graceful degradation for cache failures + */ + async getWithGracefulDegradation( + key: string, + factory: () => Promise, + options?: { + cacheTtl?: number; + maxStaleAge?: number; // Maximum age of stale data to serve + allowStale?: boolean; // Whether to serve stale data + }, + ): Promise { + try { + // Try to get fresh data from cache + const cachedValue = await this.get(key); + if (cachedValue !== undefined) { + this.logger.log(`Cache HIT for key: ${key}`); + return cachedValue; + } + } catch (cacheError) { + this.logger.error(`Cache GET failed: ${cacheError.message}, proceeding to factory`); + } + + try { + // Get fresh data from factory + const freshValue = await factory(); + + // Try to cache the fresh value + try { + await this.set(key, freshValue, { ttl: options?.cacheTtl }); + } catch (cacheError) { + this.logger.error(`Failed to cache fresh value: ${cacheError.message}, continuing without cache`); + } + + return freshValue; + } catch (factoryError) { + this.logger.error(`Factory failed: ${factoryError.message}`); + + // If we allow stale data and have it available, return it + if (options?.allowStale && options?.maxStaleAge) { + try { + // This would require additional implementation to check staleness + // For now, we'll just try to get whatever is in cache + const possiblyStaleValue = await this.get(key); + if (possiblyStaleValue !== undefined) { + this.logger.warn(`Returning stale data for key: ${key}`); + return possiblyStaleValue; + } + } catch (staleError) { + this.logger.error(`Failed to retrieve stale data: ${staleError.message}`); + } + } + + throw factoryError; + } + } +} diff --git a/src/common/services/compliance-reporting.service.ts b/src/common/services/compliance-reporting.service.ts index 24974212..4aaf843d 100644 --- a/src/common/services/compliance-reporting.service.ts +++ b/src/common/services/compliance-reporting.service.ts @@ -79,12 +79,9 @@ export class ComplianceReportingService { /** * Generate a comprehensive compliance report */ - async generateComplianceReport( - periodStart: Date, - periodEnd: Date, - ): Promise { + async generateComplianceReport(periodStart: Date, periodEnd: Date): Promise { this.logger.log( - `Generating compliance report for period: ${periodStart.toISOString()} to ${periodEnd.toISOString()}` + `Generating compliance report for period: ${periodStart.toISOString()} to ${periodEnd.toISOString()}`, ); // Get all report components @@ -110,14 +107,13 @@ export class ComplianceReportingService { // Calculate overall compliance rate const totalAudits = auditTrailSummary.totalActions; - const gdprRequests = gdprComplianceSummary.dataExports + - gdprComplianceSummary.deletions + - gdprComplianceSummary.consentUpdates; + const gdprRequests = + gdprComplianceSummary.dataExports + gdprComplianceSummary.deletions + gdprComplianceSummary.consentUpdates; const complianceRate = this.calculateComplianceRate( auditTrailSummary, gdprComplianceSummary, - securityMetricsSummary + securityMetricsSummary, ); const report: ComplianceReport = { @@ -141,11 +137,7 @@ export class ComplianceReportingService { gdprCompliance: gdprComplianceSummary, securityMetrics: securityMetricsSummary, }, - recommendations: this.generateRecommendations( - auditTrailSummary, - gdprComplianceSummary, - securityMetricsSummary - ), + recommendations: this.generateRecommendations(auditTrailSummary, gdprComplianceSummary, securityMetricsSummary), }; // Log the report generation @@ -166,10 +158,7 @@ export class ComplianceReportingService { /** * Get audit trail summary for the specified period */ - private async getAuditTrailSummary( - periodStart: Date, - periodEnd: Date, - ): Promise { + private async getAuditTrailSummary(periodStart: Date, periodEnd: Date): Promise { const audits = await this.prisma.auditLog.findMany({ where: { timestamp: { @@ -212,10 +201,7 @@ export class ComplianceReportingService { /** * Get user activity summary for the specified period */ - private async getUserActivitySummary( - periodStart: Date, - periodEnd: Date, - ): Promise { + private async getUserActivitySummary(periodStart: Date, periodEnd: Date): Promise { const [activeUsers, newUsers, roleChanges, loginAttempts] = await Promise.all([ this.prisma.user.count({ where: { @@ -244,10 +230,7 @@ export class ComplianceReportingService { // Assuming we have a way to track login attempts (maybe in audit logs) this.prisma.auditLog.count({ where: { - AND: [ - { tableName: 'auth_attempts' }, - { timestamp: { gte: periodStart, lte: periodEnd } }, - ], + AND: [{ tableName: 'auth_attempts' }, { timestamp: { gte: periodStart, lte: periodEnd } }], }, }), ]); @@ -274,10 +257,7 @@ export class ComplianceReportingService { .filter(stat => stat.compliancePercentage < 95) .map(stat => stat.tableName); - const expiredRecords = retentionStats.reduce( - (sum, stat) => sum + stat.expiredRecords, - 0 - ); + const expiredRecords = retentionStats.reduce((sum, stat) => sum + stat.expiredRecords, 0); return { compliantTables, @@ -289,10 +269,7 @@ export class ComplianceReportingService { /** * Get GDPR compliance summary */ - private async getGdprComplianceSummary( - periodStart: Date, - periodEnd: Date, - ): Promise { + private async getGdprComplianceSummary(periodStart: Date, periodEnd: Date): Promise { // In a real implementation, these would come from GDPR-specific logs // For now, we'll count GDPR-related audit logs const [dataExports, deletions, consentUpdates] = await Promise.all([ @@ -301,11 +278,11 @@ export class ComplianceReportingService { AND: [ { operation: 'READ' }, { tableName: 'users' }, - { - newData: { - path: ['action'], - string_contains: 'GDPR_DATA_EXPORT' - } + { + newData: { + path: ['action'], + string_contains: 'GDPR_DATA_EXPORT', + }, }, { timestamp: { gte: periodStart, lte: periodEnd } }, ], @@ -313,18 +290,12 @@ export class ComplianceReportingService { }), this.prisma.auditLog.count({ where: { - AND: [ - { operation: 'GDPR_DELETE' }, - { timestamp: { gte: periodStart, lte: periodEnd } }, - ], + AND: [{ operation: 'GDPR_DELETE' }, { timestamp: { gte: periodStart, lte: periodEnd } }], }, }), this.prisma.auditLog.count({ where: { - AND: [ - { tableName: 'consents' }, - { timestamp: { gte: periodStart, lte: periodEnd } }, - ], + AND: [{ tableName: 'consents' }, { timestamp: { gte: periodStart, lte: periodEnd } }], }, }), ]); @@ -343,10 +314,7 @@ export class ComplianceReportingService { /** * Get security metrics summary */ - private async getSecurityMetricsSummary( - periodStart: Date, - periodEnd: Date, - ): Promise { + private async getSecurityMetricsSummary(periodStart: Date, periodEnd: Date): Promise { // These would typically come from security logs // For now, we'll use audit logs to approximate const [failedLogins, suspiciousActivities, apiKeyUsage] = await Promise.all([ @@ -418,7 +386,7 @@ export class ComplianceReportingService { // 1. Proper audit logging (should have audit logs) // 2. GDPR compliance (should handle GDPR requests) // 3. Security incidents (lower is better) - + let score = 100; // Deduct points for lack of audit logs @@ -452,29 +420,21 @@ export class ComplianceReportingService { const recommendations: string[] = []; if (auditSummary.totalActions === 0) { - recommendations.push( - 'Implement comprehensive audit logging for all data changes' - ); + recommendations.push('Implement comprehensive audit logging for all data changes'); } if (securitySummary.failedLogins > 50) { - recommendations.push( - 'Review authentication security and implement additional measures' - ); + recommendations.push('Review authentication security and implement additional measures'); } if (gdprSummary.dataExports === 0 && gdprSummary.deletions === 0) { - recommendations.push( - 'Verify GDPR compliance features are properly tracked and logged' - ); + recommendations.push('Verify GDPR compliance features are properly tracked and logged'); } if (auditSummary.topTables.length > 0) { const mostChangedTable = auditSummary.topTables[0]; if (mostChangedTable.count > 1000) { - recommendations.push( - `Consider optimizing access patterns for ${mostChangedTable.tableName} table` - ); + recommendations.push(`Consider optimizing access patterns for ${mostChangedTable.tableName} table`); } } @@ -488,20 +448,17 @@ export class ComplianceReportingService { /** * Export compliance report in various formats */ - async exportReport( - report: ComplianceReport, - format: 'json' | 'csv' | 'pdf' = 'json', - ): Promise { + async exportReport(report: ComplianceReport, format: 'json' | 'csv' | 'pdf' = 'json'): Promise { switch (format) { case 'json': return JSON.stringify(report, null, 2); - + case 'csv': return this.convertToCsv(report); - + case 'pdf': return this.convertToPdf(report); - + default: throw new Error(`Unsupported export format: ${format}`); } @@ -521,7 +478,7 @@ export class ComplianceReportingService { csv += `GDPR Requests,${report.summary.gdprRequests}\n`; csv += `Security Incidents,${report.summary.securityIncidents}\n`; csv += `Compliance Rate,${report.summary.complianceRate}%\n`; - + return csv; } @@ -531,7 +488,10 @@ export class ComplianceReportingService { private convertToPdf(report: ComplianceReport): Buffer { // This would require a PDF generation library like pdfkit or puppeteer // For now, return a placeholder - return Buffer.from(`Compliance Report\n\nReport ID: ${report.reportId}\nGenerated: ${report.reportDate}\n\n${JSON.stringify(report, null, 2)}`, 'utf-8'); + return Buffer.from( + `Compliance Report\n\nReport ID: ${report.reportId}\nGenerated: ${report.reportDate}\n\n${JSON.stringify(report, null, 2)}`, + 'utf-8', + ); } /** @@ -563,7 +523,8 @@ export class ComplianceReportingService { periodEnd.setDate(periodEnd.getDate() + 1); } else if (interval === 'weekly') { periodEnd.setDate(periodEnd.getDate() + 7); - } else { // monthly + } else { + // monthly periodEnd.setMonth(periodEnd.getMonth() + 1); } @@ -588,7 +549,8 @@ export class ComplianceReportingService { currentDate.setDate(currentDate.getDate() + 1); } else if (interval === 'weekly') { currentDate.setDate(currentDate.getDate() + 7); - } else { // monthly + } else { + // monthly currentDate.setMonth(currentDate.getMonth() + 1); } } @@ -599,4 +561,4 @@ export class ComplianceReportingService { auditVolumes, }; } -} \ No newline at end of file +} diff --git a/src/common/services/data-retention.service.ts b/src/common/services/data-retention.service.ts index 99ec4456..05c3b674 100644 --- a/src/common/services/data-retention.service.ts +++ b/src/common/services/data-retention.service.ts @@ -69,9 +69,7 @@ export class DataRetentionService { batchSize: number = 1000, dryRun: boolean = false, ): Promise<{ deletedCount: number; tableName: string; batchSize: number }> { - const policies = tableName - ? this.defaultPolicies.filter(p => p.tableName === tableName) - : this.defaultPolicies; + const policies = tableName ? this.defaultPolicies.filter(p => p.tableName === tableName) : this.defaultPolicies; if (policies.length === 0) { throw new Error(`No retention policy found for table: ${tableName}`); @@ -83,31 +81,27 @@ export class DataRetentionService { const cutoffDate = new Date(); cutoffDate.setDate(cutoffDate.getDate() - policy.retentionPeriodDays); - this.logger.log( - `Cleaning up ${policy.tableName} data older than ${cutoffDate.toISOString()}` - ); + this.logger.log(`Cleaning up ${policy.tableName} data older than ${cutoffDate.toISOString()}`); // Calculate how many records would be affected const countResult = await this.getDeletableRecordCount(policy, cutoffDate); - + if (countResult === 0) { this.logger.log(`No records to delete for ${policy.tableName}`); continue; } if (dryRun) { - this.logger.log( - `[DRY RUN] Would delete ${countResult} records from ${policy.tableName}` - ); + this.logger.log(`[DRY RUN] Would delete ${countResult} records from ${policy.tableName}`); totalDeleted += countResult; continue; } // Actually perform the deletion in batches const deletedCount = await this.deleteInBatches(policy, cutoffDate, batchSize); - + totalDeleted += deletedCount; - + this.logger.log(`Deleted ${deletedCount} records from ${policy.tableName}`); // Log the cleanup action @@ -134,10 +128,7 @@ export class DataRetentionService { /** * Calculate how many records would be deleted for a given policy */ - async getDeletableRecordCount( - policy: RetentionPolicy, - cutoffDate: Date, - ): Promise { + async getDeletableRecordCount(policy: RetentionPolicy, cutoffDate: Date): Promise { switch (policy.tableName) { case 'audit_logs': return this.prisma.auditLog.count({ @@ -190,11 +181,7 @@ export class DataRetentionService { /** * Delete records in batches to avoid locking issues */ - private async deleteInBatches( - policy: RetentionPolicy, - cutoffDate: Date, - batchSize: number, - ): Promise { + private async deleteInBatches(policy: RetentionPolicy, cutoffDate: Date, batchSize: number): Promise { let totalDeleted = 0; let batchDeleted = 0; @@ -209,13 +196,9 @@ export class DataRetentionService { /** * Delete a single batch of records */ - private async deleteBatch( - policy: RetentionPolicy, - cutoffDate: Date, - batchSize: number, - ): Promise { + private async deleteBatch(policy: RetentionPolicy, cutoffDate: Date, batchSize: number): Promise { // Use Prisma transaction to ensure data consistency - return this.prisma.$transaction(async (tx) => { + return this.prisma.$transaction(async tx => { let deletedCount = 0; switch (policy.tableName) { @@ -227,7 +210,7 @@ export class DataRetentionService { take: batchSize, select: { id: true }, }); - + if (auditLogs.length > 0) { await tx.auditLog.deleteMany({ where: { @@ -246,7 +229,7 @@ export class DataRetentionService { take: batchSize, select: { id: true }, }); - + if (systemLogs.length > 0) { await tx.systemLog.deleteMany({ where: { @@ -260,15 +243,12 @@ export class DataRetentionService { case 'documents': const documents = await tx.document.findMany({ where: { - AND: [ - { createdAt: { lt: cutoffDate } }, - { expiresAt: { lt: cutoffDate } }, - ], + AND: [{ createdAt: { lt: cutoffDate } }, { expiresAt: { lt: cutoffDate } }], }, take: batchSize, select: { id: true }, }); - + if (documents.length > 0) { await tx.document.deleteMany({ where: { @@ -287,7 +267,7 @@ export class DataRetentionService { take: batchSize, select: { id: true }, }); - + if (transactions.length > 0) { await tx.transaction.deleteMany({ where: { @@ -306,7 +286,7 @@ export class DataRetentionService { take: batchSize, select: { id: true }, }); - + if (properties.length > 0) { await tx.property.deleteMany({ where: { @@ -325,7 +305,7 @@ export class DataRetentionService { take: batchSize, select: { id: true }, }); - + if (roleChangeLogs.length > 0) { await tx.roleChangeLog.deleteMany({ where: { @@ -356,12 +336,14 @@ export class DataRetentionService { /** * Get statistics about data retention compliance */ - async getRetentionStats(): Promise<{ - tableName: string; - totalRecords: number; - expiredRecords: number; - compliancePercentage: number; - }[]> { + async getRetentionStats(): Promise< + { + tableName: string; + totalRecords: number; + expiredRecords: number; + compliancePercentage: number; + }[] + > { const stats = []; for (const policy of this.defaultPolicies) { @@ -373,9 +355,7 @@ export class DataRetentionService { this.getDeletableRecordCount(policy, cutoffDate), ]); - const compliancePercentage = totalRecords > 0 - ? ((totalRecords - expiredRecords) / totalRecords) * 100 - : 100; + const compliancePercentage = totalRecords > 0 ? ((totalRecords - expiredRecords) / totalRecords) * 100 : 100; stats.push({ tableName: policy.tableName, @@ -431,7 +411,7 @@ export class DataRetentionService { } const deletedCount = await this.deleteInBatches(policy, cutoffDate, batchSize); - + await this.auditService.logAction({ tableName: 'data_retention', operation: AuditOperation.DELETE, @@ -445,4 +425,4 @@ export class DataRetentionService { return deletedCount; } -} \ No newline at end of file +} diff --git a/src/common/services/encryption.service.ts b/src/common/services/encryption.service.ts index ffc4e360..3526de16 100644 --- a/src/common/services/encryption.service.ts +++ b/src/common/services/encryption.service.ts @@ -9,7 +9,7 @@ export class EncryptionService { private readonly keyLength = 32; // 256 bits private readonly ivLength = 16; // 128 bits private readonly saltLength = 16; // 128 bits - + private readonly encryptionKey: Buffer; constructor(private configService: ConfigService) { @@ -34,7 +34,7 @@ export class EncryptionService { // Truncate the key if it's too long key = key.substring(0, this.keyLength); } - + return Buffer.from(key, 'utf8'); } @@ -45,10 +45,10 @@ export class EncryptionService { try { // Generate a random IV for each encryption const iv = crypto.randomBytes(this.ivLength); - + // Create cipher const cipher = crypto.createCipheriv(this.algorithm, this.encryptionKey, iv); - + // Encrypt the data let dataBuffer: Buffer; if (typeof data === 'string') { @@ -56,15 +56,12 @@ export class EncryptionService { } else { dataBuffer = data; } - - const encrypted = Buffer.concat([ - cipher.update(dataBuffer), - cipher.final(), - ]); - + + const encrypted = Buffer.concat([cipher.update(dataBuffer), cipher.final()]); + // Get the authentication tag const authTag = cipher.getAuthTag(); - + return { encrypted: encrypted.toString('hex'), iv: iv.toString('hex'), @@ -85,19 +82,16 @@ export class EncryptionService { const encryptedBuffer = Buffer.from(encryptedData.encrypted, 'hex'); const iv = Buffer.from(encryptedData.iv, 'hex'); const authTag = Buffer.from(encryptedData.authTag, 'hex'); - + // Create decipher const decipher = crypto.createDecipheriv(this.algorithm, this.encryptionKey, iv); - + // Set the authentication tag decipher.setAuthTag(authTag); - + // Decrypt the data - const decrypted = Buffer.concat([ - decipher.update(encryptedBuffer), - decipher.final(), - ]); - + const decrypted = Buffer.concat([decipher.update(encryptedBuffer), decipher.final()]); + return decrypted.toString('utf8'); } catch (error) { this.logger.error(`Decryption failed: ${error.message}`); @@ -110,10 +104,8 @@ export class EncryptionService { */ hashWithSalt(data: string): { hash: string; salt: string } { const salt = crypto.randomBytes(this.saltLength).toString('hex'); - const hash = crypto - .pbkdf2Sync(data, salt, 10000, 64, 'sha256') - .toString('hex'); - + const hash = crypto.pbkdf2Sync(data, salt, 10000, 64, 'sha256').toString('hex'); + return { hash, salt }; } @@ -121,21 +113,19 @@ export class EncryptionService { * Verify hashed data */ verifyHash(data: string, hash: string, salt: string): boolean { - const computedHash = crypto - .pbkdf2Sync(data, salt, 10000, 64, 'sha256') - .toString('hex'); - - return crypto.timingSafeEqual( - Buffer.from(hash, 'hex'), - Buffer.from(computedHash, 'hex') - ); + const computedHash = crypto.pbkdf2Sync(data, salt, 10000, 64, 'sha256').toString('hex'); + + return crypto.timingSafeEqual(Buffer.from(hash, 'hex'), Buffer.from(computedHash, 'hex')); } /** * Generate a cryptographically secure random string */ generateRandomString(length: number): string { - return crypto.randomBytes(Math.ceil(length / 2)).toString('hex').substring(0, length); + return crypto + .randomBytes(Math.ceil(length / 2)) + .toString('hex') + .substring(0, length); } /** @@ -143,7 +133,7 @@ export class EncryptionService { */ encryptObject>(obj: T, fieldsToEncrypt: string[]): T { const encryptedObj = { ...obj }; - + for (const field of fieldsToEncrypt) { if (encryptedObj.hasOwnProperty(field) && encryptedObj[field] !== null && encryptedObj[field] !== undefined) { const value = encryptedObj[field]; @@ -152,12 +142,12 @@ export class EncryptionService { // Store as encrypted object encryptedObj[field as keyof T] = { __encrypted__: true, - ...encrypted + ...encrypted, } as T[keyof T]; } } } - + return encryptedObj; } @@ -166,7 +156,7 @@ export class EncryptionService { */ decryptObject>(obj: T, fieldsToDecrypt: string[]): T { const decryptedObj = { ...obj }; - + for (const field of fieldsToDecrypt) { if ( decryptedObj.hasOwnProperty(field) && @@ -178,15 +168,15 @@ export class EncryptionService { const encryptedData = { encrypted: (decryptedObj[field] as any).encrypted, iv: (decryptedObj[field] as any).iv, - authTag: (decryptedObj[field] as any).authTag + authTag: (decryptedObj[field] as any).authTag, }; - + // Decrypt the value const decryptedValue = this.decrypt(encryptedData); decryptedObj[field as keyof T] = decryptedValue as T[keyof T]; } } - + return decryptedObj; } @@ -197,14 +187,11 @@ export class EncryptionService { try { const iv = crypto.randomBytes(this.ivLength); const cipher = crypto.createCipheriv(this.algorithm, this.encryptionKey, iv); - - const encrypted = Buffer.concat([ - cipher.update(buffer), - cipher.final(), - ]); - + + const encrypted = Buffer.concat([cipher.update(buffer), cipher.final()]); + const authTag = cipher.getAuthTag(); - + return { encryptedBuffer: encrypted, iv, @@ -223,11 +210,8 @@ export class EncryptionService { try { const decipher = crypto.createDecipheriv(this.algorithm, this.encryptionKey, encryptedData.iv); decipher.setAuthTag(encryptedData.authTag); - - return Buffer.concat([ - decipher.update(encryptedData.encryptedBuffer), - decipher.final(), - ]); + + return Buffer.concat([decipher.update(encryptedData.encryptedBuffer), decipher.final()]); } catch (error) { this.logger.error(`File decryption failed: ${error.message}`); throw new Error(`File decryption failed: ${error.message}`); @@ -245,16 +229,16 @@ export class EncryptionService { } else { dataBuffer = data; } - + const sign = crypto.createSign('SHA256'); sign.write(dataBuffer); sign.end(); - + const privateKey = this.configService.get('SIGNING_PRIVATE_KEY'); if (!privateKey) { throw new Error('SIGNING_PRIVATE_KEY environment variable is required for signing'); } - + return sign.sign(privateKey, 'hex'); } catch (error) { this.logger.error(`Signature creation failed: ${error.message}`); @@ -273,16 +257,16 @@ export class EncryptionService { } else { dataBuffer = data; } - + const verify = crypto.createVerify('SHA256'); verify.write(dataBuffer); verify.end(); - + const publicKey = this.configService.get('SIGNING_PUBLIC_KEY'); if (!publicKey) { throw new Error('SIGNING_PUBLIC_KEY environment variable is required for signature verification'); } - + return verify.verify(publicKey, signature, 'hex'); } catch (error) { this.logger.error(`Signature verification failed: ${error.message}`); @@ -305,4 +289,4 @@ export class EncryptionService { // Implementation would depend on specific requirements and data volume this.logger.warn('Key rotation functionality would need to be implemented based on specific requirements'); } -} \ No newline at end of file +} diff --git a/src/common/services/gdpr.service.ts b/src/common/services/gdpr.service.ts index 82b2823f..da9a5ccc 100644 --- a/src/common/services/gdpr.service.ts +++ b/src/common/services/gdpr.service.ts @@ -37,10 +37,7 @@ export class GdprService { }), this.prisma.transaction.findMany({ where: { - OR: [ - { fromAddress: user.walletAddress }, - { toAddress: user.walletAddress }, - ], + OR: [{ fromAddress: user.walletAddress }, { toAddress: user.walletAddress }], }, }), this.prisma.document.findMany({ @@ -60,7 +57,7 @@ export class GdprService { tableName: 'users', operation: AuditOperation.READ, newData: { action: 'GDPR_DATA_EXPORT', userId }, - userId: userId, + userId, }); return { @@ -90,8 +87,8 @@ export class GdprService { // Perform soft delete or hard delete based on compliance requirements // For financial/real estate applications, we may want to preserve records for legal reasons // So we'll anonymize the user instead of completely deleting - - await this.prisma.$transaction(async (tx) => { + + await this.prisma.$transaction(async tx => { // Anonymize user data (keep for legal compliance but remove personal info) await tx.user.update({ where: { id: userId }, @@ -139,7 +136,7 @@ export class GdprService { throw new Error('User not found'); } - await this.prisma.$transaction(async (tx) => { + await this.prisma.$transaction(async tx => { // Store original data for audit before anonymization const originalData = { id: user.id, @@ -185,7 +182,7 @@ export class GdprService { operation: AuditOperation.UPDATE, oldData: originalData, newData: { id: userId, status: 'ANONYMIZED' }, - userId: userId, + userId, }); }); } @@ -228,12 +225,12 @@ export class GdprService { */ private sanitizeUserData(user: any): any { const sanitized = { ...user }; - + // Remove sensitive fields that shouldn't be exported delete sanitized.password; delete sanitized.twoFactorSecret; delete sanitized.refreshToken; - + return sanitized; } @@ -254,10 +251,6 @@ export class GdprService { } // Check if user has been anonymized - return ( - user.email?.startsWith('anonymized-') || - user.email?.startsWith('deleted-') || - user.walletAddress === null - ); + return user.email?.startsWith('anonymized-') || user.email?.startsWith('deleted-') || user.walletAddress === null; } -} \ No newline at end of file +} diff --git a/src/common/services/redis.service.ts b/src/common/services/redis.service.ts index 1d9470bf..9144909a 100644 --- a/src/common/services/redis.service.ts +++ b/src/common/services/redis.service.ts @@ -59,4 +59,20 @@ export class RedisService { async incr(key: string): Promise { return await this.redis.incr(key); } + + async publish(channel: string, message: string): Promise { + return await this.redis.publish(channel, message); + } + + getRedisInstance(): Redis { + return this.redis; + } + + async eval(script: string, keys: string[], args: string[]): Promise { + return await this.redis.eval(script, keys.length, ...keys, ...args); + } + + async flushdb(): Promise { + return await this.redis.flushdb(); + } } diff --git a/src/rbac/rbac.module.ts b/src/rbac/rbac.module.ts index eb4786a6..2f900cdf 100644 --- a/src/rbac/rbac.module.ts +++ b/src/rbac/rbac.module.ts @@ -8,4 +8,4 @@ import { AuditModule } from '../common/audit/audit.module'; providers: [RbacService], exports: [RbacService], }) -export class RbacModule {} \ No newline at end of file +export class RbacModule {} diff --git a/src/rbac/rbac.service.ts b/src/rbac/rbac.service.ts index 6f327ef5..d563cdc7 100644 --- a/src/rbac/rbac.service.ts +++ b/src/rbac/rbac.service.ts @@ -14,11 +14,7 @@ export class RbacService { /** * Check if a user has permission to perform an action on a resource */ - async hasPermission( - userId: string, - resource: Resource, - action: Action, - ): Promise { + async hasPermission(userId: string, resource: Resource, action: Action): Promise { try { // Get user with roles and permissions const user = await this.prisma.user.findUnique({ @@ -48,9 +44,7 @@ export class RbacService { // Check if user has direct permissions if (user.userRole && user.userRole.permissions) { const hasPerm = user.userRole.permissions.some( - (rolePerm) => - rolePerm.permission.resource === resource && - rolePerm.permission.action === action, + rolePerm => rolePerm.permission.resource === resource && rolePerm.permission.action === action, ); if (hasPerm) { @@ -61,9 +55,7 @@ export class RbacService { // Also check if user has 'MANAGE' permission for the resource (full access) if (user.userRole && user.userRole.permissions) { const hasManagePerm = user.userRole.permissions.some( - (rolePerm) => - rolePerm.permission.resource === resource && - rolePerm.permission.action === Action.MANAGE, + rolePerm => rolePerm.permission.resource === resource && rolePerm.permission.action === Action.MANAGE, ); if (hasManagePerm) { @@ -318,10 +310,7 @@ export class RbacService { const transaction = await this.prisma.transaction.findUnique({ where: { id: resourceId }, }); - return ( - transaction?.fromAddress === userId || - transaction?.toAddress === userId - ); + return transaction?.fromAddress === userId || transaction?.toAddress === userId; default: return false; @@ -335,15 +324,10 @@ export class RbacService { /** * Enhanced permission check combining RBAC and ABAC */ - async hasAccess( - userId: string, - resource: Resource, - action: Action, - resourceId?: string, - ): Promise { + async hasAccess(userId: string, resource: Resource, action: Action, resourceId?: string): Promise { // First, check basic RBAC permissions const hasBasicPermission = await this.hasPermission(userId, resource, action); - + if (hasBasicPermission) { return true; } @@ -358,12 +342,8 @@ export class RbacService { const mappedResource = resourceMap[resource]; if (mappedResource) { - const hasOwnership = await this.validateResourceOwnership( - userId, - mappedResource as any, - resourceId, - ); - + const hasOwnership = await this.validateResourceOwnership(userId, mappedResource as any, resourceId); + if (hasOwnership) { return true; } @@ -372,4 +352,4 @@ export class RbacService { return false; } -} \ No newline at end of file +} diff --git a/src/valuation/valuation.service.ts b/src/valuation/valuation.service.ts index d5d6e873..2877cd3a 100644 --- a/src/valuation/valuation.service.ts +++ b/src/valuation/valuation.service.ts @@ -3,6 +3,7 @@ import { ConfigService } from '@nestjs/config'; import { PrismaService } from '../database/prisma/prisma.service'; import axios from 'axios'; import { Decimal } from '@prisma/client/runtime/library'; +import { CacheService } from '../common/services/cache.service'; export interface PropertyFeatures { id?: string; @@ -39,6 +40,7 @@ export class ValuationService { constructor( private readonly configService: ConfigService, private readonly prisma: PrismaService, + private readonly cacheService: CacheService, // Inject the cache service ) { this.externalApis = { zillow: { @@ -46,21 +48,28 @@ export class ValuationService { apiKey: this.configService.get('valuation.externalApi.zillowApiKey'), }, redfin: { - baseUrl: 'https://redfin.com/stingray/', + baseUrl: 'https://api.redfin.com/v1', apiKey: this.configService.get('valuation.externalApi.redfinApiKey'), }, corelogic: { baseUrl: 'https://api.corelogic.com/v1', - apiKey: this.configService.get('valuation.externalApi.coreLogicApiKey'), + apiKey: this.configService.get('valuation.externalApi.corelogicApiKey'), }, }; } - /** - * Main method to get property valuation - */ async getValuation(propertyId: string, features?: PropertyFeatures): Promise { const cacheKey = `valuation:${propertyId}`; + const cacheTtl = this.configService.get('valuation.valuation.cacheTtl', 86400); // 24 hours default + + // Try to get from cache first + const cachedValuation = await this.cacheService.get(cacheKey); + if (cachedValuation) { + this.logger.log(`Cache HIT for property ${propertyId}`); + return cachedValuation; + } + + this.logger.log(`Cache MISS for property ${propertyId}, fetching fresh valuation`); // Skip cache implementation for now since cache service is not available @@ -123,7 +132,11 @@ export class ValuationService { // Update property with valuation await this.updatePropertyWithValuation(propertyId, savedValuation); - // Cache implementation skipped since cache service is not available + // Cache the result with tags for easy invalidation + await this.cacheService.set(cacheKey, savedValuation, { ttl: cacheTtl }); + await this.cacheService.tagEntry(cacheKey, ['valuation', 'property', propertyId]); + + this.logger.log(`Successfully cached valuation for property ${propertyId}`); return savedValuation; } catch (error) { @@ -268,41 +281,33 @@ export class ValuationService { } /** - * Normalize property features for consistent processing + * Normalize property features to standard format */ private normalizeFeatures(features: PropertyFeatures): PropertyFeatures { - const normalized: PropertyFeatures = { ...features }; - - // Normalize location to standard format - if (normalized.location) { - normalized.location = normalized.location.trim().toLowerCase(); - } - - // Ensure numeric values are valid - if (typeof normalized.bedrooms === 'string') { - normalized.bedrooms = parseInt(normalized.bedrooms, 10); - } - if (typeof normalized.bathrooms === 'string') { - normalized.bathrooms = parseFloat(normalized.bathrooms); - } - if (typeof normalized.squareFootage === 'string') { - normalized.squareFootage = parseFloat(normalized.squareFootage); - } - if (typeof normalized.yearBuilt === 'string') { - normalized.yearBuilt = parseInt(normalized.yearBuilt, 10); - } - if (typeof normalized.lotSize === 'string') { - normalized.lotSize = parseFloat(normalized.lotSize); - } - - // Set defaults for missing values - normalized.bedrooms = normalized.bedrooms ?? 0; - normalized.bathrooms = normalized.bathrooms ?? 0; - normalized.squareFootage = normalized.squareFootage ?? 0; - normalized.yearBuilt = normalized.yearBuilt ?? 0; - normalized.lotSize = normalized.lotSize ?? 0; - - return normalized; + // Normalize location: trim and lowercase + const location = features.location ? features.location.trim().toLowerCase() : ''; + + // Convert string values to numbers where appropriate + const bedrooms = typeof features.bedrooms === 'string' ? + parseInt(features.bedrooms, 10) : features.bedrooms; + const bathrooms = typeof features.bathrooms === 'string' ? + parseFloat(features.bathrooms) : features.bathrooms; + const squareFootage = typeof features.squareFootage === 'string' ? + parseInt(features.squareFootage, 10) : features.squareFootage; + const yearBuilt = typeof features.yearBuilt === 'string' ? + parseInt(features.yearBuilt, 10) : features.yearBuilt; + const lotSize = typeof features.lotSize === 'string' ? + parseFloat(features.lotSize) : features.lotSize; + + return { + ...features, + location, + bedrooms: bedrooms || 0, + bathrooms: bathrooms || 0, + squareFootage: squareFootage || 0, + yearBuilt: yearBuilt || 0, + lotSize: lotSize || 0, + }; } /** @@ -412,18 +417,33 @@ export class ValuationService { where: { id: propertyId }, data: updateData, }); + + // Invalidate related caches when property is updated + await this.invalidateRelatedCaches(propertyId); } /** * Get historical valuations for a property */ async getPropertyHistory(propertyId: string): Promise { + const cacheKey = `valuation:history:${propertyId}`; + const cacheTtl = this.configService.get('valuation.valuation.cacheTtl', 3600); // 1 hour for history + + // Try to get from cache first + const cachedHistory = await this.cacheService.get(cacheKey); + if (cachedHistory) { + this.logger.log(`Cache HIT for property history ${propertyId}`); + return cachedHistory; + } + + this.logger.log(`Cache MISS for property history ${propertyId}, fetching fresh data`); + const valuations = await (this.prisma as any).propertyValuation?.findMany({ where: { propertyId }, orderBy: { valuationDate: 'desc' }, }); - return valuations.map(v => ({ + const result = valuations.map(v => ({ propertyId: v.propertyId, estimatedValue: Number(v.estimatedValue), confidenceScore: v.confidenceScore, @@ -433,6 +453,12 @@ export class ValuationService { featuresUsed: v.featuresUsed ? JSON.parse(v.featuresUsed as string) : undefined, rawData: v.rawData ? JSON.parse(v.rawData as string) : undefined, })); + + // Cache the result with tags + await this.cacheService.set(cacheKey, result, { ttl: cacheTtl }); + await this.cacheService.tagEntry(cacheKey, ['valuation-history', 'property', propertyId]); + + return result; } /** @@ -487,19 +513,238 @@ export class ValuationService { }; } - private calculateTrendDirection(data: any[]) { - if (data.length < 2) { - return 'insufficient_data'; + private calculateTrendDirection(marketData: any[]): 'up' | 'down' | 'stable' { + if (marketData.length < 2) { + return 'stable'; } - const firstValue = data[0]._avg.estimatedValue; - const lastValue = data[data.length - 1]._avg.estimatedValue; + const first = marketData[0]._avg.estimatedValue; + const last = marketData[marketData.length - 1]._avg.estimatedValue; - if (firstValue && lastValue) { - const changePercent = ((lastValue - firstValue) / firstValue) * 100; - return changePercent > 0 ? 'upward' : changePercent < 0 ? 'downward' : 'stable'; + const change = ((last - first) / first) * 100; + + if (change > 5) { + return 'up'; + } + if (change < -5) { + return 'down'; } + return 'stable'; + } + + /** + * Invalidate cache for a specific property valuation and related caches + */ + async invalidatePropertyCache(propertyId: string): Promise { + const cacheKey = `valuation:${propertyId}`; + + // Invalidate with cascade to handle dependent caches + await this.cacheService.invalidateWithCascade(cacheKey); + + this.logger.log(`Invalidated cache for property ${propertyId} with cascade`); + } + + /** + * Invalidate related caches when property is updated + */ + private async invalidateRelatedCaches(propertyId: string): Promise { + // Invalidate property-related caches + await this.cacheService.invalidateByPattern(`property:${propertyId}*`); + await this.cacheService.invalidateByPattern(`valuation:${propertyId}*`); + await this.cacheService.invalidateByPattern(`document:property:${propertyId}*`); + + this.logger.log(`Invalidated related caches for property ${propertyId}`); + } - return 'unknown'; + /** + * Get cache metrics for valuations + */ + getValuationCacheMetrics() { + return this.cacheService.getMetrics('valuation'); + } + + /** + * Invalidate all valuations for a specific location + */ + async invalidateLocationValuations(location: string): Promise { + // Invalidate all valuations for properties in this location + await this.cacheService.invalidateByPattern(`valuation:*${location}*`); + await this.cacheService.invalidateByPattern(`valuation:history:*${location}*`); + + this.logger.log(`Invalidated valuations for location: ${location}`); + } + + /** + * Conditional invalidation based on valuation thresholds + */ + async conditionallyInvalidateValuations(condition: (value: any) => boolean): Promise { + await this.cacheService.conditionalInvalidate('valuation:*', condition); + this.logger.log('Conditionally invalidated valuations based on provided condition'); + } + + /** + * Warm cache with frequently accessed property valuations + */ + async warmValuationCache(): Promise { + this.logger.log('Starting valuation cache warming process'); + + // Identify frequently accessed properties (e.g., recently viewed, popular locations) + const frequentlyAccessedProperties = await this.getFrequentlyAccessedProperties(); + + const warmupTasks = frequentlyAccessedProperties.map(property => ({ + key: `valuation:${property.id}`, + factory: () => this.getFreshValuation(property.id, property), + options: { ttl: this.configService.get('valuation.valuation.cacheTtl', 86400) }, + condition: () => true, // Always run warming for these properties + })); + + await this.cacheService.warmCache(warmupTasks); + + this.logger.log( + `Completed warming cache for ${frequentlyAccessedProperties.length} frequently accessed properties`, + ); + } + + /** + * Warm cache with recent property valuations + */ + async warmRecentValuations(): Promise { + this.logger.log('Starting recent valuations cache warming'); + + // Get recently valued properties + const recentProperties = await this.getRecentValuedProperties(); + + const warmupTasks = recentProperties.map(property => ({ + key: `valuation:history:${property.propertyId}`, + factory: () => this.getPropertyHistory(property.propertyId), + options: { ttl: this.configService.get('valuation.history.cacheTtl', 3600) }, + condition: () => true, + })); + + await this.cacheService.warmCache(warmupTasks); + + this.logger.log(`Completed warming cache for ${recentProperties.length} recent valuations`); + } + + /** + * Get frequently accessed properties based on access logs or analytics + */ + private async getFrequentlyAccessedProperties(limit: number = 10): Promise { + // In a real implementation, this would query analytics/access logs + // For now, we'll return recently created properties as a proxy for frequently accessed + try { + const properties = await this.prisma.property.findMany({ + orderBy: { createdAt: 'desc' }, + take: limit, + select: { + id: true, + location: true, + bedrooms: true, + bathrooms: true, + squareFootage: true, + yearBuilt: true, + propertyType: true, + lotSize: true, + }, + }); + + return properties; + } catch (error) { + this.logger.error(`Failed to get frequently accessed properties: ${error.message}`); + return []; + } + } + + /** + * Get recently valued properties + */ + private async getRecentValuedProperties(limit: number = 10): Promise> { + try { + const recentValuations = await (this.prisma as any).propertyValuation?.findMany({ + orderBy: { valuationDate: 'desc' }, + take: limit, + select: { + propertyId: true, + }, + distinct: ['propertyId'], // Get unique property IDs + }); + + return recentValuations; + } catch (error) { + this.logger.error(`Failed to get recent valued properties: ${error.message}`); + return []; + } + } + + /** + * Get fresh valuation without using cache + */ + private async getFreshValuation(propertyId: string, features?: PropertyFeatures): Promise { + this.logger.log(`Fetching fresh valuation for property ${propertyId} (bypassing cache)`); + + try { + // Get property from database if features not provided + if (!features) { + const property = await this.prisma.property.findUnique({ + where: { id: propertyId }, + }); + + if (!property) { + throw new NotFoundException(`Property with ID ${propertyId} not found`); + } + + const prop = property as any; + features = { + id: prop.id, + location: prop.location, + bedrooms: prop.bedrooms || 0, + bathrooms: prop.bathrooms || 0, + squareFootage: prop.squareFootage ? Number(prop.squareFootage) : 0, + yearBuilt: prop.yearBuilt || new Date().getFullYear(), + propertyType: prop.propertyType || 'residential', + lotSize: prop.lotSize ? Number(prop.lotSize) : 0, + }; + } + + // Normalize features + const normalizedFeatures = this.normalizeFeatures(features); + + // Get valuation from external APIs + const valuations = await Promise.all([ + this.getZillowValuation(normalizedFeatures).catch(err => { + this.logger.warn(`Zillow API failed: ${err.message}`); + return null; + }), + this.getRedfinValuation(normalizedFeatures).catch(err => { + this.logger.warn(`Redfin API failed: ${err.message}`); + return null; + }), + this.getCoreLogicValuation(normalizedFeatures).catch(err => { + this.logger.warn(`CoreLogic API failed: ${err.message}`); + return null; + }), + ]); + + // Filter out null results + const validValuations = valuations.filter(val => val !== null); + + if (validValuations.length === 0) { + throw new HttpException('All external valuation APIs failed', HttpStatus.SERVICE_UNAVAILABLE); + } + + // Combine valuations using weighted average + const combinedValuation = this.combineValuations(validValuations); + + // Save valuation to database + const savedValuation = await this.saveValuation(combinedValuation); + + // Update property with valuation + await this.updatePropertyWithValuation(propertyId, savedValuation); + + return savedValuation; + } catch (error) { + this.logger.error(`Fresh valuation failed for property ${propertyId}: ${error.message}`); + throw error; + } } } diff --git a/test/valuation/valuation.controller.spec.ts b/test/valuation/valuation.controller.spec.ts index 918b0e8d..2851d5f6 100644 --- a/test/valuation/valuation.controller.spec.ts +++ b/test/valuation/valuation.controller.spec.ts @@ -77,8 +77,8 @@ describe('ValuationController', () => { const location = 'Test Location'; const expectedResult = { location, - trendData: [], - trendDirection: 'upward', + trendData: [] as { date: Date; avgValue: number }[], + trendDirection: 'up' as const, }; jest.spyOn(service, 'getMarketTrendAnalysis').mockResolvedValue(expectedResult); diff --git a/test/valuation/valuation.service.spec.ts b/test/valuation/valuation.service.spec.ts index adbc1955..3cdff1d2 100644 --- a/test/valuation/valuation.service.spec.ts +++ b/test/valuation/valuation.service.spec.ts @@ -1,7 +1,11 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ConfigService } from '@nestjs/config'; +import { CACHE_MANAGER } from '@nestjs/cache-manager'; +import { RedisService as NestRedisService } from '@liaoliaots/nestjs-redis'; import { PrismaService } from '../../src/database/prisma/prisma.service'; import { ValuationService } from '../../src/valuation/valuation.service'; +import { CacheService } from '../../src/common/services/cache.service'; +import { RedisService } from '../../src/common/services/redis.service'; import { Decimal } from '@prisma/client/runtime/library'; describe('ValuationService', () => { @@ -13,6 +17,41 @@ describe('ValuationService', () => { const module: TestingModule = await Test.createTestingModule({ providers: [ ValuationService, + CacheService, + RedisService, + { + provide: NestRedisService, + useValue: { + getOrThrow: jest.fn().mockReturnValue({ + get: jest.fn(), + set: jest.fn(), + del: jest.fn(), + exists: jest.fn(), + expire: jest.fn(), + keys: jest.fn(), + hgetall: jest.fn(), + hset: jest.fn(), + hdel: jest.fn(), + ttl: jest.fn(), + incr: jest.fn(), + publish: jest.fn(), + eval: jest.fn(), + flushdb: jest.fn(), + }), + }, + }, + { + provide: CACHE_MANAGER, + useValue: { + get: jest.fn(), + set: jest.fn(), + del: jest.fn(), + reset: jest.fn(), + store: { + reset: jest.fn(), + }, + }, + }, { provide: ConfigService, useValue: {