diff --git a/docker-compose.monitoring.yml b/docker-compose.monitoring.yml new file mode 100644 index 00000000..c74beb25 --- /dev/null +++ b/docker-compose.monitoring.yml @@ -0,0 +1,132 @@ +version: '3.8' + +services: + # ELK Stack for Centralized Logging + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0 + container_name: quest_elasticsearch + environment: + - discovery.type=single-node + - xpack.security.enabled=false + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ports: + - "9200:9200" + volumes: + - elasticsearch_data:/usr/share/elasticsearch/data + networks: + - quest-network + + logstash: + image: docker.elastic.co/logstash/logstash:8.15.0 + container_name: quest_logstash + ports: + - "5044:5044" + - "9600:9600" + volumes: + - ./monitoring/logstash/pipeline:/usr/share/logstash/pipeline + - ./monitoring/logstash/config:/usr/share/logstash/config + depends_on: + - elasticsearch + networks: + - quest-network + + kibana: + image: docker.elastic.co/kibana/kibana:8.15.0 + container_name: quest_kibana + ports: + - "5601:5601" + environment: + - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 + depends_on: + - elasticsearch + networks: + - quest-network + + # Prometheus for Metrics Collection + prometheus: + image: prom/prometheus:v2.53.0 + container_name: quest_prometheus + ports: + - "9090:9090" + volumes: + - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + - ./monitoring/prometheus/rules:/etc/prometheus/rules + - prometheus_data:/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + - '--storage.tsdb.path=/prometheus' + - '--web.console.libraries=/etc/prometheus/console_libraries' + - '--web.console.templates=/etc/prometheus/consoles' + - '--storage.tsdb.retention.time=200h' + - '--web.enable-lifecycle' + networks: + - quest-network + + # Grafana for Visualization + grafana: + image: grafana/grafana:11.1.0 + container_name: quest_grafana + ports: + - "3003:3000" + environment: + - GF_SECURITY_ADMIN_PASSWORD=admin123 + - GF_USERS_ALLOW_SIGN_UP=false + volumes: + - grafana_data:/var/lib/grafana + - ./monitoring/grafana/provisioning:/etc/grafana/provisioning + - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards + depends_on: + - prometheus + networks: + - quest-network + + # Jaeger for Distributed Tracing + jaeger: + image: jaegertracing/all-in-one:1.54 + container_name: quest_jaeger + ports: + - "16686:16686" + - "14268:14268" + - "14250:14250" + environment: + - COLLECTOR_OTLP_ENABLED=true + networks: + - quest-network + + # AlertManager for Alerting + alertmanager: + image: prom/alertmanager:v0.27.0 + container_name: quest_alertmanager + ports: + - "9093:9093" + volumes: + - ./monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml + networks: + - quest-network + + # Node Exporter for System Metrics + node-exporter: + image: prom/node-exporter:v1.8.1 + container_name: quest_node_exporter + ports: + - "9100:9100" + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - '--path.procfs=/host/proc' + - '--path.rootfs=/rootfs' + - '--path.sysfs=/host/sys' + - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' + networks: + - quest-network + +volumes: + elasticsearch_data: + prometheus_data: + grafana_data: + +networks: + quest-network: + external: true diff --git a/microservices/database-strategy.md b/microservices/database-strategy.md new file mode 100644 index 00000000..d5f7dc3f --- /dev/null +++ b/microservices/database-strategy.md @@ -0,0 +1,397 @@ +# Microservices Database Strategy and Schema Design + +## Overview + +This document outlines the database architecture strategy for Quest Service microservices, focusing on schema isolation, connection management, and scalability. + +## Database Architecture + +### Strategy: Database per Service with Shared Infrastructure + +We're implementing a **database-per-service** pattern with the following characteristics: + +- **Isolation**: Each microservice has its own database +- **Shared Infrastructure**: Common connection pooling, monitoring, and backup systems +- **Cross-Service Communication**: Event-driven communication via message queues +- **Read Replicas**: Read replicas for read-heavy services + +## Database Instances + +### Primary Databases + +| Service | Database Name | Purpose | Replication | +|----------|---------------|---------|-------------| +| quest-service | quest_db | Main application database | Yes | +| game-session-service | game_session_db | Session management | Yes | +| economy-service | economy_db | Transactions, shop, energy | Yes | +| notification-service | notification_db | Notifications, templates | No | +| social-service | social_db | Social features, friends | Yes | +| recommendation-service | recommendation_db | User recommendations | No | + +### Shared Infrastructure Database + +| Database | Purpose | +|----------|---------| +| quest_shared | User authentication, cross-service data | + +## Schema Design Standards + +### Naming Conventions + +#### Database Names +- Format: `{service}_db` +- Example: `game_session_db`, `economy_db` + +#### Table Names +- Format: `{entity_name}` (plural) +- Examples: `sessions`, `transactions`, `shop_items` + +#### Column Names +- Use snake_case +- Primary keys: `id` (UUID) +- Foreign keys: `{table}_id` +- Timestamps: `created_at`, `updated_at` + +#### Indexes +- Primary indexes on foreign keys +- Composite indexes for common query patterns +- Partial indexes for time-based queries + +### Data Types + +#### UUIDs +- Use UUID for all primary keys +- Format: `uuid` type in PostgreSQL +- Generated via `gen_random_uuid()` + +#### Timestamps +- Use `timestamp with time zone` +- Always store in UTC +- Convert to local time in application layer + +#### JSON Data +- Use `jsonb` for structured data +- Index frequently accessed JSON fields +- Use GIN indexes for JSON queries + +## Connection Management + +### Connection Pools + +#### Main Service (quest-service) +```yaml +pool: + min: 5 + max: 20 + idle_timeout: 30000 + connection_timeout: 10000 +``` + +#### Microservices +```yaml +pool: + min: 2 + max: 10 + idle_timeout: 30000 + connection_timeout: 10000 +``` + +### Read Replicas + +#### Configuration +```yaml +read_replicas: + - host: postgres-read-1 + port: 5432 + weight: 1 + - host: postgres-read-2 + port: 5432 + weight: 1 +``` + +#### Routing Rules +- Write operations: Primary database +- Read operations: Load balanced across replicas +- Real-time data: Primary database +- Analytics queries: Read replicas + +## Migration Strategy + +### Migration Framework +- **Tool**: TypeORM migrations +- **Environment**: Separate migrations per service +- **Execution**: Ordered migrations with rollback capability + +### Migration Process + +1. **Development** + ```bash + # Generate migration + npm run migration:generate -- --name AddNewFeature + + # Run migration + npm run migration:run + ``` + +2. **Production** + ```bash + # Create migration backup + pg_dump quest_db > backup_before_migration.sql + + # Run migration with dry run + npm run migration:run -- --dry-run + + # Execute migration + npm run migration:run + ``` + +### Migration Conventions + +#### File Naming +- Format: `YYYYMMDDHHMMSS-Description.ts` +- Example: `20240424120000-AddEnergySystem.ts` + +#### Class Naming +- Format: `DescriptionTimestamp` +- Example: `AddEnergySystem20240424120000` + +## Backup Strategy + +### Automated Backups + +#### Full Backups +- **Frequency**: Daily at 2:00 AM UTC +- **Retention**: 30 days +- **Storage**: Encrypted S3 bucket + +#### Incremental Backups +- **Frequency**: Every 6 hours +- **Retention**: 7 days +- **Storage**: Local encrypted storage + +#### Point-in-Time Recovery +- **Frequency**: Every 15 minutes +- **Retention**: 24 hours +- **Storage**: High-speed SSD + +### Backup Scripts + +```bash +#!/bin/bash +# backup-database.sh + +DB_NAME=$1 +BACKUP_DIR="/backups/database" +DATE=$(date +%Y%m%d_%H%M%S) +BACKUP_FILE="$BACKUP_DIR/${DB_NAME}_${DATE}.sql" + +# Create backup +pg_dump -h $DB_HOST -U $DB_USER -d $DB_NAME > $BACKUP_FILE + +# Compress and encrypt +gzip $BACKUP_FILE +gpg --symmetric --cipher-algo AES256 $BACKUP_FILE.gz + +# Upload to S3 +aws s3 cp $BACKUP_FILE.gz.enc s3://quest-backups/database/ + +# Cleanup local files +rm $BACKUP_FILE.gz* +``` + +## Monitoring and Observability + +### Database Metrics + +#### Connection Metrics +- Active connections +- Connection pool utilization +- Connection latency +- Failed connections + +#### Performance Metrics +- Query execution time +- Slow query count +- Index usage statistics +- Table bloat + +#### Storage Metrics +- Database size growth +- Table size distribution +- Index size +- WAL (Write-Ahead Log) size + +### Alerting Rules + +#### Critical Alerts +- Database connection failures +- Disk space > 90% +- Backup failures +- Replication lag > 5 minutes + +#### Warning Alerts +- Slow queries > 1 second +- Connection pool > 80% utilization +- Query timeout rate > 1% + +## Security Considerations + +### Access Control + +#### Database Users +```sql +-- Service-specific users +CREATE USER quest_service WITH PASSWORD 'secure_password'; +CREATE USER game_session_service WITH PASSWORD 'secure_password'; +CREATE USER economy_service WITH PASSWORD 'secure_password'; + +-- Grant specific privileges +GRANT ALL PRIVILEGES ON DATABASE quest_db TO quest_service; +GRANT ALL PRIVILEGES ON DATABASE game_session_db TO game_session_service; +GRANT ALL PRIVILEGES ON DATABASE economy_db TO economy_service; +``` + +#### Network Security +- SSL/TLS encryption for all connections +- IP whitelisting for database access +- VPN requirement for admin access + +### Data Encryption + +#### At Rest +- Transparent Data Encryption (TDE) +- Encrypted backups +- Encrypted storage volumes + +#### In Transit +- TLS 1.3 for all connections +- Certificate rotation every 90 days + +## Scalability Planning + +### Horizontal Scaling + +#### Read Scaling +- Multiple read replicas +- Connection pooling +- Query caching + +#### Write Scaling +- Database sharding (future) +- Write queue optimization +- Batch operations + +### Vertical Scaling + +#### Resource Allocation +```yaml +resources: + quest_service: + cpu: 2 cores + memory: 4GB + storage: 100GB SSD + + game_session_service: + cpu: 1 core + memory: 2GB + storage: 50GB SSD + + economy_service: + cpu: 2 cores + memory: 4GB + storage: 200GB SSD +``` + +## Disaster Recovery + +### Recovery Procedures + +#### Database Corruption +1. Identify corruption extent +2. Failover to replica +3. Restore from recent backup +4. Verify data integrity +5. Update application configuration + +#### Complete Outage +1. Activate disaster recovery site +2. Restore from latest backup +3. Update DNS records +4. Monitor system performance +5. Communicate status to stakeholders + +### Testing Strategy + +#### Backup Verification +- Weekly restore tests +- Data integrity checks +- Application compatibility verification + +#### Failover Testing +- Monthly failover drills +- Replica promotion tests +- Network connectivity verification + +## Documentation Standards + +### Schema Documentation + +#### Entity Relationship Diagrams +- Use Mermaid.js for visual documentation +- Include all entities and relationships +- Update with each schema change + +#### API Documentation +- Database query examples +- Performance considerations +- Index recommendations + +### Operational Documentation + +#### Runbooks +- Database maintenance procedures +- Performance tuning guides +- Emergency response procedures +- Troubleshooting checklists + +## Implementation Checklist + +### Initial Setup +- [ ] Create databases for each service +- [ ] Configure connection pools +- [ ] Set up read replicas +- [ ] Implement backup automation +- [ ] Configure monitoring +- [ ] Test disaster recovery + +### Ongoing Maintenance +- [ ] Daily backup verification +- [ ] Weekly performance reviews +- [ ] Monthly security audits +- [ ] Quarterly capacity planning +- [ ] Annual disaster recovery tests + +## Migration Guide + +### From Monolith to Microservices + +1. **Phase 1**: Set up new databases +2. **Phase 2**: Migrate non-critical data +3. **Phase 3**: Implement data synchronization +4. **Phase 4**: Migrate critical data +5. **Phase 5**: Switch traffic to new services +6. **Phase 6**: Decommission old database + +### Data Consistency + +#### Eventual Consistency +- Use message queues for data synchronization +- Implement idempotent operations +- Handle conflicts with resolution strategies + +#### Strong Consistency (Critical Operations) +- Financial transactions +- User authentication +- Game session state + +This database strategy provides a solid foundation for scalable, maintainable microservices with proper isolation, security, and disaster recovery capabilities. diff --git a/microservices/database/init-databases.sh b/microservices/database/init-databases.sh new file mode 100644 index 00000000..d72b3afe --- /dev/null +++ b/microservices/database/init-databases.sh @@ -0,0 +1,317 @@ +#!/bin/bash + +# Quest Service Database Initialization Script +# This script creates separate databases for each microservice + +set -e + +echo "Initializing Quest Service databases..." + +# Database connection parameters +DB_HOST=${DB_HOST:-localhost} +DB_PORT=${DB_PORT:-5432} +DB_USER=${DB_USER:-postgres} +DB_PASSWORD=${DB_PASSWORD:-password} + +# Function to create database if it doesn't exist +create_database() { + local db_name=$1 + echo "Creating database: $db_name" + + PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d postgres -c " + SELECT 'CREATE DATABASE $db_name' + WHERE NOT EXISTS ( + SELECT FROM pg_database WHERE datname = '$db_name' + );" || true + + if [ $? -eq 0 ]; then + echo "✓ Database $db_name created successfully" + else + echo "✗ Failed to create database $db_name" + exit 1 + fi +} + +# Function to create user and grant privileges +create_user() { + local username=$1 + local password=$2 + local database=$3 + + echo "Creating user: $username for database: $database" + + PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d postgres -c " + DO \$\$ + BEGIN; + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '$username') THEN + CREATE USER $username WITH PASSWORD '$password'; + END IF; + GRANT ALL PRIVILEGES ON DATABASE $database TO $username; + GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $username; + GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $username; + COMMIT; + \$\$" || true + + if [ $? -eq 0 ]; then + echo "✓ User $username created and granted privileges" + else + echo "✗ Failed to create user $username" + exit 1 + fi +} + +# Wait for PostgreSQL to be ready +echo "Waiting for PostgreSQL to be ready..." +until PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d postgres -c '\q'; do + echo "PostgreSQL is ready" + break +done +sleep 2 + +# Create databases for each microservice +echo "Creating microservice databases..." + +create_database "quest_db" +create_database "game_session_db" +create_database "economy_db" +create_database "notification_db" +create_database "social_db" +create_database "recommendation_db" + +# Create service-specific users with limited privileges +echo "Creating service users..." + +# Main quest service user (full access to quest_db) +create_user "quest_service_user" "quest_secure_password_2024" "quest_db" + +# Game session service user +create_user "game_session_service_user" "session_secure_password_2024" "game_session_db" + +# Economy service user +create_user "economy_service_user" "economy_secure_password_2024" "economy_db" + +# Notification service user +create_user "notification_service_user" "notification_secure_password_2024" "notification_db" + +# Social service user +create_user "social_service_user" "social_secure_password_2024" "social_db" + +# Recommendation service user +create_user "recommendation_service_user" "recommendation_secure_password_2024" "recommendation_db" + +# Create shared user for cross-service operations +echo "Creating shared service user..." +PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d postgres -c " + DO \$\$ + BEGIN; + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'quest_shared_user') THEN + CREATE USER quest_shared_user WITH PASSWORD 'shared_secure_password_2024'; + END IF; + COMMIT; + \$\$" || true + +# Grant shared user read access to all databases +for db in "quest_db" "game_session_db" "economy_db" "notification_db" "social_db" "recommendation_db"; do + echo "Granting shared user access to: $db" + PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d postgres -c " + GRANT CONNECT ON DATABASE $db TO quest_shared_user; + GRANT USAGE ON SCHEMA public TO quest_shared_user; + GRANT SELECT ON ALL TABLES IN SCHEMA public TO quest_shared_user;" || true +done + +# Create monitoring and backup functions +echo "Creating monitoring functions..." + +PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d postgres -c " + DO \$\$ + BEGIN; + -- Function to get database size + CREATE OR REPLACE FUNCTION get_database_size(db_name TEXT) + RETURNS TEXT AS \$\$ + DECLARE + size_result TEXT; + BEGIN + EXECUTE format('SELECT pg_size_pretty(pg_database_size(%L))', db_name) INTO size_result; + RETURN size_result; + END; + \$\$ LANGUAGE plpgsql; + + -- Function to get table sizes + CREATE OR REPLACE FUNCTION get_table_sizes(db_name TEXT) + RETURNS TABLE(table_name TEXT, size TEXT) AS \$\$ + BEGIN + RETURN QUERY + SELECT + schemaname||'.'||tablename as table_name, + pg_size_pretty(pg_total_relation_size(schemaname::regclass::oid)) as size + FROM pg_tables + WHERE schemaname = 'public' + AND pg_my_temp_schema() IS NULL; + END; + \$\$ LANGUAGE plpgsql; + COMMIT; + \$\$" || true + +# Set up replication user +echo "Creating replication user..." +PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d postgres -c " + DO \$\$ + BEGIN; + IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'quest_replicator') THEN + CREATE USER quest_replicator WITH REPLICATION ENCRYPTED PASSWORD 'replication_secure_password_2024'; + END IF; + COMMIT; + \$\$" || true + +# Create monitoring views +echo "Creating monitoring views..." + +for db in "quest_db" "game_session_db" "economy_db" "notification_db" "social_db" "recommendation_db"; do + echo "Creating monitoring views for: $db" + PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $db -c " + CREATE OR REPLACE VIEW database_stats AS + SELECT + '$db' as database_name, + pg_database_size('$db') as database_size_bytes, + pg_size_pretty(pg_database_size('$db')) as database_size_human, + (SELECT COUNT(*) FROM pg_tables WHERE schemaname = 'public') as table_count, + (SELECT SUM(pg_total_relation_size(schemaname||'.'||tablename)) + FROM pg_tables WHERE schemaname = 'public') as total_table_size; + " || true +done + +# Create backup helper functions +echo "Creating backup functions..." + +PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d postgres -c " + DO \$\$ + BEGIN; + -- Function to create backup + CREATE OR REPLACE FUNCTION create_backup(db_name TEXT, backup_path TEXT) + RETURNS BOOLEAN AS \$\$ + DECLARE + backup_command TEXT; + BEGIN + backup_command := format('pg_dump %s -h %s -p %s -U %s -d %s -f %s', + db_name, '$DB_HOST', '$DB_PORT', '$DB_USER', db_name, backup_path); + + -- This would typically be executed by the backup system + -- For now, just return true to indicate success + RETURN TRUE; + END; + \$\$ LANGUAGE plpgsql; + + -- Function to verify backup integrity + CREATE OR REPLACE FUNCTION verify_backup(backup_path TEXT) + RETURNS BOOLEAN AS \$\$ + DECLARE + restore_command TEXT; + BEGIN + -- This would typically verify backup integrity + -- For now, just return true + RETURN TRUE; + END; + \$\$ LANGUAGE plpgsql; + COMMIT; + \$\$" || true + +# Set up row-level security policies (optional, for multi-tenant scenarios) +echo "Setting up RLS policies..." + +PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d economy_db -c " + -- Enable RLS on transactions table + ALTER TABLE transactions ENABLE ROW LEVEL SECURITY; + + -- Policy to allow users to see only their own transactions + CREATE POLICY user_transactions_policy ON transactions + FOR ALL + TO economy_service_user + USING (user_id = current_setting('app.current_user_id'::TEXT)); + + -- Apply the policy + ALTER TABLE transactions FORCE ROW LEVEL SECURITY; +" || true + +# Create performance monitoring indexes +echo "Creating performance indexes..." + +PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d game_session_db -c " + -- Indexes for session management + CREATE INDEX IF NOT EXISTS idx_sessions_user_id_status ON sessions(user_id, status); + CREATE INDEX IF NOT EXISTS idx_sessions_last_active_at ON sessions(last_active_at); + CREATE INDEX IF NOT EXISTS idx_sessions_session_id ON sessions(session_id); + + -- Indexes for state snapshots + CREATE INDEX IF NOT EXISTS idx_state_snapshots_session_id_created ON state_snapshots(session_id, created_at); + CREATE INDEX IF NOT EXISTS idx_state_snapshots_session_id_type ON state_snapshots(session_id, snapshot_type); + + -- Indexes for replays + CREATE INDEX IF NOT EXISTS idx_replays_session_id ON replays(session_id); + CREATE INDEX IF NOT EXISTS idx_replays_user_id_created ON replays(user_id, created_at); +" || true + +PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d economy_db -c " + -- Indexes for transactions + CREATE INDEX IF NOT EXISTS idx_transactions_user_id_created ON transactions(user_id, created_at); + CREATE INDEX IF NOT EXISTS idx_transactions_status_created ON transactions(status, created_at); + CREATE INDEX IF NOT EXISTS idx_transactions_type_status ON transactions(type, status); + CREATE INDEX IF NOT EXISTS idx_transactions_user_id_type ON transactions(user_id, type); + + -- Indexes for user energy + CREATE INDEX IF NOT EXISTS idx_user_energy_user_id_type ON user_energy(user_id, energy_type); + CREATE INDEX IF NOT EXISTS idx_user_energy_last_regeneration ON user_energy(last_regeneration_time); + + -- Indexes for shop items + CREATE INDEX IF NOT EXISTS idx_shop_items_type_status ON shop_items(item_type, status); + CREATE INDEX IF NOT EXISTS idx_shop_items_created_at ON shop_items(created_at); +" || true + +# Set up database configuration +echo "Optimizing database configuration..." + +PGPASSWORD=$DB_PASSWORD psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d postgres -c " + -- Set session timeout + ALTER SYSTEM SET session_timeout = '300s'; + + -- Set statement timeout + ALTER SYSTEM SET statement_timeout = '60s'; + + -- Set work memory + ALTER SYSTEM SET work_mem = '256MB'; + + -- Set maintenance work memory + ALTER SYSTEM SET maintenance_work_mem = '1GB'; + + -- Set checkpoint completion target + ALTER SYSTEM SET checkpoint_completion_target = '0.7'; + + -- Set random page cost + ALTER SYSTEM SET random_page_cost = '1.1'; + + -- Set effective cache size + ALTER SYSTEM SET effective_cache_size = '4GB'; + + -- Reload configuration + SELECT pg_reload_conf(); +" || true + +echo "Database initialization completed successfully!" +echo "" +echo "Summary of created databases:" +echo "- quest_db (main service)" +echo "- game_session_db (session management)" +echo "- economy_db (economy and transactions)" +echo "- notification_db (notifications)" +echo "- social_db (social features)" +echo "- recommendation_db (recommendations)" +echo "" +echo "Service users created with appropriate privileges" +echo "Monitoring views and performance indexes configured" +echo "" +echo "Next steps:" +echo "1. Update your .env files with appropriate database credentials" +echo "2. Run migrations for each service" +echo "3. Start the microservices" +echo "" +echo "Database connection string format:" +echo "postgresql://username:password@localhost:5432/database_name" diff --git a/microservices/docker-compose.databases.yml b/microservices/docker-compose.databases.yml new file mode 100644 index 00000000..53a5ab4c --- /dev/null +++ b/microservices/docker-compose.databases.yml @@ -0,0 +1,250 @@ +version: '3.8' + +services: + # PostgreSQL Primary for Main Service + postgres: + image: postgres:15-alpine + container_name: quest_postgres_main + ports: + - "5432:5432" + environment: + POSTGRES_USER: ${DB_USER:-postgres} + POSTGRES_PASSWORD: ${DB_PASSWORD:-password} + POSTGRES_DB: postgres + volumes: + - postgres_main_data:/var/lib/postgresql/data + - ./database/init-databases.sh:/docker-entrypoint-initdb.d/init-databases.sh + networks: + - quest-network + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres}"] + interval: 5s + timeout: 5s + retries: 5 + + # PostgreSQL for Game Session Service + postgres-game-session: + image: postgres:15-alpine + container_name: quest_postgres_game_session + ports: + - "5433:5432" + environment: + POSTGRES_USER: ${DB_USER:-postgres} + POSTGRES_PASSWORD: ${DB_PASSWORD:-password} + POSTGRES_DB: game_session_db + volumes: + - postgres_game_session_data:/var/lib/postgresql/data + networks: + - quest-network + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres}"] + interval: 5s + timeout: 5s + retries: 5 + + # PostgreSQL for Economy Service + postgres-economy: + image: postgres:15-alpine + container_name: quest_postgres_economy + ports: + - "5434:5432" + environment: + POSTGRES_USER: ${DB_USER:-postgres} + POSTGRES_PASSWORD: ${DB_PASSWORD:-password} + POSTGRES_DB: economy_db + volumes: + - postgres_economy_data:/var/lib/postgresql/data + networks: + - quest-network + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres}"] + interval: 5s + timeout: 5s + retries: 5 + + # PostgreSQL for Notification Service + postgres-notification: + image: postgres:15-alpine + container_name: quest_postgres_notification + ports: + - "5435:5432" + environment: + POSTGRES_USER: ${DB_USER:-postgres} + POSTGRES_PASSWORD: ${DB_PASSWORD:-password} + POSTGRES_DB: notification_db + volumes: + - postgres_notification_data:/var/lib/postgresql/data + networks: + - quest-network + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres}"] + interval: 5s + timeout: 5s + retries: 5 + + # PostgreSQL for Social Service + postgres-social: + image: postgres:15-alpine + container_name: quest_postgres_social + ports: + - "5436:5432" + environment: + POSTGRES_USER: ${DB_USER:-postgres} + POSTGRES_PASSWORD: ${DB_PASSWORD:-password} + POSTGRES_DB: social_db + volumes: + - postgres_social_data:/var/lib/postgresql/data + networks: + - quest-network + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres}"] + interval: 5s + timeout: 5s + retries: 5 + + # Read Replica for Main Service + postgres-read-replica-1: + image: postgres:15-alpine + container_name: quest_postgres_read_replica_1 + ports: + - "5437:5432" + environment: + POSTGRES_USER: ${DB_USER:-postgres} + POSTGRES_PASSWORD: ${DB_PASSWORD:-password} + POSTGRES_DB: quest_db + PGUSER: ${DB_USER:-postgres} + POSTGRES_MASTER_SERVICE: postgres + volumes: + - postgres_read_replica_1_data:/var/lib/postgresql/data + networks: + - quest-network + depends_on: + postgres: + condition: service_healthy + + # Read Replica for Game Session Service + postgres-game-session-read: + image: postgres:15-alpine + container_name: quest_postgres_game_session_read + ports: + - "5438:5432" + environment: + POSTGRES_USER: ${DB_USER:-postgres} + POSTGRES_PASSWORD: ${DB_PASSWORD:-password} + POSTGRES_DB: game_session_db + PGUSER: ${DB_USER:-postgres} + POSTGRES_MASTER_SERVICE: postgres-game-session + volumes: + - postgres_game_session_read_data:/var/lib/postgresql/data + networks: + - quest-network + depends_on: + postgres-game-session: + condition: service_healthy + + # Redis for Session Caching + redis: + image: redis:7-alpine + container_name: quest_redis + ports: + - "6379:6379" + command: redis-server --requirepass ${REDIS_PASSWORD:-redis123} + volumes: + - redis_data:/data + networks: + - quest-network + healthcheck: + test: ["CMD", "redis-cli", "--raw", "incr", "ping"] + interval: 5s + timeout: 3s + retries: 5 + + # Redis Cluster for Microservices + redis-cluster-node-1: + image: redis:7-alpine + container_name: quest_redis_cluster_node_1 + ports: + - "7001:6379" + - "17001:16379" + command: redis-server --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --appendonly yes --appendfilename appendonly.aof + volumes: + - redis_cluster_1_data:/data + networks: + - quest-network + + redis-cluster-node-2: + image: redis:7-alpine + container_name: quest_redis_cluster_node_2 + ports: + - "7002:6379" + - "17002:16379" + command: redis-server --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --appendonly yes --appendfilename appendonly.aof + volumes: + - redis_cluster_2_data:/data + networks: + - quest-network + + # Database Management Tools + pgadmin: + image: dpage/pgadmin4:latest + container_name: quest_pgadmin + ports: + - "5050:80" + environment: + PGADMIN_DEFAULT_EMAIL: ${PGADMIN_EMAIL:-admin@quest-service.com} + PGADMIN_DEFAULT_PASSWORD: ${PGADMIN_PASSWORD:-admin123} + PGADMIN_CONFIG_SERVER_MODE: 'False' + volumes: + - pgadmin_data:/var/lib/pgadmin + networks: + - quest-network + depends_on: + - postgres + - postgres-game-session + - postgres-economy + + # Database Monitoring + postgres-exporter: + image: prometheuscommunity/postgres-exporter:v0.15.0 + container_name: quest_postgres_exporter + ports: + - "9187:9187" + environment: + DATA_SOURCE_NAME: quest + DATA_SOURCE_URI: postgres://${DB_USER:-postgres}:${DB_PASSWORD:-password}@postgres:5432/quest_db?sslmode=disable + networks: + - quest-network + depends_on: + postgres: + condition: service_healthy + + redis-exporter: + image: oliver006/redis_exporter:v1.52.0 + container_name: quest_redis_exporter + ports: + - "9121:9121" + environment: + REDIS_ADDR: redis://redis:6379 + REDIS_PASSWORD: ${REDIS_PASSWORD:-redis123} + networks: + - quest-network + depends_on: + redis: + condition: service_healthy + +volumes: + postgres_main_data: + postgres_game_session_data: + postgres_economy_data: + postgres_notification_data: + postgres_social_data: + postgres_read_replica_1_data: + postgres_game_session_read_data: + redis_data: + redis_cluster_1_data: + redis_cluster_2_data: + pgadmin_data: + +networks: + quest-network: + external: true diff --git a/microservices/game-session-service/src/app.module.ts b/microservices/game-session-service/src/app.module.ts index adbb95de..a8dff8b9 100644 --- a/microservices/game-session-service/src/app.module.ts +++ b/microservices/game-session-service/src/app.module.ts @@ -9,7 +9,7 @@ import { SessionService } from './services/session.service'; import { StateSnapshotService } from './services/state-snapshot.service'; import { ReplayService } from './services/replay.service'; import { RedisCacheService } from './services/redis-cache.service'; -import { TimeoutHandlerService } from './services/timeout-handler.service'; +import { SessionTimeoutService } from './services/session-timeout.service'; import { SessionController } from './controllers/session.controller'; import { StateController } from './controllers/state.controller'; import { ReplayController } from './controllers/replay.controller'; @@ -52,7 +52,7 @@ import { AppService } from './app.service'; StateSnapshotService, ReplayService, RedisCacheService, - TimeoutHandlerService, + SessionTimeoutService, ], }) export class AppModule {} diff --git a/microservices/game-session-service/src/services/session-timeout.service.ts b/microservices/game-session-service/src/services/session-timeout.service.ts new file mode 100644 index 00000000..43d0aa37 --- /dev/null +++ b/microservices/game-session-service/src/services/session-timeout.service.ts @@ -0,0 +1,44 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { SessionService } from './session.service'; +import { SessionStatus } from '../entities/session.entity'; + +@Injectable() +export class SessionTimeoutService { + private readonly logger = new Logger(SessionTimeoutService.name); + private readonly INACTIVE_THRESHOLD_SECONDS = 1800; // 30 minutes + private readonly CLEANUP_INTERVAL_SECONDS = 300; // 5 minutes + + constructor(private readonly sessionService: SessionService) {} + + @Cron(CronExpression.EVERY_MINUTE) + async handleSessionTimeouts(): Promise { + try { + // Handle expired sessions (timeout based on timeoutAt field) + const expiredSessions = await this.sessionService.getExpiredSessions(); + + for (const session of expiredSessions) { + await this.sessionService.timeout(session.sessionId); + this.logger.warn(`Session timed out: ${session.sessionId} for user: ${session.userId}`); + } + + // Handle inactive sessions (no recent activity) + const inactiveSessions = await this.sessionService.getInactiveSessions( + this.INACTIVE_THRESHOLD_SECONDS, + ); + + for (const session of inactiveSessions) { + await this.sessionService.abandon(session.sessionId); + this.logger.warn(`Session abandoned due to inactivity: ${session.sessionId} for user: ${session.userId}`); + } + + if (expiredSessions.length > 0 || inactiveSessions.length > 0) { + this.logger.log( + `Session cleanup completed: ${expiredSessions.length} expired, ${inactiveSessions.length} abandoned`, + ); + } + } catch (error) { + this.logger.error('Error during session timeout cleanup', error); + } + } +} diff --git a/monitoring/alertmanager/alertmanager.yml b/monitoring/alertmanager/alertmanager.yml new file mode 100644 index 00000000..107bc1db --- /dev/null +++ b/monitoring/alertmanager/alertmanager.yml @@ -0,0 +1,56 @@ +global: + smtp_smarthost: 'localhost:587' + smtp_from: 'alerts@quest-service.com' + smtp_require_tls: false + +route: + group_by: ['alertname', 'job'] + group_wait: 10s + group_interval: 10s + repeat_interval: 1h + receiver: 'web.hook' + routes: + - match: + severity: critical + receiver: 'critical-alerts' + - match: + severity: warning + receiver: 'warning-alerts' + +receivers: + - name: 'web.hook' + webhook_configs: + - url: 'http://quest-service:3000/webhooks/alerts' + send_resolved: true + + - name: 'critical-alerts' + email_configs: + - to: 'admin@quest-service.com' + subject: '[CRITICAL] Quest Service Alert: {{ .GroupLabels.alertname }}' + body: | + {{ range .Alerts }} + Alert: {{ .Annotations.summary }} + Description: {{ .Annotations.description }} + Labels: {{ range .Labels.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }} + {{ end }} + webhook_configs: + - url: 'http://quest-service:3000/webhooks/critical-alerts' + send_resolved: true + + - name: 'warning-alerts' + email_configs: + - to: 'devops@quest-service.com' + subject: '[WARNING] Quest Service Alert: {{ .GroupLabels.alertname }}' + body: | + {{ range .Alerts }} + Alert: {{ .Annotations.summary }} + Description: {{ .Annotations.description }} + Labels: {{ range .Labels.SortedPairs }}{{ .Name }}={{ .Value }} {{ end }} + {{ end }} + +inhibit_rules: + - source_match: + severity: 'critical' + target_match: + severity: 'warning' + equal: ['alertname', 'job'] diff --git a/monitoring/grafana/dashboards/quest-service-overview.json b/monitoring/grafana/dashboards/quest-service-overview.json new file mode 100644 index 00000000..01bcd699 --- /dev/null +++ b/monitoring/grafana/dashboards/quest-service-overview.json @@ -0,0 +1,135 @@ +{ + "dashboard": { + "id": null, + "title": "Quest Service Overview", + "tags": ["quest-service"], + "timezone": "browser", + "panels": [ + { + "id": 1, + "title": "Service Status", + "type": "stat", + "targets": [ + { + "expr": "up{job=~\"quest-service|notification-service|social-service|game-session-service|economy-service\"}", + "legendFormat": "{{job}}" + } + ], + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + }, + "type": "value" + } + ] + } + }, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0} + }, + { + "id": 2, + "title": "Request Rate", + "type": "graph", + "targets": [ + { + "expr": "sum(rate(http_requests_total{job=~\"quest-service|notification-service|social-service|game-session-service|economy-service\"}[5m])) by (job)", + "legendFormat": "{{job}}" + } + ], + "yAxes": [ + { + "label": "Requests/sec" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0} + }, + { + "id": 3, + "title": "Response Time (95th percentile)", + "type": "graph", + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=~\"quest-service|notification-service|social-service|game-session-service|economy-service\"}[5m])) by (le, job))", + "legendFormat": "{{job}}" + } + ], + "yAxes": [ + { + "label": "Seconds" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8} + }, + { + "id": 4, + "title": "Error Rate", + "type": "graph", + "targets": [ + { + "expr": "sum(rate(http_requests_total{job=~\"quest-service|notification-service|social-service|game-session-service|economy-service\",status=~\"5..\"}[5m])) by (job)", + "legendFormat": "{{job}}" + } + ], + "yAxes": [ + { + "label": "Errors/sec" + } + ], + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8} + }, + { + "id": 5, + "title": "Memory Usage", + "type": "graph", + "targets": [ + { + "expr": "(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100", + "legendFormat": "Memory Usage %" + } + ], + "yAxes": [ + { + "label": "Percentage", + "max": 100, + "min": 0 + } + ], + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 16} + }, + { + "id": 6, + "title": "CPU Usage", + "type": "graph", + "targets": [ + { + "expr": "100 - (avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m])) * 100)", + "legendFormat": "CPU Usage %" + } + ], + "yAxes": [ + { + "label": "Percentage", + "max": 100, + "min": 0 + } + ], + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 16} + } + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "refresh": "30s" + } +} diff --git a/monitoring/grafana/provisioning/dashboards/dashboard.yml b/monitoring/grafana/provisioning/dashboards/dashboard.yml new file mode 100644 index 00000000..240cb2e8 --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/dashboard.yml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: 'quest-service-dashboards' + orgId: 1 + folder: 'Quest Service' + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 00000000..34ea3627 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,21 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + + - name: Elasticsearch + type: elasticsearch + access: proxy + url: http://elasticsearch:9200 + database: "quest-logs-*" + jsonData: + timeField: "@timestamp" + esVersion: 8.0.0 + logMessageField: "message" + logLevelField: "log_level" + editable: true diff --git a/monitoring/logstash/config/logstash.yml b/monitoring/logstash/config/logstash.yml new file mode 100644 index 00000000..3be0b810 --- /dev/null +++ b/monitoring/logstash/config/logstash.yml @@ -0,0 +1,6 @@ +http.host: "0.0.0.0" +xpack.monitoring.elasticsearch.hosts: [ "http://elasticsearch:9200" ] +path.config: /usr/share/logstash/pipeline +pipeline.workers: 2 +pipeline.batch.size: 125 +pipeline.batch.delay: 50 diff --git a/monitoring/logstash/pipeline/logstash.conf b/monitoring/logstash/pipeline/logstash.conf new file mode 100644 index 00000000..a11b8cc8 --- /dev/null +++ b/monitoring/logstash/pipeline/logstash.conf @@ -0,0 +1,92 @@ +input { + beats { + port => 5044 + } + + tcp { + port => 5000 + codec => json + } + + udp { + port => 5001 + codec => json + } +} + +filter { + # Parse JSON logs + if [message] { + json { + source => "message" + target => "parsed" + } + } + + # Add service name from container + if [container] { + mutate { + add_field => { "service" => "%{[container][name]}" } + } + } + + # Parse timestamp + if [parsed][timestamp] { + date { + match => [ "[parsed][timestamp]", "ISO8601" ] + target => "@timestamp" + } + } + + # Extract log level + if [parsed][level] { + mutate { + add_field => { "log_level" => "%{[parsed][level]}" } + } + } + + # Clean up fields + mutate { + remove_field => [ "host", "agent", "ecs", "input", "log" ] + } +} + +output { + elasticsearch { + hosts => ["elasticsearch:9200"] + index => "quest-logs-%{+YYYY.MM.dd}" + template_name => "quest-logs" + template_pattern => "quest-logs-*" + template => { + "index_patterns" => ["quest-logs-*"], + "settings" => { + "number_of_shards" => 1, + "number_of_replicas" => 0 + }, + "mappings" => { + "properties" => { + "@timestamp" => { "type" => "date" }, + "service" => { "type" => "keyword" }, + "log_level" => { "type" => "keyword" }, + "message" => { "type" => "text" }, + "parsed" => { + "properties" => { + "level" => { "type" => "keyword" }, + "message" => { "type" => "text" }, + "timestamp" => { "type" => "date" }, + "traceId" => { "type" => "keyword" }, + "spanId" => { "type" => "keyword" }, + "userId" => { "type" => "keyword" }, + "requestId" => { "type" => "keyword" } + } + } + } + } + } + } + + # Debug output + stdout { + codec => rubydebug + } +} diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 00000000..cf6708df --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,71 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +rule_files: + - "rules/*.yml" + +alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + +scrape_configs: + # Prometheus itself + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + # Main Quest Service + - job_name: 'quest-service' + static_configs: + - targets: ['quest-service:3000'] + metrics_path: '/metrics' + scrape_interval: 15s + + # Notification Service + - job_name: 'notification-service' + static_configs: + - targets: ['notification-service:3000'] + metrics_path: '/metrics' + scrape_interval: 15s + + # Social Service + - job_name: 'social-service' + static_configs: + - targets: ['social-service:3000'] + metrics_path: '/metrics' + scrape_interval: 15s + + # Game Session Service + - job_name: 'game-session-service' + static_configs: + - targets: ['game-session-service:3000'] + metrics_path: '/metrics' + scrape_interval: 15s + + # Economy Service + - job_name: 'economy-service' + static_configs: + - targets: ['economy-service:3000'] + metrics_path: '/metrics' + scrape_interval: 15s + + # Node Exporter for System Metrics + - job_name: 'node-exporter' + static_configs: + - targets: ['node-exporter:9100'] + scrape_interval: 30s + + # PostgreSQL Exporter + - job_name: 'postgres-exporter' + static_configs: + - targets: ['postgres-exporter:9187'] + scrape_interval: 30s + + # Redis Exporter + - job_name: 'redis-exporter' + static_configs: + - targets: ['redis-exporter:9121'] + scrape_interval: 30s diff --git a/monitoring/prometheus/rules/alerts.yml b/monitoring/prometheus/rules/alerts.yml new file mode 100644 index 00000000..088323d7 --- /dev/null +++ b/monitoring/prometheus/rules/alerts.yml @@ -0,0 +1,112 @@ +groups: + - name: quest-service-alerts + rules: + # Service Health Alerts + - alert: ServiceDown + expr: up == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Service {{ $labels.job }} is down" + description: "Service {{ $labels.job }} has been down for more than 1 minute." + + # High Error Rate Alerts + - alert: HighErrorRate + expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.1 + for: 2m + labels: + severity: warning + annotations: + summary: "High error rate on {{ $labels.job }}" + description: "Error rate is {{ $value }} errors per second on {{ $labels.job }}." + + # High Response Time Alerts + - alert: HighResponseTime + expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 1 + for: 2m + labels: + severity: warning + annotations: + summary: "High response time on {{ $labels.job }}" + description: "95th percentile response time is {{ $value }} seconds on {{ $labels.job }}." + + # Memory Usage Alerts + - alert: HighMemoryUsage + expr: (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes > 0.9 + for: 5m + labels: + severity: warning + annotations: + summary: "High memory usage" + description: "Memory usage is above 90% on {{ $labels.instance }}." + + # CPU Usage Alerts + - alert: HighCPUUsage + expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 + for: 5m + labels: + severity: warning + annotations: + summary: "High CPU usage" + description: "CPU usage is above 80% on {{ $labels.instance }}." + + # Database Connection Alerts + - alert: DatabaseConnectionHigh + expr: pg_stat_activity_count > 80 + for: 2m + labels: + severity: warning + annotations: + summary: "High database connections" + description: "Database has {{ $value }} active connections." + + # Redis Memory Usage Alerts + - alert: RedisMemoryHigh + expr: redis_memory_used_bytes / redis_memory_max_bytes > 0.9 + for: 5m + labels: + severity: warning + annotations: + summary: "Redis memory usage high" + description: "Redis memory usage is {{ $value | humanizePercentage }}." + + # Disk Space Alerts + - alert: DiskSpaceLow + expr: (node_filesystem_avail_bytes / node_filesystem_size_bytes) * 100 < 10 + for: 5m + labels: + severity: critical + annotations: + summary: "Low disk space" + description: "Disk space is below 10% on {{ $labels.instance }}." + + # Game Session Specific Alerts + - alert: GameSessionTimeouts + expr: rate(game_session_timeouts_total[5m]) > 0.05 + for: 2m + labels: + severity: warning + annotations: + summary: "High game session timeouts" + description: "Game session timeout rate is {{ $value }} per second." + + # Economy Service Alerts + - alert: TransactionFailures + expr: rate(economy_transaction_failures_total[5m]) > 0.02 + for: 2m + labels: + severity: warning + annotations: + summary: "High transaction failure rate" + description: "Economy transaction failure rate is {{ $value }} per second." + + # Queue Size Alerts + - alert: QueueSizeHigh + expr: bullmq_queue_size > 1000 + for: 5m + labels: + severity: warning + annotations: + summary: "High queue size" + description: "Queue {{ $labels.queue }} has {{ $value }} pending jobs." diff --git a/monitoring/runbooks/README.md b/monitoring/runbooks/README.md new file mode 100644 index 00000000..b77d73b7 --- /dev/null +++ b/monitoring/runbooks/README.md @@ -0,0 +1,483 @@ +# Quest Service Monitoring Runbooks + +## Table of Contents + +1. [Service Down Alert](#service-down-alert) +2. [High Error Rate Alert](#high-error-rate-alert) +3. [High Response Time Alert](#high-response-time-alert) +4. [High Memory Usage Alert](#high-memory-usage-alert) +5. [High CPU Usage Alert](#high-cpu-usage-alert) +6. [Database Connection Issues](#database-connection-issues) +7. [Redis Memory Issues](#redis-memory-issues) +8. [Low Disk Space Alert](#low-disk-space-alert) +9. [Game Session Timeouts](#game-session-timeouts) +10. [Economy Transaction Failures](#economy-transaction-failures) + +--- + +## Service Down Alert + +### Symptoms +- Service shows as DOWN in Grafana dashboard +- Health checks failing +- Users unable to access specific functionality + +### Possible Causes +- Service crash +- Network connectivity issues +- Resource exhaustion +- Deployment issues + +### Troubleshooting Steps + +1. **Check Service Status** + ```bash + docker ps | grep quest-service + docker logs quest-service --tail=100 + ``` + +2. **Check Resource Usage** + ```bash + docker stats quest-service + ``` + +3. **Check Network Connectivity** + ```bash + curl -f http://localhost:3000/health || echo "Service unreachable" + ``` + +4. **Restart Service if Needed** + ```bash + docker restart quest-service + ``` + +5. **Check Dependencies** + - Database connectivity + - Redis connectivity + - External API status + +### Prevention +- Set up proper health checks +- Monitor resource usage +- Implement circuit breakers for external dependencies + +--- + +## High Error Rate Alert + +### Symptoms +- Error rate > 10% for sustained period +- 5xx responses increasing +- User complaints about errors + +### Possible Causes +- Code bugs +- Database issues +- External service failures +- Resource constraints + +### Troubleshooting Steps + +1. **Check Error Logs** + ```bash + docker logs quest-service | grep ERROR | tail -50 + ``` + +2. **Analyze Error Patterns** + - Check Kibana for error patterns + - Look for recent deployments + - Check database query performance + +3. **Check Dependencies** + ```bash + # Database + docker exec -it postgres pg_isready -U postgres + + # Redis + docker exec -it redis redis-cli ping + ``` + +4. **Rollback Recent Changes** + - If recent deployment, consider rollback + - Check feature flags + +### Prevention +- Implement comprehensive error handling +- Add proper logging +- Use canary deployments +- Set up automated testing + +--- + +## High Response Time Alert + +### Symptoms +- 95th percentile response time > 1 second +- User complaints about slowness +- Timeouts occurring + +### Possible Causes +- Database performance issues +- Resource contention +- Inefficient queries +- Network latency + +### Troubleshooting Steps + +1. **Check Database Performance** + ```sql + -- Check slow queries + SELECT query, mean_time, calls + FROM pg_stat_statements + ORDER BY mean_time DESC + LIMIT 10; + ``` + +2. **Check Resource Usage** + ```bash + docker stats + top + ``` + +3. **Analyze Application Logs** + - Look for timeout errors + - Check for database connection pool issues + +4. **Profile Application** + - Use APM tools + - Check for memory leaks + - Analyze garbage collection + +### Prevention +- Implement caching strategies +- Optimize database queries +- Use connection pooling +- Monitor and optimize regularly + +--- + +## High Memory Usage Alert + +### Symptoms +- Memory usage > 90% for sustained period +- Service becoming unresponsive +- Out-of-memory errors + +### Possible Causes +- Memory leaks +- Increased traffic +- Inefficient code +- Large data processing + +### Troubleshooting Steps + +1. **Check Memory Usage** + ```bash + free -h + docker stats quest-service + ``` + +2. **Analyze Memory Patterns** + ```bash + # Check for memory leaks + docker logs quest-service | grep -i "out of memory" + ``` + +3. **Check Application Metrics** + - Monitor heap usage in Grafana + - Look for memory growth patterns + +4. **Restart Service if Needed** + ```bash + docker restart quest-service + ``` + +### Prevention +- Implement memory monitoring +- Use memory profiling tools +- Optimize data structures +- Set appropriate memory limits + +--- + +## High CPU Usage Alert + +### Symptoms +- CPU usage > 80% for sustained period +- System becoming sluggish +- Response times increasing + +### Possible Causes +- High traffic volume +- CPU-intensive operations +- Inefficient algorithms +- Infinite loops + +### Troubleshooting Steps + +1. **Check CPU Usage** + ```bash + top + docker stats quest-service + ``` + +2. **Identify CPU-Intensive Processes** + ```bash + ps aux | sort -rk 3 | head -10 + ``` + +3. **Analyze Application Performance** + - Check for infinite loops + - Profile CPU usage + - Review recent code changes + +### Prevention +- Implement CPU monitoring +- Use efficient algorithms +- Scale horizontally when needed +- Implement rate limiting + +--- + +## Database Connection Issues + +### Symptoms +- High number of active connections +- Connection timeouts +- Database errors in logs + +### Possible Causes +- Connection leaks +- High query load +- Database performance issues +- Network problems + +### Troubleshooting Steps + +1. **Check Connection Count** + ```sql + SELECT count(*) FROM pg_stat_activity; + ``` + +2. **Check Long-Running Queries** + ```sql + SELECT pid, now() - pg_stat_activity.query_start AS duration, query + FROM pg_stat_activity + WHERE state = 'active' + ORDER BY duration DESC; + ``` + +3. **Check Database Performance** + ```bash + docker exec -it postgres psql -U postgres -c "SELECT * FROM pg_stat_database;" + ``` + +### Prevention +- Implement connection pooling +- Monitor query performance +- Set connection limits +- Regular database maintenance + +--- + +## Redis Memory Issues + +### Symptoms +- Redis memory usage > 90% +- Redis evictions increasing +- Cache performance degradation + +### Possible Causes +- Memory leaks +- Large cached objects +- Insufficient memory limits +- Too many keys + +### Troubleshooting Steps + +1. **Check Redis Memory Usage** + ```bash + docker exec -it redis redis-cli info memory + ``` + +2. **Analyze Memory Usage** + ```bash + docker exec -it redis redis-cli memory usage + ``` + +3. **Check for Memory Leaks** + - Monitor key count over time + - Check for expired keys not being cleaned + +### Prevention +- Set appropriate memory limits +- Implement key expiration +- Monitor memory usage patterns +- Use Redis clustering if needed + +--- + +## Low Disk Space Alert + +### Symptoms +- Disk usage < 10% available +- Write errors in logs +- Service failures + +### Possible Causes +- Log files accumulating +- Database growth +- Temporary files not cleaned +- Backup files + +### Troubleshooting Steps + +1. **Check Disk Usage** + ```bash + df -h + du -sh /var/log/* + ``` + +2. **Clean Up Log Files** + ```bash + # Clean old logs + find /var/log -name "*.log" -mtime +7 -delete + + # Clean docker logs + docker system prune -f + ``` + +3. **Check Database Size** + ```bash + docker exec -it postgres psql -U postgres -c "SELECT pg_size_pretty(pg_database_size('quest_db'));" + ``` + +### Prevention +- Implement log rotation +- Set up automated cleanup +- Monitor disk usage trends +- Plan capacity accordingly + +--- + +## Game Session Timeouts + +### Symptoms +- High session timeout rate +- Users losing game progress +- Complaints about session issues + +### Possible Causes +- Redis connectivity issues +- Session management bugs +- Network timeouts +- Resource constraints + +### Troubleshooting Steps + +1. **Check Session Logs** + ```bash + docker logs game-session-service | grep -i timeout | tail -20 + ``` + +2. **Check Redis Connectivity** + ```bash + docker exec -it redis redis-cli ping + ``` + +3. **Analyze Session Patterns** + - Check for specific timeout patterns + - Look for correlation with system load + +### Prevention +- Implement proper session management +- Add session heartbeat mechanism +- Monitor session health +- Implement session recovery + +--- + +## Economy Transaction Failures + +### Symptoms +- High transaction failure rate +- Users losing currency/items +- Payment processing issues + +### Possible Causes +- Database transaction conflicts +- Payment gateway issues +- Insufficient funds +- Race conditions + +### Troubleshooting Steps + +1. **Check Transaction Logs** + ```bash + docker logs economy-service | grep -i "transaction.*fail" | tail -20 + ``` + +2. **Check Database Integrity** + ```sql + -- Check for orphaned transactions + SELECT * FROM economy_transactions WHERE status = 'pending' AND created_at < NOW() - INTERVAL '1 hour'; + ``` + +3. **Verify Payment Gateway** + - Check external payment service status + - Verify API credentials + - Test payment flow + +### Prevention +- Implement proper transaction handling +- Add retry mechanisms +- Use database transactions properly +- Implement audit logging + +--- + +## Emergency Contacts + +- **DevOps Team**: devops@quest-service.com +- **Development Team**: dev@quest-service.com +- **On-call Engineer**: +1-555-0123 + +## Escalation Procedures + +1. **Level 1**: Automated alerts, basic troubleshooting +2. **Level 2**: DevOps team notification (5 minutes) +3. **Level 3**: Development team notification (15 minutes) +4. **Level 4**: Management notification (30 minutes) + +## Communication Templates + +### Service Outage Template +``` +Subject: [OUTAGE] Quest Service - [Service Name] Down + +Status: INVESTIGATING +Impact: Users experiencing [specific impact] +Started: [timestamp] +Next Update: [timestamp + 15 min] + +Details: +[Brief description of issue] + +Actions: +[Current troubleshooting steps] +``` + +### Resolution Template +``` +Subject: [RESOLVED] Quest Service - [Service Name] Restored + +Status: RESOLVED +Duration: [total outage time] +Impact: [affected users/services] + +Root Cause: +[Final analysis of issue] + +Preventive Measures: +[Steps to prevent recurrence] +``` diff --git a/package.json b/package.json index 683070f2..5aaa967a 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "@types/passport-jwt": "^4.0.1", "@types/qrcode": "^1.5.6", "@willsoto/nestjs-prometheus": "^6.0.2", + "prom-client": "^15.1.3", "amqp-connection-manager": "^4.1.14", "amqplib": "^0.10.3", "axios": "^1.13.5", diff --git a/src/app.module.ts b/src/app.module.ts index 3236dd9a..984a31e0 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -55,6 +55,7 @@ import { WebhooksModule } from './webhooks/webhooks.module'; import { PlayerEventsModule } from './player-events/player-events.module'; import { AccountModule } from './account/account.module'; import { BlockchainEventsModule } from './blockchain-events/blockchain-events.module'; +import { MetricsModule } from './common/metrics/metrics.module'; @Module({ imports: [ @@ -159,6 +160,7 @@ import { BlockchainEventsModule } from './blockchain-events/blockchain-events.mo WebhooksModule, AccountModule, BlockchainEventsModule, + MetricsModule, ], controllers: [AppController], providers: [ diff --git a/src/common/logging/structured-logger.interceptor.ts b/src/common/logging/structured-logger.interceptor.ts new file mode 100644 index 00000000..bc41d30f --- /dev/null +++ b/src/common/logging/structured-logger.interceptor.ts @@ -0,0 +1,143 @@ +import { Injectable, NestInterceptor, ExecutionContext, CallHandler } from '@nestjs/common'; +import { Observable } from 'rxjs'; +import { tap } from 'rxjs/operators'; +import { Request, Response } from 'express'; +import { v4 as uuidv4 } from 'uuid'; + +export interface StructuredLog { + timestamp: string; + traceId: string; + spanId: string; + level: 'info' | 'warn' | 'error' | 'debug'; + message: string; + service: string; + method: string; + url: string; + statusCode?: number; + userId?: string; + requestId?: string; + duration?: number; + error?: any; + metadata?: any; +} + +@Injectable() +export class StructuredLoggerInterceptor implements NestInterceptor { + private serviceName = process.env.SERVICE_NAME || 'quest-service'; + + intercept(context: ExecutionContext, next: CallHandler): Observable { + const request = context.switchToHttp().getRequest(); + const response = context.switchToHttp().getResponse(); + const startTime = Date.now(); + + const traceId = this.getTraceId(request); + const spanId = uuidv4(); + const requestId = uuidv4(); + + const { method, url } = request; + + // Add trace context to request for use in controllers + request['traceId'] = traceId; + request['spanId'] = spanId; + request['requestId'] = requestId; + + this.log({ + timestamp: new Date().toISOString(), + traceId, + spanId, + level: 'info', + message: 'Request started', + service: this.serviceName, + method, + url, + userId: this.extractUserId(request), + requestId, + metadata: { + userAgent: request.get('user-agent'), + ip: request.ip, + }, + }); + + return next.handle().pipe( + tap({ + next: (data) => { + const duration = Date.now() - startTime; + const statusCode = response.statusCode; + + this.log({ + timestamp: new Date().toISOString(), + traceId, + spanId, + level: 'info', + message: 'Request completed', + service: this.serviceName, + method, + url, + statusCode, + userId: this.extractUserId(request), + requestId, + duration, + metadata: { + responseSize: JSON.stringify(data).length, + }, + }); + }, + error: (error) => { + const duration = Date.now() - startTime; + const statusCode = error.status || error.statusCode || 500; + + this.log({ + timestamp: new Date().toISOString(), + traceId, + spanId, + level: 'error', + message: 'Request failed', + service: this.serviceName, + method, + url, + statusCode, + userId: this.extractUserId(request), + requestId, + duration, + error: { + name: error.name, + message: error.message, + stack: error.stack, + }, + }); + }, + }), + ); + } + + private getTraceId(request: Request): string { + return ( + request.headers['x-trace-id'] as string || + request.headers['x-request-id'] as string || + uuidv4() + ); + } + + private extractUserId(request: Request): string | undefined { + // Try to extract user ID from various sources + return ( + (request.user as any)?.id || + request.headers['x-user-id'] as string || + request.query.userId as string + ); + } + + private log(logEntry: StructuredLog): void { + // In production, this would be sent to Logstash/Elasticsearch + // For now, we'll use console with structured format + const logOutput = JSON.stringify(logEntry); + + if (logEntry.level === 'error') { + console.error(logOutput); + } else if (logEntry.level === 'warn') { + console.warn(logOutput); + } else { + console.log(logOutput); + } + } +} diff --git a/src/common/metrics/metrics.interceptor.ts b/src/common/metrics/metrics.interceptor.ts new file mode 100644 index 00000000..601ea23d --- /dev/null +++ b/src/common/metrics/metrics.interceptor.ts @@ -0,0 +1,47 @@ +import { Injectable, NestInterceptor, ExecutionContext, CallHandler } from '@nestjs/common'; +import { Observable } from 'rxjs'; +import { tap } from 'rxjs/operators'; +import { Request, Response } from 'express'; +import { MetricsService } from './metrics.service'; + +@Injectable() +export class MetricsInterceptor implements NestInterceptor { + constructor(private readonly metricsService: MetricsService) {} + + intercept(context: ExecutionContext, next: CallHandler): Observable { + const request = context.switchToHttp().getRequest(); + const response = context.switchToHttp().getResponse(); + const startTime = Date.now(); + + const { method, url } = request; + + return next.handle().pipe( + tap({ + next: () => { + const duration = (Date.now() - startTime) / 1000; + const statusCode = response.statusCode.toString(); + const route = this.extractRoute(url); + + this.metricsService.incrementHttpRequests(method, route, statusCode); + this.metricsService.recordHttpRequestDuration(method, route, statusCode, duration); + }, + error: () => { + const duration = (Date.now() - startTime) / 1000; + const statusCode = response.statusCode?.toString() || '500'; + const route = this.extractRoute(url); + + this.metricsService.incrementHttpRequests(method, route, statusCode); + this.metricsService.recordHttpRequestDuration(method, route, statusCode, duration); + }, + }), + ); + } + + private extractRoute(url: string): string { + // Extract route from URL, removing query parameters and IDs + const route = url.split('?')[0]; + + // Replace numeric IDs with placeholder + return route.replace(/\/\d+/g, '/:id'); + } +} diff --git a/src/common/metrics/metrics.module.ts b/src/common/metrics/metrics.module.ts new file mode 100644 index 00000000..51f03f12 --- /dev/null +++ b/src/common/metrics/metrics.module.ts @@ -0,0 +1,23 @@ +import { Module } from '@nestjs/common'; +import { PrometheusModule } from '@willsoto/nestjs-prometheus'; +import { MetricsService } from './metrics.service'; + +@Module({ + imports: [ + PrometheusModule.register({ + path: '/metrics', + defaultMetrics: { + enabled: true, + config: { + labels: { + app: 'quest-service', + version: process.env.APP_VERSION || '1.0.0', + }, + }, + }, + }), + ], + providers: [MetricsService], + exports: [MetricsService], +}) +export class MetricsModule {} diff --git a/src/common/metrics/metrics.service.ts b/src/common/metrics/metrics.service.ts new file mode 100644 index 00000000..b178429b --- /dev/null +++ b/src/common/metrics/metrics.service.ts @@ -0,0 +1,114 @@ +import { Injectable } from '@nestjs/common'; +import { Counter, Histogram, Gauge, Registry } from 'prom-client'; + +@Injectable() +export class MetricsService { + private registry: Registry; + + // HTTP Metrics + private httpRequestsTotal: Counter; + private httpRequestDuration: Histogram; + + // Business Metrics + private gameSessionsTotal: Counter; + private gameSessionsActive: Gauge; + private puzzleCompletions: Counter; + private economyTransactions: Counter; + + constructor() { + this.registry = new Registry(); + this.initializeMetrics(); + } + + private initializeMetrics() { + // HTTP Request Metrics + this.httpRequestsTotal = new Counter({ + name: 'http_requests_total', + help: 'Total number of HTTP requests', + labelNames: ['method', 'route', 'status_code'], + registers: [this.registry], + }); + + this.httpRequestDuration = new Histogram({ + name: 'http_request_duration_seconds', + help: 'Duration of HTTP requests in seconds', + labelNames: ['method', 'route', 'status_code'], + buckets: [0.1, 0.3, 0.5, 0.7, 1, 3, 5, 7, 10], + registers: [this.registry], + }); + + // Game Session Metrics + this.gameSessionsTotal = new Counter({ + name: 'game_sessions_total', + help: 'Total number of game sessions created', + labelNames: ['user_id', 'puzzle_type'], + registers: [this.registry], + }); + + this.gameSessionsActive = new Gauge({ + name: 'game_sessions_active', + help: 'Number of active game sessions', + labelNames: ['puzzle_type'], + registers: [this.registry], + }); + + // Puzzle Completion Metrics + this.puzzleCompletions = new Counter({ + name: 'puzzle_completions_total', + help: 'Total number of puzzle completions', + labelNames: ['puzzle_type', 'difficulty', 'user_id'], + registers: [this.registry], + }); + + // Economy Transaction Metrics + this.economyTransactions = new Counter({ + name: 'economy_transactions_total', + help: 'Total number of economy transactions', + labelNames: ['transaction_type', 'status', 'user_id'], + registers: [this.registry], + }); + } + + // HTTP Metrics Methods + incrementHttpRequests(method: string, route: string, statusCode: string) { + this.httpRequestsTotal + .labels(method, route, statusCode) + .inc(); + } + + recordHttpRequestDuration(method: string, route: string, statusCode: string, duration: number) { + this.httpRequestDuration + .labels(method, route, statusCode) + .observe(duration); + } + + // Game Session Metrics Methods + incrementGameSessions(userId: string, puzzleType: string) { + this.gameSessionsTotal + .labels(userId, puzzleType) + .inc(); + } + + setActiveGameSessions(puzzleType: string, count: number) { + this.gameSessionsActive + .labels(puzzleType) + .set(count); + } + + incrementPuzzleCompletions(puzzleType: string, difficulty: string, userId: string) { + this.puzzleCompletions + .labels(puzzleType, difficulty, userId) + .inc(); + } + + // Economy Metrics Methods + incrementEconomyTransactions(transactionType: string, status: string, userId: string) { + this.economyTransactions + .labels(transactionType, status, userId) + .inc(); + } + + getRegistry(): Registry { + return this.registry; + } +} diff --git a/src/main.ts b/src/main.ts index d955b364..095bffe6 100644 --- a/src/main.ts +++ b/src/main.ts @@ -7,6 +7,8 @@ import helmet from 'helmet'; import { AppModule } from './app.module'; import { AllExceptionsFilter } from './common/exceptions/http-exception.filter'; import { SanitizeInterceptor } from './common/interceptors/sanitize.interceptor'; +import { MetricsInterceptor } from './common/metrics/metrics.interceptor'; +import { MetricsService } from './common/metrics/metrics.service'; import * as Sentry from '@sentry/node'; import { ThrottlerGuard } from '@nestjs/throttler'; import { MicroserviceOptions, Transport } from '@nestjs/microservices'; @@ -67,8 +69,11 @@ async function bootstrap() { // Global exception filter app.useGlobalFilters(new AllExceptionsFilter()); - // Global sanitize interceptor - app.useGlobalInterceptors(new SanitizeInterceptor()); + // Global interceptors + app.useGlobalInterceptors( + new SanitizeInterceptor(), + new MetricsInterceptor(app.get(MetricsService)), + ); app.setGlobalPrefix(apiPrefix);