Merge branch 'fix/enforce-encryption-at-rest' into 'main'

Verify and enforce encryption at rest (AES-256)

Closes #136

See merge request postgres-ai/postgresai!216

Author: Sarumyan

.gitlab-ci.yml

@@ -219,6 +219,31 @@ vm-auth:config:tests:
     - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
     - if: '$CI_COMMIT_BRANCH == "main"'
 
+terraform:aws:tests:
+  stage: test
+  image:
+    name: hashicorp/terraform:1.9
+    entrypoint: [""]
+  variables:
+    TF_IN_AUTOMATION: "true"
+  script:
+    # file("~/.ssh/test-key.pem") in terraform_data connection block is
+    # evaluated eagerly during config parsing even though the resource is
+    # overridden in tests. Create a dummy file so file() resolves; the
+    # content is never used for an actual SSH connection.
+    - mkdir -p ~/.ssh && printf 'dummy-key-for-testing\n' > ~/.ssh/test-key.pem
+    - cd terraform/aws
+    - terraform init -backend=false
+    - terraform validate
+    - terraform test -filter=tests/kms_encryption.tftest.hcl
+  rules:
+    - if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
+      changes:
+        - terraform/aws/**/*
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      changes:
+        - terraform/aws/**/*
+
 cli:node:smoke:
   stage: test
   image: node:20-alpine

postgres_ai_helm/values.yaml

@@ -22,6 +22,15 @@ storage:
   victoriaMetricsSize: 150Gi
   grafanaSize: 5Gi
   accessModes: ["ReadWriteOnce"]
+  # Storage class for persistent volumes.
+  # For encryption at rest (required for DPA/SOC2 compliance), use an
+  # encrypted storage class. Examples:
+  #   AWS EKS:  create a StorageClass with "encrypted: true" parameter,
+  #             or enable account-level default EBS encryption
+  #   GCP GKE:  "standard" or "premium-rwo" (encrypted at rest by default with Google-managed keys)
+  #   Azure AKS: "managed-premium" (encrypted by default with platform-managed keys)
+  #   Hetzner HCloud: "hcloud-volumes" (encrypted at rest by default)
+  # Leave empty to use the cluster default storage class.
   storageClassName: ""
 
 global:

terraform/aws/README.md

@@ -9,7 +9,7 @@ Single EC2 instance with Docker Compose.
 Terraform creates:
 - VPC with public subnet
 - EC2 instance (Ubuntu 22.04 LTS)
-- EBS volumes (configurable types: gp3, st1, sc1, encrypted)
+- EBS volumes (configurable types: gp3, st1, sc1; AES-256 encryption at rest enabled)
 - Security Group (SSH + Grafana ports)
 - Elastic IP (optional)
 
@@ -223,6 +223,37 @@ When enabled, all VictoriaMetrics API endpoints require authentication. The heal
 
 Grafana, the Flask backend, and the Reporter are automatically configured to use the credentials.
 
+
+### Encryption at rest
+
+All monitoring data is encrypted at rest using AES-256:
+
+- **EBS root volume** (30 GiB): `encrypted = true` in `main.tf`
+- **EBS data volume** (configurable size): `encrypted = true` in `main.tf`
+- **Encryption key**: AWS-managed `aws/ebs` key by default. To use a custom KMS key, set `encryption_kms_key_arn` in `terraform.tfvars`:
+  ```hcl
+  encryption_kms_key_arn = "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"
+  ```
+  > **WARNING — data loss risk:** Changing `encryption_kms_key_arn` in **any direction** (empty → ARN, ARN → empty, or ARN → different ARN) on an existing deployment will cause Terraform to **destroy and recreate** both the EBS data volume and the EC2 instance (`kms_key_id` is a `ForceNew` attribute). Take a snapshot of **both** the data volume and the root volume before applying this change to a live environment, then restore from snapshots after Terraform completes.
+
+Postgres (sink-postgres) and VictoriaMetrics store data on the encrypted EBS data volume mounted at `/data`. No application-level encryption is needed — the storage layer handles it transparently.
+
+To verify encryption on a running instance:
+
+```bash
+# Data volume
+aws ec2 describe-volumes \
+  --volume-ids $(terraform output -raw data_volume_id) \
+  --query 'Volumes[0].{Encrypted:Encrypted,KmsKeyId:KmsKeyId}'
+
+# Root volume
+aws ec2 describe-volumes \
+  --volume-ids $(terraform output -raw root_volume_id) \
+  --query 'Volumes[0].{Encrypted:Encrypted,KmsKeyId:KmsKeyId}'
+```
+
+**Note:** This covers encryption at rest only. For encryption in transit, configure TLS for Postgres connections and HTTPS for Grafana endpoints.
+
 ### Recommendations
 
 1. **Most secure setup (SSH tunnel only)**:
@@ -434,6 +465,29 @@ For production deployments:
 - Configure monitoring instances manually after deployment (Method 2)
 - Store state in private repositories only
 
+#### Remote state with encryption (recommended for production)
+
+Add a backend block inside `terraform {}` in `main.tf`:
+
+```hcl
+backend "s3" {
+  bucket         = "your-terraform-state-bucket"
+  key            = "postgres-ai-monitoring/terraform.tfstate"
+  region         = "us-east-1"
+  encrypt        = true        # AES-256 encryption at rest (SSE-S3)
+  dynamodb_table = "terraform-locks"  # State locking (Terraform < 1.10)
+  # use_lockfile = true               # Native S3 locking (Terraform >= 1.10, replaces dynamodb_table)
+}
+```
+
+This ensures the state file (which contains credentials) is encrypted at rest in S3.
+
+**Important:** The S3 bucket should have versioning enabled, public access blocked, and a bucket policy enforcing TLS (`aws:SecureTransport`). For SSE-KMS instead of SSE-S3, add `kms_key_id` to the backend configuration.
+
+**Note:** For existing deployments with local state, run `terraform init -migrate-state` after adding the backend block.
+
+**Note:** The application (Postgres, VictoriaMetrics) does not use S3 buckets for data storage — all data resides on the encrypted EBS data volume at `/data`.
+
 ### IMDSv2
 
 EC2 instances are configured to require IMDSv2 (Instance Metadata Service v2) for enhanced security against SSRF attacks.

terraform/aws/main.tf

@@ -145,6 +145,7 @@ resource "aws_ebs_volume" "data" {
   size              = var.data_volume_size
   type              = var.data_volume_type
   encrypted         = true
+  kms_key_id        = var.encryption_kms_key_arn != "" ? var.encryption_kms_key_arn : null
 
   tags = {
     Name = "${var.environment}-postgres-ai-data"
@@ -165,6 +166,7 @@ resource "aws_instance" "main" {
     volume_size = 30
     volume_type = var.root_volume_type
     encrypted   = true
+    kms_key_id  = var.encryption_kms_key_arn != "" ? var.encryption_kms_key_arn : null
   }
 
   user_data = templatefile("${path.module}/user_data.sh", {

terraform/aws/outputs.tf

@@ -8,6 +8,11 @@ output "data_volume_id" {
   value       = aws_ebs_volume.data.id
 }
 
+output "root_volume_id" {
+  description = "EBS root volume ID (for snapshots)"
+  value       = one(aws_instance.main.root_block_device).volume_id
+}
+
 output "public_ip" {
   description = "Public IP address"
   value       = var.use_elastic_ip ? aws_eip.main[0].public_ip : aws_instance.main.public_ip

terraform/aws/tests/kms_encryption.tftest.hcl

@@ -0,0 +1,227 @@
+# Tests for encryption_kms_key_arn variable validation and KMS key propagation.
+# Requires Terraform >= 1.7 (mock_provider support).
+# Run with: terraform test -filter=tests/kms_encryption.tftest.hcl
+
+mock_provider "aws" {
+  mock_data "aws_availability_zones" {
+    defaults = {
+      names = ["us-east-1a", "us-east-1b", "us-east-1c"]
+    }
+  }
+
+  mock_data "aws_ami" {
+    defaults = {
+      id = "ami-0123456789abcdef0"
+    }
+  }
+}
+mock_provider "local" {}
+
+# Shared base variables for all runs (required variables with mock values)
+variables {
+  aws_region         = "us-east-1"
+  environment        = "test"
+  instance_type      = "t3.micro"
+  data_volume_size   = 100
+  data_volume_type   = "gp3"
+  root_volume_type   = "gp3"
+  ssh_key_name       = "test-key"
+  allowed_ssh_cidr   = ["10.0.0.0/8"]
+  allowed_cidr_blocks = []
+  use_elastic_ip     = false
+  grafana_password   = "test-password"
+}
+
+# --- Validation: valid inputs must pass ---
+
+run "empty_string_passes_validation" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = ""
+  }
+
+  assert {
+    condition     = aws_ebs_volume.data.encrypted == true
+    error_message = "EBS data volume must be encrypted at rest"
+  }
+
+  # kms_key_id is Computed+Optional in the AWS provider schema, so its planned
+  # value is "known after apply" even when we set it to null in the config.
+  # The null-path cannot be asserted at plan time with a mock provider.
+  # Non-null propagation (ARN → kms_key_id) is verified in
+  # standard_key_arn_passes_validation and kms_key_propagates_to_* runs.
+
+  assert {
+    condition     = one(aws_instance.main.root_block_device).encrypted == true
+    error_message = "EC2 root block device must be encrypted at rest"
+  }
+}
+
+run "standard_key_arn_passes_validation" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"
+  }
+
+  assert {
+    condition     = aws_ebs_volume.data.kms_key_id == "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"
+    error_message = "EBS data volume kms_key_id must equal encryption_kms_key_arn"
+  }
+
+  assert {
+    condition     = one(aws_instance.main.root_block_device).kms_key_id == "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"
+    error_message = "Root block device kms_key_id must equal encryption_kms_key_arn"
+  }
+}
+
+run "uppercase_hex_key_arn_passes_validation" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = "arn:aws:kms:us-east-1:123456789012:key/12345678-ABCD-1234-1234-123456789012"
+  }
+}
+
+run "multi_region_key_arn_passes_validation" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = "arn:aws:kms:us-east-1:123456789012:key/mrk-1234abcd12ab34cd56ef1234567890ab"
+  }
+}
+
+run "alias_arn_passes_validation" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = "arn:aws:kms:us-east-1:123456789012:alias/my-ebs-key"
+  }
+}
+
+run "govcloud_key_arn_passes_validation" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = "arn:aws-us-gov:kms:us-gov-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"
+  }
+}
+
+# --- Validation: invalid inputs must fail ---
+
+run "plain_string_fails_validation" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = "not-an-arn"
+  }
+
+  expect_failures = [var.encryption_kms_key_arn]
+}
+
+run "wrong_service_fails_validation" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = "arn:aws:s3:::my-bucket"
+  }
+
+  expect_failures = [var.encryption_kms_key_arn]
+}
+
+run "truncated_arn_fails_validation" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = "arn:aws:kms:us-east-1:123456789012:key/..."
+  }
+
+  expect_failures = [var.encryption_kms_key_arn]
+}
+
+run "short_account_id_fails_validation" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = "arn:aws:kms:us-east-1:12345678901:key/12345678-1234-1234-1234-123456789012"
+  }
+
+  expect_failures = [var.encryption_kms_key_arn]
+}
+
+run "malformed_key_id_fails_validation" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234"
+  }
+
+  expect_failures = [var.encryption_kms_key_arn]
+}
+
+run "long_account_id_fails_validation" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = "arn:aws:kms:us-east-1:1234567890123:key/12345678-1234-1234-1234-123456789012"
+  }
+
+  expect_failures = [var.encryption_kms_key_arn]
+}
+
+run "alias_arn_with_dots_passes_validation" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = "arn:aws:kms:us-east-1:123456789012:alias/my.project.ebs-key"
+  }
+}
+
+run "short_key_id_fails_validation" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = "arn:aws:kms:us-east-1:123456789012:key/abc"
+  }
+
+  expect_failures = [var.encryption_kms_key_arn]
+}
+
+# --- Propagation: KMS key ARN must be forwarded to EBS resources ---
+
+run "kms_key_propagates_to_ebs_data_volume" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"
+  }
+
+  assert {
+    condition     = aws_ebs_volume.data.kms_key_id == "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"
+    error_message = "EBS data volume kms_key_id must equal encryption_kms_key_arn when non-empty"
+  }
+
+  assert {
+    condition     = aws_ebs_volume.data.encrypted == true
+    error_message = "EBS data volume must be encrypted at rest"
+  }
+}
+
+run "kms_key_propagates_to_root_block_device" {
+  command = plan
+
+  variables {
+    encryption_kms_key_arn = "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"
+  }
+
+  assert {
+    condition     = one(aws_instance.main.root_block_device).kms_key_id == "arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012"
+    error_message = "EC2 root block device kms_key_id must equal encryption_kms_key_arn when non-empty"
+  }
+
+  assert {
+    condition     = one(aws_instance.main.root_block_device).encrypted == true
+    error_message = "EC2 root block device must be encrypted at rest"
+  }
+}

terraform/aws/variables.tf

@@ -85,6 +85,17 @@ variable "postgres_ai_version" {
   default     = "0.10"
 }
 
+variable "encryption_kms_key_arn" {
+  description = "KMS key ARN or alias ARN for EBS encryption. Leave empty to use the default AWS-managed aws/ebs key (AES-256). Accepts key ARNs (arn:aws:kms:REGION:ACCOUNT:key/UUID) and alias ARNs (arn:aws:kms:REGION:ACCOUNT:alias/NAME). WARNING: changing this value in any direction (empty to ARN, ARN to empty, or ARN to different ARN) on existing infrastructure will destroy and recreate EBS volumes and the EC2 instance, causing data loss. Snapshot both volumes before making this change on a live deployment."
+  type        = string
+  default     = ""
+
+  validation {
+    condition     = var.encryption_kms_key_arn == "" || can(regex("^arn:aws(-[a-z-]+)?:kms:[a-z0-9-]+:[0-9]{12}:(key/((mrk-[a-fA-F0-9]{32})|([a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}))|alias/[a-zA-Z0-9/._-]+)$", var.encryption_kms_key_arn))
+    error_message = "Must be a valid KMS key ARN (e.g., 'arn:aws:kms:us-east-1:123456789012:key/12345678-1234-1234-1234-123456789012') or alias ARN (e.g., 'arn:aws:kms:us-east-1:123456789012:alias/my-ebs-key'), or empty string to use the AWS-managed aws/ebs key."
+  }
+}
+
 variable "bind_host" {
   description = "Bind host for internal service ports (127.0.0.1: for localhost only, empty for all interfaces)"
   type        = string