Skip to content

Commit 7b69090

Browse files
claudeclaude
committed
feat: add RDS/Aurora refresh component for DBLab
Add a new component that automates DBLab full refresh using temporary RDS/Aurora database clones created from snapshots. This enables a hassle-free data sync workflow that doesn't impact production. Features: - Create temporary RDS/Aurora clones from latest automated snapshots - Wait for clone availability with proper timeout handling - Trigger DBLab full refresh via API - Poll refresh status until completion - Clean up temporary clones automatically Deployment options: - AWS Lambda with SAM template and EventBridge scheduling - Standalone CLI binary for cron/manual execution - Docker container for containerized environments Includes comprehensive documentation with IAM policy examples and example configuration files.
1 parent 115bf1c commit 7b69090

File tree

13 files changed

+2325
-0
lines changed

13 files changed

+2325
-0
lines changed

‎engine/cmd/rds-refresh/main.go‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
/*
2+
2024 © Postgres.ai
3+
*/
4+
5+
// Package main provides the entry point for the rds-refresh CLI tool.
6+
// This tool automates DBLab full refresh using temporary RDS/Aurora clones.
7+
package main
8+
9+
import (
10+
"context"
11+
"flag"
12+
"fmt"
13+
"os"
14+
"os/signal"
15+
"syscall"
16+
17+
"github.com/aws/aws-lambda-go/lambda"
18+
19+
"gitlab.com/postgres-ai/database-lab/v3/internal/rdsrefresh"
20+
)
21+
22+
var (
23+
version = "dev"
24+
buildTime = "unknown"
25+
)
26+
27+
func main() {
28+
// Check if running in Lambda
29+
if os.Getenv("AWS_LAMBDA_FUNCTION_NAME") != "" {
30+
lambda.Start(rdsrefresh.HandleLambda)
31+
return
32+
}
33+
34+
// CLI mode
35+
configPath := flag.String("config", "", "Path to configuration file")
36+
dryRun := flag.Bool("dry-run", false, "Validate configuration without creating resources")
37+
showVersion := flag.Bool("version", false, "Show version information")
38+
help := flag.Bool("help", false, "Show help")
39+
40+
flag.Usage = printUsage
41+
flag.Parse()
42+
43+
if *help {
44+
printUsage()
45+
os.Exit(0)
46+
}
47+
48+
if *showVersion {
49+
fmt.Printf("rds-refresh version %s (built: %s)\n", version, buildTime)
50+
os.Exit(0)
51+
}
52+
53+
if *configPath == "" {
54+
fmt.Fprintln(os.Stderr, "error: -config flag is required")
55+
printUsage()
56+
os.Exit(1)
57+
}
58+
59+
if err := run(*configPath, *dryRun); err != nil {
60+
fmt.Fprintf(os.Stderr, "error: %v\n", err)
61+
os.Exit(1)
62+
}
63+
}
64+
65+
func run(configPath string, dryRun bool) error {
66+
cfg, err := rdsrefresh.LoadConfig(configPath)
67+
if err != nil {
68+
return fmt.Errorf("failed to load config: %w", err)
69+
}
70+
71+
ctx, cancel := context.WithCancel(context.Background())
72+
defer cancel()
73+
74+
// Handle interrupt signals
75+
sigCh := make(chan os.Signal, 1)
76+
signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
77+
78+
go func() {
79+
sig := <-sigCh
80+
fmt.Printf("\nReceived signal %v, initiating graceful shutdown...\n", sig)
81+
cancel()
82+
}()
83+
84+
logger := &rdsrefresh.DefaultLogger{}
85+
86+
refresher, err := rdsrefresh.NewRefresher(ctx, cfg, logger)
87+
if err != nil {
88+
return fmt.Errorf("failed to initialize refresher: %w", err)
89+
}
90+
91+
if dryRun {
92+
return refresher.DryRun(ctx)
93+
}
94+
95+
result := refresher.Run(ctx)
96+
97+
fmt.Println()
98+
fmt.Println("=== Refresh Summary ===")
99+
fmt.Printf("Success: %v\n", result.Success)
100+
fmt.Printf("Snapshot: %s\n", result.SnapshotID)
101+
fmt.Printf("Clone ID: %s\n", result.CloneID)
102+
fmt.Printf("Duration: %v\n", result.Duration.Round(1e9))
103+
104+
if result.Error != nil {
105+
return result.Error
106+
}
107+
108+
return nil
109+
}
110+
111+
func printUsage() {
112+
fmt.Fprintf(os.Stderr, `rds-refresh - Automate DBLab full refresh using RDS/Aurora snapshots
113+
114+
This tool creates a temporary RDS/Aurora clone from a snapshot, triggers
115+
a DBLab Engine full refresh, and then cleans up the temporary clone.
116+
117+
USAGE:
118+
rds-refresh -config <path> [options]
119+
120+
OPTIONS:
121+
-config <path> Path to YAML configuration file (required)
122+
-dry-run Validate configuration without creating resources
123+
-version Show version information
124+
-help Show this help message
125+
126+
LAMBDA MODE:
127+
When running as an AWS Lambda function (detected via AWS_LAMBDA_FUNCTION_NAME
128+
environment variable), configuration is loaded from environment variables:
129+
130+
Required:
131+
RDS_SOURCE_IDENTIFIER Source RDS instance or Aurora cluster ID
132+
RDS_CLONE_INSTANCE_CLASS Instance class for the clone (e.g., db.t3.medium)
133+
DBLAB_API_ENDPOINT DBLab Engine API endpoint
134+
DBLAB_TOKEN DBLab verification token
135+
AWS_REGION AWS region
136+
137+
Optional:
138+
RDS_SOURCE_TYPE "rds" or "aurora-cluster" (default: rds)
139+
RDS_SNAPSHOT_IDENTIFIER Specific snapshot ID (default: latest)
140+
RDS_CLONE_SUBNET_GROUP DB subnet group name
141+
RDS_CLONE_SECURITY_GROUPS JSON array of security group IDs
142+
RDS_CLONE_PUBLIC "true" to make clone publicly accessible
143+
RDS_CLONE_PARAMETER_GROUP DB parameter group name
144+
RDS_CLONE_ENABLE_IAM_AUTH "true" to enable IAM authentication
145+
RDS_CLONE_STORAGE_TYPE Storage type (gp2, gp3, io1, etc.)
146+
RDS_CLONE_TAGS JSON object of additional tags
147+
DBLAB_INSECURE "true" to skip TLS verification
148+
149+
EXAMPLE CONFIGURATION:
150+
151+
source:
152+
type: rds
153+
identifier: production-db
154+
155+
clone:
156+
instanceClass: db.t3.medium
157+
subnetGroup: default-vpc-subnet
158+
securityGroups:
159+
- sg-12345678
160+
publiclyAccessible: false
161+
enableIAMAuth: true
162+
163+
dblab:
164+
apiEndpoint: https://dblab.example.com:2345
165+
token: ${DBLAB_TOKEN}
166+
pollInterval: 30s
167+
timeout: 4h
168+
169+
aws:
170+
region: us-east-1
171+
172+
For more information, see:
173+
https://postgres.ai/docs/database-lab-engine
174+
175+
`)
176+
}

‎engine/configs/rds-refresh.example.yaml‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
# Example configuration for rds-refresh component
2+
#
3+
# This component automates DBLab full refresh using temporary RDS/Aurora clones.
4+
# Copy this file and customize for your environment.
5+
#
6+
# For Lambda deployment, see deploy/rds-refresh/template.yaml
7+
# For CLI usage: rds-refresh -config rds-refresh.yaml
8+
9+
# Source database configuration
10+
source:
11+
# Type of source database:
12+
# - "rds" for RDS DB instance
13+
# - "aurora-cluster" for Aurora cluster
14+
type: rds
15+
16+
# RDS DB instance identifier or Aurora cluster identifier
17+
identifier: production-db
18+
19+
# Optional: Specific snapshot identifier to use
20+
# If empty, the latest automated snapshot will be used
21+
# snapshotIdentifier: rds:production-db-2024-01-15-02-00
22+
23+
# Temporary clone configuration
24+
clone:
25+
# Instance class for the clone (can be smaller than production)
26+
instanceClass: db.t3.medium
27+
28+
# DB subnet group (must be in a VPC accessible from DBLab Engine)
29+
subnetGroup: default-vpc-subnet
30+
31+
# VPC security groups for the clone
32+
# Must allow inbound connections from DBLab Engine on PostgreSQL port
33+
securityGroups:
34+
- sg-12345678
35+
- sg-87654321
36+
37+
# Whether the clone should be publicly accessible
38+
# Set to false if DBLab is in the same VPC
39+
publiclyAccessible: false
40+
41+
# Enable IAM database authentication (recommended)
42+
enableIAMAuth: true
43+
44+
# Optional: DB parameter group name
45+
# parameterGroup: custom-postgres-params
46+
47+
# Optional: DB option group name (RDS only)
48+
# optionGroup: custom-options
49+
50+
# Optional: Cluster parameter group (Aurora only)
51+
# clusterParameterGroup: aurora-postgres-params
52+
53+
# Optional: Engine version override
54+
# engineVersion: "15.4"
55+
56+
# Optional: Custom port (default: 5432)
57+
# port: 5432
58+
59+
# Optional: Storage type (gp2, gp3, io1, io2)
60+
# storageType: gp3
61+
62+
# Deletion protection (should be false for temporary clones)
63+
deletionProtection: false
64+
65+
# Additional tags for the clone
66+
tags:
67+
Environment: dblab-refresh
68+
Team: platform
69+
CostCenter: engineering
70+
71+
# DBLab Engine configuration
72+
dblab:
73+
# DBLab Engine API endpoint
74+
apiEndpoint: https://dblab.example.com:2345
75+
76+
# Verification token for DBLab API
77+
# Use environment variable expansion for security
78+
token: ${DBLAB_TOKEN}
79+
80+
# Skip TLS certificate verification (not recommended for production)
81+
insecure: false
82+
83+
# How often to poll DBLab status during refresh
84+
pollInterval: 30s
85+
86+
# Maximum time to wait for refresh to complete
87+
timeout: 4h
88+
89+
# AWS configuration
90+
aws:
91+
# AWS region where RDS/Aurora resources are located
92+
region: us-east-1
93+
94+
# Optional: Custom AWS endpoint (for testing with LocalStack)
95+
# endpoint: http://localhost:4566

‎engine/deploy/rds-refresh/Dockerfile‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
# Build stage
2+
FROM golang:1.23-alpine AS builder
3+
4+
RUN apk add --no-cache git ca-certificates
5+
6+
WORKDIR /build
7+
8+
# Copy go mod files first for better caching
9+
COPY engine/go.mod engine/go.sum ./
10+
RUN go mod download
11+
12+
# Copy source code
13+
COPY engine/ ./
14+
15+
# Build the binary
16+
ARG VERSION=dev
17+
ARG BUILD_TIME=unknown
18+
19+
RUN CGO_ENABLED=0 GOOS=linux go build \
20+
-ldflags="-s -w -X main.version=${VERSION} -X main.buildTime=${BUILD_TIME}" \
21+
-o /rds-refresh \
22+
./cmd/rds-refresh
23+
24+
# Runtime stage
25+
FROM alpine:3.19
26+
27+
RUN apk add --no-cache ca-certificates tzdata
28+
29+
# Create non-root user
30+
RUN adduser -D -u 1000 appuser
31+
32+
WORKDIR /app
33+
34+
COPY --from=builder /rds-refresh /usr/local/bin/rds-refresh
35+
36+
USER appuser
37+
38+
ENTRYPOINT ["/usr/local/bin/rds-refresh"]
39+
CMD ["--help"]

0 commit comments

Comments
 (0)