diff --git a/backend/cmd/api/admin_routes_test.go b/backend/cmd/api/admin_routes_test.go index 6896cac68..a28204f5b 100644 --- a/backend/cmd/api/admin_routes_test.go +++ b/backend/cmd/api/admin_routes_test.go @@ -88,7 +88,7 @@ func newAdminRouteTestServer(t *testing.T, signingKey ...byte) http.Handler { ID: uuid.New(), ProjectID: projectID, Platform: "wechat", - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, }).Error) server, err := newServer(serverConfig{ diff --git a/backend/cmd/api/sticky_writer_test.go b/backend/cmd/api/sticky_writer_test.go index 9989570ae..5c315fa54 100644 --- a/backend/cmd/api/sticky_writer_test.go +++ b/backend/cmd/api/sticky_writer_test.go @@ -27,7 +27,7 @@ func TestServerStickyWriterCookieRoutesAdminStatsToWriter(t *testing.T) { writer := testsupport.SetupTestDB() reader := testsupport.SetupTestDB() router := dbrouter.NewRouter(writer, dbrouter.WithReader(reader)) - seedAdminStatsRouteDatabase(t, writer, "writer", 1, models.PublicationStatusPublished) + seedAdminStatsRouteDatabase(t, writer, "writer", 1, models.PublicationStatusSucceeded) seedAdminStatsRouteDatabase(t, reader, "reader", 2, models.PublicationStatusFailed) server, err := newServer(serverConfig{ @@ -65,7 +65,7 @@ func TestServerRejectsForgedStickyWriterCookie(t *testing.T) { writer := testsupport.SetupTestDB() reader := testsupport.SetupTestDB() router := dbrouter.NewRouter(writer, dbrouter.WithReader(reader)) - seedAdminStatsRouteDatabase(t, writer, "writer", 1, models.PublicationStatusPublished) + seedAdminStatsRouteDatabase(t, writer, "writer", 1, models.PublicationStatusSucceeded) seedAdminStatsRouteDatabase(t, reader, "reader", 2, models.PublicationStatusFailed) server, err := newServer(serverConfig{ @@ -102,7 +102,7 @@ func TestServerEventualAdminStatsStillUseReaderWithoutStickyCookie(t *testing.T) writer := testsupport.SetupTestDB() reader := testsupport.SetupTestDB() router := dbrouter.NewRouter(writer, dbrouter.WithReader(reader)) - seedAdminStatsRouteDatabase(t, writer, "writer", 1, models.PublicationStatusPublished) + seedAdminStatsRouteDatabase(t, writer, "writer", 1, models.PublicationStatusSucceeded) seedAdminStatsRouteDatabase(t, reader, "reader", 2, models.PublicationStatusFailed) server, err := newServer(serverConfig{ diff --git a/backend/internal/db/db.go b/backend/internal/db/db.go index 153cda004..260080393 100644 --- a/backend/internal/db/db.go +++ b/backend/internal/db/db.go @@ -10,10 +10,8 @@ import ( "strings" "time" - "github.com/google/uuid" "gorm.io/driver/postgres" "gorm.io/gorm" - "gorm.io/gorm/clause" "github.com/kurodakayn/mpp-backend/internal/models" ) @@ -24,10 +22,7 @@ var DefaultRouter *Router //go:embed seed/seed_data.sql var seedDataSQL string -// Stable app-specific key for the Postgres transaction advisory lock around migrations. -const migrationAdvisoryLockKey = 776770001 -const devFallbackPasswordHash = "$2a$10$JuGX0AMl3DS3eGm/yRvY2OZLm4QuTuoIgRT4ucmVs/BCwoPYARN4C" //nolint:gosec // Development fallback is a bcrypt hash, not a plaintext password. -const disabledPasswordHash = "legacy-password-reset-required" +const schemaAdvisoryLockKey = 776770001 const ( dbMaxOpenConnsEnv = "DB_MAX_OPEN_CONNS" @@ -74,8 +69,8 @@ func InitDB() { DefaultRouter = NewRouter(database) fmt.Println("Database connection established") - if err := migrate(database); err != nil { - log.Fatal("Failed to migrate database:", err) + if err := syncSchema(database); err != nil { + log.Fatal("Failed to initialize database schema:", err) } reader, err := optionalPostgresReadReplicaFromEnv() @@ -331,339 +326,92 @@ func durationFromEnv(name string, fallback time.Duration) (time.Duration, error) return value, nil } -func migrate(database *gorm.DB) error { - return withMigrationLock(database, func(migrationDB *gorm.DB) error { - if err := prepareUserEmailMigration(migrationDB); err != nil { - return err - } - if err := prepareUserPasswordHashMigration(migrationDB); err != nil { - return err - } - if err := preparePlatformAccountWorkspaceMigration(migrationDB); err != nil { - return err - } - if err := prepareRemoteBrowserSessionRuntimeReferenceMigration(migrationDB); err != nil { - return err - } - if err := migrationDB.AutoMigrate( - &models.User{}, - &models.Workspace{}, - &models.WorkspaceMember{}, - &models.WorkspaceInvite{}, - &models.WorkspaceActivity{}, - &models.WorkspaceDashboardStats{}, - &models.Notification{}, - &models.PlatformAccount{}, - &models.PlatformAccountGrant{}, - &models.Project{}, - &models.ContentTemplate{}, - &models.BrandProfile{}, - &models.ProjectCollaborator{}, - &models.ProjectActivity{}, - &models.ProjectComment{}, - &models.ProjectVersion{}, - &models.ProjectShareLink{}, - &models.MediaAsset{}, - &models.MediaAssetUsage{}, - &models.ProjectPlatformPublication{}, - &models.ProjectListSummary{}, - &models.ScheduledPublication{}, - &models.PublishAttempt{}, - &models.RemoteBrowserSession{}, - &models.PublishEvent{}, - &models.AIContextSnapshot{}, - &models.AIGrowthOptimizationRun{}, - &models.AIProposal{}, - &models.AIDraftingSession{}, - &models.AIDraftingMessage{}, - &models.AIToolCall{}, - &models.AIDraftingSessionSummary{}, - &models.AISessionEvent{}, - &models.AIUsageRecord{}, - &models.WorkspaceQuotaAggregate{}, - &models.OutboxEvent{}, - &models.CollabDocument{}, - &models.CollabDocumentCollaborator{}, - &models.CollabDocumentState{}, - &models.CollabDocumentUpdateBatch{}, - &models.ExtensionCallbackToken{}, - &models.ExtensionExecutionEvent{}, - ); err != nil { - return err - } - - if err := migratePublicationStatuses(migrationDB); err != nil { - return err - } - - if err := backfillPersonalWorkspaces(migrationDB); err != nil { - return err - } - if err := backfillPlatformAccountWorkspaces(migrationDB); err != nil { - return err - } +func syncSchema(database *gorm.DB) error { + return withSchemaLock(database, syncSchemaUnlocked) +} + +func syncSchemaUnlocked(database *gorm.DB) error { + if err := database.AutoMigrate( + &models.User{}, + &models.Workspace{}, + &models.WorkspaceMember{}, + &models.WorkspaceInvite{}, + &models.WorkspaceActivity{}, + &models.WorkspaceDashboardStats{}, + &models.Notification{}, + &models.PlatformAccount{}, + &models.PlatformAccountGrant{}, + &models.Project{}, + &models.ContentTemplate{}, + &models.BrandProfile{}, + &models.ProjectCollaborator{}, + &models.ProjectActivity{}, + &models.ProjectComment{}, + &models.ProjectVersion{}, + &models.ProjectShareLink{}, + &models.MediaAsset{}, + &models.MediaAssetUsage{}, + &models.ProjectPlatformPublication{}, + &models.ProjectListSummary{}, + &models.ScheduledPublication{}, + &models.PublishAttempt{}, + &models.RemoteBrowserSession{}, + &models.PublishEvent{}, + &models.AIContextSnapshot{}, + &models.AIGrowthOptimizationRun{}, + &models.AIProposal{}, + &models.AIDraftingSession{}, + &models.AIDraftingMessage{}, + &models.AIToolCall{}, + &models.AIDraftingSessionSummary{}, + &models.AISessionEvent{}, + &models.AIUsageRecord{}, + &models.WorkspaceQuotaAggregate{}, + &models.OutboxEvent{}, + &models.CollabDocument{}, + &models.CollabDocumentCollaborator{}, + &models.CollabDocumentState{}, + &models.CollabDocumentUpdateBatch{}, + &models.ExtensionCallbackToken{}, + &models.ExtensionExecutionEvent{}, + ); err != nil { + return err + } - // Redis owns normal active-session locking; this index is the atomic fallback when Redis is disabled. - if err := migrationDB.Exec(` + // Redis owns normal active-session locking; this index is the atomic fallback when Redis is disabled. + if err := database.Exec(` CREATE UNIQUE INDEX IF NOT EXISTS ux_remote_browser_sessions_active_user_platform ON remote_browser_sessions (user_id, platform) WHERE status IN ('pending', 'ready', 'login_detected', 'capturing') `).Error; err != nil { - return err - } - if migrationDB.Name() == "postgres" { - if err := migrationDB.Exec(` - CREATE UNIQUE INDEX IF NOT EXISTS ux_platform_accounts_workspace_platform_remote - ON platform_accounts (workspace_id, platform, platform_user_id) - WHERE platform_user_id IS NOT NULL AND platform_user_id <> '' - `).Error; err != nil { - return err - } - if err := migrationDB.Exec(` - CREATE UNIQUE INDEX IF NOT EXISTS ux_platform_accounts_workspace_platform_display - ON platform_accounts (workspace_id, platform, display_name) - WHERE (platform_user_id IS NULL OR platform_user_id = '') AND display_name IS NOT NULL AND display_name <> '' - `).Error; err != nil { - return err - } - } - return nil - }) -} - -func migratePublicationStatuses(database *gorm.DB) error { - if !database.Migrator().HasTable(&models.ProjectPlatformPublication{}) { - return nil - } - - statusMap := map[string]string{ - "pending": models.PublicationStatusDraft, - "adapted": models.PublicationStatusDraft, - "published": models.PublicationStatusSucceeded, - "disabled": models.PublicationStatusCancelled, - } - for oldStatus, newStatus := range statusMap { - if err := database.Model(&models.ProjectPlatformPublication{}). - Where("status = ?", oldStatus). - Update("status", newStatus).Error; err != nil { - return err - } - } - return nil -} - -func backfillPersonalWorkspaces(database *gorm.DB) error { - var users []models.User - return database.FindInBatches(&users, 100, func(tx *gorm.DB, _ int) error { - for _, user := range users { - workspaceID := models.PersonalWorkspaceID(user.ID) - workspace := models.Workspace{ - ID: workspaceID, - OwnerUserID: user.ID, - Name: models.PersonalWorkspaceName, - Slug: models.PersonalWorkspaceSlug(user.ID), - Status: models.WorkspaceStatusActive, - } - if err := tx.Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "id"}}, - DoNothing: true, - }).Create(&workspace).Error; err != nil { - return err - } - - now := time.Now() - member := models.WorkspaceMember{ - WorkspaceID: workspaceID, - UserID: user.ID, - Role: models.WorkspaceRoleOwner, - JoinedAt: &now, - } - if err := tx.Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "workspace_id"}, {Name: "user_id"}}, - DoNothing: true, - }).Create(&member).Error; err != nil { - return err - } - - if err := tx.Model(&models.Project{}). - Where("user_id = ? AND workspace_id IS NULL", user.ID). - Update("workspace_id", workspaceID).Error; err != nil { - return err - } - } - return nil - }).Error -} - -func preparePlatformAccountWorkspaceMigration(database *gorm.DB) error { - if database.Name() != "postgres" { - return nil - } - if !database.Migrator().HasTable(&models.PlatformAccount{}) { - return nil - } - return database.Exec(`DROP INDEX IF EXISTS idx_platform_accounts_user_platform`).Error -} - -func prepareRemoteBrowserSessionRuntimeReferenceMigration(database *gorm.DB) error { - if database.Name() != "postgres" { - return nil - } - if !database.Migrator().HasTable(&models.RemoteBrowserSession{}) { - return nil - } - if !database.Migrator().HasColumn(&models.RemoteBrowserSession{}, "runtime_reference") { - if err := database.Exec(`ALTER TABLE remote_browser_sessions ADD COLUMN runtime_reference jsonb NOT NULL DEFAULT '{}'::jsonb`).Error; err != nil { - return err - } + return err } - if database.Migrator().HasColumn(&models.RemoteBrowserSession{}, "container_id") { + if database.Name() == "postgres" { if err := database.Exec(` - UPDATE remote_browser_sessions - SET runtime_reference = jsonb_build_object( - 'driver', 'docker', - 'runtime_id', container_id, - 'cdp_endpoint', jsonb_build_object('host', '', 'port', 0), - 'stream_endpoint', jsonb_build_object('host', '', 'port', 0), - 'cleanup_labels', '{}'::jsonb - ) - WHERE container_id <> '' - AND (runtime_reference IS NULL OR runtime_reference = '{}'::jsonb) + CREATE UNIQUE INDEX IF NOT EXISTS ux_platform_accounts_workspace_platform_remote + ON platform_accounts (workspace_id, platform, platform_user_id) + WHERE platform_user_id IS NOT NULL AND platform_user_id <> '' `).Error; err != nil { return err } - if err := database.Exec(`ALTER TABLE remote_browser_sessions DROP COLUMN IF EXISTS container_id`).Error; err != nil { - return err - } - } - if database.Migrator().HasColumn(&models.RemoteBrowserSession{}, "cdp_endpoint_ref") { - if err := database.Exec(`ALTER TABLE remote_browser_sessions DROP COLUMN IF EXISTS cdp_endpoint_ref`).Error; err != nil { + if err := database.Exec(` + CREATE UNIQUE INDEX IF NOT EXISTS ux_platform_accounts_workspace_platform_display + ON platform_accounts (workspace_id, platform, display_name) + WHERE (platform_user_id IS NULL OR platform_user_id = '') AND display_name IS NOT NULL AND display_name <> '' + `).Error; err != nil { return err } } return nil } -func backfillPlatformAccountWorkspaces(database *gorm.DB) error { - if !database.Migrator().HasTable(&models.PlatformAccount{}) { - return nil - } - var accounts []models.PlatformAccount - return database.FindInBatches(&accounts, 100, func(tx *gorm.DB, _ int) error { - for _, account := range accounts { - updates := map[string]any{} - if account.WorkspaceID == nil || *account.WorkspaceID == uuid.Nil { - workspaceID := models.PersonalWorkspaceID(account.UserID) - updates["workspace_id"] = workspaceID - } - if account.OwnerUserID == nil { - updates["owner_user_id"] = account.UserID - } - if account.ConnectedByUserID == nil { - updates["connected_by_user_id"] = account.UserID - } - if strings.TrimSpace(account.DisplayName) == "" { - displayName := account.Username - if strings.TrimSpace(displayName) == "" { - displayName = account.Platform - } - updates["display_name"] = displayName - } - if strings.TrimSpace(account.ShareScope) == "" { - updates["share_scope"] = models.PlatformAccountSharePrivate - } - if strings.TrimSpace(account.HealthStatus) == "" { - updates["health_status"] = healthStatusForPlatformAccountStatus(account.Status) - } - if strings.TrimSpace(account.CredentialSecretRef) == "" { - updates["credential_secret_ref"] = "platform-account:" + account.ID.String() - } - if len(updates) > 0 { - if err := tx.Model(&models.PlatformAccount{}).Where("id = ?", account.ID).Updates(updates).Error; err != nil { - return err - } - } - } - return nil - }).Error -} - -func healthStatusForPlatformAccountStatus(status string) string { - switch status { - case models.PlatformAccountStatusConnected: - return models.PlatformAccountHealthHealthy - case models.PlatformAccountStatusFailed: - return models.PlatformAccountHealthFailed - case models.PlatformAccountStatusNeedsReauth: - return models.PlatformAccountHealthNeedsReauth - default: - return models.PlatformAccountHealthUnknown - } -} - -func prepareUserEmailMigration(database *gorm.DB) error { - if database.Name() != "postgres" { - return nil - } - if !database.Migrator().HasTable(&models.User{}) { - return nil - } - - if !database.Migrator().HasColumn(&models.User{}, "email") { - if err := database.Exec(`ALTER TABLE users ADD COLUMN email text`).Error; err != nil { - return err - } - } - - if err := database.Exec(` - UPDATE users - SET email = username || '-' || substring(id::text, 1, 8) || '@local.invalid' - WHERE email IS NULL OR email = '' - `).Error; err != nil { - return err - } - if err := database.Exec(`ALTER TABLE users ALTER COLUMN email SET NOT NULL`).Error; err != nil { - return err - } - return database.Exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_users_email ON users (email)`).Error -} - -func prepareUserPasswordHashMigration(database *gorm.DB) error { - if database.Name() != "postgres" { - return nil - } - if !database.Migrator().HasTable(&models.User{}) { - return nil - } - - if !database.Migrator().HasColumn(&models.User{}, "password_hash") { - if err := database.Exec(`ALTER TABLE users ADD COLUMN password_hash text`).Error; err != nil { - return err - } - } - - passwordHash := disabledPasswordHash - if devSeedEnabled() { - passwordHash = devFallbackPasswordHash - } - - if err := database.Exec(` - UPDATE users - SET password_hash = ? - WHERE password_hash IS NULL OR password_hash = '' - `, passwordHash).Error; err != nil { - return err - } - return database.Exec(`ALTER TABLE users ALTER COLUMN password_hash SET NOT NULL`).Error -} - -func withMigrationLock(database *gorm.DB, run func(*gorm.DB) error) error { +func withSchemaLock(database *gorm.DB, run func(*gorm.DB) error) error { if database.Name() != "postgres" { return run(database) } return database.Transaction(func(tx *gorm.DB) error { - if err := tx.Exec("SELECT pg_advisory_xact_lock(?)", migrationAdvisoryLockKey).Error; err != nil { + if err := tx.Exec("SELECT pg_advisory_xact_lock(?)", schemaAdvisoryLockKey).Error; err != nil { return err } return run(tx) diff --git a/backend/internal/db/db_test.go b/backend/internal/db/db_test.go index 415ccb276..1e3163609 100644 --- a/backend/internal/db/db_test.go +++ b/backend/internal/db/db_test.go @@ -22,12 +22,12 @@ func (r *recordingQueryObserver) ObserveQuery(_ context.Context, observation Que r.observations = append(r.observations, observation) } -func TestMigrateKeepsActiveBrowserSessionUniquenessFallback(t *testing.T) { +func TestSyncSchemaKeepsActiveBrowserSessionUniquenessFallback(t *testing.T) { database, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{ Logger: logger.Default.LogMode(logger.Silent), }) require.NoError(t, err) - require.NoError(t, migrate(database)) + require.NoError(t, syncSchema(database)) userID := uuid.New() now := time.Now() @@ -62,19 +62,19 @@ func TestMigrateKeepsActiveBrowserSessionUniquenessFallback(t *testing.T) { require.NoError(t, database.Create(&expiredSession).Error) } -func TestMigrateAddsProjectCollabDocumentLink(t *testing.T) { +func TestSyncSchemaAddsProjectCollabDocumentLink(t *testing.T) { database, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) require.NoError(t, err) - require.NoError(t, migrate(database)) + require.NoError(t, syncSchema(database)) require.True(t, database.Migrator().HasColumn(&models.Project{}, "collab_document_id")) require.True(t, database.Migrator().HasIndex(&models.Project{}, "ux_projects_collab_document")) } -func TestMigrateAddsWorkspaceTeamModel(t *testing.T) { +func TestSyncSchemaAddsWorkspaceTeamModel(t *testing.T) { database, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) require.NoError(t, err) - require.NoError(t, migrate(database)) + require.NoError(t, syncSchema(database)) require.True(t, database.Migrator().HasTable(&models.Workspace{})) require.True(t, database.Migrator().HasTable(&models.WorkspaceMember{})) @@ -120,10 +120,10 @@ func TestMigrateAddsWorkspaceTeamModel(t *testing.T) { require.Equal(t, workspace.Name, loadedProject.Workspace.Name) } -func TestMigrateAddsArchiveScanIndexes(t *testing.T) { +func TestSyncSchemaAddsArchiveScanIndexes(t *testing.T) { database, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) require.NoError(t, err) - require.NoError(t, migrate(database)) + require.NoError(t, syncSchema(database)) require.True(t, database.Migrator().HasIndex(&models.PublishEvent{}, "idx_publish_events_archive_created_id")) require.True(t, database.Migrator().HasIndex(&models.ExtensionExecutionEvent{}, "idx_extension_execution_events_archive_created_id")) @@ -132,103 +132,6 @@ func TestMigrateAddsArchiveScanIndexes(t *testing.T) { require.True(t, database.Migrator().HasIndex(&models.RemoteBrowserSession{}, "idx_remote_browser_sessions_archive_status_created_id")) } -func TestMigrateBackfillsPersonalWorkspaces(t *testing.T) { - database, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) - require.NoError(t, err) - - require.NoError(t, database.Exec(`CREATE TABLE users ( - id TEXT PRIMARY KEY, - username TEXT NOT NULL UNIQUE, - email TEXT NOT NULL UNIQUE, - is_email_verified BOOLEAN NOT NULL DEFAULT 0, - password_hash TEXT NOT NULL, - role TEXT NOT NULL DEFAULT 'user', - created_at DATETIME, - updated_at DATETIME - )`).Error) - require.NoError(t, database.Exec(`CREATE TABLE projects ( - id TEXT PRIMARY KEY, - user_id TEXT NOT NULL, - collab_document_id TEXT UNIQUE, - title TEXT NOT NULL, - source_content TEXT NOT NULL, - status TEXT NOT NULL, - created_at DATETIME, - updated_at DATETIME - )`).Error) - - ownerID := uuid.New() - emptyUserID := uuid.New() - projectID := uuid.New() - require.NoError(t, database.Exec( - `INSERT INTO users (id, username, email, password_hash, role) VALUES (?, ?, ?, ?, ?)`, - ownerID.String(), - "owner", - "owner@example.com", - "hash", - "user", - ).Error) - require.NoError(t, database.Exec( - `INSERT INTO users (id, username, email, password_hash, role) VALUES (?, ?, ?, ?, ?)`, - emptyUserID.String(), - "empty-user", - "empty-user@example.com", - "hash", - "user", - ).Error) - require.NoError(t, database.Exec( - `INSERT INTO projects (id, user_id, title, source_content, status) VALUES (?, ?, ?, ?, ?)`, - projectID.String(), - ownerID.String(), - "Legacy project", - "content", - models.ProjectStatusDraft, - ).Error) - - require.NoError(t, migrate(database)) - - ownerWorkspaceID := models.PersonalWorkspaceID(ownerID) - emptyUserWorkspaceID := models.PersonalWorkspaceID(emptyUserID) - - var workspaceCount int64 - require.NoError(t, database.Model(&models.Workspace{}).Count(&workspaceCount).Error) - require.Equal(t, int64(2), workspaceCount) - - var ownerWorkspace models.Workspace - require.NoError(t, database.First(&ownerWorkspace, "id = ?", ownerWorkspaceID).Error) - require.Equal(t, ownerID, ownerWorkspace.OwnerUserID) - require.Equal(t, models.PersonalWorkspaceName, ownerWorkspace.Name) - require.Equal(t, models.PersonalWorkspaceSlug(ownerID), ownerWorkspace.Slug) - require.Equal(t, models.WorkspaceStatusActive, ownerWorkspace.Status) - - var emptyUserWorkspace models.Workspace - require.NoError(t, database.First(&emptyUserWorkspace, "id = ?", emptyUserWorkspaceID).Error) - require.Equal(t, emptyUserID, emptyUserWorkspace.OwnerUserID) - - var ownerMembership models.WorkspaceMember - require.NoError(t, database.First(&ownerMembership, "workspace_id = ? AND user_id = ?", ownerWorkspaceID, ownerID).Error) - require.Equal(t, models.WorkspaceRoleOwner, ownerMembership.Role) - require.NotNil(t, ownerMembership.JoinedAt) - - var project models.Project - require.NoError(t, database.First(&project, "id = ?", projectID).Error) - require.NotNil(t, project.WorkspaceID) - require.Equal(t, ownerWorkspaceID, *project.WorkspaceID) - - require.NoError(t, migrate(database)) - require.NoError(t, database.Model(&models.Workspace{}).Count(&workspaceCount).Error) - require.Equal(t, int64(2), workspaceCount) - - var membershipCount int64 - require.NoError(t, database.Model(&models.WorkspaceMember{}).Count(&membershipCount).Error) - require.Equal(t, int64(2), membershipCount) - - var reloadedProject models.Project - require.NoError(t, database.First(&reloadedProject, "id = ?", projectID).Error) - require.NotNil(t, reloadedProject.WorkspaceID) - require.Equal(t, ownerWorkspaceID, *reloadedProject.WorkspaceID) -} - func TestConnectionPoolConfigFromEnvUsesDefaults(t *testing.T) { clearConnectionPoolEnv(t) diff --git a/backend/internal/handlers/dashboard_test.go b/backend/internal/handlers/dashboard_test.go index df154a7ed..674462ca8 100644 --- a/backend/internal/handlers/dashboard_test.go +++ b/backend/internal/handlers/dashboard_test.go @@ -574,7 +574,7 @@ func TestUserDashboardHandlerListExtensionPrepublishReturnsCurrentUserItems(t *t ProjectID: project.ID, Platform: "douyin", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: []byte(`{"text":"douyin preview"}`), }).Error) @@ -622,7 +622,7 @@ func TestUserDashboardHandlerCreateExtensionHandoffReturnsHandoff(t *testing.T) ProjectID: project.ID, Platform: "douyin", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: []byte(`{"format":"text","text":"douyin body"}`), }).Error) @@ -675,7 +675,7 @@ func TestUserDashboardHandlerRecordExtensionEventAcceptsCallbackTokenWithoutUser ProjectID: project.ID, Platform: "douyin", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: []byte(`{"format":"text","text":"douyin body"}`), }).Error) handoff, err := service.CreateExtensionHandoff(user.ID, dto.CreateExtensionHandoffRequest{ @@ -816,7 +816,7 @@ func TestUserDashboardHandlerGetAndUpdateProject(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, }).Error) getContext, getRecorder := newHandlerTestContext(e, http.MethodGet, "/api/user/dashboard/projects/"+project.ID.String()) @@ -1434,7 +1434,7 @@ func TestUserDashboardHandlerSaveProjectContentPreservesPrepublishDraft(t *testi ProjectID: project.ID, Platform: "zhihu", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: []byte(`{"format":"markdown","markdown":"AI draft"}`), }).Error) @@ -1460,7 +1460,7 @@ func TestUserDashboardHandlerSaveProjectContentPreservesPrepublishDraft(t *testi var publication models.ProjectPlatformPublication require.NoError(t, db.First(&publication, "project_id = ? AND platform = ?", project.ID, "zhihu").Error) - require.Equal(t, models.PublicationStatusAdapted, publication.Status) + require.Equal(t, models.PublicationStatusDraft, publication.Status) require.JSONEq(t, `{"format":"markdown","markdown":"AI draft"}`, string(publication.AdaptedContent)) } @@ -1483,14 +1483,14 @@ func TestUserDashboardHandlerSaveProjectPlatformsPreservesSelectedDrafts(t *test ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: []byte(`{"format":"html","html":"Wechat draft"}`), }).Error) require.NoError(t, db.Create(&models.ProjectPlatformPublication{ ProjectID: project.ID, Platform: "zhihu", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: []byte(`{"format":"markdown","markdown":"Zhihu AI draft"}`), }).Error) @@ -1512,12 +1512,12 @@ func TestUserDashboardHandlerSaveProjectPlatformsPreservesSelectedDrafts(t *test var wechat models.ProjectPlatformPublication require.NoError(t, db.First(&wechat, "project_id = ? AND platform = ?", project.ID, "wechat").Error) require.False(t, wechat.Enabled) - require.Equal(t, models.PublicationStatusDisabled, wechat.Status) + require.Equal(t, models.PublicationStatusCancelled, wechat.Status) var zhihu models.ProjectPlatformPublication require.NoError(t, db.First(&zhihu, "project_id = ? AND platform = ?", project.ID, "zhihu").Error) require.True(t, zhihu.Enabled) - require.Equal(t, models.PublicationStatusAdapted, zhihu.Status) + require.Equal(t, models.PublicationStatusDraft, zhihu.Status) require.JSONEq(t, `{"format":"markdown","markdown":"Zhihu AI draft"}`, string(zhihu.AdaptedContent)) } @@ -1573,7 +1573,7 @@ func TestUserDashboardHandlerSyncProjectPrepublish(t *testing.T) { ProjectID: project.ID, Platform: "zhihu", Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, Config: []byte(`{"title":"Sync title"}`), }).Error) @@ -1597,7 +1597,7 @@ func TestUserDashboardHandlerSyncProjectPrepublish(t *testing.T) { require.Equal(t, project.ID, resp.ProjectID) require.Len(t, resp.Items, 1) require.Equal(t, "zhihu", resp.Items[0].Platform) - require.Equal(t, models.PublicationStatusAdapted, resp.Items[0].Status) + require.Equal(t, models.PublicationStatusDraft, resp.Items[0].Status) require.Equal(t, "markdown", resp.Items[0].AdaptedContent["format"]) require.Contains(t, resp.Items[0].AdaptedContent["markdown"], "**sync**") } @@ -1671,7 +1671,7 @@ func TestUserDashboardHandlerUpdateProjectPrepublishDraft(t *testing.T) { ProjectID: project.ID, Platform: "zhihu", Enabled: true, - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, AdaptedContent: []byte(`{"format":"markdown","markdown":"# Old"}`), RemoteID: "remote-id", PublishURL: "https://example.com/post", @@ -1696,7 +1696,7 @@ func TestUserDashboardHandlerUpdateProjectPrepublishDraft(t *testing.T) { var resp dto.ProjectPublicationsResponse require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) require.Len(t, resp.Items, 1) - require.Equal(t, models.PublicationStatusAdapted, resp.Items[0].Status) + require.Equal(t, models.PublicationStatusDraft, resp.Items[0].Status) require.Equal(t, "## Updated", resp.Items[0].AdaptedContent["markdown"]) require.Empty(t, resp.Items[0].PublishURL) require.Empty(t, resp.Items[0].RemoteID) @@ -1930,7 +1930,7 @@ func TestUserDashboardHandlerPublishProjectRejectsDisabledPublication(t *testing ProjectID: project.ID, Platform: "wechat", Enabled: false, - Status: models.PublicationStatusDisabled, + Status: models.PublicationStatusCancelled, }).Error) req := httptest.NewRequest( @@ -1972,7 +1972,7 @@ func TestUserDashboardHandlerCreatesXManualPublishIntent(t *testing.T) { ProjectID: project.ID, Platform: "x", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: []byte(`{"text":"manual x post"}`), }).Error) diff --git a/backend/internal/models/models.go b/backend/internal/models/models.go index ceed530bb..65c41d9f4 100644 --- a/backend/internal/models/models.go +++ b/backend/internal/models/models.go @@ -28,13 +28,6 @@ const ( PublicationStatusSucceeded = string(contracts.PublicationStatusSucceeded) PublicationStatusFailed = string(contracts.PublicationStatusFailed) PublicationStatusCancelled = string(contracts.PublicationStatusCancelled) - - // Deprecated compatibility aliases. New code should use draft/syncing/queued/ - // publishing/succeeded/failed/cancelled names. - PublicationStatusPending = PublicationStatusDraft - PublicationStatusAdapted = PublicationStatusDraft - PublicationStatusPublished = PublicationStatusSucceeded - PublicationStatusDisabled = PublicationStatusCancelled ) // Platform account status constants diff --git a/backend/internal/redisclient/redisclient.go b/backend/internal/redisclient/redisclient.go index df8a9f478..1d2a73c8b 100644 --- a/backend/internal/redisclient/redisclient.go +++ b/backend/internal/redisclient/redisclient.go @@ -23,6 +23,8 @@ const ( tlsEnv = "REDIS_TLS" tlsCACertEnv = "REDIS_TLS_CA_CERT" tlsCAFileEnv = "REDIS_TLS_CA_FILE" + tlsCertFileEnv = "REDIS_TLS_CERT_FILE" + tlsKeyFileEnv = "REDIS_TLS_KEY_FILE" tlsServerNameEnv = "REDIS_TLS_SERVER_NAME" sentinelAddrsEnv = "REDIS_SENTINEL_ADDRS" sentinelMasterEnv = "REDIS_SENTINEL_MASTER_NAME" @@ -56,6 +58,8 @@ type Config struct { TLS bool TLSCACert string TLSCAFile string + TLSCertFile string + TLSKeyFile string TLSServerName string ClusterAddrs []string SentinelAddrs []string @@ -156,6 +160,8 @@ func ConfigFromEnv() (Config, error) { TLS: envFlagEnabled(tlsEnv), TLSCACert: strings.TrimSpace(os.Getenv(tlsCACertEnv)), TLSCAFile: strings.TrimSpace(os.Getenv(tlsCAFileEnv)), + TLSCertFile: strings.TrimSpace(os.Getenv(tlsCertFileEnv)), + TLSKeyFile: strings.TrimSpace(os.Getenv(tlsKeyFileEnv)), TLSServerName: strings.TrimSpace(os.Getenv(tlsServerNameEnv)), } switch endpointMode { @@ -644,9 +650,30 @@ func redisTLSConfig(config Config) (*tls.Config, error) { } else { tlsConfig.RootCAs = certPool } + clientCertificate, ok, err := redisTLSClientCertificate(config.TLSCertFile, config.TLSKeyFile) + if err != nil { + return nil, err + } + if ok { + tlsConfig.Certificates = []tls.Certificate{clientCertificate} + } return tlsConfig, nil } +func redisTLSClientCertificate(certFile string, keyFile string) (tls.Certificate, bool, error) { + if certFile == "" && keyFile == "" { + return tls.Certificate{}, false, nil + } + if certFile == "" || keyFile == "" { + return tls.Certificate{}, false, fmt.Errorf("%s and %s must be set together", tlsCertFileEnv, tlsKeyFileEnv) + } + certificate, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return tls.Certificate{}, false, fmt.Errorf("failed to load Redis TLS client certificate from %s and %s: %w", tlsCertFileEnv, tlsKeyFileEnv, err) + } + return certificate, true, nil +} + func redisTLSRootCAs(inlineCert string, certFile string) (*x509.CertPool, error) { if inlineCert == "" && certFile == "" { return nil, errRedisTLSRootCAsNotConfigured @@ -680,6 +707,9 @@ func (c Config) tlsConfig() *tls.Config { ServerName: c.TLSServerName, } tlsConfig.RootCAs, _ = redisTLSRootCAs(c.TLSCACert, c.TLSCAFile) + if clientCertificate, ok, err := redisTLSClientCertificate(c.TLSCertFile, c.TLSKeyFile); err == nil && ok { + tlsConfig.Certificates = []tls.Certificate{clientCertificate} + } return tlsConfig } return c.tls.Clone() diff --git a/backend/internal/redisclient/redisclient_test.go b/backend/internal/redisclient/redisclient_test.go index 408f62055..58d0e1a20 100644 --- a/backend/internal/redisclient/redisclient_test.go +++ b/backend/internal/redisclient/redisclient_test.go @@ -2,8 +2,15 @@ package redisclient import ( "context" + "crypto/rand" + "crypto/rsa" "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" "os" + "path/filepath" "testing" "time" @@ -225,6 +232,22 @@ func TestConfigFromEnvBuildsTLSOptionsFromCAFile(t *testing.T) { require.NotNil(t, options.TLSConfig.RootCAs) } +func TestConfigFromEnvBuildsTLSOptionsWithClientCertificate(t *testing.T) { + clearRedisEnv(t) + certFile, keyFile := writeTestRedisClientCertificate(t) + t.Setenv(addrEnv, "redis.example.invalid:6379") + t.Setenv(tlsEnv, "true") + t.Setenv(tlsCertFileEnv, certFile) + t.Setenv(tlsKeyFileEnv, keyFile) + + config, err := ConfigFromEnv() + require.NoError(t, err) + + options := options(config, RoleDefault) + require.NotNil(t, options.TLSConfig) + require.Len(t, options.TLSConfig.Certificates, 1) +} + func TestConfigFromEnvRejectsInvalidTLSCA(t *testing.T) { tests := []struct { name string @@ -256,6 +279,33 @@ func TestConfigFromEnvRejectsInvalidTLSCA(t *testing.T) { } } +func TestConfigFromEnvRejectsIncompleteTLSClientCertificate(t *testing.T) { + tests := []struct { + name string + env map[string]string + }{ + {name: "missing key", env: map[string]string{tlsCertFileEnv: "/tmp/redis-client.crt"}}, + {name: "missing cert", env: map[string]string{tlsKeyFileEnv: "/tmp/redis-client.key"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clearRedisEnv(t) + t.Setenv(addrEnv, "redis.example.invalid:6379") + t.Setenv(tlsEnv, "true") + for key, value := range tt.env { + t.Setenv(key, value) + } + + _, err := ConfigFromEnv() + + require.Error(t, err) + require.Contains(t, err.Error(), tlsCertFileEnv) + require.Contains(t, err.Error(), tlsKeyFileEnv) + }) + } +} + func TestRoleSettingsMatchExpectedBaselines(t *testing.T) { tests := []struct { role Role @@ -448,6 +498,8 @@ func clearRedisEnv(t *testing.T) { tlsEnv, tlsCACertEnv, tlsCAFileEnv, + tlsCertFileEnv, + tlsKeyFileEnv, tlsServerNameEnv, sentinelAddrsEnv, sentinelMasterEnv, @@ -461,6 +513,31 @@ func clearRedisEnv(t *testing.T) { } } +func writeTestRedisClientCertificate(t *testing.T) (string, string) { + t.Helper() + + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "redis-client"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + certDER, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey) + require.NoError(t, err) + keyDER := x509.MarshalPKCS1PrivateKey(privateKey) + + dir := t.TempDir() + certFile := filepath.Join(dir, "redis-client.crt") + keyFile := filepath.Join(dir, "redis-client.key") + require.NoError(t, os.WriteFile(certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), 0o600)) + require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: keyDER}), 0o600)) + return certFile, keyFile +} + const testRedisCACertPEM = `-----BEGIN CERTIFICATE----- MIIDEzCCAfugAwIBAgIUb15xgBiiAVRKRFX/A/p9TvypqJwwDQYJKoZIhvcNAQEL BQAwGTEXMBUGA1UEAwwObXBwLXJlZGlzLXRlc3QwHhcNMjYwNjE4MTQyOTQ5WhcN diff --git a/backend/internal/services/dashboard/facade_test.go b/backend/internal/services/dashboard/facade_test.go index 2c5456f30..478fa34e7 100644 --- a/backend/internal/services/dashboard/facade_test.go +++ b/backend/internal/services/dashboard/facade_test.go @@ -24,7 +24,7 @@ func TestDashboardServiceWithContextIsolatesStickyWriterAcrossConcurrentScopes(t limitDashboardFacadeDBConnections(t, writer) limitDashboardFacadeDBConnections(t, reader) router := dbrouter.NewRouter(writer, dbrouter.WithReader(reader)) - seedDashboardFacadeStats(t, writer, "writer", 1, models.PublicationStatusPublished) + seedDashboardFacadeStats(t, writer, "writer", 1, models.PublicationStatusSucceeded) seedDashboardFacadeStats(t, reader, "reader", 2, models.PublicationStatusFailed) service := services.NewDashboardServiceWithRouter(writer, router) diff --git a/backend/internal/services/extension/handoffs.go b/backend/internal/services/extension/handoffs.go index 659cc5627..8f985ef01 100644 --- a/backend/internal/services/extension/handoffs.go +++ b/backend/internal/services/extension/handoffs.go @@ -138,7 +138,7 @@ func (s *Service) CreateExtensionHandoff(userID uuid.UUID, req dto.CreateExtensi } return err } - if !publication.Enabled || publication.Status == models.PublicationStatusDisabled { + if !publication.Enabled || publication.Status == models.PublicationStatusCancelled { return ErrPublicationDisabled } adaptedContent, err := extensionHandoffAdaptedContent(publication.AdaptedContent) @@ -269,7 +269,7 @@ func extensionPublicationUpdatesForEvent(event models.ExtensionExecutionEvent) m switch event.Status { case "user_review": updates := map[string]any{ - "status": models.PublicationStatusAdapted, + "status": models.PublicationStatusDraft, "draft_status": models.PublicationDraftStatusReady, "review_status": models.PublicationReviewStatusReviewing, "error_message": "", diff --git a/backend/internal/services/extension/handoffs_test.go b/backend/internal/services/extension/handoffs_test.go index 407761a99..149ef817a 100644 --- a/backend/internal/services/extension/handoffs_test.go +++ b/backend/internal/services/extension/handoffs_test.go @@ -95,14 +95,14 @@ func TestListExtensionPrepublishReturnsCurrentUserDouyinDrafts(t *testing.T) { ProjectID: olderProject.ID, Platform: "douyin", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: datatypes.JSON(`{"format":"text","text":"` + longText + `"}`), }).Error) disabledPublication := models.ProjectPlatformPublication{ ProjectID: newerProject.ID, Platform: "douyin", Enabled: false, - Status: models.PublicationStatusDisabled, + Status: models.PublicationStatusCancelled, AdaptedContent: datatypes.JSON(`{"format":"text","text":"disabled draft"}`), } require.NoError(t, db.Create(&disabledPublication).Error) @@ -111,14 +111,14 @@ func TestListExtensionPrepublishReturnsCurrentUserDouyinDrafts(t *testing.T) { ProjectID: unsupportedProject.ID, Platform: "zhihu", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: datatypes.JSON(`{"markdown":"zhihu draft"}`), }).Error) require.NoError(t, db.Create(&models.ProjectPlatformPublication{ ProjectID: otherProject.ID, Platform: "douyin", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: datatypes.JSON(`{"text":"other draft"}`), }).Error) @@ -150,7 +150,7 @@ func TestListExtensionPrepublishReturnsXDrafts(t *testing.T) { ProjectID: project.ID, Platform: "x", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: datatypes.JSON(`{"format":"text","text":"x draft"}`), }).Error) @@ -186,7 +186,7 @@ func TestListExtensionPrepublishUsesReader(t *testing.T) { ProjectID: project.ID, Platform: "douyin", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: datatypes.JSON(`{"format":"text","text":"reader draft"}`), }).Error) @@ -214,7 +214,7 @@ func TestCreateExtensionHandoffReturnsDouyinArticleHandoff(t *testing.T) { ProjectID: project.ID, Platform: "douyin", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: datatypes.JSON(`{"schema_version":1,"format":"text","text":"ready text"}`), } require.NoError(t, db.Create(&publication).Error) @@ -272,7 +272,7 @@ func TestCreateExtensionHandoffReturnsXPostHandoff(t *testing.T) { ProjectID: project.ID, Platform: "x", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: datatypes.JSON(`{"schema_version":1,"format":"text","text":"x ready text"}`), }).Error) @@ -315,14 +315,14 @@ func TestCreateExtensionHandoffReturnsMultiplePlatformHandoff(t *testing.T) { ProjectID: project.ID, Platform: "douyin", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: datatypes.JSON(`{"format":"text","text":"douyin ready"}`), }).Error) require.NoError(t, db.Create(&models.ProjectPlatformPublication{ ProjectID: project.ID, Platform: "x", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: datatypes.JSON(`{"format":"text","text":"x ready"}`), }).Error) @@ -362,7 +362,7 @@ func TestRecordExtensionEventAcceptsKnownTokenAndDeduplicatesEventID(t *testing. ProjectID: project.ID, Platform: "douyin", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: datatypes.JSON(`{"format":"text","text":"ready text"}`), }).Error) @@ -420,7 +420,7 @@ func TestRecordExtensionEventMarksXPublicationReadyForUserReview(t *testing.T) { ProjectID: project.ID, Platform: "x", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, DraftStatus: models.PublicationDraftStatusReady, ReviewStatus: models.PublicationReviewStatusDraft, ErrorMessage: "old error", @@ -446,7 +446,7 @@ func TestRecordExtensionEventMarksXPublicationReadyForUserReview(t *testing.T) { assert.False(t, resp.Duplicate) var publication models.ProjectPlatformPublication require.NoError(t, db.First(&publication, "project_id = ? AND platform = ?", project.ID, "x").Error) - assert.Equal(t, models.PublicationStatusAdapted, publication.Status) + assert.Equal(t, models.PublicationStatusDraft, publication.Status) assert.Equal(t, models.PublicationDraftStatusReady, publication.DraftStatus) assert.Equal(t, models.PublicationReviewStatusReviewing, publication.ReviewStatus) assert.Empty(t, publication.ErrorMessage) @@ -469,7 +469,7 @@ func TestRecordExtensionEventMarksXPublicationFailedWithMessage(t *testing.T) { ProjectID: project.ID, Platform: "x", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, DraftStatus: models.PublicationDraftStatusReady, ReviewStatus: models.PublicationReviewStatusDraft, AdaptedContent: datatypes.JSON(`{"format":"text","text":"ready text"}`), @@ -515,7 +515,7 @@ func TestRecordExtensionEventDoesNotApplyDuplicatePublicationUpdate(t *testing.T ProjectID: project.ID, Platform: "x", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: datatypes.JSON(`{"format":"text","text":"ready text"}`), }).Error) @@ -574,7 +574,7 @@ func TestRecordExtensionEventRejectsExpiredToken(t *testing.T) { ProjectID: project.ID, Platform: "douyin", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, AdaptedContent: datatypes.JSON(`{"format":"text","text":"ready text"}`), }).Error) handoff, err := s.CreateExtensionHandoff(user.ID, dto.CreateExtensionHandoffRequest{ @@ -635,7 +635,7 @@ func TestCreateExtensionHandoffRejectsDisabledPublication(t *testing.T) { ProjectID: project.ID, Platform: "douyin", Enabled: false, - Status: models.PublicationStatusDisabled, + Status: models.PublicationStatusCancelled, AdaptedContent: datatypes.JSON(`{"format":"text","text":"ready text"}`), } require.NoError(t, db.Create(&publication).Error) @@ -665,7 +665,7 @@ func TestCreateExtensionHandoffRejectsMissingAdaptedText(t *testing.T) { ProjectID: project.ID, Platform: "douyin", Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, AdaptedContent: datatypes.JSON(`{}`), }).Error) diff --git a/backend/internal/services/prepublish/drafts.go b/backend/internal/services/prepublish/drafts.go index 143f571ce..9c064efa3 100644 --- a/backend/internal/services/prepublish/drafts.go +++ b/backend/internal/services/prepublish/drafts.go @@ -46,7 +46,7 @@ func (s *Service) SyncProjectPrepublish(projectID uuid.UUID, userID uuid.UUID, r } if len(platforms) == 0 { for _, publication := range project.Publications { - if publication.Enabled && publication.Status != models.PublicationStatusDisabled { + if publication.Enabled && publication.Status != models.PublicationStatusCancelled { platforms = append(platforms, publication.Platform) } } @@ -112,7 +112,7 @@ func (s *Service) ensurePrepublishPublications(project *models.Project, platform return err } - if !publication.Enabled || publication.Status == models.PublicationStatusDisabled { + if !publication.Enabled || publication.Status == models.PublicationStatusCancelled { if err := tx.Model(&publication).Updates(map[string]any{ "draft_status": models.PublicationDraftStatusUnsynced, "enabled": true, @@ -250,7 +250,7 @@ func (s *Service) UpdateProjectPrepublishDraft(projectID uuid.UUID, userID uuid. "remote_id": "", "review_status": models.PublicationReviewStatusDraft, "retry_count": 0, - "status": models.PublicationStatusAdapted, + "status": models.PublicationStatusDraft, "sync_required": false, }).Error; err != nil { return nil, err diff --git a/backend/internal/services/prepublish/drafts_test.go b/backend/internal/services/prepublish/drafts_test.go index 4ee0de1d9..a2c7a20ca 100644 --- a/backend/internal/services/prepublish/drafts_test.go +++ b/backend/internal/services/prepublish/drafts_test.go @@ -41,28 +41,28 @@ func TestSyncProjectPrepublishGeneratesPlatformDrafts(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Platform title"}`), }) db.Create(&models.ProjectPlatformPublication{ ProjectID: project.ID, Platform: "zhihu", Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Platform title"}`), }) db.Create(&models.ProjectPlatformPublication{ ProjectID: project.ID, Platform: "x", Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Platform title"}`), }) db.Create(&models.ProjectPlatformPublication{ ProjectID: project.ID, Platform: "douyin", Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Platform title"}`), }) @@ -78,7 +78,7 @@ func TestSyncProjectPrepublishGeneratesPlatformDrafts(t *testing.T) { var wechatPub models.ProjectPlatformPublication require.NoError(t, db.First(&wechatPub, "project_id = ? AND platform = ?", project.ID, "wechat").Error) - assert.Equal(t, models.PublicationStatusAdapted, wechatPub.Status) + assert.Equal(t, models.PublicationStatusDraft, wechatPub.Status) var wechatContent map[string]any require.NoError(t, json.Unmarshal(wechatPub.AdaptedContent, &wechatContent)) @@ -87,7 +87,7 @@ func TestSyncProjectPrepublishGeneratesPlatformDrafts(t *testing.T) { var zhihuPub models.ProjectPlatformPublication require.NoError(t, db.First(&zhihuPub, "project_id = ? AND platform = ?", project.ID, "zhihu").Error) - assert.Equal(t, models.PublicationStatusAdapted, zhihuPub.Status) + assert.Equal(t, models.PublicationStatusDraft, zhihuPub.Status) var zhihuContent map[string]any require.NoError(t, json.Unmarshal(zhihuPub.AdaptedContent, &zhihuContent)) @@ -97,7 +97,7 @@ func TestSyncProjectPrepublishGeneratesPlatformDrafts(t *testing.T) { var xPub models.ProjectPlatformPublication require.NoError(t, db.First(&xPub, "project_id = ? AND platform = ?", project.ID, "x").Error) - assert.Equal(t, models.PublicationStatusAdapted, xPub.Status) + assert.Equal(t, models.PublicationStatusDraft, xPub.Status) var xContent map[string]any require.NoError(t, json.Unmarshal(xPub.AdaptedContent, &xContent)) @@ -107,7 +107,7 @@ func TestSyncProjectPrepublishGeneratesPlatformDrafts(t *testing.T) { var douyinPub models.ProjectPlatformPublication require.NoError(t, db.First(&douyinPub, "project_id = ? AND platform = ?", project.ID, "douyin").Error) - assert.Equal(t, models.PublicationStatusAdapted, douyinPub.Status) + assert.Equal(t, models.PublicationStatusDraft, douyinPub.Status) var douyinContent map[string]any require.NoError(t, json.Unmarshal(douyinPub.AdaptedContent, &douyinContent)) @@ -189,7 +189,7 @@ func TestSyncProjectPrepublishSanitizesHTMLDraftsBeforePersisting(t *testing.T) ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Platform title"}`), }).Error) @@ -249,7 +249,7 @@ func TestSyncProjectPrepublishReadsLatestCollabSnapshot(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Platform title"}`), }).Error) @@ -290,7 +290,7 @@ func TestSyncProjectPrepublishMarksFailedWhenContentPipelineCompilerFails(t *tes ProjectID: project.ID, Platform: "zhihu", Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Platform title"}`), AdaptedContent: datatypes.JSON(`{}`), }).Error) @@ -343,7 +343,7 @@ func TestSyncProjectPrepublishRejectsActivePublishWithoutMarkingSyncing(t *testi ProjectID: project.ID, Platform: "zhihu", Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Platform title"}`), AdaptedContent: datatypes.JSON(`{}`), }).Error) @@ -367,7 +367,7 @@ func TestSyncProjectPrepublishRejectsActivePublishWithoutMarkingSyncing(t *testi var pendingPublication models.ProjectPlatformPublication require.NoError(t, db.First(&pendingPublication, "project_id = ? AND platform = ?", project.ID, "zhihu").Error) - require.Equal(t, models.PublicationStatusPending, pendingPublication.Status) + require.Equal(t, models.PublicationStatusDraft, pendingPublication.Status) } func TestSyncProjectPrepublishInvalidatesCachesAfterCommittedEnsureBeforeActivePublish(t *testing.T) { @@ -395,7 +395,7 @@ func TestSyncProjectPrepublishInvalidatesCachesAfterCommittedEnsureBeforeActiveP ProjectID: project.ID, Platform: "zhihu", Enabled: false, - Status: models.PublicationStatusDisabled, + Status: models.PublicationStatusCancelled, Config: datatypes.JSON(`{"title":"Prepublish active cache"}`), AdaptedContent: datatypes.JSON(`{"summary":"disabled"}`), }).Error) @@ -457,7 +457,7 @@ func TestSyncProjectPrepublishDoesNotApplyDraftWhenPublicationBecomesPublishing( ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Platform title"}`), AdaptedContent: datatypes.JSON(`{"format":"html","html":"old draft"}`), RemoteID: "active-remote", @@ -512,7 +512,7 @@ func TestUpdateProjectPrepublishDraftSanitizesHTMLBeforePersisting(t *testing.T) ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Platform title"}`), AdaptedContent: datatypes.JSON(`{}`), }).Error) diff --git a/backend/internal/services/project/experience_test.go b/backend/internal/services/project/experience_test.go index f04b8dd9e..a01930e45 100644 --- a/backend/internal/services/project/experience_test.go +++ b/backend/internal/services/project/experience_test.go @@ -84,7 +84,7 @@ func TestProjectVersionsRestoreSavedContent(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, DraftStatus: models.PublicationDraftStatusReady, ReviewStatus: models.PublicationReviewStatusApproved, SyncRequired: false, diff --git a/backend/internal/services/project/lifecycle_test.go b/backend/internal/services/project/lifecycle_test.go index 337f6632b..1ffb93e7e 100644 --- a/backend/internal/services/project/lifecycle_test.go +++ b/backend/internal/services/project/lifecycle_test.go @@ -35,7 +35,7 @@ func TestListProjects(t *testing.T) { db.Create(&p2) db.Create(&p3) - db.Create(&models.ProjectPlatformPublication{ProjectID: p1.ID, Platform: "wechat", Status: models.PublicationStatusPublished, PublishURL: "url1"}) + db.Create(&models.ProjectPlatformPublication{ProjectID: p1.ID, Platform: "wechat", Status: models.PublicationStatusSucceeded, PublishURL: "url1"}) // Test global admin pagination res, err := s.ListProjects(1, 10, "", "", "", nil) @@ -218,7 +218,7 @@ func TestListProjectsUsesReaderForAdminList(t *testing.T) { require.NoError(t, reader.Create(&models.ProjectPlatformPublication{ ProjectID: project.ID, Platform: "wechat", - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, }).Error) var writerProjects int64 @@ -303,7 +303,7 @@ func TestGetProjectUsesReaderForAdminDetail(t *testing.T) { require.NoError(t, reader.Create(&models.ProjectPlatformPublication{ ProjectID: project.ID, Platform: "wechat", - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, PublishURL: "https://example.test/reader", }).Error) @@ -408,7 +408,7 @@ func TestCreateProjectCreatesSelectedPublications(t *testing.T) { var wechatPub models.ProjectPlatformPublication require.NoError(t, db.First(&wechatPub, "project_id = ? AND platform = ?", resp.ID, "wechat").Error) - assert.Equal(t, models.PublicationStatusPending, wechatPub.Status) + assert.Equal(t, models.PublicationStatusDraft, wechatPub.Status) var config map[string]string require.NoError(t, json.Unmarshal(wechatPub.Config, &config)) @@ -422,7 +422,7 @@ func TestCreateProjectCreatesSelectedPublications(t *testing.T) { var douyinPub models.ProjectPlatformPublication require.NoError(t, db.First(&douyinPub, "project_id = ? AND platform = ?", resp.ID, "douyin").Error) - assert.Equal(t, models.PublicationStatusPending, douyinPub.Status) + assert.Equal(t, models.PublicationStatusDraft, douyinPub.Status) } func TestCreateProjectAppliesContentTemplateDefaults(t *testing.T) { @@ -591,7 +591,7 @@ func TestGetProjectReturnsSourceContentForOwner(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, }) db.Create(&models.ProjectCollaborator{ ProjectID: project.ID, @@ -638,7 +638,7 @@ func TestUpdateProjectRebuildsSelectedPublications(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, PublishURL: "https://example.com/old", RemoteID: "old-remote", PublishedAt: &publishedAt, @@ -674,12 +674,12 @@ func TestUpdateProjectRebuildsSelectedPublications(t *testing.T) { var wechatPub models.ProjectPlatformPublication require.NoError(t, db.First(&wechatPub, "project_id = ? AND platform = ?", project.ID, "wechat").Error) assert.False(t, wechatPub.Enabled) - assert.Equal(t, models.PublicationStatusDisabled, wechatPub.Status) + assert.Equal(t, models.PublicationStatusCancelled, wechatPub.Status) var zhihuPub models.ProjectPlatformPublication require.NoError(t, db.First(&zhihuPub, "project_id = ? AND platform = ?", project.ID, "zhihu").Error) assert.True(t, zhihuPub.Enabled) - assert.Equal(t, models.PublicationStatusPending, zhihuPub.Status) + assert.Equal(t, models.PublicationStatusDraft, zhihuPub.Status) assert.Empty(t, zhihuPub.ErrorMessage) assert.Empty(t, zhihuPub.PublishURL) assert.Nil(t, zhihuPub.PublishedAt) @@ -687,7 +687,7 @@ func TestUpdateProjectRebuildsSelectedPublications(t *testing.T) { var douyinPub models.ProjectPlatformPublication require.NoError(t, db.First(&douyinPub, "project_id = ? AND platform = ?", project.ID, "douyin").Error) assert.True(t, douyinPub.Enabled) - assert.Equal(t, models.PublicationStatusPending, douyinPub.Status) + assert.Equal(t, models.PublicationStatusDraft, douyinPub.Status) _, err = s.UpdateProject(project.ID, stranger.ID, dto.UpdateProjectRequest{ Title: "Not allowed", @@ -740,7 +740,7 @@ func TestUpdateProjectSyncsLinkedCollabDocumentSnapshot(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, }).Error) updated, err := s.UpdateProject(project.ID, owner.ID, dto.UpdateProjectRequest{ @@ -795,7 +795,7 @@ func TestUpdateProjectPreservesRequestContentForUninitializedLinkedCollabDocumen ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, }).Error) updated, err := s.UpdateProject(project.ID, owner.ID, dto.UpdateProjectRequest{ @@ -837,7 +837,7 @@ func TestUpdateProjectAllowsEditorAndRejectsViewer(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, }).Error) require.NoError(t, db.Create(&models.ProjectCollaborator{ ProjectID: project.ID, @@ -865,12 +865,12 @@ func TestUpdateProjectAllowsEditorAndRejectsViewer(t *testing.T) { var wechatPub models.ProjectPlatformPublication require.NoError(t, db.First(&wechatPub, "project_id = ? AND platform = ?", project.ID, "wechat").Error) require.False(t, wechatPub.Enabled) - require.Equal(t, models.PublicationStatusDisabled, wechatPub.Status) + require.Equal(t, models.PublicationStatusCancelled, wechatPub.Status) var zhihuPub models.ProjectPlatformPublication require.NoError(t, db.First(&zhihuPub, "project_id = ? AND platform = ?", project.ID, "zhihu").Error) require.True(t, zhihuPub.Enabled) - require.Equal(t, models.PublicationStatusPending, zhihuPub.Status) + require.Equal(t, models.PublicationStatusDraft, zhihuPub.Status) _, err = s.UpdateProject(project.ID, viewer.ID, dto.UpdateProjectRequest{ Title: "Viewer title", @@ -1102,7 +1102,7 @@ func TestSaveProjectPlatformsAllowsEditorAndRejectsViewer(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, }).Error) require.NoError(t, db.Create(&models.ProjectCollaborator{ ProjectID: project.ID, @@ -1126,7 +1126,7 @@ func TestSaveProjectPlatformsAllowsEditorAndRejectsViewer(t *testing.T) { var zhihuPub models.ProjectPlatformPublication require.NoError(t, db.First(&zhihuPub, "project_id = ? AND platform = ?", project.ID, "zhihu").Error) require.True(t, zhihuPub.Enabled) - require.Equal(t, models.PublicationStatusPending, zhihuPub.Status) + require.Equal(t, models.PublicationStatusDraft, zhihuPub.Status) _, err = s.SaveProjectPlatforms(project.ID, viewer.ID, dto.SaveProjectPlatformsRequest{ Platforms: []string{"wechat"}, diff --git a/backend/internal/services/project/listing/list_cache_integration_test.go b/backend/internal/services/project/listing/list_cache_integration_test.go index 70cf65b5c..59945c003 100644 --- a/backend/internal/services/project/listing/list_cache_integration_test.go +++ b/backend/internal/services/project/listing/list_cache_integration_test.go @@ -713,7 +713,7 @@ func seedProjectListCacheProject(t *testing.T, db *gorm.DB, user models.User, ti require.NoError(t, db.Create(&models.ProjectPlatformPublication{ ProjectID: project.ID, Platform: platform, - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, }).Error) } return project @@ -747,7 +747,7 @@ func seedWorkspaceProjectListCacheProject(t *testing.T, db *gorm.DB, user models require.NoError(t, db.Create(&models.ProjectPlatformPublication{ ProjectID: project.ID, Platform: platform, - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, }).Error) } return project diff --git a/backend/internal/services/project/publications_test.go b/backend/internal/services/project/publications_test.go index 52aaf5b42..4491fe52c 100644 --- a/backend/internal/services/project/publications_test.go +++ b/backend/internal/services/project/publications_test.go @@ -35,7 +35,7 @@ func TestGetProjectPublications(t *testing.T) { pub := models.ProjectPlatformPublication{ ProjectID: p.ID, Platform: "wechat", - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, Config: datatypes.JSON(configJSON), AdaptedContent: datatypes.JSON(contentJSON), } @@ -79,7 +79,7 @@ func TestGetProjectPublicationsUsesReaderForAdminDetail(t *testing.T) { require.NoError(t, reader.Create(&models.ProjectPlatformPublication{ ProjectID: project.ID, Platform: "wechat", - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, PublishURL: "https://example.test/reader", }).Error) @@ -88,7 +88,7 @@ func TestGetProjectPublicationsUsesReaderForAdminDetail(t *testing.T) { require.Equal(t, project.ID, res.ProjectID) require.Len(t, res.Items, 1) require.Equal(t, "wechat", res.Items[0].Platform) - require.Equal(t, models.PublicationStatusPublished, res.Items[0].Status) + require.Equal(t, models.PublicationStatusSucceeded, res.Items[0].Status) } func TestGetProjectPublicationsUsesWriterForScopedDetail(t *testing.T) { @@ -103,7 +103,7 @@ func TestGetProjectPublicationsUsesWriterForScopedDetail(t *testing.T) { require.NoError(t, writer.Create(&models.ProjectPlatformPublication{ ProjectID: currentProject.ID, Platform: "wechat", - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, }).Error) staleReaderProject := models.Project{ @@ -125,7 +125,7 @@ func TestGetProjectPublicationsUsesWriterForScopedDetail(t *testing.T) { require.Equal(t, currentProject.ID, res.ProjectID) require.Len(t, res.Items, 1) require.Equal(t, "wechat", res.Items[0].Platform) - require.Equal(t, models.PublicationStatusPublished, res.Items[0].Status) + require.Equal(t, models.PublicationStatusSucceeded, res.Items[0].Status) } func TestGetProjectPublicationsUsesWriterForStickyAdminDetail(t *testing.T) { @@ -142,7 +142,7 @@ func TestGetProjectPublicationsUsesWriterForStickyAdminDetail(t *testing.T) { require.NoError(t, writer.Create(&models.ProjectPlatformPublication{ ProjectID: currentProject.ID, Platform: "wechat", - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, }).Error) staleReaderProject := models.Project{ @@ -164,5 +164,5 @@ func TestGetProjectPublicationsUsesWriterForStickyAdminDetail(t *testing.T) { require.Equal(t, currentProject.ID, res.ProjectID) require.Len(t, res.Items, 1) require.Equal(t, "wechat", res.Items[0].Platform) - require.Equal(t, models.PublicationStatusPublished, res.Items[0].Status) + require.Equal(t, models.PublicationStatusSucceeded, res.Items[0].Status) } diff --git a/backend/internal/services/project/publicationselection/selection.go b/backend/internal/services/project/publicationselection/selection.go index 5c6f69296..6e8a0d093 100644 --- a/backend/internal/services/project/publicationselection/selection.go +++ b/backend/internal/services/project/publicationselection/selection.go @@ -52,7 +52,7 @@ func ReconcileSelected(tx *gorm.DB, projectID uuid.UUID, platforms []string, mod continue } - if mode == ReconcileResetAll || !publication.Enabled || publication.Status == models.PublicationStatusDisabled { + if mode == ReconcileResetAll || !publication.Enabled || publication.Status == models.PublicationStatusCancelled { config, err := configForPlatform(publication.Platform) if err != nil { return nil, err @@ -104,7 +104,7 @@ func createPendingPublication(projectID uuid.UUID, platform string, config datat ProjectID: projectID, Platform: platform, Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, SyncRequired: true, Config: config, AdaptedContent: datatypes.JSON([]byte(`{}`)), @@ -115,7 +115,7 @@ func disablePublication(tx *gorm.DB, publication models.ProjectPlatformPublicati return tx.Model(&publication).Updates(map[string]any{ "enabled": false, "error_message": "", - "status": models.PublicationStatusDisabled, + "status": models.PublicationStatusCancelled, }) } @@ -124,7 +124,7 @@ func resetPublicationForDraft(tx *gorm.DB, publication models.ProjectPlatformPub "draft_status": models.PublicationDraftStatusUnsynced, "enabled": true, "review_status": models.PublicationReviewStatusDraft, - "status": models.PublicationStatusPending, + "status": models.PublicationStatusDraft, "sync_required": true, } if mode == ReconcileResetAll { diff --git a/backend/internal/services/publicationpayload/payload.go b/backend/internal/services/publicationpayload/payload.go index c5a457273..3098c0015 100644 --- a/backend/internal/services/publicationpayload/payload.go +++ b/backend/internal/services/publicationpayload/payload.go @@ -16,7 +16,7 @@ func BuildPending(title, summary, coverImageURL string) (datatypes.JSON, datatyp return nil, nil, "", err } - return config, datatypes.JSON([]byte(`{}`)), models.PublicationStatusPending, nil + return config, datatypes.JSON([]byte(`{}`)), models.PublicationStatusDraft, nil } func DefaultConfig(title, summary, coverImageURL string) (datatypes.JSON, error) { diff --git a/backend/internal/services/publish/browser_session.go b/backend/internal/services/publish/browser_session.go index 4d788c14b..545c61d57 100644 --- a/backend/internal/services/publish/browser_session.go +++ b/backend/internal/services/publish/browser_session.go @@ -37,7 +37,7 @@ func (s *Service) StartDouyinPublishSession(ctx context.Context, projectID uuid. } return nil, err } - if !pub.Enabled || pub.Status == models.PublicationStatusDisabled { + if !pub.Enabled || pub.Status == models.PublicationStatusCancelled { return nil, ErrPublicationDisabled } if len(pub.AdaptedContent) == 0 || string(pub.AdaptedContent) == "{}" { diff --git a/backend/internal/services/publish/publication_flow_test.go b/backend/internal/services/publish/publication_flow_test.go index 50aaf58c2..5cafa25df 100644 --- a/backend/internal/services/publish/publication_flow_test.go +++ b/backend/internal/services/publish/publication_flow_test.go @@ -84,13 +84,13 @@ func TestBatchPublishProject(t *testing.T) { db.Create(&models.ProjectPlatformPublication{ ProjectID: p.ID, Platform: "wechat", - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"app_id": "test", "app_secret": "test"}`), }) db.Create(&models.ProjectPlatformPublication{ ProjectID: p.ID, Platform: "zhihu", - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, }) // Test batch publish @@ -125,7 +125,7 @@ func TestPublishProjectUsesSavedWechatCredentials(t *testing.T) { pub := models.ProjectPlatformPublication{ ProjectID: project.ID, Platform: "wechat", - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"app_id":"stale","app_secret":"stale-secret","title":"Title"}`), AdaptedContent: datatypes.JSON(`{"summary":"ready"}`), } @@ -144,7 +144,7 @@ func TestPublishProjectUsesSavedWechatCredentials(t *testing.T) { result, err := s.PublishProject(project.ID, "wechat", &user.ID, uuid.Nil) require.NoError(t, err) - assert.Equal(t, models.PublicationStatusPublished, result.Status) + assert.Equal(t, models.PublicationStatusSucceeded, result.Status) var completedActivity models.ProjectActivity require.NoError(t, db.First( @@ -199,7 +199,7 @@ func TestPublishProjectInvalidatesDashboardCaches(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"app_id":"wx","app_secret":"secret","title":"Title"}`), AdaptedContent: datatypes.JSON(`{"summary":"ready"}`), }).Error) @@ -215,7 +215,7 @@ func TestPublishProjectInvalidatesDashboardCaches(t *testing.T) { resp, err := s.WithContext(context.Background()).PublishProject(project.ID, "wechat", &user.ID, uuid.Nil) require.NoError(t, err) - assert.Equal(t, models.PublicationStatusPublished, resp.Status) + assert.Equal(t, models.PublicationStatusSucceeded, resp.Status) requirePublishCacheKeys(t, redisClient, "mpp:dashboard:projects:list:*", 0) require.Contains(t, requirePublishCacheKeys(t, redisClient, "mpp:dashboard:stats:*", 1), staleStatsKey) @@ -650,7 +650,7 @@ func TestPublishProjectAllowsEmbeddedWechatCredentialsWithoutSavedAccount(t *tes ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"app_id":"wx","app_secret":"secret","title":"Title"}`), AdaptedContent: datatypes.JSON(`{"summary":"ready"}`), } @@ -659,7 +659,7 @@ func TestPublishProjectAllowsEmbeddedWechatCredentialsWithoutSavedAccount(t *tes result, err := s.PublishProject(project.ID, "wechat", &user.ID, uuid.Nil) require.NoError(t, err) - assert.Equal(t, models.PublicationStatusPublished, result.Status) + assert.Equal(t, models.PublicationStatusSucceeded, result.Status) assert.JSONEq(t, `{"app_id":"wx","app_secret":"secret","title":"Title"}`, string(fakePublisher.Config)) } @@ -724,7 +724,7 @@ func TestPublishProjectPresignsReadyMediaRefsWithoutPersistingSignedURLs(t *test ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(config), AdaptedContent: datatypes.JSON(adaptedContent), }).Error) @@ -732,7 +732,7 @@ func TestPublishProjectPresignsReadyMediaRefsWithoutPersistingSignedURLs(t *test result, err := s.PublishProject(project.ID, "wechat", &user.ID, uuid.Nil) require.NoError(t, err) - assert.Equal(t, models.PublicationStatusPublished, result.Status) + assert.Equal(t, models.PublicationStatusSucceeded, result.Status) expectedURL := "fake://get/mpp-media/" + asset.ObjectKey assert.Contains(t, string(fakePublisher.Config), expectedURL) assert.NotContains(t, string(fakePublisher.Config), mediaRef) @@ -803,7 +803,7 @@ func TestPublishProjectValidatesWorkspaceLibraryMediaReadyState(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(config), AdaptedContent: datatypes.JSON(`{"format":"html","html":"ready"}`), }).Error) @@ -819,7 +819,7 @@ func TestPublishProjectValidatesWorkspaceLibraryMediaReadyState(t *testing.T) { result, err = s.PublishProject(project.ID, "wechat", &user.ID, uuid.Nil) require.NoError(t, err) - assert.Equal(t, models.PublicationStatusPublished, result.Status) + assert.Equal(t, models.PublicationStatusSucceeded, result.Status) assert.Contains(t, string(fakePublisher.Config), "fake://get/mpp-media/"+asset.ObjectKey) } @@ -886,7 +886,7 @@ func TestPublishProjectPreservesReadyMediaRefsWhenContentPipelineResolverIsConfi ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(config), AdaptedContent: datatypes.JSON(adaptedContent), }).Error) @@ -894,7 +894,7 @@ func TestPublishProjectPreservesReadyMediaRefsWhenContentPipelineResolverIsConfi result, err := s.PublishProject(project.ID, "wechat", &user.ID, uuid.Nil) require.NoError(t, err) - assert.Equal(t, models.PublicationStatusPublished, result.Status) + assert.Equal(t, models.PublicationStatusSucceeded, result.Status) assert.Contains(t, string(fakePublisher.Config), mediaRef) assert.NotContains(t, string(fakePublisher.Config), "fake://get/") assert.Contains(t, string(fakePublisher.AdaptedContent), mediaRef) @@ -923,7 +923,7 @@ func TestPublishProjectPassesDecryptedBrowserCookiesToPublisher(t *testing.T) { ProjectID: project.ID, Platform: "douyin", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{}`), AdaptedContent: datatypes.JSON(`{"summary":"ready"}`), } @@ -941,7 +941,7 @@ func TestPublishProjectPassesDecryptedBrowserCookiesToPublisher(t *testing.T) { result, err := s.PublishProject(project.ID, "douyin", &user.ID, uuid.Nil) require.NoError(t, err) - assert.Equal(t, models.PublicationStatusPublished, result.Status) + assert.Equal(t, models.PublicationStatusSucceeded, result.Status) assert.Contains(t, string(fakePublisher.AccountCookies), "secret-value") assert.NotContains(t, string(fakePublisher.AccountCookies), "ciphertext") @@ -971,7 +971,7 @@ func TestPublishProjectIgnoresBrowserSessionIDForAsyncPublishing(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Title"}`), AdaptedContent: datatypes.JSON(`{"summary":"ready"}`), } @@ -989,7 +989,7 @@ func TestPublishProjectIgnoresBrowserSessionIDForAsyncPublishing(t *testing.T) { result, err := s.PublishProject(project.ID, "wechat", &user.ID, sessionID) require.NoError(t, err) - assert.Equal(t, models.PublicationStatusPublished, result.Status) + assert.Equal(t, models.PublicationStatusSucceeded, result.Status) assert.Empty(t, fakePublisher.RemoteURL) } @@ -1013,7 +1013,7 @@ func TestPublishProjectRequiresSavedCookiesForBrowserCookiePlatforms(t *testing. ProjectID: project.ID, Platform: "douyin", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{}`), AdaptedContent: datatypes.JSON(`{"summary":"ready"}`), } @@ -1054,7 +1054,7 @@ func TestPublishProjectBlocksUnhealthyAccountAndCreatesNotification(t *testing.T Platform: "wechat", PlatformAccountID: &account.ID, Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Title"}`), AdaptedContent: datatypes.JSON(`{"format":"html","html":"ready"}`), }).Error) @@ -1094,7 +1094,7 @@ func TestPublishProjectRequiresPrepublishSyncForPendingPublication(t *testing.T) ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Title"}`), AdaptedContent: datatypes.JSON(`{}`), } @@ -1107,7 +1107,7 @@ func TestPublishProjectRequiresPrepublishSyncForPendingPublication(t *testing.T) var saved models.ProjectPlatformPublication require.NoError(t, db.First(&saved, "id = ?", pub.ID).Error) - assert.Equal(t, models.PublicationStatusPending, saved.Status) + assert.Equal(t, models.PublicationStatusDraft, saved.Status) assert.Empty(t, saved.ErrorMessage) } @@ -1185,7 +1185,7 @@ func TestPublishProjectRejectsProjectEditor(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Title"}`), AdaptedContent: datatypes.JSON(`{"format":"html","html":"ready"}`), }).Error) @@ -1226,7 +1226,7 @@ func TestPublishProjectRejectsProjectViewer(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Title"}`), AdaptedContent: datatypes.JSON(`{"format":"html","html":"ready"}`), }).Error) @@ -1257,7 +1257,7 @@ func TestPublishProjectUsesSavedXOAuth2Credentials(t *testing.T) { pub := models.ProjectPlatformPublication{ ProjectID: project.ID, Platform: "x", - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"api_key":"stale","api_secret":"stale","access_token":"stale","access_token_secret":"stale","title":"Title"}`), AdaptedContent: datatypes.JSON(`{"text":"ready"}`), } @@ -1278,7 +1278,7 @@ func TestPublishProjectUsesSavedXOAuth2Credentials(t *testing.T) { result, err := s.PublishProject(project.ID, "x", &user.ID, uuid.Nil) require.NoError(t, err) - assert.Equal(t, models.PublicationStatusPublished, result.Status) + assert.Equal(t, models.PublicationStatusSucceeded, result.Status) var config map[string]any require.NoError(t, json.Unmarshal(fakePublisher.Config, &config)) @@ -1324,7 +1324,7 @@ func TestPublishProjectRefreshesExpiredXOAuth2Token(t *testing.T) { pub := models.ProjectPlatformPublication{ ProjectID: project.ID, Platform: "x", - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Title"}`), AdaptedContent: datatypes.JSON(`{"text":"ready"}`), } @@ -1350,7 +1350,7 @@ func TestPublishProjectRefreshesExpiredXOAuth2Token(t *testing.T) { result, err := s.PublishProject(project.ID, "x", &user.ID, uuid.Nil) require.NoError(t, err) - assert.Equal(t, models.PublicationStatusPublished, result.Status) + assert.Equal(t, models.PublicationStatusSucceeded, result.Status) assert.Equal(t, "oauth2-refresh", provider.RefreshToken) assert.Equal(t, "client-id", provider.RefreshConfig.ClientID) assert.Equal(t, "client-secret", provider.RefreshConfig.ClientSecret) @@ -1389,7 +1389,7 @@ func TestCreateXPostIntentReturnsManualPublishURL(t *testing.T) { ProjectID: project.ID, Platform: "x", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Title"}`), AdaptedContent: datatypes.JSON(`{"text":"hello x & \u4e2d\u6587"}`), } @@ -1411,7 +1411,7 @@ func TestCreateXPostIntentReturnsManualPublishURL(t *testing.T) { var saved models.ProjectPlatformPublication require.NoError(t, db.First(&saved, "id = ?", pub.ID).Error) - assert.Equal(t, models.PublicationStatusAdapted, saved.Status) + assert.Equal(t, models.PublicationStatusDraft, saved.Status) assert.Equal(t, publishURL, saved.PublishURL) assert.Empty(t, saved.ErrorMessage) } @@ -1433,7 +1433,7 @@ func TestCreateXPostIntentRequiresPrepublishSyncForPendingPublication(t *testing ProjectID: project.ID, Platform: "x", Enabled: true, - Status: models.PublicationStatusPending, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Title"}`), AdaptedContent: datatypes.JSON(`{}`), } @@ -1446,7 +1446,7 @@ func TestCreateXPostIntentRequiresPrepublishSyncForPendingPublication(t *testing var saved models.ProjectPlatformPublication require.NoError(t, db.First(&saved, "id = ?", pub.ID).Error) - assert.Equal(t, models.PublicationStatusPending, saved.Status) + assert.Equal(t, models.PublicationStatusDraft, saved.Status) assert.JSONEq(t, `{}`, string(saved.AdaptedContent)) } @@ -1475,7 +1475,7 @@ func TestCreateXPostIntentRejectsProjectEditor(t *testing.T) { ProjectID: project.ID, Platform: "x", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Title"}`), AdaptedContent: datatypes.JSON(`{"text":"hello x"}`), } @@ -1516,7 +1516,7 @@ func TestCreateXPostIntentRejectsProjectViewer(t *testing.T) { ProjectID: project.ID, Platform: "x", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Title"}`), AdaptedContent: datatypes.JSON(`{"text":"hello x"}`), } @@ -1549,7 +1549,7 @@ func TestPublishProjectRejectsDisabledPublication(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: false, - Status: models.PublicationStatusDisabled, + Status: models.PublicationStatusCancelled, Config: datatypes.JSON(`{"title":"Title"}`), AdaptedContent: datatypes.JSON(`{"summary":"ready"}`), }) diff --git a/backend/internal/services/publish/queue_test.go b/backend/internal/services/publish/queue_test.go index e0557359d..5212d5cba 100644 --- a/backend/internal/services/publish/queue_test.go +++ b/backend/internal/services/publish/queue_test.go @@ -395,7 +395,7 @@ func TestEnqueuePublishProjectQueuesAndLocksPublication(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Queued post"}`), AdaptedContent: datatypes.JSON(`{"format":"html","html":"ready"}`), }).Error) @@ -452,7 +452,7 @@ func TestEnqueuePublishProjectInvalidatesDashboardCaches(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Queued cache post"}`), AdaptedContent: datatypes.JSON(`{"format":"html","html":"ready"}`), }).Error) @@ -503,7 +503,7 @@ func TestEnqueuePublishProjectRejectsProjectEditor(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Queued post"}`), AdaptedContent: datatypes.JSON(`{"format":"html","html":"ready"}`), }).Error) @@ -543,7 +543,7 @@ func TestEnqueuePublishProjectRejectsProjectViewer(t *testing.T) { ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Queued post"}`), AdaptedContent: datatypes.JSON(`{"format":"html","html":"ready"}`), }).Error) @@ -578,7 +578,7 @@ func TestEnqueuePublishProjectReplaysDuplicateWhenLockWinsBeforeQueuedEvent(t *t ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Queued post"}`), AdaptedContent: datatypes.JSON(`{"format":"html","html":"ready"}`), } @@ -635,7 +635,7 @@ func TestEnqueuePublishProjectDoesNotPersistScheduleWhenLockIsHeld(t *testing.T) ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Queued post"}`), AdaptedContent: datatypes.JSON(`{"format":"html","html":"ready"}`), } @@ -681,7 +681,7 @@ func TestFlushScheduledPublicationsPublishesDueSchedules(t *testing.T) { Platform: "wechat", PlatformAccountID: &account.ID, Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Scheduled post"}`), AdaptedContent: datatypes.JSON(`{"format":"html","html":"ready"}`), } @@ -811,7 +811,7 @@ func TestEnqueuePublishProjectLeavesFailedDispatchInOutboxForRetry(t *testing.T) ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Queued post"}`), AdaptedContent: datatypes.JSON(`{"format":"html","html":"ready"}`), }).Error) @@ -1086,7 +1086,7 @@ func TestEnqueuePublishProjectRejectsPublicationChangedToSyncingAfterLock(t *tes ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusAdapted, + Status: models.PublicationStatusDraft, Config: datatypes.JSON(`{"title":"Queued post"}`), AdaptedContent: datatypes.JSON(`{"format":"html","html":"ready"}`), } diff --git a/backend/internal/services/readmodel/service.go b/backend/internal/services/readmodel/service.go index 3ef303fd5..f10241e5d 100644 --- a/backend/internal/services/readmodel/service.go +++ b/backend/internal/services/readmodel/service.go @@ -162,7 +162,7 @@ func (s *Service) RefreshWorkspace(workspaceID uuid.UUID) error { Count(&stats.TotalProjects).Error; err != nil { return err } - if err := s.countWorkspacePublications(workspaceID, models.PublicationStatusPublished, &stats.TotalPublishedPublications); err != nil { + if err := s.countWorkspacePublications(workspaceID, models.PublicationStatusSucceeded, &stats.TotalPublishedPublications); err != nil { return err } if err := s.countWorkspacePublications(workspaceID, models.PublicationStatusFailed, &stats.TotalFailedPublications); err != nil { diff --git a/backend/internal/services/readmodel/service_test.go b/backend/internal/services/readmodel/service_test.go index af7fa0ea6..57f7e5024 100644 --- a/backend/internal/services/readmodel/service_test.go +++ b/backend/internal/services/readmodel/service_test.go @@ -40,7 +40,7 @@ func TestRefreshProjectUpsertsProjectListSummaryAndWorkspaceStats(t *testing.T) ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, DraftStatus: models.PublicationDraftStatusReady, ReviewStatus: models.PublicationReviewStatusDraft, }).Error) @@ -64,7 +64,7 @@ func TestRefreshProjectUpsertsProjectListSummaryAndWorkspaceStats(t *testing.T) require.NoError(t, json.Unmarshal(summary.Publications, &publications)) require.Len(t, publications, 2) require.Equal(t, "wechat", publications[0].Platform) - require.Equal(t, models.PublicationStatusPublished, publications[0].Status) + require.Equal(t, models.PublicationStatusSucceeded, publications[0].Status) var stats models.WorkspaceDashboardStats require.NoError(t, db.First(&stats, "workspace_id = ?", workspaceID).Error) @@ -130,7 +130,7 @@ func TestRebuildDashboardReplaysFactsAndRemovesOrphanReadModels(t *testing.T) { ProjectID: projectID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, DraftStatus: models.PublicationDraftStatusReady, ReviewStatus: models.PublicationReviewStatusDraft, }).Error) diff --git a/backend/internal/services/stats/overview.go b/backend/internal/services/stats/overview.go index 3367091c8..c36aee16f 100644 --- a/backend/internal/services/stats/overview.go +++ b/backend/internal/services/stats/overview.go @@ -75,7 +75,7 @@ func (s *Service) computeStats(scopeUserID *uuid.UUID) (*dto.DashboardStatsRespo } // Published publications count - pubPubQuery := readDB.Model(&models.ProjectPlatformPublication{}).Where("project_platform_publications.status = ?", models.PublicationStatusPublished) + pubPubQuery := readDB.Model(&models.ProjectPlatformPublication{}).Where("project_platform_publications.status = ?", models.PublicationStatusSucceeded) if scopeUserID != nil { pubPubQuery = pubPubQuery.Joins("JOIN projects ON projects.id = project_platform_publications.project_id"). Scopes(func(db *gorm.DB) *gorm.DB { @@ -330,7 +330,7 @@ func (s *Service) computeWorkspaceStats(workspaceID uuid.UUID) (*dto.DashboardSt pubQuery := readDB.Model(&models.ProjectPlatformPublication{}). Joins("JOIN projects ON projects.id = project_platform_publications.project_id"). Where(where, args...) - if err := pubQuery.Where("project_platform_publications.status = ?", models.PublicationStatusPublished). + if err := pubQuery.Where("project_platform_publications.status = ?", models.PublicationStatusSucceeded). Count(&stats.TotalPublishedPublications).Error; err != nil { return nil, err } diff --git a/backend/internal/services/stats/overview_test.go b/backend/internal/services/stats/overview_test.go index 7169988c9..b8803c78f 100644 --- a/backend/internal/services/stats/overview_test.go +++ b/backend/internal/services/stats/overview_test.go @@ -36,7 +36,7 @@ func TestGetStats(t *testing.T) { db.Create(&p1) db.Create(&p2) - db.Create(&models.ProjectPlatformPublication{ProjectID: p1.ID, Platform: "wechat", Status: models.PublicationStatusPublished}) + db.Create(&models.ProjectPlatformPublication{ProjectID: p1.ID, Platform: "wechat", Status: models.PublicationStatusSucceeded}) db.Create(&models.ProjectPlatformPublication{ProjectID: p2.ID, Platform: "zhihu", Status: models.PublicationStatusFailed}) // Test Admin scope (nil scopeUserID) @@ -77,7 +77,7 @@ func TestGetStatsUsesCompleteWorkspaceReadModel(t *testing.T) { UpdatedAt: now, } require.NoError(t, db.Create(&project).Error) - publicationStatus := models.PublicationStatusPublished + publicationStatus := models.PublicationStatusSucceeded if i >= 5 { publicationStatus = models.PublicationStatusFailed } @@ -146,7 +146,7 @@ func TestGetStatsCachesGlobalDashboardStats(t *testing.T) { s := services.NewDashboardService(db) s.UseRedis(redisClient) - seedStatsOverviewProject(t, db, "cached-a", models.PublicationStatusPublished) + seedStatsOverviewProject(t, db, "cached-a", models.PublicationStatusSucceeded) stats, err := s.WithContext(context.Background()).GetStats(nil) require.NoError(t, err) @@ -185,7 +185,7 @@ func TestCreateProjectInvalidatesDashboardStatsCache(t *testing.T) { s := services.NewDashboardService(db) s.UseRedis(redisClient) - user := seedStatsOverviewProject(t, db, "stats-create", models.PublicationStatusPublished) + user := seedStatsOverviewProject(t, db, "stats-create", models.PublicationStatusSucceeded) stats, err := s.WithContext(context.Background()).GetStats(nil) require.NoError(t, err) @@ -213,7 +213,7 @@ func TestStatsCacheIgnoresStaleRefillAfterInvalidation(t *testing.T) { s.UseRedis(redisClient) ctx := context.Background() - user := seedStatsOverviewProject(t, db, "stats-cache-race", models.PublicationStatusPublished) + user := seedStatsOverviewProject(t, db, "stats-cache-race", models.PublicationStatusSucceeded) first, err := s.WithContext(ctx).GetStats(nil) require.NoError(t, err) @@ -307,7 +307,7 @@ func TestGetStatsBypassesCacheForStickyEventualCounts(t *testing.T) { require.NoError(t, err) assert.Equal(t, int64(1), readerStats.TotalFailedPublications) - seedStatsOverviewProject(t, writer, "writer", models.PublicationStatusPublished) + seedStatsOverviewProject(t, writer, "writer", models.PublicationStatusSucceeded) stickyCtx := dbrouter.WithStickyWriter(context.Background(), time.Now().Add(time.Minute)) stickyStats, err := s.WithContext(stickyCtx).GetStats(nil) @@ -324,7 +324,7 @@ func TestGetStatsFallsBackToDatabaseWhenCachedPayloadIsInvalid(t *testing.T) { s := services.NewDashboardService(db) s.UseRedis(redisClient) - seedStatsOverviewProject(t, db, "fallback", models.PublicationStatusPublished) + seedStatsOverviewProject(t, db, "fallback", models.PublicationStatusSucceeded) _, err := s.WithContext(context.Background()).GetStats(nil) require.NoError(t, err) cacheKey := requireSingleStatsCacheKey(t, redisClient) @@ -364,7 +364,7 @@ func TestGetStatsRepairsSemanticallyInvalidCachedPayload(t *testing.T) { s := services.NewDashboardService(db) s.UseRedis(redisClient) - seedStatsOverviewProject(t, db, "semantic-invalid-a", models.PublicationStatusPublished) + seedStatsOverviewProject(t, db, "semantic-invalid-a", models.PublicationStatusSucceeded) _, err := s.WithContext(context.Background()).GetStats(nil) require.NoError(t, err) cacheKey := requireSingleStatsCacheKey(t, redisClient) @@ -398,7 +398,7 @@ func TestGetStatsCachesScopedUserDashboardStats(t *testing.T) { s := services.NewDashboardService(db) s.UseRedis(redisClient) - user := seedStatsOverviewProject(t, db, "scoped-a", models.PublicationStatusPublished) + user := seedStatsOverviewProject(t, db, "scoped-a", models.PublicationStatusSucceeded) stats, err := s.WithContext(context.Background()).GetStats(&user.ID) require.NoError(t, err) @@ -453,7 +453,7 @@ func TestGetStatsBypassesScopedCacheForStickyWriter(t *testing.T) { s := services.NewDashboardService(db) s.UseRedis(redisClient) - user := seedStatsOverviewProject(t, db, "scoped-sticky", models.PublicationStatusPublished) + user := seedStatsOverviewProject(t, db, "scoped-sticky", models.PublicationStatusSucceeded) stats, err := s.WithContext(context.Background()).GetStats(&user.ID) require.NoError(t, err) @@ -492,7 +492,7 @@ func TestGetWorkspaceStatsCachesScopedDashboardStats(t *testing.T) { owner := seedStatsUser(t, db, "workspace-cache-owner") workspaceID := seedStatsWorkspace(t, db, owner, "workspace-cache") - seedStatsWorkspaceProject(t, db, owner.ID, workspaceID, "workspace-cache-a", models.PublicationStatusPublished) + seedStatsWorkspaceProject(t, db, owner.ID, workspaceID, "workspace-cache-a", models.PublicationStatusSucceeded) stats, err := s.WithContext(context.Background()).GetWorkspaceStats(workspaceID, owner.ID) require.NoError(t, err) @@ -578,7 +578,7 @@ func TestWorkspaceMembershipInvalidatesScopedStatsCache(t *testing.T) { owner := seedStatsUser(t, db, "membership-cache-owner") member := seedStatsUser(t, db, "membership-cache-member") workspaceID := seedStatsWorkspace(t, db, owner, "membership-cache") - seedStatsWorkspaceProject(t, db, owner.ID, workspaceID, "membership-cache-project", models.PublicationStatusPublished) + seedStatsWorkspaceProject(t, db, owner.ID, workspaceID, "membership-cache-project", models.PublicationStatusSucceeded) beforeStats, err := s.WithContext(ctx).GetStats(&member.ID) require.NoError(t, err) @@ -708,7 +708,7 @@ func TestGetWorkspaceStatsFallbackIncludesLegacyPersonalProjects(t *testing.T) { ID: uuid.New(), ProjectID: project.ID, Platform: "wechat", - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, }).Error) require.NoError(t, db.Create(&models.ProjectPlatformPublication{ ID: uuid.New(), @@ -737,7 +737,7 @@ func TestGetStatsUsesReaderForEventualCounts(t *testing.T) { require.NoError(t, reader.Create(&models.ProjectPlatformPublication{ ProjectID: project.ID, Platform: "wechat", - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, }).Error) var writerUsers int64 @@ -762,7 +762,7 @@ func TestGetStatsCollapsesConcurrentCacheMisses(t *testing.T) { s := services.NewDashboardService(db) s.UseRedis(redisClient) - seedStatsOverviewProject(t, db, "stats-singleflight", models.PublicationStatusPublished) + seedStatsOverviewProject(t, db, "stats-singleflight", models.PublicationStatusSucceeded) queryCount := registerBlockingStatsQueryCounter(t, db) results := runConcurrentStatsRequests(t, s, queryCount) @@ -985,7 +985,7 @@ func seedStatsLifecycleProject(t *testing.T, db *gorm.DB, prefix string) (models ProjectID: project.ID, Platform: "wechat", Enabled: true, - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, }).Error) require.NoError(t, db.Create(&models.ProjectPlatformPublication{ ProjectID: project.ID, @@ -1066,7 +1066,7 @@ func TestGetStatsUsesWriterForStickyEventualCounts(t *testing.T) { require.NoError(t, writer.Create(&models.ProjectPlatformPublication{ ProjectID: writerProject.ID, Platform: "wechat", - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, }).Error) readerUser := models.User{Username: "stale-reader-user", Email: "stale-reader-user@example.com", PasswordHash: "hash"} @@ -1104,7 +1104,7 @@ func TestGetStatsUsesWriterForScopedCounts(t *testing.T) { require.NoError(t, writer.Create(&models.ProjectPlatformPublication{ ProjectID: currentProject.ID, Platform: "wechat", - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, }).Error) staleProject := models.Project{UserID: user.ID, Title: "stale-reader-project", SourceContent: "content", Status: models.ProjectStatusReady} @@ -1112,7 +1112,7 @@ func TestGetStatsUsesWriterForScopedCounts(t *testing.T) { require.NoError(t, reader.Create(&models.ProjectPlatformPublication{ ProjectID: staleProject.ID, Platform: "wechat", - Status: models.PublicationStatusPublished, + Status: models.PublicationStatusSucceeded, }).Error) require.NoError(t, reader.Create(&models.ProjectPlatformPublication{ ProjectID: staleProject.ID, diff --git a/browser-worker/internal/session/redis_state.go b/browser-worker/internal/session/redis_state.go index aee681186..b90dac79f 100644 --- a/browser-worker/internal/session/redis_state.go +++ b/browser-worker/internal/session/redis_state.go @@ -23,6 +23,8 @@ const ( redisTLSEnv = "REDIS_TLS" redisTLSCACertEnv = "REDIS_TLS_CA_CERT" redisTLSCAFileEnv = "REDIS_TLS_CA_FILE" + redisTLSCertFileEnv = "REDIS_TLS_CERT_FILE" + redisTLSKeyFileEnv = "REDIS_TLS_KEY_FILE" redisTLSServerNameEnv = "REDIS_TLS_SERVER_NAME" redisSentinelAddrsEnv = "REDIS_SENTINEL_ADDRS" redisSentinelMasterEnv = "REDIS_SENTINEL_MASTER_NAME" @@ -68,6 +70,8 @@ type redisConnectionConfig struct { TLS bool TLSCACert string TLSCAFile string + TLSCertFile string + TLSKeyFile string TLSServerName string ClusterAddrs []string SentinelAddrs []string @@ -132,6 +136,8 @@ func redisConnectionConfigFromEnv() (redisConnectionConfig, error) { TLS: redisEnvFlagEnabled(redisTLSEnv), TLSCACert: strings.TrimSpace(os.Getenv(redisTLSCACertEnv)), TLSCAFile: strings.TrimSpace(os.Getenv(redisTLSCAFileEnv)), + TLSCertFile: strings.TrimSpace(os.Getenv(redisTLSCertFileEnv)), + TLSKeyFile: strings.TrimSpace(os.Getenv(redisTLSKeyFileEnv)), TLSServerName: strings.TrimSpace( os.Getenv(redisTLSServerNameEnv), ), @@ -463,9 +469,30 @@ func redisTLSConfig(config redisConnectionConfig) (*tls.Config, error) { } else { tlsConfig.RootCAs = rootCAs } + clientCertificate, ok, err := redisTLSClientCertificate(config.TLSCertFile, config.TLSKeyFile) + if err != nil { + return nil, err + } + if ok { + tlsConfig.Certificates = []tls.Certificate{clientCertificate} + } return tlsConfig, nil } +func redisTLSClientCertificate(certFile string, keyFile string) (tls.Certificate, bool, error) { + if certFile == "" && keyFile == "" { + return tls.Certificate{}, false, nil + } + if certFile == "" || keyFile == "" { + return tls.Certificate{}, false, fmt.Errorf("%s and %s must be set together", redisTLSCertFileEnv, redisTLSKeyFileEnv) + } + certificate, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return tls.Certificate{}, false, fmt.Errorf("failed to load Redis TLS client certificate from %s and %s: %w", redisTLSCertFileEnv, redisTLSKeyFileEnv, err) + } + return certificate, true, nil +} + func redisTLSRootCAs(inlineCert string, certFile string) (*x509.CertPool, error) { if inlineCert == "" && certFile == "" { return nil, errRedisTLSRootCAsNotConfigured @@ -515,6 +542,9 @@ func (c redisConnectionConfig) tlsConfig() *tls.Config { ServerName: c.TLSServerName, } tlsConfig.RootCAs, _ = redisTLSRootCAs(c.TLSCACert, c.TLSCAFile) + if clientCertificate, ok, err := redisTLSClientCertificate(c.TLSCertFile, c.TLSKeyFile); err == nil && ok { + tlsConfig.Certificates = []tls.Certificate{clientCertificate} + } return tlsConfig } return c.tls.Clone() diff --git a/browser-worker/internal/session/redis_state_test.go b/browser-worker/internal/session/redis_state_test.go index aea44839e..cf94240aa 100644 --- a/browser-worker/internal/session/redis_state_test.go +++ b/browser-worker/internal/session/redis_state_test.go @@ -1,9 +1,17 @@ package session import ( + "crypto/rand" + "crypto/rsa" "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" "os" + "path/filepath" "testing" + "time" "github.com/redis/go-redis/v9" "github.com/stretchr/testify/require" @@ -83,6 +91,22 @@ func TestRedisConnectionConfigFromEnvBuildsTLSOptionsFromCAFile(t *testing.T) { require.NotNil(t, options.TLSConfig.RootCAs) } +func TestRedisConnectionConfigFromEnvBuildsTLSOptionsWithClientCertificate(t *testing.T) { + clearRedisEnv(t) + certFile, keyFile := writeTestRedisClientCertificate(t) + t.Setenv(redisAddrEnv, "redis.example.invalid:6379") + t.Setenv(redisTLSEnv, "true") + t.Setenv(redisTLSCertFileEnv, certFile) + t.Setenv(redisTLSKeyFileEnv, keyFile) + + config, err := redisConnectionConfigFromEnv() + require.NoError(t, err) + + options := redisOptions(config) + require.NotNil(t, options.TLSConfig) + require.Len(t, options.TLSConfig.Certificates, 1) +} + func TestRedisConnectionConfigFromEnvRejectsInvalidTLSCA(t *testing.T) { tests := []struct { name string @@ -114,6 +138,33 @@ func TestRedisConnectionConfigFromEnvRejectsInvalidTLSCA(t *testing.T) { } } +func TestRedisConnectionConfigFromEnvRejectsIncompleteTLSClientCertificate(t *testing.T) { + tests := []struct { + name string + env map[string]string + }{ + {name: "missing key", env: map[string]string{redisTLSCertFileEnv: "/tmp/redis-client.crt"}}, + {name: "missing cert", env: map[string]string{redisTLSKeyFileEnv: "/tmp/redis-client.key"}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clearRedisEnv(t) + t.Setenv(redisAddrEnv, "redis.example.invalid:6379") + t.Setenv(redisTLSEnv, "true") + for key, value := range tt.env { + t.Setenv(key, value) + } + + _, err := redisConnectionConfigFromEnv() + + require.Error(t, err) + require.Contains(t, err.Error(), redisTLSCertFileEnv) + require.Contains(t, err.Error(), redisTLSKeyFileEnv) + }) + } +} + func TestRedisConnectionConfigFromEnvUsesSentinelEndpoint(t *testing.T) { clearRedisEnv(t) t.Setenv(redisEndpointModeEnv, redisEndpointModeSentinel) @@ -269,6 +320,8 @@ func clearRedisEnv(t *testing.T) { redisTLSEnv, redisTLSCACertEnv, redisTLSCAFileEnv, + redisTLSCertFileEnv, + redisTLSKeyFileEnv, redisTLSServerNameEnv, redisSentinelAddrsEnv, redisSentinelMasterEnv, @@ -277,6 +330,31 @@ func clearRedisEnv(t *testing.T) { } } +func writeTestRedisClientCertificate(t *testing.T) (string, string) { + t.Helper() + + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "redis-client"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } + certDER, err := x509.CreateCertificate(rand.Reader, template, template, &privateKey.PublicKey, privateKey) + require.NoError(t, err) + keyDER := x509.MarshalPKCS1PrivateKey(privateKey) + + dir := t.TempDir() + certFile := filepath.Join(dir, "redis-client.crt") + keyFile := filepath.Join(dir, "redis-client.key") + require.NoError(t, os.WriteFile(certFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), 0o600)) + require.NoError(t, os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: keyDER}), 0o600)) + return certFile, keyFile +} + const testRedisCACertPEM = `-----BEGIN CERTIFICATE----- MIIDEzCCAfugAwIBAgIUb15xgBiiAVRKRFX/A/p9TvypqJwwDQYJKoZIhvcNAQEL BQAwGTEXMBUGA1UEAwwObXBwLXJlZGlzLXRlc3QwHhcNMjYwNjE4MTQyOTQ5WhcN diff --git a/collab-service/src/collab/redis-pubsub.test.ts b/collab-service/src/collab/redis-pubsub.test.ts index c3eebefc1..e5a781449 100644 --- a/collab-service/src/collab/redis-pubsub.test.ts +++ b/collab-service/src/collab/redis-pubsub.test.ts @@ -178,6 +178,42 @@ describe("redisClientOptionsFromConfig", () => { }); }); + it("reads TLS client certificate options from files", () => { + const dir = mkdtempSync(join(tmpdir(), "mpp-redis-client-cert-")); + const certFile = join(dir, "client.crt"); + const keyFile = join(dir, "client.key"); + writeFileSync(certFile, testRedisClientCertPEM); + writeFileSync(keyFile, testRedisClientKeyPEM); + const config = loadConfig({ + NODE_ENV: "test", + REDIS_ADDR: "redis.example.invalid:6379", + REDIS_TLS: "true", + REDIS_TLS_CERT_FILE: certFile, + REDIS_TLS_KEY_FILE: keyFile, + }); + + const options = redisClientOptionsFromConfig(config); + + expect(options.socket).toMatchObject({ + cert: testRedisClientCertPEM, + key: testRedisClientKeyPEM, + tls: true, + }); + }); + + it("rejects incomplete TLS client certificate options", () => { + const config = loadConfig({ + NODE_ENV: "test", + REDIS_ADDR: "redis.example.invalid:6379", + REDIS_TLS: "true", + REDIS_TLS_CERT_FILE: "/tmp/redis-client.crt", + }); + + expect(() => redisClientOptionsFromConfig(config)).toThrow( + /REDIS_TLS_CERT_FILE/, + ); + }); + it("upgrades explicit redis URLs when REDIS_TLS is enabled", () => { const config = loadConfig({ NODE_ENV: "test", @@ -376,3 +412,20 @@ tYGsODtHm/A35rOUUfx34E9PUIQXrm7HPIHbThi64/vJFd2dzvB/966Z2YCtkBf2 eXFaNn/Uv31V+R4jo/IoXT3Ge5aU2/HCF4GLt86Hny8lrZI/rzBtD+mvxHiPCeVH kXlb94L5hmllJh6r7idCx5YrKWYGYCc= -----END CERTIFICATE-----`; + +const testRedisClientCertPEM = `-----BEGIN CERTIFICATE----- +MIIBfTCCASOgAwIBAgIBATAKBggqhkjOPQQDAjAXMRUwEwYDVQQDEwxyZWRpcy1j +bGllbnQwHhcNMjYwMTAxMDAwMDAwWhcNMjcwMTAxMDAwMDAwWjAXMRUwEwYDVQQD +EwxyZWRpcy1jbGllbnQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASlQ67fO3vT +YCz7zD0MyoQ7gGmC6a9LdbwH4Q4YHjKiNF9JpEORxk4g7FND6dH1grn6iJ2IGlzE +N0R0B3kzVYwWo1MwUTAdBgNVHQ4EFgQU2a9Z5v6f5uUeV2l6EcnfwbkwvRswHwYD +VR0jBBgwFoAU2a9Z5v6f5uUeV2l6EcnfwbkwvRswDwYDVR0TAQH/BAUwAwEB/zAK +BggqhkjOPQQDAgNJADBGAiEAyBTnx37EGUIo7PrpV67EMAWs7N8hJlmdGr6vH3Of +tGQCIQD5VdY7B/zag98SfOvdiX5j1EbkruWyXEFUvc+hjRLCTg== +-----END CERTIFICATE-----`; + +const testRedisClientKeyPEM = `-----BEGIN EC PRIVATE KEY----- +MHcCAQEEILMuW4ZfYwR5l+8a9XJduP/9O6n2pMSYjRoXrEGLZQ2ZoAoGCCqGSM49 +AwEHoUQDQgAEpUOu3zt702As+8w9DMqEO4BpgumvS3W8B+EOGB4yojRfSaRDkcZO +IOxTQ+nR9YK5+oidGDpcxDdEdAd5M1WMFg== +-----END EC PRIVATE KEY-----`; diff --git a/collab-service/src/collab/redis-pubsub.ts b/collab-service/src/collab/redis-pubsub.ts index 08b203c4e..3811a9d49 100644 --- a/collab-service/src/collab/redis-pubsub.ts +++ b/collab-service/src/collab/redis-pubsub.ts @@ -299,6 +299,11 @@ function redisTLSOptionsFromConfig( if (ca) { (options as TLSConnectionOptions).ca = ca; } + const clientCertificate = redisTLSClientCertificateFromConfig(config); + if (clientCertificate) { + (options as TLSConnectionOptions).cert = clientCertificate.cert; + (options as TLSConnectionOptions).key = clientCertificate.key; + } if (config.REDIS_TLS_SERVER_NAME.trim()) { (options as TLSConnectionOptions).servername = config.REDIS_TLS_SERVER_NAME.trim(); @@ -318,6 +323,25 @@ function redisTLSCAFromConfig(config: CollabConfig): string | undefined { return undefined; } +function redisTLSClientCertificateFromConfig( + config: CollabConfig, +): { cert: string; key: string } | undefined { + const certFile = config.REDIS_TLS_CERT_FILE.trim(); + const keyFile = config.REDIS_TLS_KEY_FILE.trim(); + if (!certFile && !keyFile) { + return undefined; + } + if (!certFile || !keyFile) { + throw new Error( + "REDIS_TLS_CERT_FILE and REDIS_TLS_KEY_FILE must be set together", + ); + } + return { + cert: readFileSync(certFile, "utf8"), + key: readFileSync(keyFile, "utf8"), + }; +} + export function redisSentinelOptionsFromConfig( config: CollabConfig, ): RedisSentinelOptions { diff --git a/collab-service/src/config.ts b/collab-service/src/config.ts index 28de53cf5..4cb7aa49f 100644 --- a/collab-service/src/config.ts +++ b/collab-service/src/config.ts @@ -79,6 +79,8 @@ const BaseEnvSchema = z.object({ REDIS_TLS: EnvBoolean.default(false), REDIS_TLS_CA_CERT: z.string().default(""), REDIS_TLS_CA_FILE: z.string().default(""), + REDIS_TLS_CERT_FILE: z.string().default(""), + REDIS_TLS_KEY_FILE: z.string().default(""), REDIS_TLS_SERVER_NAME: z.string().default(""), REDIS_SENTINEL_ADDRS: z.string().default(""), REDIS_SENTINEL_MASTER_NAME: z.preprocess( diff --git a/contracts/env.schema.yaml b/contracts/env.schema.yaml index 768603bf2..b244aa223 100644 --- a/contracts/env.schema.yaml +++ b/contracts/env.schema.yaml @@ -50,11 +50,15 @@ profiles: value: "0" - name: REDIS_TLS value: "false" - - comment: Optional managed Redis TLS trust material. Use REDIS_TLS_CA_CERT for an inline PEM CA bundle, or REDIS_TLS_CA_FILE when you mount provider CA material into the container. REDIS_TLS_SERVER_NAME overrides certificate hostname verification only when the provider requires a different SNI name. + - comment: Optional Redis TLS trust material. Use REDIS_TLS_CA_CERT for an inline PEM CA bundle, REDIS_TLS_CA_FILE when you mount CA material into the container, and REDIS_TLS_CERT_FILE plus REDIS_TLS_KEY_FILE when the Redis endpoint requires client certificate auth. REDIS_TLS_SERVER_NAME overrides certificate hostname verification only when the provider requires a different SNI name. - name: REDIS_TLS_CA_CERT value: "" - name: REDIS_TLS_CA_FILE value: "" + - name: REDIS_TLS_CERT_FILE + value: "" + - name: REDIS_TLS_KEY_FILE + value: "" - name: REDIS_TLS_SERVER_NAME value: "" - comment: Set these only when REDIS_ENDPOINT_MODE=sentinel. For the non-production HA topology, use REDIS_SENTINEL_ADDRS=redis-ha-sentinel:26379 and REDIS_SENTINEL_MASTER_NAME=mpp-redis-ha. @@ -327,11 +331,15 @@ profiles: value: "0" - name: REDIS_TLS value: "false" - - comment: Optional managed Redis TLS trust material. Use REDIS_TLS_CA_CERT for an inline PEM CA bundle, or REDIS_TLS_CA_FILE when you mount provider CA material into the container. REDIS_TLS_SERVER_NAME overrides certificate hostname verification only when the provider requires a different SNI name. + - comment: Optional Redis TLS trust material. Use REDIS_TLS_CA_CERT for an inline PEM CA bundle, REDIS_TLS_CA_FILE when you mount CA material into the container, and REDIS_TLS_CERT_FILE plus REDIS_TLS_KEY_FILE when the Redis endpoint requires client certificate auth. REDIS_TLS_SERVER_NAME overrides certificate hostname verification only when the provider requires a different SNI name. - name: REDIS_TLS_CA_CERT value: "" - name: REDIS_TLS_CA_FILE value: "" + - name: REDIS_TLS_CERT_FILE + value: "" + - name: REDIS_TLS_KEY_FILE + value: "" - name: REDIS_TLS_SERVER_NAME value: "" - comment: Set these only when REDIS_ENDPOINT_MODE=sentinel. For the non-production HA topology, use REDIS_SENTINEL_ADDRS=redis-ha-sentinel:26379 and REDIS_SENTINEL_MASTER_NAME=mpp-redis-ha. @@ -793,6 +801,14 @@ variables: category: redis type: file_path services: [backend, publish-worker, browser-worker, collab-service] + REDIS_TLS_CERT_FILE: + category: redis + type: file_path + services: [backend, publish-worker, browser-worker, collab-service] + REDIS_TLS_KEY_FILE: + category: redis + type: file_path + services: [backend, publish-worker, browser-worker, collab-service] REDIS_TLS_SERVER_NAME: category: redis type: hostname diff --git a/deploy/docker/.env.deploy.example b/deploy/docker/.env.deploy.example index 9381e31b2..b88ce14eb 100644 --- a/deploy/docker/.env.deploy.example +++ b/deploy/docker/.env.deploy.example @@ -26,9 +26,11 @@ REDIS_ADDR=redis:6379 REDIS_PASSWORD=replace-with-a-strong-redis-password REDIS_DB=0 REDIS_TLS=false -# Optional managed Redis TLS trust material. Use REDIS_TLS_CA_CERT for an inline PEM CA bundle, or REDIS_TLS_CA_FILE when you mount provider CA material into the container. REDIS_TLS_SERVER_NAME overrides certificate hostname verification only when the provider requires a different SNI name. +# Optional Redis TLS trust material. Use REDIS_TLS_CA_CERT for an inline PEM CA bundle, REDIS_TLS_CA_FILE when you mount CA material into the container, and REDIS_TLS_CERT_FILE plus REDIS_TLS_KEY_FILE when the Redis endpoint requires client certificate auth. REDIS_TLS_SERVER_NAME overrides certificate hostname verification only when the provider requires a different SNI name. REDIS_TLS_CA_CERT= REDIS_TLS_CA_FILE= +REDIS_TLS_CERT_FILE= +REDIS_TLS_KEY_FILE= REDIS_TLS_SERVER_NAME= # Set these only when REDIS_ENDPOINT_MODE=sentinel. For the non-production HA topology, use REDIS_SENTINEL_ADDRS=redis-ha-sentinel:26379 and REDIS_SENTINEL_MASTER_NAME=mpp-redis-ha. REDIS_SENTINEL_ADDRS= diff --git a/deploy/docker/.env.dev.example b/deploy/docker/.env.dev.example index 3f6936efc..b2ee11fc8 100644 --- a/deploy/docker/.env.dev.example +++ b/deploy/docker/.env.dev.example @@ -26,9 +26,11 @@ REDIS_ADDR=redis:6379 REDIS_PASSWORD= REDIS_DB=0 REDIS_TLS=false -# Optional managed Redis TLS trust material. Use REDIS_TLS_CA_CERT for an inline PEM CA bundle, or REDIS_TLS_CA_FILE when you mount provider CA material into the container. REDIS_TLS_SERVER_NAME overrides certificate hostname verification only when the provider requires a different SNI name. +# Optional Redis TLS trust material. Use REDIS_TLS_CA_CERT for an inline PEM CA bundle, REDIS_TLS_CA_FILE when you mount CA material into the container, and REDIS_TLS_CERT_FILE plus REDIS_TLS_KEY_FILE when the Redis endpoint requires client certificate auth. REDIS_TLS_SERVER_NAME overrides certificate hostname verification only when the provider requires a different SNI name. REDIS_TLS_CA_CERT= REDIS_TLS_CA_FILE= +REDIS_TLS_CERT_FILE= +REDIS_TLS_KEY_FILE= REDIS_TLS_SERVER_NAME= # Set these only when REDIS_ENDPOINT_MODE=sentinel. For the non-production HA topology, use REDIS_SENTINEL_ADDRS=redis-ha-sentinel:26379 and REDIS_SENTINEL_MASTER_NAME=mpp-redis-ha. REDIS_SENTINEL_ADDRS= diff --git a/deploy/kubernetes/app-baseline/README.md b/deploy/kubernetes/app-baseline/README.md index 141bd180f..f3596950c 100644 --- a/deploy/kubernetes/app-baseline/README.md +++ b/deploy/kubernetes/app-baseline/README.md @@ -20,8 +20,9 @@ Required overlay inputs: `REDIS_ENDPOINT_MODE=cluster` with `REDIS_ADDR` set to one or more comma-separated seed nodes. Cluster mode requires `REDIS_DB=0`. - Redis TLS policy through `REDIS_TLS`; set `REDIS_TLS_CA_CERT`, - `REDIS_TLS_CA_FILE`, or `REDIS_TLS_SERVER_NAME` only when the provider - requires custom trust material or a documented SNI override. + `REDIS_TLS_CA_FILE`, `REDIS_TLS_CERT_FILE`, `REDIS_TLS_KEY_FILE`, or + `REDIS_TLS_SERVER_NAME` only when the endpoint requires custom trust + material, client certificate auth, or a documented SNI override. - Public collaboration routing through `COLLAB_WEBSOCKET_URL_BASE`. - Public HTTP routing through the `mpp-public-gateway` Ingress. The baseline routes `/collab` to `collab-service` for WebSocket traffic and all remaining diff --git a/deploy/kubernetes/app-baseline/app-config.yaml b/deploy/kubernetes/app-baseline/app-config.yaml index 7a50c4625..094e4d50a 100644 --- a/deploy/kubernetes/app-baseline/app-config.yaml +++ b/deploy/kubernetes/app-baseline/app-config.yaml @@ -45,6 +45,8 @@ data: REDIS_TLS: "false" REDIS_TLS_CA_CERT: "" REDIS_TLS_CA_FILE: "" + REDIS_TLS_CERT_FILE: "" + REDIS_TLS_KEY_FILE: "" REDIS_TLS_SERVER_NAME: "" REDIS_SENTINEL_ADDRS: "" REDIS_SENTINEL_MASTER_NAME: mpp-redis-ha diff --git a/deploy/kubernetes/app-baseline/deployments.yaml b/deploy/kubernetes/app-baseline/deployments.yaml index 2966c71b6..eae2866e0 100644 --- a/deploy/kubernetes/app-baseline/deployments.yaml +++ b/deploy/kubernetes/app-baseline/deployments.yaml @@ -173,6 +173,10 @@ spec: secretKeyRef: name: mpp-app-secrets key: X_OAUTH2_CLIENT_SECRET + volumeMounts: + - name: redis-tls + mountPath: /etc/mpp/redis + readOnly: true readinessProbe: httpGet: path: /ready @@ -196,6 +200,11 @@ spec: limits: cpu: "1" memory: 1Gi + volumes: + - name: redis-tls + secret: + secretName: mpp-redis-cluster-tls + optional: true --- apiVersion: apps/v1 kind: Deployment @@ -303,6 +312,10 @@ spec: secretKeyRef: name: mpp-app-secrets key: X_OAUTH2_CLIENT_SECRET + volumeMounts: + - name: redis-tls + mountPath: /etc/mpp/redis + readOnly: true readinessProbe: httpGet: path: /ready @@ -326,6 +339,11 @@ spec: limits: cpu: "1" memory: 1Gi + volumes: + - name: redis-tls + secret: + secretName: mpp-redis-cluster-tls + optional: true --- apiVersion: apps/v1 kind: Deployment @@ -398,6 +416,10 @@ spec: secretKeyRef: name: mpp-app-secrets key: BROWSER_WORKER_INTERNAL_TOKEN + volumeMounts: + - name: redis-tls + mountPath: /etc/mpp/redis + readOnly: true readinessProbe: httpGet: path: /ready @@ -421,6 +443,11 @@ spec: limits: cpu: "1" memory: 512Mi + volumes: + - name: redis-tls + secret: + secretName: mpp-redis-cluster-tls + optional: true --- apiVersion: apps/v1 kind: Deployment @@ -657,6 +684,10 @@ spec: name: mpp-app-secrets key: REDIS_PASSWORD optional: true + volumeMounts: + - name: redis-tls + mountPath: /etc/mpp/redis + readOnly: true readinessProbe: httpGet: path: /ready @@ -680,3 +711,8 @@ spec: limits: cpu: 500m memory: 512Mi + volumes: + - name: redis-tls + secret: + secretName: mpp-redis-cluster-tls + optional: true diff --git a/deploy/kubernetes/data-services/redis-cluster-nonprod/README.md b/deploy/kubernetes/data-services/redis-cluster-nonprod/README.md index 138028737..502e7208f 100644 --- a/deploy/kubernetes/data-services/redis-cluster-nonprod/README.md +++ b/deploy/kubernetes/data-services/redis-cluster-nonprod/README.md @@ -7,11 +7,13 @@ the intended production-style shape without changing production Redis: - TLS-only Redis traffic on `6379` and Cluster bus traffic on `16379`. - Password auth from `mpp-app-secrets/REDIS_PASSWORD`. - TLS material from a pre-created `mpp-redis-cluster-tls` Secret with - `ca.crt`, `tls.crt`, and `tls.key`. + `ca.crt`, `tls.crt`, and `tls.key`; app clients authenticate with Redis + password auth plus the mounted client certificate and key. - A bootstrap Job that runs `redis-cli --cluster create` with one replica per master. - A cluster-aware `redis_exporter` Deployment for per-node and slot metrics. -- A backup CronJob that schedules node snapshots and records Cluster topology. +- A backup CronJob that exports RDB files from primary nodes and records + Cluster topology. It intentionally does not replace the existing `redis` Service or switch app traffic by default. Non-production apps keep using `REDIS_ENDPOINT_MODE=direct` @@ -53,7 +55,7 @@ Record every drill in this table before promoting Cluster settings elsewhere. | Slots | `redis-cli --tls --cacert ca.crt -a "$REDIS_PASSWORD" -h redis-cluster-0.redis-cluster-headless.mpp-system.svc.cluster.local CLUSTER SLOTS` | Slots `0..16383` covered | | | Failover | `redis-cli --tls --cacert ca.crt -a "$REDIS_PASSWORD" -h CLUSTER FAILOVER` | Replica promoted and clients recover | | | Reshard | `redis-cli --tls --cacert ca.crt -a "$REDIS_PASSWORD" --cluster reshard :6379` | Slots move without uncovered slots | | -| Backup | `kubectl create job --from=cronjob/redis-cluster-backup redis-cluster-backup-manual -n "$MPP_APP_NS"` | Backup marker and topology files created | | +| Backup | `kubectl create job --from=cronjob/redis-cluster-backup redis-cluster-backup-manual -n "$MPP_APP_NS"` | RDB files for primary nodes plus topology files created | | | Restore | Restore RDB/AOF data into a fresh non-prod Cluster and run `CLUSTER INFO` | `cluster_state:ok`; sampled app keys read back | | | Metrics | Scrape `redis-cluster-exporter:9121/metrics` | Per-node, memory, command, keyspace, and Cluster metrics visible | | | Hot keys | `redis-cli --tls --cacert ca.crt -a "$REDIS_PASSWORD" --hotkeys -h ` | Hot-key sample recorded or provider gap noted | | @@ -78,6 +80,9 @@ Only use Cluster mode in non-production after the validation record is complete: REDIS_ENDPOINT_MODE: cluster REDIS_ADDR: redis-cluster.mpp-system.svc.cluster.local:6379 REDIS_TLS: "true" +REDIS_TLS_CA_FILE: /etc/mpp/redis/ca.crt +REDIS_TLS_CERT_FILE: /etc/mpp/redis/tls.crt +REDIS_TLS_KEY_FILE: /etc/mpp/redis/tls.key ``` To roll back app traffic, patch the direct-mode values and restart the diff --git a/deploy/kubernetes/data-services/redis-cluster-nonprod/redis-cluster.yaml b/deploy/kubernetes/data-services/redis-cluster-nonprod/redis-cluster.yaml index 5b74396b0..ee5d09c3b 100644 --- a/deploy/kubernetes/data-services/redis-cluster-nonprod/redis-cluster.yaml +++ b/deploy/kubernetes/data-services/redis-cluster-nonprod/redis-cluster.yaml @@ -516,15 +516,22 @@ spec: backup_dir="/backups/redis-cluster/$(date -u +%Y%m%dT%H%M%SZ)" mkdir -p "$backup_dir" first_node="${REDIS_CLUSTER_NODES%% *}" - # redis-cluster-0.redis-cluster-headless.mpp-system.svc.cluster.local is the backup anchor. + # REDIS_CLUSTER_NODES targets redis-cluster-headless.mpp-system.svc.cluster.local pod DNS entries. for node in $REDIS_CLUSTER_NODES; do host="${node%:*}" port="${node##*:}" - redis-cli --tls --cacert /tls/ca.crt --cert /tls/tls.crt --key /tls/tls.key --no-auth-warning -a "$REDIS_PASSWORD" -h "$host" -p "$port" BGSAVE SCHEDULE || true - redis-cli --tls --cacert /tls/ca.crt --cert /tls/tls.crt --key /tls/tls.key --no-auth-warning -a "$REDIS_PASSWORD" -h "$host" -p "$port" CLUSTER NODES > "$backup_dir/${host}.cluster-nodes.txt" + redis_cli() { + redis-cli --tls --cacert /tls/ca.crt --cert /tls/tls.crt --key /tls/tls.key --no-auth-warning -a "$REDIS_PASSWORD" -h "$host" -p "$port" "$@" + } + redis_cli BGSAVE SCHEDULE || true + redis_cli CLUSTER NODES > "$backup_dir/${host}.cluster-nodes.txt" + role="$(redis_cli INFO replication | tr -d '\r' | awk -F: '/^role:/{print $2}')" + if [ "$role" = "master" ]; then + redis-cli --tls --cacert /tls/ca.crt --cert /tls/tls.crt --key /tls/tls.key --no-auth-warning -a "$REDIS_PASSWORD" -h "$host" -p "$port" --rdb "$backup_dir/${host}.rdb" + fi done echo "$first_node" > "$backup_dir/cluster-source.txt" - echo "Redis Cluster backup marker written to $backup_dir" + echo "Redis Cluster RDB exports and topology files written to $backup_dir" resources: requests: cpu: 25m diff --git a/doc/plan/database-optimization.md b/doc/plan/database-optimization.md index 8a9445f2c..a6789dfe7 100644 --- a/doc/plan/database-optimization.md +++ b/doc/plan/database-optimization.md @@ -15,13 +15,13 @@ Current overall progress: about `58%`. This number is manually estimated by phas | Phase | Weight | Current completion | Status | Completed | Not done / next steps | | ----- | ------ | ------------------ | ------ | --------- | --------------------- | -| Phase 0: Data-layer baseline inventory | 10% | 100% | Done | GORM query observability, `mpp_db_*` metrics, dashboard query-plan audit script, `pg_stat_statements`, database baseline audit script, PostgreSQL exporter table-level health and 24h row-growth panels, read/write consistency classification, and versioned migration rules for complex DDL | Continue implementing code routing, migration executor, partitioning, and archiving according to this checklist in later phases | +| Phase 0: Data-layer baseline inventory | 10% | 100% | Done | GORM query observability, `mpp_db_*` metrics, dashboard query-plan audit script, `pg_stat_statements`, database baseline audit script, PostgreSQL exporter table-level health and 24h row-growth panels, and read/write consistency classification | Continue implementing code routing, partitioning, and archiving according to this checklist in later phases | | Phase 1: Single-database connection pool, indexing, pagination, and lifecycle governance | 15% | 100% | Done | backend/publish-worker/collab-service application connection pools, Redis client connection pool, PgBouncer writer pool, composite indexes, keyset list pagination, list queries avoiding the large `source_content` field, event/session history retention periods, and R2/S3 cold-event archive worker | None; Phase 2 is complete, and later work moves into Phase 3/4 read replicas, partitioning, and recovery flows | | Phase 2: Read models and cache first | 15% | 100% | Done | Redis and Asynq dependencies are reusable; admin dashboard stats, admin project list, and dashboard account summary have short-TTL Redis cache; stats/project list/account cache misses are merged with singleflight; project, prepublish, publish, and account write paths invalidate the related dashboard cache; `workspace_dashboard_stats` and `project_list_summaries` read models are in place, and APIs prefer read models when coverage is complete; async refresh triggers after project save, platform sync, publish completion, and member changes; admin rebuild API, Asynq queue, and worker support full read-model rebuild | None | | Phase 3: Read/write splitting | 15% | 100% | Done | Optional `DB_READER_*` connection, application-level DB Router, signed sticky writer, consistency routing for project/stats/workspace/platform_account/publish/prepublish/mediaasset/browser_session/extension, consistency-level inventories for dashboard/publish/collab-service, self-hosted PostgreSQL read replica, managed `postgres-reader` entry point, PgBouncer reader pool, replica lag monitoring and automatic fallback to writer when over threshold | None; Phase 4 continues partitioning, archiving, and recovery flows | | Phase 4: Single-database partitioning, archiving, and hot/cold tiering | 15% | 15% | In progress | Collaborative editing already has state + update batch + compaction foundation; event and terminal-session history already have row-level R2/S3 archive worker | Time partitioning for event tables, hash partitioning for collaborative batches, cold partition export, recovery flow | -| Phase 5: Citus preparation | 20% | 5% | Not started | Workspace model, `projects.workspace_id`, and personal workspace ID already exist | Global `workspace_id`, Citus distribution column/colocation design, unique constraint and foreign-key review, migration rehearsal | -| Phase 6: Citus distributed PostgreSQL operation | 10% | 0% | Deferred | None | Citus shadow cluster, small-tenant migration rehearsal, worker/coordinator monitoring and backup, large-tenant isolation strategy | +| Phase 5: Citus preparation | 20% | 5% | Not started | Workspace model, `projects.workspace_id`, and personal workspace ID already exist | Global `workspace_id`, Citus distribution column/colocation design, unique constraint and foreign-key review | +| Phase 6: Citus distributed PostgreSQL operation | 10% | 0% | Deferred | None | Future Citus cluster design, worker/coordinator monitoring and backup, large-tenant isolation strategy | ### 0.1 Progress Update Rules @@ -42,8 +42,8 @@ Current overall progress = Σ(phase weight * phase current completion) Phase completion can be estimated from the checklist, but must be adjusted by acceptance quality: - Design-only work without code, configuration, scripts, or a verification entry point can count for at most `20%` of that phase. -- Implementation without monitoring, rollback, or rebuild paths can count for at most `60%` of that phase. -- A phase can be marked `Done` only when implementation, verification entry point, rollback/rebuild method, and documentation status are all updated. +- Implementation without monitoring or rebuild paths can count for at most `60%` of that phase. +- A phase can be marked `Done` only when implementation, verification entry point, rebuild method where applicable, and documentation status are all updated. - A `Deferred` phase does not gain completion just because it is intentionally not being done; only trigger conditions are recorded. Atomic commit guidance: @@ -67,8 +67,7 @@ Atomic commit guidance: | Event-table partitioning and archiving | In progress | `publish_events`, `extension_execution_events`, `project_activities`, `workspace_activities`, and terminal `remote_browser_sessions` have default retention periods; the `archive` worker can batch-export JSONL to R2/S3 and delete old hot-table rows after successful upload | Time partitioning, cold partition export, and recovery flow are not implemented | `backend/internal/services/archive/config.go`, `backend/internal/services/archive/worker.go`, `backend/internal/services/archive/worker_test.go` | | Collaboration batch governance | In progress | `collab_document_states`, `collab_document_update_batches`, and compaction/retention foundations exist | `document_id` hash partitioning and cold archiving are not implemented | `backend/internal/models/collab.go`, `collab-service/src/persistence/document-persistence.ts` | | Outbox/CDC/event stream | In progress | The publishing queue path has a transactional Outbox: `EnqueuePublishProject` writes `outbox_events` in the same transaction and dispatches immediately after commit; publish worker starts an outbox dispatcher and supports retries for failed/stale processing records; Asynq continues to serve as the task-execution queue, and `PublishEvent` continues to serve as publishing audit | Currently covers only `publish.job_requested`; general business-event outbox, Debezium, and Redpanda/Kafka CDC are not implemented | `backend/internal/services/publish/queue.go`, `backend/internal/services/publish/outbox.go`, `backend/internal/models/models.go` | -| Citus target state | Not started | Confirmed `workspace_id` as the most suitable distribution-column direction | Citus distributed tables, reference tables, colocation, and migration rehearsal are not implemented | Phase 5/6 in this document | -| Complex DDL migration rules | Done | Versioned migration rules are defined, including complex DDL trigger conditions, script naming, expand/contract flow, rollback, and verification requirements | Before the first complex DDL, choose and implement either `goose` or Atlas as the executor | Phase 0 complex DDL versioned migration rules in this document | +| Citus target state | Not started | Confirmed `workspace_id` as the most suitable distribution-column direction | Citus distributed tables, reference tables, and colocation are not implemented | Phase 5/6 in this document | ### 0.3 Phase Checklist @@ -80,7 +79,6 @@ Atomic commit guidance: - [x] Add a baseline audit script for query fingerprints, table sizes, index sizes, and dead tuples. - [x] Build dashboards for table row count, 24h row growth, table size, index size, dead tuples, vacuum state. Verification entry point: `deploy/docker/observability/postgres-exporter/queries.yml`, `deploy/docker/observability/grafana/dashboards/mpp-observability-baseline.json`. - [x] Mark DB calls in dashboard, publish, and collab-service with consistency levels. Verification entry point: Phase 0 consistency-level inventory in this document. -- [x] Define versioned migration rules for complex DDL. Verification entry point: Phase 0 complex DDL versioned migration rules in this document. #### Phase 1: Single-database Connection Pool, Indexing, Pagination, and Lifecycle Governance @@ -133,13 +131,11 @@ Atomic commit guidance: - [ ] Design Citus distributed tables, reference tables, and colocated table groups. - [ ] Review unique constraints, foreign keys, and cross-tenant joins. - [ ] Add `workspace_id` to worker payloads. -- [ ] Write a shadow migration and rollback flow from single-database PostgreSQL to Citus. #### Phase 6: Citus Distributed PostgreSQL Operation -- [ ] Set up a Citus shadow cluster. -- [ ] Import test workspaces or new workspaces for validation. -- [ ] Rehearse migration of one low-risk existing workspace. +- [ ] Set up a Citus validation cluster when horizontal scaling is actually needed. +- [ ] Import synthetic or new workspaces for validation. - [ ] Configure PgBouncer, monitoring, backup, and failure drills for Citus coordinator/workers. - [ ] Govern worker concurrency by `workspace_id`. - [ ] Define large-tenant isolation strategy using an independent Citus cluster or independent PostgreSQL. @@ -234,9 +230,9 @@ Core principles: | Collaborative update batch growth | PostgreSQL hash partition by `document_id` + compaction + retention | The unique key of `collab_document_update_batches` contains `document_id`, and document-hash partitioning matches the load path better | Partitioning by `created_at`, which breaks document-local uniqueness and loading efficiency | | Cold-data archive | R2/S3 + Parquet/JSONL + background archive worker | The project already points toward R2/S3 object storage, and archived data remains traceable offline | Deleting historical events with no recovery path | | Multi-consumer event stream | Outbox + Debezium + Redpanda/Kafka | Introduce only when notifications, auditing, search, recommendations, and data warehouse all consume events | Introducing Kafka/Pulsar directly at the current publishing-queue stage | -| Horizontal table/database splitting | Citus distributed PostgreSQL + application-level workspace data-access boundary | MPP is tenant-oriented SaaS, `workspace_id` can serve as a stable distribution column, and Citus preserves the PostgreSQL ecosystem while reducing custom shard-routing cost | Vitess/MySQL migration; ShardingSphere transparent proxy as the first choice | +| Horizontal table/database splitting | Citus distributed PostgreSQL + application-level workspace data-access boundary | MPP is tenant-oriented SaaS, `workspace_id` can serve as a stable distribution column, and Citus preserves the PostgreSQL ecosystem while reducing custom shard-routing cost | Rewriting around Vitess/MySQL; ShardingSphere transparent proxy as the first choice | | Full-text search | Start with PostgreSQL full-text/trigram, later OpenSearch | Early content search can reuse PostgreSQL; split out when search becomes an independent product capability | Adding OpenSearch early for simple title search | -| Online migration governance | Versioned migration tool `goose` or Atlas + `CREATE INDEX CONCURRENTLY` | Current GORM AutoMigrate fits early-stage simple models; partitioning/sharding/online indexes require explicit migration scripts | Continuing to rely on AutoMigrate for complex DDL | +| Schema initialization | GORM AutoMigrate for undeployed/local environments | The project has no production dataset yet, so the current priority is a clean target schema | Keeping startup-time legacy compatibility code and compatibility DDL | ## 6. Progressive Phase Roadmap @@ -252,7 +248,7 @@ Work items: - Build table-level growth dashboards: row count, table size, index size, dead tuples, and vacuum state. - Enable `pg_stat_statements` for PostgreSQL and align SQL hashes with the existing GORM QueryObserver hashes. - Review whether all high-frequency queries carry stable filters such as `workspace_id`, `user_id`, `project_id`, and `document_id`. -- Define versioned migration rules; complex DDL no longer depends only on GORM AutoMigrate. +- Keep the target schema clean; before real deployment, prefer direct model/schema changes over compatibility code. Acceptance criteria: @@ -288,7 +284,7 @@ dashboard / publish / collab-service DB-call consistency inventory: | dashboard | Platform account connection state, account grants, account credentials / cookie reads | `read-your-write` | Cache miss and grant checks continue using `StrongRead`; publishing credentials must not use Redis display cache | `backend/internal/services/platform_account/service.go` | | publish | Pre-publish reads for project, publication, platform account, and media resource availability | `read-your-write` | Uses DB Router `StrongRead`/sticky writer; publishing credentials, media references, and publication state machine do not use replicas | `backend/internal/services/publish/service.go`, `backend/internal/services/publish/media_refs.go` | | publish | Publish queueing, idempotency checks, publish event, outbox event, publication status update, retry/claim | `writer` | Uses DB Router `Writer`; transactional outbox and state machine must not use reader | `backend/internal/services/publish/queue.go`, `backend/internal/services/publish/outbox.go` | -| publish | Publishing history, low-priority audit reports, growth analysis | `analytics-read` | Not currently a separate read model; can migrate to reader/archive query after Phase 4/Outbox | `backend/internal/models/models.go` | +| publish | Publishing history, low-priority audit reports, growth analysis | `analytics-read` | Not currently a separate read model; can move to reader/archive query after Phase 4/Outbox | `backend/internal/models/models.go` | | prepublish | Pre-read for project, publication, and draft sync | `read-your-write` | Uses DB Router `StrongRead` to avoid syncing stale content before replica catches up after user save | `backend/internal/services/prepublish/drafts.go` | | prepublish | Draft sync, publication update, and cache invalidation trigger | `writer` | Uses DB Router `Writer`; transaction reads do not use replicas | `backend/internal/services/prepublish/drafts.go` | | mediaasset | Upload completion, deletion, object-storage status update | `writer` | Uses DB Router `Writer`; metadata state changes do not use replicas | `backend/internal/services/mediaasset/assets.go` | @@ -298,20 +294,13 @@ dashboard / publish / collab-service DB-call consistency inventory: | extension | Prepublish candidate list display | `eventual-read` | Uses DB Router `EventualRead`, allows short delay, and is protected by replica-lag fallback | `backend/internal/services/extension/handoffs.go` | | collab-service | Reading state and update batches when opening a document | `read-your-write` | Current node-postgres pool connects only to writer; before introducing reader, latest flush for the same document must be visible | `collab-service/src/persistence/document-persistence.ts` | | collab-service | Initialize document state, append update batch, `FOR UPDATE` seq lock, store compacted state, sync project source content, prune compacted batches | `writer` | All currently go to writer; transactions, locks, writes, and compaction forbid reader | `collab-service/src/persistence/document-persistence.ts` | -| collab-service | Offline collaboration batch growth analysis, archive validation, rebuild rehearsal | `analytics-read` | After Phase 4, can execute on reader or archive replica; must not affect online flush | Phase 4 in this document | +| collab-service | Offline collaboration batch growth analysis, archive validation, rebuild checks | `analytics-read` | After Phase 4, can execute on reader or archive replica; must not affect online flush | Phase 4 in this document | -Complex DDL versioned migration rules: +Pre-deployment schema rule: -| Rule | Requirement | -| ---- | ----------- | -| Trigger conditions | Any partitioning, table/column rename, column type rewrite, non-null/unique/foreign-key constraint, bulk backfill, `CREATE INDEX CONCURRENTLY`, large-table delete/archive, Citus distribution-column change, or colocate change must use versioned migration and cannot depend only on GORM AutoMigrate. | -| Tool and directory | Phase 0 completes the rule definition; before the first complex DDL, choose either `goose` or Atlas and put scripts into one migration directory. Naming uses `YYYYMMDDHHMMSS__.up.sql` / `.down.sql` or the equivalent format for the selected tool. | -| expand/contract | Split by default into expand, backfill, verify, contract. expand only adds backward-compatible structures; backfill is batched, rate-limited, and resumable; contract runs only after all application versions no longer read the old structure. | -| Locks and timeouts | Each migration explicitly sets `lock_timeout` and `statement_timeout`; large-table indexes use `CREATE INDEX CONCURRENTLY`; PostgreSQL `CONCURRENTLY` statements must not be placed inside a transaction block. | -| Constraint rollout | Large-table constraints prefer `NOT VALID` first, then `VALIDATE CONSTRAINT` after validation passes; `NOT NULL` requires backfill first, then a check constraint or tightening during a maintenance window. | -| Idempotency and rollback | Scripts must include existence checks or repeat-execution protection; `.down.sql` must at least roll back metadata changes. Irreversible data deletion must first archive to R2/S3 and declare the recovery path in migration notes. | -| Verification entry point | Migration PRs must include `EXPLAIN` or `script/db/audit_dashboard_query_plans.sh` results, before/after summaries from `script/db/audit_database_baseline.sh`, estimated lock time, and rollback plan. | -| AutoMigrate boundary | AutoMigrate is kept only for early simple model sync and local development; production complex DDL is run by versioned migration first, and application startup migration must not handle large-table rewrites, partitioning, concurrent indexes, or dangerous constraints. | +- The project has not been deployed with real customer data yet, so database changes should target the clean desired schema directly. +- Keep startup schema initialization focused on the desired current schema. +- Keep `AutoMigrate` as the local/development schema initializer before real deployment policy exists. Benefits: @@ -321,7 +310,7 @@ Benefits: Costs: - Requires adding database-side observability and a small amount of query annotation. -- The team needs discipline around migration scripts. +- The team needs discipline to avoid carrying compatibility code before it is needed. ### Phase 1: Single-database Connection Pool, Indexing, Pagination, and Lifecycle Governance @@ -342,13 +331,13 @@ Recommended technology: - PgBouncer. - PostgreSQL `pg_stat_statements`, `VACUUM`/`ANALYZE` monitoring. -- `goose` or Atlas for complex migrations. +- GORM AutoMigrate for local/undeployed schema initialization. - R2/S3 archive. Benefits: - Improves stability without changing the broad business-code structure. -- Reduces migration risk for later read/write splitting and sharding. +- Keeps the target schema straightforward before real deployment. Costs: @@ -452,8 +441,8 @@ Work items: - Partition `remote_browser_sessions` by `created_at` or `expires_at`, and archive completed historical sessions by retention period. - Prefer hash partitioning `collab_document_update_batches` by `document_id`, because the query path loads by document and sorts by seq. - Keep compaction for collaborative update batches: `collab_document_states` stores compacted state, and old batches are kept only for the auditable window. -- Build an archive flow: after a partition freezes, export it to R2/S3, then detach/drop the partition. -- Write a clear recovery flow for each archived data domain: import from object storage into a temporary table without polluting hot tables directly. +- Build an archive flow: after data leaves the hot window, export it to R2/S3, then delete or detach cold data according to the chosen storage layout. +- Write a clear restore flow for each archived data domain when archived records need inspection. Recommended technology: @@ -471,7 +460,7 @@ Benefits: Costs: - Unique constraints on partitioned tables must include the partition key, so existing unique indexes need redesign. -- GORM AutoMigrate is not suitable for directly managing complex partitioning; explicit migrations are required. +- GORM AutoMigrate is not suitable for directly managing complex partitioning; design partition DDL separately when this phase becomes real. - Queries must include time or tenant filters, otherwise they still scan many partitions. ### Phase 5: Citus Preparation @@ -483,26 +472,25 @@ When to apply: - One workspace or a few large customers become clear hotspots that affect other tenants. - Single-database backup/restore time already exceeds the business RTO. -Goal: first make the code and data model satisfy Citus distributed-table constraints, then actually move into distributed PostgreSQL. +Goal: shape the code and data model so a future Citus deployment is possible without rewriting business logic. Work items: -- Confirm `workspace_id` is required on all project-domain tables; tables without it need to add it or derive it stably from `project_id`. +- Confirm `workspace_id` is required on all project-domain tables before production schema freezes. - Use `workspace_id` as the Citus distribution column, and colocate projects, publishing targets, publishing events, project activities, comments, versions, and media metadata as much as possible. - Revisit unique constraints and foreign keys: unique constraints on distributed tables must include the distribution column, and cross-distribution-column foreign keys and cross-tenant joins must not be core paths. - Design small tables such as `users`, global configuration, and platform dictionaries as reference tables or keep them in the control domain; whether `workspaces` and `workspace_members` should be reference tables depends on scale and permission-query paths. -- `platform_accounts` is currently user + platform scoped, so first keep it as control-domain data; if accounts later become workspace assets, add `workspace_id` and move them into the colocated table group. +- `platform_accounts` is currently user + platform scoped, so first keep it as control-domain data; if accounts later become workspace assets, model them directly in the colocated table group. - Add `workspace_id` to every asynchronous task payload; workers must not locate data by `project_id` alone. - Change repository/service DB entry points to `ForWorkspace(ctx, workspaceID)` or an equivalent data-access boundary, so Citus/single-database differences are not scattered through business code. - Define cross-tenant restrictions: no cross-workspace transactions; global statistics are generated through read models, CDC, or offline warehouse. -- Define migration from single-database PostgreSQL to Citus: shadow cluster, data replication, row-count and checksum validation, read-only gray rollout, write freeze, final cutover, and rollback window. Recommended technology: - Citus coordinator + worker nodes. - Citus colocated distributed tables, with `workspace_id` preferred as the distribution column. - Reference tables for small global dimension tables. -- Application-level workspace data-access boundary for expressing tenant context, read/write consistency, and migration gray rollout; it is not the primary sharding implementation. +- Application-level workspace data-access boundary for expressing tenant context and read/write consistency; it is not the primary sharding implementation. Benefits: @@ -512,24 +500,24 @@ Benefits: Costs: -- This is the highest-cost phase: distributed-table DDL, unique constraints, foreign keys, migration, backup, monitoring, and failure recovery all become more complex. +- This is the highest-cost phase: distributed-table DDL, unique constraints, foreign keys, backup, monitoring, and failure recovery all become more complex. - Cross-workspace queries, admin statistics, and admin lists need read models or asynchronous aggregation. - If a single workspace becomes a super-hotspot, Citus can expand overall multi-tenant capacity but cannot automatically spread one tenant's write hotspot infinitely; large tenants or business hotspots need separate isolation. ### Phase 6: Citus Distributed PostgreSQL Operation -When to apply: after Phase 5 Citus constraint refactoring, task routing, migration tooling, and rollback rehearsals are complete, migrate the first non-core small tenant. +When to apply: after Phase 5 Citus constraint refactoring and task routing are complete, and new growth clearly requires horizontal write capacity. -Goal: gradually move from single-database PostgreSQL into a Citus coordinator + worker nodes distributed PostgreSQL runtime. +Goal: operate Citus as a future PostgreSQL-compatible distributed runtime for new or synthetic workspaces first. Work items: -- Set up a Citus shadow cluster first, and import a batch of new or test workspaces. -- Choose one low-risk existing workspace for migration rehearsal, validating colocated joins, publishing-state transitions, collaborative editing, and dashboard read models. +- Set up a Citus validation cluster when horizontal scaling is actually needed. +- Import synthetic or new workspaces for validation, checking colocated joins, publishing-state transitions, collaborative editing, and dashboard read models. - Configure PgBouncer, backups, monitoring, capacity thresholds, and failure drills separately for Citus coordinator and worker nodes. - Apply Asynq worker concurrency limits by `workspace_id` and Citus cluster capacity, so single-tenant tasks do not overwhelm workers or the database. - Build cross-tenant admin read models: tenant list, project count, publish count, and capacity watermarks. -- Build a large-tenant isolation strategy: when necessary, migrate large workspaces to an independent Citus cluster or independent PostgreSQL. +- Build a large-tenant isolation strategy: when necessary, place large workspaces on an independent Citus cluster or independent PostgreSQL. Recommended technology: @@ -555,7 +543,7 @@ Costs: | Query type | Default route | Example | Notes | | ---------- | ------------- | ------- | ----- | | Write transaction | Writer | Create project, save content, sync platform draft, advance publishing state | All reads and writes inside the transaction stay on writer | -| Immediate read after write | Writer or sticky writer | Refresh details after saving a project, poll status after publishing | Avoid UI state rollback caused by replica lag | +| Immediate read after write | Writer or sticky writer | Refresh details after saving a project, poll status after publishing | Avoid stale UI state caused by replica lag | | Delay-tolerant business read | Reader | Project list, historical publishing events, workspace activities | Must tolerate second-level delay | | Statistical read | Read model / Reader | dashboard totals, success/failure statistics | Prefer read model, then reader | | Audit archive read | Archive / offline | Publishing events beyond retention period | Does not query hot tables by default | @@ -571,11 +559,11 @@ Reasons: - Project and publishing flows inside one workspace need local transactions; cross-workspace transactions should not become core paths. - Citus colocated distributed tables require a stable distribution column. Designing around `workspace_id` keeps common joins inside the same workspace within one colocated table group. -Compatibility strategy: +Pre-deployment modeling strategy: - Personal user projects belong to personal workspaces, using `PersonalWorkspaceID(userID)`. -- Historical tables that only have `user_id` should backfill `workspace_id` through project or personal workspace before migration. -- `platform_accounts` is currently user + platform scoped and is better kept in the control domain first; if accounts later become workspace assets, migrate them into a colocated table group distributed by `workspace_id`. +- New project-domain tables should include `workspace_id` from the beginning when they need tenant-local querying. +- `platform_accounts` is currently user + platform scoped and is better kept in the control domain first; if accounts later become workspace assets, model them in a colocated table group distributed by `workspace_id`. ### 7.3 Table Partitioning Recommendations @@ -610,12 +598,12 @@ Trigger conditions for introducing Redpanda/Kafka: | Phase | Main benefits | Main costs | Risks | | ----- | ------------- | ---------- | ----- | | Phase 0 | See real bottlenecks and avoid mistaken decomposition | Add monitoring and classification | Metrics definitions are inconsistent | -| Phase 1 | Improve single-database stability and control connections/slow SQL | PgBouncer and migration discipline | Misconfigured connection pool causes cascading failure | +| Phase 1 | Improve single-database stability and control connections/slow SQL | PgBouncer and lifecycle discipline | Misconfigured connection pool causes cascading failure | | Phase 2 | Significantly reduce high-frequency dashboard read pressure | Read-model delay and rebuild logic | Read model briefly inconsistent with fact tables | -| Phase 3 | Horizontally scale read traffic | Complex read-after-write consistency | Replica lag makes UI state roll back | +| Phase 3 | Horizontally scale read traffic | Complex read-after-write consistency | Replica lag makes UI state stale | | Phase 4 | Reduce hot-table and index bloat | Partition DDL and archive flow | Wrong partition key makes queries slower | -| Phase 5 | Data model becomes Citus-ready | Distribution column, constraints, migration, testing complexity | Cross-tenant transactions enter core paths accidentally | -| Phase 6 | Horizontal scaling within PostgreSQL ecosystem | Highest Citus operational cost | Distributed-table migration rollback and global statistics are complex | +| Phase 5 | Data model becomes Citus-ready | Distribution column, constraints, testing complexity | Cross-tenant transactions enter core paths accidentally | +| Phase 6 | Horizontal scaling within PostgreSQL ecosystem | Highest Citus operational cost | Distributed-table operation and global statistics are complex | ## 9. Recommended Execution Order @@ -623,7 +611,7 @@ Short term: 1. Build database observability and query classification. 2. Add PostgreSQL-side metrics and `pg_stat_statements`. -3. Move complex DDL out of GORM AutoMigrate into versioned migrations. +3. Keep schema initialization clean and remove legacy compatibility paths before deployment. 4. Design dashboard read models and Redis cache strategy. 5. Define retention and archive strategy for event tables and collaborative update batches. @@ -640,5 +628,5 @@ Long term: 1. Complete `workspace_id` across all domains. 2. Design Citus distributed tables and reference tables. 3. Workspace-aware worker concurrency governance. -4. Small-tenant migration rehearsal. -5. Citus production cluster cutover. +4. Validate Citus with synthetic or new workspaces before any real tenant data exists. +5. Decide whether Citus production operation is still justified. diff --git a/script/kubernetes/smoke-test/checks/defaults.go b/script/kubernetes/smoke-test/checks/defaults.go index ae36c79ce..5935a84f5 100644 --- a/script/kubernetes/smoke-test/checks/defaults.go +++ b/script/kubernetes/smoke-test/checks/defaults.go @@ -40,6 +40,8 @@ var requiredConfigKeys = []string{ "REDIS_TLS", "REDIS_TLS_CA_CERT", "REDIS_TLS_CA_FILE", + "REDIS_TLS_CERT_FILE", + "REDIS_TLS_KEY_FILE", "REDIS_TLS_SERVER_NAME", "REDIS_SENTINEL_ADDRS", "REDIS_SENTINEL_MASTER_NAME", diff --git a/script/kubernetes/smoke-test/checks/fixtures.go b/script/kubernetes/smoke-test/checks/fixtures.go index 5f5188f07..30ff1b0ac 100644 --- a/script/kubernetes/smoke-test/checks/fixtures.go +++ b/script/kubernetes/smoke-test/checks/fixtures.go @@ -138,6 +138,8 @@ func DryRunConfigMap() Object { "REDIS_TLS": "true", "REDIS_TLS_CA_CERT": "", "REDIS_TLS_CA_FILE": "", + "REDIS_TLS_CERT_FILE": "", + "REDIS_TLS_KEY_FILE": "", "REDIS_TLS_SERVER_NAME": "", "REDIS_SENTINEL_ADDRS": "", "REDIS_SENTINEL_MASTER_NAME": "mpp-redis-ha", diff --git a/script/kubernetes/validation/app_baseline.rb b/script/kubernetes/validation/app_baseline.rb index e53f612ee..b8fe61231 100644 --- a/script/kubernetes/validation/app_baseline.rb +++ b/script/kubernetes/validation/app_baseline.rb @@ -47,6 +47,8 @@ module AppBaseline "REDIS_TLS", "REDIS_TLS_CA_CERT", "REDIS_TLS_CA_FILE", + "REDIS_TLS_CERT_FILE", + "REDIS_TLS_KEY_FILE", "REDIS_TLS_SERVER_NAME", "REDIS_SENTINEL_ADDRS", "REDIS_SENTINEL_MASTER_NAME",