ACC-2624 Proposal of deletion policy / document lifecycle

CLAUDE.md

@@ -0,0 +1,5 @@
+# contentgrid-appserver
+
+## Testing
+
+Always run `./gradlew check` before considering work done.

contentgrid-appserver-autoconfigure/build.gradle

@@ -9,6 +9,8 @@ dependencies {
     implementation 'com.contentgrid.thunx:thunx-autoconfigure'
 
     compileOnly project(':contentgrid-appserver-application-model')
+    compileOnly project(':contentgrid-appserver-content-lifecycle')
+    compileOnly 'io.micrometer:micrometer-core'
     compileOnly project(':contentgrid-appserver-actuators')
     compileOnly project(':contentgrid-appserver-contentstore-api')
     compileOnly project(':contentgrid-appserver-contentstore-impl-encryption')

contentgrid-appserver-autoconfigure/src/main/java/com/contentgrid/appserver/autoconfigure/domain/ContentGridDomainAutoConfiguration.java

@@ -2,6 +2,8 @@
 
 import com.contentgrid.appserver.application.model.Application;
 import com.contentgrid.appserver.autoconfigure.events.ContentGridEventsAutoConfiguration;
+import com.contentgrid.appserver.autoconfigure.lifecycle.ContentLifecycleAutoConfiguration;
+import com.contentgrid.appserver.content.lifecycle.ContentReferenceTracker;
 import com.contentgrid.appserver.contentstore.api.ContentStore;
 import com.contentgrid.appserver.domain.ContentApi;
 import com.contentgrid.appserver.domain.ContentApiImpl;
@@ -18,7 +20,7 @@
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.context.annotation.Bean;
 
-@AutoConfiguration(after={ContentGridEventsAutoConfiguration.class})
+@AutoConfiguration(after={ContentGridEventsAutoConfiguration.class, ContentLifecycleAutoConfiguration.class})
 @ConditionalOnClass({DatamodelApiImpl.class})
 public class ContentGridDomainAutoConfiguration {
 
@@ -45,8 +47,8 @@ public void dispatchDelete(Application application, EntityInstance instance) {}
 
     @Bean
     DatamodelApiImpl datamodelApi(QueryEngine queryEngine, ContentStore contentStore, DomainEventDispatcher dispatcher,
-            CursorCodec cursorCodec, Clock clock) {
-        return new DatamodelApiImpl(queryEngine, contentStore, dispatcher, cursorCodec, clock);
+            CursorCodec cursorCodec, Clock clock, ContentReferenceTracker contentReferenceTracker) {
+        return new DatamodelApiImpl(queryEngine, contentStore, dispatcher, cursorCodec, clock, contentReferenceTracker);
     }
 
     @Bean

contentgrid-appserver-autoconfigure/src/main/java/com/contentgrid/appserver/autoconfigure/lifecycle/ContentLifecycleAutoConfiguration.java

@@ -0,0 +1,61 @@
+package com.contentgrid.appserver.autoconfigure.lifecycle;
+
+import com.contentgrid.appserver.application.model.values.ApplicationName;
+import com.contentgrid.appserver.autoconfigure.json.schema.ApplicationResolverAutoConfiguration;
+import com.contentgrid.appserver.content.lifecycle.ContentDeletionJob;
+import com.contentgrid.appserver.content.lifecycle.ContentLifecycleProperties;
+import com.contentgrid.appserver.content.lifecycle.ContentReferenceTracker;
+import com.contentgrid.appserver.content.lifecycle.ContentReferenceVerificationQuery;
+import com.contentgrid.appserver.content.lifecycle.DeferredContentReferenceTracker;
+import com.contentgrid.appserver.content.lifecycle.JooqContentReferenceTracker;
+import com.contentgrid.appserver.content.lifecycle.JooqContentReferenceVerificationQuery;
+import com.contentgrid.appserver.contentstore.api.ContentStore;
+import com.contentgrid.appserver.registry.ApplicationResolver;
+import io.micrometer.core.instrument.MeterRegistry;
+import org.jooq.DSLContext;
+import org.springframework.boot.autoconfigure.AutoConfiguration;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Primary;
+
+@AutoConfiguration(after = ApplicationResolverAutoConfiguration.class)
+@ConditionalOnClass(JooqContentReferenceTracker.class)
+@EnableConfigurationProperties(ContentLifecycleProperties.class)
+public class ContentLifecycleAutoConfiguration {
+
+    @Bean
+    JooqContentReferenceTracker jooqContentReferenceTracker(DSLContext dslContext) {
+        return new JooqContentReferenceTracker(dslContext);
+    }
+
+    @Bean
+    @Primary
+    ContentReferenceTracker contentReferenceTracker(JooqContentReferenceTracker jooqTracker) {
+        return new DeferredContentReferenceTracker(jooqTracker);
+    }
+
+    @Bean
+    ContentReferenceVerificationQuery contentReferenceVerificationQuery(DSLContext dslContext) {
+        return new JooqContentReferenceVerificationQuery(dslContext);
+    }
+
+    @Bean
+    @ConditionalOnBean({ApplicationResolver.class, ContentStore.class, MeterRegistry.class})
+    ContentDeletionJob contentDeletionJob(
+            DSLContext dslContext,
+            ContentStore contentStore,
+            ContentReferenceVerificationQuery verificationQuery,
+            ApplicationResolver applicationResolver,
+            MeterRegistry meterRegistry,
+            ContentLifecycleProperties properties) {
+        return new ContentDeletionJob(
+                dslContext,
+                contentStore,
+                verificationQuery,
+                applicationResolver.resolve(ApplicationName.of("default")),
+                meterRegistry,
+                properties);
+    }
+}

contentgrid-appserver-autoconfigure/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports

@@ -1,3 +1,4 @@
+com.contentgrid.appserver.autoconfigure.lifecycle.ContentLifecycleAutoConfiguration
 com.contentgrid.appserver.autoconfigure.contentstore.EncryptedContentStoreAutoConfiguration
 com.contentgrid.appserver.autoconfigure.contentstore.FilesystemContentStoreAutoConfiguration
 com.contentgrid.appserver.autoconfigure.contentstore.S3ContentStoreAutoConfiguration

contentgrid-appserver-autoconfigure/src/test/java/com/contentgrid/appserver/autoconfigure/rest/ContentGridRestAutoConfigurationTest.java

@@ -7,6 +7,7 @@
 import com.contentgrid.appserver.autoconfigure.contentstore.FilesystemContentStoreAutoConfiguration;
 import com.contentgrid.appserver.autoconfigure.domain.ContentGridDomainAutoConfiguration;
 import com.contentgrid.appserver.autoconfigure.events.ContentGridEventsAutoConfiguration;
+import com.contentgrid.appserver.autoconfigure.lifecycle.ContentLifecycleAutoConfiguration;
 import com.contentgrid.appserver.autoconfigure.query.engine.JOOQQueryEngineAutoConfiguration;
 import com.contentgrid.appserver.registry.ApplicationResolver;
 import com.contentgrid.appserver.registry.SingleApplicationResolver;
@@ -42,6 +43,8 @@ class ContentGridRestAutoConfigurationTest {
                     JOOQQueryEngineAutoConfiguration.class,
                     // autoconfiguration for content store
                     FilesystemContentStoreAutoConfiguration.class,
+                    // autoconfiguration for content lifecycle
+                    ContentLifecycleAutoConfiguration.class,
                     // autoconfiguration for domain
                     ContentGridDomainAutoConfiguration.class,
                     ContentGridEventsAutoConfiguration.class,

contentgrid-appserver-content-lifecycle/README.md

@@ -0,0 +1,46 @@
+# contentgrid-appserver-content-lifecycle
+
+Tracks content object references and performs safe, grace-period-based deletion from the underlying `ContentStore`.
+
+## How it works
+
+- A `_content_references` table tracks a reference count for every content object.
+- When content is uploaded, `ContentReferenceTracker.incrementReference()` is called.
+- When content is cleared from an entity (set to null, replaced, or entity deleted), `ContentReferenceTracker.decrementReference()` is called **after the entity transaction commits** (via `DeferredContentReferenceTracker`).
+- When the reference count reaches zero, the row is marked with `marked_for_deletion_at = NOW()`.
+- `ContentDeletionJob` (an `ApplicationRunner`) processes candidates past the grace period, performs a safety verification query, then deletes from the `ContentStore` and removes the tracking row.
+
+## Running the deletion job
+
+The `ContentDeletionJob` implements Spring Boot's `ApplicationRunner`. It is intended to be invoked as a **K8s CronJob** using a separate Spring Boot process. Configure the grace period and batch size in `application.properties`:
+
+```properties
+contentgrid.appserver.content.lifecycle.deletion.grace-period=P7D
+contentgrid.appserver.content.lifecycle.deletion.batch-size=100
+```
+
+## Backfill for existing data
+
+When deploying this module to an existing database, the `_content_references` table will be empty. Content that was already uploaded before this feature was introduced will never be deleted, which is safe (conservative). If you want to reclaim storage for content that was already dereferenced, a manual backfill is required.
+
+**Chosen approach: manual operation** — no automatic Flyway migration is provided. The backfill is complex and tenant-specific (depends on the entity/column layout), and running it automatically could be disruptive. Perform the backfill as a one-time manual SQL operation if needed.
+
+### Backfill SQL pattern
+
+The following pattern inserts a row into `_content_references` for every content object currently referenced in any entity table. Adjust table and column names to match your application model:
+
+```sql
+INSERT INTO _content_references (content_id, reference_count, first_referenced_at)
+SELECT content_id, COUNT(*) AS reference_count, NOW() AS first_referenced_at
+FROM (
+    SELECT invoice_content_id AS content_id FROM invoice WHERE invoice_content_id IS NOT NULL
+    UNION ALL
+    SELECT document_file_id  AS content_id FROM document  WHERE document_file_id  IS NOT NULL
+    -- ... add one UNION ALL per content attribute per entity table
+) AS all_refs
+GROUP BY content_id
+ON CONFLICT (content_id) DO UPDATE
+    SET reference_count = EXCLUDED.reference_count;
+```
+
+Run this in a maintenance window before enabling the deletion job for the first time.

contentgrid-appserver-content-lifecycle/build.gradle

@@ -0,0 +1,47 @@
+plugins {
+    id 'java-library'
+    id 'maven-publish'
+    id 'io.freefair.lombok'
+}
+
+dependencies {
+    api project(':contentgrid-appserver-contentstore-api')
+    implementation project(':contentgrid-appserver-application-model')
+    implementation project(':contentgrid-appserver-query-engine-api')
+    implementation 'org.jooq:jooq'
+    implementation 'org.springframework:spring-tx'
+    implementation 'org.springframework.boot:spring-boot-autoconfigure'
+    implementation 'io.micrometer:micrometer-core'
+    implementation 'org.slf4j:slf4j-api'
+    runtimeOnly 'org.postgresql:postgresql'
+
+    annotationProcessor "org.jooq:jooq-checker:3.20.11"
+    testAnnotationProcessor "org.jooq:jooq-checker:3.20.11"
+
+    testImplementation 'org.springframework.boot:spring-boot-starter-test'
+    testImplementation 'org.springframework.boot:spring-boot-starter-jooq'
+    testImplementation 'org.assertj:assertj-core'
+    testImplementation 'org.mockito:mockito-junit-jupiter'
+    testImplementation testFixtures(project(':contentgrid-appserver-application-model'))
+    testImplementation testFixtures(project(':contentgrid-appserver-contentstore-impl-utils'))
+}
+
+tasks.withType(JavaCompile) {
+    options.fork = true
+    options.compilerArgs += [
+            '-processor', 'org.jooq.checker.PlainSQLChecker' +
+            ',lombok.launch.AnnotationProcessorHider$AnnotationProcessor'
+    ]
+    options.forkOptions.jvmArgs += [
+            "--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED",
+            "--add-exports=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED",
+            "--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED",
+            "--add-exports=jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED",
+            "--add-exports=jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED",
+            "--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED",
+            "--add-exports=jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED",
+            "--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED",
+            "--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED",
+            "--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED"
+    ]
+}

contentgrid-appserver-content-lifecycle/gradle.properties

@@ -0,0 +1 @@
+description=Content lifecycle tracking and deletion for the Appserver

contentgrid-appserver-content-lifecycle/src/main/java/com/contentgrid/appserver/content/lifecycle/ContentDeletionJob.java

@@ -0,0 +1,101 @@
+package com.contentgrid.appserver.content.lifecycle;
+
+import com.contentgrid.appserver.application.model.Application;
+import com.contentgrid.appserver.contentstore.api.ContentReference;
+import com.contentgrid.appserver.contentstore.api.ContentStore;
+import io.micrometer.core.instrument.Counter;
+import io.micrometer.core.instrument.MeterRegistry;
+import java.time.OffsetDateTime;
+import lombok.extern.slf4j.Slf4j;
+import org.jooq.DSLContext;
+import org.springframework.boot.ApplicationArguments;
+import org.springframework.boot.ApplicationRunner;
+
+/**
+ * Deletes content objects from the {@link ContentStore} that have been marked for deletion and whose grace period
+ * has elapsed. Intended to be invoked as a K8s CronJob via Spring Boot's {@link ApplicationRunner} mechanism.
+ *
+ * <p>For each candidate:
+ * <ol>
+ *   <li>Performs a safety check to confirm the content is truly unreferenced (drift detection).
+ *   <li>If unreferenced: deletes from the content store and removes the tracking row.
+ *   <li>If still referenced (drift): clears the deletion mark and records a drift metric.
+ *   <li>On any failure: records a failure metric and continues to the next candidate.
+ * </ol>
+ */
+@Slf4j
+public class ContentDeletionJob implements ApplicationRunner {
+
+    private final DSLContext dslContext;
+    private final ContentStore contentStore;
+    private final ContentReferenceVerificationQuery verificationQuery;
+    private final Application application;
+    private final ContentLifecycleProperties properties;
+    private final Counter successCounter;
+    private final Counter failureCounter;
+    private final Counter driftCounter;
+
+    public ContentDeletionJob(
+            DSLContext dslContext,
+            ContentStore contentStore,
+            ContentReferenceVerificationQuery verificationQuery,
+            Application application,
+            MeterRegistry meterRegistry,
+            ContentLifecycleProperties properties) {
+        this.dslContext = dslContext;
+        this.contentStore = contentStore;
+        this.verificationQuery = verificationQuery;
+        this.application = application;
+        this.properties = properties;
+        this.successCounter = Counter.builder("content.deletion.success")
+                .description("Number of content objects successfully deleted")
+                .register(meterRegistry);
+        this.failureCounter = Counter.builder("content.deletion.failure")
+                .description("Number of content deletion attempts that failed")
+                .register(meterRegistry);
+        this.driftCounter = Counter.builder("content.deletion.drift")
+                .description("Number of content objects marked for deletion but found to still be referenced")
+                .register(meterRegistry);
+    }
+
+    @Override
+    public void run(ApplicationArguments args) {
+        var cutoff = OffsetDateTime.now().minus(properties.getDeletion().getGracePeriod());
+
+        var candidates = dslContext
+                .select(ContentReferenceTable.CONTENT_ID)
+                .from(ContentReferenceTable.TABLE)
+                .where(ContentReferenceTable.MARKED_FOR_DELETION_AT.lessOrEqual(cutoff))
+                .limit(properties.getDeletion().getBatchSize())
+                .fetch(ContentReferenceTable.CONTENT_ID);
+
+        log.info("Processing {} content deletion candidates", candidates.size());
+
+        for (var contentId : candidates) {
+            processDeletionCandidate(ContentReference.of(contentId));
+        }
+    }
+
+    private void processDeletionCandidate(ContentReference ref) {
+        try {
+            if (verificationQuery.isReferenced(application, ref)) {
+                dslContext.update(ContentReferenceTable.TABLE)
+                        .set(ContentReferenceTable.MARKED_FOR_DELETION_AT, (OffsetDateTime) null)
+                        .where(ContentReferenceTable.CONTENT_ID.eq(ref.getValue()))
+                        .execute();
+                driftCounter.increment();
+                log.warn("Drift detected: content {} is still referenced; clearing deletion mark", ref.getValue());
+            } else {
+                contentStore.remove(ref);
+                dslContext.deleteFrom(ContentReferenceTable.TABLE)
+                        .where(ContentReferenceTable.CONTENT_ID.eq(ref.getValue()))
+                        .execute();
+                successCounter.increment();
+                log.debug("Deleted content {}", ref.getValue());
+            }
+        } catch (Exception e) {
+            failureCounter.increment();
+            log.error("Failed to process content deletion for {}", ref.getValue(), e);
+        }
+    }
+}

contentgrid-appserver-content-lifecycle/src/main/java/com/contentgrid/appserver/content/lifecycle/ContentLifecycleProperties.java

@@ -0,0 +1,30 @@
+package com.contentgrid.appserver.content.lifecycle;
+
+import java.time.Duration;
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+
+@ConfigurationProperties("contentgrid.appserver.content.lifecycle")
+@Getter
+@Setter
+public class ContentLifecycleProperties {
+
+    private Deletion deletion = new Deletion();
+
+    @Getter
+    @Setter
+    public static class Deletion {
+
+        /**
+         * Grace period after a content object is dereferenced before it becomes eligible for deletion.
+         * Expressed as an ISO-8601 duration (e.g. {@code P7D} for 7 days).
+         */
+        private Duration gracePeriod = Duration.parse("P7D");
+
+        /**
+         * Maximum number of deletion candidates processed per job run.
+         */
+        private int batchSize = 100;
+    }
+}

contentgrid-appserver-content-lifecycle/src/main/java/com/contentgrid/appserver/content/lifecycle/ContentReferenceTable.java

@@ -0,0 +1,22 @@
+package com.contentgrid.appserver.content.lifecycle;
+
+import java.time.OffsetDateTime;
+import org.jooq.Allow;
+import org.jooq.Field;
+import org.jooq.Table;
+import org.jooq.impl.DSL;
+import org.jooq.impl.SQLDataType;
+
+@Allow.PlainSQL
+final class ContentReferenceTable {
+
+    static final Table<?> TABLE = DSL.table("_content_references");
+
+    static final Field<String> CONTENT_ID = DSL.field("content_id", SQLDataType.VARCHAR.nullable(false));
+    static final Field<Integer> REFERENCE_COUNT = DSL.field("reference_count", SQLDataType.INTEGER.nullable(false));
+    static final Field<OffsetDateTime> FIRST_REFERENCED_AT = DSL.field("first_referenced_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(false));
+    static final Field<OffsetDateTime> LAST_DEREFERENCED_AT = DSL.field("last_dereferenced_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(true));
+    static final Field<OffsetDateTime> MARKED_FOR_DELETION_AT = DSL.field("marked_for_deletion_at", SQLDataType.TIMESTAMPWITHTIMEZONE.nullable(true));
+
+    private ContentReferenceTable() {}
+}

contentgrid-appserver-content-lifecycle/src/main/java/com/contentgrid/appserver/content/lifecycle/ContentReferenceTracker.java

@@ -0,0 +1,11 @@
+package com.contentgrid.appserver.content.lifecycle;
+
+import com.contentgrid.appserver.contentstore.api.ContentReference;
+
+public interface ContentReferenceTracker {
+
+    void incrementReference(ContentReference ref);
+
+    void decrementReference(ContentReference ref);
+
+}