[ACC-2342] Add SearchFilter for FTS on content attributes

contentgrid-appserver-application-model/src/main/java/com/contentgrid/appserver/application/model/Entity.java

@@ -6,6 +6,8 @@
 import com.contentgrid.appserver.application.model.attributes.ContentAttribute;
 import com.contentgrid.appserver.application.model.attributes.SimpleAttribute;
 import com.contentgrid.appserver.application.model.attributes.SimpleAttribute.Type;
+import com.contentgrid.appserver.application.model.attributes.flags.SyntheticAttributeFlag;
+import com.contentgrid.appserver.application.model.attributes.flags.IgnoredFlag;
 import com.contentgrid.appserver.application.model.attributes.flags.ReadOnlyFlag;
 import com.contentgrid.appserver.application.model.exceptions.AttributeNotFoundException;
 import com.contentgrid.appserver.application.model.exceptions.DuplicateElementException;
@@ -17,6 +19,7 @@
 import com.contentgrid.appserver.application.model.i18n.TranslatableImpl;
 import com.contentgrid.appserver.application.model.i18n.TranslationBuilderSupport;
 import com.contentgrid.appserver.application.model.i18n.UnconfigurableTranslatable;
+import com.contentgrid.appserver.application.model.searchfilters.FullTextSearchContentAttributeSearchFilter;
 import com.contentgrid.appserver.application.model.searchfilters.SearchFilter;
 import com.contentgrid.appserver.application.model.sortable.SortableField;
 import com.contentgrid.appserver.application.model.values.AttributeName;
@@ -197,6 +200,24 @@ public static class ConfigurableEntityTranslations implements EntityTranslations
                 }
         );
         this.attributes.remove(this.primaryKey.getName());
+
+        // Look for any FullTextSearchContentAttributeSearchFilter defined on a content attribute of this entity.
+        // If these exist, create an attribute on the entity that will be used to store the content text.
+        searchFilters.forEach(
+                searchFilter -> {
+                    if (searchFilter instanceof FullTextSearchContentAttributeSearchFilter ftsContentFilter) {
+                        // Create a hidden attribute on this entity that will be used to store the extracted text.
+                        String attributeName = ftsContentFilter.getHiddenTextAttributeFormattedName();
+                        Attribute hiddenTextAttribute = SimpleAttribute.builder()
+                                .name(AttributeName.of(attributeName))
+                                .column(ColumnName.of(attributeName))
+                                .type(Type.TEXT)
+                                .flag(SyntheticAttributeFlag.INSTANCE)
+                                .build();
+                        this.attributes.put(AttributeName.of(attributeName), hiddenTextAttribute);
+                    }
+                }
+        );
     }
 
     /**

contentgrid-appserver-application-model/src/main/java/com/contentgrid/appserver/application/model/attributes/flags/SyntheticAttributeFlag.java

@@ -0,0 +1,10 @@
+package com.contentgrid.appserver.application.model.attributes.flags;
+
+/**
+ * Marks an attribute as <i>synthetic</i>; generated internally for the benefit of the application itself
+ * <p>
+ * Synthetic attributes are not part of the application model, and are always hidden as well
+ */
+public interface SyntheticAttributeFlag extends IgnoredFlag {
+    SyntheticAttributeFlag INSTANCE = attribute -> {};
+}

contentgrid-appserver-application-model/src/main/java/com/contentgrid/appserver/application/model/searchfilters/FullTextSearchContentAttributeSearchFilter.java

@@ -0,0 +1,77 @@
+package com.contentgrid.appserver.application.model.searchfilters;
+
+import com.contentgrid.appserver.application.model.attributes.ContentAttribute;
+import com.contentgrid.appserver.application.model.i18n.ConfigurableTranslatable;
+import com.contentgrid.appserver.application.model.i18n.TranslatableImpl;
+import com.contentgrid.appserver.application.model.i18n.TranslationBuilderSupport;
+import com.contentgrid.appserver.application.model.searchfilters.flags.SearchFilterFlag;
+import com.contentgrid.appserver.application.model.values.AttributeName;
+import com.contentgrid.appserver.application.model.values.FilterName;
+import com.contentgrid.appserver.application.model.values.PropertyPath;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NonNull;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * FullTextSearchContentAttributeSearchFilter is a search filter that performs full-text search against
+ * a specific content attribute.
+ * <br>
+ * It needs to be different from the {@link FullTextSearchAttributeSearchFilter} because it uses
+ * custom logic to resolve the content attribute to the correct database table with the extracted
+ * text when querying.
+ */
+@Getter
+public class FullTextSearchContentAttributeSearchFilter extends BaseAttributeSearchFilter implements LocaleAwareSearchFilter {
+    public static final String HIDDEN_EXTRACTED_TEXT_ATTRIBUTE_FORMAT = "%s__cg_text";
+
+    @NonNull private final Locale locale;
+
+    @Builder
+    public FullTextSearchContentAttributeSearchFilter(
+            @NonNull FilterName name,
+            @NonNull ConfigurableTranslatable<SearchFilterTranslations, ConfigurableSearchFilterTranslations> translations,
+            @NonNull PropertyPath attributePath,
+            @NonNull Set<SearchFilterFlag> flags,
+            @NonNull Locale locale) {
+        super(name, translations, attributePath, flags);
+
+        this.locale = locale;
+    }
+
+    public static @NonNull FullTextSearchContentAttributeSearchFilterBuilder builder() {
+        return new FullTextSearchContentAttributeSearchFilterBuilder()
+                .translations(new TranslatableImpl<>(ConfigurableSearchFilterTranslations::new))
+                .flags(Set.of());
+    }
+
+    public String getHiddenTextAttributeFormattedName() {
+        List<String> path = this.getAttributePath().toList();
+        // Format the paths from x.y.z to x_y_z so they can be used to construct
+        // the attribute name. However, if a path is formatted like
+        // some.path_to.attribute, and another attribute is formatted like
+        // some.path.to.attribute, both will end up with the same generated name for
+        // the extracted text attribute. To get around this, we can replace all underscores in each
+        // path element's name with two underscores. This way, the paths above become
+        // some_path__to_attribute and some_path_to_attribute.
+        String formattedPath = path.stream()
+                .map(element -> element.replace("_", "__"))
+                .collect(Collectors.joining("_"));
+        return HIDDEN_EXTRACTED_TEXT_ATTRIBUTE_FORMAT.formatted(formattedPath);
+    }
+
+    public static class FullTextSearchContentAttributeSearchFilterBuilder extends TranslationBuilderSupport<SearchFilterTranslations, ConfigurableSearchFilterTranslations, FullTextSearchContentAttributeSearchFilterBuilder> {
+        {
+            getTranslations = () -> translations;
+        }
+
+        public FullTextSearchContentAttributeSearchFilterBuilder attribute(@NonNull ContentAttribute attribute) {
+            this.attributePath = PropertyPath.of(attribute.getName());
+            return this;
+        }
+    }
+}

contentgrid-appserver-application-model/src/test/java/com/contentgrid/appserver/application/model/EntityTest.java

@@ -2,6 +2,7 @@
 
 import static org.junit.jupiter.api.Assertions.*;
 
+import com.contentgrid.appserver.application.model.attributes.Attribute;
 import com.contentgrid.appserver.application.model.attributes.CompositeAttribute;
 import com.contentgrid.appserver.application.model.attributes.CompositeAttributeImpl;
 import com.contentgrid.appserver.application.model.attributes.ContentAttribute;
@@ -16,6 +17,7 @@
 import com.contentgrid.appserver.application.model.i18n.UserLocales;
 import com.contentgrid.appserver.application.model.searchfilters.AttributeSearchFilter;
 import com.contentgrid.appserver.application.model.searchfilters.AttributeSearchFilter.Operation;
+import com.contentgrid.appserver.application.model.searchfilters.FullTextSearchContentAttributeSearchFilter;
 import com.contentgrid.appserver.application.model.searchfilters.SearchFilter;
 import com.contentgrid.appserver.application.model.sortable.SortableField;
 import com.contentgrid.appserver.application.model.values.ApplicationName;
@@ -30,6 +32,8 @@
 import com.contentgrid.appserver.application.model.values.TableName;
 import java.util.List;
 import java.util.Locale;
+import java.util.Optional;
+
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
@@ -495,4 +499,26 @@ void entity_translations() {
         assertEquals("Couleur", entity.getTranslations(Locale.FRENCH).getSingularName());
         assertNull(entity.getTranslations(Locale.FRENCH).getPluralName());
     }
+
+    @Test
+    void entity_createsHiddenTextAttribute() {
+        FullTextSearchContentAttributeSearchFilter filter = FullTextSearchContentAttributeSearchFilter
+                .builder()
+                .name(FilterName.of("file-fts-content"))
+                .locale(Locale.ENGLISH)
+                .attribute(CONTENT1)
+                .build();
+        var entity = Entity.builder()
+                .name(EntityName.of("file"))
+                .pathSegment(PathSegmentName.of("files"))
+                .linkName(LinkName.of("file"))
+                .table(TableName.of("file"))
+                .attribute(CONTENT1)
+                .searchFilter(filter)
+                .build();
+        String attrName = filter.getHiddenTextAttributeFormattedName();
+        Optional<Attribute> attribute = entity.getAttributeByName(AttributeName.of(attrName));
+        assertTrue(attribute.isPresent());
+        assertTrue(attribute.get().isIgnored());
+    }
 }

contentgrid-appserver-domain/src/main/java/com/contentgrid/appserver/domain/ThunkExpressionGenerator.java

@@ -8,7 +8,9 @@
 import com.contentgrid.appserver.application.model.searchfilters.AttributeSearchFilter;
 import com.contentgrid.appserver.application.model.searchfilters.BaseAttributeSearchFilter;
 import com.contentgrid.appserver.application.model.searchfilters.FullTextSearchAttributeSearchFilter;
+import com.contentgrid.appserver.application.model.searchfilters.FullTextSearchContentAttributeSearchFilter;
 import com.contentgrid.appserver.application.model.searchfilters.SearchFilter;
+import com.contentgrid.appserver.application.model.values.AttributeName;
 import com.contentgrid.appserver.application.model.values.AttributePath;
 import com.contentgrid.appserver.application.model.values.FilterName;
 import com.contentgrid.appserver.application.model.values.PropertyPath;
@@ -47,13 +49,17 @@ static ThunkExpression<Boolean> from(Application application, Entity entity, Map
 
             SearchFilter searchFilter = maybeSearchFilter.get();
 
-            // currently only handle attribute search filters
             if (searchFilter instanceof BaseAttributeSearchFilter attributeSearchFilter) {
-                var attribute = application.resolvePropertyPath(entity, attributeSearchFilter.getAttributePath());
+                PropertyPath propertyPath = attributeSearchFilter.getAttributePath();
+                // FTS content search filter targets a different attribute than is specified in its filter spec
+                if (searchFilter instanceof FullTextSearchContentAttributeSearchFilter ftsSearchFilter) {
+                    propertyPath = PropertyPath.of(AttributeName.of(ftsSearchFilter.getHiddenTextAttributeFormattedName()));
+                }
+                var attribute = application.resolvePropertyPath(entity, propertyPath);
                 List<PathElement> pathElements;
 
                 try {
-                    pathElements = convertPath(application, entity, attributeSearchFilter.getAttributePath());
+                    pathElements = convertPath(application, entity, propertyPath);
                 } catch (IllegalArgumentException e) {
                     throw new InvalidParameterException(entity.getName().getValue(), entry.getKey(),
                             attribute.getType(), entry.getValue().toString(), e);
@@ -121,6 +127,7 @@ private static ThunkExpression<Boolean> createExpression(BaseAttributeSearchFilt
         SymbolicReference attr = SymbolicReference.of(Variable.named("entity"), pathElements);
 
         if (filter instanceof FullTextSearchAttributeSearchFilter ftsSearchFilter) return createExpression(ftsSearchFilter, attr, value);
+        if (filter instanceof FullTextSearchContentAttributeSearchFilter ftsContentSearchFilter) return createExpression(ftsContentSearchFilter, attr, value);
         if (filter instanceof AttributeSearchFilter attrSearchFilter) return createExpression(attrSearchFilter, attr, value);
 
         throw new IllegalArgumentException("Received unknown filter type (%s).".formatted(filter.getClass().getName()));
@@ -143,6 +150,12 @@ private static ThunkExpression<Boolean> createExpression(FullTextSearchAttribute
         return StringComparison.contentGridFullTextSearchMatch(attr, value.assertResultType(String.class), filter.getLocale());
     }
 
+    private static ThunkExpression<Boolean> createExpression(FullTextSearchContentAttributeSearchFilter filter,
+                                                             SymbolicReference attr,
+                                                             Scalar<?> value) {
+        return StringComparison.contentGridFullTextSearchMatch(attr, value.assertResultType(String.class), filter.getLocale());
+    }
+
     private static List<PathElement> convertPath(Application application, Entity entity, PropertyPath path) {
         List<PathElement> pathElements = new ArrayList<>();
         Entity currentEntity = entity;

contentgrid-appserver-domain/src/test/java/com/contentgrid/appserver/domain/ThunkExpressionGeneratorTest.java

@@ -8,6 +8,7 @@
 import com.contentgrid.appserver.application.model.Entity;
 import com.contentgrid.appserver.application.model.attributes.CompositeAttribute;
 import com.contentgrid.appserver.application.model.attributes.CompositeAttributeImpl;
+import com.contentgrid.appserver.application.model.attributes.ContentAttribute;
 import com.contentgrid.appserver.application.model.attributes.SimpleAttribute;
 import com.contentgrid.appserver.application.model.attributes.SimpleAttribute.Type;
 import com.contentgrid.appserver.application.model.attributes.flags.ReadOnlyFlag;
@@ -19,6 +20,7 @@
 import com.contentgrid.appserver.application.model.relations.SourceOneToOneRelation;
 import com.contentgrid.appserver.application.model.searchfilters.AttributeSearchFilter;
 import com.contentgrid.appserver.application.model.searchfilters.AttributeSearchFilter.Operation;
+import com.contentgrid.appserver.application.model.searchfilters.FullTextSearchContentAttributeSearchFilter;
 import com.contentgrid.appserver.application.model.values.ApplicationName;
 import com.contentgrid.appserver.application.model.values.AttributeName;
 import com.contentgrid.appserver.application.model.values.ColumnName;
@@ -30,6 +32,7 @@
 import com.contentgrid.appserver.application.model.values.RelationName;
 import com.contentgrid.appserver.application.model.values.TableName;
 import com.contentgrid.appserver.exception.InvalidParameterException;
+import com.contentgrid.appserver.query.engine.api.thunx.expression.StringComparison;
 import com.contentgrid.thunx.predicates.model.Comparison;
 import com.contentgrid.thunx.predicates.model.FunctionExpression.Operator;
 import com.contentgrid.thunx.predicates.model.LogicalOperation;
@@ -41,6 +44,7 @@
 import java.time.Instant;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.UUID;
 import org.junit.jupiter.api.Test;
@@ -110,6 +114,16 @@ class ThunkExpressionGeneratorTest {
                     .build())
             .build();
 
+    private static final ContentAttribute CONTENT_ATTR = ContentAttribute.builder()
+            .name(AttributeName.of("content"))
+            .pathSegment(PathSegmentName.of("content"))
+            .linkName(LinkName.of("content"))
+            .idColumn(ColumnName.of("content__id"))
+            .filenameColumn(ColumnName.of("content__filename"))
+            .mimetypeColumn(ColumnName.of("content__mimetype"))
+            .lengthColumn(ColumnName.of("content__length"))
+            .build();
+
     private static final Entity testEntity = Entity.builder()
             .name(EntityName.of("testEntity"))
             .table(TableName.of("test_entity"))
@@ -122,6 +136,7 @@ class ThunkExpressionGeneratorTest {
             .attribute(TEXT_ATTR)
             .attribute(DATETIME_ATTR)
             .attribute(COMP_ATTR)
+            .attribute(CONTENT_ATTR)
             .searchFilter(AttributeSearchFilter.builder()
                     .operation(Operation.EXACT)
                     .name(FilterName.of("count"))
@@ -227,6 +242,11 @@ class ThunkExpressionGeneratorTest {
                     .name(FilterName.of("shipment.destination"))
                     .attributePath(PropertyPath.of(RelationName.of("shipment"), AttributeName.of("destination")))
                     .build())
+            .searchFilter(FullTextSearchContentAttributeSearchFilter.builder()
+                    .name(FilterName.of("content~fts"))
+                    .attribute(CONTENT_ATTR)
+                    .locale(Locale.ENGLISH)
+                    .build())
             .build();
 
     private static final Entity shipmentEntity = Entity.builder()
@@ -764,4 +784,27 @@ void acrossManyToManyRelationAttributeIsValid() {
         );
         assertEquals(Scalar.of("A unicorn"), comparison.getRightTerm());
     }
+
+    @Test
+    void contentFullTextSearch() {
+        Map<String, List<String>> params = Map.of("content~fts", List.of("I am your father"));
+        var entity = testApplication.getEntityByName(EntityName.of("testEntity")).orElseThrow();
+        ThunkExpression<Boolean> result = ThunkExpressionGenerator.from(testApplication, entity, params);
+
+        assertInstanceOf(StringComparison.ContentGridFullTextSearch.class, result);
+        StringComparison.ContentGridFullTextSearch comparison = (StringComparison.ContentGridFullTextSearch) result;
+
+        assertEquals(Locale.ENGLISH, comparison.getLocale());
+        // Check that left term of comparison is the correct reference to the hidden text attribute
+        String textAttrName = FullTextSearchContentAttributeSearchFilter.HIDDEN_EXTRACTED_TEXT_ATTRIBUTE_FORMAT.formatted(CONTENT_ATTR.getName());
+        assertEquals(
+                SymbolicReference.of(
+                        Variable.named("entity"),
+                        SymbolicReference.path(textAttrName)
+                ),
+                comparison.getLeftTerm()
+        );
+        // Right term should be the search term
+        assertEquals(Scalar.of("I am your father"), comparison.getRightTerm());
+    }
 }