feat(search): explicit modes, match provenance, and strict filtering

open_index/brain.py

@@ -268,9 +268,16 @@ def search(
         min_confidence: float = 0.0,
         as_of: Optional[str] = None,
         source: Optional[str] = None,
+        mode: str = "hybrid",
+        filters: Optional[dict[str, Any]] = None,
     ) -> SearchResults:
         """Search, optionally filtered by trust and by validity window.
 
+        `mode` is "hybrid" (default), "keyword" or "semantic", and decides which
+        candidates exist rather than merely how they are weighted. `filters` is
+        an exact-match predicate pushed into the backend query; filtering on a
+        field not declared `filterable` raises.
+
         `min_confidence` drops claims that cannot clear the floor, INCLUDING
         unattributed ones — see `Entity.trusted`. `as_of` drops claims whose
         validity window excludes that instant. Both default to off so existing
@@ -284,7 +291,8 @@ def search(
         started = perf_counter()
         try:
             results = self.backend.search(query, doc_types, limit, counts_only,
-                                          semantic_weight=semantic_weight)
+                                          semantic_weight=semantic_weight,
+                                          mode=mode, filters=filters)
         except Exception as exc:
             if source:
                 self._record_fetch(

open_index/config.py

@@ -187,6 +187,11 @@ def doc_type_to_yaml_dict(dt: DocType) -> dict:
         }
         if f.required:
             entry["required"] = True
+        # Emitted only when set, like `required` — but it must be emitted, or a
+        # doc_type written back loses the flag and every filter on that field is
+        # then refused as undeclared.
+        if f.filterable:
+            entry["filterable"] = True
         if f.description:
             entry["description"] = f.description
         fields.append(entry)

open_index/mcp_server.py

@@ -99,15 +99,40 @@ def search_brain(
         query: Optional[str] = None,
         doc_types: Optional[list[str]] = None,
         limit: int = 20,
+        mode: str = "hybrid",
+        filters: Optional[dict[str, Any]] = None,
     ) -> str:
-        """Search the brain. `query` is free text; `doc_types` optionally filters
-        to specific concepts (e.g. ["product", "issue"]). Returns matching
-        entities ranked by relevance, plus per-doc_type counts."""
+        """Search the brain.
+
+        `query` is free text. `doc_types` narrows to specific concepts
+        (e.g. ["product", "issue"]).
+
+        `mode` chooses how matching works:
+          hybrid    (default) keyword matches plus the nearest by meaning
+          keyword   literal term matching only — use when the query is an exact
+                    name, code or identifier and a near-miss is not acceptable
+          semantic  meaning only — use when the right words may not appear in
+                    the document at all
+
+        `filters` is an exact-match constraint, e.g. {"tenant_id": "acme"}. It
+        is a hard predicate, not a ranking hint: non-matching documents cannot
+        appear at any score. Only fields declared `filterable` can be used, and
+        filtering on any other field is an error rather than being ignored — so
+        a filter never silently fails open.
+
+        Each result carries `match`, saying why it came back: type is
+        "keyword", "semantic", "both", "filter" or "none", with the normalised
+        score from each arm.
+        """
         results = brain.search(
-            query=query, doc_types=doc_types, limit=limit, source="mcp"
+            query=query, doc_types=doc_types, limit=limit, source="mcp",
+            mode=mode, filters=filters,
         )
         return json.dumps(
             {
+                "query": query,
+                "mode": mode,
+                "filters": filters or {},
                 "total": results.total,
                 "doc_type_counts": results.doc_type_counts,
                 "results": results.results,
@@ -352,6 +377,12 @@ def create_doc_type(
                 boost      — number > 0, default 1. Search weight: a hit in a
                              boost-6 field outranks a boost-1 hit 6-to-1.
                 required   — true to reject entities missing it
+                filterable — true to allow exact filtering on this field via
+                             search_brain(filters=...). Set it on anything that
+                             identifies *whose* data a document is (a tenant,
+                             account or user id): filtering is refused on fields
+                             that do not declare it, so that a filter can never
+                             fail open.
                 Convention: a high-boost `name` field, plus a `description` field
                 with search "semantic" so entities are findable by meaning.
             relationships: The edge vocabulary for this type, each

open_index/schema.py

@@ -78,6 +78,13 @@ class FieldSpec(BaseModel):
     boost: float = 1.0
     required: bool = False
     description: Optional[str] = None
+    # Opt-in to exact filtering (`filters={"tenant_id": "acme"}`), which is a
+    # hard predicate in the backend query rather than anything ranked. Opt-in
+    # because it is a promise: a filterable field is indexed for equality and
+    # filtering on a field that is not declared filterable raises instead of
+    # quietly returning everything. That failure mode is the whole point when a
+    # filter is carrying a tenant or user boundary.
+    filterable: bool = False
 
     @field_validator("boost")
     @classmethod

open_index/storage/base.py

@@ -72,6 +72,13 @@ def iter_semantic_entities(
             yield entity
 
 
+# The three ways a search can be run. `mode` is not sugar over semantic_weight:
+# it decides which candidates exist at all. Weighting alone would let a document
+# that matched no keyword into a keyword search at score 0, which is a wrong
+# answer rather than a low-ranked one.
+SEARCH_MODES = ("hybrid", "keyword", "semantic")
+
+
 @dataclass
 class SearchResults:
     total: int
@@ -81,6 +88,85 @@ class SearchResults:
     limited: bool = False
 
 
+# Every result dict carries `match`, so a caller (and anyone debugging an
+# agent's memory) can see *why* a document came back rather than only how highly
+# it ranked:
+#
+#   {"type": "keyword" | "semantic" | "both" | "filter" | "none",
+#    "keyword_score": float,     # 0.0 when this arm did not match
+#    "semantic_score": float}    # cosine rescaled to [0, 1]
+#
+# "semantic" means the document was among the nearest by vector — NOT that its
+# cosine was above zero, which is true of very nearly every embedded document
+# and would label the entire index a semantic match.
+#
+# "filter" and "none" are the honest answers when nothing was ranked at all: a
+# pure filter, or a plain listing. Calling those a keyword match at score 0
+# would misreport why the document came back, which is the one thing this field
+# exists to get right.
+
+
+def _match_info(keyword: bool, semantic: bool, keyword_score: float,
+                semantic_score: float, filtered: bool = False) -> dict[str, Any]:
+    """The `match` block for one result."""
+    if keyword and semantic:
+        kind = "both"
+    elif keyword:
+        kind = "keyword"
+    elif semantic:
+        kind = "semantic"
+    else:
+        kind = "filter" if filtered else "none"
+    return {
+        "type": kind,
+        "keyword_score": round(float(keyword_score), 3),
+        "semantic_score": round(float(semantic_score), 3),
+    }
+
+
+def resolve_filters(
+    doc_types_map: dict[str, DocType],
+    filters: Optional[dict[str, Any]],
+    scope: Optional[list[str]] = None,
+) -> list[tuple[str, Any]]:
+    """Validate a strict filter against the schema, or raise.
+
+    Fails closed on purpose. A filter is the one search input that may be
+    carrying a security boundary — a tenant, a user, an account — and the
+    dangerous outcome is not an error but a silent one: a typo'd or undeclared
+    field that quietly matches nothing, and so filters nothing, and returns the
+    whole index.
+
+    A field must be declared `filterable: true` on at least one doc_type in
+    scope. Entities of a type that does not carry the field never match, which
+    falls out of the backends comparing a missing value: this is deliberate, and
+    is why a filter cannot leak across doc_types that do not model it.
+    """
+    if not filters:
+        return []
+
+    in_scope = [dt for name, dt in doc_types_map.items()
+                if not scope or name in scope]
+    allowed = {
+        f.name
+        for dt in in_scope
+        for f in dt.fields
+        if f.filterable
+    }
+
+    unknown = [k for k in filters if k not in allowed]
+    if unknown:
+        known = ", ".join(sorted(allowed)) or "none"
+        raise ValueError(
+            "cannot filter on " + ", ".join(f"{k!r}" for k in sorted(unknown))
+            + f" — filterable fields here are: {known}. "
+            "Declare `filterable: true` on the field in its doc_type to filter "
+            "by it. (Refusing rather than ignoring: a filter that is silently "
+            "dropped returns everything.)"
+        )
+    return sorted(filters.items())
+
+
 @runtime_checkable
 class SearchBackend(Protocol):
     """Persist entities + their relationships, and search over them."""
@@ -126,10 +212,27 @@ def search(
         limit: int = 20,
         counts_only: bool = False,
         semantic_weight: Optional[float] = None,
+        mode: str = "hybrid",
+        filters: Optional[dict[str, Any]] = None,
     ) -> SearchResults:
-        """Search entities. `semantic_weight` overrides the config's
-        search.semantic_weight for this call: 0.0 = keyword-only,
-        1.0 = semantic-only, None = the brain's configured default."""
+        """Search entities.
+
+        `mode` selects which candidates exist:
+            hybrid    keyword hits UNION the nearest by vector (the default)
+            keyword   keyword hits only
+            semantic  nearest by vector only
+
+        `semantic_weight` blends the two arms *within* hybrid: 0.0 leans
+        keyword, 1.0 leans semantic, None uses the brain's configured default.
+        It does not decide membership — that is `mode`, so that a keyword search
+        cannot return something which matched no keyword.
+
+        `filters` is an exact-match predicate applied in the query itself, never
+        after ranking. Every field in it must be declared `filterable: true`;
+        see `resolve_filters`, which raises rather than ignoring an unknown one.
+
+        Each result dict carries `match` — see the note above SearchResults.
+        """
         ...
 
     def counts(self) -> dict[str, int]: