diff --git a/open_index/brain.py b/open_index/brain.py index aff0100..0f3deb8 100644 --- a/open_index/brain.py +++ b/open_index/brain.py @@ -268,9 +268,16 @@ def search( min_confidence: float = 0.0, as_of: Optional[str] = None, source: Optional[str] = None, + mode: str = "hybrid", + filters: Optional[dict[str, Any]] = None, ) -> SearchResults: """Search, optionally filtered by trust and by validity window. + `mode` is "hybrid" (default), "keyword" or "semantic", and decides which + candidates exist rather than merely how they are weighted. `filters` is + an exact-match predicate pushed into the backend query; filtering on a + field not declared `filterable` raises. + `min_confidence` drops claims that cannot clear the floor, INCLUDING unattributed ones — see `Entity.trusted`. `as_of` drops claims whose validity window excludes that instant. Both default to off so existing @@ -284,7 +291,8 @@ def search( started = perf_counter() try: results = self.backend.search(query, doc_types, limit, counts_only, - semantic_weight=semantic_weight) + semantic_weight=semantic_weight, + mode=mode, filters=filters) except Exception as exc: if source: self._record_fetch( diff --git a/open_index/config.py b/open_index/config.py index 9ad116e..117ae3c 100644 --- a/open_index/config.py +++ b/open_index/config.py @@ -187,6 +187,11 @@ def doc_type_to_yaml_dict(dt: DocType) -> dict: } if f.required: entry["required"] = True + # Emitted only when set, like `required` — but it must be emitted, or a + # doc_type written back loses the flag and every filter on that field is + # then refused as undeclared. + if f.filterable: + entry["filterable"] = True if f.description: entry["description"] = f.description fields.append(entry) diff --git a/open_index/mcp_server.py b/open_index/mcp_server.py index 196d04e..a51dd91 100644 --- a/open_index/mcp_server.py +++ b/open_index/mcp_server.py @@ -99,15 +99,40 @@ def search_brain( query: Optional[str] = None, doc_types: Optional[list[str]] = None, limit: int = 20, + mode: str = "hybrid", + filters: Optional[dict[str, Any]] = None, ) -> str: - """Search the brain. `query` is free text; `doc_types` optionally filters - to specific concepts (e.g. ["product", "issue"]). Returns matching - entities ranked by relevance, plus per-doc_type counts.""" + """Search the brain. + + `query` is free text. `doc_types` narrows to specific concepts + (e.g. ["product", "issue"]). + + `mode` chooses how matching works: + hybrid (default) keyword matches plus the nearest by meaning + keyword literal term matching only — use when the query is an exact + name, code or identifier and a near-miss is not acceptable + semantic meaning only — use when the right words may not appear in + the document at all + + `filters` is an exact-match constraint, e.g. {"tenant_id": "acme"}. It + is a hard predicate, not a ranking hint: non-matching documents cannot + appear at any score. Only fields declared `filterable` can be used, and + filtering on any other field is an error rather than being ignored — so + a filter never silently fails open. + + Each result carries `match`, saying why it came back: type is + "keyword", "semantic", "both", "filter" or "none", with the normalised + score from each arm. + """ results = brain.search( - query=query, doc_types=doc_types, limit=limit, source="mcp" + query=query, doc_types=doc_types, limit=limit, source="mcp", + mode=mode, filters=filters, ) return json.dumps( { + "query": query, + "mode": mode, + "filters": filters or {}, "total": results.total, "doc_type_counts": results.doc_type_counts, "results": results.results, @@ -352,6 +377,12 @@ def create_doc_type( boost — number > 0, default 1. Search weight: a hit in a boost-6 field outranks a boost-1 hit 6-to-1. required — true to reject entities missing it + filterable — true to allow exact filtering on this field via + search_brain(filters=...). Set it on anything that + identifies *whose* data a document is (a tenant, + account or user id): filtering is refused on fields + that do not declare it, so that a filter can never + fail open. Convention: a high-boost `name` field, plus a `description` field with search "semantic" so entities are findable by meaning. relationships: The edge vocabulary for this type, each diff --git a/open_index/schema.py b/open_index/schema.py index 2d24b0d..c1d6dd8 100644 --- a/open_index/schema.py +++ b/open_index/schema.py @@ -78,6 +78,13 @@ class FieldSpec(BaseModel): boost: float = 1.0 required: bool = False description: Optional[str] = None + # Opt-in to exact filtering (`filters={"tenant_id": "acme"}`), which is a + # hard predicate in the backend query rather than anything ranked. Opt-in + # because it is a promise: a filterable field is indexed for equality and + # filtering on a field that is not declared filterable raises instead of + # quietly returning everything. That failure mode is the whole point when a + # filter is carrying a tenant or user boundary. + filterable: bool = False @field_validator("boost") @classmethod diff --git a/open_index/storage/base.py b/open_index/storage/base.py index 092c013..37f0c6e 100644 --- a/open_index/storage/base.py +++ b/open_index/storage/base.py @@ -72,6 +72,13 @@ def iter_semantic_entities( yield entity +# The three ways a search can be run. `mode` is not sugar over semantic_weight: +# it decides which candidates exist at all. Weighting alone would let a document +# that matched no keyword into a keyword search at score 0, which is a wrong +# answer rather than a low-ranked one. +SEARCH_MODES = ("hybrid", "keyword", "semantic") + + @dataclass class SearchResults: total: int @@ -81,6 +88,85 @@ class SearchResults: limited: bool = False +# Every result dict carries `match`, so a caller (and anyone debugging an +# agent's memory) can see *why* a document came back rather than only how highly +# it ranked: +# +# {"type": "keyword" | "semantic" | "both" | "filter" | "none", +# "keyword_score": float, # 0.0 when this arm did not match +# "semantic_score": float} # cosine rescaled to [0, 1] +# +# "semantic" means the document was among the nearest by vector — NOT that its +# cosine was above zero, which is true of very nearly every embedded document +# and would label the entire index a semantic match. +# +# "filter" and "none" are the honest answers when nothing was ranked at all: a +# pure filter, or a plain listing. Calling those a keyword match at score 0 +# would misreport why the document came back, which is the one thing this field +# exists to get right. + + +def _match_info(keyword: bool, semantic: bool, keyword_score: float, + semantic_score: float, filtered: bool = False) -> dict[str, Any]: + """The `match` block for one result.""" + if keyword and semantic: + kind = "both" + elif keyword: + kind = "keyword" + elif semantic: + kind = "semantic" + else: + kind = "filter" if filtered else "none" + return { + "type": kind, + "keyword_score": round(float(keyword_score), 3), + "semantic_score": round(float(semantic_score), 3), + } + + +def resolve_filters( + doc_types_map: dict[str, DocType], + filters: Optional[dict[str, Any]], + scope: Optional[list[str]] = None, +) -> list[tuple[str, Any]]: + """Validate a strict filter against the schema, or raise. + + Fails closed on purpose. A filter is the one search input that may be + carrying a security boundary — a tenant, a user, an account — and the + dangerous outcome is not an error but a silent one: a typo'd or undeclared + field that quietly matches nothing, and so filters nothing, and returns the + whole index. + + A field must be declared `filterable: true` on at least one doc_type in + scope. Entities of a type that does not carry the field never match, which + falls out of the backends comparing a missing value: this is deliberate, and + is why a filter cannot leak across doc_types that do not model it. + """ + if not filters: + return [] + + in_scope = [dt for name, dt in doc_types_map.items() + if not scope or name in scope] + allowed = { + f.name + for dt in in_scope + for f in dt.fields + if f.filterable + } + + unknown = [k for k in filters if k not in allowed] + if unknown: + known = ", ".join(sorted(allowed)) or "none" + raise ValueError( + "cannot filter on " + ", ".join(f"{k!r}" for k in sorted(unknown)) + + f" — filterable fields here are: {known}. " + "Declare `filterable: true` on the field in its doc_type to filter " + "by it. (Refusing rather than ignoring: a filter that is silently " + "dropped returns everything.)" + ) + return sorted(filters.items()) + + @runtime_checkable class SearchBackend(Protocol): """Persist entities + their relationships, and search over them.""" @@ -126,10 +212,27 @@ def search( limit: int = 20, counts_only: bool = False, semantic_weight: Optional[float] = None, + mode: str = "hybrid", + filters: Optional[dict[str, Any]] = None, ) -> SearchResults: - """Search entities. `semantic_weight` overrides the config's - search.semantic_weight for this call: 0.0 = keyword-only, - 1.0 = semantic-only, None = the brain's configured default.""" + """Search entities. + + `mode` selects which candidates exist: + hybrid keyword hits UNION the nearest by vector (the default) + keyword keyword hits only + semantic nearest by vector only + + `semantic_weight` blends the two arms *within* hybrid: 0.0 leans + keyword, 1.0 leans semantic, None uses the brain's configured default. + It does not decide membership — that is `mode`, so that a keyword search + cannot return something which matched no keyword. + + `filters` is an exact-match predicate applied in the query itself, never + after ranking. Every field in it must be declared `filterable: true`; + see `resolve_filters`, which raises rather than ignoring an unknown one. + + Each result dict carries `match` — see the note above SearchResults. + """ ... def counts(self) -> dict[str, int]: diff --git a/open_index/storage/opensearch_backend.py b/open_index/storage/opensearch_backend.py index 92fc3bb..3103e74 100644 --- a/open_index/storage/opensearch_backend.py +++ b/open_index/storage/opensearch_backend.py @@ -31,8 +31,11 @@ from open_index.schema import DocType from open_index.storage.base import ( NO_EMBEDDING_PROVIDER_WARNING, + SEARCH_MODES, SearchResults, + _match_info, iter_semantic_entities, + resolve_filters, semantic_doc_types, semantic_fields_in_scope, semantic_text_for, @@ -283,11 +286,28 @@ def _search_fields(self, doc_types: Optional[list[str]]) -> list[str]: fields += [f"fields.{name}^{b:g}" for name, b in boosts.items()] return fields + @staticmethod + def _term_filters(pairs: list[tuple[str, Any]]) -> list[dict]: + """Exact-match clauses for a strict filter. + + `.keyword` because the dynamic mapping gives a string field a text arm + for search and a keyword subfield for exact matching; a `term` against + the analysed arm would match tokens rather than the whole value, which + for a tenant id is a wrong answer that looks like a right one. + Non-strings are matched on the field itself, which is already exact. + """ + clauses = [] + for name, value in pairs: + field = f"fields.{name}.keyword" if isinstance(value, str) else f"fields.{name}" + clauses.append({"term": {field: value}}) + return clauses + def build_search_body( self, query: Optional[str], doc_types: Optional[list[str]], limit: int, - counts_only: bool, + counts_only: bool, filter_pairs: Optional[list[tuple[str, Any]]] = None, ) -> dict: filters = [{"terms": {"doc_type": doc_types}}] if doc_types else [] + filters += self._term_filters(filter_pairs or []) if query: must: dict = { "multi_match": { @@ -542,14 +562,21 @@ def reembed(self) -> None: logger.warning("clear_scroll failed: %s", exc) def _build_knn_body( - self, vector: list[float], doc_types: Optional[list[str]], k: int + self, vector: list[float], doc_types: Optional[list[str]], k: int, + filter_pairs: Optional[list[tuple[str, Any]]] = None, ) -> dict: knn_clause: dict[str, Any] = { "vector": vector, "k": k, } - if doc_types: - knn_clause["filter"] = {"terms": {"doc_type": doc_types}} + # The strict filter applies to the vector arm too. Filtering only the + # keyword arm would let a semantic search return exactly the documents + # the filter exists to exclude. + clauses = ([{"terms": {"doc_type": doc_types}}] if doc_types else []) + clauses += self._term_filters(filter_pairs or []) + if clauses: + knn_clause["filter"] = ({"bool": {"filter": clauses}} + if len(clauses) > 1 else clauses[0]) return { "size": k, "query": {"knn": {"embedding": knn_clause}}, @@ -563,8 +590,10 @@ def _warn_no_embedding_provider(self) -> None: def _run_keyword_search( self, query: Optional[str], doc_types: Optional[list[str]], limit: int, counts_only: bool, + filter_pairs: Optional[list[tuple[str, Any]]] = None, ) -> SearchResults: - body = self.build_search_body(query, doc_types, limit, counts_only) + body = self.build_search_body(query, doc_types, limit, counts_only, + filter_pairs) res = self._client.search(index=self.index, body=body) total = res["hits"]["total"]["value"] doc_type_counts = { @@ -574,15 +603,23 @@ def _run_keyword_search( if counts_only: return SearchResults(total=total, results=[], doc_type_counts=doc_type_counts) + hits = res["hits"]["hits"] + top = max((float(h["_score"] or 0.0) for h in hits), default=0.0) results = [] - for h in res["hits"]["hits"]: + for h in hits: src = h["_source"] + raw = float(h["_score"]) if h.get("_score") is not None else 0.0 results.append({ "id": src["id"], "doc_type": src["doc_type"], "name": src.get("name", ""), - "score": float(h["_score"]) if h.get("_score") is not None else 0.0, + "score": raw, "entity": self.doc_to_entity(src).to_json(), + # No query means nothing was ranked: this is a listing or a + # pure filter, and saying "keyword" would misreport it. + "match": _match_info(bool(query), False, + raw / top if top else 0.0, 0.0, + filtered=bool(filter_pairs)), }) return SearchResults( total=total, results=results, doc_type_counts=doc_type_counts, limited=total > limit, @@ -592,37 +629,57 @@ def search( self, query: Optional[str] = None, doc_types: Optional[list[str]] = None, limit: int = 20, counts_only: bool = False, semantic_weight: Optional[float] = None, + mode: str = "hybrid", filters: Optional[dict[str, Any]] = None, ) -> SearchResults: - if counts_only or not query: - return self._run_keyword_search(query, doc_types, limit, counts_only) + if mode not in SEARCH_MODES: + raise ValueError(f"unknown search mode {mode!r} — expected one of " + + ", ".join(SEARCH_MODES)) + # Raises on an undeclared field rather than filtering nothing. + filter_pairs = resolve_filters(self._doc_types, filters, doc_types) + + if counts_only or not query or mode == "keyword": + return self._run_keyword_search(query, doc_types, limit, counts_only, + filter_pairs) w = semantic_weight if semantic_weight is not None else ( self._config.search.semantic_weight if self._config else 0.3) - semantic_scope = w > 0 and semantic_fields_in_scope(self._doc_types, doc_types) + # In semantic mode the vector arm runs whatever the weight says; the + # weight only blends the arms in hybrid. + wants_semantic = mode == "semantic" or w > 0 + semantic_scope = wants_semantic and semantic_fields_in_scope( + self._doc_types, doc_types) provider = self._get_embedding_provider() if semantic_scope else None if provider is None: if semantic_scope: self._warn_no_embedding_provider() - return self._run_keyword_search(query, doc_types, limit, counts_only) + # Semantic was asked for and cannot be served. Falling back to + # keyword is the established behaviour, and the results say + # "keyword" so the caller can see what actually happened. + return self._run_keyword_search(query, doc_types, limit, counts_only, + filter_pairs) # Hybrid search: run the keyword and k-NN queries in parallel, merge by id, # normalize each source by its own max _score, and combine with # search.semantic_weight. The k-NN filter uses doc_types so the merged # candidate set and doc_type_counts stay consistent with the keyword arm. k = max(limit, 50) - kw_body = self.build_search_body(query, doc_types, k, counts_only=False) - kw_res = self._client.search(index=self.index, body=kw_body) - - vector = provider.encode([query])[0] - knn_body = self._build_knn_body(vector, doc_types, k) - knn_res = self._client.search(index=self.index, body=knn_body) - candidates: dict[str, dict] = {} kw_scores: dict[str, float] = {} sem_scores: dict[str, float] = {} - for h in kw_res["hits"]["hits"]: - candidates[h["_id"]] = h["_source"] - kw_scores[h["_id"]] = float(h["_score"]) if h.get("_score") is not None else 0.0 + + # Only hybrid runs the keyword arm; in semantic mode a keyword hit that + # is not near the query has no business in the candidate set. + if mode == "hybrid": + kw_body = self.build_search_body(query, doc_types, k, counts_only=False, + filter_pairs=filter_pairs) + kw_res = self._client.search(index=self.index, body=kw_body) + for h in kw_res["hits"]["hits"]: + candidates[h["_id"]] = h["_source"] + kw_scores[h["_id"]] = float(h["_score"]) if h.get("_score") is not None else 0.0 + + vector = provider.encode([query])[0] + knn_body = self._build_knn_body(vector, doc_types, k, filter_pairs) + knn_res = self._client.search(index=self.index, body=knn_body) for h in knn_res["hits"]["hits"]: candidates[h["_id"]] = h["_source"] sem_scores[h["_id"]] = float(h["_score"]) if h.get("_score") is not None else 0.0 @@ -634,8 +691,10 @@ def search( for eid, src in candidates.items(): kw_norm = kw_scores.get(eid, 0.0) / kw_max if kw_max else 0.0 sem_norm = sem_scores.get(eid, 0.0) / sem_max if sem_max else 0.0 - score = (1 - w) * kw_norm + w * sem_norm - merged.append((score, src)) + score = sem_norm if mode == "semantic" else (1 - w) * kw_norm + w * sem_norm + # Membership, not score: which arm actually retrieved it. + merged.append((score, src, eid in kw_scores, eid in sem_scores, + kw_norm, sem_norm)) merged.sort(key=lambda s: (-s[0], s[1].get("name", ""))) picked = merged[:limit] results = [ @@ -645,8 +704,9 @@ def search( "name": src.get("name", ""), "score": round(score, 3), "entity": self.doc_to_entity(src).to_json(), + "match": _match_info(kw_hit, sem_hit, kw_norm, sem_norm), } - for score, src in picked + for score, src, kw_hit, sem_hit, kw_norm, sem_norm in picked ] doc_type_counts = {} for src in candidates.values(): diff --git a/open_index/storage/sqlite_backend.py b/open_index/storage/sqlite_backend.py index 814823e..a353f77 100644 --- a/open_index/storage/sqlite_backend.py +++ b/open_index/storage/sqlite_backend.py @@ -26,8 +26,11 @@ from open_index.schema import DocType from open_index.storage.base import ( NO_EMBEDDING_PROVIDER_WARNING, + SEARCH_MODES, SearchResults, + _match_info, iter_semantic_entities, + resolve_filters, semantic_doc_types, semantic_fields_in_scope, semantic_text_for, @@ -451,17 +454,45 @@ def _fts_query(query: str) -> str: return "" return " OR ".join(f"{t}*" for t in terms) - def _entities_in_scope(self, doc_types: Optional[list[str]], limit: int) -> list[sqlite3.Row]: + @staticmethod + def _filter_sql(pairs: list[tuple[str, Any]]) -> tuple[str, list[Any]]: + """SQL for an exact-match filter over schema fields. + + json_extract rather than a column because schema fields live inside the + stored JSON blob. They sit at its top level, not under a `fields` key — + the blob is the flat shape `_fields_of` reads back — so the path is + `$.`. A missing field extracts to NULL and `NULL = ?` is never + true, so an entity of a doc_type that does not model the field is + excluded, which is the safe direction for a filter that may be carrying + a tenant boundary. + """ + if not pairs: + return "", [] + sql = "".join(" AND json_extract(e.data, '$.' || ?) = ?" for _ in pairs) + params: list[Any] = [] + for name, value in pairs: + params.extend([name, value]) + return sql, params + + def _entities_in_scope(self, doc_types: Optional[list[str]], limit: int, + filter_pairs: Optional[list[tuple[str, Any]]] = None + ) -> list[sqlite3.Row]: + """Rows the semantic arm may consider. + + Filters are applied here too, not only on the keyword query: this is the + set the vector scan ranks, and leaving it unfiltered would let a + semantic search return exactly the rows a filter was meant to exclude. + """ + filter_sql, filter_params = self._filter_sql(filter_pairs or []) + where = ["1=1"] + params: list[Any] = [] if doc_types: - placeholders = ",".join("?" * len(doc_types)) - return self._conn.execute( - f"SELECT id, doc_type, name, data FROM entities " - f"WHERE doc_type IN ({placeholders}) ORDER BY name LIMIT ?", - list(doc_types) + [limit], - ).fetchall() + where.append("doc_type IN (%s)" % ",".join("?" * len(doc_types))) + params.extend(doc_types) return self._conn.execute( - "SELECT id, doc_type, name, data FROM entities ORDER BY name LIMIT ?", - (limit,), + f"SELECT id, doc_type, name, data FROM entities e " + f"WHERE {' AND '.join(where)}{filter_sql} ORDER BY name LIMIT ?", + params + filter_params + [limit], ).fetchall() def search( @@ -471,13 +502,23 @@ def search( limit: int = 20, counts_only: bool = False, semantic_weight: Optional[float] = None, + mode: str = "hybrid", + filters: Optional[dict[str, Any]] = None, ) -> SearchResults: + if mode not in SEARCH_MODES: + raise ValueError(f"unknown search mode {mode!r} — expected one of " + + ", ".join(SEARCH_MODES)) w = semantic_weight if semantic_weight is not None else ( self._config.search.semantic_weight if self._config else 0.3) + # Raises on an undeclared field rather than filtering nothing. + filter_pairs = resolve_filters(self._doc_types, filters, doc_types) params: list[Any] = [] where: list[str] = [] - match_expr = self._fts_query(query) if query else "" + # A keyword search must not run the FTS query in semantic mode: the + # candidate set is the vector neighbourhood, and a keyword hit that is + # not near the query has no business being in it. + match_expr = self._fts_query(query) if (query and mode != "semantic") else "" if match_expr: base = ( "FROM entities_fts " @@ -493,6 +534,12 @@ def search( params.extend(doc_types) where_sql = (" AND " + " AND ".join(where)) if where else "" + # Appended after the doc_type clause so the filter constrains the counts + # and every candidate query below, not just the rows finally shown. + filter_sql, filter_params = self._filter_sql(filter_pairs) + where_sql += filter_sql + params += filter_params + # Per-doc_type aggregate counts (always computed — the map's spoke data). count_rows = self._conn.execute( f"SELECT e.doc_type AS dt, COUNT(*) AS n {base}{where_sql} GROUP BY e.doc_type", @@ -504,13 +551,20 @@ def search( if counts_only: return SearchResults(total=total, results=[], doc_type_counts=doc_type_counts) - semantic_scope = query and w > 0 and semantic_fields_in_scope(self._doc_types, doc_types) + # In semantic mode the vector arm runs whatever the weight says: the + # weight blends the two arms in hybrid, and letting a configured 0.0 + # silence an explicitly semantic search would answer a different + # question than the one asked. + wants_semantic = mode == "semantic" or (mode == "hybrid" and w > 0) + semantic_scope = (query and wants_semantic + and semantic_fields_in_scope(self._doc_types, doc_types)) provider = self._get_embedding_provider() if semantic_scope else None if semantic_scope and provider is not None: # Brute-force semantic scan over entities in scope. This is the SQLite # equivalent of a vector search; it is intentionally simple and capped # so it stays practical until sqlite-vec is adopted. - rows = self._entities_in_scope(doc_types, limit=_MAX_SEMANTIC_SCAN) + rows = self._entities_in_scope(doc_types, limit=_MAX_SEMANTIC_SCAN, + filter_pairs=filter_pairs) query_emb = provider.encode([query])[0] query_terms = [ t for t in "".join(c if c.isalnum() else " " for c in query.lower()).split() if t @@ -553,14 +607,26 @@ def search( # so `total`/`doc_type_counts` mean the same thing on both backends. # ((cos+1)/2 is > 0 for virtually every vector, so raw "sem > 0" # would count every embedded entity as a match.) + # Membership is tracked, not inferred from the scores: cosine is + # positive for very nearly every vector, so "sem > 0" would call the + # whole index a semantic match. A document is a semantic match when + # it is among the K nearest — that is what the arm actually + # retrieved. K = max(limit, 50) candidates: dict[str, tuple[float, float, sqlite3.Row, dict]] = {} - for kw, sem, r, data in scored: - if kw > 0: - candidates[r["id"]] = (kw, sem, r, data) - for kw, sem, r, data in sorted(scored, key=lambda s: -s[1])[:K]: - if sem > 0: - candidates.setdefault(r["id"], (kw, sem, r, data)) + kw_hits: set[str] = set() + sem_hits: set[str] = set() + + if mode in ("hybrid", "keyword"): + for kw, sem, r, data in scored: + if kw > 0: + candidates[r["id"]] = (kw, sem, r, data) + kw_hits.add(r["id"]) + if mode in ("hybrid", "semantic"): + for kw, sem, r, data in sorted(scored, key=lambda s: -s[1])[:K]: + if sem > 0: + candidates.setdefault(r["id"], (kw, sem, r, data)) + sem_hits.add(r["id"]) final = [] for kw, sem, r, data in candidates.values(): @@ -568,15 +634,22 @@ def search( # (cos+1)/2 is already bounded to [0, 1]; keep it absolute so a # weak best-match doesn't inflate the semantic arm. sem_norm = sem if sem_max else 0.0 - score = (1 - w) * kw_norm + w * sem_norm - final.append((score, r, data)) + if mode == "keyword": + score = kw_norm + elif mode == "semantic": + score = sem_norm + else: + score = (1 - w) * kw_norm + w * sem_norm + final.append((score, kw_norm, sem_norm, r, data)) # Highest combined score first; stable by name for ties. - final.sort(key=lambda s: (-s[0], s[1]["name"])) + final.sort(key=lambda s: (-s[0], s[3]["name"])) picked = final[:limit] results = [ {"id": r["id"], "doc_type": r["doc_type"], "name": r["name"], - "score": round(score, 3), "entity": data} - for (score, r, data) in picked + "score": round(score, 3), "entity": data, + "match": _match_info(r["id"] in kw_hits, r["id"] in sem_hits, + kw_norm, sem_norm)} + for (score, kw_norm, sem_norm, r, data) in picked ] doc_type_counts = {} for _kw, _sem, r, _data in candidates.values(): @@ -611,10 +684,12 @@ def search( scored.append((score, r, data)) # Highest weighted score first; stable by name for ties. scored.sort(key=lambda s: (-s[0], s[1]["name"])) + top = scored[0][0] if scored else 0.0 picked = scored[:limit] results = [ {"id": r["id"], "doc_type": r["doc_type"], "name": r["name"], - "score": round(score, 3), "entity": data} + "score": round(score, 3), "entity": data, + "match": _match_info(True, False, score / top if top else 0.0, 0.0)} for (score, r, data) in picked ] else: @@ -623,9 +698,13 @@ def search( f"ORDER BY e.name LIMIT ?", params + [limit], ).fetchall() + # Nothing was ranked here — this is a listing, or a pure filter. + # Saying so beats implying a relevance match that never happened. results = [ {"id": r["id"], "doc_type": r["doc_type"], "name": r["name"], - "score": 0.0, "entity": json.loads(r["data"])} + "score": 0.0, "entity": json.loads(r["data"]), + "match": _match_info(False, False, 0.0, 0.0, + filtered=bool(filter_pairs))} for r in rows ] diff --git a/open_index/ui/templates/explore.html b/open_index/ui/templates/explore.html index 9902633..7f102fd 100644 --- a/open_index/ui/templates/explore.html +++ b/open_index/ui/templates/explore.html @@ -58,6 +58,14 @@

Explore

{{ r.name }} + {% if r.badge %} + {# Why this came back, not just how highly it ranked — the thing you + need when an agent retrieved something surprising. #} + + {{ r.badge.label }} + + {% endif %} {{ r.doc_type }} {% endfor %} diff --git a/open_index/ui/view.py b/open_index/ui/view.py index 44ecea1..01fa309 100644 --- a/open_index/ui/view.py +++ b/open_index/ui/view.py @@ -190,16 +190,45 @@ def provenance_row(entity) -> Optional[dict[str, Any]]: } -# Search modes offered in the UI, mapped to a semantic_weight override. +# Search modes offered in the UI, mapped to the backend's mode. +# +# These used to map to a semantic_weight instead, which was subtly wrong: a +# weight of 0.0 still let semantically-matched documents into the candidate set +# at score 0, so "Keyword" returned things that matched no keyword. Mode decides +# membership, so the label now means what it says. SEARCH_MODES = { - "Hybrid": None, # the brain's configured blend - "Keyword": 0.0, - "Semantic": 1.0, + "Hybrid": "hybrid", + "Keyword": "keyword", + "Semantic": "semantic", } -def semantic_weight_for(mode: str) -> Optional[float]: - return SEARCH_MODES.get(mode) +def backend_mode_for(label: str) -> str: + """The backend mode for a UI label, defaulting to hybrid for anything odd.""" + return SEARCH_MODES.get(label, "hybrid") + + +# How a result's `match.type` reads on the page, and the colour it carries. +MATCH_LABELS = { + "both": ("keyword + meaning", "#7c3aed"), + "keyword": ("keyword", "#2563eb"), + "semantic": ("meaning", "#0d9488"), + "filter": ("filtered", "#6b7280"), + "none": ("listed", "#6b7280"), +} + + +def match_badge(match: Optional[dict]) -> Optional[dict]: + """Label, colour and scores for one result's match, or None if absent.""" + if not match: + return None + label, color = MATCH_LABELS.get(match.get("type", ""), (match.get("type", ""), "#6b7280")) + return { + "label": label, + "color": color, + "keyword_score": match.get("keyword_score", 0.0), + "semantic_score": match.get("semantic_score", 0.0), + } # -- map rendering ------------------------------------------------------------ diff --git a/open_index/ui/web.py b/open_index/ui/web.py index e2bba32..0b4eb80 100644 --- a/open_index/ui/web.py +++ b/open_index/ui/web.py @@ -190,11 +190,13 @@ def page_explore(request, name: str, brain: Brain) -> dict[str, Any]: try: found = brain.search( query=query, doc_types=selected or None, limit=50, - semantic_weight=view.semantic_weight_for(mode), source="ui") + mode=view.backend_mode_for(mode), source="ui") ctx["results"] = { "total": found.total, "rows": [ - {**r, "color": view.color_for(brain, r["doc_type"])} + {**r, + "color": view.color_for(brain, r["doc_type"]), + "badge": view.match_badge(r.get("match"))} for r in found.results ], } diff --git a/tests/test_search_modes_and_filters.py b/tests/test_search_modes_and_filters.py new file mode 100644 index 0000000..529478d --- /dev/null +++ b/tests/test_search_modes_and_filters.py @@ -0,0 +1,261 @@ +"""Search modes, match provenance, and strict filtering. + +The three properties worth defending here: + + mode decides membership, not just weight — a keyword search must not return a + document that matched no keyword, however low it scores. + + `match` says why a document came back, and "semantic" means it was among the + nearest by vector. Cosine is positive for nearly every embedded document, so a + naive `sem > 0` would label the whole index a semantic match. + + a filter is a hard predicate on every path. The dangerous bug is not an error + but a silent one: an unfiltered semantic arm, or a typo'd field that quietly + matches everything. +""" + +import shutil + +import pytest +import yaml + +from open_index.brain import Brain +from open_index.config import load_brain_config +from open_index.embeddings import FakeEmbedProvider +from open_index.models import Entity +from open_index.storage import get_backend +from open_index.storage.base import resolve_filters + +EXAMPLE = "examples/support-brain" + + +@pytest.fixture +def brain(tmp_path): + """The example brain plus a filterable `tenant_id` on `issue`. + + Uses FakeEmbedProvider rather than the real model: these tests are about + which arm retrieved a document, not about embedding quality, and CI does not + install the `semantic` extra. Without it the semantic arm silently falls + back to keyword — which is exactly the behaviour these tests exist to tell + apart, so they would pass locally and fail in CI. Which they did. + """ + d = tmp_path / "b" + shutil.copytree(EXAMPLE, d) + spec_path = d / "doc_types" / "issue.yaml" + spec = yaml.safe_load(spec_path.read_text()) + spec["schema"]["fields"].append({ + "name": "tenant_id", "type": "string", "processing": "keyword", + "search": "none", "boost": 1.0, "filterable": True, + }) + spec_path.write_text(yaml.safe_dump(spec, sort_keys=False)) + + config = load_brain_config(d) + backend = get_backend(config) + backend._embedding_provider = FakeEmbedProvider(dim=32) + backend._embedding_provider_initialized = True + b = Brain(config, backend=backend) + b.index() + for i, tenant in [(1, "acme"), (2, "globex")]: + b.put_entity(Entity( + id=f"issue:t{i}", doc_type="issue", + name=f"payment gateway timeout {tenant}", + fields={"tenant_id": tenant, + "description": "card payments fail at checkout"}, + )) + return b + + +def ids(results): + return [r["id"] for r in results.results] + + +# -- mode decides membership --------------------------------------------------- + + +def test_keyword_mode_returns_nothing_that_missed_the_keywords(brain): + """The bug a weight of 0.0 could not fix: semantic candidates entered the + set anyway and merely scored 0.""" + res = brain.search(query="payment", mode="keyword") + assert res.results + for row in res.results: + assert row["match"]["type"] == "keyword" + + +def test_keyword_mode_is_narrower_than_hybrid(brain): + hybrid = brain.search(query="card declined at till", mode="hybrid") + keyword = brain.search(query="card declined at till", mode="keyword") + assert keyword.total < hybrid.total + + +def test_semantic_mode_finds_documents_that_share_no_words(brain): + res = brain.search(query="customer cannot pay with their card", mode="semantic") + assert res.results + assert all(r["match"]["type"] == "semantic" for r in res.results) + + +def test_semantic_mode_runs_even_when_the_configured_weight_is_zero(brain): + """Asking for semantic and getting keyword back would answer a different + question than the one put.""" + res = brain.search(query="card trouble", mode="semantic", semantic_weight=0.0) + assert res.results + assert all(r["match"]["type"] == "semantic" for r in res.results) + + +def test_an_unknown_mode_is_refused(brain): + with pytest.raises(ValueError, match="unknown search mode"): + brain.search(query="payment", mode="telepathy") + + +# -- match provenance ---------------------------------------------------------- + + +def test_every_result_says_why_it_came_back(brain): + res = brain.search(query="payment") + assert res.results + for row in res.results: + m = row["match"] + assert m["type"] in ("keyword", "semantic", "both", "filter", "none") + assert 0.0 <= m["keyword_score"] <= 1.0 + assert 0.0 <= m["semantic_score"] <= 1.0 + + +def test_hybrid_can_report_both_arms(brain): + res = brain.search(query="payment", mode="hybrid") + assert any(r["match"]["type"] == "both" for r in res.results) + + +def test_semantic_match_is_bounded_by_the_nearest_neighbourhood(brain): + """cos rescaled to [0,1] is > 0 almost everywhere, so membership by score + would make every embedded document a semantic match. Membership is instead + "among the K nearest", which only shows on a corpus bigger than K — on a + small index every document genuinely is in the neighbourhood. + """ + for i in range(80): + brain.put_entity(Entity( + id=f"issue:bulk-{i}", doc_type="issue", name=f"unrelated topic {i}", + fields={"description": f"a wholly different subject number {i}"})) + + total_entities = sum(brain.counts().values()) + assert total_entities > 50, "the K bound is only observable past K" + + res = brain.search(query="payment", mode="semantic", limit=5) + assert res.total <= 50 + assert res.total < total_entities + + +def test_a_listing_is_not_reported_as_a_keyword_match(brain): + res = brain.search() + assert res.results + assert all(r["match"]["type"] == "none" for r in res.results) + + +def test_a_pure_filter_reports_itself_as_filtered(brain): + res = brain.search(filters={"tenant_id": "acme"}) + assert ids(res) == ["issue:t1"] + assert res.results[0]["match"]["type"] == "filter" + + +# -- strict filtering ---------------------------------------------------------- + + +@pytest.mark.parametrize("mode", ["hybrid", "keyword", "semantic"]) +def test_a_filter_holds_in_every_mode(brain, mode): + """Filtering only the keyword arm would let a semantic search return exactly + the documents the filter exists to exclude.""" + res = brain.search(query="payment", mode=mode, filters={"tenant_id": "acme"}) + assert ids(res) == ["issue:t1"] + + +def test_a_filter_excludes_the_other_tenant(brain): + res = brain.search(query="payment", filters={"tenant_id": "globex"}) + assert ids(res) == ["issue:t2"] + + +def test_a_filter_narrows_the_totals_and_counts_too(brain): + """Counts that disagree with the rows are a display that lies.""" + res = brain.search(query="payment", filters={"tenant_id": "acme"}) + assert res.total == 1 + assert sum(res.doc_type_counts.values()) == 1 + + +def test_entities_without_the_field_never_match(brain): + """An entity of a doc_type that does not model the field must not slip + through a tenant filter.""" + res = brain.search(filters={"tenant_id": "acme"}) + assert all(r["doc_type"] == "issue" for r in res.results) + + +def test_filtering_on_an_undeclared_field_raises(brain): + with pytest.raises(ValueError, match="cannot filter on"): + brain.search(query="payment", filters={"nope": "x"}) + + +def test_filtering_on_a_non_filterable_field_raises(brain): + """`description` exists but is not declared filterable — a promise not made + is not a promise kept.""" + with pytest.raises(ValueError, match="cannot filter on"): + brain.search(query="payment", filters={"description": "x"}) + + +def test_the_refusal_names_what_can_be_filtered(brain): + with pytest.raises(ValueError, match="tenant_id"): + brain.search(filters={"nope": "x"}) + + +def test_no_filter_is_not_a_filter(brain): + assert resolve_filters({}, None) == [] + assert resolve_filters({}, {}) == [] + + +def test_a_filter_outside_the_doc_type_scope_raises(brain): + """tenant_id is declared on `issue`; filtering a search scoped to `product` + is a mistake worth surfacing rather than a silent empty result.""" + with pytest.raises(ValueError, match="cannot filter on"): + brain.search(doc_types=["product"], filters={"tenant_id": "acme"}) + + +def test_filters_combine_with_doc_types(brain): + res = brain.search(doc_types=["issue"], filters={"tenant_id": "acme"}) + assert ids(res) == ["issue:t1"] + + +# -- the flag has to survive a round trip -------------------------------------- + + +def test_filterable_survives_being_written_back_to_yaml(): + """It was dropped on serialization, which is a quiet failure: the field + comes back non-filterable and every filter on it is then refused as + undeclared, long after the doc_type was written.""" + from open_index.config import doc_type_to_yaml_dict + from open_index.schema import DocType + + dt = DocType.from_dict({ + "doc_type": "issue", "description": "d", + "schema": {"fields": [ + {"name": "tenant_id", "type": "string", "search": "none", + "filterable": True}, + {"name": "body", "type": "text", "search": "semantic"}, + ]}, + }) + back = DocType.from_dict(doc_type_to_yaml_dict(dt)) + assert {f.name: f.filterable for f in back.fields} == { + "tenant_id": True, "body": False} + + +def test_a_doc_type_created_through_the_agent_can_declare_filterable(tmp_path): + """create_doc_type writes the schema to disk; the flag must reach it.""" + import shutil + + from open_index.config import load_brain_config, write_doc_type + from open_index.schema import DocType + + d = tmp_path / "b" + shutil.copytree(EXAMPLE, d) + dt = DocType.from_dict({ + "doc_type": "record", "description": "d", + "schema": {"fields": [{"name": "account_id", "type": "string", + "search": "none", "filterable": True}]}, + }) + write_doc_type(d, dt) + reloaded = load_brain_config(d).doc_type("record") + assert reloaded.fields[0].filterable is True diff --git a/tests/test_ui_view.py b/tests/test_ui_view.py index 2852fdb..f5df166 100644 --- a/tests/test_ui_view.py +++ b/tests/test_ui_view.py @@ -164,14 +164,28 @@ def test_empty_provenance_block_counts_as_unattributed(brain): # -- search modes ------------------------------------------------------------- -@pytest.mark.parametrize("mode,expected", - [("Hybrid", None), ("Keyword", 0.0), ("Semantic", 1.0)]) -def test_search_mode_weights(mode, expected): - assert view.semantic_weight_for(mode) == expected +@pytest.mark.parametrize("label,expected", + [("Hybrid", "hybrid"), ("Keyword", "keyword"), + ("Semantic", "semantic")]) +def test_search_labels_map_to_backend_modes(label, expected): + """These used to map to a semantic_weight, which let 'Keyword' return + documents that matched no keyword. The label now selects the mode.""" + assert view.backend_mode_for(label) == expected -def test_unknown_search_mode_falls_back_to_configured(brain): - assert view.semantic_weight_for("nonsense") is None +def test_unknown_search_mode_falls_back_to_hybrid(brain): + assert view.backend_mode_for("nonsense") == "hybrid" + + +def test_match_badge_describes_why_a_result_came_back(): + badge = view.match_badge({"type": "both", "keyword_score": 0.9, + "semantic_score": 0.4}) + assert "keyword" in badge["label"] and "meaning" in badge["label"] + assert badge["keyword_score"] == 0.9 + + +def test_match_badge_is_absent_when_the_backend_sent_none(): + assert view.match_badge(None) is None def test_color_for_unknown_doc_type_is_the_default(brain):