Skip to content

Commit 4c2e2ea

Browse files
srimon12srimon12
authored
feat: enhance search and recommend functionality with offset, score threshold, and cross-collection lookup support (#37)
1 parent 05be76d commit 4c2e2ea

9 files changed

Lines changed: 292 additions & 9 deletions

File tree

‎docs/programmatic.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ print(result.data) # {"id": "<uuid>", "collection": "notes"}
2727

2828
# Search
2929
result = conn.run_query(
30-
"SEARCH notes SIMILAR TO 'hello' LIMIT 5 WHERE year >= 2023"
30+
"SEARCH notes SIMILAR TO 'hello' LIMIT 5 SCORE THRESHOLD 0.8 WHERE year >= 2023"
3131
)
3232
for hit in result.data:
3333
print(hit["score"], hit["payload"])
@@ -124,7 +124,7 @@ with Connection("http://localhost:6333") as conn:
124124

125125
# Recommend similar points
126126
result = conn.run_query(
127-
"RECOMMEND FROM notes POSITIVE IDS (1, 2) NEGATIVE IDS (3) LIMIT 5"
127+
"RECOMMEND FROM notes POSITIVE IDS (1, 2) NEGATIVE IDS (3) LIMIT 5 SCORE THRESHOLD 0.6"
128128
)
129129
for hit in result.data:
130130
print(hit["score"], hit["payload"])

‎docs/reference.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@ Qdrant/bm25
3333
INSERT INTO docs VALUES {'text': 'hello'} USING MODEL 'BAAI/bge-small-en-v1.5'
3434
SEARCH docs SIMILAR TO 'hello' LIMIT 5 USING MODEL 'BAAI/bge-small-en-v1.5'
3535

36+
-- Pagination and score filtering
37+
SEARCH docs SIMILAR TO 'hello' LIMIT 5 OFFSET 10 SCORE THRESHOLD 0.8
38+
39+
-- Cross-collection retrieval
40+
SEARCH docs SIMILAR TO 'hello' LIMIT 5 LOOKUP FROM user_profiles VECTOR 'preferences'
41+
3642
-- Explicit vector names
3743
INSERT INTO docs VALUES {'text': 'hello'} USING VECTOR 'body'
3844
SEARCH docs SIMILAR TO 'hello' LIMIT 5 USING VECTOR 'body'
@@ -172,7 +178,7 @@ Tests do not require a running Qdrant instance — the Qdrant client is mocked.
172178
pytest tests/ -v
173179
```
174180

175-
Expected output: **549 tests passing**.
181+
Expected output: **604 tests passing**.
176182

177183
---
178184

‎docs/search.md‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ An optional `WHERE` clause filters the candidate set **before** similarity ranki
1010

1111
**Syntax:**
1212
```
13-
SEARCH <collection_name> SIMILAR TO '<query_text>' LIMIT <n>
13+
SEARCH <collection_name> SIMILAR TO '<query_text>' LIMIT <n> [OFFSET <n>] [SCORE THRESHOLD <f>] [LOOKUP FROM <collection> [VECTOR '<name>']]
1414
SEARCH <collection_name> SIMILAR TO '<query_text>' LIMIT <n> USING MODEL '<model_name>'
1515
SEARCH <collection_name> SIMILAR TO '<query_text>' LIMIT <n> USING VECTOR '<dense_vector_name>'
1616
SEARCH <collection_name> SIMILAR TO '<query_text>' LIMIT <n> [USING MODEL '<model>'] WHERE <filter>
@@ -29,6 +29,21 @@ Basic search, return top 5 results:
2929
SEARCH articles SIMILAR TO 'machine learning algorithms' LIMIT 5
3030
```
3131

32+
Pagination with OFFSET:
33+
```sql
34+
SEARCH articles SIMILAR TO 'machine learning' LIMIT 10 OFFSET 20
35+
```
36+
37+
Filter low-quality matches with SCORE THRESHOLD:
38+
```sql
39+
SEARCH articles SIMILAR TO 'deep learning' LIMIT 10 SCORE THRESHOLD 0.8
40+
```
41+
42+
Cross-collection vector lookup:
43+
```sql
44+
SEARCH articles SIMILAR TO 'deep learning' LIMIT 5 LOOKUP FROM user_profiles VECTOR 'preferences'
45+
```
46+
3247
Search only papers published after 2020:
3348
```sql
3449
SEARCH articles SIMILAR TO 'deep learning' LIMIT 10 WHERE year > 2020
@@ -71,6 +86,10 @@ Search with native MMR diversification:
7186
SEARCH articles SIMILAR TO 'attention mechanism' LIMIT 10 WITH { mmr_diversity: 0.5, mmr_candidates: 50 }
7287
```
7388

89+
**Clause Order:**
90+
`SEARCH` requires clauses to appear in this strict order if used:
91+
`LIMIT` → `OFFSET` → `SCORE THRESHOLD` → `LOOKUP FROM` → `USING ...` → `WHERE` → `RERANK` → `WITH` → `GROUP BY`
92+
7493
**Output:**
7594

7695
Results are displayed as a table with three columns:
@@ -394,6 +413,7 @@ SEARCH <collection> SIMILAR TO '<query>' LIMIT <n> USING HYBRID GROUP BY <field>
394413
- **`GROUP_SIZE <m>`** — maximum number of points per group (default: **3**).
395414
- **`GROUP BY <field>`** — the payload field whose values define the groups. **Must be a string (keyword) or number (integer) field** — this is enforced by Qdrant. Dot-notation is supported (e.g. `meta.author`). Array-valued fields are allowed: a point with multiple values for the field can appear in multiple groups. The field should be indexed as `keyword` or `integer` for best performance (see [CREATE INDEX](collections.md)).
396415
- `WHERE` filters, `USING HYBRID`, and `USING MODEL` are all compatible with GROUP BY.
416+
- ⚠️ **Incompatibility:** `GROUP BY` is not compatible with `OFFSET` or `RERANK`. Use cursors (not currently supported in QQL) for paginating grouped results in Qdrant.
397417
- **`GROUP BY` and `RERANK` cannot be combined** in the same statement — this raises a syntax error.
398418

399419
**Examples:**

‎src/qql/ast_nodes.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -295,6 +295,9 @@ class SearchStmt:
295295
group_size: int = 3 # max points per group (ignored when group_by is None)
296296
dense_vector: str | None = None
297297
sparse_vector: str | None = None
298+
offset: int = 0 # skip first N results
299+
score_threshold: float | None = None # drop results below this score
300+
lookup_from: tuple[str, str | None] | None = None # cross-collection retrieval: (collection_name, vector_name)
298301

299302

300303
@dataclass(frozen=True)
@@ -305,10 +308,10 @@ class RecommendStmt:
305308
limit: int = 10
306309
strategy: str | None = None
307310
query_filter: FilterExpr | None = None
308-
offset: int = 0
309-
score_threshold: float | None = None
311+
offset: int = 0 # skip first N results
312+
score_threshold: float | None = None # drop results below this score
310313
with_clause: SearchWith | None = None
311-
lookup_from: tuple[str, str | None] | None = None
314+
lookup_from: tuple[str, str | None] | None = None # cross-collection retrieval: (collection_name, vector_name)
312315
using: str | None = None
313316

314317

‎src/qql/cli.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,9 @@
8888
8989
[yellow]SEARCH[/yellow] <name> [yellow]SIMILAR TO[/yellow] '<text>' [yellow]LIMIT[/yellow] <n>
9090
Semantic search by vector similarity.
91+
Optional: [yellow]OFFSET[/yellow] <n>
92+
Optional: [yellow]SCORE THRESHOLD[/yellow] <float|int>
93+
Optional: [yellow]LOOKUP FROM[/yellow] <collection> [[yellow]VECTOR[/yellow] '<vector_name>']
9194
Optional: [yellow]USING MODEL[/yellow] '<model>'
9295
Optional: [yellow]USING VECTOR[/yellow] '<dense_vector>'
9396
Optional: [yellow]USING HYBRID[/yellow] [FUSION 'rrf|dbsf'] [DENSE MODEL '<model>'] [DENSE VECTOR '<name>'] [SPARSE MODEL '<model>'] [SPARSE VECTOR '<name>']
@@ -99,13 +102,19 @@
99102
Optional: [yellow]GROUP BY[/yellow] <field> [[yellow]GROUP_SIZE[/yellow] <n>]
100103
Group results by a payload field value (default GROUP_SIZE: 3).
101104
Field must be keyword or integer type. RERANK and GROUP BY cannot be combined.
105+
OFFSET is not supported with GROUP BY.
102106
103107
[yellow]RECOMMEND FROM[/yellow] <name> [yellow]POSITIVE IDS[/yellow] (<id>, ...)
104108
Find points similar to known examples.
105109
Optional: [yellow]NEGATIVE IDS[/yellow] (<id>, ...)
106110
Optional: [yellow]STRATEGY[/yellow] 'average_vector|best_score|sum_scores'
107-
Optional: [yellow]WHERE[/yellow] <filter>
111+
Optional: [yellow]LOOKUP FROM[/yellow] <collection> [[yellow]VECTOR[/yellow] '<vector_name>']
112+
Optional: [yellow]USING[/yellow] '<vector_name>'
108113
Requires: [yellow]LIMIT[/yellow] <n>
114+
Optional: [yellow]OFFSET[/yellow] <n>
115+
Optional: [yellow]SCORE THRESHOLD[/yellow] <float|int>
116+
Optional: [yellow]WHERE[/yellow] <filter>
117+
Optional: [yellow]WITH[/yellow] { hnsw_ef: <int>, exact: <bool>, acorn: <bool>, indexed_only: <bool>, quantization: { ignore: <bool>, rescore: <bool>, oversampling: <n> } }
109118
110119
[yellow]DELETE FROM[/yellow] <name> [yellow]WHERE id =[/yellow] '<id>'
111120
Delete a point by its ID.

‎src/qql/executor.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -846,6 +846,13 @@ def _execute_search(self, node: SearchStmt) -> ExecutionResult:
846846
# enough material to reorder; only `node.limit` results are returned.
847847
fetch_limit = node.limit * _RERANK_FETCH_MULTIPLIER if node.rerank else node.limit
848848

849+
lookup_from: LookupLocation | None = None
850+
if node.lookup_from is not None:
851+
lookup_from = LookupLocation(
852+
collection=node.lookup_from[0],
853+
vector=node.lookup_from[1],
854+
)
855+
849856
# ── GROUP BY SEARCH: delegate to query_points_groups() ─────────────
850857
if node.group_by is not None:
851858
return self._execute_search_groups(
@@ -879,7 +886,10 @@ def _execute_search(self, node: SearchStmt) -> ExecutionResult:
879886
],
880887
query=FusionQuery(fusion=self._resolve_hybrid_fusion(node.fusion)),
881888
limit=fetch_limit,
889+
offset=node.offset or None,
882890
query_filter=qdrant_filter,
891+
score_threshold=node.score_threshold,
892+
lookup_from=lookup_from,
883893
)
884894
except UnexpectedResponse as e:
885895
raise QQLRuntimeError(f"Qdrant error during SEARCH: {e}") from e
@@ -919,8 +929,11 @@ def _execute_search(self, node: SearchStmt) -> ExecutionResult:
919929
query=sparse_vector,
920930
using=topology.sparse_using(node.sparse_vector),
921931
limit=fetch_limit,
932+
offset=node.offset or None,
922933
query_filter=qdrant_filter,
923934
search_params=search_params,
935+
score_threshold=node.score_threshold,
936+
lookup_from=lookup_from,
924937
)
925938
except UnexpectedResponse as e:
926939
raise QQLRuntimeError(f"Qdrant error during SEARCH: {e}") from e
@@ -956,8 +969,11 @@ def _execute_search(self, node: SearchStmt) -> ExecutionResult:
956969
query=self._build_dense_query(vector, node.with_clause),
957970
using=query_using,
958971
limit=fetch_limit,
972+
offset=node.offset or None,
959973
query_filter=qdrant_filter,
960974
search_params=search_params,
975+
score_threshold=node.score_threshold,
976+
lookup_from=lookup_from,
961977
)
962978
except UnexpectedResponse as e:
963979
raise QQLRuntimeError(f"Qdrant error during SEARCH: {e}") from e
@@ -1599,6 +1615,14 @@ def _execute_search_groups(
15991615
topology: CollectionTopology,
16001616
) -> ExecutionResult:
16011617
"""Execute SEARCH ... GROUP BY using query_points_groups()."""
1618+
1619+
lookup_from: LookupLocation | None = None
1620+
if node.lookup_from is not None:
1621+
lookup_from = LookupLocation(
1622+
collection=node.lookup_from[0],
1623+
vector=node.lookup_from[1],
1624+
)
1625+
16021626
try:
16031627
if node.hybrid:
16041628
dense_model = node.model or self._config.default_model
@@ -1627,6 +1651,8 @@ def _execute_search_groups(
16271651
limit=node.limit,
16281652
group_size=node.group_size,
16291653
query_filter=qdrant_filter,
1654+
score_threshold=node.score_threshold,
1655+
lookup_from=lookup_from,
16301656
)
16311657
label = "hybrid, grouped"
16321658
elif node.sparse_only:
@@ -1645,6 +1671,8 @@ def _execute_search_groups(
16451671
group_size=node.group_size,
16461672
query_filter=qdrant_filter,
16471673
search_params=search_params,
1674+
score_threshold=node.score_threshold,
1675+
lookup_from=lookup_from,
16481676
)
16491677
label = "sparse, grouped"
16501678
else:
@@ -1660,6 +1688,8 @@ def _execute_search_groups(
16601688
group_size=node.group_size,
16611689
query_filter=qdrant_filter,
16621690
search_params=search_params,
1691+
score_threshold=node.score_threshold,
1692+
lookup_from=lookup_from,
16631693
)
16641694
label = "grouped"
16651695
except UnexpectedResponse as e:

‎src/qql/parser.py‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -682,6 +682,31 @@ def _parse_search(self) -> SearchStmt:
682682
self._expect(TokenKind.LIMIT)
683683
limit = int(self._expect(TokenKind.INTEGER).value)
684684

685+
offset: int = 0
686+
if self._peek().kind == TokenKind.OFFSET:
687+
self._advance()
688+
offset_tok = self._peek()
689+
offset = int(self._expect(TokenKind.INTEGER).value)
690+
if offset < 0:
691+
raise QQLSyntaxError("OFFSET must be a non-negative integer", offset_tok.pos)
692+
693+
score_threshold: float | None = None
694+
if self._peek().kind == TokenKind.SCORE:
695+
self._advance()
696+
self._expect(TokenKind.THRESHOLD)
697+
score_threshold = float(self._parse_number())
698+
699+
lookup_from: tuple[str, str | None] | None = None
700+
if self._peek().kind == TokenKind.LOOKUP:
701+
self._advance()
702+
self._expect(TokenKind.FROM)
703+
lookup_collection = self._parse_identifier()
704+
lookup_vector: str | None = None
705+
if self._peek().kind == TokenKind.VECTOR:
706+
self._advance()
707+
lookup_vector = self._expect(TokenKind.STRING).value
708+
lookup_from = (lookup_collection, lookup_vector)
709+
685710
with_clause: SearchWith | None = None
686711
if self._peek().kind == TokenKind.EXACT:
687712
self._advance()
@@ -757,6 +782,7 @@ def _parse_search(self) -> SearchStmt:
757782
if self._peek().kind == TokenKind.MODEL:
758783
self._advance() # consume MODEL
759784
rerank_model = self._expect(TokenKind.STRING).value
785+
760786
if self._peek().kind == TokenKind.EXACT:
761787
self._advance()
762788
if with_clause is None:
@@ -771,6 +797,7 @@ def _parse_search(self) -> SearchStmt:
771797
mmr_diversity=with_clause.mmr_diversity,
772798
mmr_candidates=with_clause.mmr_candidates,
773799
)
800+
774801
if self._peek().kind == TokenKind.WITH:
775802
self._advance() # consume WITH
776803
parsed_with = self._parse_with_clause()
@@ -793,6 +820,8 @@ def _parse_search(self) -> SearchStmt:
793820
group_by: str | None = None
794821
group_size: int = 3
795822
if self._peek().kind == TokenKind.GROUP:
823+
if offset > 0:
824+
raise QQLSyntaxError("OFFSET cannot be used with GROUP BY", self._peek().pos)
796825
self._advance() # consume GROUP
797826
self._expect(TokenKind.BY)
798827
group_by = self._parse_field_path()
@@ -827,6 +856,9 @@ def _parse_search(self) -> SearchStmt:
827856
group_size=group_size,
828857
dense_vector=dense_vector,
829858
sparse_vector=sparse_vector,
859+
offset=offset,
860+
score_threshold=score_threshold,
861+
lookup_from=lookup_from,
830862
)
831863

832864
def _parse_recommend(self) -> RecommendStmt:
@@ -870,13 +902,16 @@ def _parse_recommend(self) -> RecommendStmt:
870902
offset: int = 0
871903
if self._peek().kind == TokenKind.OFFSET:
872904
self._advance()
905+
offset_tok = self._peek()
873906
offset = int(self._expect(TokenKind.INTEGER).value)
907+
if offset < 0:
908+
raise QQLSyntaxError("OFFSET must be a non-negative integer", offset_tok.pos)
874909

875910
score_threshold: float | None = None
876911
if self._peek().kind == TokenKind.SCORE:
877912
self._advance()
878913
self._expect(TokenKind.THRESHOLD)
879-
score_threshold = float(self._expect(TokenKind.FLOAT).value)
914+
score_threshold = float(self._parse_number())
880915

881916
query_filter: FilterExpr | None = None
882917
if self._peek().kind == TokenKind.WHERE:

0 commit comments

Comments
 (0)