diff --git a/performer/proto/operational/sdk.caps.proto b/performer/proto/operational/sdk.caps.proto index bd6df94a..7f450a99 100644 --- a/performer/proto/operational/sdk.caps.proto +++ b/performer/proto/operational/sdk.caps.proto @@ -152,4 +152,8 @@ enum Caps { // The SDK supports the RESUMING status in the result of functions_status() of the eventing management API. SDK_EVENTING_RESUMING_FUNCTION_STATUS = 40; + + // The SDK supports score fusion for hybrid search (SDK-RFC 52). + // Support for SDK_VECTOR_SEARCH is a prerequisite. + SDK_SEARCH_SCORE_FUSION = 41; } diff --git a/performer/proto/operational/sdk.search.proto b/performer/proto/operational/sdk.search.proto index a5c430b3..666da31b 100644 --- a/performer/proto/operational/sdk.search.proto +++ b/performer/proto/operational/sdk.search.proto @@ -7,9 +7,7 @@ option go_package = "github.com/couchbaselabs/transactions-fit-performer/protoco option java_multiple_files = true; option ruby_package = "FIT::Protocol::SDK::Search"; -import "shared.doc_location.proto"; import "shared.content.proto"; -import "shared.collection.proto"; import "shared.basic.proto"; import "google/protobuf/timestamp.proto"; import "streams.top_level.proto"; @@ -91,6 +89,27 @@ enum VectorQueryCombination { OR = 1; } +message SearchScoringReciprocalRankFusion { + optional uint32 rank_constant = 1; + optional uint32 window_size = 2; +} + +message SearchScoringRelativeScoreFusion { + optional uint32 window_size = 1; +} + +message SearchScoringNone { +} + +// Controls the top-level search score field. +message SearchScoring { + oneof mode { + SearchScoringReciprocalRankFusion reciprocal_rank_fusion = 1; + SearchScoringRelativeScoreFusion relative_score_fusion = 2; + // Equivalent to SearchOptions.disable_scoring. + SearchScoringNone none = 3; + } +} // Executing a `cluster.searchQuery()` FTS query. // @@ -321,6 +340,10 @@ message SearchOptions { map raw = 12; optional bool include_locations = 13; optional protocol.shared.JsonSerializer serialize = 14; + // Top-level search score mode. Mutually exclusive with disable_scoring. + optional SearchScoring scoring = 15; + // Maps to the SDK's deprecated disableScoring option. + optional bool disable_scoring = 16 [deprecated = true]; } enum MatchOperator { diff --git a/performer/src/performer.rs b/performer/src/performer.rs index 254b0c39..ef1cfd92 100644 --- a/performer/src/performer.rs +++ b/performer/src/performer.rs @@ -74,6 +74,7 @@ impl PerformerService for Performer { sdk::Caps::SdkSearch, sdk::Caps::SdkVectorSearch, sdk::Caps::SdkVectorSearchBase64, + sdk::Caps::SdkSearchScoreFusion, sdk::Caps::SdkScopeSearch, sdk::Caps::SdkSearchRfcRevision11, sdk::Caps::SdkScopeSearchIndexManagement, diff --git a/performer/src/translations/search_options.rs b/performer/src/translations/search_options.rs index 7443506d..bc4c0f76 100644 --- a/performer/src/translations/search_options.rs +++ b/performer/src/translations/search_options.rs @@ -1,11 +1,13 @@ use crate::errors::error::{Error, Result}; use crate::proto::protocol::sdk::search::search_facet::Facet; use crate::proto::protocol::sdk::search::{ - search_sort, Highlight, HighlightStyle, SearchFacet, SearchGeoDistanceUnits, SearchOptions, - SearchScanConsistency, SearchSort, VectorQueryCombination, VectorSearchOptions, + search_scoring, search_sort, Highlight, HighlightStyle, SearchFacet, SearchGeoDistanceUnits, + SearchOptions, SearchScanConsistency, SearchScoring, SearchSort, VectorQueryCombination, + VectorSearchOptions, }; use chrono::{DateTime, FixedOffset, Utc}; use couchbase::options::search_options::ScanConsistency::NotBounded; +use couchbase::search::scoring::{ReciprocalRankScoreFusion, RelativeScoreScoreFusion, Scoring}; use couchbase::search::sort::{Sort, SortFieldMode, SortFieldType, SortGeoDistanceUnit}; use prost_types::Timestamp; use std::time::SystemTime; @@ -72,10 +74,48 @@ impl TryFrom for couchbase::options::search_options::SearchOption if let Some(_serializer) = opts.serialize { return Err(Error::unimplemented("serializer is unimplemented")); } + if let Some(scoring) = opts.scoring { + copts = copts.scoring(scoring.try_into()?); + } + #[allow(deprecated)] + if let Some(disable_scoring) = opts.disable_scoring { + copts = copts.disable_scoring(disable_scoring); + } Ok(copts) } } +impl TryFrom for Scoring { + type Error = Box; + + fn try_from(proto: SearchScoring) -> Result { + let mode = proto + .mode + .ok_or_else(|| Error::invalid_argument("Search scoring mode is not set"))?; + + match mode { + search_scoring::Mode::None(_) => Ok(Scoring::None), + search_scoring::Mode::ReciprocalRankFusion(rrf) => { + let mut fusion = ReciprocalRankScoreFusion::new(); + if let Some(rank_constant) = rrf.rank_constant { + fusion = fusion.rank_constant(rank_constant); + } + if let Some(window_size) = rrf.window_size { + fusion = fusion.window_size(window_size); + } + Ok(Scoring::ReciprocalRankFusion(fusion)) + } + search_scoring::Mode::RelativeScoreFusion(rsf) => { + let mut fusion = RelativeScoreScoreFusion::new(); + if let Some(window_size) = rsf.window_size { + fusion = fusion.window_size(window_size); + } + Ok(Scoring::RelativeScoreFusion(fusion)) + } + } + } +} + impl TryFrom for couchbase::options::search_options::Highlight { type Error = Box; diff --git a/sdk/couchbase-core/src/componentconfigs.rs b/sdk/couchbase-core/src/componentconfigs.rs index ca981a5d..897ebd9b 100644 --- a/sdk/couchbase-core/src/componentconfigs.rs +++ b/sdk/couchbase-core/src/componentconfigs.rs @@ -303,6 +303,9 @@ impl AgentComponentConfigs { vector_search_enabled: config .features .contains(&ParsedConfigFeature::FtsVectorSearch), + score_fusion_enabled: config + .features + .contains(&ParsedConfigFeature::FtsScoreFusion), }, mgmt_config: MgmtComponentConfig { endpoints: mgmt_endpoints, diff --git a/sdk/couchbase-core/src/configparser.rs b/sdk/couchbase-core/src/configparser.rs index f839e3e1..6e507669 100644 --- a/sdk/couchbase-core/src/configparser.rs +++ b/sdk/couchbase-core/src/configparser.rs @@ -104,6 +104,9 @@ impl ConfigParser { if caps.contains(&"vectorSearch".to_string()) { features.push(ParsedConfigFeature::FtsVectorSearch); } + if caps.contains(&"scoreFusion".to_string()) { + features.push(ParsedConfigFeature::FtsScoreFusion); + } } let cluster_labels = if config.cluster_name.is_some() || config.cluster_uuid.is_some() { diff --git a/sdk/couchbase-core/src/options/search.rs b/sdk/couchbase-core/src/options/search.rs index 90296a77..1d959c2f 100644 --- a/sdk/couchbase-core/src/options/search.rs +++ b/sdk/couchbase-core/src/options/search.rs @@ -21,7 +21,7 @@ use crate::retry::{RetryStrategy, DEFAULT_RETRY_STRATEGY}; use crate::searchx; use crate::searchx::facets::Facet; use crate::searchx::queries::Query; -use crate::searchx::query_options::{Control, Highlight, KnnOperator, KnnQuery}; +use crate::searchx::query_options::{Control, Highlight, KnnOperator, KnnQuery, Params}; use crate::searchx::sort::Sort; use std::collections::HashMap; use std::sync::Arc; @@ -46,6 +46,7 @@ pub struct SearchOptions { pub sort: Option>, pub knn: Option>, pub knn_operator: Option, + pub params: Option, pub raw: Option>, @@ -79,6 +80,7 @@ impl SearchOptions { sort: None, knn: None, knn_operator: None, + params: None, raw: None, index_name: index_name.into(), scope_name: None, @@ -174,6 +176,11 @@ impl SearchOptions { self } + pub fn params(mut self, params: impl Into>) -> Self { + self.params = params.into(); + self + } + pub fn raw(mut self, raw: impl Into>>) -> Self { self.raw = raw.into(); self @@ -225,6 +232,7 @@ impl From for searchx::query_options::QueryOptions { sort: opts.sort, knn: opts.knn, knn_operator: opts.knn_operator, + params: opts.params, raw: opts.raw, index_name: opts.index_name, scope_name: opts.scope_name, diff --git a/sdk/couchbase-core/src/parsedconfig.rs b/sdk/couchbase-core/src/parsedconfig.rs index 8ce40262..38a40943 100644 --- a/sdk/couchbase-core/src/parsedconfig.rs +++ b/sdk/couchbase-core/src/parsedconfig.rs @@ -63,6 +63,7 @@ pub(crate) struct ParsedConfigNode { #[derive(Debug, Clone, Eq, PartialEq)] pub(crate) enum ParsedConfigFeature { FtsVectorSearch, + FtsScoreFusion, Unknown, } diff --git a/sdk/couchbase-core/src/searchcomponent.rs b/sdk/couchbase-core/src/searchcomponent.rs index f7a87959..7015f265 100644 --- a/sdk/couchbase-core/src/searchcomponent.rs +++ b/sdk/couchbase-core/src/searchcomponent.rs @@ -67,6 +67,7 @@ pub(crate) struct SearchComponent { #[derive(Debug)] pub(crate) struct SearchComponentState { pub vector_search_enabled: bool, + pub score_fusion_enabled: bool, } pub(crate) struct SearchComponentConfig { @@ -74,6 +75,7 @@ pub(crate) struct SearchComponentConfig { pub authenticator: Authenticator, pub vector_search_enabled: bool, + pub score_fusion_enabled: bool, } #[derive(Debug)] @@ -102,6 +104,7 @@ impl SearchComponent { retry_manager, state: ArcSwap::new(Arc::new(SearchComponentState { vector_search_enabled: config.vector_search_enabled, + score_fusion_enabled: config.score_fusion_enabled, })), } } @@ -120,6 +123,7 @@ impl SearchComponent { self.state.swap(Arc::new(SearchComponentState { vector_search_enabled: config.vector_search_enabled, + score_fusion_enabled: config.score_fusion_enabled, })); } @@ -157,6 +161,7 @@ impl SearchComponent { auth, vector_search_enabled: self.state.load().vector_search_enabled, + score_fusion_enabled: self.state.load().score_fusion_enabled, tracing: self.tracing.clone(), } .query(&copts) @@ -475,6 +480,7 @@ impl SearchComponent { canonical_endpoint: target.canonical_endpoint, auth: target.auth, vector_search_enabled: false, + score_fusion_enabled: false, tracing: self.tracing.clone(), }; @@ -511,6 +517,7 @@ impl SearchComponent { auth: target.auth, vector_search_enabled: self.state.load().vector_search_enabled, + score_fusion_enabled: self.state.load().score_fusion_enabled, tracing: self.tracing.clone(), }; @@ -598,6 +605,7 @@ impl SearchComponent { auth, vector_search_enabled: self.state.load().vector_search_enabled, + score_fusion_enabled: self.state.load().score_fusion_enabled, tracing: self.tracing.clone(), }) .await @@ -641,6 +649,7 @@ impl SearchComponent { auth, vector_search_enabled: self.state.load().vector_search_enabled, + score_fusion_enabled: self.state.load().score_fusion_enabled, tracing: self.tracing.clone(), }) .await diff --git a/sdk/couchbase-core/src/searchx/ensure_index_helper.rs b/sdk/couchbase-core/src/searchx/ensure_index_helper.rs index 63bbea74..dd05c421 100644 --- a/sdk/couchbase-core/src/searchx/ensure_index_helper.rs +++ b/sdk/couchbase-core/src/searchx/ensure_index_helper.rs @@ -81,6 +81,7 @@ impl<'a> EnsureIndexHelper<'a> { canonical_endpoint: target.canonical_endpoint.to_string(), auth: target.auth.clone(), vector_search_enabled: true, + score_fusion_enabled: true, tracing: Default::default(), } .get_index(&GetIndexOptions { @@ -126,6 +127,7 @@ impl<'a> EnsureIndexHelper<'a> { canonical_endpoint: target.canonical_endpoint.to_string(), auth: target.auth.clone(), vector_search_enabled: true, + score_fusion_enabled: true, tracing: Default::default(), } .refresh_config(&RefreshConfigOptions::new()) diff --git a/sdk/couchbase-core/src/searchx/query_options.rs b/sdk/couchbase-core/src/searchx/query_options.rs index 30695ef2..d25f270f 100644 --- a/sdk/couchbase-core/src/searchx/query_options.rs +++ b/sdk/couchbase-core/src/searchx/query_options.rs @@ -99,6 +99,31 @@ impl Highlight { } } +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)] +#[non_exhaustive] +pub struct Params { + #[serde(skip_serializing_if = "Option::is_none")] + pub score_rank_constant: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub score_window_size: Option, +} + +impl Params { + pub fn new() -> Self { + Default::default() + } + + pub fn score_rank_constant(mut self, value: impl Into>) -> Self { + self.score_rank_constant = value.into(); + self + } + + pub fn score_window_size(mut self, value: impl Into>) -> Self { + self.score_window_size = value.into(); + self + } +} + pub type ConsistencyVectors = HashMap>; #[derive(Debug, Default, Clone, PartialEq, Eq, Serialize)] @@ -256,6 +281,8 @@ pub struct QueryOptions { pub(crate) knn: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub(crate) knn_operator: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub(crate) params: Option, #[serde(skip_serializing_if = "Option::is_none", flatten)] pub(crate) raw: Option>, @@ -291,6 +318,7 @@ impl QueryOptions { sort: None, knn: None, knn_operator: None, + params: None, raw: None, index_name: index_name.into(), scope_name: None, @@ -384,6 +412,11 @@ impl QueryOptions { self } + pub fn params(mut self, params: impl Into>) -> Self { + self.params = params.into(); + self + } + pub fn raw(mut self, raw: impl Into>>) -> Self { self.raw = raw.into(); self diff --git a/sdk/couchbase-core/src/searchx/search.rs b/sdk/couchbase-core/src/searchx/search.rs index d2f3e791..84af727e 100644 --- a/sdk/couchbase-core/src/searchx/search.rs +++ b/sdk/couchbase-core/src/searchx/search.rs @@ -54,6 +54,7 @@ pub struct Search { pub auth: Auth, pub vector_search_enabled: bool, + pub score_fusion_enabled: bool, pub(crate) tracing: Arc, } @@ -112,6 +113,16 @@ impl Search { )); } + if !self.score_fusion_enabled { + if let Some(score) = opts.score.as_ref() { + if score == "rrf" || score == "rsf" { + return Err(Error::new_unsupported_feature_error( + "score fusion".to_string(), + )); + } + } + } + let req_uri = if let Some(bucket) = &opts.bucket_name { if let Some(scope) = &opts.scope_name { format!( diff --git a/sdk/couchbase/src/clients/search_client.rs b/sdk/couchbase/src/clients/search_client.rs index ab0d9eb4..d5666c3d 100644 --- a/sdk/couchbase/src/clients/search_client.rs +++ b/sdk/couchbase/src/clients/search_client.rs @@ -23,9 +23,10 @@ use crate::options::search_options::SearchOptions; use crate::results::search_results::SearchResult; use crate::retry::RetryStrategy; use crate::search::request::SearchRequest; +use crate::search::scoring::Scoring; use couchbase_core::searchx; use couchbase_core::searchx::query_options::{ - Consistency, ConsistencyLevel, ConsistencyVectors, Control, KnnOperator, KnnQuery, + Consistency, ConsistencyLevel, ConsistencyVectors, Control, KnnOperator, KnnQuery, Params, }; use std::collections::HashMap; use std::sync::Arc; @@ -105,6 +106,7 @@ impl CouchbaseSearchClient { } } + #[allow(deprecated)] pub async fn search( &self, index_name: String, @@ -113,14 +115,43 @@ impl CouchbaseSearchClient { ) -> error::Result { let opts = opts.unwrap_or_default(); - let score = if let Some(disable_scoring) = opts.disable_scoring { - if disable_scoring { - Some("none".to_string()) - } else { - None - } + if opts.disable_scoring.is_some() && opts.scoring.is_some() { + return Err(error::Error::invalid_argument( + "scoring", + "cannot be used with disable_scoring", + )); + } + + let (score, params) = if let Some(scoring) = opts.scoring { + let score = match &scoring { + Scoring::None => "none", + Scoring::ReciprocalRankFusion(_) => "rrf", + Scoring::RelativeScoreFusion(_) => "rsf", + }; + + let params = match scoring { + Scoring::None => None, + Scoring::ReciprocalRankFusion(rank) => { + if rank.rank_constant.is_some() || rank.window_size.is_some() { + Some( + Params::default() + .score_rank_constant(rank.rank_constant) + .score_window_size(rank.window_size), + ) + } else { + None + } + } + Scoring::RelativeScoreFusion(rsf) => rsf + .window_size + .map(|window_size| Params::default().score_window_size(window_size)), + }; + + (Some(score.to_string()), params) + } else if opts.disable_scoring == Some(true) { + (Some("none".to_string()), None) } else { - None + (None, None) }; if opts.consistent_with.is_some() && opts.scan_consistency.is_some() { @@ -231,6 +262,7 @@ impl CouchbaseSearchClient { .sort(opts.sort.map(|s| s.into_iter().map(|i| i.into()).collect())) .knn(knn) .knn_operator(knn_operator) + .params(params) .raw(opts.raw) .scope_name(scope_name) .bucket_name(bucket_name) diff --git a/sdk/couchbase/src/options/search_options.rs b/sdk/couchbase/src/options/search_options.rs index 1ed84577..8e332d7a 100644 --- a/sdk/couchbase/src/options/search_options.rs +++ b/sdk/couchbase/src/options/search_options.rs @@ -22,6 +22,7 @@ use crate::error::Error; use crate::mutation_state::MutationState; use crate::retry::RetryStrategy; use crate::search::facets::Facet; +use crate::search::scoring::Scoring; use crate::search::sort::Sort; use serde::Serialize; use serde_json::Value; @@ -149,11 +150,16 @@ pub struct SearchOptions { pub sort: Option>, /// Facets to include in the result for aggregation. pub facets: Option>, + /// Controls how results are scored. See [`Scoring`] for the available modes. + /// + /// **Volatile: This API is subject to change at any time.** + pub scoring: Option, /// Raw key/value parameters passed directly to the search request body. pub raw: Option>, /// If `true`, includes term location information in the result. pub include_locations: Option, /// If `true`, disables scoring (useful when only sorting or filtering). + #[deprecated(since = "1.1.0", note = "Use `scoring(Scoring::None)` instead")] pub disable_scoring: Option, /// Server-side timeout for the search. pub server_timeout: Option, @@ -227,6 +233,14 @@ impl SearchOptions { self } + /// Sets how results are scored. See [`Scoring`] for the available modes. + /// + /// **Volatile: This API is subject to change at any time.** + pub fn scoring(mut self, scoring: Scoring) -> Self { + self.scoring = Some(scoring); + self + } + /// Adds a raw key/value parameter to the search request body. pub fn add_raw( mut self, @@ -256,6 +270,8 @@ impl SearchOptions { } /// If `true`, disables scoring (useful when only sorting or filtering). + #[deprecated(since = "1.1.0", note = "Use `scoring(Scoring::None)` instead")] + #[allow(deprecated)] pub fn disable_scoring(mut self, disable_scoring: bool) -> Self { self.disable_scoring = Some(disable_scoring); self diff --git a/sdk/couchbase/src/search/mod.rs b/sdk/couchbase/src/search/mod.rs index 90bb9e57..bb231a89 100644 --- a/sdk/couchbase/src/search/mod.rs +++ b/sdk/couchbase/src/search/mod.rs @@ -34,5 +34,6 @@ pub mod facets; pub mod location; pub mod queries; pub mod request; +pub mod scoring; pub mod sort; pub mod vector; diff --git a/sdk/couchbase/src/search/scoring.rs b/sdk/couchbase/src/search/scoring.rs new file mode 100644 index 00000000..ec1aca7f --- /dev/null +++ b/sdk/couchbase/src/search/scoring.rs @@ -0,0 +1,93 @@ +/* + * + * * Copyright (c) 2025 Couchbase, Inc. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +//! Scoring definitions for Full-Text Search results, including score fusion. +//! +//! Score fusion controls how the FTS and vector result sets of a hybrid search are merged +//! into a single ranked list. It is only meaningful for a hybrid request (both an FTS query +//! and a vector search); applied to a single result set, it re-scores the hits but leaves +//! their ordering unchanged. +//! +//! Use the [`Scoring`] enum to pass a scoring mode to +//! [`SearchOptions::scoring`](crate::options::search_options::SearchOptions::scoring). +//! +//! **Volatile: This API is subject to change at any time.** + +/// Selects how search results are scored. +/// +/// Use with [`SearchOptions::scoring`](crate::options::search_options::SearchOptions::scoring). +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub enum Scoring { + /// Disables scoring entirely. Works on any server version. + None, + /// Reciprocal Rank Fusion: merges by rank rather than raw score. + ReciprocalRankFusion(ReciprocalRankScoreFusion), + /// Relative Score Fusion: merges by normalized score rather than rank. + RelativeScoreFusion(RelativeScoreScoreFusion), +} + +/// Tuning parameters for [`Scoring::ReciprocalRankFusion`]. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct ReciprocalRankScoreFusion { + /// The rank constant used when merging by rank. + pub rank_constant: Option, + /// How many results per list are considered for fusion. + pub window_size: Option, +} + +impl ReciprocalRankScoreFusion { + /// Creates a new `ReciprocalRankScoreFusion` with default settings. + pub fn new() -> Self { + Self::default() + } + + /// Sets the rank constant. + pub fn rank_constant(mut self, rank_constant: u32) -> Self { + self.rank_constant = Some(rank_constant); + self + } + + /// Sets how many results per list are considered for fusion. + pub fn window_size(mut self, window_size: u32) -> Self { + self.window_size = Some(window_size); + self + } +} + +/// Tuning parameters for [`Scoring::RelativeScoreFusion`]. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +#[non_exhaustive] +pub struct RelativeScoreScoreFusion { + /// How many results per list are considered for fusion. + pub window_size: Option, +} + +impl RelativeScoreScoreFusion { + /// Creates a new `RelativeScoreScoreFusion` with default settings. + pub fn new() -> Self { + Self::default() + } + + /// Sets how many results per list are considered for fusion. + pub fn window_size(mut self, window_size: u32) -> Self { + self.window_size = Some(window_size); + self + } +}