Merge pull request #45 from qdrant/update-v1.2.0

Update v1.2.0

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "qdrant-client"
-version = "1.1.2"
+version = "1.2.0"
 edition = "2021"
 authors = ["Qdrant Team <[email protected]>"]
 description = "Rust client for Qdrant Vector Search Engine"

examples/search.rs

@@ -34,6 +34,7 @@ async fn main() -> Result<()> {
                     distance: Distance::Cosine.into(),
                     hnsw_config: None,
                     quantization_config: None,
+                    on_disk: None,
                 })),
             }),
             ..Default::default()

proto/collections.proto

@@ -6,6 +6,7 @@ message VectorParams {
   Distance distance = 2; // Distance function used for comparing vectors
   optional HnswConfigDiff hnsw_config = 3; // Configuration of vector HNSW graph. If omitted - the collection configuration will be used
   optional QuantizationConfig quantization_config = 4; // Configuration of vector quantization config. If omitted - the collection configuration will be used
+  optional bool on_disk = 5; // If true - serve vectors from disk. If set to false, the vectors will be loaded in RAM.
 }
 
 message VectorParamsMap {
@@ -68,6 +69,14 @@ enum QuantizationType {
   Int8 = 1;
 }
 
+enum CompressionRatio {
+  x4 = 0;
+  x8 = 1;
+  x16 = 2;
+  x32 = 3;
+  x64 = 4;
+}
+
 message OptimizerStatus {
   bool ok = 1;
   string error = 2;
@@ -129,26 +138,35 @@ message OptimizersConfigDiff {
   */
   optional uint64 default_segment_number = 3;
   /*
-  Do not create segments larger this size (in KiloBytes).
+  Do not create segments larger this size (in kilobytes).
   Large segments might require disproportionately long indexation times,
   therefore it makes sense to limit the size of segments.
-
-  If indexation speed has more priority for you - make this parameter lower.
+    
+  If indexing speed is more important - make this parameter lower.
   If search speed is more important - make this parameter higher.
   Note: 1Kb = 1 vector of size 256
+  If not set, will be automatically selected considering the number of available CPUs.
   */
   optional uint64 max_segment_size = 4;
   /*
-  Maximum size (in KiloBytes) of vectors to store in-memory per segment.
-  Segments larger than this threshold will be stored as a read-only memmaped file.
-  To enable memmap storage, lower the threshold
+  Maximum size (in kilobytes) of vectors to store in-memory per segment.
+  Segments larger than this threshold will be stored as read-only memmaped file.
+
+  Memmap storage is disabled by default, to enable it, set this threshold to a reasonable value.
+
+  To disable memmap storage, set this to `0`.
+
   Note: 1Kb = 1 vector of size 256
   */
   optional uint64 memmap_threshold = 5;
   /*
-  Maximum size (in KiloBytes) of vectors allowed for plain index.
-  Default value based on https://github.com/google-research/google-research/blob/master/scann/docs/algorithms.md
-  Note: 1Kb = 1 vector of size 256
+  Maximum size (in kilobytes) of vectors allowed for plain index, exceeding this threshold will enable vector indexing
+  
+  Default value is 20,000, based on <https://github.com/google-research/google-research/blob/master/scann/docs/algorithms.md>.
+  
+  To disable vector indexing, set to `0`.
+  
+  Note: 1kB = 1 vector of size 256.
   */
   optional uint64 indexing_threshold = 6;
   /*
@@ -167,9 +185,15 @@ message ScalarQuantization {
   optional bool always_ram = 3; // If true - quantized vectors always will be stored in RAM, ignoring the config of main storage
 }
 
+message ProductQuantization {
+  CompressionRatio compression = 1; // Compression ratio
+  optional bool always_ram = 2; // If true - quantized vectors always will be stored in RAM, ignoring the config of main storage
+}
+
 message QuantizationConfig {
   oneof quantization {
     ScalarQuantization scalar = 1;
+    ProductQuantization product = 2;
   }
 }
 
@@ -312,3 +336,68 @@ message ListAliasesResponse {
   repeated AliasDescription aliases = 1;
   double time = 2; // Time spent to process
 }
+
+message CollectionClusterInfoRequest {
+  string collection_name = 1; // Name of the collection
+}
+
+enum ReplicaState {
+  Active = 0; // Active and sound
+  Dead = 1; // Failed for some reason
+  Partial = 2; // The shard is partially loaded and is currently receiving data from other shards
+  Initializing = 3; // Collection is being created
+  Listener = 4; // A shard which receives data, but is not used for search; Useful for backup shards
+}
+
+message LocalShardInfo {
+  uint32 shard_id = 1; // Local shard id
+  uint64 points_count = 2; // Number of points in the shard
+  ReplicaState state = 3;  // Is replica active
+}
+
+message RemoteShardInfo {
+  uint32 shard_id = 1; // Local shard id
+  uint64 peer_id = 2; // Remote peer id
+  ReplicaState state = 3; // Is replica active
+}
+
+message ShardTransferInfo {
+  uint32 shard_id = 1; // Local shard id
+  uint64 from = 2;
+  uint64 to = 3;
+  bool sync = 4; // If `true` transfer is a synchronization of a replicas; If `false` transfer is a moving of a shard from one peer to another
+}
+
+message CollectionClusterInfoResponse {
+  uint64 peer_id = 1;  // ID of this peer 
+  uint64 shard_count = 2; // Total number of shards
+  repeated LocalShardInfo local_shards = 3; // Local shards
+  repeated RemoteShardInfo remote_shards = 4; // Remote shards
+  repeated ShardTransferInfo shard_transfers = 5; // Shard transfers
+}
+
+message MoveShard {
+  uint32 shard_id = 1; // Local shard id
+  uint64 from_peer_id = 2;
+  uint64 to_peer_id = 3;
+}
+
+message Replica {
+  uint32 shard_id = 1; 
+  uint64 peer_id = 2; 
+}
+
+message UpdateCollectionClusterSetupRequest {
+  string collection_name = 1; // Name of the collection
+  oneof operation {
+    MoveShard move_shard = 2;
+    MoveShard replicate_shard = 3;
+    MoveShard abort_transfer = 4;
+    Replica drop_replica = 5;
+  }
+  optional uint64 timeout = 6; // Wait timeout for operation commit in seconds, if not specified - default value will be supplied
+}
+
+message UpdateCollectionClusterSetupResponse {
+  bool result = 1;
+}

proto/collections_service.proto

@@ -37,4 +37,12 @@ service Collections {
   Get list of all aliases for all existing collections
   */
   rpc ListAliases (ListAliasesRequest) returns (ListAliasesResponse) {}
+  /*
+  Get cluster information for a collection
+  */
+  rpc CollectionClusterInfo (CollectionClusterInfoRequest) returns (CollectionClusterInfoResponse) {}
+  /*
+  Update cluster setup for a collection
+  */
+  rpc UpdateCollectionClusterSetup (UpdateCollectionClusterSetupRequest) returns (UpdateCollectionClusterSetupResponse) {}
 }

proto/points.proto

@@ -71,6 +71,26 @@ message GetPoints {
   optional ReadConsistency read_consistency = 6; // Options for specifying read consistency guarantees
 }
 
+message UpdatePointVectors {
+  string collection_name = 1; // name of the collection
+  optional bool wait = 2; // Wait until the changes have been applied?
+  repeated PointVectors points = 3; // List of points and vectors to update
+  optional WriteOrdering ordering = 4; // Write ordering guarantees
+}
+
+message PointVectors {
+  PointId id = 1; // ID to update vectors for
+  Vectors vectors = 2; // Named vectors to update, leave others intact
+}
+
+message DeletePointVectors {
+  string collection_name = 1; // name of the collection
+  optional bool wait = 2; // Wait until the changes have been applied?
+  PointsSelector points_selector = 3; // Affected points
+  VectorsSelector vectors = 4; // List of vector names to delete
+  optional WriteOrdering ordering = 5; // Write ordering guarantees
+}
+
 message SetPayloadPoints {
   string collection_name = 1; // name of the collection
   optional bool wait = 2; // Wait until the changes have been applied?
@@ -209,6 +229,21 @@ message SearchBatchPoints {
   optional ReadConsistency read_consistency = 3; // Options for specifying read consistency guarantees
 }
 
+message SearchPointGroups {
+  string collection_name = 1; // Name of the collection
+  repeated float vector = 2; // Vector to compare against
+  Filter filter = 3; // Filter conditions - return only those points that satisfy the specified conditions
+  uint32 limit = 4; // Max number of result
+  WithPayloadSelector with_payload = 5; // Options for specifying which payload to include or not
+  SearchParams params = 6; // Search config
+  optional float score_threshold = 7; // If provided - cut off results with worse scores
+  optional string vector_name = 8; // Which vector to use for search, if not specified - use default vector
+  optional WithVectorsSelector with_vectors = 9; // Options for specifying which vectors to include into response
+  string group_by = 10; // Payload field to group by, must be a string or number field. If there are multiple values for the field, all of them will be used. One point can be in multiple groups.
+  uint32 group_size = 11; // Maximum amount of points to return per group
+  optional ReadConsistency read_consistency = 12; // Options for specifying read consistency guarantees
+}
+
 message ScrollPoints {
   string collection_name = 1;
   Filter filter = 2; // Filter conditions - return only those points that satisfy the specified conditions
@@ -248,6 +283,23 @@ message RecommendBatchPoints {
   optional ReadConsistency read_consistency = 3; // Options for specifying read consistency guarantees
 }
 
+message RecommendPointGroups {
+  string collection_name = 1; // Name of the collection
+  repeated PointId positive = 2; // Look for vectors closest to those
+  repeated PointId negative = 3; // Try to avoid vectors like this
+  Filter filter = 4; // Filter conditions - return only those points that satisfy the specified conditions
+  uint32 limit = 5; // Max number of groups in result
+  WithPayloadSelector with_payload = 6; // Options for specifying which payload to include or not
+  SearchParams params = 7; // Search config
+  optional float score_threshold = 8; // If provided - cut off results with worse scores
+  optional string using = 9; // Define which vector to use for recommendation, if not specified - default vector
+  optional WithVectorsSelector with_vectors = 10; // Options for specifying which vectors to include into response
+  optional LookupLocation lookup_from = 11; // Name of the collection to use for points lookup, if not specified - use current collection
+  string group_by = 12; // Payload field to group by, must be a string or number field. If there are multiple values for the field, all of them will be used. One point can be in multiple groups.
+  uint32 group_size = 13; // Maximum amount of points to return per group
+  optional ReadConsistency read_consistency = 14; // Options for specifying read consistency guarantees
+}
+
 message CountPoints {
   string collection_name = 1; // name of the collection
   Filter filter = 2; // Filter conditions - return only those points that satisfy the specified conditions
@@ -283,6 +335,26 @@ message ScoredPoint {
   optional Vectors vectors = 6; // Vectors to search
 }
 
+message GroupId {
+  oneof kind {
+    // Represents a double value.
+    uint64 unsigned_value = 1;
+    // Represents an integer value
+    int64 integer_value = 2;
+    // Represents a string value.
+    string string_value = 3;
+  }
+}
+
+message PointGroup {
+  GroupId id = 1; // Group id
+  repeated ScoredPoint hits = 2; // Points in the group
+}
+
+message GroupsResult {
+  repeated PointGroup groups = 1; // Groups
+}
+
 message SearchResponse {
   repeated ScoredPoint result = 1;
   double time = 2; // Time spent to process
@@ -297,6 +369,11 @@ message SearchBatchResponse {
   double time = 2; // Time spent to process
 }
 
+message SearchGroupsResponse {
+  GroupsResult result = 1;
+  double time = 2; // Time spent to process
+}
+
 message CountResponse {
   CountResult result = 1;
   double time = 2; // Time spent to process
@@ -334,6 +411,11 @@ message RecommendBatchResponse {
   double time = 2; // Time spent to process
 }
 
+message RecommendGroupsResponse {
+  GroupsResult result = 1;
+  double time = 2; // Time spent to process
+}
+
 // ---------------------------------------------
 // ------------- Filter Conditions -------------
 // ---------------------------------------------
@@ -351,6 +433,7 @@ message Condition {
     HasIdCondition has_id = 3;
     Filter filter = 4;
     IsNullCondition is_null = 5;
+    NestedCondition nested = 6;
   }
 }
 
@@ -366,6 +449,11 @@ message HasIdCondition {
   repeated PointId has_id = 1;
 }
 
+message NestedCondition {
+  string key = 1; // Path to nested object
+  Filter filter = 2; // Filter condition
+}
+
 message FieldCondition {
   string key = 1;
   Match match = 2; // Check if point has field with a given value
@@ -383,6 +471,8 @@ message Match {
     string text = 4; // Match text
     RepeatedStrings keywords = 5; // Match multiple keywords
     RepeatedIntegers integers = 6; // Match multiple integers
+    RepeatedIntegers except_integers = 7; // Match any other value except those integers
+    RepeatedStrings except_keywords = 8; // Match any other value except those keywords
   }
 }
 

proto/points_service.proto

@@ -20,6 +20,14 @@ service Points {
    */
   rpc Get (GetPoints) returns (GetResponse) {}
   /*
+  Update named vectors for point
+   */
+  rpc UpdateVectors (UpdatePointVectors) returns (PointsOperationResponse) {}
+  /*
+  Delete named vectors for points
+   */
+  rpc DeleteVectors (DeletePointVectors) returns (PointsOperationResponse) {}
+  /*
   Set payload for points
    */
   rpc SetPayload (SetPayloadPoints) returns (PointsOperationResponse) {}
@@ -52,6 +60,10 @@ service Points {
     */
   rpc SearchBatch (SearchBatchPoints) returns (SearchBatchResponse) {}
   /*
+  Retrieve closest points based on vector similarity and given filtering conditions, grouped by a given field
+   */
+  rpc SearchGroups (SearchPointGroups) returns (SearchGroupsResponse) {}
+  /*
   Iterate over all or filtered points points
   */
   rpc Scroll (ScrollPoints) returns (ScrollResponse) {}
@@ -63,6 +75,10 @@ service Points {
   Look for the points which are closer to stored positive examples and at the same time further to negative examples.
    */
   rpc RecommendBatch (RecommendBatchPoints) returns (RecommendBatchResponse) {}
+    /*
+  Look for the points which are closer to stored positive examples and at the same time further to negative examples, grouped by a given field
+   */
+  rpc RecommendGroups (RecommendPointGroups) returns (RecommendGroupsResponse) {}
   /*
    Count points in collection with given filtering conditions
    */