Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add vector search for Redis JSON and HashSet #220

Merged
merged 9 commits into from
Sep 23, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package com.microsoft.semantickernel.tests.connectors.memory;
package com.microsoft.semantickernel.tests.connectors.memory.jdbc;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
Expand All @@ -12,7 +12,7 @@ public class Hotel {
@VectorStoreRecordKeyAttribute
private final String id;

@VectorStoreRecordDataAttribute
@VectorStoreRecordDataAttribute(isFilterable = true)
private final String name;

@VectorStoreRecordDataAttribute
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import com.microsoft.semantickernel.data.vectorsearch.VectorSearchResult;
import com.microsoft.semantickernel.data.vectorstorage.options.GetRecordOptions;
import com.microsoft.semantickernel.data.vectorstorage.options.VectorSearchOptions;
import com.microsoft.semantickernel.tests.connectors.memory.Hotel;
import com.mysql.cj.jdbc.MysqlDataSource;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import com.microsoft.semantickernel.connectors.data.jdbc.JDBCVectorStoreRecordCollectionOptions;
import com.microsoft.semantickernel.connectors.data.mysql.MySQLVectorStoreQueryProvider;
import com.microsoft.semantickernel.connectors.data.postgres.PostgreSQLVectorStoreQueryProvider;
import com.microsoft.semantickernel.tests.connectors.memory.Hotel;
import com.mysql.cj.jdbc.MysqlDataSource;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package com.microsoft.semantickernel.tests.connectors.memory.redis;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordDataAttribute;
import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordKeyAttribute;
import com.microsoft.semantickernel.data.vectorstorage.attributes.VectorStoreRecordVectorAttribute;

import java.util.List;

public class Hotel {
@VectorStoreRecordKeyAttribute
private final String id;

@VectorStoreRecordDataAttribute(isFilterable = true)
private final String name;

@VectorStoreRecordDataAttribute
private final int code;

@JsonProperty("summary")
@VectorStoreRecordDataAttribute()
private final String description;

@JsonProperty("summaryEmbedding1")
@VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "euclidean")
private final List<Float> euclidean;

@JsonProperty("summaryEmbedding2")
@VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "cosineDistance")
private final List<Float> cosineDistance;

@JsonProperty("summaryEmbedding3")
@VectorStoreRecordVectorAttribute(dimensions = 8, distanceFunction = "dotProduct")
private final List<Float> dotProduct;
@VectorStoreRecordDataAttribute
private double rating;

public Hotel() {
this(null, null, 0, null, null, null, null, 0.0);
}

@JsonCreator
public Hotel(
@JsonProperty("id") String id,
@JsonProperty("name") String name,
@JsonProperty("code") int code,
@JsonProperty("summary") String description,
@JsonProperty("summaryEmbedding1") List<Float> euclidean,
@JsonProperty("summaryEmbedding2") List<Float> cosineDistance,
@JsonProperty("summaryEmbedding3") List<Float> dotProduct,
@JsonProperty("rating") double rating) {
this.id = id;
this.name = name;
this.code = code;
this.description = description;
this.euclidean = euclidean;
this.cosineDistance = euclidean;
this.dotProduct = euclidean;
this.rating = rating;
}

public String getId() {
return id;
}

public String getName() {
return name;
}

public int getCode() {
return code;
}

public String getDescription() {
return description;
}

public List<Float> getEuclidean() {
return euclidean;
}

public List<Float> getCosineDistance() {
return cosineDistance;
}

public List<Float> getDotProduct() {
return dotProduct;
}

public double getRating() {
return rating;
}

public void setRating(double rating) {
this.rating = rating;
}
}

Large diffs are not rendered by default.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStore;
import com.microsoft.semantickernel.connectors.data.redis.RedisVectorStoreOptions;
import com.microsoft.semantickernel.data.vectorstorage.VectorStoreRecordCollectionOptions;
import com.microsoft.semantickernel.tests.connectors.memory.Hotel;
import com.microsoft.semantickernel.tests.connectors.memory.jdbc.Hotel;
import com.redis.testcontainers.RedisContainer;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
Expand Down
2 changes: 1 addition & 1 deletion connectors/semantickernel-connectors-memory-redis/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>5.1.0</version>
<version>5.2.0-beta5</version>
<scope>provided</scope>
</dependency>
<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -118,17 +118,19 @@ public static void dataStorageWithAzureAISearch(
OpenAITextEmbeddingGenerationService embeddingGeneration) {

// Create a new Azure AI Search vector store
var azureAISearchVectorStore = com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore.builder()
var azureAISearchVectorStore = com.microsoft.semantickernel.connectors.data.azureaisearch.AzureAISearchVectorStore
.builder()
.withSearchIndexAsyncClient(searchClient)
.withOptions(new AzureAISearchVectorStoreOptions())
.build();

String collectionName = "skgithubfiles";
var collection = (AzureAISearchVectorStoreRecordCollection<GitHubFile>) azureAISearchVectorStore.getCollection(
collectionName,
AzureAISearchVectorStoreRecordCollectionOptions.<GitHubFile>builder()
.withRecordClass(GitHubFile.class)
.build());
var collection = (AzureAISearchVectorStoreRecordCollection<GitHubFile>) azureAISearchVectorStore
.getCollection(
collectionName,
AzureAISearchVectorStoreRecordCollectionOptions.<GitHubFile>builder()
.withRecordClass(GitHubFile.class)
.build());

// Create collection if it does not exist and store data
collection
Expand All @@ -146,17 +148,17 @@ public static void dataStorageWithAzureAISearch(
}
var searchResult = results.get(0);
System.out.printf("Search result with score: %f.%n Link: %s, Description: %s%n",
searchResult.getScore(), searchResult.getRecord().link, searchResult.getRecord().description);
searchResult.getScore(), searchResult.getRecord().link,
searchResult.getRecord().description);
}


private static Mono<List<VectorSearchResult<GitHubFile>>> search(
String searchText,
AzureAISearchVectorStoreRecordCollection<GitHubFile> recordCollection,
OpenAITextEmbeddingGenerationService embeddingGeneration) {
String searchText,
AzureAISearchVectorStoreRecordCollection<GitHubFile> recordCollection,
OpenAITextEmbeddingGenerationService embeddingGeneration) {

return embeddingGeneration.generateEmbeddingsAsync(Collections.singletonList(searchText))
.flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null));
.flatMap(r -> recordCollection.searchAsync(r.get(0).getVector(), null));
}

private static Mono<List<String>> storeData(
Expand Down
2 changes: 1 addition & 1 deletion semantickernel-bom/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>5.1.0</version>
<version>5.2.0-beta5</version>
</dependency>

<dependency>
Expand Down
Loading