Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,21 +1,24 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>netcoreapp2.1</TargetFramework>
<TargetFramework>netcoreapp3.1</TargetFramework>

<IsPackable>false</IsPackable>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Azure.Search" Version="9.0.1" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="2.1.1" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="2.1.1" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="2.1.1" />
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="2.1.1" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="2.1.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.8.0" />
<PackageReference Include="xunit" Version="2.3.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.3.1" />
<PackageReference Include="Azure.Search.Documents" Version="11.1.1" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="3.1.8" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="3.1.8" />
<PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="3.1.8" />
<PackageReference Include="Microsoft.Extensions.Configuration.FileExtensions" Version="3.1.8" />
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="3.1.8" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.7.1" />
<PackageReference Include="xunit" Version="2.4.1" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>

<ItemGroup>
Expand Down
16 changes: 8 additions & 8 deletions AzureSearchToolkit.IntegrationTest/Data.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ class Data
public const string Index = "azure-search-toolkit-integration-test";

const int ExpectedDataCount = 100;

static readonly AzureSearchConnection azureSearchConnection;
static readonly AzureSearchContext azureSearchContext;

Expand Down Expand Up @@ -58,7 +58,7 @@ public void LoadToMemoryFromAzureSearch()
{
throw new InvalidOperationException(
$"Tests expect {ExpectedDataCount} entries but {memory.Count} loaded from AzureSearch index '{Index}'");
}
}
}

public void LoadFromJsonToAzureSearch()
Expand All @@ -72,30 +72,30 @@ public void LoadFromJsonToAzureSearch()
throw createIndexServiceResult.PotentialException.GetException();
}

var countServiceResult = AsyncHelper.RunSync(() => azureSearchHelper.CountDocuments<Listing>(new SearchParameters(), indexName: Index));
var countServiceResult = AsyncHelper.RunSync(() => azureSearchHelper.CountDocuments<Listing>(new SearchOptions(), indexName: Index));

if (!countServiceResult.IsStatusOk() || countServiceResult.Data != ExpectedDataCount)
{
var baseDirectory = AppContext.BaseDirectory;
var mockedDataPath = Path.Combine(baseDirectory, "App_Data\\listings-mocked.json");

var searchParameters = new SearchParameters()
var searchOptions = new SearchOptions()
{
Select = new List<string>() { "id" },
Top = 1000
};

if (countServiceResult.Data > 0)
{
var allDocuments = AsyncHelper.RunSync(() => azureSearchHelper.SearchDocuments<Listing>(searchParameters, indexName: Index));
var allDocuments = AsyncHelper.RunSync(() => azureSearchHelper.SearchDocuments<Listing>(searchOptions, indexName: Index));

AsyncHelper.RunSync(() => azureSearchHelper
.DeleteDocumentsInIndex(allDocuments.Data.Results.Select(q => new Listing { Id = q.Document.Id }), Index));

azureSearchHelper.WaitForSearchOperationCompletion<Listing>(0, Index);
}
var listings = JsonConvert.DeserializeObject<List<Listing>>(File.ReadAllText(mockedDataPath), new JsonSerializerSettings

var listings = JsonConvert.DeserializeObject<List<Listing>>(File.ReadAllText(mockedDataPath), new JsonSerializerOptions
{
Converters = new List<JsonConverter> { new GeographyPointJsonConverter() }
});
Expand All @@ -114,7 +114,7 @@ public void LoadFromJsonToAzureSearch()

public void WaitForSearchOperationCompletion(int numberOfRequiredItemsInSearch)
{
using (var azureSearchHelper = new AzureSearchHelper(LamaConfiguration.Current(), NullLogger.Instance))
using (var azureSearchHelper = new AzureSearchHelper(LamaConfiguration.Current()))
{
azureSearchHelper.WaitForSearchOperationCompletion<Listing>(numberOfRequiredItemsInSearch, Index);
}
Expand Down
4 changes: 2 additions & 2 deletions AzureSearchToolkit.IntegrationTest/DataAssert.cs
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,13 @@ public static void SetDefaultOrderForType<TSource, TKey>(Expression<Func<TSource
}
}

public static void Same<TSource>(Func<IQueryable<TSource>, IQueryable<TSource>> query,
public static void Same<TSource>(Func<IQueryable<TSource>, IQueryable<TSource>> query,
bool useDefaultOrder = true, bool ignoreOrder = false) where TSource : class
{
Same<TSource, TSource>(query, useDefaultOrder, ignoreOrder);
}

public static void Same<TSource, TTarget>(Func<IQueryable<TSource>, IQueryable<TTarget>> query,
public static void Same<TSource, TTarget>(Func<IQueryable<TSource>, IQueryable<TTarget>> query,
bool useDefaultOrder = true, bool ignoreOrder = false) where TSource : class
{
var expectQuery = query(Data.Memory<TSource>());
Expand Down
2 changes: 0 additions & 2 deletions AzureSearchToolkit.IntegrationTest/Models/Listing.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
using AzureSearchToolkit.Attributes;
using Microsoft.Azure.Search;
using Microsoft.Azure.Search.Models;
using Microsoft.Spatial;
using System;
using System.Collections.Generic;
Expand Down
99 changes: 50 additions & 49 deletions AzureSearchToolkit.IntegrationTest/Utilities/AzureSearchHelper.cs
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
using AzureSearchToolkit.IntegrationTest.Configuration;
using AzureSearchToolkit.Logging;
using Azure.Search.Documents;
using Azure.Search.Documents.Indexes;
using Azure.Search.Documents.Models;

using AzureSearchToolkit.IntegrationTest.Configuration;
using AzureSearchToolkit.Request;
using Microsoft.Azure.Search;
using Microsoft.Azure.Search.Models;
using Microsoft.Extensions.Logging;
using System;
using System.Collections.Generic;
using System.Linq;
Expand All @@ -13,25 +15,25 @@ namespace AzureSearchToolkit.IntegrationTest.Utilities
{
class AzureSearchHelper: IDisposable
{
private string _searchName;
private string endpoint;

private LamaConfiguration _configuration;
private SearchServiceClient _serviceClient;
private LamaConfiguration configuration;
private SearchClient serviceClient;

private ILogger _logger { get; set; }

public AzureSearchHelper(LamaConfiguration configuration, ILogger logger)
{
_configuration = configuration;
this.configuration = configuration;
_logger = logger;

var searchKey = _configuration.GetModel().SearchKey;
var searchKey = this.configuration.GetModel().SearchKey;

_searchName = _configuration.GetModel().SearchName;
endpoint = this.configuration.GetModel().SearchName;

if (!string.IsNullOrWhiteSpace(_searchName) && !string.IsNullOrWhiteSpace(searchKey))
if (!string.IsNullOrWhiteSpace(endpoint) && !string.IsNullOrWhiteSpace(searchKey))
{
_serviceClient = new SearchServiceClient(_searchName, new SearchCredentials(searchKey));
serviceClient = new SearchClient(endpoint, new SearchCredentials(searchKey));
}
}

Expand All @@ -44,7 +46,7 @@ public async Task<ServiceResult<bool>> CreateSearchIndex<T>(string indexName = n

try
{
indexExists = await _serviceClient.Indexes.ExistsAsync(indexName);
indexExists = await serviceClient.Indexes.ExistsAsync(indexName);
}
catch (Exception e)
{
Expand All @@ -70,7 +72,7 @@ public async Task<ServiceResult<bool>> CreateSearchIndex<T>(string indexName = n

try
{
var result = await _serviceClient.Indexes.CreateAsync(definition);
var result = await serviceClient.Indexes.CreateAsync(definition);

if (result != null)
{
Expand All @@ -79,7 +81,7 @@ public async Task<ServiceResult<bool>> CreateSearchIndex<T>(string indexName = n
}
catch (Exception e)
{
_logger.Log(TraceEventType.Error, e, null, "Index {0} was not created!", indexName);
_logger.LogError(e, "Index {indexName} was not created!", indexName);
}
}

Expand Down Expand Up @@ -137,14 +139,14 @@ public async Task<ServiceResult<bool>> ChangeDocumentsInIndex<T>(IEnumerable<T>
return serviceResult.CopyStatus(searchIndexCreateServiceResult);
}

var indexActions = new List<IndexAction<T>>();
var indexActions = new List<IndexDocumentsAction<T>>();

var documentCounter = 0;

foreach (var document in documents)
{
var crudType = AzureSearchIndexType.Upload;
IndexAction<T> indexAction = null;
IndexDocumentsAction<T> indexAction = null;

if (crudTypes.Count() > documentCounter)
{
Expand All @@ -158,16 +160,16 @@ public async Task<ServiceResult<bool>> ChangeDocumentsInIndex<T>(IEnumerable<T>
switch (crudType)
{
case AzureSearchIndexType.Upload:
indexAction = IndexAction.Upload(document);
indexAction = IndexDocumentsAction.Upload(document);
break;
case AzureSearchIndexType.Delete:
indexAction = IndexAction.Delete(document);
indexAction = IndexDocumentsAction.Delete(document);
break;
case AzureSearchIndexType.Merge:
indexAction = IndexAction.Merge(document);
indexAction = IndexDocumentsAction.Merge(document);
break;
default:
indexAction = IndexAction.MergeOrUpload(document);
indexAction = IndexDocumentsAction.MergeOrUpload(document);
break;
}

Expand All @@ -177,7 +179,7 @@ public async Task<ServiceResult<bool>> ChangeDocumentsInIndex<T>(IEnumerable<T>
}

var batch = IndexBatch.New(indexActions);
var indexClient = _serviceClient.Indexes.GetClient(indexName);
var indexClient = serviceClient.Indexes.GetClient(indexName);

try
{
Expand All @@ -191,19 +193,18 @@ public async Task<ServiceResult<bool>> ChangeDocumentsInIndex<T>(IEnumerable<T>
// the batch. Depending on your application, you can take compensating actions like delaying and
// retrying. For this simple demo, we just log the failed document keys and continue.
serviceResult.SetException(e);
_logger.Log(TraceEventType.Error, e, null, "Failed to index some of the documents: {0}",
string.Join(", ", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key)));
_logger.LogError(e, $"Failed to index some of the documents: {Environment.NewLine}{{documents}}", e.IndexingResults.Where(r => !r.Succeeded).Select(r => r.Key));
}
catch (Exception e)
{
serviceResult.SetException(e);
_logger.Log(TraceEventType.Error, e, null, "Search index failed");
_logger.LogError(e, "Search index failed");
}

return serviceResult;
}

public async Task<ServiceResult<DocumentSearchResult<T>>> SearchDocuments<T>(SearchParameters searchParameters, string searchText = null,
public async Task<ServiceResult<DocumentSearchResult<T>>> SearchDocuments<T>(SearchOptions searchOptions, string searchText = null,
string indexName = null) where T : class
{
var serviceResult = new ServiceResult<DocumentSearchResult<T>>();
Expand All @@ -215,15 +216,15 @@ public async Task<ServiceResult<DocumentSearchResult<T>>> SearchDocuments<T>(Sea

indexName = GetIndexName<T>(indexName);

var indexClient = _serviceClient.Indexes.GetClient(indexName);
var indexClient = serviceClient.Indexes.GetClient(indexName);

if (indexClient != null)
{
var headers = new Dictionary<string, List<string>>() { { "x-ms-azs-return-searchid", new List<string>() { "true" } } };

try
{
var response = await indexClient.Documents.SearchWithHttpMessagesAsync<T>(searchText, searchParameters, customHeaders: headers);
var response = await indexClient.Documents.SearchWithHttpMessagesAsync<T>(searchText, searchOptions, customHeaders: headers);

if (response.Response.IsSuccessStatusCode)
{
Expand All @@ -233,9 +234,9 @@ public async Task<ServiceResult<DocumentSearchResult<T>>> SearchDocuments<T>(Sea
{
var searchId = headerValues.FirstOrDefault();

_logger.Log(TraceEventType.Information, null, new Dictionary<string, object>
_logger.LogInformation(, new Dictionary<string, object>
{
{"SearchServiceName", _searchName },
{"SearchServiceName", endpoint },
{"SearchId", searchId},
{"IndexName", indexName},
{"QueryTerms", searchText}
Expand All @@ -248,31 +249,30 @@ public async Task<ServiceResult<DocumentSearchResult<T>>> SearchDocuments<T>(Sea
{
serviceResult.SetStatusWithMessage(response.Response.StatusCode, $"Search failed for indexName {indexName}.");

_logger.Log(TraceEventType.Warning, null, null, $"Search failed for indexName {indexName}. Reason: {response.Response.ReasonPhrase}");
_logger.LogWarning("Search failed for indexName {indexName}. Reason: {reason}", indexName, response.Response.ReasonPhrase);
}
}
catch (Exception e)
{
serviceResult
.SetException(e)
.SetMessage($"Search failed for indexName {indexName}.");
_logger.Log(TraceEventType.Error, e, null,
$"Search failed for indexName {indexName}. Query text: {searchText}, Query: {searchParameters.ToString()}");
_logger.LogError(e, "Search failed for indexName {indexName}. Query text: {searchText}, Query: {searchOptions}", indexName, searchText, searchOptions);
}
}

return serviceResult;
}

public async Task<ServiceResult<long>> CountDocuments<T>(SearchParameters searchParameters, string searchText = null,
public async Task<ServiceResult<long>> CountDocuments<T>(SearchOptions searchOptions, string searchText = null,
string indexName = null) where T : class
{
var serviceResult = new ServiceResult<long>();

searchParameters.Top = 0;
searchParameters.IncludeTotalResultCount = true;
searchOptions.Top = 0;
searchOptions.IncludeTotalResultCount = true;

var documentSearchServiceResult = await SearchDocuments<T>(searchParameters, searchText, indexName);
var documentSearchServiceResult = await SearchDocuments<T>(searchOptions, searchText, indexName);

if (documentSearchServiceResult.IsStatusOk())
{
Expand All @@ -286,20 +286,21 @@ public async Task<ServiceResult<long>> CountDocuments<T>(SearchParameters search
return serviceResult;
}

public SearchParameters GetSearchParameters(ApiParameters apiParameters)
public SearchOptions GetSearchParameters(ApiParameters apiParameters)
{
var searchParameters = new SearchParameters
var searchOptions = new SearchOptions
{
IncludeTotalResultCount = true,
IncludeTotalCount = true,
SearchMode = SearchMode.Any,
Top = apiParameters.Limit,
Size = apiParameters.Limit,
Skip = (apiParameters.Page - 1) * apiParameters.Limit,
QueryType = QueryType.Full
QueryType = SearchQueryType.Full
};

if (apiParameters.IsSearchQuery())
{
searchParameters.SearchFields = apiParameters.GetSplittedQueryBy();
var searchFields = searchOptions.SearchFields;
apiParameters.GetSplittedQueryBy().ForEach(f => searchFields.Add(f));
}

var orderBy = "";
Expand Down Expand Up @@ -341,9 +342,9 @@ public SearchParameters GetSearchParameters(ApiParameters apiParameters)
orderBy = "createdAt desc";
}

searchParameters.OrderBy = new List<string> { orderBy };
searchOptions.OrderBy.Add(orderBy);

return searchParameters;
return searchOptions;
}

public string JoinFilters(string firstFilter, string secondFilter, bool isAnd = true)
Expand Down Expand Up @@ -549,10 +550,10 @@ public async Task WaitForSearchOperationCompletionAsync<T>(int numberOfRequiredI
var numberOfRetries = 0;
var numberOfItemsInSearch = -1;

var searchParemeters = new SearchParameters
var searchParemeters = new SearchOptions
{
Top = 1,
IncludeTotalResultCount = true
Size = 1,
IncludeTotalCount = true
};

do
Expand Down Expand Up @@ -595,7 +596,7 @@ private string GetIndexName(string index)

public void Dispose()
{
_serviceClient.Dispose();
serviceClient.Dispose();
}
}
}
Loading