Skip to content

Commit e8e3dd4

Browse files
sa-es-irsa-es-ir
authored
Merge pull request #27 from sa-es-ir/26-extend-string-level-semantics
Extend string level semantics
2 parents ce1facd + e5a60d9 commit e8e3dd4

9 files changed

Lines changed: 513 additions & 31 deletions

‎Directory.Packages.props‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,16 @@
11
<Project>
2-
<PropertyGroup>
3-
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
4-
</PropertyGroup>
5-
<ItemGroup>
6-
<PackageVersion Include="xunit.v3" Version="3.0.1" />
7-
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="10.0.1" />
8-
<PackageVersion Include="Microsoft.Extensions.AI" Version="10.0.1" />
9-
<PackageVersion Include="xunit.runner.visualstudio" Version="3.1.4" />
10-
<PackageVersion Include="coverlet.collector" Version="6.0.4" />
11-
<GlobalPackageReference Include="StyleCop.Analyzers" Version="1.1.118">
12-
<PrivateAssets>all</PrivateAssets>
13-
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
14-
</GlobalPackageReference>
15-
16-
</ItemGroup>
2+
<PropertyGroup>
3+
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
4+
</PropertyGroup>
5+
<ItemGroup>
6+
<PackageVersion Include="xunit.v3" Version="3.2.2" />
7+
<PackageVersion Include="Microsoft.Extensions.AI.Abstractions" Version="10.2.0" />
8+
<PackageVersion Include="Microsoft.Extensions.AI" Version="10.2.0" />
9+
<PackageVersion Include="xunit.runner.visualstudio" Version="3.1.5" />
10+
<PackageVersion Include="coverlet.collector" Version="6.0.4" />
11+
<GlobalPackageReference Include="StyleCop.Analyzers" Version="1.1.118">
12+
<PrivateAssets>all</PrivateAssets>
13+
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
14+
</GlobalPackageReference>
15+
</ItemGroup>
1716
</Project>

‎src/Detester/Abstractions/IDetesterBuilder.cs‎

Lines changed: 68 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,64 @@ public interface IDetesterBuilder
4949
/// <returns>The builder instance for method chaining.</returns>
5050
IDetesterBuilder ShouldContainResponse(string expectedText);
5151

52+
/// <summary>
53+
/// Asserts that the AI response does not contain the specified text.
54+
/// </summary>
55+
/// <param name="unexpectedText">The text that should not be present in the response.</param>
56+
/// <returns>The builder instance for method chaining.</returns>
57+
IDetesterBuilder ShouldNotContainResponse(string unexpectedText);
58+
59+
/// <summary>
60+
/// Asserts that the AI response does not contain any of the specified texts.
61+
/// This ensures that none of the provided texts appear in the response and is
62+
/// equivalent to calling <see cref="ShouldNotContainResponse(string)"/> once
63+
/// for each value in <paramref name="unexpectedTexts"/>.
64+
/// </summary>
65+
/// <param name="unexpectedTexts">The texts that must not be present anywhere in the response.</param>
66+
/// <returns>The builder instance for method chaining.</returns>
67+
IDetesterBuilder ShouldNotContainAnyResponse(params string[] unexpectedTexts);
68+
69+
/// <summary>
70+
/// Asserts that the AI response matches the specified regular expression pattern.
71+
/// </summary>
72+
/// <param name="pattern">The regular expression pattern the response must match.</param>
73+
/// <returns>The builder instance for method chaining.</returns>
74+
IDetesterBuilder ShouldMatchRegex(string pattern);
75+
76+
/// <summary>
77+
/// Asserts that the AI response does not contain the specified text.
78+
/// Alias for <see cref="ShouldNotContainResponse"/> for semantic clarity.
79+
/// </summary>
80+
/// <param name="unexpectedText">The text that should not be present in the response.</param>
81+
/// <returns>The builder instance for method chaining.</returns>
82+
IDetesterBuilder ShouldNotContain(string unexpectedText);
83+
84+
/// <summary>
85+
/// Asserts that the AI response contains all of the specified substrings.
86+
/// </summary>
87+
/// <param name="expectedSubstrings">The substrings that must all be present in the response.</param>
88+
/// <returns>The builder instance for method chaining.</returns>
89+
IDetesterBuilder ShouldContainAll(params string[] expectedSubstrings);
90+
91+
/// <summary>
92+
/// Asserts that the AI response contains at least one of the specified substrings.
93+
/// </summary>
94+
/// <param name="expectedSubstrings">The substrings where at least one must be present in the response.</param>
95+
/// <returns>The builder instance for method chaining.</returns>
96+
IDetesterBuilder ShouldContainAny(params string[] expectedSubstrings);
97+
98+
/// <summary>
99+
/// Asserts that the AI response is equal to the specified text using the provided string comparison.
100+
/// By default, this uses a case-insensitive comparison for consistency with other string assertion methods.
101+
/// </summary>
102+
/// <param name="expected">The expected response text.</param>
103+
/// <param name="comparison">
104+
/// The string comparison to use when comparing the response to the expected text.
105+
/// Defaults to <see cref="StringComparison.OrdinalIgnoreCase"/>.
106+
/// </param>
107+
/// <returns>The builder instance for method chaining.</returns>
108+
IDetesterBuilder ShouldBeEqualTo(string expected, StringComparison comparison = StringComparison.OrdinalIgnoreCase);
109+
52110
/// <summary>
53111
/// Asserts that the AI response contains the specified text as an alternative to the previous assertion.
54112
/// This creates an OR condition where at least one of the options in the OR group must match.
@@ -57,6 +115,16 @@ public interface IDetesterBuilder
57115
/// <returns>The builder instance for method chaining.</returns>
58116
IDetesterBuilder OrShouldContainResponse(string expectedText);
59117

118+
/// <summary>
119+
/// Asserts that the AI response contains valid JSON that can be deserialized to the specified type.
120+
/// Optionally validates the deserialized object using a predicate function.
121+
/// </summary>
122+
/// <typeparam name="T">The type to deserialize the JSON response into.</typeparam>
123+
/// <param name="options">JSON serializer options to use for deserialization. If null, default options are used.</param>
124+
/// <param name="validator">Optional predicate to validate the deserialized object. Returns true if validation passes.</param>
125+
/// <returns>The builder instance for method chaining.</returns>
126+
IDetesterBuilder ShouldHaveJsonOfType<T>(System.Text.Json.JsonSerializerOptions? options = null, Func<T, bool>? validator = null);
127+
60128
/// <summary>
61129
/// Asserts that the AI model called the specified function/tool.
62130
/// </summary>
@@ -72,16 +140,6 @@ public interface IDetesterBuilder
72140
/// <returns>The builder instance for method chaining.</returns>
73141
IDetesterBuilder ShouldCallFunctionWithParameters(string functionName, IDictionary<string, object?> expectedParameters);
74142

75-
/// <summary>
76-
/// Asserts that the AI response contains valid JSON that can be deserialized to the specified type.
77-
/// Optionally validates the deserialized object using a predicate function.
78-
/// </summary>
79-
/// <typeparam name="T">The type to deserialize the JSON response into.</typeparam>
80-
/// <param name="options">JSON serializer options to use for deserialization. If null, default options are used.</param>
81-
/// <param name="validator">Optional predicate to validate the deserialized object. Returns true if validation passes.</param>
82-
/// <returns>The builder instance for method chaining.</returns>
83-
IDetesterBuilder ShouldHaveJsonOfType<T>(System.Text.Json.JsonSerializerOptions? options = null, Func<T, bool>? validator = null);
84-
85143
/// <summary>
86144
/// Asserts the test asynchronously by executing the configured prompts and validating responses.
87145
/// </summary>

‎src/Detester/Detester.csproj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
<!-- NuGet Package Metadata -->
1111
<PackageId>Detester</PackageId>
12-
<Version>1.0.1</Version>
12+
<Version>1.0.2</Version>
1313
<Authors>Saeed Esmaeelinejad</Authors>
1414
<Company>Detester</Company>
1515
<PackageIcon>packageIcon.jpeg</PackageIcon>

‎src/Detester/DetesterBuilder.cs‎

Lines changed: 197 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,12 @@ public class DetesterBuilder : IDetesterBuilder
1313
private readonly List<string> prompts = [];
1414
private readonly List<string> expectedResponses = [];
1515
private readonly List<List<string>> orResponseGroups = [];
16+
private readonly List<string> unexpectedResponses = [];
17+
private readonly List<string> unexpectedAnyResponses = [];
18+
private readonly List<string> regexPatterns = [];
19+
private readonly List<string> containAllSubstrings = [];
20+
private readonly List<List<string>> containAnyGroups = [];
21+
private readonly List<EqualityExpectation> equalityExpectations = [];
1622
private readonly List<FunctionCallExpectation> expectedFunctionCalls = [];
1723
private readonly List<JsonExpectation> jsonExpectations = [];
1824
private string? instruction;
@@ -144,6 +150,119 @@ public IDetesterBuilder ShouldContainResponse(string expectedText)
144150
return this;
145151
}
146152

153+
/// <inheritdoc/>
154+
public IDetesterBuilder ShouldNotContainResponse(string unexpectedText)
155+
{
156+
if (string.IsNullOrWhiteSpace(unexpectedText))
157+
{
158+
throw new ArgumentException("Unexpected text cannot be null or whitespace.", nameof(unexpectedText));
159+
}
160+
161+
unexpectedResponses.Add(unexpectedText);
162+
return this;
163+
}
164+
165+
/// <inheritdoc/>
166+
public IDetesterBuilder ShouldNotContainAnyResponse(params string[] unexpectedTexts)
167+
{
168+
if (unexpectedTexts == null || unexpectedTexts.Length == 0)
169+
{
170+
throw new ArgumentException("Unexpected texts cannot be null or empty.", nameof(unexpectedTexts));
171+
}
172+
173+
foreach (var text in unexpectedTexts)
174+
{
175+
if (string.IsNullOrWhiteSpace(text))
176+
{
177+
throw new ArgumentException("Individual unexpected texts cannot be null or whitespace.", nameof(unexpectedTexts));
178+
}
179+
180+
unexpectedAnyResponses.Add(text);
181+
}
182+
183+
return this;
184+
}
185+
186+
/// <inheritdoc/>
187+
public IDetesterBuilder ShouldMatchRegex(string pattern)
188+
{
189+
if (string.IsNullOrWhiteSpace(pattern))
190+
{
191+
throw new ArgumentException("Pattern cannot be null or whitespace.", nameof(pattern));
192+
}
193+
194+
regexPatterns.Add(pattern);
195+
return this;
196+
}
197+
198+
/// <inheritdoc/>
199+
public IDetesterBuilder ShouldNotContain(string unexpectedText)
200+
{
201+
return ShouldNotContainResponse(unexpectedText);
202+
}
203+
204+
/// <inheritdoc/>
205+
public IDetesterBuilder ShouldContainAll(params string[] expectedSubstrings)
206+
{
207+
if (expectedSubstrings == null || expectedSubstrings.Length == 0)
208+
{
209+
throw new ArgumentException("Expected substrings cannot be null or empty.", nameof(expectedSubstrings));
210+
}
211+
212+
foreach (var substring in expectedSubstrings)
213+
{
214+
if (string.IsNullOrWhiteSpace(substring))
215+
{
216+
throw new ArgumentException("Individual expected substrings cannot be null or whitespace.", nameof(expectedSubstrings));
217+
}
218+
219+
containAllSubstrings.Add(substring);
220+
}
221+
222+
return this;
223+
}
224+
225+
/// <inheritdoc/>
226+
public IDetesterBuilder ShouldContainAny(params string[] expectedSubstrings)
227+
{
228+
if (expectedSubstrings == null || expectedSubstrings.Length == 0)
229+
{
230+
throw new ArgumentException("Expected substrings cannot be null or empty.", nameof(expectedSubstrings));
231+
}
232+
233+
var group = new List<string>();
234+
235+
foreach (var substring in expectedSubstrings)
236+
{
237+
if (string.IsNullOrWhiteSpace(substring))
238+
{
239+
throw new ArgumentException("Individual expected substrings cannot be null or whitespace.", nameof(expectedSubstrings));
240+
}
241+
242+
group.Add(substring);
243+
}
244+
245+
containAnyGroups.Add(group);
246+
return this;
247+
}
248+
249+
/// <inheritdoc/>
250+
public IDetesterBuilder ShouldBeEqualTo(string expected, StringComparison comparison = StringComparison.OrdinalIgnoreCase)
251+
{
252+
if (expected is null)
253+
{
254+
throw new ArgumentNullException(nameof(expected));
255+
}
256+
257+
equalityExpectations.Add(new EqualityExpectation
258+
{
259+
Expected = expected,
260+
Comparison = comparison,
261+
});
262+
263+
return this;
264+
}
265+
147266
/// <inheritdoc/>
148267
public IDetesterBuilder OrShouldContainResponse(string expectedText)
149268
{
@@ -248,8 +367,11 @@ public async Task AssertAsync(CancellationToken cancellationToken = default)
248367

249368
chatHistory.Add(new ChatMessage(ChatRole.Assistant, response.Text));
250369

251-
// Check if response contains expected text for any of the assertions
252-
if (expectedResponses.Count > 0 || orResponseGroups.Count > 0)
370+
// Check if response contains expected or unexpected text for any of the assertions
371+
if (expectedResponses.Count > 0 || orResponseGroups.Count > 0 ||
372+
unexpectedResponses.Count > 0 || unexpectedAnyResponses.Count > 0 ||
373+
regexPatterns.Count > 0 || containAllSubstrings.Count > 0 ||
374+
containAnyGroups.Count > 0 || equalityExpectations.Count > 0)
253375
{
254376
var responseText = response.Text ?? string.Empty;
255377

@@ -266,6 +388,79 @@ public async Task AssertAsync(CancellationToken cancellationToken = default)
266388
$"Actual response: {responseText}");
267389
}
268390

391+
// Check individual NOT-CONTAINS assertions
392+
var violatingUnexpected = unexpectedResponses
393+
.Where(unexpected => responseText.Contains(unexpected, StringComparison.OrdinalIgnoreCase))
394+
.ToList();
395+
396+
if (violatingUnexpected.Count > 0)
397+
{
398+
var violatingText = string.Join(", ", violatingUnexpected.Select(e => $"'{e}'"));
399+
throw new DetesterException(
400+
$"Response contained unexpected text(s): {violatingText}. " +
401+
$"Actual response: {responseText}");
402+
}
403+
404+
// Check NOT-CONTAINS-ANY assertions (ensure all specified texts are absent)
405+
var violatingAny = unexpectedAnyResponses
406+
.Where(unexpected => responseText.Contains(unexpected, StringComparison.OrdinalIgnoreCase))
407+
.ToList();
408+
409+
if (violatingAny.Count > 0)
410+
{
411+
var violatingText = string.Join(", ", violatingAny.Select(e => $"'{e}'"));
412+
throw new DetesterException(
413+
$"Response contained one or more texts that should not appear: {violatingText}. " +
414+
$"Actual response: {responseText}");
415+
}
416+
417+
// Check regex patterns
418+
foreach (var pattern in regexPatterns)
419+
{
420+
if (!System.Text.RegularExpressions.Regex.IsMatch(responseText, pattern))
421+
{
422+
throw new DetesterException(
423+
$"Response did not match the required regular expression pattern '{pattern}'. " +
424+
$"Actual response: {responseText}");
425+
}
426+
}
427+
428+
// Check that response contains all required substrings
429+
var missingAllSubstrings = containAllSubstrings
430+
.Where(s => !responseText.Contains(s, StringComparison.OrdinalIgnoreCase))
431+
.ToList();
432+
433+
if (missingAllSubstrings.Count > 0)
434+
{
435+
var missingText = string.Join(", ", missingAllSubstrings.Select(e => $"'{e}'"));
436+
throw new DetesterException(
437+
$"Response did not contain all required substrings: {missingText}. " +
438+
$"Actual response: {responseText}");
439+
}
440+
441+
// Check ANY-groups (at least one in each group must match)
442+
foreach (var group in containAnyGroups)
443+
{
444+
var hasAny = group.Any(s => responseText.Contains(s, StringComparison.OrdinalIgnoreCase));
445+
if (!hasAny)
446+
{
447+
var options = string.Join("' OR '", group);
448+
throw new DetesterException(
449+
$"Response did not contain any of the required alternatives: '{options}'. " +
450+
$"Actual response: {responseText}");
451+
}
452+
}
453+
454+
// Check equality expectations
455+
foreach (var expectation in equalityExpectations)
456+
{
457+
if (!string.Equals(responseText, expectation.Expected, expectation.Comparison))
458+
{
459+
throw new DetesterException(
460+
$"Response was not equal to the expected text. Expected: '{expectation.Expected}', Actual: '{responseText}'.");
461+
}
462+
}
463+
269464
// Check OR assertions (at least one in each OR group must match)
270465
foreach (var orGroup in orResponseGroups)
271466
{

‎src/Detester/EqualityExpectation.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
// Copyright (c) Detester. All rights reserved.
2+
3+
namespace Detester;
4+
5+
internal sealed class EqualityExpectation
6+
{
7+
public string Expected { get; set; } = string.Empty;
8+
9+
public StringComparison Comparison { get; set; }
10+
}

‎src/Detester/FunctionCallExpectation.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ namespace Detester;
55
/// <summary>
66
/// Represents an expectation for a function call in an AI response.
77
/// </summary>
8-
internal class FunctionCallExpectation
8+
internal sealed class FunctionCallExpectation
99
{
1010
/// <summary>
1111
/// Gets or sets the name of the function that is expected to be called.

‎src/Detester/JsonExpectation.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ namespace Detester;
77
/// <summary>
88
/// Represents an expectation for JSON deserialization in an AI response.
99
/// </summary>
10-
internal class JsonExpectation
10+
internal sealed class JsonExpectation
1111
{
1212
/// <summary>
1313
/// Gets or sets the type to deserialize the JSON response into.

0 commit comments

Comments
 (0)