Add Retry and Retry with RetryExceptions examples (#1099)

* Add Retry and Retry with RetryExceptions examples

* Update to latest nunit alpha package

* Add a note that RetryExceptions is only available from NUnit 4.5.0 d7f76c4

Author: SeanKilleen

articles/nunit/getting-started/dotnet-core-and-dotnet-standard.html

@@ -116,7 +116,7 @@ <h2 id="tldr">TL;DR</h2>
 
     &lt;ItemGroup&gt;
         &lt;PackageReference Include=&quot;Microsoft.NET.Test.Sdk&quot; Version=&quot;18.0.1&quot; /&gt;
-        &lt;PackageReference Include=&quot;NUnit&quot; Version=&quot;4.4.0&quot; /&gt;
+        &lt;PackageReference Include=&quot;NUnit&quot; Version=&quot;4.5.0-alpha.0.17&quot; /&gt;
         &lt;PackageReference Include=&quot;NUnit3TestAdapter&quot; Version=&quot;5.2.0&quot; /&gt;
         &lt;PackageReference Include=&quot;NUnit.Analyzers&quot; Version=&quot;4.11.2&quot;&gt;
             &lt;PrivateAssets&gt;all&lt;/PrivateAssets&gt;

articles/nunit/writing-tests/attributes/retry.html

@@ -87,17 +87,76 @@
 <h1 id="retry">Retry</h1>
 
 <p>RetryAttribute is used on a test method to specify that it should be rerun if it fails, up to a maximum number of times.</p>
+<pre><code class="lang-csharp" name="Retry">[TestFixture]
+public sealed class RetryTests
+{
+    private readonly Random _random = new(42);
+
+    [Test]
+    [Retry(5)]
+    public async Task OperationShouldPassIn1s()
+    {
+        var sw = Stopwatch.StartNew();
+        string result = await ExpensiveOperation();
+        sw.Stop();
+        Assert.That(sw.ElapsedMilliseconds, Is.LessThan(1000), &quot;Operation did not complete in time&quot;);
+        Assert.That(result, Is.Not.Null);
+    }
+
+    private async Task&lt;string&gt; ExpensiveOperation()
+    {
+        // Simulate an expensive operation
+        int duration = _random.Next(500, 1500);
+        await Task.Delay(duration); // Simulate work
+        return &quot;Actual Result&quot;; // Simulate a response
+    }
+}
+</code></pre>
 <p>Notes:</p>
 <ol>
-<li>The argument you specify is the total number of attempts and <strong>not</strong> the number of retries after an initial failure.
-So <code>[Retry(1)]</code> does nothing and should not be used.</li>
-<li>It is not currently possible to use <code>RetryAttribute</code> on a <code>TestFixture</code> or any other type of test suite. Only single
-tests may be repeated.</li>
-<li>If a test has an unexpected exception, an error result is returned and it is not retried. Only assertion failures can
-trigger a retry. To convert an unexpected exception into an assertion failure, see the
-<a class="xref" href="../constraints/ThrowsConstraint.html">ThrowsConstraint</a>.</li>
+<li><p>The argument you specify is the total number of attempts and <strong>not</strong> the number of retries after an initial failure.
+So <code>[Retry(1)]</code> does nothing and should not be used.</p>
+</li>
+<li><p>It is not currently possible to use <code>RetryAttribute</code> on a <code>TestFixture</code> or any other type of test suite.
+Only single tests may be repeated.</p>
+</li>
+<li><p>If a test has an unexpected exception, an error result is returned and it is not retried.</p>
+<p>From NUnit 4.5.0 you can enable retry on an expected exception such as <code>TimeoutException</code>
+by setting the <code>RetryExceptions</code> property. The value of this property is an array of anticipated exceptions that should be retried.</p>
+</li>
 </ol>
+<pre><code class="lang-csharp" name="RetryWithRetryExceptions">[TestFixture]
+public sealed class Retry
+{
+    private int _delayInMilliseconds;
+
+    [OneTimeSetUp]
+    public void Setup()
+    {
+        _delayInMilliseconds = 2500;
+    }
+
+    [Test]
+    [Retry(5, RetryExceptions = [typeof(OperationCanceledException)])]
+    [CancelAfter(2000)]
+    public async Task QueryServiceAsync(CancellationToken cancellationToken)
+    {
+        string result = await CallExternalServiceAsync(cancellationToken);
+        Assert.That(result, Is.Not.Null);
+    }
+
+    private async Task&lt;string&gt; CallExternalServiceAsync(CancellationToken cancellationToken)
+    {
+        // Call an external service that may time out
+        int delayInMilliseconds = _delayInMilliseconds;
+        if (_delayInMilliseconds &gt; 1000)
+            _delayInMilliseconds -= 1000; // Decrease delay for next attempt
 
+        await Task.Delay(delayInMilliseconds, cancellationToken); // Simulate a delay that may exceed
+        return &quot;Actual Result&quot;; // Simulate a response
+    }
+}
+</code></pre>
 </article>
           </div>
 

index.json

@@ -2432,7 +2432,7 @@
   "articles/nunit/getting-started/dotnet-core-and-dotnet-standard.html": {
     "href": "articles/nunit/getting-started/dotnet-core-and-dotnet-standard.html",
     "title": ".NET Core | NUnit Docs",
-    "summary": ".NET Core More information and getting started tutorials are available for NUnit and .NET Core targeting C#, F# and Visual Basic in the .NET Core documentation's Unit Testing in .NET Core page. The other information on this page is older documentation. If you follow the instructions in the Installation section your project will work with .NET Core. The test projects have to be .NET (Core) or .NET Framework; .NET Standard can't be used as a test project, since it can't be run on its own, but any code in a .NET Standard library can be tested from a .NET (Core) or .NET Framework test project. TL;DR Adding the adapter and Microsoft.NET.Test.Sdk version 17.8.0 or greater to your NUnit test projects will also enable the dotnet test command for .NET Core projects. Any tests using the new style CSPROJ format, either .NET Core or .NET 4.x, need to add a PackageReference to the NuGet package Microsoft.NET.Test.Sdk. Your test assemblies must also be .NET Core or .NET 4.x, not .NET Standard. You can create a new NUnit test project using dotnet new nunit. It will create an ItemGroup in the csproj file with the necessary references. The .csproj will look similar to below. <Project Sdk=\"Microsoft.NET.Sdk\"> <PropertyGroup> <TargetFrameworks>net8.0</TargetFrameworks> <ImplicitUsings>enable</ImplicitUsings> <Nullable>enable</Nullable> <NoWarn>NUnit2007;NUnit2009;CS7022</NoWarn> <IsPackable>false</IsPackable> </PropertyGroup> <ItemGroup> <PackageReference Include=\"Microsoft.NET.Test.Sdk\" Version=\"18.0.1\" /> <PackageReference Include=\"NUnit\" Version=\"4.4.0\" /> <PackageReference Include=\"NUnit3TestAdapter\" Version=\"5.2.0\" /> <PackageReference Include=\"NUnit.Analyzers\" Version=\"4.11.2\"> <PrivateAssets>all</PrivateAssets> <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> </PackageReference> <PackageReference Include=\"coverlet.collector\" Version=\"6.0.4\"> <PrivateAssets>all</PrivateAssets> <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> </PackageReference> </ItemGroup> </Project> .NET Core test can be run on the command line with dotnet test, for example, From the solution root folder dotnet test or from the test project folder dotnet test Or you can specify the csproj file you want to test dotnet test .\\test\\NetCore10Tests\\NetCore10Tests.csproj For a more complete walk-through, please see Testing .NET Core with NUnit in Visual Studio 2017 Using the NUnit project templates The NUnit test project templates come included with dotnet. You can run dotnet new nunit to create an NUnit test project. FAQ Why can't my tests target .NET Standard Visual Studio and VSTest require that the tests target a specific platform. .NET Standard is like a Portable library in that it does not target any specific platform, but can run on any supported platform. Microsoft decided that your tests should be compiled to target a platform so they know which platform to run your tests under and you get the behavior you expect for the platform you are targeting. You can however target multiple platforms in your tests and compile and run each from the command line. It still only runs one platform from Visual Studio, but I would hope that is being worked on. I haven't tested 15.3 yet. It is similar to a console application, it cannot be .NET Standard, it must target a platform, .NET Core or .NET Framework. This limitation is the same for all test adapters including xUnit and MSTest2. My tests aren't showing up in Visual Studio 2022 or later Are you using the NuGet adapter package? Are you using version 4.5.0 or newer of the NuGet adapter package? Do your tests target .NET Core or the full .NET Framework? (see above) Have you added a Package Reference to Microsoft.NET.Test.Sdk? Have you restarted Visual Studio? It is still a bit temperamental. Are you doing integration testing with Asp.Net Core WebApplicationFactory and using minimal API, and it complains about missing testhost.deps.json? Then you're accessing the wrong Program class in your WebApplicationFactory. Fix it by adding a code line at the end of the Program.cs file, and verify that your WebApplicationFactory is actually using this Program class: public partial class Program { } The reason for this is that the generated/hidden Minimal API (Top Level statement) Program class is internal. By declaring the partial part, it is changed to public. See Microsoft docs for more info, and some alternatives. My tests multi-target .NET Core and .NET Framework, why can't I run both in Visual Studio This was a limitation in earlier versions of Visual Studio, but is fixed in Visual Studio 2022. If you must run an earlier version and have this issue, you can run specific tests using the --framework command line option of dotnet test How do I produce a test results file dotnet test can generate an NUnit3 test result file by adding a runsettings property. The property to add is TestOutputXml. This generation is done using the NUnit Engine report service, and produce the same result as the NUnit3-console. This is available through the NUnit3TestAdapter. You run it by adding the setting on the command line (or in a runsettings file): dotnet test -- NUnit.TestOutputXml=yourfoldername The folder is relative to a base folder determined by the OutputXmlFolder settings, or you can use an absolute path. The latter is useful in CI scenarios. Alternatively, there is a 3rd party package, NUnitXml.TestLogger which also produces a NUnit3 xml format. Details for use see here. Note that this is a re-implementation of the NUnit3 format, and may differ."
+    "summary": ".NET Core More information and getting started tutorials are available for NUnit and .NET Core targeting C#, F# and Visual Basic in the .NET Core documentation's Unit Testing in .NET Core page. The other information on this page is older documentation. If you follow the instructions in the Installation section your project will work with .NET Core. The test projects have to be .NET (Core) or .NET Framework; .NET Standard can't be used as a test project, since it can't be run on its own, but any code in a .NET Standard library can be tested from a .NET (Core) or .NET Framework test project. TL;DR Adding the adapter and Microsoft.NET.Test.Sdk version 17.8.0 or greater to your NUnit test projects will also enable the dotnet test command for .NET Core projects. Any tests using the new style CSPROJ format, either .NET Core or .NET 4.x, need to add a PackageReference to the NuGet package Microsoft.NET.Test.Sdk. Your test assemblies must also be .NET Core or .NET 4.x, not .NET Standard. You can create a new NUnit test project using dotnet new nunit. It will create an ItemGroup in the csproj file with the necessary references. The .csproj will look similar to below. <Project Sdk=\"Microsoft.NET.Sdk\"> <PropertyGroup> <TargetFrameworks>net8.0</TargetFrameworks> <ImplicitUsings>enable</ImplicitUsings> <Nullable>enable</Nullable> <NoWarn>NUnit2007;NUnit2009;CS7022</NoWarn> <IsPackable>false</IsPackable> </PropertyGroup> <ItemGroup> <PackageReference Include=\"Microsoft.NET.Test.Sdk\" Version=\"18.0.1\" /> <PackageReference Include=\"NUnit\" Version=\"4.5.0-alpha.0.17\" /> <PackageReference Include=\"NUnit3TestAdapter\" Version=\"5.2.0\" /> <PackageReference Include=\"NUnit.Analyzers\" Version=\"4.11.2\"> <PrivateAssets>all</PrivateAssets> <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> </PackageReference> <PackageReference Include=\"coverlet.collector\" Version=\"6.0.4\"> <PrivateAssets>all</PrivateAssets> <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> </PackageReference> </ItemGroup> </Project> .NET Core test can be run on the command line with dotnet test, for example, From the solution root folder dotnet test or from the test project folder dotnet test Or you can specify the csproj file you want to test dotnet test .\\test\\NetCore10Tests\\NetCore10Tests.csproj For a more complete walk-through, please see Testing .NET Core with NUnit in Visual Studio 2017 Using the NUnit project templates The NUnit test project templates come included with dotnet. You can run dotnet new nunit to create an NUnit test project. FAQ Why can't my tests target .NET Standard Visual Studio and VSTest require that the tests target a specific platform. .NET Standard is like a Portable library in that it does not target any specific platform, but can run on any supported platform. Microsoft decided that your tests should be compiled to target a platform so they know which platform to run your tests under and you get the behavior you expect for the platform you are targeting. You can however target multiple platforms in your tests and compile and run each from the command line. It still only runs one platform from Visual Studio, but I would hope that is being worked on. I haven't tested 15.3 yet. It is similar to a console application, it cannot be .NET Standard, it must target a platform, .NET Core or .NET Framework. This limitation is the same for all test adapters including xUnit and MSTest2. My tests aren't showing up in Visual Studio 2022 or later Are you using the NuGet adapter package? Are you using version 4.5.0 or newer of the NuGet adapter package? Do your tests target .NET Core or the full .NET Framework? (see above) Have you added a Package Reference to Microsoft.NET.Test.Sdk? Have you restarted Visual Studio? It is still a bit temperamental. Are you doing integration testing with Asp.Net Core WebApplicationFactory and using minimal API, and it complains about missing testhost.deps.json? Then you're accessing the wrong Program class in your WebApplicationFactory. Fix it by adding a code line at the end of the Program.cs file, and verify that your WebApplicationFactory is actually using this Program class: public partial class Program { } The reason for this is that the generated/hidden Minimal API (Top Level statement) Program class is internal. By declaring the partial part, it is changed to public. See Microsoft docs for more info, and some alternatives. My tests multi-target .NET Core and .NET Framework, why can't I run both in Visual Studio This was a limitation in earlier versions of Visual Studio, but is fixed in Visual Studio 2022. If you must run an earlier version and have this issue, you can run specific tests using the --framework command line option of dotnet test How do I produce a test results file dotnet test can generate an NUnit3 test result file by adding a runsettings property. The property to add is TestOutputXml. This generation is done using the NUnit Engine report service, and produce the same result as the NUnit3-console. This is available through the NUnit3TestAdapter. You run it by adding the setting on the command line (or in a runsettings file): dotnet test -- NUnit.TestOutputXml=yourfoldername The folder is relative to a base folder determined by the OutputXmlFolder settings, or you can use an absolute path. The latter is useful in CI scenarios. Alternatively, there is a 3rd party package, NUnitXml.TestLogger which also produces a NUnit3 xml format. Details for use see here. Note that this is a re-implementation of the NUnit3 format, and may differ."
   },
   "articles/nunit/getting-started/installation.html": {
     "href": "articles/nunit/getting-started/installation.html",
@@ -3122,7 +3122,7 @@
   "articles/nunit/writing-tests/attributes/retry.html": {
     "href": "articles/nunit/writing-tests/attributes/retry.html",
     "title": "Retry | NUnit Docs",
-    "summary": "Retry RetryAttribute is used on a test method to specify that it should be rerun if it fails, up to a maximum number of times. Notes: The argument you specify is the total number of attempts and not the number of retries after an initial failure. So [Retry(1)] does nothing and should not be used. It is not currently possible to use RetryAttribute on a TestFixture or any other type of test suite. Only single tests may be repeated. If a test has an unexpected exception, an error result is returned and it is not retried. Only assertion failures can trigger a retry. To convert an unexpected exception into an assertion failure, see the ThrowsConstraint."
+    "summary": "Retry RetryAttribute is used on a test method to specify that it should be rerun if it fails, up to a maximum number of times. [TestFixture] public sealed class RetryTests { private readonly Random _random = new(42); [Test] [Retry(5)] public async Task OperationShouldPassIn1s() { var sw = Stopwatch.StartNew(); string result = await ExpensiveOperation(); sw.Stop(); Assert.That(sw.ElapsedMilliseconds, Is.LessThan(1000), \"Operation did not complete in time\"); Assert.That(result, Is.Not.Null); } private async Task<string> ExpensiveOperation() { // Simulate an expensive operation int duration = _random.Next(500, 1500); await Task.Delay(duration); // Simulate work return \"Actual Result\"; // Simulate a response } } Notes: The argument you specify is the total number of attempts and not the number of retries after an initial failure. So [Retry(1)] does nothing and should not be used. It is not currently possible to use RetryAttribute on a TestFixture or any other type of test suite. Only single tests may be repeated. If a test has an unexpected exception, an error result is returned and it is not retried. From NUnit 4.5.0 you can enable retry on an expected exception such as TimeoutException by setting the RetryExceptions property. The value of this property is an array of anticipated exceptions that should be retried. [TestFixture] public sealed class Retry { private int _delayInMilliseconds; [OneTimeSetUp] public void Setup() { _delayInMilliseconds = 2500; } [Test] [Retry(5, RetryExceptions = [typeof(OperationCanceledException)])] [CancelAfter(2000)] public async Task QueryServiceAsync(CancellationToken cancellationToken) { string result = await CallExternalServiceAsync(cancellationToken); Assert.That(result, Is.Not.Null); } private async Task<string> CallExternalServiceAsync(CancellationToken cancellationToken) { // Call an external service that may time out int delayInMilliseconds = _delayInMilliseconds; if (_delayInMilliseconds > 1000) _delayInMilliseconds -= 1000; // Decrease delay for next attempt await Task.Delay(delayInMilliseconds, cancellationToken); // Simulate a delay that may exceed return \"Actual Result\"; // Simulate a response } }"
   },
   "articles/nunit/writing-tests/attributes/sequential.html": {
     "href": "articles/nunit/writing-tests/attributes/sequential.html",