Avoid SQL connection flakines in integration tests (#7854)

NachoEchevarria · web-flow · commit 2d191f038c79 · 2025-11-28T11:42:40.000+01:00
## Summary of changes We are getting [the following error](https://dev.azure.com/datadoghq/a51c4863-3eb4-4c5d-878a-58b41a049e4e/_apis/build/builds/191432/logs/22447) in our integration tests: ``` 2025-11-21T04:47:15.5891593Z 04:47:15 [DBG] Stack Trace: 2025-11-21T04:47:15.5892025Z 04:47:15 [DBG] at Datadog.Trace.TestHelpers.ExitCodeException.Throw(Int32 actualExitCode, Int32 expectedExitCode, String message) in /project/tracer/test/Datadog.Trace.TestHelpers/ExitCodeException.cs:line 44 2025-11-21T04:47:15.5892326Z 04:47:15 [DBG] at Datadog.Trace.TestHelpers.TestHelper.WaitForProcessResult(ProcessHelper helper, Int32 expectedExitCode, Boolean dumpChildProcesses) in /project/tracer/test/Datadog.Trace.TestHelpers.AutoInstrumentation/TestHelper.cs:line 212 2025-11-21T04:47:15.5892650Z 04:47:15 [DBG] at Datadog.Trace.TestHelpers.TestHelper.RunSampleAndWaitForExit(MockTracerAgent agent, String arguments, String packageVersion, String framework, Int32 aspNetCorePort, Boolean usePublishWithRID, String dotnetRuntimeArgs) in /project/tracer/test/Datadog.Trace.TestHelpers.AutoInstrumentation/TestHelper.cs:line 172 2025-11-21T04:47:15.5893224Z 04:47:15 [DBG] at Datadog.Trace.ClrProfiler.IntegrationTests.AdoNet.MicrosoftDataSqlClientTests.SubmitsTraces(String packageVersion, String metadataSchemaVersion, String propagation) in /project/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AdoNet/MicrosoftDataSqlClientTests.cs:line 62 2025-11-21T04:47:15.5893396Z 04:47:15 [DBG] --- End of stack trace from previous location --- 2025-11-21T04:47:15.5893558Z 04:47:15 [DBG] Standard Output Messages: 2025-11-21T04:47:15.5893704Z 04:47:15 [DBG] Platform: Arm64 2025-11-21T04:47:15.5893857Z 04:47:15 [DBG] TargetPlatform: ARM64 2025-11-21T04:47:15.5894007Z 04:47:15 [DBG] Configuration: Release 2025-11-21T04:47:15.5894160Z 04:47:15 [DBG] TargetFramework: net8.0 2025-11-21T04:47:15.5894453Z 04:47:15 [DBG] .NET Core: True 2025-11-21T04:47:15.5894656Z 04:47:15 [DBG] Native Loader DLL: /project/shared/bin/monitoring-home/linux-arm64/Datadog.Trace.ClrProfiler.Native.so 2025-11-21T04:47:15.5894828Z 04:47:15 [DBG] Agent listener info: Traces at port 45159 2025-11-21T04:47:15.5895049Z 04:47:15 [DBG] Starting Application: /project/artifacts/publish/Samples.Microsoft.Data.SqlClient/release_net8.0_6.1.3/Samples.Microsoft.Data.SqlClient.dll 2025-11-21T04:47:15.5895201Z 04:47:15 [DBG] ProcessId: 6967 2025-11-21T04:47:15.5895347Z 04:47:15 [DBG] StandardOutput: 2025-11-21T04:47:15.5895700Z 04:47:15 [DBG] Microsoft.Data.SqlClient.SqlException (0x80131904): A network-related or instance-specific error occurred while establishing a connection to SQL Server. The server was not found or was not accessible. Verify that the instance name is correct and that SQL Server is configured to allow remote connections. (provider: TCP Provider, error: 35 - An internal exception was caught) 2025-11-21T04:47:15.5895896Z 04:47:15 [DBG] ---> System.Net.Sockets.SocketException (00000005, 0xFFFDFFFF): Name or service not known 2025-11-21T04:47:15.5896124Z 04:47:15 [DBG] at System.Net.Dns.GetHostEntryOrAddressesCore(String hostName, Boolean justAddresses, AddressFamily addressFamily, Nullable`1 startingTimestamp) 2025-11-21T04:47:15.5896315Z 04:47:15 [DBG] at System.Net.Dns.GetHostAddresses(String hostNameOrAddress, AddressFamily family) 2025-11-21T04:47:15.5896553Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.ManagedSni.SniTcpHandle.GetHostAddressesSortedByPreference(String serverName, SqlConnectionIPAddressPreference ipPreference)+MoveNext() 2025-11-21T04:47:15.5896817Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.ManagedSni.SniTcpHandle.Connect(String serverName, Int32 port, TimeoutTimer timeout, SqlConnectionIPAddressPreference ipPreference, String cachedFQDN, SQLDNSInfo& pendingDNSInfo) 2025-11-21T04:47:15.5897128Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.ManagedSni.SniTcpHandle..ctor(String serverName, Int32 port, TimeoutTimer timeout, Boolean parallel, SqlConnectionIPAddressPreference ipPreference, String cachedFQDN, SQLDNSInfo& pendingDNSInfo, Boolean tlsFirst, String hostNameInCertificate, String serverCertificateFilename) 2025-11-21T04:47:15.5897354Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.SqlInternalConnection.OnError(SqlException exception, Boolean breakConnection, Action`1 wrapCloseInAction) 2025-11-21T04:47:15.5897591Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.TdsParser.ThrowExceptionAndWarning(TdsParserStateObject stateObj, SqlCommand command, Boolean callerHasConnectionLock, Boolean asyncClose) 2025-11-21T04:47:15.5897851Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.TdsParser.Connect(ServerInfo serverInfo, SqlInternalConnectionTds connHandler, TimeoutTimer timeout, SqlConnectionString connectionOptions, Boolean withFailover) 2025-11-21T04:47:15.5898098Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.SqlInternalConnectionTds.AttemptOneLogin(ServerInfo serverInfo, String newPassword, SecureString newSecurePassword, TimeoutTimer timeout, Boolean withFailover) 2025-11-21T04:47:15.5898394Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.SqlInternalConnectionTds.LoginNoFailover(ServerInfo serverInfo, String newPassword, SecureString newSecurePassword, Boolean redirectedUserInstance, SqlConnectionString connectionOptions, SqlCredential credential, TimeoutTimer timeout) 2025-11-21T04:47:15.5898801Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.SqlInternalConnectionTds.OpenLoginEnlist(TimeoutTimer timeout, SqlConnectionString connectionOptions, SqlCredential credential, String newPassword, SecureString newSecurePassword, Boolean redirectedUserInstance) 2025-11-21T04:47:15.5899380Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.SqlInternalConnectionTds..ctor(DbConnectionPoolIdentity identity, SqlConnectionString connectionOptions, SqlCredential credential, Object providerInfo, String newPassword, SecureString newSecurePassword, Boolean redirectedUserInstance, SqlConnectionString userConnectionOptions, SessionData reconnectSessionData, Boolean applyTransientFaultHandling, String access***, IDbConnectionPool pool, Func`3 access***Callback) 2025-11-21T04:47:15.5899800Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.SqlConnectionFactory.CreateConnection(DbConnectionOptions options, DbConnectionPoolKey poolKey, DbConnectionPoolGroupProviderInfo poolGroupProviderInfo, IDbConnectionPool pool, DbConnection owningConnection, DbConnectionOptions userOptions) 2025-11-21T04:47:15.5900076Z 04:47:15 [DBG] at Microsoft.Data.ProviderBase.DbConnectionFactory.CreatePooledConnection(IDbConnectionPool pool, DbConnection owningObject, DbConnectionOptions options, DbConnectionPoolKey poolKey, DbConnectionOptions userOptions) 2025-11-21T04:47:15.5900319Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.ConnectionPool.WaitHandleDbConnectionPool.CreateObject(DbConnection owningObject, DbConnectionOptions userOptions, DbConnectionInternal oldConnection) 2025-11-21T04:47:15.5900568Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.ConnectionPool.WaitHandleDbConnectionPool.UserCreateRequest(DbConnection owningObject, DbConnectionOptions userOptions, DbConnectionInternal oldConnection) 2025-11-21T04:47:15.5900866Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.ConnectionPool.WaitHandleDbConnectionPool.TryGetConnection(DbConnection owningObject, UInt32 waitForMultipleObjectsTimeout, Boolean allowCreate, Boolean onlyOneCheckConnection, DbConnectionOptions userOptions, DbConnectionInternal& connection) 2025-11-21T04:47:15.5901142Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.ConnectionPool.WaitHandleDbConnectionPool.TryGetConnection(DbConnection owningObject, TaskCompletionSource`1 taskCompletionSource, DbConnectionOptions userOptions, DbConnectionInternal& connection) 2025-11-21T04:47:15.5901419Z 04:47:15 [DBG] at Microsoft.Data.ProviderBase.DbConnectionFactory.TryGetConnection(DbConnection owningConnection, TaskCompletionSource`1 retry, DbConnectionOptions userOptions, DbConnectionInternal oldConnection, DbConnectionInternal& connection) 2025-11-21T04:47:15.5901688Z 04:47:15 [DBG] at Microsoft.Data.ProviderBase.DbConnectionInternal.TryOpenConnectionInternal(DbConnection outerConnection, DbConnectionFactory connectionFactory, TaskCompletionSource`1 retry, DbConnectionOptions userOptions) 2025-11-21T04:47:15.5902144Z 04:47:15 [DBG] at Microsoft.Data.ProviderBase.DbConnectionClosed.TryOpenConnection(DbConnection outerConnection, DbConnectionFactory connectionFactory, TaskCompletionSource`1 retry, DbConnectionOptions userOptions) 2025-11-21T04:47:15.5902394Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.SqlConnection.TryOpen(TaskCompletionSource`1 retry, SqlConnectionOverrides overrides) 2025-11-21T04:47:15.5902583Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.SqlConnection.Open(SqlConnectionOverrides overrides) 2025-11-21T04:47:15.5902754Z 04:47:15 [DBG] at Microsoft.Data.SqlClient.SqlConnection.Open() 2025-11-21T04:47:15.5903048Z 04:47:15 [DBG] at Samples.Microsoft.Data.SqlClient.Program.OpenConnection(Type connectionType) in D:\a\_work\1\s\tracer\test\test-applications\integrations\Samples.Microsoft.Data.SqlClient\Program.cs:line 49 ``` CI tests occasionally fail with DNS/network errors (Error 11001, Class 20) when SQL Server isn't immediately available. These are infrastructure issues, not test failures. By catching connection errors and returning exit code 13, the test framework can skip these tests instead of failing the build. SQL exceptions during test execution (after successful connection) will still fail the test as expected. ### Changes - Catch `SqlException` during connection attempts in `Samples.Microsoft.Data.SqlClient` - Exit with code 13 (skip) when SQL Server is unavailable after 3 retry attempts - Refactor retry logic from `goto` to a clean `for` loop - Improve logging to show attempt progress and detailed error information - Add some sleep time between tries ## Reason for change ## Implementation details ## Test coverage ## Other details
diff --git a/tracer/test/test-applications/integrations/Samples.Microsoft.Data.SqlClient/Program.cs b/tracer/test/test-applications/integrations/Samples.Microsoft.Data.SqlClient/Program.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Data.Common;
+using System.Net.Sockets;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Data.SqlClient;
@@ -17,15 +18,28 @@ private static async Task<int> Main()
 
             using (var connection = OpenConnection(typeof(SqlConnection)))
             {
+                if (connection is null)
+                {
+                    Console.WriteLine("No connection could be established. Exiting with skip code (13)");
+                    return 13;
+                }
+
                 await RelationalDatabaseTestHarness.RunAllAsync<SqlCommand>(connection, commandFactory, commandExecutor, cts.Token);
             }
 
             // Test the result when the ADO.NET provider assembly is loaded through Assembly.LoadFile
             // On .NET Core this results in a new assembly being loaded whose types are not considered the same
             // as the types loaded through the default loading mechanism, potentially causing type casting issues in CallSite instrumentation
             var loadFileType = AssemblyHelpers.LoadFileAndRetrieveType(typeof(SqlConnection));
+
             using (var connection = OpenConnection(loadFileType))
             {
+                if (connection is null)
+                {
+                    Console.WriteLine("No connection could be established. Exiting with skip code (13)");
+                    return 13;
+                }
+
                 // Do not use the strongly typed SqlCommandExecutor because the type casts will fail
                 await RelationalDatabaseTestHarness.RunBaseClassesAsync(connection, commandFactory, cts.Token);
             }
@@ -37,31 +51,80 @@ private static async Task<int> Main()
 
         private static DbConnection OpenConnection(Type connectionType)
         {
-            var remainingAttempts = 3;
+            const int maxAttempts = 3;
             var connectionString = Environment.GetEnvironmentVariable("SQLSERVER_CONNECTION_STRING") ??
 @"Server=(localdb)\MSSQLLocalDB;Integrated Security=true;Connection Timeout=60";
 
-            DbConnection connection = null;
-            retry:
-            try
-            {
-                connection = Activator.CreateInstance(connectionType, connectionString) as DbConnection;
-                connection.Open();
-                return connection;
-            }
-            catch (Exception ex)
+            SqlException lastException = null;
+
+            for (int attempt = 1; attempt <= maxAttempts; attempt++)
             {
-                if (remainingAttempts > 0)
+                DbConnection connection = null;
+                try
                 {
-                    Console.WriteLine(ex);
+                    connection = Activator.CreateInstance(connectionType, connectionString) as DbConnection;
+                    connection.Open();
+                    return connection;
+                }
+                catch (SqlException ex) when (IsRetryableConnectionError(ex))
+                {
+                    lastException = ex;
                     connection?.Dispose();
-                    remainingAttempts--;
-                    goto retry;
+
+                    if (attempt < maxAttempts)
+                    {
+                        Console.WriteLine($"Connection attempt {attempt}/{maxAttempts} failed. Retrying...");
+                        Console.WriteLine($"SqlException Number: {ex.Number}, State: {ex.State}, Class: {ex.Class}");
+                        Console.WriteLine($"Message: {ex.Message}");
+                        Thread.Sleep(1000 * attempt);
+                    }
+                }
+                catch (SqlException ex) when (!IsRetryableConnectionError(ex))
+                {
+                    Console.WriteLine($"Fatal SqlException Number: {ex.Number}, State: {ex.State}, Class: {ex.Class}");
+                    Console.WriteLine($"Message: {ex.Message}");
+                    throw;
+                }
+                catch (Exception ex)
+                {
+                    // Other errors (reflection issues, etc.) should fail the test
+                    Console.WriteLine($"Unexpected error opening connection: {ex}");
+                    throw;
                 }
+            }
+
+            // After all retry attempts exhausted, return null to signal connection failure
+            Console.WriteLine($"Unable to establish SQL connection after {maxAttempts} attempts.");
+            if (lastException != null)
+            {
+                Console.WriteLine($"Final SqlException Number: {lastException.Number}, State: {lastException.State}, Class: {lastException.Class}");
+                Console.WriteLine($"Message: {lastException.Message}");
+            }
+            return null;
+        }
 
-                // else
-                throw;
+        static bool IsRetryableConnectionError(SqlException ex)
+        {
+            // Known retryable error codes
+            if (ex.Number == -1 ||      // Generic network error
+                ex.Number == -2 ||      // Connection timeout
+                ex.Number == 53 ||      // SQL Server not found
+                ex.Number == 258 ||     // Connection timeout
+                ex.Number == 10053 ||   // Connection aborted
+                ex.Number == 10054 ||   // Connection reset
+                ex.Number == 10060 ||   // Connection timeout
+                ex.Number == 11001)     // DNS failure
+            {
+                return true;
             }
+
+            // Number=0 with SocketException indicates network issue
+            if (ex.Number == 0 && ex.InnerException is SocketException)
+            {
+                return true;
+            }
+
+            return false;
         }
     }
 }
