Support for .akv files to AKV variable replacement. (#2957)

aaronburtle · Aniruddh25 · web-flow · commit 499cd2402085 · 2025-11-26T14:08:05.000-08:00
## Why make this change? Closes #2748 ## What is this change? Adds the option to use a local .akv file instead of Azure Key Vault for @akv('') replacement in the config file during deserialization. Similar to how we handle .env files. ## How was this tested? A new test was added that verifies we are able to do the replacement and get the correct resultant configuration. --------- Co-authored-by: Aniruddh Munde <anmunde@microsoft.com>
diff --git a/src/Config/DeserializationVariableReplacementSettings.cs b/src/Config/DeserializationVariableReplacementSettings.cs
@@ -8,6 +8,7 @@
 using Azure.DataApiBuilder.Service.Exceptions;
 using Azure.Identity;
 using Azure.Security.KeyVault.Secrets;
+using Microsoft.Extensions.Logging;
 
 namespace Azure.DataApiBuilder.Config
 {
@@ -43,19 +44,23 @@ public class DeserializationVariableReplacementSettings
 
         private readonly AzureKeyVaultOptions? _azureKeyVaultOptions;
         private readonly SecretClient? _akvClient;
+        private readonly Dictionary<string, string>? _akvFileSecrets;
+        private readonly ILogger? _logger;
 
         public Dictionary<Regex, Func<Match, string>> ReplacementStrategies { get; private set; } = new();
 
         public DeserializationVariableReplacementSettings(
             AzureKeyVaultOptions? azureKeyVaultOptions = null,
             bool doReplaceEnvVar = false,
             bool doReplaceAkvVar = false,
-            EnvironmentVariableReplacementFailureMode envFailureMode = EnvironmentVariableReplacementFailureMode.Throw)
+            EnvironmentVariableReplacementFailureMode envFailureMode = EnvironmentVariableReplacementFailureMode.Throw,
+            ILogger? logger = null)
         {
             _azureKeyVaultOptions = azureKeyVaultOptions;
             DoReplaceEnvVar = doReplaceEnvVar;
             DoReplaceAkvVar = doReplaceAkvVar;
             EnvFailureMode = envFailureMode;
+            _logger = logger;
 
             if (DoReplaceEnvVar)
             {
@@ -66,13 +71,68 @@ public DeserializationVariableReplacementSettings(
 
             if (DoReplaceAkvVar && _azureKeyVaultOptions is not null)
             {
-                _akvClient = CreateSecretClient(_azureKeyVaultOptions);
+                // Determine if endpoint points to a local .akv file. If so, load secrets from file; otherwise, use remote AKV.
+                if (IsLocalAkvFileEndpoint(_azureKeyVaultOptions.Endpoint))
+                {
+                    _akvFileSecrets = LoadAkvFileSecrets(_azureKeyVaultOptions.Endpoint!, _logger);
+                }
+                else
+                {
+                    _akvClient = CreateSecretClient(_azureKeyVaultOptions);
+                }
+
                 ReplacementStrategies.Add(
                     new Regex(OUTER_AKV_PATTERN, RegexOptions.Compiled),
                     ReplaceAkvVariable);
             }
         }
 
+        // Checks if the endpoint is a path to a local .akv file.
+        private static bool IsLocalAkvFileEndpoint(string? endpoint)
+            => !string.IsNullOrWhiteSpace(endpoint)
+               && endpoint.EndsWith(".akv", StringComparison.OrdinalIgnoreCase)
+               && File.Exists(endpoint);
+
+        // Loads key=value pairs from a .akv file, similar to .env style. Lines starting with '#' are comments.
+        private static Dictionary<string, string> LoadAkvFileSecrets(string filePath, ILogger? logger = null)
+        {
+            Dictionary<string, string> secrets = new(StringComparer.OrdinalIgnoreCase);
+            foreach (string rawLine in File.ReadAllLines(filePath))
+            {
+                string line = rawLine.Trim();
+                if (string.IsNullOrEmpty(line) || line.StartsWith('#'))
+                {
+                    continue;
+                }
+
+                int eqIndex = line.IndexOf('=');
+                if (eqIndex <= 0)
+                {
+                    logger?.LogDebug("Ignoring malformed line in AKV secrets file {FilePath}: {Line}", filePath, rawLine);
+                    continue;
+                }
+
+                string key = line.Substring(0, eqIndex).Trim();
+                string value = line[(eqIndex + 1)..].Trim();
+
+                // Remove optional surrounding quotes
+                if (value.Length >= 2 && ((value.StartsWith('"') && value.EndsWith('"')) || (value.StartsWith('\'') && value.EndsWith('\''))))
+                {
+                    value = value[1..^1];
+                }
+
+                if (!string.IsNullOrEmpty(key))
+                {
+                    if (!secrets.TryAdd(key, value))
+                    {
+                        logger?.LogDebug("Duplicate key '{Key}' encountered in AKV secrets file {FilePath}. Skipping later value.", key, filePath);
+                    }
+                }
+            }
+
+            return secrets;
+        }
+
         private string ReplaceEnvVariable(Match match)
         {
             // strips first and last characters, ie: '''hello'' --> ''hello'
@@ -170,6 +230,15 @@ private static SecretClient CreateSecretClient(AzureKeyVaultOptions options)
                     DataApiBuilderException.SubStatusCodes.ErrorInInitialization);
             }
 
+            // If endpoint is a local .akv file, we should not create a SecretClient.
+            if (IsLocalAkvFileEndpoint(options.Endpoint))
+            {
+                throw new DataApiBuilderException(
+                    "Attempted to create Azure Key Vault client for local .akv file endpoint.",
+                    System.Net.HttpStatusCode.InternalServerError,
+                    DataApiBuilderException.SubStatusCodes.ErrorInInitialization);
+            }
+
             SecretClientOptions clientOptions = new();
 
             if (options.RetryPolicy is not null)
@@ -195,6 +264,12 @@ private static SecretClient CreateSecretClient(AzureKeyVaultOptions options)
 
         private string? GetAkvVariable(string name)
         {
+            // If using local .akv file secrets, return from dictionary.
+            if (_akvFileSecrets is not null)
+            {
+                return _akvFileSecrets.TryGetValue(name, out string? value) ? value : null;
+            }
+
             if (_akvClient is null)
             {
                 throw new InvalidOperationException("Azure Key Vault client is not initialized.");
diff --git a/src/Service.Tests/UnitTests/RuntimeConfigLoaderJsonDeserializerTests.cs b/src/Service.Tests/UnitTests/RuntimeConfigLoaderJsonDeserializerTests.cs
@@ -13,6 +13,7 @@
 using Azure.DataApiBuilder.Config.Converters;
 using Azure.DataApiBuilder.Config.ObjectModel;
 using Azure.DataApiBuilder.Service.Exceptions;
+using Microsoft.Data.SqlClient;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 
 namespace Azure.DataApiBuilder.Service.Tests.UnitTests
@@ -240,6 +241,7 @@ public void CheckCommentParsingInConfigFile()
         /// but have the effect of default values when deserialized.
         /// It starts with a minimal config and incrementally
         /// adds the optional subproperties. At each step, tests for valid deserialization.
+        /// </summary>
         [TestMethod]
         public void TestNullableOptionalProps()
         {
@@ -431,7 +433,7 @@ public static string GetModifiedJsonString(string[] reps, string enumString)
     ""host"": {
       ""mode"": ""development"",
       ""cors"": {
-        ""origins"": [ """ + reps[++index % reps.Length] + @""", """ + reps[++index % reps.Length] + @""" ],
+        ""origins"": [ """ + reps[++index % reps.Length] + @""", """ + reps[++index % reps.Length] + @"""],
         ""allow-credentials"": true
       },
       ""authentication"": {
@@ -671,5 +673,179 @@ private static bool TryParseAndAssertOnDefaults(string json, out RuntimeConfig p
         #endregion Helper Functions
 
         record StubJsonType(string Foo);
+
+        /// <summary>
+        /// Test to verify Azure Key Vault variable replacement from local .akv file.
+        /// </summary>
+        [TestMethod]
+        public void TestAkvVariableReplacementFromLocalFile()
+        {
+            // Arrange: create a temporary .akv secrets file
+            string akvFilePath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString() + ".akv");
+            string secretConnectionString = "Server=tcp:127.0.0.1,1433;Persist Security Info=False;Trusted_Connection=True;TrustServerCertificate=True;MultipleActiveResultSets=False;Connection Timeout=5;";
+            File.WriteAllText(akvFilePath, $"DBCONN={secretConnectionString}\nAPI_KEY=abcd\n# Comment line should be ignored\n MALFORMEDLINE \n");
+
+            // Escape backslashes for JSON
+            string escapedPath = akvFilePath.Replace("\\", "\\\\");
+
+            string jsonConfig = $$"""
+            {
+              "$schema": "https://github.com/Azure/data-api-builder/releases/download/vmajor.minor.patch-alpha/dab.draft.schema.json",
+              "data-source": {
+                "database-type": "mssql",
+                "connection-string": "@akv('DBCONN')"
+              },
+              "azure-key-vault": {
+                "endpoint": "{{escapedPath}}"
+              },
+              "entities": { }
+            }
+            """;
+
+            try
+            {
+                // Act
+                DeserializationVariableReplacementSettings replacementSettings = new(
+                    azureKeyVaultOptions: null,
+                    doReplaceEnvVar: false,
+                    doReplaceAkvVar: true);
+                bool parsed = RuntimeConfigLoader.TryParseConfig(jsonConfig, out RuntimeConfig config, replacementSettings: replacementSettings);
+
+                // Assert
+                Assert.IsTrue(parsed, "Config should parse successfully with local AKV file replacement.");
+                Assert.IsNotNull(config, "Config should not be null.");
+                Assert.AreEqual(secretConnectionString, config.DataSource.ConnectionString, "Connection string should be replaced from AKV local file secret.");
+            }
+            finally
+            {
+                // Cleanup
+                if (File.Exists(akvFilePath))
+                {
+                    File.Delete(akvFilePath);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Validates that when an AKV secret's value itself contains an @env('...') pattern, it is NOT further resolved
+        /// because replacement only runs once per original JSON token. Demonstrates that nested env patterns inside
+        /// AKV secret values are left intact.
+        /// </summary>
+        [TestMethod]
+        public void TestAkvSecretValueContainingEnvPatternIsNotEnvExpanded()
+        {
+            string akvFilePath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString() + ".akv");
+            // Valid MSSQL connection string which embeds an @env('env') pattern in the Database value.
+            // This pattern should NOT be expanded because replacement only runs once on the original JSON token (@akv('DBCONN')).
+            string secretValueWithEnvPattern = "Server=localhost;Database=@env('env');User Id=sa;Password=XXXX;";
+            File.WriteAllText(akvFilePath, $"DBCONN={secretValueWithEnvPattern}\n");
+            string escapedPath = akvFilePath.Replace("\\", "\\\\");
+
+            // Set env variable to prove it would be different if expansion occurred.
+            Environment.SetEnvironmentVariable("env", "SHOULD_NOT_APPEAR");
+
+            string jsonConfig = $$"""
+            {
+              "$schema": "https://github.com/Azure/data-api-builder/releases/download/vmajor.minor.patch-alpha/dab.draft.schema.json",
+              "data-source": {
+                "database-type": "mssql",
+                "connection-string": "@akv('DBCONN')"
+              },
+              "azure-key-vault": {
+                "endpoint": "{{escapedPath}}"
+              },
+              "entities": { }
+            }
+            """;
+
+            try
+            {
+                DeserializationVariableReplacementSettings replacementSettings = new(
+                    azureKeyVaultOptions: null,
+                    doReplaceEnvVar: true,
+                    doReplaceAkvVar: true);
+                bool parsed = RuntimeConfigLoader.TryParseConfig(jsonConfig, out RuntimeConfig config, replacementSettings: replacementSettings);
+                Assert.IsTrue(parsed, "Config should parse successfully.");
+                Assert.IsNotNull(config);
+
+                string actual = config.DataSource.ConnectionString;
+                Assert.IsTrue(actual.Contains("@env('env')"), "Nested @env pattern inside AKV secret should remain unexpanded.");
+                Assert.IsFalse(actual.Contains("SHOULD_NOT_APPEAR"), "Env var value should not be expanded inside AKV secret.");
+                Assert.IsTrue(actual.Contains("Application Name="), "Application Name should be appended for MSSQL when env replacement is enabled.");
+
+                var builderOriginal = new SqlConnectionStringBuilder(secretValueWithEnvPattern.Replace("Server=", "Data Source=").Replace("Database=", "Initial Catalog="));
+                var builderActual = new SqlConnectionStringBuilder(actual);
+                Assert.AreEqual(builderOriginal["Data Source"], builderActual["Data Source"], "Server/Data Source should match.");
+                Assert.AreEqual(builderOriginal["Initial Catalog"], builderActual["Initial Catalog"], "Database/Initial Catalog should match (with env pattern retained).");
+                Assert.AreEqual(builderOriginal["User ID"], builderActual["User ID"], "User Id should match.");
+                Assert.AreEqual(builderOriginal["Password"], builderActual["Password"], "Password should match.");
+            }
+            finally
+            {
+                if (File.Exists(akvFilePath))
+                {
+                    File.Delete(akvFilePath);
+                }
+
+                Environment.SetEnvironmentVariable("env", null);
+            }
+        }
+
+        /// <summary>
+        /// Validates two-pass replacement where an env var resolves to an AKV pattern which then resolves to the secret value.
+        /// connection-string = @env('env_variable'), env_variable value = @akv('DBCONN'), AKV secret DBCONN holds the final connection string.
+        /// </summary>
+        [TestMethod]
+        public void TestEnvVariableResolvingToAkvPatternIsExpandedInSecondPass()
+        {
+            string akvFilePath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString() + ".akv");
+            string finalSecretValue = "Server=localhost;Database=Test;User Id=sa;Password=XXXX;";
+            File.WriteAllText(akvFilePath, $"DBCONN={finalSecretValue}\n");
+            string escapedPath = akvFilePath.Replace("\\", "\\\\");
+            Environment.SetEnvironmentVariable("env_variable", "@akv('DBCONN')");
+
+            string jsonConfig = $$"""
+            {
+              "$schema": "https://github.com/Azure/data-api-builder/releases/download/vmajor.minor.patch-alpha/dab.draft.schema.json",
+              "data-source": {
+                "database-type": "mssql",
+                "connection-string": "@env('env_variable')"
+              },
+              "azure-key-vault": {
+                "endpoint": "{{escapedPath}}"
+              },
+              "entities": { }
+            }
+            """;
+
+            try
+            {
+                DeserializationVariableReplacementSettings replacementSettings = new(
+                    azureKeyVaultOptions: null,
+                    doReplaceEnvVar: true,
+                    doReplaceAkvVar: true);
+                bool parsed = RuntimeConfigLoader.TryParseConfig(jsonConfig, out RuntimeConfig config, replacementSettings: replacementSettings);
+                Assert.IsTrue(parsed, "Config should parse successfully.");
+                Assert.IsNotNull(config);
+
+                string expected = RuntimeConfigLoader.GetConnectionStringWithApplicationName(finalSecretValue);
+                var builderExpected = new SqlConnectionStringBuilder(expected);
+                var builderActual = new SqlConnectionStringBuilder(config.DataSource.ConnectionString);
+                Assert.AreEqual(builderExpected["Data Source"], builderActual["Data Source"], "Data Source should match.");
+                Assert.AreEqual(builderExpected["Initial Catalog"], builderActual["Initial Catalog"], "Initial Catalog should match.");
+                Assert.AreEqual(builderExpected["User ID"], builderActual["User ID"], "User ID should match.");
+                Assert.AreEqual(builderExpected["Password"], builderActual["Password"], "Password should match.");
+                Assert.IsTrue(builderActual.ApplicationName?.Contains("dab_"), "Application Name should be appended including product identifier.");
+            }
+            finally
+            {
+                if (File.Exists(akvFilePath))
+                {
+                    File.Delete(akvFilePath);
+                }
+
+                Environment.SetEnvironmentVariable("env_variable", null);
+            }
+        }
     }
 }
