From 0de3ccfafeac91849875f7d0d9d19e39fd419909 Mon Sep 17 00:00:00 2001 From: Yan Zhang <2351748+Eskibear@users.noreply.github.com> Date: Tue, 9 May 2023 09:45:36 +0800 Subject: [PATCH 01/17] parse requirement type of feature flags (#418) --- .../FeatureManagement/FeatureConditions.cs | 3 + .../FeatureManagementConstants.cs | 1 + .../FeatureManagementKeyValueAdapter.cs | 9 +++ .../FeatureManagementTests.cs | 76 +++++++++++++++++++ 4 files changed, 89 insertions(+) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureConditions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureConditions.cs index 27aa9ee5..6927d310 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureConditions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureConditions.cs @@ -10,5 +10,8 @@ internal class FeatureConditions { [JsonPropertyName("client_filters")] public List ClientFilters { get; set; } = new List(); + + [JsonPropertyName("requirement_type")] + public string RequirementType { get; set; } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs index 1ef33aad..b0e09723 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs @@ -9,5 +9,6 @@ internal class FeatureManagementConstants public const string ContentType = "application/vnd.microsoft.appconfig.ff+json"; public const string SectionName = "FeatureManagement"; public const string EnabledFor = "EnabledFor"; + public const string RequirementType = "RequirementType"; } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs index 713efe8f..3de20b3f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs @@ -51,6 +51,15 @@ public Task>> ProcessKeyValue(Configura keyValues.Add(new KeyValuePair($"{FeatureManagementConstants.SectionName}:{featureFlag.Id}:{FeatureManagementConstants.EnabledFor}:{i}:Parameters:{kvp.Key}", kvp.Value)); } } + + // + // process RequirementType only when filters are not empty + if (featureFlag.Conditions.RequirementType != null) + { + keyValues.Add(new KeyValuePair( + $"{FeatureManagementConstants.SectionName}:{featureFlag.Id}:{FeatureManagementConstants.RequirementType}", + featureFlag.Conditions.RequirementType)); + } } } else diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index dcca8506..c50994e0 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -185,6 +185,21 @@ public class FeatureManagementTests eTag: new ETag("0a76e3d7-7ec1-4e37-883c-9ea6d0d89e63"), contentType: "text"); + readonly ConfigurationSetting Feature_RequirementTypeAll = ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + "Feature_All", + value: @" + { + ""id"": ""Feature_All"", + ""enabled"": true, + ""conditions"": { + ""requirement_type"": ""All"", + ""client_filters"": [] + } + } + ", + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); + TimeSpan CacheExpirationTime = TimeSpan.FromSeconds(1); [Fact] @@ -1129,5 +1144,66 @@ Response GetTestKey(string key, string label, Cancellation { return Response.FromValue(TestHelpers.CloneSetting(FirstKeyValue), new Mock().Object); } + + [Fact] + public void WithRequirementType() + { + var emptyFilters = "[]"; + var nonEmptyFilters = @"[ + { + ""name"": ""FilterA"", + ""parameters"": { + ""Foo"": ""Bar"" + } + }, + { + ""name"": ""FilterB"" + } + ]"; + var featureFlags = new List() + { + _kv2, + featureWithRequirementType("Feature_NoFilters", "All", emptyFilters), + featureWithRequirementType("Feature_RequireAll", "All", nonEmptyFilters), + featureWithRequirementType("Feature_RequireAny", "Any", nonEmptyFilters) + }; + + var mockResponse = new Mock(); + var mockClient = new Mock(MockBehavior.Strict); + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns(new MockAsyncPageable(featureFlags)); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.UseFeatureFlags(); + }) + .Build(); + + Assert.Null(config["FeatureManagement:MyFeature2:RequirementType"]); + Assert.Null(config["FeatureManagement:Feature_NoFilters:RequirementType"]); + Assert.Equal("All", config["FeatureManagement:Feature_RequireAll:RequirementType"]); + Assert.Equal("Any", config["FeatureManagement:Feature_RequireAny:RequirementType"]); + } + + private ConfigurationSetting featureWithRequirementType(string featureId, string requirementType, string clientFiltersJsonString) + { + return ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + featureId, + value: $@" + {{ + ""id"": ""{featureId}"", + ""enabled"": true, + ""conditions"": {{ + ""requirement_type"": ""{requirementType}"", + ""client_filters"": {clientFiltersJsonString} + }} + }} + ", + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); + } } } From 6c2934c1cb3044d9dfee13509d06fc050778d02c Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Wed, 31 May 2023 14:42:48 -0500 Subject: [PATCH 02/17] Upgrade package versions to 6.0.2-preview (#425) * upgrade package versions to 6.0.2 * preview versions --- .../Microsoft.Azure.AppConfiguration.AspNetCore.csproj | 2 +- .../Microsoft.Azure.AppConfiguration.Functions.Worker.csproj | 2 +- ...rosoft.Extensions.Configuration.AzureAppConfiguration.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj index 5c0333cd..e628659f 100644 --- a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj +++ b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj @@ -28,7 +28,7 @@ - 6.0.1 + 6.0.2-preview diff --git a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj index 83c92083..a1ddbb00 100644 --- a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj +++ b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj @@ -24,7 +24,7 @@ - 6.0.1 + 6.0.2-preview diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index df233019..cf8935df 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -41,7 +41,7 @@ - 6.0.1 + 6.0.2-preview From f84d48d2e68062d4a1e2282581d943624745ddc6 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Wed, 19 Jul 2023 12:06:53 -0500 Subject: [PATCH 03/17] Add SelectSnapshot (#435) * first draft of selectsnapshot * add null check for name * add to summary of selectsnapshot, logic for selecting snapshots in provider * in progress testing createsnapshot and getsnapshot * fix null error * working for 4/20 version of sdk, need unit tests and updates with new changes as needed * fix comment for keyvalueselector snapshotname * fix selectsnapshot comment, null checks and remove repeated code * add check for composition type before loading snapshot * simplify composition type check * fix line endings * fix configureawait line format * add configureawait for getsnapshot * catch snapshot not found exception, catch invalidoperationexception when optional is true * add name for snapshot error * add feature flag adapter by default, fix tests for sdk package and adapter changes * throw invalidoperation only when snapshot not found, fix structure of defaultquery and null checks in selects * change exception message to say snapshot was not found * change catch statement * add catch in tryrefreshasync, chain request failed exception * use refreshfailederror * update to 1.3.0-beta.2, fix tests and options class --- .../AzureAppConfigurationOptions.cs | 30 +++++++++-- .../AzureAppConfigurationProvider.cs | 52 ++++++++++++++++--- ...Configuration.AzureAppConfiguration.csproj | 2 +- .../Models/KeyValueSelector.cs | 7 ++- .../FeatureManagementTests.cs | 4 +- .../JsonContentTypeTests.cs | 4 +- .../LoggingTests.cs | 42 +++++++++++++++ tests/Tests.AzureAppConfiguration/Tests.cs | 2 +- 8 files changed, 126 insertions(+), 17 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 5aee3744..850aa5ca 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -27,7 +27,8 @@ public class AzureAppConfigurationOptions private List _adapters = new List() { new AzureKeyVaultKeyValueAdapter(new AzureKeyVaultSecretProvider()), - new JsonKeyValueAdapter() + new JsonKeyValueAdapter(), + new FeatureManagementKeyValueAdapter() }; private List>> _mappers = new List>>(); private List _kvSelectors = new List(); @@ -148,7 +149,7 @@ public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter throw new ArgumentException("The characters '*' and ',' are not supported in label filters.", nameof(labelFilter)); } - if (!_kvSelectors.Any(s => s.KeyFilter.Equals(keyFilter) && s.LabelFilter.Equals(labelFilter))) + if (!_kvSelectors.Any(s => string.Equals(s.KeyFilter, keyFilter) && string.Equals(s.LabelFilter, labelFilter))) { _kvSelectors.Add(new KeyValueSelector { @@ -160,6 +161,29 @@ public AzureAppConfigurationOptions Select(string keyFilter, string labelFilter return this; } + /// + /// Specify a snapshot and include its contained key-values in the configuration provider. + /// can be called multiple times to include key-values from multiple snapshots. + /// + /// The name of the snapshot in Azure App Configuration. + public AzureAppConfigurationOptions SelectSnapshot(string name) + { + if (string.IsNullOrEmpty(name)) + { + throw new ArgumentNullException(nameof(name)); + } + + if (!_kvSelectors.Any(s => string.Equals(s.SnapshotName, name))) + { + _kvSelectors.Add(new KeyValueSelector + { + SnapshotName = name + }); + } + + return this; + } + /// /// Enables Azure App Configuration feature flags to be parsed and transformed into feature management configuration. /// @@ -406,7 +430,7 @@ public AzureAppConfigurationOptions Map(Func TryRefreshAsync(CancellationToken cancellationToken) _logger.LogWarning(LogHelper.BuildRefreshCanceledErrorMessage()); return false; } + catch (InvalidOperationException e) + { + _logger.LogWarning(LogHelper.BuildRefreshFailedErrorMessage(e.Message)); + return false; + } return true; } @@ -562,7 +567,8 @@ await ExecuteWithFailOverPolicyAsync( catch (Exception exception) when (ignoreFailures && (exception is RequestFailedException || ((exception as AggregateException)?.InnerExceptions?.All(e => e is RequestFailedException) ?? false) || - exception is OperationCanceledException)) + exception is OperationCanceledException || + exception is InvalidOperationException)) { } // Update the cache expiration time for all refresh registered settings and feature flags @@ -598,7 +604,8 @@ private async Task> LoadSelectedKeyValu var serverData = new Dictionary(StringComparer.OrdinalIgnoreCase); // Use default query if there are no key-values specified for use other than the feature flags - bool useDefaultQuery = !_options.KeyValueSelectors.Any(selector => !selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)); + bool useDefaultQuery = !_options.KeyValueSelectors.Any(selector => selector.KeyFilter == null || + !selector.KeyFilter.StartsWith(FeatureManagementConstants.FeatureFlagMarker)); if (useDefaultQuery) { @@ -618,17 +625,46 @@ await CallWithRequestTracing(async () => }).ConfigureAwait(false); } - foreach (var loadOption in _options.KeyValueSelectors) + foreach (KeyValueSelector loadOption in _options.KeyValueSelectors) + { + IAsyncEnumerable settingsEnumerable; + + if (string.IsNullOrEmpty(loadOption.SnapshotName)) { - var selector = new SettingSelector + settingsEnumerable = client.GetConfigurationSettingsAsync( + new SettingSelector + { + KeyFilter = loadOption.KeyFilter, + LabelFilter = loadOption.LabelFilter + }, + cancellationToken); + } + else + { + ConfigurationSettingsSnapshot snapshot; + + try + { + snapshot = await client.GetSnapshotAsync(loadOption.SnapshotName).ConfigureAwait(false); + } + catch (RequestFailedException rfe) when (rfe.Status == (int)HttpStatusCode.NotFound) + { + throw new InvalidOperationException($"Could not find snapshot with name '{loadOption.SnapshotName}'.", rfe); + } + + if (snapshot.CompositionType != CompositionType.Key) { - KeyFilter = loadOption.KeyFilter, - LabelFilter = loadOption.LabelFilter - }; + throw new InvalidOperationException($"{nameof(snapshot.CompositionType)} for the selected snapshot with name '{snapshot.Name}' must be 'key', found '{snapshot.CompositionType}'."); + } + + settingsEnumerable = client.GetConfigurationSettingsForSnapshotAsync( + loadOption.SnapshotName, + cancellationToken); + } await CallWithRequestTracing(async () => { - await foreach (ConfigurationSetting setting in client.GetConfigurationSettingsAsync(selector, cancellationToken).ConfigureAwait(false)) + await foreach (ConfigurationSetting setting in settingsEnumerable.ConfigureAwait(false)) { serverData[setting.Key] = setting; } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index cf8935df..cd44a448 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -14,7 +14,7 @@ - + diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs index 1b288c89..49b60661 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Models/KeyValueSelector.cs @@ -15,8 +15,13 @@ public class KeyValueSelector public string KeyFilter { get; set; } /// - /// A filter that determines what label to use when selecting key-values for the the configuration provider + /// A filter that determines what label to use when selecting key-values for the the configuration provider. /// public string LabelFilter { get; set; } + + /// + /// The name of the Azure App Configuration snapshot to use when selecting key-values for the configuration provider. + /// + public string SnapshotName { get; set; } } } diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index c50994e0..b00d187e 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -394,7 +394,7 @@ public void PreservesDefaultQuery() options.UseFeatureFlags(); }).Build(); - bool performedDefaultQuery = mockTransport.Requests.Any(r => r.Uri.PathAndQuery.Contains("/kv/?key=%2A&label=%00")); + bool performedDefaultQuery = mockTransport.Requests.Any(r => r.Uri.PathAndQuery.Contains("/kv?key=%2A&label=%00")); bool queriedFeatureFlags = mockTransport.Requests.Any(r => r.Uri.PathAndQuery.Contains(Uri.EscapeDataString(FeatureManagementConstants.FeatureFlagMarker))); Assert.True(performedDefaultQuery); @@ -422,7 +422,7 @@ public void QueriesFeatureFlags() }) .Build(); - bool performedDefaultQuery = mockTransport.Requests.Any(r => r.Uri.PathAndQuery.Contains("/kv/?key=%2A&label=%00")); + bool performedDefaultQuery = mockTransport.Requests.Any(r => r.Uri.PathAndQuery.Contains("/kv?key=%2A&label=%00")); bool queriedFeatureFlags = mockTransport.Requests.Any(r => r.Uri.PathAndQuery.Contains(Uri.EscapeDataString(FeatureManagementConstants.FeatureFlagMarker))); Assert.True(performedDefaultQuery); diff --git a/tests/Tests.AzureAppConfiguration/JsonContentTypeTests.cs b/tests/Tests.AzureAppConfiguration/JsonContentTypeTests.cs index 4bb42394..7723a612 100644 --- a/tests/Tests.AzureAppConfiguration/JsonContentTypeTests.cs +++ b/tests/Tests.AzureAppConfiguration/JsonContentTypeTests.cs @@ -221,7 +221,9 @@ public void JsonContentTypeTests_DontFlattenFeatureFlagAsJsonObject() .AddAzureAppConfiguration(options => options.ClientManager = mockClientManager) .Build(); - Assert.Equal(compactJsonValue, config[FeatureManagementConstants.FeatureFlagMarker + "Beta"]); + Assert.Equal("Browser", config["FeatureManagement:Beta:EnabledFor:0:Name"]); + Assert.Equal("Firefox", config["FeatureManagement:Beta:EnabledFor:0:Parameters:AllowedBrowsers:0"]); + Assert.Equal("Safari", config["FeatureManagement:Beta:EnabledFor:0:Parameters:AllowedBrowsers:1"]); } [Fact] diff --git a/tests/Tests.AzureAppConfiguration/LoggingTests.cs b/tests/Tests.AzureAppConfiguration/LoggingTests.cs index 8c8ac9a0..e83509f5 100644 --- a/tests/Tests.AzureAppConfiguration/LoggingTests.cs +++ b/tests/Tests.AzureAppConfiguration/LoggingTests.cs @@ -139,6 +139,48 @@ public void ValidateUnauthorizedExceptionLoggedDuringRefresh() Assert.Contains(LoggingConstants.RefreshFailedDueToAuthenticationError, warningInvocation); } + [Fact] + public void ValidateInvalidOperationExceptionLoggedDuringRefresh() + { + IConfigurationRefresher refresher = null; + var mockClient = GetMockConfigurationClient(); + mockClient.Setup(c => c.GetConfigurationSettingAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Throws(new InvalidOperationException()); + + string warningInvocation = ""; + using var _ = new AzureEventSourceListener( + (args, s) => + { + if (args.Level == EventLevel.Warning) + { + warningInvocation += s; + } + }, EventLevel.Verbose); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.ConfigureRefresh(refreshOptions => + { + refreshOptions.Register("TestKey1", "label") + .SetCacheExpiration(CacheExpirationTime); + }); + + refresher = options.GetRefresher(); + }) + .Build(); + + Assert.Equal("TestValue1", config["TestKey1"]); + FirstKeyValue.Value = "newValue1"; + + Thread.Sleep(CacheExpirationTime); + refresher.TryRefreshAsync().Wait(); + + Assert.NotEqual("newValue1", config["TestKey1"]); + Assert.Contains(LoggingConstants.RefreshFailedError, warningInvocation); + } + [Fact] public void ValidateKeyVaultExceptionLoggedDuringRefresh() { diff --git a/tests/Tests.AzureAppConfiguration/Tests.cs b/tests/Tests.AzureAppConfiguration/Tests.cs index 1d8b33a0..29415be6 100644 --- a/tests/Tests.AzureAppConfiguration/Tests.cs +++ b/tests/Tests.AzureAppConfiguration/Tests.cs @@ -187,7 +187,7 @@ public void TestUserAgentHeader() // 4. Contains the name and version of the App Configuration SDK package // 5. Contains the runtime information (target framework, OS description etc.) in the format set by the SDK // 6. Does not contain any additional components - string userAgentRegex = @"^Microsoft\.Extensions\.Configuration\.AzureAppConfiguration/\d+\.\d+\.\d+(-preview)?(-\d+-\d+)?,azsdk-net-Data.AppConfiguration/[.+\w]+ \([.;\w\s]+\)$"; + string userAgentRegex = @"^Microsoft\.Extensions\.Configuration\.AzureAppConfiguration/\d+\.\d+\.\d+(-preview)?(-\d+-\d+)?,azsdk-net-Data.AppConfiguration/[.+\w-]+ \([.;\w\s]+\)$"; var response = new MockResponse(200); response.SetContent(SerializationHelpers.Serialize(_kvCollectionPageOne.ToArray(), TestHelpers.SerializeBatch)); From b0ceb703e44112ac20029164135032612bdadfb9 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Wed, 19 Jul 2023 12:20:33 -0500 Subject: [PATCH 04/17] upgrade versions to 7.0.0-preview (#436) --- .../Microsoft.Azure.AppConfiguration.AspNetCore.csproj | 2 +- .../Microsoft.Azure.AppConfiguration.Functions.Worker.csproj | 2 +- ...rosoft.Extensions.Configuration.AzureAppConfiguration.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj index e628659f..ceae2885 100644 --- a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj +++ b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj @@ -28,7 +28,7 @@ - 6.0.2-preview + 7.0.0-preview diff --git a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj index a1ddbb00..0a2a76a8 100644 --- a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj +++ b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj @@ -24,7 +24,7 @@ - 6.0.2-preview + 7.0.0-preview diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index cd44a448..8307c796 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -41,7 +41,7 @@ - 6.0.2-preview + 7.0.0-preview From 18a4c866a8f5b7014d7b2bc5c50b2682b8e83c17 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Wed, 19 Jul 2023 14:00:02 -0500 Subject: [PATCH 05/17] Revert "parse requirement type of feature flags (#418)" (#438) This reverts commit 0de3ccfafeac91849875f7d0d9d19e39fd419909. --- .../FeatureManagement/FeatureConditions.cs | 3 - .../FeatureManagementConstants.cs | 1 - .../FeatureManagementKeyValueAdapter.cs | 9 --- .../FeatureManagementTests.cs | 76 ------------------- 4 files changed, 89 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureConditions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureConditions.cs index 6927d310..27aa9ee5 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureConditions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureConditions.cs @@ -10,8 +10,5 @@ internal class FeatureConditions { [JsonPropertyName("client_filters")] public List ClientFilters { get; set; } = new List(); - - [JsonPropertyName("requirement_type")] - public string RequirementType { get; set; } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs index b0e09723..1ef33aad 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs @@ -9,6 +9,5 @@ internal class FeatureManagementConstants public const string ContentType = "application/vnd.microsoft.appconfig.ff+json"; public const string SectionName = "FeatureManagement"; public const string EnabledFor = "EnabledFor"; - public const string RequirementType = "RequirementType"; } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs index 3de20b3f..713efe8f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs @@ -51,15 +51,6 @@ public Task>> ProcessKeyValue(Configura keyValues.Add(new KeyValuePair($"{FeatureManagementConstants.SectionName}:{featureFlag.Id}:{FeatureManagementConstants.EnabledFor}:{i}:Parameters:{kvp.Key}", kvp.Value)); } } - - // - // process RequirementType only when filters are not empty - if (featureFlag.Conditions.RequirementType != null) - { - keyValues.Add(new KeyValuePair( - $"{FeatureManagementConstants.SectionName}:{featureFlag.Id}:{FeatureManagementConstants.RequirementType}", - featureFlag.Conditions.RequirementType)); - } } } else diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index b00d187e..6c6b6b14 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -185,21 +185,6 @@ public class FeatureManagementTests eTag: new ETag("0a76e3d7-7ec1-4e37-883c-9ea6d0d89e63"), contentType: "text"); - readonly ConfigurationSetting Feature_RequirementTypeAll = ConfigurationModelFactory.ConfigurationSetting( - key: FeatureManagementConstants.FeatureFlagMarker + "Feature_All", - value: @" - { - ""id"": ""Feature_All"", - ""enabled"": true, - ""conditions"": { - ""requirement_type"": ""All"", - ""client_filters"": [] - } - } - ", - contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); - TimeSpan CacheExpirationTime = TimeSpan.FromSeconds(1); [Fact] @@ -1144,66 +1129,5 @@ Response GetTestKey(string key, string label, Cancellation { return Response.FromValue(TestHelpers.CloneSetting(FirstKeyValue), new Mock().Object); } - - [Fact] - public void WithRequirementType() - { - var emptyFilters = "[]"; - var nonEmptyFilters = @"[ - { - ""name"": ""FilterA"", - ""parameters"": { - ""Foo"": ""Bar"" - } - }, - { - ""name"": ""FilterB"" - } - ]"; - var featureFlags = new List() - { - _kv2, - featureWithRequirementType("Feature_NoFilters", "All", emptyFilters), - featureWithRequirementType("Feature_RequireAll", "All", nonEmptyFilters), - featureWithRequirementType("Feature_RequireAny", "Any", nonEmptyFilters) - }; - - var mockResponse = new Mock(); - var mockClient = new Mock(MockBehavior.Strict); - - mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) - .Returns(new MockAsyncPageable(featureFlags)); - - var config = new ConfigurationBuilder() - .AddAzureAppConfiguration(options => - { - options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); - options.UseFeatureFlags(); - }) - .Build(); - - Assert.Null(config["FeatureManagement:MyFeature2:RequirementType"]); - Assert.Null(config["FeatureManagement:Feature_NoFilters:RequirementType"]); - Assert.Equal("All", config["FeatureManagement:Feature_RequireAll:RequirementType"]); - Assert.Equal("Any", config["FeatureManagement:Feature_RequireAny:RequirementType"]); - } - - private ConfigurationSetting featureWithRequirementType(string featureId, string requirementType, string clientFiltersJsonString) - { - return ConfigurationModelFactory.ConfigurationSetting( - key: FeatureManagementConstants.FeatureFlagMarker + featureId, - value: $@" - {{ - ""id"": ""{featureId}"", - ""enabled"": true, - ""conditions"": {{ - ""requirement_type"": ""{requirementType}"", - ""client_filters"": {clientFiltersJsonString} - }} - }} - ", - contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", - eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); - } } } From 49f6d10961d1b79ae950833f9ea9630856dc4544 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Tue, 22 Aug 2023 16:45:23 -0500 Subject: [PATCH 06/17] Update UseFeatureFlags for snapshot changes (#449) * update usefeatureflags to fit new use case * change back to use * add mention of feature flag refresh in summary * add default query mention * fix mention of null label in comment * revise comment on deafult query * Update src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs Co-authored-by: Jimmy Campbell * Update src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs Co-authored-by: Jimmy Campbell --------- Co-authored-by: Jimmy Campbell --- .../AzureAppConfigurationOptions.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index d9df3baa..a6746e04 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -188,7 +188,9 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) } /// - /// Enables Azure App Configuration feature flags to be parsed and transformed into feature management configuration. + /// Configures options for Azure App Configuration feature flags that will be parsed and transformed into feature management configuration. + /// If no filtering is specified via the then all feature flags with no label are loaded. + /// All loaded feature flags will be automatically registered for refresh on an individual flag level. /// /// A callback used to configure feature flag options. public AzureAppConfigurationOptions UseFeatureFlags(Action configure = null) @@ -245,11 +247,6 @@ public AzureAppConfigurationOptions UseFeatureFlags(Action c } } - if (!_adapters.Any(a => a is FeatureManagementKeyValueAdapter)) - { - _adapters.Add(new FeatureManagementKeyValueAdapter()); - } - return this; } From e0438a45e81a129b3f1098b13200952fea88723f Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Wed, 23 Aug 2023 16:41:13 -0500 Subject: [PATCH 07/17] Upgrade packages to 7.0.0-preview2 (#451) * upgrade packages to 7.0.0-preview2 * change to preview.2 --- .../Microsoft.Azure.AppConfiguration.AspNetCore.csproj | 2 +- .../Microsoft.Azure.AppConfiguration.Functions.Worker.csproj | 2 +- ...rosoft.Extensions.Configuration.AzureAppConfiguration.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj index ceae2885..4a625fc4 100644 --- a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj +++ b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj @@ -28,7 +28,7 @@ - 7.0.0-preview + 7.0.0-preview.2 diff --git a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj index 0a2a76a8..5e31320e 100644 --- a/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj +++ b/src/Microsoft.Azure.AppConfiguration.Functions.Worker/Microsoft.Azure.AppConfiguration.Functions.Worker.csproj @@ -24,7 +24,7 @@ - 7.0.0-preview + 7.0.0-preview.2 diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index 8307c796..2f853530 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -41,7 +41,7 @@ - 7.0.0-preview + 7.0.0-preview.2 From 4cb5c6123065913edbbcf0dda96d2f685c6a2327 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Mon, 28 Aug 2023 16:44:29 -0500 Subject: [PATCH 08/17] set failTaskOnFailedTests to true (#453) --- .pipelines/OneBranch.Official.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pipelines/OneBranch.Official.yml b/.pipelines/OneBranch.Official.yml index cad3f96b..1297da99 100644 --- a/.pipelines/OneBranch.Official.yml +++ b/.pipelines/OneBranch.Official.yml @@ -171,5 +171,5 @@ extends: testResultsFormat: 'vstest' testResultsFiles: '**/*.trx' searchFolder: '' - failTaskOnFailedTests: False + failTaskOnFailedTests: True testRunTitle: Unit Tests \ No newline at end of file From dc794061acce5ebe318c0dde592c8001147733e8 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Mon, 18 Sep 2023 13:34:31 -0700 Subject: [PATCH 09/17] upgrade Microsoft.NET.test.sdk packages (#462) --- .../Tests.AzureAppConfiguration.AspNetCore.csproj | 2 +- .../Tests.AzureAppConfiguration.Functions.Worker.csproj | 2 +- .../Tests.AzureAppConfiguration.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Tests.AzureAppConfiguration.AspNetCore/Tests.AzureAppConfiguration.AspNetCore.csproj b/tests/Tests.AzureAppConfiguration.AspNetCore/Tests.AzureAppConfiguration.AspNetCore.csproj index 1d531112..fd919761 100644 --- a/tests/Tests.AzureAppConfiguration.AspNetCore/Tests.AzureAppConfiguration.AspNetCore.csproj +++ b/tests/Tests.AzureAppConfiguration.AspNetCore/Tests.AzureAppConfiguration.AspNetCore.csproj @@ -11,7 +11,7 @@ - + diff --git a/tests/Tests.AzureAppConfiguration.Functions.Worker/Tests.AzureAppConfiguration.Functions.Worker.csproj b/tests/Tests.AzureAppConfiguration.Functions.Worker/Tests.AzureAppConfiguration.Functions.Worker.csproj index f3cbe5bb..88a8f633 100644 --- a/tests/Tests.AzureAppConfiguration.Functions.Worker/Tests.AzureAppConfiguration.Functions.Worker.csproj +++ b/tests/Tests.AzureAppConfiguration.Functions.Worker/Tests.AzureAppConfiguration.Functions.Worker.csproj @@ -11,7 +11,7 @@ - + diff --git a/tests/Tests.AzureAppConfiguration/Tests.AzureAppConfiguration.csproj b/tests/Tests.AzureAppConfiguration/Tests.AzureAppConfiguration.csproj index 855b5c48..0cf7ee17 100644 --- a/tests/Tests.AzureAppConfiguration/Tests.AzureAppConfiguration.csproj +++ b/tests/Tests.AzureAppConfiguration/Tests.AzureAppConfiguration.csproj @@ -13,7 +13,7 @@ - + From 5883b3ae8394dd183a473dab1f772f5386f83ca7 Mon Sep 17 00:00:00 2001 From: Avani Gupta Date: Thu, 21 Sep 2023 16:38:45 -0700 Subject: [PATCH 10/17] Make AzureAppConfigurationOptions.KeyValueSelectors internal (#465) --- .../AzureAppConfigurationOptions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index a6746e04..c9e855e5 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -58,7 +58,7 @@ public class AzureAppConfigurationOptions /// /// A collection of . /// - public IEnumerable KeyValueSelectors => _kvSelectors; + internal IEnumerable KeyValueSelectors => _kvSelectors; /// /// A collection of . From 6ce2e10c1d96c7720e2ee276aaf989a6469d6896 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Fri, 22 Sep 2023 15:37:44 -0700 Subject: [PATCH 11/17] Fix user agent header test (#466) * fix useragentheader test * fix test for 7.0.0.3 case --- tests/Tests.AzureAppConfiguration/Tests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Tests.AzureAppConfiguration/Tests.cs b/tests/Tests.AzureAppConfiguration/Tests.cs index 13994a75..e12e7663 100644 --- a/tests/Tests.AzureAppConfiguration/Tests.cs +++ b/tests/Tests.AzureAppConfiguration/Tests.cs @@ -186,7 +186,7 @@ public void TestUserAgentHeader() // 4. Contains the name and version of the App Configuration SDK package // 5. Contains the runtime information (target framework, OS description etc.) in the format set by the SDK // 6. Does not contain any additional components - string userAgentRegex = @"^Microsoft\.Extensions\.Configuration\.AzureAppConfiguration/\d+\.\d+\.\d+(-preview)?(-\d+-\d+)?,azsdk-net-Data.AppConfiguration/[.+\w-]+ \([.;\w\s]+\)$"; + string userAgentRegex = @"^Microsoft\.Extensions\.Configuration\.AzureAppConfiguration/\d+\.\d+\.\d+(-preview(\.\d+)?)?,azsdk-net-Data.AppConfiguration/[.+\w-]+ \([.;\w\s]+\)$"; var response = new MockResponse(200); response.SetContent(SerializationHelpers.Serialize(_kvCollectionPageOne.ToArray(), TestHelpers.SerializeBatch)); From 34ba1490880a75c3361b8c436738f07ff4505ee7 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Thu, 5 Oct 2023 13:50:41 -0700 Subject: [PATCH 12/17] Improve summaries for methods in AzureAppConfigurationOptions (#467) * first draft clarify select summaries and fix usefeatureflags summary * move default query message to class summary * change mention of null label to no label Co-authored-by: Jimmy Campbell --------- Co-authored-by: Jimmy Campbell --- .../AzureAppConfigurationOptions.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index c9e855e5..087ed96d 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -15,7 +15,8 @@ namespace Microsoft.Extensions.Configuration.AzureAppConfiguration { /// - /// Options used to configure the behavior of an Azure App Configuration provider. + /// Options used to configure the behavior of an Azure App Configuration provider. + /// If neither nor is ever called, all key-values with no label are included in the configuration provider. /// public class AzureAppConfigurationOptions { @@ -189,7 +190,7 @@ public AzureAppConfigurationOptions SelectSnapshot(string name) /// /// Configures options for Azure App Configuration feature flags that will be parsed and transformed into feature management configuration. - /// If no filtering is specified via the then all feature flags with no label are loaded. + /// If no filtering is specified via the then all feature flags with no label are loaded. /// All loaded feature flags will be automatically registered for refresh on an individual flag level. /// /// A callback used to configure feature flag options. From 3f47e0b9dc7a6ad6181ae594ae838ad3dd37ffa1 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Tue, 24 Oct 2023 10:30:34 -0700 Subject: [PATCH 13/17] Add back RequirementType support (#475) * Revert "Revert "parse requirement type of feature flags (#418)" (#438)" This reverts commit 18a4c866a8f5b7014d7b2bc5c50b2682b8e83c17. * update package version * fix method names and order, remove unused vars --- .../AzureAppConfigurationProvider.cs | 6 +- .../FeatureManagement/FeatureConditions.cs | 3 + .../FeatureManagementConstants.cs | 1 + .../FeatureManagementKeyValueAdapter.cs | 9 +++ ...Configuration.AzureAppConfiguration.csproj | 2 +- .../FeatureManagementTests.cs | 62 +++++++++++++++++++ 6 files changed, 79 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 4cbefed3..551794c3 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -655,7 +655,7 @@ await CallWithRequestTracing(async () => } else { - ConfigurationSettingsSnapshot snapshot; + ConfigurationSnapshot snapshot; try { @@ -666,9 +666,9 @@ await CallWithRequestTracing(async () => throw new InvalidOperationException($"Could not find snapshot with name '{loadOption.SnapshotName}'.", rfe); } - if (snapshot.CompositionType != CompositionType.Key) + if (snapshot.SnapshotComposition != SnapshotComposition.Key) { - throw new InvalidOperationException($"{nameof(snapshot.CompositionType)} for the selected snapshot with name '{snapshot.Name}' must be 'key', found '{snapshot.CompositionType}'."); + throw new InvalidOperationException($"{nameof(snapshot.SnapshotComposition)} for the selected snapshot with name '{snapshot.Name}' must be 'key', found '{snapshot.SnapshotComposition}'."); } settingsEnumerable = client.GetConfigurationSettingsForSnapshotAsync( diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureConditions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureConditions.cs index 27aa9ee5..6927d310 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureConditions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureConditions.cs @@ -10,5 +10,8 @@ internal class FeatureConditions { [JsonPropertyName("client_filters")] public List ClientFilters { get; set; } = new List(); + + [JsonPropertyName("requirement_type")] + public string RequirementType { get; set; } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs index 1ef33aad..b0e09723 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementConstants.cs @@ -9,5 +9,6 @@ internal class FeatureManagementConstants public const string ContentType = "application/vnd.microsoft.appconfig.ff+json"; public const string SectionName = "FeatureManagement"; public const string EnabledFor = "EnabledFor"; + public const string RequirementType = "RequirementType"; } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs index 713efe8f..3de20b3f 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/FeatureManagement/FeatureManagementKeyValueAdapter.cs @@ -51,6 +51,15 @@ public Task>> ProcessKeyValue(Configura keyValues.Add(new KeyValuePair($"{FeatureManagementConstants.SectionName}:{featureFlag.Id}:{FeatureManagementConstants.EnabledFor}:{i}:Parameters:{kvp.Key}", kvp.Value)); } } + + // + // process RequirementType only when filters are not empty + if (featureFlag.Conditions.RequirementType != null) + { + keyValues.Add(new KeyValuePair( + $"{FeatureManagementConstants.SectionName}:{featureFlag.Id}:{FeatureManagementConstants.RequirementType}", + featureFlag.Conditions.RequirementType)); + } } } else diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index 2f853530..66540883 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -14,7 +14,7 @@ - + diff --git a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs index 6c6b6b14..bc1e74f4 100644 --- a/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs +++ b/tests/Tests.AzureAppConfiguration/FeatureManagementTests.cs @@ -1120,6 +1120,50 @@ public void MapTransformFeatureFlagWithRefresh() Assert.Equal("newValue1", config["TestKey1"]); Assert.Equal("NoUsers", config["FeatureManagement:MyFeature:EnabledFor:0:Name"]); } + + [Fact] + public void WithRequirementType() + { + var emptyFilters = "[]"; + var nonEmptyFilters = @"[ + { + ""name"": ""FilterA"", + ""parameters"": { + ""Foo"": ""Bar"" + } + }, + { + ""name"": ""FilterB"" + } + ]"; + var featureFlags = new List() + { + _kv2, + FeatureWithRequirementType("Feature_NoFilters", "All", emptyFilters), + FeatureWithRequirementType("Feature_RequireAll", "All", nonEmptyFilters), + FeatureWithRequirementType("Feature_RequireAny", "Any", nonEmptyFilters) + }; + + var mockResponse = new Mock(); + var mockClient = new Mock(MockBehavior.Strict); + + mockClient.Setup(c => c.GetConfigurationSettingsAsync(It.IsAny(), It.IsAny())) + .Returns(new MockAsyncPageable(featureFlags)); + + var config = new ConfigurationBuilder() + .AddAzureAppConfiguration(options => + { + options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object); + options.UseFeatureFlags(); + }) + .Build(); + + Assert.Null(config["FeatureManagement:MyFeature2:RequirementType"]); + Assert.Null(config["FeatureManagement:Feature_NoFilters:RequirementType"]); + Assert.Equal("All", config["FeatureManagement:Feature_RequireAll:RequirementType"]); + Assert.Equal("Any", config["FeatureManagement:Feature_RequireAny:RequirementType"]); + } + Response GetIfChanged(ConfigurationSetting setting, bool onlyIfChanged, CancellationToken cancellationToken) { return Response.FromValue(FirstKeyValue, new MockResponse(200)); @@ -1129,5 +1173,23 @@ Response GetTestKey(string key, string label, Cancellation { return Response.FromValue(TestHelpers.CloneSetting(FirstKeyValue), new Mock().Object); } + + private ConfigurationSetting FeatureWithRequirementType(string featureId, string requirementType, string clientFiltersJsonString) + { + return ConfigurationModelFactory.ConfigurationSetting( + key: FeatureManagementConstants.FeatureFlagMarker + featureId, + value: $@" + {{ + ""id"": ""{featureId}"", + ""enabled"": true, + ""conditions"": {{ + ""requirement_type"": ""{requirementType}"", + ""client_filters"": {clientFiltersJsonString} + }} + }} + ", + contentType: FeatureManagementConstants.ContentType + ";charset=utf-8", + eTag: new ETag("c3c231fd-39a0-4cb6-3237-4614474b92c1")); + } } } From 94e687187b80a658c1f96dd0b05e90a063791217 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Fri, 3 Nov 2023 11:21:48 -0700 Subject: [PATCH 14/17] Remove netstandard2.0 support from ASP.NET. Remove netcoreapp3.1 target framework. (#482) * remove netcoreapp3.1 from target frameworks * fix indentation * fix target framework indentation * remove netstandard2.0 * use channel instead of version for install-dotnet.ps1 * update packages to 6.x.x versions from 3.1.x, fix asyncinterfaces issue * fix tests file package version downgrades * fix bcl.asyncinterfaces issue, remove unused test package * move system.text.json to only netstandard2.0 * remove system.text.json --- build/install-dotnet.ps1 | 4 ++-- ...soft.Azure.AppConfiguration.AspNetCore.csproj | 11 ++--------- ...ns.Configuration.AzureAppConfiguration.csproj | 16 ++++------------ ...Tests.AzureAppConfiguration.AspNetCore.csproj | 4 ++-- ...AzureAppConfiguration.Functions.Worker.csproj | 2 +- .../Tests.AzureAppConfiguration.csproj | 3 +-- 6 files changed, 12 insertions(+), 28 deletions(-) diff --git a/build/install-dotnet.ps1 b/build/install-dotnet.ps1 index 898b6409..11552a78 100644 --- a/build/install-dotnet.ps1 +++ b/build/install-dotnet.ps1 @@ -3,6 +3,6 @@ [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; -&([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Version 7.0.100 +&([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Channel 6.0 -&([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Version 6.0.301 \ No newline at end of file +&([scriptblock]::Create((Invoke-WebRequest -UseBasicParsing 'https://dot.net/v1/dotnet-install.ps1'))) -Channel 7.0 \ No newline at end of file diff --git a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj index 4a625fc4..8f3e8171 100644 --- a/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj +++ b/src/Microsoft.Azure.AppConfiguration.AspNetCore/Microsoft.Azure.AppConfiguration.AspNetCore.csproj @@ -2,7 +2,7 @@ - netstandard2.0;netcoreapp3.1;net6.0;net7.0 + net6.0;net7.0 Microsoft.Azure.AppConfiguration.AspNetCore allows developers to use Microsoft Azure App Configuration service as a configuration source in their applications. This package adds additional features for ASP.NET Core applications to the existing package Microsoft.Extensions.Configuration.AzureAppConfiguration. true false @@ -15,14 +15,7 @@ - - - - - - - - + diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj index 66540883..5a4bdec2 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Microsoft.Extensions.Configuration.AzureAppConfiguration.csproj @@ -2,7 +2,7 @@ - netstandard2.0 + netstandard2.0;netstandard2.1 8.0 Microsoft.Extensions.Configuration.AzureAppConfiguration is a configuration provider for the .NET Core framework that allows developers to use Microsoft Azure App Configuration service as a configuration source in their applications. true @@ -17,17 +17,9 @@ - - - - - - - - - - - + + + diff --git a/tests/Tests.AzureAppConfiguration.AspNetCore/Tests.AzureAppConfiguration.AspNetCore.csproj b/tests/Tests.AzureAppConfiguration.AspNetCore/Tests.AzureAppConfiguration.AspNetCore.csproj index fd919761..ef89659c 100644 --- a/tests/Tests.AzureAppConfiguration.AspNetCore/Tests.AzureAppConfiguration.AspNetCore.csproj +++ b/tests/Tests.AzureAppConfiguration.AspNetCore/Tests.AzureAppConfiguration.AspNetCore.csproj @@ -1,7 +1,7 @@  - netcoreapp2.1;netcoreapp3.1;net6.0;net7.0 + net6.0;net7.0 8.0 false true @@ -10,7 +10,7 @@ - + diff --git a/tests/Tests.AzureAppConfiguration.Functions.Worker/Tests.AzureAppConfiguration.Functions.Worker.csproj b/tests/Tests.AzureAppConfiguration.Functions.Worker/Tests.AzureAppConfiguration.Functions.Worker.csproj index 88a8f633..0340afd5 100644 --- a/tests/Tests.AzureAppConfiguration.Functions.Worker/Tests.AzureAppConfiguration.Functions.Worker.csproj +++ b/tests/Tests.AzureAppConfiguration.Functions.Worker/Tests.AzureAppConfiguration.Functions.Worker.csproj @@ -10,7 +10,7 @@ - + diff --git a/tests/Tests.AzureAppConfiguration/Tests.AzureAppConfiguration.csproj b/tests/Tests.AzureAppConfiguration/Tests.AzureAppConfiguration.csproj index 0cf7ee17..19955a28 100644 --- a/tests/Tests.AzureAppConfiguration/Tests.AzureAppConfiguration.csproj +++ b/tests/Tests.AzureAppConfiguration/Tests.AzureAppConfiguration.csproj @@ -1,7 +1,7 @@  - netcoreapp2.1;netcoreapp3.1;net6.0;net7.0 + net48;net6.0;net7.0 8.0 false true @@ -11,7 +11,6 @@ - From 56e544cf7b01d853cc9dd3b67ae3fcd213de07bf Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Thu, 16 Nov 2023 14:29:28 -0800 Subject: [PATCH 15/17] Add StartupOptions for time-based retries on startup (#488) * first draft time-based retries in startup * set startupdelay from configureclientoptions, fix syntax and spacing * allow configuring of startupclientoptions from options * rough draft startupoptions * fix draft * clarify summary for startupoptions * testing out retryoptions settings * update with new structure, need logic to calculate retry options for specified timeout * fix usage of startupconfigclientmanager, logic for retry time to use cancellationtoken * in progress fixing connect tests * fix connecttest, clientoptions in progress, need to fix maxretry logic * fix tests * extend startup timeout on test * remove unused variable * remove unused usings * remove max retries change * in progress working on fix for multiple clients with timeout * in progress fix initializeasync * in progress fixing failover logic * progress * fix with updates to logic for replicas * fix tests and logic in initializeasync * remove unused using * in progress fixing timing of cts * stuck on logic for srv dns changes, iasyncenumerable * progress * progress changing timeout to per store/replica, change options properties to internal * fix logic to be per store/replica * remove cancellationtoken from func * progress on looping over clients * fix iterating over clients, unit tests in progress * add backoff to if statement * working draft of design to retry replicas until timeout * updates to fix unit tests, bugs * fix variable naming hasNextClient * add comment to catch in initializeasync * prevent unnecessary delay when load finished * remove custom exception, fix logic in executewithfailover * move comment * fix small mistakes, rename variables * PR revisions * restructure startup retry logic, fix tests * Update src/Microsoft.Extensions.Configuration.AzureAppConfiguration/StartupOptions.cs Co-authored-by: Jimmy Campbell * remove unnecessary param isstartup, fix text * PR revisions, fix configurestartupoptions summary * Apply suggestions from code review Co-authored-by: Jimmy Campbell * combine if statemenets * simplify logic for catching operationcanceledexception Co-authored-by: Avani Gupta * move call to getclients * PR revisions, make tests more specific * add new methods for jitter and startup backoff calculation, fix smaller issues with namespace/names * use new fixed + exponential backoff function * remove debugging statemenets * PR revisions * PR revisions, remove new exponential backoff method and update old one to use jitter * update jitter range logic * fix isFailoverable, change jitter ratio to 0.25 * update IsFailoverable --------- Co-authored-by: Jimmy Campbell Co-authored-by: Avani Gupta --- .../AzureAppConfigurationOptions.cs | 17 +- .../AzureAppConfigurationProvider.cs | 204 +++++++++++++----- .../ConfigurationClientManager.cs | 9 +- .../Constants/FailOverConstants.cs | 5 +- .../Extensions/TimeSpanExtensions.cs | 59 ++++- .../IConfigurationClientManager.cs | 2 + .../StartupOptions.cs | 19 ++ .../ClientOptionsTests.cs | 42 ++-- .../ConnectTests.cs | 18 +- .../FailoverTests.cs | 12 +- .../KeyVaultReferenceTests.cs | 4 + .../MockedConfigurationClientManager.cs | 8 +- .../RefreshTests.cs | 4 + 13 files changed, 320 insertions(+), 83 deletions(-) create mode 100644 src/Microsoft.Extensions.Configuration.AzureAppConfiguration/StartupOptions.cs diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 087ed96d..068ddbfa 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -116,6 +116,11 @@ internal IEnumerable Adapters /// internal FeatureFilterTelemetry FeatureFilterTelemetry { get; set; } = new FeatureFilterTelemetry(); + /// + /// Options used to configure provider startup. + /// + internal StartupOptions Startup { get; set; } = new StartupOptions(); + /// /// Specify what key-values to include in the configuration provider. /// can be called multiple times to include multiple sets of key-values. @@ -351,7 +356,7 @@ public AzureAppConfigurationOptions TrimKeyPrefix(string prefix) } /// - /// Configure the client used to communicate with Azure App Configuration. + /// Configure the client(s) used to communicate with Azure App Configuration. /// /// A callback used to configure Azure App Configuration client options. public AzureAppConfigurationOptions ConfigureClientOptions(Action configure) @@ -429,6 +434,16 @@ public AzureAppConfigurationOptions Map(Func + /// Configure the provider behavior when loading data from Azure App Configuration on startup. + /// + /// A callback used to configure Azure App Configuration startup options. + public AzureAppConfigurationOptions ConfigureStartupOptions(Action configure) + { + configure?.Invoke(Startup); + return this; + } + private static ConfigurationClientOptions GetDefaultClientOptions() { var clientOptions = new ConfigurationClientOptions(ConfigurationClientOptions.ServiceVersion.V2022_11_01_Preview); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs index 551794c3..34a60350 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationProvider.cs @@ -88,9 +88,9 @@ public ILoggerFactory LoggerFactory } } - public AzureAppConfigurationProvider(IConfigurationClientManager clientManager, AzureAppConfigurationOptions options, bool optional) + public AzureAppConfigurationProvider(IConfigurationClientManager configClientManager, AzureAppConfigurationOptions options, bool optional) { - _configClientManager = clientManager ?? throw new ArgumentNullException(nameof(clientManager)); + _configClientManager = configClientManager ?? throw new ArgumentNullException(nameof(configClientManager)); _options = options ?? throw new ArgumentNullException(nameof(options)); _optional = optional; @@ -127,16 +127,13 @@ public override void Load() { var watch = Stopwatch.StartNew(); - var loadStartTime = DateTimeOffset.UtcNow; - - // Guaranteed to have atleast one available client since it is a application startup path. - IEnumerable availableClients = _configClientManager.GetAvailableClients(loadStartTime); - try { + using var startupCancellationTokenSource = new CancellationTokenSource(_options.Startup.Timeout); + // Load() is invoked only once during application startup. We don't need to check for concurrent network // operations here because there can't be any other startup or refresh operation in progress at this time. - InitializeAsync(_optional, availableClients, CancellationToken.None).ConfigureAwait(false).GetAwaiter().GetResult(); + LoadAsync(_optional, startupCancellationTokenSource.Token).ConfigureAwait(false).GetAwaiter().GetResult(); } catch (ArgumentException) { @@ -205,7 +202,8 @@ public async Task RefreshAsync(CancellationToken cancellationToken) if (InitializationCacheExpires < utcNow) { InitializationCacheExpires = utcNow.Add(MinCacheExpirationInterval); - await InitializeAsync(ignoreFailures: false, availableClients, cancellationToken).ConfigureAwait(false); + + await InitializeAsync(availableClients, cancellationToken).ConfigureAwait(false); } return; @@ -548,42 +546,135 @@ private async Task> PrepareData(Dictionary availableClients, CancellationToken cancellationToken = default) + private async Task LoadAsync(bool ignoreFailures, CancellationToken cancellationToken) { - Dictionary data = null; - Dictionary watchedSettings = null; - + var startupStopwatch = Stopwatch.StartNew(); + + int postFixedWindowAttempts = 0; + + var startupExceptions = new List(); + try { - await ExecuteWithFailOverPolicyAsync( - availableClients, - async (client) => + while (true) + { + IEnumerable clients = _configClientManager.GetAllClients(); + + if (await TryInitializeAsync(clients, startupExceptions, cancellationToken).ConfigureAwait(false)) { - data = await LoadSelectedKeyValues( - client, - cancellationToken) - .ConfigureAwait(false); - - watchedSettings = await LoadKeyValuesRegisteredForRefresh( - client, - data, - cancellationToken) - .ConfigureAwait(false); - - watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); - }, - cancellationToken) - .ConfigureAwait(false); + break; + } + + TimeSpan delay; + + if (startupStopwatch.Elapsed.TryGetFixedBackoff(out TimeSpan backoff)) + { + delay = backoff; + } + else + { + postFixedWindowAttempts++; + + delay = FailOverConstants.MinStartupBackoffDuration.CalculateBackoffDuration( + FailOverConstants.MaxBackoffDuration, + postFixedWindowAttempts); + } + + try + { + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw new TimeoutException( + $"The provider timed out while attempting to load.", + new AggregateException(startupExceptions)); + } + } } catch (Exception exception) when ( ignoreFailures && (exception is RequestFailedException || + exception is KeyVaultReferenceException || + exception is TimeoutException || exception is OperationCanceledException || exception is InvalidOperationException || ((exception as AggregateException)?.InnerExceptions?.Any(e => e is RequestFailedException || e is OperationCanceledException) ?? false))) { } + } + + private async Task TryInitializeAsync(IEnumerable clients, List startupExceptions, CancellationToken cancellationToken = default) + { + try + { + await InitializeAsync(clients, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return false; + } + catch (RequestFailedException exception) + { + if (IsFailOverable(exception)) + { + startupExceptions.Add(exception); + + return false; + } + + throw; + } + catch (AggregateException exception) + { + if (exception.InnerExceptions?.Any(e => e is OperationCanceledException) ?? false) + { + if (!cancellationToken.IsCancellationRequested) + { + startupExceptions.Add(exception); + } + + return false; + } + + if (IsFailOverable(exception)) + { + startupExceptions.Add(exception); + + return false; + } + + throw; + } + + return true; + } + + private async Task InitializeAsync(IEnumerable clients, CancellationToken cancellationToken = default) + { + Dictionary data = null; + Dictionary watchedSettings = null; + + await ExecuteWithFailOverPolicyAsync( + clients, + async (client) => + { + data = await LoadSelectedKeyValues( + client, + cancellationToken) + .ConfigureAwait(false); + + watchedSettings = await LoadKeyValuesRegisteredForRefresh( + client, + data, + cancellationToken) + .ConfigureAwait(false); + + watchedSettings = UpdateWatchedKeyValueCollections(watchedSettings, data); + }, + cancellationToken) + .ConfigureAwait(false); // Update the cache expiration time for all refresh registered settings and feature flags foreach (KeyValueWatcher changeWatcher in _options.ChangeWatchers.Concat(_options.MultiKeyWatchers)) @@ -599,17 +690,10 @@ e is RequestFailedException || adapter.InvalidateCache(); } - try - { - Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); - SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); - _watchedSettings = watchedSettings; - _mappedData = mappedData; - } - catch (KeyVaultReferenceException) when (ignoreFailures) - { - // ignore failures - } + Dictionary mappedData = await MapConfigurationSettings(data).ConfigureAwait(false); + SetData(await PrepareData(mappedData, cancellationToken).ConfigureAwait(false)); + _watchedSettings = watchedSettings; + _mappedData = mappedData; } } @@ -856,7 +940,10 @@ private void UpdateCacheExpirationTime(KeyValueWatcher changeWatcher) changeWatcher.CacheExpires = DateTimeOffset.UtcNow.Add(cacheExpirationTime); } - private async Task ExecuteWithFailOverPolicyAsync(IEnumerable clients, Func> funcToExecute, CancellationToken cancellationToken = default) + private async Task ExecuteWithFailOverPolicyAsync( + IEnumerable clients, + Func> funcToExecute, + CancellationToken cancellationToken = default) { using IEnumerator clientEnumerator = clients.GetEnumerator(); @@ -929,7 +1016,10 @@ private async Task ExecuteWithFailOverPolicyAsync(IEnumerable clients, Func funcToExecute, CancellationToken cancellationToken = default) + private async Task ExecuteWithFailOverPolicyAsync( + IEnumerable clients, + Func funcToExecute, + CancellationToken cancellationToken = default) { await ExecuteWithFailOverPolicyAsync(clients, async (client) => { @@ -948,18 +1038,28 @@ private bool IsFailOverable(AggregateException ex) private bool IsFailOverable(RequestFailedException rfe) { + if (rfe.Status == HttpStatusCodes.TooManyRequests || + rfe.Status == (int)HttpStatusCode.RequestTimeout || + rfe.Status >= (int)HttpStatusCode.InternalServerError) + { + return true; + } + + Exception innerException; - // The InnerException could be SocketException or WebException when endpoint is invalid and IOException if it is network issue. - if (rfe.InnerException != null && rfe.InnerException is HttpRequestException hre && hre.InnerException != null) + if (rfe.InnerException is HttpRequestException hre) + { + innerException = hre.InnerException; + } + else { - return hre.InnerException is WebException || - hre.InnerException is SocketException || - hre.InnerException is IOException; + innerException = rfe.InnerException; } - return rfe.Status == HttpStatusCodes.TooManyRequests || - rfe.Status == (int)HttpStatusCode.RequestTimeout || - rfe.Status >= (int)HttpStatusCode.InternalServerError; + // The InnerException could be SocketException or WebException when an endpoint is invalid and IOException if it's a network issue. + return innerException is WebException || + innerException is SocketException || + innerException is IOException; } private async Task> MapConfigurationSettings(Dictionary data) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs index b0b9dc80..b56b7acf 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/ConfigurationClientManager.cs @@ -4,12 +4,10 @@ using Azure.Core; using Azure.Data.AppConfiguration; -using Microsoft.Extensions.Configuration.AzureAppConfiguration.Constants; using Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions; using System; using System.Collections.Generic; using System.Linq; -using System.Net; namespace Microsoft.Extensions.Configuration.AzureAppConfiguration { @@ -62,6 +60,11 @@ public IEnumerable GetAvailableClients(DateTimeOffset time) return _clients.Where(client => client.BackoffEndTime <= time).Select(c => c.Client).ToList(); } + public IEnumerable GetAllClients() + { + return _clients.Select(c => c.Client).ToList(); + } + public void UpdateClientStatus(ConfigurationClient client, bool successful) { if (client == null) @@ -96,7 +99,7 @@ public bool UpdateSyncToken(Uri endpoint, string syncToken) throw new ArgumentNullException(nameof(syncToken)); } - ConfigurationClientWrapper clientWrapper = this._clients.SingleOrDefault(c => new EndpointComparer().Equals(c.Endpoint, endpoint)); + ConfigurationClientWrapper clientWrapper = _clients.SingleOrDefault(c => new EndpointComparer().Equals(c.Endpoint, endpoint)); if (clientWrapper != null) { diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/FailOverConstants.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/FailOverConstants.cs index bfdac9d9..f4836ced 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/FailOverConstants.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Constants/FailOverConstants.cs @@ -4,12 +4,15 @@ using System; -namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.Constants +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration { internal class FailOverConstants { // Timeouts to retry requests to config stores and their replicas after failure. + public static readonly TimeSpan MinBackoffDuration = TimeSpan.FromSeconds(30); public static readonly TimeSpan MaxBackoffDuration = TimeSpan.FromMinutes(10); + // Minimum backoff duration for retries that occur after the fixed backoff window during startup. + public static readonly TimeSpan MinStartupBackoffDuration = TimeSpan.FromSeconds(30); } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/TimeSpanExtensions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/TimeSpanExtensions.cs index a3196656..e6836d69 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/TimeSpanExtensions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/Extensions/TimeSpanExtensions.cs @@ -1,14 +1,22 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. // -using Microsoft.Extensions.Configuration.AzureAppConfiguration.Constants; using System; +using System.Collections.Generic; namespace Microsoft.Extensions.Configuration.AzureAppConfiguration.Extensions { internal static class TimeSpanExtensions { private const int MaxAttempts = 63; + private const double JitterRatio = 0.25; + + private static readonly IList> StartupMaxBackoffDurationIntervals = new List> + { + new KeyValuePair(TimeSpan.FromSeconds(100), TimeSpan.FromSeconds(5)), + new KeyValuePair(TimeSpan.FromSeconds(200), TimeSpan.FromSeconds(10)), + new KeyValuePair(TimeSpan.FromSeconds(600), FailOverConstants.MinStartupBackoffDuration), + }; /// /// This method calculates randomized exponential backoff times for operations that occur periodically on a given . @@ -38,7 +46,7 @@ public static TimeSpan CalculateBackoffTime(this TimeSpan interval, TimeSpan min } /// - /// This method calculates the randomized exponential backoff duration for the configuration store after a failure + /// This method calculates the jittered exponential backoff duration for the configuration store after a failure /// which lies between and . /// /// The minimum duration to retry after. @@ -70,7 +78,52 @@ public static TimeSpan CalculateBackoffDuration(this TimeSpan minDuration, TimeS calculatedMilliseconds = maxDuration.TotalMilliseconds; } - return TimeSpan.FromMilliseconds(minDuration.TotalMilliseconds + new Random().NextDouble() * (calculatedMilliseconds - minDuration.TotalMilliseconds)); + return TimeSpan.FromMilliseconds(calculatedMilliseconds).Jitter(JitterRatio); + } + + /// + /// This method tries to get the fixed backoff duration for the elapsed startup time. + /// + /// The time elapsed since the current startup began. + /// The backoff time span if getting the fixed backoff is successful. + /// A boolean indicating if getting the fixed backoff duration was successful. Returns false + /// if the elapsed startup time is greater than the fixed backoff window. + public static bool TryGetFixedBackoff(this TimeSpan startupTimeElapsed, out TimeSpan backoff) + { + foreach (KeyValuePair interval in StartupMaxBackoffDurationIntervals) + { + if (startupTimeElapsed < interval.Key) + { + backoff = interval.Value; + + return true; + } + } + + backoff = TimeSpan.Zero; + + return false; + } + + private static TimeSpan Jitter(this TimeSpan timeSpan, double ratio) + { + if (ratio < 0 || ratio > 1) + { + throw new ArgumentOutOfRangeException(nameof(ratio)); + } + + if (ratio == 0) + { + return timeSpan; + } + + var rand = new Random(); + + double jitter = ratio * (rand.NextDouble() * 2 - 1); + + double interval = timeSpan.TotalMilliseconds * (1 + jitter); + + return TimeSpan.FromMilliseconds(interval); } } } diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationClientManager.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationClientManager.cs index dfae119f..be56f5ff 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationClientManager.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/IConfigurationClientManager.cs @@ -12,6 +12,8 @@ internal interface IConfigurationClientManager { IEnumerable GetAvailableClients(DateTimeOffset time); + IEnumerable GetAllClients(); + void UpdateClientStatus(ConfigurationClient client, bool successful); bool UpdateSyncToken(Uri endpoint, String syncToken); diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/StartupOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/StartupOptions.cs new file mode 100644 index 00000000..56d06c19 --- /dev/null +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/StartupOptions.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +// + +using System; + +namespace Microsoft.Extensions.Configuration.AzureAppConfiguration +{ + /// + /// Options used when initially loading data into the configuration provider. + /// + public class StartupOptions + { + /// + /// The amount of time allowed to load data from Azure App Configuration on startup. + /// + public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(100); + } +} diff --git a/tests/Tests.AzureAppConfiguration/ClientOptionsTests.cs b/tests/Tests.AzureAppConfiguration/ClientOptionsTests.cs index 1d40f7d5..8e1c310c 100644 --- a/tests/Tests.AzureAppConfiguration/ClientOptionsTests.cs +++ b/tests/Tests.AzureAppConfiguration/ClientOptionsTests.cs @@ -3,11 +3,9 @@ // using Azure; using Azure.Core; -using Azure.Core.Testing; -using Azure.Data.AppConfiguration; using Microsoft.Extensions.Configuration; -using Moq; using System; +using System.Linq; using Xunit; namespace Tests.AzureAppConfiguration @@ -19,40 +17,54 @@ public void ClientOptionsTests_OverridesDefaultClientOptions() { // Arrange var requestCountPolicy = new HttpRequestCountPipelinePolicy(); - int defaultMaxRetries = 0; var configBuilder = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { + options.ConfigureStartupOptions(startupOptions => + { + startupOptions.Timeout = TimeSpan.FromSeconds(15); + }); options.Connect(TestHelpers.CreateMockEndpointString()); options.ClientOptions.AddPolicy(requestCountPolicy, HttpPipelinePosition.PerRetry); - defaultMaxRetries = options.ClientOptions.Retry.MaxRetries; }); // Act - Build - Assert.Throws(configBuilder.Build); + Exception exception = Assert.Throws(() => configBuilder.Build()); - // Assert defaultMaxRetries + 1 original request = totalRequestCount - Assert.Equal(defaultMaxRetries + 1, requestCountPolicy.RequestCount); + // Assert the inner aggregate exception + Assert.IsType(exception.InnerException); - requestCountPolicy.ResetRequestCount(); + // Assert the second inner aggregate exception + Assert.IsType(exception.InnerException.InnerException); - // Arrange - int maxRetries = defaultMaxRetries + 1; + var exponentialRequestCount = requestCountPolicy.RequestCount; + + requestCountPolicy.ResetRequestCount(); configBuilder = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { + options.ConfigureStartupOptions(startupOptions => + { + startupOptions.Timeout = TimeSpan.FromSeconds(15); + }); options.Connect(TestHelpers.CreateMockEndpointString()); - options.ConfigureClientOptions(clientOptions => clientOptions.Retry.MaxRetries = maxRetries); + options.ConfigureClientOptions(clientOptions => clientOptions.Retry.Delay = TimeSpan.FromSeconds(60)); options.ClientOptions.AddPolicy(requestCountPolicy, HttpPipelinePosition.PerRetry); }); // Act - Build - Assert.Throws(configBuilder.Build); + exception = Assert.Throws(() => configBuilder.Build()); + + // Assert the inner aggregate exception + Assert.IsType(exception.InnerException); + + // Assert the inner request failed exceptions + Assert.True((exception.InnerException as AggregateException)?.InnerExceptions?.All(e => e is RequestFailedException) ?? false); - // Assert maxRetries + 1 original request = totalRequestCount. - Assert.Equal(maxRetries + 1, requestCountPolicy.RequestCount); + // Assert less retries due to increased delay + Assert.True(requestCountPolicy.RequestCount < exponentialRequestCount); } } } diff --git a/tests/Tests.AzureAppConfiguration/ConnectTests.cs b/tests/Tests.AzureAppConfiguration/ConnectTests.cs index 6f65b537..22a1507a 100644 --- a/tests/Tests.AzureAppConfiguration/ConnectTests.cs +++ b/tests/Tests.AzureAppConfiguration/ConnectTests.cs @@ -2,12 +2,12 @@ // Licensed under the MIT license. // using Azure.Core; -using Azure.Core.Testing; using Azure.Data.AppConfiguration; using Microsoft.Extensions.Configuration; using Moq; using System; using System.Collections.Generic; +using System.Linq; using System.Threading; using System.Threading.Tasks; using Xunit; @@ -74,17 +74,27 @@ public void ConnectTests_UsesParametersFromLatestConnectCall() configBuilder = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { + options.ConfigureStartupOptions(startupOptions => + { + startupOptions.Timeout = TimeSpan.FromSeconds(15); + }); options.Connect("invalid_connection_string"); options.Connect(TestHelpers.PrimaryConfigStoreEndpoint, mockTokenCredential.Object); options.ClientOptions.AddPolicy(requestCountPolicy, HttpPipelinePosition.PerRetry); defaultMaxRetries = options.ClientOptions.Retry.MaxRetries; }); - // Act - Assert.Throws(configBuilder.Build); + // Act - Build + Exception exception = Assert.Throws(() => configBuilder.Build()); + + // Assert the inner aggregate exception + Assert.IsType(exception.InnerException); + + // Assert the second inner aggregate exception + Assert.IsType(exception.InnerException.InnerException); // Assert the second connect call was successful and it made requests to the configuration store. - Assert.Equal(defaultMaxRetries + 1, requestCountPolicy.RequestCount); + Assert.True(requestCountPolicy.RequestCount >= defaultMaxRetries + 1); } } } diff --git a/tests/Tests.AzureAppConfiguration/FailoverTests.cs b/tests/Tests.AzureAppConfiguration/FailoverTests.cs index e480e571..78d1e858 100644 --- a/tests/Tests.AzureAppConfiguration/FailoverTests.cs +++ b/tests/Tests.AzureAppConfiguration/FailoverTests.cs @@ -111,6 +111,10 @@ public void FailOverTests_ReturnsAllClientsIfAllBackedOff() var configBuilder = new ConfigurationBuilder() .AddAzureAppConfiguration(options => { + options.ConfigureStartupOptions(startupOptions => + { + startupOptions.Timeout = TimeSpan.FromSeconds(15); + }); options.ClientManager = configClientManager; options.Select("TestKey*"); options.ConfigureRefresh(refreshOptions => @@ -123,7 +127,13 @@ public void FailOverTests_ReturnsAllClientsIfAllBackedOff() }); // Throws last exception when all clients fail. - Assert.Throws(configBuilder.Build); + Exception exception = Assert.Throws(() => configBuilder.Build()); + + // Assert the inner aggregate exception + Assert.IsType(exception.InnerException); + + // Assert the inner request failed exceptions + Assert.True((exception.InnerException as AggregateException)?.InnerExceptions?.All(e => e is RequestFailedException) ?? false); // The client manager should return no clients since all clients are in the back-off state. Assert.False(configClientManager.GetAvailableClients(DateTimeOffset.UtcNow).Any()); diff --git a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs index deb0138b..0cccf3cd 100644 --- a/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs +++ b/tests/Tests.AzureAppConfiguration/KeyVaultReferenceTests.cs @@ -274,6 +274,10 @@ public void CancellationToken() { new ConfigurationBuilder().AddAzureAppConfiguration(options => { + options.ConfigureStartupOptions(startupOptions => + { + startupOptions.Timeout = TimeSpan.FromSeconds(5); + }); options.ClientManager = TestHelpers.CreateMockedConfigurationClientManager(mockClient.Object);; options.ConfigureKeyVault(kv => kv.Register(mockSecretClient.Object)); }) diff --git a/tests/Tests.AzureAppConfiguration/MockedConfigurationClientManager.cs b/tests/Tests.AzureAppConfiguration/MockedConfigurationClientManager.cs index 9d76a781..c0859d86 100644 --- a/tests/Tests.AzureAppConfiguration/MockedConfigurationClientManager.cs +++ b/tests/Tests.AzureAppConfiguration/MockedConfigurationClientManager.cs @@ -20,12 +20,14 @@ internal class MockedConfigurationClientManager : IConfigurationClientManager public MockedConfigurationClientManager(IEnumerable clients) { - this._clients = clients.ToList(); + _clients = clients.ToList(); } - public IEnumerable GetAvailableClients(DateTimeOffset time) + public IEnumerable GetAvailableClients(DateTimeOffset time) => GetAllClients(); + + public IEnumerable GetAllClients() { - return this._clients.Select(cw => cw.Client); + return _clients.Select(cw => cw.Client); } public void UpdateClientStatus(ConfigurationClient client, bool successful) diff --git a/tests/Tests.AzureAppConfiguration/RefreshTests.cs b/tests/Tests.AzureAppConfiguration/RefreshTests.cs index c829dfe2..7d797dba 100644 --- a/tests/Tests.AzureAppConfiguration/RefreshTests.cs +++ b/tests/Tests.AzureAppConfiguration/RefreshTests.cs @@ -1076,6 +1076,10 @@ public void RefreshTests_ChainedConfigurationProviderUsedAsRootForRefresherProvi private void optionsInitializer(AzureAppConfigurationOptions options) { + options.ConfigureStartupOptions(startupOptions => + { + startupOptions.Timeout = TimeSpan.FromSeconds(5); + }); options.Connect(TestHelpers.CreateMockEndpointString()); options.ConfigureClientOptions(clientOptions => clientOptions.Retry.MaxRetries = 0); } From 46f347a09f52b289cf25de871a748632fa2b48e0 Mon Sep 17 00:00:00 2001 From: Amer Jusupovic <32405726+amerjusupovic@users.noreply.github.com> Date: Thu, 16 Nov 2023 14:50:33 -0800 Subject: [PATCH 16/17] Update Azure.Data.AppConfiguration to stable version 1.3.0 (#492) * update sdk package to 1.3.0 * update test file package for security alert --- .../AzureAppConfigurationOptions.cs | 2 +- ...rosoft.Extensions.Configuration.AzureAppConfiguration.csproj | 2 +- .../Tests.AzureAppConfiguration.csproj | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index 068ddbfa..1edf1b7c 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -446,7 +446,7 @@ public AzureAppConfigurationOptions ConfigureStartupOptions(Action - + diff --git a/tests/Tests.AzureAppConfiguration/Tests.AzureAppConfiguration.csproj b/tests/Tests.AzureAppConfiguration/Tests.AzureAppConfiguration.csproj index 19955a28..4ed6e40f 100644 --- a/tests/Tests.AzureAppConfiguration/Tests.AzureAppConfiguration.csproj +++ b/tests/Tests.AzureAppConfiguration/Tests.AzureAppConfiguration.csproj @@ -10,7 +10,7 @@ - + From 5301c7824bc89ca59d513b6755970edea2f6d7e4 Mon Sep 17 00:00:00 2001 From: AMER JUSUPOVIC Date: Thu, 16 Nov 2023 16:53:18 -0800 Subject: [PATCH 17/17] fix merge conflict with FeatureFilterTracing in AzureAppConfigurationOptions --- .../AzureAppConfigurationOptions.cs | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs index ca3fb4f7..02ceb495 100644 --- a/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs +++ b/src/Microsoft.Extensions.Configuration.AzureAppConfiguration/AzureAppConfigurationOptions.cs @@ -25,12 +25,7 @@ public class AzureAppConfigurationOptions private List _changeWatchers = new List(); private List _multiKeyWatchers = new List(); - private List _adapters = new List() - { - new AzureKeyVaultKeyValueAdapter(new AzureKeyVaultSecretProvider()), - new JsonKeyValueAdapter(), - new FeatureManagementKeyValueAdapter() - }; + private List _adapters; private List>> _mappers = new List>>(); private List _kvSelectors = new List(); private IConfigurationRefresher _refresher = new AzureAppConfigurationRefresher(); @@ -121,6 +116,19 @@ internal IEnumerable Adapters /// internal StartupOptions Startup { get; set; } = new StartupOptions(); + /// + /// Initializes a new instance of the class. + /// + public AzureAppConfigurationOptions() + { + _adapters = new List() + { + new AzureKeyVaultKeyValueAdapter(new AzureKeyVaultSecretProvider()), + new JsonKeyValueAdapter(), + new FeatureManagementKeyValueAdapter(FeatureFilterTracing) + }; + } + /// /// Specify what key-values to include in the configuration provider. /// can be called multiple times to include multiple sets of key-values.