From 4ca4b0a76a32bd974757def2a2620103f88d465a Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Wed, 28 Feb 2024 18:06:58 -0800 Subject: [PATCH 01/47] Improve fhirtimer --- .../Features/Watchdogs/DefragWatchdog.cs | 5 +- .../Features/Watchdogs/FhirTimer.cs | 100 ++++-------------- .../Features/Watchdogs/Watchdog.cs | 23 ---- .../Features/Watchdogs/WatchdogLease.cs | 4 +- .../Watchdogs/WatchdogsBackgroundService.cs | 18 +--- .../Persistence/SqlServerWatchdogTests.cs | 8 -- 6 files changed, 33 insertions(+), 125 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/DefragWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/DefragWatchdog.cs index 7cba164b44..191cea4f68 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/DefragWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/DefragWatchdog.cs @@ -57,8 +57,9 @@ internal DefragWatchdog() internal async Task StartAsync(CancellationToken cancellationToken) { _cancellationToken = cancellationToken; - await StartAsync(false, 24 * 3600, 2 * 3600, cancellationToken); - await InitDefragParamsAsync(); + await Task.WhenAll( + StartAsync(false, 24 * 3600, 2 * 3600, cancellationToken), + InitDefragParamsAsync()); } protected override async Task ExecuteAsync() diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs index df33cba028..a5bbc7276a 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs @@ -8,108 +8,54 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.Health.Core; +using Microsoft.Health.Fhir.Core.Extensions; namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - public abstract class FhirTimer : IDisposable + public abstract class FhirTimer(ILogger logger = null) { - private Timer _timer; - private bool _disposed = false; - private bool _isRunning; private bool _isFailing; - private bool _isStarted; - private string _lastException; - private readonly ILogger _logger; - private CancellationToken _cancellationToken; - - protected FhirTimer(ILogger logger = null) - { - _logger = logger; - _isFailing = false; - _lastException = null; - LastRunDateTime = DateTime.Parse("2017-12-01"); - } internal double PeriodSec { get; set; } - internal DateTime LastRunDateTime { get; private set; } - - internal bool IsRunning => _isRunning; + internal DateTimeOffset LastRunDateTime { get; private set; } = DateTime.Parse("2017-12-01"); internal bool IsFailing => _isFailing; - internal bool IsStarted => _isStarted; - - internal string LastException => _lastException; - protected async Task StartAsync(double periodSec, CancellationToken cancellationToken) { PeriodSec = periodSec; - _cancellationToken = cancellationToken; - - // WARNING: Avoid using 'async' lambda when delegate type returns 'void' - _timer = new Timer(async _ => await RunInternalAsync(), null, TimeSpan.FromSeconds(PeriodSec * RandomNumberGenerator.GetInt32(1000) / 1000), TimeSpan.FromSeconds(PeriodSec)); - - _isStarted = true; - await Task.CompletedTask; - } - protected abstract Task RunAsync(); - - private async Task RunInternalAsync() - { - if (_isRunning || _cancellationToken.IsCancellationRequested) - { - return; - } + await Task.Delay(TimeSpan.FromSeconds(PeriodSec * RandomNumberGenerator.GetInt32(1000) / 1000), cancellationToken); + using var periodicTimer = new PeriodicTimer(TimeSpan.FromSeconds(PeriodSec)); - try - { - _isRunning = true; - await RunAsync(); - _isFailing = false; - _lastException = null; - LastRunDateTime = DateTime.UtcNow; - } - catch (Exception e) + while (!cancellationToken.IsCancellationRequested) { try { - _logger.LogWarning(e, "Error executing FHIR Timer"); // exceptions in logger should never bubble up + await periodicTimer.WaitForNextTickAsync(cancellationToken); } - catch + catch (OperationCanceledException) { - // ignored + // Time to exit + break; } - _isFailing = true; - _lastException = e.ToString(); - } - finally - { - _isRunning = false; + try + { + await RunAsync(); + LastRunDateTime = Clock.UtcNow; + _isFailing = false; + } + catch (Exception e) + { + logger.LogWarning(e, "Error executing timer"); + _isFailing = true; + } } } - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - - protected virtual void Dispose(bool disposing) - { - if (_disposed) - { - return; - } - - if (disposing) - { - _timer?.Dispose(); - } - - _disposed = true; - } + protected abstract Task RunAsync(); } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs index 65c4926787..d68b671a44 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs @@ -19,7 +19,6 @@ public abstract class Watchdog : FhirTimer private ISqlRetryService _sqlRetryService; private readonly ILogger _logger; private readonly WatchdogLease _watchdogLease; - private bool _disposed = false; private double _periodSec; private double _leasePeriodSec; @@ -138,27 +137,5 @@ protected async Task GetLongParameterByIdAsync(string id, CancellationToke return (long)value; } - - public new void Dispose() - { - Dispose(true); - } - - protected override void Dispose(bool disposing) - { - if (_disposed) - { - return; - } - - if (disposing) - { - _watchdogLease?.Dispose(); - } - - base.Dispose(disposing); - - _disposed = true; - } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs index cbf92d3f9c..ee4a595cd5 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs @@ -17,7 +17,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs internal class WatchdogLease : FhirTimer { private const double TimeoutFactor = 0.25; - private readonly object _locker = new object(); + private readonly object _locker = new(); private readonly ISqlRetryService _sqlRetryService; private readonly ILogger _logger; @@ -70,7 +70,7 @@ protected override async Task RunAsync() cmd.Parameters.AddWithValue("@Watchdog", _watchdogName); cmd.Parameters.AddWithValue("@Worker", _worker); cmd.Parameters.AddWithValue("@AllowRebalance", _allowRebalance); - cmd.Parameters.AddWithValue("@WorkerIsRunning", IsRunning); + cmd.Parameters.AddWithValue("@WorkerIsRunning", true); cmd.Parameters.AddWithValue("@ForceAcquire", false); // TODO: Provide ability to set. usefull for tests cmd.Parameters.AddWithValue("@LeasePeriodSec", PeriodSec + _leaseTimeoutSec); var leaseEndTimePar = cmd.Parameters.AddWithValue("@LeaseEndTime", DateTime.UtcNow); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs index eea8ded412..85dc260a8d 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs @@ -43,16 +43,11 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); } - await _defragWatchdog.StartAsync(stoppingToken); - await _cleanupEventLogWatchdog.StartAsync(stoppingToken); - await _transactionWatchdog.Value.StartAsync(stoppingToken); - await _invisibleHistoryCleanupWatchdog.StartAsync(stoppingToken); - - while (true) - { - stoppingToken.ThrowIfCancellationRequested(); - await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); - } + await Task.WhenAll( + _defragWatchdog.StartAsync(stoppingToken), + _cleanupEventLogWatchdog.StartAsync(stoppingToken), + _transactionWatchdog.Value.StartAsync(stoppingToken), + _invisibleHistoryCleanupWatchdog.StartAsync(stoppingToken)); } public Task Handle(StorageInitializedNotification notification, CancellationToken cancellationToken) @@ -63,10 +58,7 @@ public Task Handle(StorageInitializedNotification notification, CancellationToke public override void Dispose() { - _defragWatchdog.Dispose(); - _cleanupEventLogWatchdog.Dispose(); _transactionWatchdog.Dispose(); - _invisibleHistoryCleanupWatchdog.Dispose(); base.Dispose(); } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs index 1c258a3749..127eb7a94b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs @@ -99,8 +99,6 @@ COMMIT TRANSACTION var sizeAfter = GetSize(); Assert.True(sizeAfter * 9 < sizeBefore, $"{sizeAfter} * 9 < {sizeBefore}"); - - wd.Dispose(); } [Fact] @@ -142,8 +140,6 @@ WHILE @i < 10000 _testOutputHelper.WriteLine($"EventLog.Count={GetCount("EventLog")}."); Assert.True(GetCount("EventLog") <= 1000, "Count is low"); - - wd.Dispose(); } [Fact] @@ -209,8 +205,6 @@ FOR INSERT } Assert.Equal(1, GetCount("NumberSearchParam")); // wd rolled forward transaction - - wd.Dispose(); } [Fact] @@ -278,8 +272,6 @@ public async Task AdvanceVisibility() _testOutputHelper.WriteLine($"Visibility={visibility}"); Assert.Equal(tran3.TransactionId, visibility); - - wd.Dispose(); } private ResourceWrapperFactory CreateResourceWrapperFactory() From 4331d99ae67d6733d951f72d88bd780d81064ae0 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Mon, 15 Apr 2024 17:04:04 -0700 Subject: [PATCH 02/47] Subscription infra --- Microsoft.Health.Fhir.sln | 7 ++ .../Microsoft.Health.Fhir.Api.csproj | 1 + .../FhirServerServiceCollectionExtensions.cs | 2 + .../Features/Operations/JobType.cs | 2 + .../Features/Operations/QueueType.cs | 1 + .../Storage/SqlRetry/ISqlRetryService.cs | 4 +- .../Storage/SqlRetry/SqlCommandExtensions.cs | 6 +- .../Storage/SqlRetry/SqlRetryService.cs | 8 +- .../Storage/SqlServerFhirDataStore.cs | 9 +- .../Features/Storage/SqlStoreClient.cs | 7 +- .../Watchdogs/EventProcessorWatchdog.cs | 109 ++++++++++++++++++ .../InvisibleHistoryCleanupWatchdog.cs | 4 +- .../Watchdogs/WatchdogsBackgroundService.cs | 8 +- .../Microsoft.Health.Fhir.SqlServer.csproj | 1 + ...rBuilderSqlServerRegistrationExtensions.cs | 6 +- .../Channels/ISubscriptionChannel.cs | 17 +++ .../Channels/StorageChannel.cs | 17 +++ ...Microsoft.Health.Fhir.Subscriptions.csproj | 12 ++ .../Models/ChannelInfo.cs | 35 ++++++ .../Models/SubscriptionChannelType.cs | 21 ++++ .../Models/SubscriptionContentType.cs | 14 +++ .../Models/SubscriptionInfo.cs | 24 ++++ .../Models/SubscriptionJobDefinition.cs | 31 +++++ .../Operations/SubscriptionProcessingJob.cs | 24 ++++ .../SubscriptionsOrchestratorJob.cs | 51 ++++++++ .../Registration/SubscriptionsModule.cs | 28 +++++ .../SubscriptionManager.cs | 22 ++++ ...erFhirResourceChangeCaptureEnabledTests.cs | 4 +- .../Persistence/SqlRetryServiceTests.cs | 4 +- 29 files changed, 452 insertions(+), 27 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionChannelType.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionContentType.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/SubscriptionManager.cs diff --git a/Microsoft.Health.Fhir.sln b/Microsoft.Health.Fhir.sln index 788977da40..bc333070ea 100644 --- a/Microsoft.Health.Fhir.sln +++ b/Microsoft.Health.Fhir.sln @@ -205,6 +205,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Cosmo EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.CosmosDb.Core", "src\Microsoft.Health.Fhir.CosmosDb.Core\Microsoft.Health.Fhir.CosmosDb.Core.csproj", "{1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Subscriptions", "src\Microsoft.Health.Fhir.Subscriptions\Microsoft.Health.Fhir.Subscriptions.csproj", "{BD8F3137-89F5-4EE5-B269-24D73081E00A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -483,6 +485,10 @@ Global {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75}.Debug|Any CPU.Build.0 = Debug|Any CPU {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75}.Release|Any CPU.ActiveCfg = Release|Any CPU {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75}.Release|Any CPU.Build.0 = Release|Any CPU + {BD8F3137-89F5-4EE5-B269-24D73081E00A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BD8F3137-89F5-4EE5-B269-24D73081E00A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BD8F3137-89F5-4EE5-B269-24D73081E00A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BD8F3137-89F5-4EE5-B269-24D73081E00A}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -578,6 +584,7 @@ Global {10661BC9-01B0-4E35-9751-3B5CE97E25C0} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} {B9AAA11D-8C8C-44C3-AADE-801376EF82F0} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} + {BD8F3137-89F5-4EE5-B269-24D73081E00A} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution RESX_SortFileContentOnSave = True diff --git a/src/Microsoft.Health.Fhir.Api/Microsoft.Health.Fhir.Api.csproj b/src/Microsoft.Health.Fhir.Api/Microsoft.Health.Fhir.Api.csproj index 815c230b38..e91e43ea78 100644 --- a/src/Microsoft.Health.Fhir.Api/Microsoft.Health.Fhir.Api.csproj +++ b/src/Microsoft.Health.Fhir.Api/Microsoft.Health.Fhir.Api.csproj @@ -26,6 +26,7 @@ + diff --git a/src/Microsoft.Health.Fhir.Api/Registration/FhirServerServiceCollectionExtensions.cs b/src/Microsoft.Health.Fhir.Api/Registration/FhirServerServiceCollectionExtensions.cs index 855a5dee9f..d22a5b88dd 100644 --- a/src/Microsoft.Health.Fhir.Api/Registration/FhirServerServiceCollectionExtensions.cs +++ b/src/Microsoft.Health.Fhir.Api/Registration/FhirServerServiceCollectionExtensions.cs @@ -8,6 +8,7 @@ using Microsoft.Health.Api.Modules; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Api.Configs; +using Microsoft.Health.Fhir.Subscriptions.Registration; namespace Microsoft.Extensions.DependencyInjection { @@ -20,6 +21,7 @@ public static IServiceCollection AddFhirServerBase(this IServiceCollection servi services.RegisterAssemblyModules(Assembly.GetExecutingAssembly(), fhirServerConfiguration); services.RegisterAssemblyModules(typeof(InitializationModule).Assembly, fhirServerConfiguration); + services.RegisterAssemblyModules(typeof(SubscriptionsModule).Assembly, fhirServerConfiguration); return services; } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/JobType.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/JobType.cs index f51fc9064d..73dc713b99 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/JobType.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/JobType.cs @@ -14,5 +14,7 @@ public enum JobType : int ExportOrchestrator = 4, BulkDeleteProcessing = 5, BulkDeleteOrchestrator = 6, + SubscriptionsProcessing = 7, + SubscriptionsOrchestrator = 8, } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/QueueType.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/QueueType.cs index 9affedc7c0..cd1529e32f 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/QueueType.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/QueueType.cs @@ -12,6 +12,7 @@ public enum QueueType : byte Import = 2, Defrag = 3, BulkDelete = 4, + Subscriptions = 5, } } #pragma warning restore CA1028 // Enum Storage should be Int32 diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/ISqlRetryService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/ISqlRetryService.cs index ab00060ffd..9a1928a0c2 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/ISqlRetryService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/ISqlRetryService.cs @@ -18,8 +18,8 @@ public interface ISqlRetryService Task ExecuteSql(Func action, ILogger logger, CancellationToken cancellationToken, bool isReadOnly = false); - Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false); + Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false); - Task> ExecuteReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false); + Task> ExecuteReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false); } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlCommandExtensions.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlCommandExtensions.cs index 809c30f686..1bc3b13281 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlCommandExtensions.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlCommandExtensions.cs @@ -14,17 +14,17 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage { public static class SqlCommandExtensions { - public static async Task ExecuteNonQueryAsync(this SqlCommand cmd, ISqlRetryService retryService, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false, bool disableRetries = false) + public static async Task ExecuteNonQueryAsync(this SqlCommand cmd, ISqlRetryService retryService, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false, bool disableRetries = false) { await retryService.ExecuteSql(cmd, async (sql, cancel) => await sql.ExecuteNonQueryAsync(cancel), logger, logMessage, cancellationToken, isReadOnly, disableRetries); } - public static async Task> ExecuteReaderAsync(this SqlCommand cmd, ISqlRetryService retryService, Func readerToResult, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false) + public static async Task> ExecuteReaderAsync(this SqlCommand cmd, ISqlRetryService retryService, Func readerToResult, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false) { return await retryService.ExecuteReaderAsync(cmd, readerToResult, logger, logMessage, cancellationToken, isReadOnly); } - public static async Task ExecuteScalarAsync(this SqlCommand cmd, ISqlRetryService retryService, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false, bool disableRetries = false) + public static async Task ExecuteScalarAsync(this SqlCommand cmd, ISqlRetryService retryService, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false, bool disableRetries = false) { object scalar = null; await retryService.ExecuteSql(cmd, async (sql, cancel) => { scalar = await sql.ExecuteScalarAsync(cancel); }, logger, logMessage, cancellationToken, isReadOnly, disableRetries); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs index c230103a2e..ac2c53d1e4 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs @@ -246,7 +246,6 @@ public async Task ExecuteSql(Func - /// Type used for the . /// SQL command to be executed. /// Delegate to be executed by passing as input parameter. /// Logger used on first try error (or retry error) and connection open. @@ -256,7 +255,7 @@ public async Task ExecuteSql(Func"Flag indicating whether retries are disabled." /// A task representing the asynchronous operation. /// When executing this method, if exception is thrown that is not retriable or if last retry fails, then same exception is thrown by this method. - public async Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false) + public async Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false) { EnsureArg.IsNotNull(sqlCommand, nameof(sqlCommand)); EnsureArg.IsNotNull(action, nameof(action)); @@ -308,7 +307,7 @@ public async Task ExecuteSql(SqlCommand sqlCommand, Func> ExecuteSqlDataReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, bool allRows, bool isReadOnly, CancellationToken cancellationToken) + private async Task> ExecuteSqlDataReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, bool allRows, bool isReadOnly, CancellationToken cancellationToken) { EnsureArg.IsNotNull(sqlCommand, nameof(sqlCommand)); EnsureArg.IsNotNull(readerToResult, nameof(readerToResult)); @@ -344,7 +343,6 @@ await ExecuteSql( /// SQL connection error. In the case if non-retriable exception or if the last retry failed tha same exception is thrown. /// /// Defines data type for the returned SQL rows. - /// Type used for the . /// SQL command to be executed. /// Translation delegate that translates the row returned by execution into the data type. /// Logger used on first try error or retry error. @@ -354,7 +352,7 @@ await ExecuteSql( /// A task representing the asynchronous operation that returns all the rows that result from execution. The rows are translated by delegate /// into data type. /// When executing this method, if exception is thrown that is not retriable or if last retry fails, then same exception is thrown by this method. - public async Task> ExecuteReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false) + public async Task> ExecuteReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false) { return await ExecuteSqlDataReaderAsync(sqlCommand, readerToResult, logger, logMessage, true, isReadOnly, cancellationToken); } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs index 72b116da30..4a7a1767da 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs @@ -52,7 +52,7 @@ internal class SqlServerFhirDataStore : IFhirDataStore, IProvideCapability private readonly IBundleOrchestrator _bundleOrchestrator; private readonly CoreFeatureConfiguration _coreFeatures; private readonly ISqlRetryService _sqlRetryService; - private readonly SqlStoreClient _sqlStoreClient; + private readonly SqlStoreClient _sqlStoreClient; private readonly SqlConnectionWrapperFactory _sqlConnectionWrapperFactory; private readonly ICompressedRawResourceConverter _compressedRawResourceConverter; private readonly ILogger _logger; @@ -76,14 +76,15 @@ public SqlServerFhirDataStore( SchemaInformation schemaInformation, IModelInfoProvider modelInfoProvider, RequestContextAccessor requestContextAccessor, - IImportErrorSerializer importErrorSerializer) + IImportErrorSerializer importErrorSerializer, + SqlStoreClient storeClient) { _model = EnsureArg.IsNotNull(model, nameof(model)); _searchParameterTypeMap = EnsureArg.IsNotNull(searchParameterTypeMap, nameof(searchParameterTypeMap)); _coreFeatures = EnsureArg.IsNotNull(coreFeatures?.Value, nameof(coreFeatures)); _bundleOrchestrator = EnsureArg.IsNotNull(bundleOrchestrator, nameof(bundleOrchestrator)); _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); - _sqlStoreClient = new SqlStoreClient(_sqlRetryService, logger); + _sqlStoreClient = EnsureArg.IsNotNull(storeClient, nameof(storeClient)); _sqlConnectionWrapperFactory = EnsureArg.IsNotNull(sqlConnectionWrapperFactory, nameof(sqlConnectionWrapperFactory)); _compressedRawResourceConverter = EnsureArg.IsNotNull(compressedRawResourceConverter, nameof(compressedRawResourceConverter)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); @@ -119,7 +120,7 @@ public SqlServerFhirDataStore( } } - internal SqlStoreClient StoreClient => _sqlStoreClient; + internal SqlStoreClient StoreClient => _sqlStoreClient; internal static TimeSpan MergeResourcesTransactionHeartbeatPeriod => TimeSpan.FromSeconds(10); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs index 309656f327..5d4daf17b3 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs @@ -25,14 +25,13 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage /// /// Lightweight SQL store client. /// - /// class used in logger - internal class SqlStoreClient + internal class SqlStoreClient { private readonly ISqlRetryService _sqlRetryService; - private readonly ILogger _logger; + private readonly ILogger _logger; private const string _invisibleResource = " "; - public SqlStoreClient(ISqlRetryService sqlRetryService, ILogger logger) + public SqlStoreClient(ISqlRetryService sqlRetryService, ILogger logger) { _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs new file mode 100644 index 0000000000..b8270b7ad7 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs @@ -0,0 +1,109 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.SqlServer.Features.Storage; +using Microsoft.Health.Fhir.Subscriptions.Models; +using Microsoft.Health.JobManagement; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs +{ + internal class EventProcessorWatchdog : Watchdog + { + private readonly SqlStoreClient _store; + private readonly ILogger _logger; + private readonly ISqlRetryService _sqlRetryService; + private readonly IQueueClient _queueClient; + private CancellationToken _cancellationToken; + + public EventProcessorWatchdog( + SqlStoreClient store, + ISqlRetryService sqlRetryService, + IQueueClient queueClient, + ILogger logger) + : base(sqlRetryService, logger) + { + _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); + _store = EnsureArg.IsNotNull(store, nameof(store)); + _logger = EnsureArg.IsNotNull(logger, nameof(logger)); + _logger = EnsureArg.IsNotNull(logger, nameof(logger)); + _queueClient = EnsureArg.IsNotNull(queueClient, nameof(queueClient)); + } + + internal string LastEventProcessedTransactionId => $"{Name}.{nameof(LastEventProcessedTransactionId)}"; + + internal async Task StartAsync(CancellationToken cancellationToken, double? periodSec = null, double? leasePeriodSec = null, double? retentionPeriodDays = null) + { + _cancellationToken = cancellationToken; + await InitLastProcessedTransactionId(); + await StartAsync(true, periodSec ?? 3600, leasePeriodSec ?? 2 * 3600, cancellationToken); + } + + protected override async Task ExecuteAsync() + { + _logger.LogInformation($"{Name}: starting..."); + var lastTranId = await GetLastTransactionId(); + var visibility = await _store.MergeResourcesGetTransactionVisibilityAsync(_cancellationToken); + + _logger.LogDebug($"{Name}: last cleaned up transaction={lastTranId} visibility={visibility}."); + + var transactionsToProcess = await _store.GetTransactionsAsync(lastTranId, visibility, _cancellationToken, DateTime.UtcNow.AddDays((-1) * 7)); + _logger.LogDebug($"{Name}: found transactions={transactionsToProcess.Count}."); + + if (transactionsToProcess.Count == 0) + { + _logger.LogDebug($"{Name}: completed. transactions=0."); + return; + } + + var transactionsToQueue = new List(); + var totalRows = 0; + foreach (var tran in transactionsToProcess.Where(x => x.VisibleDate.HasValue).OrderBy(x => x.TransactionId)) + { + var jobDefinition = new SubscriptionJobDefinition(Core.Features.Operations.JobType.SubscriptionsOrchestrator) + { + TransactionId = tran.TransactionId, + VisibleDate = tran.VisibleDate.Value, + }; + + transactionsToQueue.Add(jobDefinition); + } + + await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken: _cancellationToken, definitions: transactionsToQueue.ToArray()); + await UpdateLastEventProcessedTransactionId(transactionsToProcess.Max(x => x.TransactionId)); + + _logger.LogDebug($"{Name}: completed. transactions={transactionsToProcess.Count} removed rows={totalRows}"); + } + + private async Task GetLastTransactionId() + { + return await GetLongParameterByIdAsync(LastEventProcessedTransactionId, _cancellationToken); + } + + private async Task InitLastProcessedTransactionId() + { + using var cmd = new SqlCommand("INSERT INTO dbo.Parameters (Id, Bigint) SELECT @Id, 5105975696064002770"); // surrogate id for the past. does not matter. + cmd.Parameters.AddWithValue("@Id", LastEventProcessedTransactionId); + await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); + } + + private async Task UpdateLastEventProcessedTransactionId(long lastTranId) + { + using var cmd = new SqlCommand("UPDATE dbo.Parameters SET Bigint = @LastTranId WHERE Id = @Id"); + cmd.Parameters.AddWithValue("@Id", LastEventProcessedTransactionId); + cmd.Parameters.AddWithValue("@LastTranId", lastTranId); + await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs index 8e978eb98c..eddcded1d8 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs @@ -16,13 +16,13 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { internal class InvisibleHistoryCleanupWatchdog : Watchdog { - private readonly SqlStoreClient _store; + private readonly SqlStoreClient _store; private readonly ILogger _logger; private readonly ISqlRetryService _sqlRetryService; private CancellationToken _cancellationToken; private double _retentionPeriodDays = 7; - public InvisibleHistoryCleanupWatchdog(SqlStoreClient store, ISqlRetryService sqlRetryService, ILogger logger) + public InvisibleHistoryCleanupWatchdog(SqlStoreClient store, ISqlRetryService sqlRetryService, ILogger logger) : base(sqlRetryService, logger) { _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs index 85dc260a8d..63f3cfb639 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs @@ -22,17 +22,20 @@ internal class WatchdogsBackgroundService : BackgroundService, INotificationHand private readonly CleanupEventLogWatchdog _cleanupEventLogWatchdog; private readonly IScoped _transactionWatchdog; private readonly InvisibleHistoryCleanupWatchdog _invisibleHistoryCleanupWatchdog; + private readonly EventProcessorWatchdog _eventProcessorWatchdog; public WatchdogsBackgroundService( DefragWatchdog defragWatchdog, CleanupEventLogWatchdog cleanupEventLogWatchdog, IScopeProvider transactionWatchdog, - InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog) + InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog, + EventProcessorWatchdog eventProcessorWatchdog) { _defragWatchdog = EnsureArg.IsNotNull(defragWatchdog, nameof(defragWatchdog)); _cleanupEventLogWatchdog = EnsureArg.IsNotNull(cleanupEventLogWatchdog, nameof(cleanupEventLogWatchdog)); _transactionWatchdog = EnsureArg.IsNotNull(transactionWatchdog, nameof(transactionWatchdog)).Invoke(); _invisibleHistoryCleanupWatchdog = EnsureArg.IsNotNull(invisibleHistoryCleanupWatchdog, nameof(invisibleHistoryCleanupWatchdog)); + _eventProcessorWatchdog = EnsureArg.IsNotNull(eventProcessorWatchdog, nameof(eventProcessorWatchdog)); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -47,7 +50,8 @@ await Task.WhenAll( _defragWatchdog.StartAsync(stoppingToken), _cleanupEventLogWatchdog.StartAsync(stoppingToken), _transactionWatchdog.Value.StartAsync(stoppingToken), - _invisibleHistoryCleanupWatchdog.StartAsync(stoppingToken)); + _invisibleHistoryCleanupWatchdog.StartAsync(stoppingToken), + _eventProcessorWatchdog.StartAsync(stoppingToken)); } public Task Handle(StorageInitializedNotification notification, CancellationToken cancellationToken) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj index 062b3dd869..fffb1ebd3a 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj +++ b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj @@ -54,6 +54,7 @@ + diff --git a/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs index c706e47741..eb94941a91 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs @@ -162,7 +162,9 @@ public static IFhirServerBuilder AddSqlServer(this IFhirServerBuilder fhirServer .Singleton() .AsSelf(); - services.Add>().Singleton().AsSelf(); + services.Add() + .Singleton() + .AsSelf(); services.Add().Singleton().AsSelf(); @@ -173,6 +175,8 @@ public static IFhirServerBuilder AddSqlServer(this IFhirServerBuilder fhirServer services.Add().Singleton().AsSelf(); + services.Add().Singleton().AsSelf(); + services.RemoveServiceTypeExact>() // Mediatr registers handlers as Transient by default, this extension ensures these aren't still there, only needed when service != Transient .Add() .Singleton() diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs new file mode 100644 index 0000000000..fc13912923 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Health.Fhir.Subscriptions.Channels +{ + internal interface ISubscriptionChannel + { + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs new file mode 100644 index 0000000000..fa0c938cdc --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Health.Fhir.Subscriptions.Channels +{ + public class StorageChannel : ISubscriptionChannel + { + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj new file mode 100644 index 0000000000..586eb07eae --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj @@ -0,0 +1,12 @@ + + + + enable + enable + + + + + + + diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs new file mode 100644 index 0000000000..a5b9aaf389 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs @@ -0,0 +1,35 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Health.Fhir.Subscriptions.Models +{ + public class ChannelInfo + { + /// + /// Interval to send 'heartbeat' notification + /// + public TimeSpan HeartBeatPeriod { get; set; } + + /// + /// Timeout to attempt notification delivery + /// + public TimeSpan Timeout { get; set; } + + /// + /// Maximum number of triggering resources included in notification bundles + /// + public int MaxCount { get; set; } + + public SubscriptionChannelType ChannelType { get; set; } + + public SubscriptionContentType ContentType { get; set; } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionChannelType.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionChannelType.cs new file mode 100644 index 0000000000..5b67f98271 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionChannelType.cs @@ -0,0 +1,21 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Subscriptions.Models +{ + public enum SubscriptionChannelType + { + None = 0, + RestHook = 1, + WebSocket = 2, + Email = 3, + FhirMessaging = 4, + + // Custom Channels + EventGrid = 5, + Storage = 6, + DatalakeContract = 7, + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionContentType.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionContentType.cs new file mode 100644 index 0000000000..1eee162f7d --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionContentType.cs @@ -0,0 +1,14 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Subscriptions.Models +{ + public enum SubscriptionContentType + { + Empty, + IdOnly, + FullResource, + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs new file mode 100644 index 0000000000..31d2c782ec --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using EnsureThat; + +namespace Microsoft.Health.Fhir.Subscriptions.Models +{ + public class SubscriptionInfo + { + public SubscriptionInfo(string filterCriteria) + { + FilterCriteria = EnsureArg.IsNotNullOrEmpty(filterCriteria, nameof(filterCriteria)); + } + + public string FilterCriteria { get; set; } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs new file mode 100644 index 0000000000..a5202183e6 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.JobManagement; +using Newtonsoft.Json; + +namespace Microsoft.Health.Fhir.Subscriptions.Models +{ + public class SubscriptionJobDefinition : IJobData + { + public SubscriptionJobDefinition(JobType jobType) + { + TypeId = (int)jobType; + } + + [JsonProperty(JobRecordProperties.TypeId)] + public int TypeId { get; set; } + + public long TransactionId { get; set; } + + public DateTime VisibleDate { get; set; } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs new file mode 100644 index 0000000000..b1b37e5eaf --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.JobManagement; + +namespace Microsoft.Health.Fhir.Subscriptions.Operations +{ + [JobTypeId((int)JobType.SubscriptionsProcessing)] + public class SubscriptionProcessingJob : IJob + { + public Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) + { + return Task.FromResult("Done!"); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs new file mode 100644 index 0000000000..c4809bc6eb --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs @@ -0,0 +1,51 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Subscriptions.Models; +using Microsoft.Health.JobManagement; + +namespace Microsoft.Health.Fhir.Subscriptions.Operations +{ + [JobTypeId((int)JobType.SubscriptionsOrchestrator)] + public class SubscriptionsOrchestratorJob : IJob + { + private readonly IQueueClient _queueClient; + private readonly Func> _searchService; + private const string OperationCompleted = "Completed"; + + public SubscriptionsOrchestratorJob( + IQueueClient queueClient, + Func> searchService) + { + EnsureArg.IsNotNull(queueClient, nameof(queueClient)); + EnsureArg.IsNotNull(searchService, nameof(searchService)); + + _queueClient = queueClient; + _searchService = searchService; + } + + public Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) + { + EnsureArg.IsNotNull(jobInfo, nameof(jobInfo)); + + SubscriptionJobDefinition definition = jobInfo.DeserializeDefinition(); + + // Get and evaluate the active subscriptions ... + + // await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken, jobInfo.GroupId, definitions: processingDefinition); + + return Task.FromResult(OperationCompleted); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs new file mode 100644 index 0000000000..69cc743e18 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.JobManagement; + +namespace Microsoft.Health.Fhir.Subscriptions.Registration +{ + public class SubscriptionsModule : IStartupModule + { + public void Load(IServiceCollection services) + { + services.TypesInSameAssemblyAs() + .AssignableTo() + .Transient() + .AsSelf() + .AsService(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/SubscriptionManager.cs new file mode 100644 index 0000000000..b7c03130e0 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/SubscriptionManager.cs @@ -0,0 +1,22 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions +{ + public class SubscriptionManager + { + public Task> GetActiveSubscriptionsAsync() + { + throw new NotImplementedException(); + } + } +} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs index 0428f93648..9655ceaa84 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs @@ -140,7 +140,7 @@ public async Task GivenChangeCaptureEnabledAndNoVersionPolicy_AfterUpdating_Invi using var cts = new CancellationTokenSource(); cts.CancelAfter(TimeSpan.FromSeconds(60)); - var storeClient = new SqlStoreClient(_fixture.SqlRetryService, NullLogger.Instance); + var storeClient = new SqlStoreClient(_fixture.SqlRetryService, NullLogger.Instance); var wd = new InvisibleHistoryCleanupWatchdog(storeClient, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)); await wd.StartAsync(cts.Token, 1, 2, 2.0 / 24 / 3600); // retention 2 seconds var startTime = DateTime.UtcNow; @@ -188,7 +188,7 @@ public async Task GivenChangeCaptureEnabledAndNoVersionPolicy_AfterHardDeleting_ using var cts = new CancellationTokenSource(); cts.CancelAfter(TimeSpan.FromSeconds(60)); - var storeClient = new SqlStoreClient(_fixture.SqlRetryService, NullLogger.Instance); + var storeClient = new SqlStoreClient(_fixture.SqlRetryService, NullLogger.Instance); var wd = new InvisibleHistoryCleanupWatchdog(storeClient, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)); await wd.StartAsync(cts.Token, 1, 2, 2.0 / 24 / 3600); // retention 2 seconds var startTime = DateTime.UtcNow; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs index 19e1d124a6..1178d612b1 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs @@ -380,7 +380,7 @@ private async Task SingleConnectionRetryTest(Func testStor using var sqlCommand = new SqlCommand(); sqlCommand.CommandText = $"dbo.{storedProcedureName}"; - var result = await sqlRetryService.ExecuteReaderAsync( + var result = await sqlRetryService.ExecuteReaderAsync( sqlCommand, testConnectionInitializationFailure ? ReaderToResult : ReaderToResultAndKillConnection, logger, @@ -420,7 +420,7 @@ private async Task AllConnectionRetriesTest(Func testStore try { _output.WriteLine($"{DateTime.Now:O}: Start executing ExecuteSqlDataReader."); - await sqlRetryService.ExecuteReaderAsync( + await sqlRetryService.ExecuteReaderAsync( sqlCommand, testConnectionInitializationFailure ? ReaderToResult : ReaderToResultAndKillConnection, logger, From 1e7eb6dc4c1150b14c581d03a1523ee93e4b18ba Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Mon, 15 Apr 2024 20:17:58 -0700 Subject: [PATCH 03/47] Fixes wiring up of Transaction Watchdog => Orchestrator --- .../Features/Persistence/ResourceKey.cs | 6 +++ .../appsettings.json | 4 ++ .../Storage/SqlServerFhirDataStore.cs | 5 ++- .../Watchdogs/EventProcessorWatchdog.cs | 13 +++--- .../Features/Watchdogs/Watchdog.cs | 7 ++- ...Microsoft.Health.Fhir.Subscriptions.csproj | 1 - .../Models/SubscriptionInfo.cs | 5 ++- .../Models/SubscriptionJobDefinition.cs | 16 +++++++ .../Operations/SubscriptionProcessingJob.cs | 2 + .../SubscriptionsOrchestratorJob.cs | 44 +++++++++++++++---- .../Persistence/ISubscriptionManager.cs | 14 ++++++ .../ITransactionDataStore.cs} | 11 ++--- .../Persistence/SubscriptionManager.cs | 34 ++++++++++++++ .../Registration/SubscriptionsModule.cs | 15 ++++++- .../JobHosting.cs | 2 +- .../SqlServerFhirStorageTestsFixture.cs | 3 +- 16 files changed, 151 insertions(+), 31 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs rename src/Microsoft.Health.Fhir.Subscriptions/{SubscriptionManager.cs => Persistence/ITransactionDataStore.cs} (62%) create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs diff --git a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs index 9f27f96022..08d83caa78 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs @@ -7,6 +7,7 @@ using System.Text; using EnsureThat; using Microsoft.Health.Fhir.Core.Models; +using Newtonsoft.Json; namespace Microsoft.Health.Fhir.Core.Features.Persistence { @@ -23,6 +24,11 @@ public ResourceKey(string resourceType, string id, string versionId = null) ResourceType = resourceType; } + [JsonConstructor] + protected ResourceKey() + { + } + public string Id { get; } public string VersionId { get; } diff --git a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json index 2b7fb43427..c91847252a 100644 --- a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json +++ b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json @@ -59,6 +59,10 @@ { "Queue": "BulkDelete", "UpdateProgressOnHeartbeat": false + }, + { + "Queue": "Subscriptions", + "UpdateProgressOnHeartbeat": false } ], "Export": { diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs index 4a7a1767da..20854aec0e 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs @@ -29,6 +29,7 @@ using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model; using Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration; using Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration.Merge; +using Microsoft.Health.Fhir.Subscriptions.Persistence; using Microsoft.Health.Fhir.ValueSets; using Microsoft.Health.SqlServer.Features.Client; using Microsoft.Health.SqlServer.Features.Schema; @@ -41,7 +42,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage /// /// A SQL Server-backed . /// - internal class SqlServerFhirDataStore : IFhirDataStore, IProvideCapability + internal class SqlServerFhirDataStore : IFhirDataStore, IProvideCapability, ITransactionDataStore { private const string InitialVersion = "1"; @@ -945,7 +946,7 @@ public void Build(ICapabilityStatementBuilder builder) } } - internal async Task> GetResourcesByTransactionIdAsync(long transactionId, CancellationToken cancellationToken) + public async Task> GetResourcesByTransactionIdAsync(long transactionId, CancellationToken cancellationToken) { return await _sqlStoreClient.GetResourcesByTransactionIdAsync(transactionId, _compressedRawResourceConverter.ReadCompressedRawResource, _model.GetResourceTypeName, cancellationToken); } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs index b8270b7ad7..31660110de 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs @@ -47,7 +47,7 @@ internal async Task StartAsync(CancellationToken cancellationToken, double? peri { _cancellationToken = cancellationToken; await InitLastProcessedTransactionId(); - await StartAsync(true, periodSec ?? 3600, leasePeriodSec ?? 2 * 3600, cancellationToken); + await StartAsync(true, periodSec ?? 3, leasePeriodSec ?? 20, cancellationToken); } protected override async Task ExecuteAsync() @@ -56,19 +56,20 @@ protected override async Task ExecuteAsync() var lastTranId = await GetLastTransactionId(); var visibility = await _store.MergeResourcesGetTransactionVisibilityAsync(_cancellationToken); - _logger.LogDebug($"{Name}: last cleaned up transaction={lastTranId} visibility={visibility}."); + _logger.LogInformation($"{Name}: last transaction={lastTranId} visibility={visibility}."); - var transactionsToProcess = await _store.GetTransactionsAsync(lastTranId, visibility, _cancellationToken, DateTime.UtcNow.AddDays((-1) * 7)); + var transactionsToProcess = await _store.GetTransactionsAsync(lastTranId, visibility, _cancellationToken); _logger.LogDebug($"{Name}: found transactions={transactionsToProcess.Count}."); if (transactionsToProcess.Count == 0) { - _logger.LogDebug($"{Name}: completed. transactions=0."); + await UpdateLastEventProcessedTransactionId(visibility); + _logger.LogInformation($"{Name}: completed. transactions=0."); return; } var transactionsToQueue = new List(); - var totalRows = 0; + foreach (var tran in transactionsToProcess.Where(x => x.VisibleDate.HasValue).OrderBy(x => x.TransactionId)) { var jobDefinition = new SubscriptionJobDefinition(Core.Features.Operations.JobType.SubscriptionsOrchestrator) @@ -83,7 +84,7 @@ protected override async Task ExecuteAsync() await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken: _cancellationToken, definitions: transactionsToQueue.ToArray()); await UpdateLastEventProcessedTransactionId(transactionsToProcess.Max(x => x.TransactionId)); - _logger.LogDebug($"{Name}: completed. transactions={transactionsToProcess.Count} removed rows={totalRows}"); + _logger.LogInformation($"{Name}: completed. transactions={transactionsToProcess.Count}"); } private async Task GetLastTransactionId() diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs index d68b671a44..95d2917234 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs @@ -51,8 +51,11 @@ protected internal async Task StartAsync(bool allowRebalance, double periodSec, { _logger.LogInformation($"{Name}.StartAsync: starting..."); await InitParamsAsync(periodSec, leasePeriodSec); - await StartAsync(_periodSec, cancellationToken); - await _watchdogLease.StartAsync(allowRebalance, _leasePeriodSec, cancellationToken); + + await Task.WhenAll( + StartAsync(_periodSec, cancellationToken), + _watchdogLease.StartAsync(allowRebalance, _leasePeriodSec, cancellationToken)); + _logger.LogInformation($"{Name}.StartAsync: completed."); } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj index 586eb07eae..a90c13a333 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj +++ b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj @@ -2,7 +2,6 @@ enable - enable diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs index 31d2c782ec..b2d2e39591 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs @@ -14,11 +14,14 @@ namespace Microsoft.Health.Fhir.Subscriptions.Models { public class SubscriptionInfo { - public SubscriptionInfo(string filterCriteria) + public SubscriptionInfo(string filterCriteria, ChannelInfo channel) { FilterCriteria = EnsureArg.IsNotNullOrEmpty(filterCriteria, nameof(filterCriteria)); + Channel = EnsureArg.IsNotNull(channel, nameof(channel)); } public string FilterCriteria { get; set; } + + public ChannelInfo Channel { get; set; } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs index a5202183e6..7ef7fade06 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs @@ -5,10 +5,12 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Text; using System.Threading.Tasks; using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.JobManagement; using Newtonsoft.Json; @@ -21,11 +23,25 @@ public SubscriptionJobDefinition(JobType jobType) TypeId = (int)jobType; } + [JsonConstructor] + protected SubscriptionJobDefinition() + { + } + [JsonProperty(JobRecordProperties.TypeId)] public int TypeId { get; set; } + [JsonProperty("transactionId")] public long TransactionId { get; set; } + [JsonProperty("visibleDate")] public DateTime VisibleDate { get; set; } + + [JsonProperty("resourceReferences")] + [SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "JSON Poco")] + public IList ResourceReferences { get; set; } + + [JsonProperty("channel")] + public ChannelInfo Channel { get; set; } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs index b1b37e5eaf..d62f584a63 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -18,6 +18,8 @@ public class SubscriptionProcessingJob : IJob { public Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) { + // TODO: Write resource to channel + return Task.FromResult("Done!"); } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs index c4809bc6eb..9c2efab149 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs @@ -10,9 +10,12 @@ using System.Threading.Tasks; using EnsureThat; using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Subscriptions.Models; +using Microsoft.Health.Fhir.Subscriptions.Persistence; using Microsoft.Health.JobManagement; namespace Microsoft.Health.Fhir.Subscriptions.Operations @@ -21,31 +24,56 @@ namespace Microsoft.Health.Fhir.Subscriptions.Operations public class SubscriptionsOrchestratorJob : IJob { private readonly IQueueClient _queueClient; - private readonly Func> _searchService; + private readonly ITransactionDataStore _transactionDataStore; + private readonly ISubscriptionManager _subscriptionManager; private const string OperationCompleted = "Completed"; public SubscriptionsOrchestratorJob( IQueueClient queueClient, - Func> searchService) + ITransactionDataStore transactionDataStore, + ISubscriptionManager subscriptionManager) { EnsureArg.IsNotNull(queueClient, nameof(queueClient)); - EnsureArg.IsNotNull(searchService, nameof(searchService)); + EnsureArg.IsNotNull(transactionDataStore, nameof(transactionDataStore)); + EnsureArg.IsNotNull(subscriptionManager, nameof(subscriptionManager)); _queueClient = queueClient; - _searchService = searchService; + _transactionDataStore = transactionDataStore; + _subscriptionManager = subscriptionManager; } - public Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) + public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) { EnsureArg.IsNotNull(jobInfo, nameof(jobInfo)); SubscriptionJobDefinition definition = jobInfo.DeserializeDefinition(); + var resources = await _transactionDataStore.GetResourcesByTransactionIdAsync(definition.TransactionId, cancellationToken); - // Get and evaluate the active subscriptions ... + var processingDefinition = new List(); - // await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken, jobInfo.GroupId, definitions: processingDefinition); + foreach (var sub in await _subscriptionManager.GetActiveSubscriptionsAsync(cancellationToken)) + { + var chunk = resources + //// TODO: .Where(r => sub.FilterCriteria does something??); + .Chunk(sub.Channel.MaxCount); - return Task.FromResult(OperationCompleted); + foreach (var batch in chunk) + { + var cloneDefinition = jobInfo.DeserializeDefinition(); + cloneDefinition.TypeId = (int)JobType.SubscriptionsProcessing; + cloneDefinition.ResourceReferences = batch.Select(r => new ResourceKey(r.ResourceTypeName, r.ResourceId, r.Version)).ToList(); + cloneDefinition.Channel = sub.Channel; + + processingDefinition.Add(cloneDefinition); + } + } + + if (processingDefinition.Count > 0) + { + await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken, jobInfo.GroupId, definitions: processingDefinition.ToArray()); + } + + return OperationCompleted; } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs new file mode 100644 index 0000000000..a5f2738457 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs @@ -0,0 +1,14 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Persistence +{ + public interface ISubscriptionManager + { + Task> GetActiveSubscriptionsAsync(CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs similarity index 62% rename from src/Microsoft.Health.Fhir.Subscriptions/SubscriptionManager.cs rename to src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs index b7c03130e0..6a1ca37224 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/SubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs @@ -8,15 +8,12 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -using Microsoft.Health.Fhir.Subscriptions.Models; +using Microsoft.Health.Fhir.Core.Features.Persistence; -namespace Microsoft.Health.Fhir.Subscriptions +namespace Microsoft.Health.Fhir.Subscriptions.Persistence { - public class SubscriptionManager + public interface ITransactionDataStore { - public Task> GetActiveSubscriptionsAsync() - { - throw new NotImplementedException(); - } + Task> GetResourcesByTransactionIdAsync(long transactionId, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs new file mode 100644 index 0000000000..b53ef16587 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Persistence +{ + public class SubscriptionManager : ISubscriptionManager + { + public Task> GetActiveSubscriptionsAsync(CancellationToken cancellationToken) + { + IReadOnlyCollection list = new List + { + new SubscriptionInfo( + "Resource", + new ChannelInfo + { + ChannelType = SubscriptionChannelType.Storage, + ContentType = SubscriptionContentType.FullResource, + MaxCount = 100, + }), + }; + + return Task.FromResult(list); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs index 69cc743e18..bbf12b8002 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs @@ -9,7 +9,9 @@ using System.Text; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Subscriptions.Persistence; using Microsoft.Health.JobManagement; namespace Microsoft.Health.Fhir.Subscriptions.Registration @@ -18,11 +20,20 @@ public class SubscriptionsModule : IStartupModule { public void Load(IServiceCollection services) { - services.TypesInSameAssemblyAs() + IEnumerable jobs = services.TypesInSameAssemblyAs() .AssignableTo() .Transient() + .AsSelf(); + + foreach (TypeRegistrationBuilder job in jobs) + { + job.AsDelegate>(); + } + + services.Add() + .Singleton() .AsSelf() - .AsService(); + .AsImplementedInterfaces(); } } } diff --git a/src/Microsoft.Health.TaskManagement/JobHosting.cs b/src/Microsoft.Health.TaskManagement/JobHosting.cs index 7bca9a274f..6c6707dfd8 100644 --- a/src/Microsoft.Health.TaskManagement/JobHosting.cs +++ b/src/Microsoft.Health.TaskManagement/JobHosting.cs @@ -61,7 +61,7 @@ public async Task ExecuteAsync(byte queueType, short runningJobCount, string wor { try { - _logger.LogInformation("Dequeuing next job."); + _logger.LogInformation("Dequeuing next job on {QueueType}.", queueType); if (checkTimeoutJobStopwatch.Elapsed.TotalSeconds > 600) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs index ad53e6ecef..2610fbf534 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs @@ -244,7 +244,8 @@ public async Task InitializeAsync() SchemaInformation, ModelInfoProvider.Instance, _fhirRequestContextAccessor, - importErrorSerializer); + importErrorSerializer, + new SqlStoreClient(SqlRetryService, NullLogger.Instance)); _fhirOperationDataStore = new SqlServerFhirOperationDataStore(SqlConnectionWrapperFactory, queueClient, NullLogger.Instance, NullLoggerFactory.Instance); From 2c47f3543201d744c2275a93c76f7cefdf27583f Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Tue, 16 Apr 2024 09:03:35 -0700 Subject: [PATCH 04/47] Adding code for writing to storage --- R4.slnf | 3 +- ...og.cs => SubscriptionProcessorWatchdog.cs} | 8 +-- .../Watchdogs/WatchdogsBackgroundService.cs | 32 ++++++--- ...rBuilderSqlServerRegistrationExtensions.cs | 2 +- ...Microsoft.Health.Fhir.Subscriptions.csproj | 1 + .../Operations/SubscriptionProcessingJob.cs | 68 ++++++++++++++++++- 6 files changed, 97 insertions(+), 17 deletions(-) rename src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/{EventProcessorWatchdog.cs => SubscriptionProcessorWatchdog.cs} (94%) diff --git a/R4.slnf b/R4.slnf index f3207945d8..7adecaed1e 100644 --- a/R4.slnf +++ b/R4.slnf @@ -29,6 +29,7 @@ "src\\Microsoft.Health.Fhir.Shared.Web\\Microsoft.Health.Fhir.Shared.Web.shproj", "src\\Microsoft.Health.Fhir.SqlServer.UnitTests\\Microsoft.Health.Fhir.SqlServer.UnitTests.csproj", "src\\Microsoft.Health.Fhir.SqlServer\\Microsoft.Health.Fhir.SqlServer.csproj", + "src\\Microsoft.Health.Fhir.Subscriptions\\Microsoft.Health.Fhir.Subscriptions.csproj", "src\\Microsoft.Health.Fhir.Tests.Common\\Microsoft.Health.Fhir.Tests.Common.csproj", "src\\Microsoft.Health.TaskManagement\\Microsoft.Health.TaskManagement.csproj", "src\\Microsoft.Health.TaskManagement.UnitTests\\Microsoft.Health.TaskManagement.UnitTests.csproj", @@ -43,4 +44,4 @@ "test\\Microsoft.Health.Fhir.Shared.Tests.Integration\\Microsoft.Health.Fhir.Shared.Tests.Integration.shproj" ] } -} +} \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs similarity index 94% rename from src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs rename to src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs index 31660110de..5f31720eda 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs @@ -19,19 +19,19 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - internal class EventProcessorWatchdog : Watchdog + internal class SubscriptionProcessorWatchdog : Watchdog { private readonly SqlStoreClient _store; - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly ISqlRetryService _sqlRetryService; private readonly IQueueClient _queueClient; private CancellationToken _cancellationToken; - public EventProcessorWatchdog( + public SubscriptionProcessorWatchdog( SqlStoreClient store, ISqlRetryService sqlRetryService, IQueueClient queueClient, - ILogger logger) + ILogger logger) : base(sqlRetryService, logger) { _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs index 63f3cfb639..bafa0b1c03 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs @@ -4,12 +4,14 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using EnsureThat; using MediatR; using Microsoft.Extensions.Hosting; using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Messages.Storage; @@ -22,14 +24,14 @@ internal class WatchdogsBackgroundService : BackgroundService, INotificationHand private readonly CleanupEventLogWatchdog _cleanupEventLogWatchdog; private readonly IScoped _transactionWatchdog; private readonly InvisibleHistoryCleanupWatchdog _invisibleHistoryCleanupWatchdog; - private readonly EventProcessorWatchdog _eventProcessorWatchdog; + private readonly SubscriptionProcessorWatchdog _eventProcessorWatchdog; public WatchdogsBackgroundService( DefragWatchdog defragWatchdog, CleanupEventLogWatchdog cleanupEventLogWatchdog, IScopeProvider transactionWatchdog, InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog, - EventProcessorWatchdog eventProcessorWatchdog) + SubscriptionProcessorWatchdog eventProcessorWatchdog) { _defragWatchdog = EnsureArg.IsNotNull(defragWatchdog, nameof(defragWatchdog)); _cleanupEventLogWatchdog = EnsureArg.IsNotNull(cleanupEventLogWatchdog, nameof(cleanupEventLogWatchdog)); @@ -46,12 +48,26 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); } - await Task.WhenAll( - _defragWatchdog.StartAsync(stoppingToken), - _cleanupEventLogWatchdog.StartAsync(stoppingToken), - _transactionWatchdog.Value.StartAsync(stoppingToken), - _invisibleHistoryCleanupWatchdog.StartAsync(stoppingToken), - _eventProcessorWatchdog.StartAsync(stoppingToken)); + using var continuationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); + + var tasks = new List + { + _defragWatchdog.StartAsync(continuationTokenSource.Token), + _cleanupEventLogWatchdog.StartAsync(continuationTokenSource.Token), + _transactionWatchdog.Value.StartAsync(continuationTokenSource.Token), + _invisibleHistoryCleanupWatchdog.StartAsync(continuationTokenSource.Token), + _eventProcessorWatchdog.StartAsync(continuationTokenSource.Token), + }; + + await Task.WhenAny(tasks); + + if (!stoppingToken.IsCancellationRequested) + { + // If any of the watchdogs fail, cancel all the other watchdogs + await continuationTokenSource.CancelAsync(); + } + + await Task.WhenAll(tasks); } public Task Handle(StorageInitializedNotification notification, CancellationToken cancellationToken) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs index eb94941a91..c980fdde97 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs @@ -175,7 +175,7 @@ public static IFhirServerBuilder AddSqlServer(this IFhirServerBuilder fhirServer services.Add().Singleton().AsSelf(); - services.Add().Singleton().AsSelf(); + services.Add().Singleton().AsSelf(); services.RemoveServiceTypeExact>() // Mediatr registers handlers as Transient by default, this extension ensures these aren't still there, only needed when service != Transient .Add() diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj index a90c13a333..50f7da1e86 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj +++ b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj @@ -5,6 +5,7 @@ + diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs index d62f584a63..8600df54f4 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -6,9 +6,16 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net; using System.Text; using System.Threading.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Extensions.Logging; using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Operations.Export; +using Microsoft.Health.Fhir.Core.Features.Operations.Export.ExportDestinationClient; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Subscriptions.Models; using Microsoft.Health.JobManagement; namespace Microsoft.Health.Fhir.Subscriptions.Operations @@ -16,11 +23,66 @@ namespace Microsoft.Health.Fhir.Subscriptions.Operations [JobTypeId((int)JobType.SubscriptionsProcessing)] public class SubscriptionProcessingJob : IJob { - public Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) + private readonly IResourceToByteArraySerializer _resourceToByteArraySerializer; + private readonly IExportDestinationClient _exportDestinationClient; + private readonly IResourceDeserializer _resourceDeserializer; + private readonly IFhirDataStore _dataStore; + private readonly ILogger _logger; + + public SubscriptionProcessingJob( + IResourceToByteArraySerializer resourceToByteArraySerializer, + IExportDestinationClient exportDestinationClient, + IResourceDeserializer resourceDeserializer, + IFhirDataStore dataStore, + ILogger logger) { - // TODO: Write resource to channel + _resourceToByteArraySerializer = resourceToByteArraySerializer; + _exportDestinationClient = exportDestinationClient; + _resourceDeserializer = resourceDeserializer; + _dataStore = dataStore; + _logger = logger; + } + + public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) + { + SubscriptionJobDefinition definition = jobInfo.DeserializeDefinition(); + + if (definition.Channel == null) + { + return HttpStatusCode.BadRequest.ToString(); + } + + if (definition.Channel.ChannelType == SubscriptionChannelType.Storage) + { + try + { + await _exportDestinationClient.ConnectAsync(cancellationToken, "SubscriptionContainer"); + + foreach (var resourceKey in definition.ResourceReferences) + { + var resource = await _dataStore.GetAsync(resourceKey, cancellationToken); + + string fileName = $"{resource.ResourceTypeName}/{resource.ResourceId}/_history/{resource.Version}.json"; + + _exportDestinationClient.WriteFilePart( + fileName, + resource.RawResource.Data); + + _exportDestinationClient.CommitFile(fileName); + } + } + catch (Exception ex) + { + _logger.LogJobError(jobInfo, ex.ToString()); + return HttpStatusCode.InternalServerError.ToString(); + } + } + else + { + return HttpStatusCode.BadRequest.ToString(); + } - return Task.FromResult("Done!"); + return HttpStatusCode.OK.ToString(); } } } From 85f8ba0ddd8bc1c3c34043d3800c0d75efdf5810 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Tue, 16 Apr 2024 10:02:46 -0700 Subject: [PATCH 05/47] Allow resourceKey to deserialize --- .../Features/Persistence/ResourceKey.cs | 6 +++--- .../Operations/SubscriptionProcessingJob.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs index 08d83caa78..b5316f471a 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs @@ -29,11 +29,11 @@ protected ResourceKey() { } - public string Id { get; } + public string Id { get; protected set; } - public string VersionId { get; } + public string VersionId { get; protected set; } - public string ResourceType { get; } + public string ResourceType { get; protected set; } public bool Equals(ResourceKey other) { diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs index 8600df54f4..40d582d7aa 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -56,7 +56,7 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel { try { - await _exportDestinationClient.ConnectAsync(cancellationToken, "SubscriptionContainer"); + await _exportDestinationClient.ConnectAsync(cancellationToken, "fhirsync"); foreach (var resourceKey in definition.ResourceReferences) { From 70578fdd52bb6fb1c67fbba2590cb8c7fa936028 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Tue, 16 Apr 2024 13:00:00 -0700 Subject: [PATCH 06/47] Implement basic subscription filtering --- .../Features/Persistence/ResourceKey.cs | 3 + .../Models/ChannelInfo.cs | 3 + .../Models/SubscriptionInfo.cs | 2 +- .../Operations/SubscriptionProcessingJob.cs | 4 +- .../SubscriptionsOrchestratorJob.cs | 58 ++++++++++++++++++- .../Persistence/SubscriptionManager.cs | 20 ++++++- 6 files changed, 83 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs index b5316f471a..b568089315 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs @@ -29,10 +29,13 @@ protected ResourceKey() { } + [JsonProperty("id")] public string Id { get; protected set; } + [JsonProperty("versionId")] public string VersionId { get; protected set; } + [JsonProperty("resourceType")] public string ResourceType { get; protected set; } public bool Equals(ResourceKey other) diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs index a5b9aaf389..5d99a57c2e 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs @@ -31,5 +31,8 @@ public class ChannelInfo public SubscriptionChannelType ChannelType { get; set; } public SubscriptionContentType ContentType { get; set; } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "Json Poco")] + public IDictionary Properties { get; set; } = new Dictionary(); } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs index b2d2e39591..038865d938 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs @@ -16,7 +16,7 @@ public class SubscriptionInfo { public SubscriptionInfo(string filterCriteria, ChannelInfo channel) { - FilterCriteria = EnsureArg.IsNotNullOrEmpty(filterCriteria, nameof(filterCriteria)); + FilterCriteria = filterCriteria; Channel = EnsureArg.IsNotNull(channel, nameof(channel)); } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs index 40d582d7aa..277113f818 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -56,13 +56,13 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel { try { - await _exportDestinationClient.ConnectAsync(cancellationToken, "fhirsync"); + await _exportDestinationClient.ConnectAsync(cancellationToken, definition.Channel.Properties["container"]); foreach (var resourceKey in definition.ResourceReferences) { var resource = await _dataStore.GetAsync(resourceKey, cancellationToken); - string fileName = $"{resource.ResourceTypeName}/{resource.ResourceId}/_history/{resource.Version}.json"; + string fileName = $"{resourceKey}.json"; _exportDestinationClient.WriteFilePart( fileName, diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs index 9c2efab149..9cc1fd6dc5 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs @@ -5,14 +5,17 @@ using System; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.Linq; using System.Text; using System.Threading.Tasks; using EnsureThat; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Routing; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Subscriptions.Models; using Microsoft.Health.Fhir.Subscriptions.Persistence; @@ -25,12 +28,16 @@ public class SubscriptionsOrchestratorJob : IJob { private readonly IQueueClient _queueClient; private readonly ITransactionDataStore _transactionDataStore; + private readonly ISearchService _searchService; + private readonly IQueryStringParser _queryStringParser; private readonly ISubscriptionManager _subscriptionManager; private const string OperationCompleted = "Completed"; public SubscriptionsOrchestratorJob( IQueueClient queueClient, ITransactionDataStore transactionDataStore, + ISearchService searchService, + IQueryStringParser queryStringParser, ISubscriptionManager subscriptionManager) { EnsureArg.IsNotNull(queueClient, nameof(queueClient)); @@ -39,6 +46,8 @@ public SubscriptionsOrchestratorJob( _queueClient = queueClient; _transactionDataStore = transactionDataStore; + _searchService = searchService; + _queryStringParser = queryStringParser; _subscriptionManager = subscriptionManager; } @@ -48,20 +57,63 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel SubscriptionJobDefinition definition = jobInfo.DeserializeDefinition(); var resources = await _transactionDataStore.GetResourcesByTransactionIdAsync(definition.TransactionId, cancellationToken); + var resourceKeys = resources.Select(r => new ResourceKey(r.ResourceTypeName, r.ResourceId, r.Version)).ToList().AsReadOnly(); var processingDefinition = new List(); foreach (var sub in await _subscriptionManager.GetActiveSubscriptionsAsync(cancellationToken)) { - var chunk = resources - //// TODO: .Where(r => sub.FilterCriteria does something??); + var channelResources = new List(); + + if (!string.IsNullOrEmpty(sub.FilterCriteria)) + { + var criteriaSegments = sub.FilterCriteria.Split('?'); + + List> query = new List>(); + + if (criteriaSegments.Length > 1) + { + query = _queryStringParser.Parse(criteriaSegments[1]) + .Select(x => new Tuple(x.Key, x.Value)) + .ToList(); + } + + var limitIds = string.Join(",", resourceKeys.Select(x => x.Id)); + var idParam = query.Where(x => x.Item1 == KnownQueryParameterNames.Id).FirstOrDefault(); + if (idParam != null) + { + query.Remove(idParam); + limitIds += "," + idParam.Item2; + } + + query.Add(new Tuple(KnownQueryParameterNames.Id, limitIds)); + + var results = await _searchService.SearchAsync(criteriaSegments[0], new ReadOnlyCollection>(query), cancellationToken, true, ResourceVersionType.Latest, onlyIds: true); + + channelResources.AddRange( + results.Results + .Where(x => x.SearchEntryMode == ValueSets.SearchEntryMode.Match + || x.SearchEntryMode == ValueSets.SearchEntryMode.Include) + .Select(x => x.Resource.ToResourceKey())); + } + else + { + channelResources.AddRange(resourceKeys); + } + + if (channelResources.Count == 0) + { + continue; + } + + var chunk = resourceKeys .Chunk(sub.Channel.MaxCount); foreach (var batch in chunk) { var cloneDefinition = jobInfo.DeserializeDefinition(); cloneDefinition.TypeId = (int)JobType.SubscriptionsProcessing; - cloneDefinition.ResourceReferences = batch.Select(r => new ResourceKey(r.ResourceTypeName, r.ResourceId, r.Version)).ToList(); + cloneDefinition.ResourceReferences = batch.ToList(); cloneDefinition.Channel = sub.Channel; processingDefinition.Add(cloneDefinition); diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs index b53ef16587..be8078dc2a 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs @@ -18,13 +18,31 @@ public Task> GetActiveSubscriptionsAsync(C { IReadOnlyCollection list = new List { + // "reason": "Alert on Diabetes with Complications Diagnosis", + // "criteria": "Condition?code=http://hl7.org/fhir/sid/icd-10|E11.6", new SubscriptionInfo( - "Resource", + null, new ChannelInfo { ChannelType = SubscriptionChannelType.Storage, ContentType = SubscriptionContentType.FullResource, MaxCount = 100, + Properties = new Dictionary + { + { "container", "sync-all" }, + }, + }), + new SubscriptionInfo( + "Patient", + new ChannelInfo + { + ChannelType = SubscriptionChannelType.Storage, + ContentType = SubscriptionContentType.FullResource, + MaxCount = 100, + Properties = new Dictionary + { + { "container", "sync-patient" }, + }, }), }; From 82f5c9de2030c2bb3cf2149bdd925af46d4edc28 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Tue, 16 Apr 2024 15:02:08 -0700 Subject: [PATCH 07/47] Implements Channel Interface --- .../Configs/CoreFeatureConfiguration.cs | 5 ++ .../appsettings.json | 1 + .../SubscriptionProcessorWatchdog.cs | 30 +++++++---- .../Watchdogs/WatchdogsBackgroundService.cs | 6 +-- .../Channels/ChannelTypeAttribute.cs | 25 +++++++++ .../Channels/ISubscriptionChannel.cs | 5 +- .../Channels/StorageChannel.cs | 29 +++++++++++ .../Channels/StorageChannelFactory.cs | 42 +++++++++++++++ .../Operations/SubscriptionProcessingJob.cs | 51 ++++--------------- .../Registration/SubscriptionsModule.cs | 11 ++++ 10 files changed, 149 insertions(+), 56 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Channels/ChannelTypeAttribute.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannelFactory.cs diff --git a/src/Microsoft.Health.Fhir.Core/Configs/CoreFeatureConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/CoreFeatureConfiguration.cs index 62b585c53a..ca780a0bab 100644 --- a/src/Microsoft.Health.Fhir.Core/Configs/CoreFeatureConfiguration.cs +++ b/src/Microsoft.Health.Fhir.Core/Configs/CoreFeatureConfiguration.cs @@ -81,5 +81,10 @@ public class CoreFeatureConfiguration /// Gets or sets a value indicating whether the server supports the $bulk-delete. /// public bool SupportsBulkDelete { get; set; } + + /// + /// Gets or set a value indicating whether the server supports Subscription processing. + /// + public bool SupportsSubscriptions { get; set; } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json index c91847252a..6a8add7b32 100644 --- a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json +++ b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json @@ -30,6 +30,7 @@ "ProfileValidationOnUpdate": false, "SupportsResourceChangeCapture": false, "SupportsBulkDelete": true, + "SupportsSubscriptions": true, "Versioning": { "Default": "versioned", "ResourceTypeOverrides": null diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs index 5f31720eda..8dc1f8e829 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs @@ -12,6 +12,8 @@ using EnsureThat; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.SqlServer.Features.Storage; using Microsoft.Health.Fhir.Subscriptions.Models; @@ -25,12 +27,14 @@ internal class SubscriptionProcessorWatchdog : Watchdog _logger; private readonly ISqlRetryService _sqlRetryService; private readonly IQueueClient _queueClient; + private readonly CoreFeatureConfiguration _config; private CancellationToken _cancellationToken; public SubscriptionProcessorWatchdog( SqlStoreClient store, ISqlRetryService sqlRetryService, IQueueClient queueClient, + IOptions coreConfiguration, ILogger logger) : base(sqlRetryService, logger) { @@ -39,6 +43,7 @@ public SubscriptionProcessorWatchdog( _logger = EnsureArg.IsNotNull(logger, nameof(logger)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); _queueClient = EnsureArg.IsNotNull(queueClient, nameof(queueClient)); + _config = EnsureArg.IsNotNull(coreConfiguration?.Value, nameof(coreConfiguration)); } internal string LastEventProcessedTransactionId => $"{Name}.{nameof(LastEventProcessedTransactionId)}"; @@ -68,20 +73,24 @@ protected override async Task ExecuteAsync() return; } - var transactionsToQueue = new List(); - - foreach (var tran in transactionsToProcess.Where(x => x.VisibleDate.HasValue).OrderBy(x => x.TransactionId)) + if (_config.SupportsSubscriptions) { - var jobDefinition = new SubscriptionJobDefinition(Core.Features.Operations.JobType.SubscriptionsOrchestrator) + var transactionsToQueue = new List(); + + foreach (var tran in transactionsToProcess.Where(x => x.VisibleDate.HasValue).OrderBy(x => x.TransactionId)) { - TransactionId = tran.TransactionId, - VisibleDate = tran.VisibleDate.Value, - }; + var jobDefinition = new SubscriptionJobDefinition(JobType.SubscriptionsOrchestrator) + { + TransactionId = tran.TransactionId, + VisibleDate = tran.VisibleDate.Value, + }; + + transactionsToQueue.Add(jobDefinition); + } - transactionsToQueue.Add(jobDefinition); + await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken: _cancellationToken, definitions: transactionsToQueue.ToArray()); } - await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken: _cancellationToken, definitions: transactionsToQueue.ToArray()); await UpdateLastEventProcessedTransactionId(transactionsToProcess.Max(x => x.TransactionId)); _logger.LogInformation($"{Name}: completed. transactions={transactionsToProcess.Count}"); @@ -94,8 +103,9 @@ private async Task GetLastTransactionId() private async Task InitLastProcessedTransactionId() { - using var cmd = new SqlCommand("INSERT INTO dbo.Parameters (Id, Bigint) SELECT @Id, 5105975696064002770"); // surrogate id for the past. does not matter. + using var cmd = new SqlCommand("INSERT INTO dbo.Parameters (Id, Bigint) SELECT @Id, @LastTranId"); cmd.Parameters.AddWithValue("@Id", LastEventProcessedTransactionId); + cmd.Parameters.AddWithValue("@LastTranId", await _store.MergeResourcesGetTransactionVisibilityAsync(_cancellationToken)); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs index bafa0b1c03..2e411bd8ac 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs @@ -24,7 +24,7 @@ internal class WatchdogsBackgroundService : BackgroundService, INotificationHand private readonly CleanupEventLogWatchdog _cleanupEventLogWatchdog; private readonly IScoped _transactionWatchdog; private readonly InvisibleHistoryCleanupWatchdog _invisibleHistoryCleanupWatchdog; - private readonly SubscriptionProcessorWatchdog _eventProcessorWatchdog; + private readonly SubscriptionProcessorWatchdog _subscriptionsProcessorWatchdog; public WatchdogsBackgroundService( DefragWatchdog defragWatchdog, @@ -37,7 +37,7 @@ public WatchdogsBackgroundService( _cleanupEventLogWatchdog = EnsureArg.IsNotNull(cleanupEventLogWatchdog, nameof(cleanupEventLogWatchdog)); _transactionWatchdog = EnsureArg.IsNotNull(transactionWatchdog, nameof(transactionWatchdog)).Invoke(); _invisibleHistoryCleanupWatchdog = EnsureArg.IsNotNull(invisibleHistoryCleanupWatchdog, nameof(invisibleHistoryCleanupWatchdog)); - _eventProcessorWatchdog = EnsureArg.IsNotNull(eventProcessorWatchdog, nameof(eventProcessorWatchdog)); + _subscriptionsProcessorWatchdog = EnsureArg.IsNotNull(eventProcessorWatchdog, nameof(eventProcessorWatchdog)); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -56,7 +56,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) _cleanupEventLogWatchdog.StartAsync(continuationTokenSource.Token), _transactionWatchdog.Value.StartAsync(continuationTokenSource.Token), _invisibleHistoryCleanupWatchdog.StartAsync(continuationTokenSource.Token), - _eventProcessorWatchdog.StartAsync(continuationTokenSource.Token), + _subscriptionsProcessorWatchdog.StartAsync(continuationTokenSource.Token), }; await Task.WhenAny(tasks); diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ChannelTypeAttribute.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ChannelTypeAttribute.cs new file mode 100644 index 0000000000..8c52493d98 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ChannelTypeAttribute.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Channels +{ + [AttributeUsage(AttributeTargets.Class)] + public sealed class ChannelTypeAttribute : Attribute + { + public ChannelTypeAttribute(SubscriptionChannelType channelType) + { + ChannelType = channelType; + } + + public SubscriptionChannelType ChannelType { get; } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs index fc13912923..35cb6da2b0 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs @@ -8,10 +8,13 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Subscriptions.Models; namespace Microsoft.Health.Fhir.Subscriptions.Channels { - internal interface ISubscriptionChannel + public interface ISubscriptionChannel { + Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs index fa0c938cdc..3f32ec3a5b 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs @@ -8,10 +8,39 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Health.Fhir.Core.Features.Operations.Export; +using Microsoft.Health.Fhir.Core.Features.Operations.Export.ExportDestinationClient; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Subscriptions.Models; namespace Microsoft.Health.Fhir.Subscriptions.Channels { + [ChannelType(SubscriptionChannelType.Storage)] public class StorageChannel : ISubscriptionChannel { + private readonly IExportDestinationClient _exportDestinationClient; + + public StorageChannel( + IExportDestinationClient exportDestinationClient) + { + _exportDestinationClient = exportDestinationClient; + } + + public async Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) + { + await _exportDestinationClient.ConnectAsync(cancellationToken, channelInfo.Properties["container"]); + + foreach (var resource in resources) + { + string fileName = $"{resource.ToResourceKey()}.json"; + + _exportDestinationClient.WriteFilePart( + fileName, + resource.RawResource.Data); + + _exportDestinationClient.CommitFile(fileName); + } + } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannelFactory.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannelFactory.cs new file mode 100644 index 0000000000..47bb1e1dfd --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannelFactory.cs @@ -0,0 +1,42 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Channels +{ + public class StorageChannelFactory + { + private IServiceProvider _serviceProvider; + private Dictionary _channelTypeMap; + + public StorageChannelFactory(IServiceProvider serviceProvider) + { + _serviceProvider = EnsureArg.IsNotNull(serviceProvider, nameof(serviceProvider)); + + _channelTypeMap = + typeof(ISubscriptionChannel).Assembly.GetTypes() + .Where(t => typeof(ISubscriptionChannel).IsAssignableFrom(t) && !t.IsInterface && !t.IsAbstract) + .Select(t => new + { + Type = t, + Attribute = t.GetCustomAttributes(typeof(ChannelTypeAttribute), false).FirstOrDefault() as ChannelTypeAttribute, + }) + .Where(t => t.Attribute != null) + .ToDictionary(t => t.Attribute.ChannelType, t => t.Type); + } + + public ISubscriptionChannel Create(SubscriptionChannelType type) + { + return (ISubscriptionChannel)_serviceProvider.GetService(_channelTypeMap[type]); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs index 277113f818..10c4afd4f6 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -15,6 +15,7 @@ using Microsoft.Health.Fhir.Core.Features.Operations.Export; using Microsoft.Health.Fhir.Core.Features.Operations.Export.ExportDestinationClient; using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Subscriptions.Channels; using Microsoft.Health.Fhir.Subscriptions.Models; using Microsoft.Health.JobManagement; @@ -23,24 +24,13 @@ namespace Microsoft.Health.Fhir.Subscriptions.Operations [JobTypeId((int)JobType.SubscriptionsProcessing)] public class SubscriptionProcessingJob : IJob { - private readonly IResourceToByteArraySerializer _resourceToByteArraySerializer; - private readonly IExportDestinationClient _exportDestinationClient; - private readonly IResourceDeserializer _resourceDeserializer; + private readonly StorageChannelFactory _storageChannelFactory; private readonly IFhirDataStore _dataStore; - private readonly ILogger _logger; - public SubscriptionProcessingJob( - IResourceToByteArraySerializer resourceToByteArraySerializer, - IExportDestinationClient exportDestinationClient, - IResourceDeserializer resourceDeserializer, - IFhirDataStore dataStore, - ILogger logger) + public SubscriptionProcessingJob(StorageChannelFactory storageChannelFactory, IFhirDataStore dataStore) { - _resourceToByteArraySerializer = resourceToByteArraySerializer; - _exportDestinationClient = exportDestinationClient; - _resourceDeserializer = resourceDeserializer; + _storageChannelFactory = storageChannelFactory; _dataStore = dataStore; - _logger = logger; } public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) @@ -52,35 +42,12 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel return HttpStatusCode.BadRequest.ToString(); } - if (definition.Channel.ChannelType == SubscriptionChannelType.Storage) - { - try - { - await _exportDestinationClient.ConnectAsync(cancellationToken, definition.Channel.Properties["container"]); - - foreach (var resourceKey in definition.ResourceReferences) - { - var resource = await _dataStore.GetAsync(resourceKey, cancellationToken); - - string fileName = $"{resourceKey}.json"; - - _exportDestinationClient.WriteFilePart( - fileName, - resource.RawResource.Data); + var allResources = await Task.WhenAll( + definition.ResourceReferences + .Select(async x => await _dataStore.GetAsync(x, cancellationToken))); - _exportDestinationClient.CommitFile(fileName); - } - } - catch (Exception ex) - { - _logger.LogJobError(jobInfo, ex.ToString()); - return HttpStatusCode.InternalServerError.ToString(); - } - } - else - { - return HttpStatusCode.BadRequest.ToString(); - } + var channel = _storageChannelFactory.Create(definition.Channel.ChannelType); + await channel.PublishAsync(allResources, definition.Channel, definition.VisibleDate, cancellationToken); return HttpStatusCode.OK.ToString(); } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs index bbf12b8002..d58ff3085e 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Subscriptions.Channels; using Microsoft.Health.Fhir.Subscriptions.Persistence; using Microsoft.Health.JobManagement; @@ -34,6 +35,16 @@ public void Load(IServiceCollection services) .Singleton() .AsSelf() .AsImplementedInterfaces(); + + services.TypesInSameAssemblyAs() + .AssignableTo() + .Transient() + .AsSelf() + .AsImplementedInterfaces(); + + services.Add() + .Singleton() + .AsSelf(); } } } From 35954c77b5dfcdc8aaa58b5735896d3ed8fc2e33 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Tue, 16 Apr 2024 17:28:32 -0700 Subject: [PATCH 08/47] Add example subscription --- docs/rest/Subscriptions.http | 72 ++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 docs/rest/Subscriptions.http diff --git a/docs/rest/Subscriptions.http b/docs/rest/Subscriptions.http new file mode 100644 index 0000000000..000ef07cb0 --- /dev/null +++ b/docs/rest/Subscriptions.http @@ -0,0 +1,72 @@ +# # .SUMMARY Sample requests to verify FHIR Conditional Delete +# The assumption for the requests and resources below: +# The FHIR version is R4 + +@hostname = localhost:44348 + +### Get the bearer token, if authentication is enabled +# @name bearer +POST https://{{hostname}}/connect/token +content-type: application/x-www-form-urlencoded + +grant_type=client_credentials +&client_id=globalAdminServicePrincipal +&client_secret=globalAdminServicePrincipal +&scope=fhir-api + +### PUT Subscription for Rest-hook +## More examples: https://build.fhir.org/ig/HL7/fhir-subscription-backport-ig/artifacts.html +PUT https://{{hostname}}/Subscription/example-backport-storage +content-type: application/json +Authorization: Bearer {{bearer.response.body.access_token}} + +{ + "resourceType": "Subscription", + "id": "example-backport-storage", + "meta" : { + "profile": ["http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"] + }, + "status": "requested", + "end": "2031-01-01T12:00:00", + "reason": "Test subscription based on transactions", + "criteria" : "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions", + "_criteria": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-filter-criteria", + "valueString": "Patient" + } + ] + }, + "channel": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-heartbeat-period", + "valueInteger": 120 + } + ], + "type" : "rest-hook", + "_type" : { + "extension" : [ + { + "url" : "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type", + "valueCoding" : { + "system" : "http://azurehealthcareapis.com/data-extentions/subscription-channel-type", + "code" : "azure-storage", + "display" : "Azure Blob Storage" + } + } + ] + }, + "endpoint": "https://127.0.0.1:10000/devstoreaccount1/sync-all", + "payload": "application/fhir+json", + "_payload": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content", + "valueCode": "full-resource" + } + ] + } + } +} From cb40eb2fc865666815b22ba554cdeb12d237cd8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Henrique=20Inoc=C3=AAncio=20Borba=20Ferreira?= Date: Wed, 17 Apr 2024 11:41:20 -0700 Subject: [PATCH 09/47] DataLakeChannel. --- .../Channels/DataLakeChannel.cs | 52 +++++++++++++++++++ .../Persistence/SubscriptionManager.cs | 12 +++++ tools/EventsReader/Program.cs | 2 +- tools/PerfTester/Program.cs | 4 +- 4 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs new file mode 100644 index 0000000000..efbffbabe9 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs @@ -0,0 +1,52 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Globalization; +using Microsoft.Health.Fhir.Core.Features.Operations.Export.ExportDestinationClient; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Channels +{ + [ChannelType(SubscriptionChannelType.DatalakeContract)] + public class DataLakeChannel : ISubscriptionChannel + { + private readonly IExportDestinationClient _exportDestinationClient; + + public DataLakeChannel(IExportDestinationClient exportDestinationClient) + { + _exportDestinationClient = exportDestinationClient; + } + + public async Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) + { + try + { + await _exportDestinationClient.ConnectAsync(cancellationToken, channelInfo.Properties["container"]); + + IReadOnlyList> resourceGroupedByResourceType = resources.GroupBy(x => x.ResourceTypeName.ToLower(CultureInfo.InvariantCulture)).ToList(); + + foreach (IGrouping groupOfResources in resourceGroupedByResourceType) + { + string blobName = $"{groupOfResources.Key}/{transactionTime.Year:D4}/{transactionTime.Month:D2}/{transactionTime.Day:D2}/{transactionTime.ToUniversalTime().ToString("yyyy-MM-ddTHH.mm.ss.fffZ")}.ndjson"; + + foreach (ResourceWrapper item in groupOfResources) + { + // TODO: implement the soft-delete property addition. + string json = item.RawResource.Data; + + _exportDestinationClient.WriteFilePart(blobName, json); + } + + _exportDestinationClient.Commit(); + } + } + catch (Exception ex) + { + throw new InvalidOperationException("Failure in DatalakeChannel", ex); + } + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs index be8078dc2a..8bbe3bd896 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs @@ -44,6 +44,18 @@ public Task> GetActiveSubscriptionsAsync(C { "container", "sync-patient" }, }, }), + new SubscriptionInfo( + null, + new ChannelInfo + { + ChannelType = SubscriptionChannelType.DatalakeContract, + ContentType = SubscriptionContentType.FullResource, + MaxCount = 100, + Properties = new Dictionary + { + { "container", "lake" }, + }, + }), }; return Task.FromResult(list); diff --git a/tools/EventsReader/Program.cs b/tools/EventsReader/Program.cs index 372de29ffc..d6dcf988d1 100644 --- a/tools/EventsReader/Program.cs +++ b/tools/EventsReader/Program.cs @@ -23,7 +23,7 @@ public static void Main() { ISqlConnectionBuilder iSqlConnectionBuilder = new Sql.SqlConnectionBuilder(_connectionString); _sqlRetryService = SqlRetryService.GetInstance(iSqlConnectionBuilder); - _store = new SqlStoreClient(_sqlRetryService, NullLogger.Instance); + _store = new SqlStoreClient(_sqlRetryService, NullLogger.Instance); ExecuteAsync().Wait(); } diff --git a/tools/PerfTester/Program.cs b/tools/PerfTester/Program.cs index 8f9000ccc8..6abb1bb46f 100644 --- a/tools/PerfTester/Program.cs +++ b/tools/PerfTester/Program.cs @@ -48,14 +48,14 @@ public static class Program private static readonly int _repeat = int.Parse(ConfigurationManager.AppSettings["Repeat"]); private static SqlRetryService _sqlRetryService; - private static SqlStoreClient _store; + private static SqlStoreClient _store; public static void Main() { Console.WriteLine("!!!See App.config for the details!!!"); ISqlConnectionBuilder iSqlConnectionBuilder = new Sql.SqlConnectionBuilder(_connectionString); _sqlRetryService = SqlRetryService.GetInstance(iSqlConnectionBuilder); - _store = new SqlStoreClient(_sqlRetryService, NullLogger.Instance); + _store = new SqlStoreClient(_sqlRetryService, NullLogger.Instance); DumpResourceIds(); From 1eb5264c83919e47cc5582da22dddf57dacfd6cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Henrique=20Inoc=C3=AAncio=20Borba=20Ferreira?= Date: Wed, 17 Apr 2024 17:57:17 -0700 Subject: [PATCH 10/47] Changes in DataLakeChannel and the project config. --- .../Channels/DataLakeChannel.cs | 22 ++++++++++++++++--- .../Channels/ISubscriptionChannel.cs | 3 +-- .../Channels/StorageChannel.cs | 5 +---- ...Microsoft.Health.Fhir.Subscriptions.csproj | 4 ---- .../Operations/SubscriptionProcessingJob.cs | 8 +------ .../SubscriptionsOrchestratorJob.cs | 4 +--- .../Persistence/ISubscriptionManager.cs | 3 +++ .../Persistence/ITransactionDataStore.cs | 4 +--- .../Persistence/SubscriptionManager.cs | 4 +--- 9 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs index efbffbabe9..a957b88592 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs @@ -3,7 +3,12 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; +using System.Collections.Generic; using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Health.Fhir.Core.Features.Operations.Export.ExportDestinationClient; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Subscriptions.Models; @@ -14,10 +19,12 @@ namespace Microsoft.Health.Fhir.Subscriptions.Channels public class DataLakeChannel : ISubscriptionChannel { private readonly IExportDestinationClient _exportDestinationClient; + private readonly IResourceDeserializer _resourceDeserializer; - public DataLakeChannel(IExportDestinationClient exportDestinationClient) + public DataLakeChannel(IExportDestinationClient exportDestinationClient, IResourceDeserializer resourceDeserializer) { _exportDestinationClient = exportDestinationClient; + _resourceDeserializer = resourceDeserializer; } public async Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) @@ -28,15 +35,24 @@ public async Task PublishAsync(IReadOnlyCollection resources, C IReadOnlyList> resourceGroupedByResourceType = resources.GroupBy(x => x.ResourceTypeName.ToLower(CultureInfo.InvariantCulture)).ToList(); + DateTimeOffset transactionTimeInUtc = transactionTime.ToUniversalTime(); + foreach (IGrouping groupOfResources in resourceGroupedByResourceType) { - string blobName = $"{groupOfResources.Key}/{transactionTime.Year:D4}/{transactionTime.Month:D2}/{transactionTime.Day:D2}/{transactionTime.ToUniversalTime().ToString("yyyy-MM-ddTHH.mm.ss.fffZ")}.ndjson"; + string blobName = $"{groupOfResources.Key}/{transactionTimeInUtc.Year:D4}/{transactionTimeInUtc.Month:D2}/{transactionTimeInUtc.Day:D2}/{transactionTimeInUtc.ToString("yyyy-MM-ddTHH.mm.ss.fffZ")}.ndjson"; foreach (ResourceWrapper item in groupOfResources) { - // TODO: implement the soft-delete property addition. string json = item.RawResource.Data; + /* + // TODO: Add logic to handle soft-deleted resources. + if (item.IsDeleted) + { + ResourceElement element = _resourceDeserializer.Deserialize(item); + } + */ + _exportDestinationClient.WriteFilePart(blobName, json); } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs index 35cb6da2b0..e98211970b 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs @@ -5,8 +5,7 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; +using System.Threading; using System.Threading.Tasks; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Subscriptions.Models; diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs index 3f32ec3a5b..427e5f7246 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs @@ -5,11 +5,8 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; +using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Health.Fhir.Core.Features.Operations.Export; using Microsoft.Health.Fhir.Core.Features.Operations.Export.ExportDestinationClient; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Subscriptions.Models; diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj index 50f7da1e86..13218ee495 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj +++ b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj @@ -1,9 +1,5 @@  - - enable - - diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs index 10c4afd4f6..7c38327e10 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -3,17 +3,11 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System; -using System.Collections.Generic; using System.Linq; using System.Net; -using System.Text; +using System.Threading; using System.Threading.Tasks; -using Microsoft.Build.Framework; -using Microsoft.Extensions.Logging; using Microsoft.Health.Fhir.Core.Features.Operations; -using Microsoft.Health.Fhir.Core.Features.Operations.Export; -using Microsoft.Health.Fhir.Core.Features.Operations.Export.ExportDestinationClient; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Subscriptions.Channels; using Microsoft.Health.Fhir.Subscriptions.Models; diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs index 9cc1fd6dc5..9c48d1ce72 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs @@ -7,11 +7,9 @@ using System.Collections.Generic; using System.Collections.ObjectModel; using System.Linq; -using System.Text; +using System.Threading; using System.Threading.Tasks; using EnsureThat; -using Microsoft.Health.Extensions.DependencyInjection; -using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Persistence; diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs index a5f2738457..7b1132370e 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs @@ -3,6 +3,9 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Health.Fhir.Subscriptions.Models; namespace Microsoft.Health.Fhir.Subscriptions.Persistence diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs index 6a1ca37224..52d5cf3223 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs @@ -3,10 +3,8 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System; using System.Collections.Generic; -using System.Linq; -using System.Text; +using System.Threading; using System.Threading.Tasks; using Microsoft.Health.Fhir.Core.Features.Persistence; diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs index 8bbe3bd896..cd53fa90db 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs @@ -3,10 +3,8 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System; using System.Collections.Generic; -using System.Linq; -using System.Text; +using System.Threading; using System.Threading.Tasks; using Microsoft.Health.Fhir.Subscriptions.Models; From 03977d34bfb47467dfdf9a1b73c22b6cc7e7f517 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Thu, 18 Apr 2024 09:30:37 -0700 Subject: [PATCH 11/47] Load from DB --- Microsoft.Health.Fhir.sln | 7 + docs/rest/Subscriptions.http | 117 +++++++++++- .../Models/KnownResourceTypes.cs | 2 + ...oft.Health.Fhir.Subscriptions.Tests.csproj | 27 +++ .../Peristence/SubscriptionManagerTests.cs | 49 +++++ .../AssemblyInfo.cs | 11 ++ .../Channels/DataLakeChannel.cs | 2 +- .../Channels/StorageChannel.cs | 2 +- .../Models/ChannelInfo.cs | 2 + .../SubscriptionsOrchestratorJob.cs | 7 + .../Persistence/ISubscriptionManager.cs | 2 + .../Persistence/SubscriptionManager.cs | 171 +++++++++++++----- .../Registration/SubscriptionsModule.cs | 7 +- .../CommonSamples.cs | 52 ++++++ .../EmbeddedResourceManager.cs | 11 +- .../Microsoft.Health.Fhir.Tests.Common.csproj | 2 + .../TestFiles/R4/Subscription-Backport.json | 54 ++++++ 17 files changed, 472 insertions(+), 53 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj create mode 100644 src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionManagerTests.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/AssemblyInfo.cs create mode 100644 src/Microsoft.Health.Fhir.Tests.Common/CommonSamples.cs create mode 100644 src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/Subscription-Backport.json diff --git a/Microsoft.Health.Fhir.sln b/Microsoft.Health.Fhir.sln index bc333070ea..0a0b88fedf 100644 --- a/Microsoft.Health.Fhir.sln +++ b/Microsoft.Health.Fhir.sln @@ -207,6 +207,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Cosmo EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Subscriptions", "src\Microsoft.Health.Fhir.Subscriptions\Microsoft.Health.Fhir.Subscriptions.csproj", "{BD8F3137-89F5-4EE5-B269-24D73081E00A}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Subscriptions.Tests", "src\Microsoft.Health.Fhir.Subscriptions.Tests\Microsoft.Health.Fhir.Subscriptions.Tests.csproj", "{AA73AB9D-52EF-4172-9911-3C9D661C8D48}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -489,6 +491,10 @@ Global {BD8F3137-89F5-4EE5-B269-24D73081E00A}.Debug|Any CPU.Build.0 = Debug|Any CPU {BD8F3137-89F5-4EE5-B269-24D73081E00A}.Release|Any CPU.ActiveCfg = Release|Any CPU {BD8F3137-89F5-4EE5-B269-24D73081E00A}.Release|Any CPU.Build.0 = Release|Any CPU + {AA73AB9D-52EF-4172-9911-3C9D661C8D48}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AA73AB9D-52EF-4172-9911-3C9D661C8D48}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AA73AB9D-52EF-4172-9911-3C9D661C8D48}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AA73AB9D-52EF-4172-9911-3C9D661C8D48}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -585,6 +591,7 @@ Global {B9AAA11D-8C8C-44C3-AADE-801376EF82F0} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} {BD8F3137-89F5-4EE5-B269-24D73081E00A} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4} + {AA73AB9D-52EF-4172-9911-3C9D661C8D48} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution RESX_SortFileContentOnSave = True diff --git a/docs/rest/Subscriptions.http b/docs/rest/Subscriptions.http index 000ef07cb0..68bcb743e1 100644 --- a/docs/rest/Subscriptions.http +++ b/docs/rest/Subscriptions.http @@ -14,21 +14,21 @@ grant_type=client_credentials &client_secret=globalAdminServicePrincipal &scope=fhir-api -### PUT Subscription for Rest-hook +### PUT Subscription for Blob Storage ## More examples: https://build.fhir.org/ig/HL7/fhir-subscription-backport-ig/artifacts.html -PUT https://{{hostname}}/Subscription/example-backport-storage +PUT https://{{hostname}}/Subscription/example-backport-storage-patient content-type: application/json Authorization: Bearer {{bearer.response.body.access_token}} { "resourceType": "Subscription", - "id": "example-backport-storage", + "id": "example-backport-storage-patient", "meta" : { "profile": ["http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"] }, "status": "requested", "end": "2031-01-01T12:00:00", - "reason": "Test subscription based on transactions", + "reason": "Test subscription based on transactions, filtered by Patient", "criteria" : "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions", "_criteria": { "extension": [ @@ -58,7 +58,7 @@ Authorization: Bearer {{bearer.response.body.access_token}} } ] }, - "endpoint": "https://127.0.0.1:10000/devstoreaccount1/sync-all", + "endpoint": "sync-patient", "payload": "application/fhir+json", "_payload": { "extension": [ @@ -70,3 +70,110 @@ Authorization: Bearer {{bearer.response.body.access_token}} } } } + +### PUT Subscription for Blob Storage +## More examples: https://build.fhir.org/ig/HL7/fhir-subscription-backport-ig/artifacts.html +PUT https://{{hostname}}/Subscription/example-backport-storage-all +content-type: application/json +Authorization: Bearer {{bearer.response.body.access_token}} + +{ + "resourceType": "Subscription", + "id": "example-backport-storage-all", + "meta" : { + "profile": ["http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"] + }, + "status": "requested", + "end": "2031-01-01T12:00:00", + "reason": "Test subscription based on transactions", + "criteria" : "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions", + "channel": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-heartbeat-period", + "valueInteger": 120 + }, + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-max-count", + "valuePositiveInt": 20 + } + ], + "type" : "rest-hook", + "_type" : { + "extension" : [ + { + "url" : "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type", + "valueCoding" : { + "system" : "http://azurehealthcareapis.com/data-extentions/subscription-channel-type", + "code" : "azure-storage", + "display" : "Azure Blob Storage" + } + } + ] + }, + "endpoint": "sync-all", + "payload": "application/fhir+json", + "_payload": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content", + "valueCode": "full-resource" + } + ] + } + } +} + +### PUT Subscription for Fabric +## More examples: https://build.fhir.org/ig/HL7/fhir-subscription-backport-ig/artifacts.html +PUT https://{{hostname}}/Subscription/example-backport-lake +content-type: application/json +Authorization: Bearer {{bearer.response.body.access_token}} + +{ + "resourceType": "Subscription", + "id": "example-backport-lake", + "meta" : { + "profile": ["http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"] + }, + "status": "requested", + "end": "2031-01-01T12:00:00", + "reason": "Test subscription based on transactions", + "criteria" : "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions", + "channel": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-heartbeat-period", + "valueInteger": 120 + } + ], + "type" : "rest-hook", + "_type" : { + "extension" : [ + { + "url" : "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type", + "valueCoding" : { + "system" : "http://azurehealthcareapis.com/data-extentions/subscription-channel-type", + "code" : "azure-lake-storage", + "display" : "Azure Data Lake Contract Storage" + } + } + ] + }, + "endpoint": "sync-lake", + "payload": "application/fhir+ndjson", + "_payload": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content", + "valueCode": "full-resource" + } + ] + } + } +} + +### +DELETE https://{{hostname}}/Subscription/example-backport-storage-all +content-type: application/json +Authorization: Bearer {{bearer.response.body.access_token}} \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Core/Models/KnownResourceTypes.cs b/src/Microsoft.Health.Fhir.Core/Models/KnownResourceTypes.cs index 2a2708c938..96e4099ce8 100644 --- a/src/Microsoft.Health.Fhir.Core/Models/KnownResourceTypes.cs +++ b/src/Microsoft.Health.Fhir.Core/Models/KnownResourceTypes.cs @@ -55,6 +55,8 @@ public static class KnownResourceTypes public const string SearchParameter = "SearchParameter"; + public const string Subscription = "Subscription"; + public const string Patient = "Patient"; public const string ValueSet = "ValueSet"; diff --git a/src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj new file mode 100644 index 0000000000..f2ca89213d --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj @@ -0,0 +1,27 @@ + + + + enable + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + diff --git a/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionManagerTests.cs b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionManagerTests.cs new file mode 100644 index 0000000000..253e151730 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionManagerTests.cs @@ -0,0 +1,49 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Hl7.Fhir.ElementModel; +using Hl7.Fhir.Model; +using Hl7.Fhir.Serialization; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Subscriptions.Models; +using Microsoft.Health.Fhir.Subscriptions.Persistence; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Subscriptions.Tests.Peristence +{ + public class SubscriptionManagerTests + { + private IModelInfoProvider _modelInfo; + + public SubscriptionManagerTests() + { + _modelInfo = MockModelInfoProviderBuilder + .Create(FhirSpecification.R4) + .AddKnownTypes(KnownResourceTypes.Subscription) + .Build(); + } + + [Fact] + public void GivenAnR4BackportSubscription_WhenConvertingToInfo_ThenTheInformationIsCorrect() + { + var subscription = CommonSamples.GetJsonSample("Subscription-Backport", FhirSpecification.R4, s => s.ToTypedElement(ModelInfo.ModelInspector)); + + var info = SubscriptionManager.ConvertToInfo(subscription); + + Assert.Equal("Patient", info.FilterCriteria); + Assert.Equal("sync-all", info.Channel.Endpoint); + Assert.Equal(20, info.Channel.MaxCount); + Assert.Equal(SubscriptionContentType.FullResource, info.Channel.ContentType); + Assert.Equal(SubscriptionChannelType.Storage, info.Channel.ChannelType); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/AssemblyInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/AssemblyInfo.cs new file mode 100644 index 0000000000..04b1e9fede --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/AssemblyInfo.cs @@ -0,0 +1,11 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Resources; +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Microsoft.Health.Fhir.Subscriptions.Tests")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] +[assembly: NeutralResourcesLanguage("en-us")] diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs index a957b88592..f41a460134 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs @@ -31,7 +31,7 @@ public async Task PublishAsync(IReadOnlyCollection resources, C { try { - await _exportDestinationClient.ConnectAsync(cancellationToken, channelInfo.Properties["container"]); + await _exportDestinationClient.ConnectAsync(cancellationToken, channelInfo.Endpoint); IReadOnlyList> resourceGroupedByResourceType = resources.GroupBy(x => x.ResourceTypeName.ToLower(CultureInfo.InvariantCulture)).ToList(); diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs index 427e5f7246..d7d0c2ad74 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs @@ -26,7 +26,7 @@ public StorageChannel( public async Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) { - await _exportDestinationClient.ConnectAsync(cancellationToken, channelInfo.Properties["container"]); + await _exportDestinationClient.ConnectAsync(cancellationToken, channelInfo.Endpoint); foreach (var resource in resources) { diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs index 5d99a57c2e..863f320d9e 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs @@ -32,6 +32,8 @@ public class ChannelInfo public SubscriptionContentType ContentType { get; set; } + public string Endpoint { get; set; } + [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "Json Poco")] public IDictionary Properties { get; set; } = new Dictionary(); } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs index 9c48d1ce72..acca62e009 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs @@ -15,6 +15,7 @@ using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Routing; using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Subscriptions.Models; using Microsoft.Health.Fhir.Subscriptions.Persistence; using Microsoft.Health.JobManagement; @@ -57,6 +58,12 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel var resources = await _transactionDataStore.GetResourcesByTransactionIdAsync(definition.TransactionId, cancellationToken); var resourceKeys = resources.Select(r => new ResourceKey(r.ResourceTypeName, r.ResourceId, r.Version)).ToList().AsReadOnly(); + // Sync subscriptions if a change is detected + if (resources.Any(x => string.Equals(x.ResourceTypeName, KnownResourceTypes.Subscription, StringComparison.Ordinal))) + { + await _subscriptionManager.SyncSubscriptionsAsync(cancellationToken); + } + var processingDefinition = new List(); foreach (var sub in await _subscriptionManager.GetActiveSubscriptionsAsync(cancellationToken)) diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs index 7b1132370e..180df430ba 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs @@ -13,5 +13,7 @@ namespace Microsoft.Health.Fhir.Subscriptions.Persistence public interface ISubscriptionManager { Task> GetActiveSubscriptionsAsync(CancellationToken cancellationToken); + + Task SyncSubscriptionsAsync(CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs index cd53fa90db..915ae83e63 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs @@ -3,60 +3,147 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using EnsureThat; +using MediatR; +using Microsoft.Build.Framework; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Messages.Storage; +using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Subscriptions.Models; namespace Microsoft.Health.Fhir.Subscriptions.Persistence { - public class SubscriptionManager : ISubscriptionManager + public sealed class SubscriptionManager : ISubscriptionManager, INotificationHandler { - public Task> GetActiveSubscriptionsAsync(CancellationToken cancellationToken) + private readonly IScopeProvider _dataStoreProvider; + private readonly IScopeProvider _searchServiceProvider; + private List _subscriptions = new List(); + private readonly IResourceDeserializer _resourceDeserializer; + private readonly ILogger _logger; + private static readonly object _lock = new object(); + private const string MetaString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"; + private const string CriteriaString = "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions"; + private const string CriteriaExtensionString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-filter-criteria"; + private const string ChannelTypeString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type"; + ////private const string AzureChannelTypeString = "http://azurehealthcareapis.com/data-extentions/subscription-channel-type"; + private const string PayloadTypeString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content"; + private const string MaxCountString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-max-count"; + + public SubscriptionManager( + IScopeProvider dataStoreProvider, + IScopeProvider searchServiceProvider, + IResourceDeserializer resourceDeserializer, + ILogger logger) { - IReadOnlyCollection list = new List + _dataStoreProvider = EnsureArg.IsNotNull(dataStoreProvider, nameof(dataStoreProvider)); + _searchServiceProvider = EnsureArg.IsNotNull(searchServiceProvider, nameof(searchServiceProvider)); + _resourceDeserializer = resourceDeserializer; + _logger = logger; + } + + public async Task SyncSubscriptionsAsync(CancellationToken cancellationToken) + { + // requested | active | error | off + + var updatedSubscriptions = new List(); + + using var search = _searchServiceProvider.Invoke(); + + // Get all the active subscriptions + var activeSubscriptions = await search.Value.SearchAsync( + KnownResourceTypes.Subscription, + [ + Tuple.Create("status", "active,requested"), + ], + cancellationToken); + + foreach (var param in activeSubscriptions.Results) { - // "reason": "Alert on Diabetes with Complications Diagnosis", - // "criteria": "Condition?code=http://hl7.org/fhir/sid/icd-10|E11.6", - new SubscriptionInfo( - null, - new ChannelInfo - { - ChannelType = SubscriptionChannelType.Storage, - ContentType = SubscriptionContentType.FullResource, - MaxCount = 100, - Properties = new Dictionary - { - { "container", "sync-all" }, - }, - }), - new SubscriptionInfo( - "Patient", - new ChannelInfo - { - ChannelType = SubscriptionChannelType.Storage, - ContentType = SubscriptionContentType.FullResource, - MaxCount = 100, - Properties = new Dictionary - { - { "container", "sync-patient" }, - }, - }), - new SubscriptionInfo( - null, - new ChannelInfo - { - ChannelType = SubscriptionChannelType.DatalakeContract, - ContentType = SubscriptionContentType.FullResource, - MaxCount = 100, - Properties = new Dictionary - { - { "container", "lake" }, - }, - }), + var resource = _resourceDeserializer.Deserialize(param.Resource); + + SubscriptionInfo info = ConvertToInfo(resource); + + if (info == null) + { + _logger.LogWarning("Subscription with id {SubscriptionId} is valid", resource.Id); + continue; + } + + updatedSubscriptions.Add(info); + } + + lock (_lock) + { + _subscriptions = updatedSubscriptions; + } + } + + internal static SubscriptionInfo ConvertToInfo(ResourceElement resource) + { + var profile = resource.Scalar("Subscription.meta.profile"); + + if (profile != MetaString) + { + return null; + } + + var criteria = resource.Scalar($"Subscription.criteria"); + + if (criteria != CriteriaString) + { + return null; + } + + var criteriaExt = resource.Scalar($"Subscription.criteria.extension.where(url = '{CriteriaExtensionString}').value"); + var channelTypeExt = resource.Scalar($"Subscription.channel.type.extension.where(url = '{ChannelTypeString}').value.code"); + var payloadType = resource.Scalar($"Subscription.channel.payload.extension.where(url = '{PayloadTypeString}').value"); + var maxCount = resource.Scalar($"Subscription.channel.extension.where(url = '{MaxCountString}').value"); + + var channelInfo = new ChannelInfo + { + Endpoint = resource.Scalar($"Subscription.channel.endpoint"), + ChannelType = channelTypeExt switch + { + "azure-storage" => SubscriptionChannelType.Storage, + "azure-lake-storage" => SubscriptionChannelType.DatalakeContract, + _ => SubscriptionChannelType.None, + }, + ContentType = payloadType switch + { + "full-resource" => SubscriptionContentType.FullResource, + "id-only" => SubscriptionContentType.IdOnly, + _ => SubscriptionContentType.Empty, + }, + MaxCount = maxCount ?? 100, }; - return Task.FromResult(list); + var info = new SubscriptionInfo(criteriaExt, channelInfo); + + return info; + } + + public async Task> GetActiveSubscriptionsAsync(CancellationToken cancellationToken) + { + if (_subscriptions.Count == 0) + { + await SyncSubscriptionsAsync(cancellationToken); + } + + return _subscriptions; + } + + public async Task Handle(StorageInitializedNotification notification, CancellationToken cancellationToken) + { + // Preload subscriptions when storage becomes available + await SyncSubscriptionsAsync(cancellationToken); } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs index d58ff3085e..b12ce1f5f7 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs @@ -8,9 +8,12 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using MediatR; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Messages.Storage; using Microsoft.Health.Fhir.Subscriptions.Channels; using Microsoft.Health.Fhir.Subscriptions.Persistence; using Microsoft.Health.JobManagement; @@ -31,7 +34,9 @@ public void Load(IServiceCollection services) job.AsDelegate>(); } - services.Add() + services + .RemoveServiceTypeExact>() + .Add() .Singleton() .AsSelf() .AsImplementedInterfaces(); diff --git a/src/Microsoft.Health.Fhir.Tests.Common/CommonSamples.cs b/src/Microsoft.Health.Fhir.Tests.Common/CommonSamples.cs new file mode 100644 index 0000000000..c89df50779 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Tests.Common/CommonSamples.cs @@ -0,0 +1,52 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using EnsureThat; +using Hl7.Fhir.ElementModel; +using Hl7.Fhir.Serialization; +using Hl7.Fhir.Specification; +using Microsoft.AspNetCore.SignalR; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Tests.Common +{ + public class CommonSamples + { + /// + /// Loads a sample Resource + /// + public static ResourceElement GetJsonSample(string fileName, IModelInfoProvider modelInfoProvider = null) + { + EnsureArg.IsNotNullOrWhiteSpace(fileName, nameof(fileName)); + + if (modelInfoProvider == null) + { + modelInfoProvider = MockModelInfoProviderBuilder + .Create(FhirSpecification.R4) + .Build(); + } + + return GetJsonSample(fileName, modelInfoProvider.Version, node => modelInfoProvider.ToTypedElement(node)); + } + + public static ResourceElement GetJsonSample(string fileName, FhirSpecification fhirSpecification, Func convert) + { + EnsureArg.IsNotNullOrWhiteSpace(fileName, nameof(fileName)); + + var fhirSource = EmbeddedResourceManager.GetStringContent("TestFiles", fileName, "json", fhirSpecification); + + var node = FhirJsonNode.Parse(fhirSource); + + var instance = convert(node); + + return new ResourceElement(instance); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Tests.Common/EmbeddedResourceManager.cs b/src/Microsoft.Health.Fhir.Tests.Common/EmbeddedResourceManager.cs index 44aec6ea0a..bdc47d0606 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/EmbeddedResourceManager.cs +++ b/src/Microsoft.Health.Fhir.Tests.Common/EmbeddedResourceManager.cs @@ -11,13 +11,13 @@ namespace Microsoft.Health.Fhir.Tests.Common { public static class EmbeddedResourceManager { - public static string GetStringContent(string embeddedResourceSubNamespace, string fileName, string extension) + public static string GetStringContent(string embeddedResourceSubNamespace, string fileName, string extension, FhirSpecification version) { - string resourceName = $"{typeof(EmbeddedResourceManager).Namespace}.{embeddedResourceSubNamespace}.{ModelInfoProvider.Version}.{fileName}.{extension}"; + string resourceName = $"{typeof(EmbeddedResourceManager).Namespace}.{embeddedResourceSubNamespace}.{version}.{fileName}.{extension}"; var resourceInfo = Assembly.GetExecutingAssembly().GetManifestResourceInfo(resourceName); - if (resourceInfo == null && ModelInfoProvider.Version == FhirSpecification.R4B) + if (resourceInfo == null && version == FhirSpecification.R4B) { // Try R4 version resourceName = $"{typeof(EmbeddedResourceManager).Namespace}.{embeddedResourceSubNamespace}.R4.{fileName}.{extension}"; @@ -38,5 +38,10 @@ public static string GetStringContent(string embeddedResourceSubNamespace, strin } } } + + public static string GetStringContent(string embeddedResourceSubNamespace, string fileName, string extension) + { + return GetStringContent(embeddedResourceSubNamespace, fileName, extension, ModelInfoProvider.Version); + } } } diff --git a/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj b/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj index 89524374d1..9dc53d5c30 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj +++ b/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj @@ -16,6 +16,7 @@ + @@ -92,6 +93,7 @@ + diff --git a/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/Subscription-Backport.json b/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/Subscription-Backport.json new file mode 100644 index 0000000000..aa41774c65 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/Subscription-Backport.json @@ -0,0 +1,54 @@ +{ + "resourceType": "Subscription", + "id": "example-backport-storage", + "meta": { + "profile": [ "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription" ] + }, + "status": "requested", + "end": "2031-01-01T12:00:00", + "reason": "Test subscription based on transactions, filtered by Patient", + "criteria": "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions", + "_criteria": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-filter-criteria", + "valueString": "Patient" + } + ] + }, + "channel": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-heartbeat-period", + "valueInteger": 120 + }, + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-max-count", + "valuePositiveInt": 20 + } + ], + "type": "rest-hook", + "_type": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type", + "valueCoding": { + "system": "http://azurehealthcareapis.com/data-extentions/subscription-channel-type", + "code": "azure-storage", + "display": "Azure Blob Storage" + } + } + ] + }, + "endpoint": "sync-all", + "payload": "application/fhir+json", + "_payload": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content", + "valueCode": "full-resource" + } + ] + } + } +} From fcd87c085982dcefb0e9699ba974b235cbfa065d Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Mon, 22 Apr 2024 09:29:47 -0700 Subject: [PATCH 12/47] EventGrid WIP --- Directory.Packages.props | 1 + docs/rest/Subscriptions.http | 10 +++ .../Storage/SqlRetry/SqlRetryService.cs | 2 +- .../Channels/EventGridChannel.cs | 81 +++++++++++++++++++ ...Microsoft.Health.Fhir.Subscriptions.csproj | 4 + 5 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 9df8c6869d..7479c7f93d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -31,6 +31,7 @@ + diff --git a/docs/rest/Subscriptions.http b/docs/rest/Subscriptions.http index 68bcb743e1..cab4f1c4ae 100644 --- a/docs/rest/Subscriptions.http +++ b/docs/rest/Subscriptions.http @@ -176,4 +176,14 @@ Authorization: Bearer {{bearer.response.body.access_token}} ### DELETE https://{{hostname}}/Subscription/example-backport-storage-all content-type: application/json +Authorization: Bearer {{bearer.response.body.access_token}} + +### +DELETE https://{{hostname}}/Subscription/example-backport-storage-patient +content-type: application/json +Authorization: Bearer {{bearer.response.body.access_token}} + +### +DELETE https://{{hostname}}/Subscription/example-backport-lake +content-type: application/json Authorization: Bearer {{bearer.response.body.access_token}} \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs index ac2c53d1e4..a4f4d6c8ba 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs @@ -255,7 +255,7 @@ public async Task ExecuteSql(Func"Flag indicating whether retries are disabled." /// A task representing the asynchronous operation. /// When executing this method, if exception is thrown that is not retriable or if last retry fails, then same exception is thrown by this method. - public async Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false) + public async Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false) { EnsureArg.IsNotNull(sqlCommand, nameof(sqlCommand)); EnsureArg.IsNotNull(action, nameof(action)); diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs new file mode 100644 index 0000000000..9476649097 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs @@ -0,0 +1,81 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure.Messaging.EventGrid; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Channels +{ + [ChannelType(SubscriptionChannelType.EventGrid)] + public class EventGridChannel : ISubscriptionChannel + { + public EventGridChannel() + { + } + + public Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + /* + public EventGridEvent CreateEventGridEvent(ResourceWrapper rcd) + { + EnsureArg.IsNotNull(rcd); + + string resourceId = rcd.ResourceId; + string resourceTypeName = rcd.ResourceTypeName; + string resourceVersion = rcd.Version; + string dataVersion = resourceVersion.ToString(CultureInfo.InvariantCulture); + string fhirAccountDomainName = _workerConfiguration.FhirAccount; + + string eventSubject = GetEventSubject(rcd); + string eventType = _workerConfiguration.ResourceChangeTypeMap[rcd.ResourceChangeTypeId]; + string eventGuid = rcd.GetSha256BasedGuid(); + + // The swagger specification requires the response JSON to have all properties use camelcasing + // and hence the dataPayload properties below have to use camelcase. + var dataPayload = new BinaryData(new + { + resourceType = resourceTypeName, + resourceFhirAccount = fhirAccountDomainName, + resourceFhirId = resourceId, + resourceVersionId = resourceVersion, + }); + + return new EventGridEvent( + subject: eventSubject, + eventType: eventType, + dataVersion: dataVersion, + data: dataPayload) + { + Topic = _workerConfiguration.EventGridTopic, + Id = eventGuid, + EventTime = rcd.Timestamp, + }; + } + + public string GetEventSubject(ResourceChangeData rcd) + { + EnsureArg.IsNotNull(rcd); + + // Example: "myfhirserver.contoso.com/Observation/cb875194-1195-4617-b2e9-0966bd6b8a10" + var fhirAccountDomainName = "fhirevents"; + var subjectSegements = new string[] { fhirAccountDomainName, rcd.ResourceTypeName, rcd.ResourceId }; + var subject = string.Join("/", subjectSegements); + return subject; + } + */ + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj index 13218ee495..7ec3054f0c 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj +++ b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj @@ -1,5 +1,9 @@  + + + + From 4fde508bec244f3bd58c5fec582c855de03216d6 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Wed, 28 Feb 2024 18:06:58 -0800 Subject: [PATCH 13/47] Improve fhirtimer --- .../Storage/SqlRetry/SqlRetryService.cs | 2 +- .../Features/Storage/SqlStoreClient.cs | 24 ++--- .../Watchdogs/CleanupEventLogWatchdog.cs | 25 ++--- .../Features/Watchdogs/DefragWatchdog.cs | 28 +++--- .../Features/Watchdogs/FhirTimer.cs | 73 +++++++++------ .../InvisibleHistoryCleanupWatchdog.cs | 54 ++++++----- .../Features/Watchdogs/TransactionWatchdog.cs | 52 ++++++----- .../Features/Watchdogs/Watchdog.cs | 92 +++++++++++-------- .../Features/Watchdogs/WatchdogLease.cs | 53 +++++++---- .../Watchdogs/WatchdogsBackgroundService.cs | 14 +-- ...erFhirResourceChangeCaptureEnabledTests.cs | 56 +++++++---- .../Persistence/FhirStorageTestsFixture.cs | 3 +- .../Persistence/SqlServerWatchdogTests.cs | 58 ++++++++---- tools/EventsReader/Program.cs | 2 +- 14 files changed, 310 insertions(+), 226 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs index a4f4d6c8ba..a544c7b915 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs @@ -369,7 +369,7 @@ public async Task TryLogEvent(string process, string status, string text, DateTi { try { - using var cmd = new SqlCommand() { CommandType = CommandType.StoredProcedure, CommandText = "dbo.LogEvent" }; + await using var cmd = new SqlCommand { CommandType = CommandType.StoredProcedure, CommandText = "dbo.LogEvent" }; cmd.Parameters.AddWithValue("@Process", process); cmd.Parameters.AddWithValue("@Status", status); cmd.Parameters.AddWithValue("@Text", text); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs index 5d4daf17b3..f6b27e3848 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs @@ -146,7 +146,7 @@ private static Lazy ReadRawResource(SqlDataReader reader, Func> GetResourcesByTransactionIdAsync(long transactionId, Func decompress, Func getResourceTypeName, CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.GetResourcesByTransactionId", CommandType = CommandType.StoredProcedure, CommandTimeout = 600 }; + await using var cmd = new SqlCommand() { CommandText = "dbo.GetResourcesByTransactionId", CommandType = CommandType.StoredProcedure, CommandTimeout = 600 }; cmd.Parameters.AddWithValue("@TransactionId", transactionId); //// ignore invisible resources return (await cmd.ExecuteReaderAsync(_sqlRetryService, (reader) => { return ReadResourceWrapper(reader, true, decompress, getResourceTypeName); }, _logger, cancellationToken)).Where(_ => _.RawResource.Data != _invisibleResource).ToList(); @@ -186,7 +186,7 @@ internal async Task MergeResourcesPutTransactionHeartbeatAsync(long transactionI { try { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesPutTransactionHeartbeat", CommandType = CommandType.StoredProcedure, CommandTimeout = (heartbeatPeriod.Seconds / 3) + 1 }; // +1 to avoid = SQL default timeout value + await using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesPutTransactionHeartbeat", CommandType = CommandType.StoredProcedure, CommandTimeout = (heartbeatPeriod.Seconds / 3) + 1 }; // +1 to avoid = SQL default timeout value cmd.Parameters.AddWithValue("@TransactionId", transactionId); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken); } @@ -209,7 +209,7 @@ private ResourceDateKey ReadResourceDateKeyWrapper(SqlDataReader reader) internal async Task MergeResourcesGetTransactionVisibilityAsync(CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesGetTransactionVisibility", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesGetTransactionVisibility", CommandType = CommandType.StoredProcedure }; var transactionIdParam = new SqlParameter("@TransactionId", SqlDbType.BigInt) { Direction = ParameterDirection.Output }; cmd.Parameters.Add(transactionIdParam); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken); @@ -218,7 +218,7 @@ internal async Task MergeResourcesGetTransactionVisibilityAsync(Cancellati internal async Task<(long TransactionId, int Sequence)> MergeResourcesBeginTransactionAsync(int resourceVersionCount, CancellationToken cancellationToken, DateTime? heartbeatDate = null) { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesBeginTransaction", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesBeginTransaction", CommandType = CommandType.StoredProcedure }; cmd.Parameters.AddWithValue("@Count", resourceVersionCount); var transactionIdParam = new SqlParameter("@TransactionId", SqlDbType.BigInt) { Direction = ParameterDirection.Output }; cmd.Parameters.Add(transactionIdParam); @@ -258,7 +258,7 @@ internal async Task MergeResourcesGetTransactionVisibilityAsync(Cancellati internal async Task MergeResourcesDeleteInvisibleHistory(long transactionId, CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesDeleteInvisibleHistory", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesDeleteInvisibleHistory", CommandType = CommandType.StoredProcedure }; cmd.Parameters.AddWithValue("@TransactionId", transactionId); var affectedRowsParam = new SqlParameter("@affectedRows", SqlDbType.Int) { Direction = ParameterDirection.Output }; cmd.Parameters.Add(affectedRowsParam); @@ -268,7 +268,7 @@ internal async Task MergeResourcesDeleteInvisibleHistory(long transactionId internal async Task MergeResourcesCommitTransactionAsync(long transactionId, string failureReason, CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesCommitTransaction", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesCommitTransaction", CommandType = CommandType.StoredProcedure }; cmd.Parameters.AddWithValue("@TransactionId", transactionId); if (failureReason != null) { @@ -280,14 +280,14 @@ internal async Task MergeResourcesCommitTransactionAsync(long transactionId, str internal async Task MergeResourcesPutTransactionInvisibleHistoryAsync(long transactionId, CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesPutTransactionInvisibleHistory", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesPutTransactionInvisibleHistory", CommandType = CommandType.StoredProcedure }; cmd.Parameters.AddWithValue("@TransactionId", transactionId); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken); } internal async Task MergeResourcesAdvanceTransactionVisibilityAsync(CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesAdvanceTransactionVisibility", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand { CommandText = "dbo.MergeResourcesAdvanceTransactionVisibility", CommandType = CommandType.StoredProcedure }; var affectedRowsParam = new SqlParameter("@AffectedRows", SqlDbType.Int) { Direction = ParameterDirection.Output }; cmd.Parameters.Add(affectedRowsParam); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken); @@ -297,14 +297,14 @@ internal async Task MergeResourcesAdvanceTransactionVisibilityAsync(Cancell internal async Task> MergeResourcesGetTimeoutTransactionsAsync(int timeoutSec, CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesGetTimeoutTransactions", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand { CommandText = "dbo.MergeResourcesGetTimeoutTransactions", CommandType = CommandType.StoredProcedure }; cmd.Parameters.AddWithValue("@TimeoutSec", timeoutSec); - return await cmd.ExecuteReaderAsync(_sqlRetryService, (reader) => { return reader.GetInt64(0); }, _logger, cancellationToken); + return await cmd.ExecuteReaderAsync(_sqlRetryService, reader => reader.GetInt64(0), _logger, cancellationToken); } internal async Task> GetTransactionsAsync(long startNotInclusiveTranId, long endInclusiveTranId, CancellationToken cancellationToken, DateTime? endDate = null) { - using var cmd = new SqlCommand() { CommandText = "dbo.GetTransactions", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand { CommandText = "dbo.GetTransactions", CommandType = CommandType.StoredProcedure }; cmd.Parameters.AddWithValue("@StartNotInclusiveTranId", startNotInclusiveTranId); cmd.Parameters.AddWithValue("@EndInclusiveTranId", endInclusiveTranId); if (endDate.HasValue) @@ -326,7 +326,7 @@ internal async Task> MergeResourcesGetTimeoutTransactionsAsy internal async Task> GetResourceDateKeysByTransactionIdAsync(long transactionId, CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.GetResourcesByTransactionId", CommandType = CommandType.StoredProcedure, CommandTimeout = 600 }; + await using var cmd = new SqlCommand { CommandText = "dbo.GetResourcesByTransactionId", CommandType = CommandType.StoredProcedure, CommandTimeout = 600 }; cmd.Parameters.AddWithValue("@TransactionId", transactionId); cmd.Parameters.AddWithValue("@IncludeHistory", true); cmd.Parameters.AddWithValue("@ReturnResourceKeysOnly", true); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/CleanupEventLogWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/CleanupEventLogWatchdog.cs index 23d1e1b0e5..1b01cafdd7 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/CleanupEventLogWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/CleanupEventLogWatchdog.cs @@ -13,13 +13,10 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - public class CleanupEventLogWatchdog : Watchdog + internal sealed class CleanupEventLogWatchdog : Watchdog { private readonly ISqlRetryService _sqlRetryService; private readonly ILogger _logger; - private CancellationToken _cancellationToken; - private const double _periodSec = 12 * 3600; - private const double _leasePeriodSec = 3600; public CleanupEventLogWatchdog(ISqlRetryService sqlRetryService, ILogger logger) : base(sqlRetryService, logger) @@ -29,25 +26,23 @@ public CleanupEventLogWatchdog(ISqlRetryService sqlRetryService, ILogger + internal sealed class DefragWatchdog : Watchdog { private const byte QueueType = (byte)Core.Features.Operations.QueueType.Defrag; private int _threads; private int _heartbeatPeriodSec; private int _heartbeatTimeoutSec; - private CancellationToken _cancellationToken; private static readonly string[] Definitions = { "Defrag" }; private readonly ISqlRetryService _sqlRetryService; @@ -41,7 +40,6 @@ public DefragWatchdog( } internal DefragWatchdog() - : base() { // this is used to get param names for testing } @@ -54,24 +52,22 @@ internal DefragWatchdog() internal string IsEnabledId => $"{Name}.IsEnabled"; - internal async Task StartAsync(CancellationToken cancellationToken) - { - _cancellationToken = cancellationToken; - await Task.WhenAll( - StartAsync(false, 24 * 3600, 2 * 3600, cancellationToken), - InitDefragParamsAsync()); - } + public override double LeasePeriodSec { get; internal set; } = 2 * 3600; + + public override bool AllowRebalance { get; internal set; } = false; + + public override double PeriodSec { get; internal set; } = 24 * 3600; - protected override async Task ExecuteAsync() + protected override async Task RunWorkAsync(CancellationToken cancellationToken) { - if (!await IsEnabledAsync(_cancellationToken)) + if (!await IsEnabledAsync(cancellationToken)) { _logger.LogInformation("Watchdog is not enabled. Exiting..."); return; } - using var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_cancellationToken); - var job = await GetCoordinatorJobAsync(_cancellationToken); + using var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + (long groupId, long jobId, long version, int activeDefragItems) job = await GetCoordinatorJobAsync(cancellationToken); if (job.jobId == -1) { @@ -123,7 +119,7 @@ await JobHosting.ExecuteJobWithHeartbeatsAsync( TimeSpan.FromSeconds(_heartbeatPeriodSec), cancellationTokenSource); - await CompleteJobAsync(job.jobId, job.version, false, _cancellationToken); + await CompleteJobAsync(job.jobId, job.version, false, cancellationToken); } private async Task ChangeDatabaseSettingsAsync(bool isOn, CancellationToken cancellationToken) @@ -301,7 +297,7 @@ private async Task GetHeartbeatTimeoutAsync(CancellationToken cancellationT return (int)value; } - private async Task InitDefragParamsAsync() // No CancellationToken is passed since we shouldn't cancel initialization. + protected override async Task InitAdditionalParamsAsync() { _logger.LogInformation("InitDefragParamsAsync starting..."); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs index a5bbc7276a..a85499c997 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs @@ -7,55 +7,76 @@ using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; +using EnsureThat; using Microsoft.Extensions.Logging; using Microsoft.Health.Core; using Microsoft.Health.Fhir.Core.Extensions; namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - public abstract class FhirTimer(ILogger logger = null) + public class FhirTimer(ILogger logger = null) { + private bool _active; + private bool _isFailing; - internal double PeriodSec { get; set; } + public double PeriodSec { get; private set; } + + public DateTimeOffset LastRunDateTime { get; private set; } = DateTimeOffset.Parse("2017-12-01"); - internal DateTimeOffset LastRunDateTime { get; private set; } = DateTime.Parse("2017-12-01"); + public bool IsFailing => _isFailing; - internal bool IsFailing => _isFailing; + public bool IsRunning { get; private set; } - protected async Task StartAsync(double periodSec, CancellationToken cancellationToken) + /// + /// Runs the execution of the timer until the is cancelled. + /// + public async Task ExecuteAsync(double periodSec, Func onNextTick, CancellationToken cancellationToken) { + EnsureArg.IsNotNull(onNextTick, nameof(onNextTick)); PeriodSec = periodSec; + if (_active) + { + throw new InvalidOperationException("Timer is already running"); + } + + _active = true; await Task.Delay(TimeSpan.FromSeconds(PeriodSec * RandomNumberGenerator.GetInt32(1000) / 1000), cancellationToken); using var periodicTimer = new PeriodicTimer(TimeSpan.FromSeconds(PeriodSec)); - while (!cancellationToken.IsCancellationRequested) + try { - try - { - await periodicTimer.WaitForNextTickAsync(cancellationToken); - } - catch (OperationCanceledException) + while (!cancellationToken.IsCancellationRequested) { - // Time to exit - break; - } + try + { + await periodicTimer.WaitForNextTickAsync(cancellationToken); + } + catch (OperationCanceledException) + { + break; + } - try - { - await RunAsync(); - LastRunDateTime = Clock.UtcNow; - _isFailing = false; - } - catch (Exception e) - { - logger.LogWarning(e, "Error executing timer"); - _isFailing = true; + try + { + IsRunning = true; + await onNextTick(cancellationToken); + LastRunDateTime = Clock.UtcNow; + _isFailing = false; + } + catch (Exception e) + { + logger?.LogWarning(e, "Error executing timer"); + _isFailing = true; + } } } + finally + { + _active = false; + IsRunning = false; + } } - - protected abstract Task RunAsync(); } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs index eddcded1d8..7efa16a6c7 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -14,13 +15,11 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - internal class InvisibleHistoryCleanupWatchdog : Watchdog + internal sealed class InvisibleHistoryCleanupWatchdog : Watchdog { private readonly SqlStoreClient _store; private readonly ILogger _logger; private readonly ISqlRetryService _sqlRetryService; - private CancellationToken _cancellationToken; - private double _retentionPeriodDays = 7; public InvisibleHistoryCleanupWatchdog(SqlStoreClient store, ISqlRetryService sqlRetryService, ILogger logger) : base(sqlRetryService, logger) @@ -31,48 +30,47 @@ public InvisibleHistoryCleanupWatchdog(SqlStoreClient store, ISqlRetryService sq } internal InvisibleHistoryCleanupWatchdog() - : base() { // this is used to get param names for testing } - internal string LastCleanedUpTransactionId => $"{Name}.LastCleanedUpTransactionId"; + public string LastCleanedUpTransactionId => $"{Name}.LastCleanedUpTransactionId"; - internal async Task StartAsync(CancellationToken cancellationToken, double? periodSec = null, double? leasePeriodSec = null, double? retentionPeriodDays = null) - { - _cancellationToken = cancellationToken; - await InitLastCleanedUpTransactionId(); - await StartAsync(true, periodSec ?? 3600, leasePeriodSec ?? 2 * 3600, cancellationToken); - if (retentionPeriodDays.HasValue) - { - _retentionPeriodDays = retentionPeriodDays.Value; - } - } + public override double LeasePeriodSec { get; internal set; } = 2 * 3600; - protected override async Task ExecuteAsync() + public override bool AllowRebalance { get; internal set; } = true; + + public override double PeriodSec { get; internal set; } = 3600; + + public double RetentionPeriodDays { get; internal set; } = 7; + + protected override async Task RunWorkAsync(CancellationToken cancellationToken) { _logger.LogInformation($"{Name}: starting..."); - var lastTranId = await GetLastCleanedUpTransactionId(); - var visibility = await _store.MergeResourcesGetTransactionVisibilityAsync(_cancellationToken); + var lastTranId = await GetLastCleanedUpTransactionIdAsync(cancellationToken); + var visibility = await _store.MergeResourcesGetTransactionVisibilityAsync(cancellationToken); _logger.LogInformation($"{Name}: last cleaned up transaction={lastTranId} visibility={visibility}."); - var transToClean = await _store.GetTransactionsAsync(lastTranId, visibility, _cancellationToken, DateTime.UtcNow.AddDays((-1) * _retentionPeriodDays)); + IReadOnlyList<(long TransactionId, DateTime? VisibleDate, DateTime? InvisibleHistoryRemovedDate)> transToClean = + await _store.GetTransactionsAsync(lastTranId, visibility, cancellationToken, DateTime.UtcNow.AddDays(-1 * RetentionPeriodDays)); + _logger.LogInformation($"{Name}: found transactions={transToClean.Count}."); if (transToClean.Count == 0) { - _logger.LogInformation($"{Name}: completed. transactions=0."); + _logger.LogDebug($"{Name}: completed. transactions=0."); return; } var totalRows = 0; - foreach (var tran in transToClean.Where(_ => !_.InvisibleHistoryRemovedDate.HasValue).OrderBy(_ => _.TransactionId)) + foreach ((long TransactionId, DateTime? VisibleDate, DateTime? InvisibleHistoryRemovedDate) tran in + transToClean.Where(x => !x.InvisibleHistoryRemovedDate.HasValue).OrderBy(x => x.TransactionId)) { - var rows = await _store.MergeResourcesDeleteInvisibleHistory(tran.TransactionId, _cancellationToken); + var rows = await _store.MergeResourcesDeleteInvisibleHistory(tran.TransactionId, cancellationToken); _logger.LogInformation($"{Name}: transaction={tran.TransactionId} removed rows={rows}."); totalRows += rows; - await _store.MergeResourcesPutTransactionInvisibleHistoryAsync(tran.TransactionId, _cancellationToken); + await _store.MergeResourcesPutTransactionInvisibleHistoryAsync(tran.TransactionId, cancellationToken); } await UpdateLastCleanedUpTransactionId(transToClean.Max(_ => _.TransactionId)); @@ -80,21 +78,21 @@ protected override async Task ExecuteAsync() _logger.LogInformation($"{Name}: completed. transactions={transToClean.Count} removed rows={totalRows}"); } - private async Task GetLastCleanedUpTransactionId() + private async Task GetLastCleanedUpTransactionIdAsync(CancellationToken cancellationToken) { - return await GetLongParameterByIdAsync(LastCleanedUpTransactionId, _cancellationToken); + return await GetLongParameterByIdAsync(LastCleanedUpTransactionId, cancellationToken); } - private async Task InitLastCleanedUpTransactionId() + protected override async Task InitAdditionalParamsAsync() { - using var cmd = new SqlCommand("INSERT INTO dbo.Parameters (Id, Bigint) SELECT @Id, 5105975696064002770"); // surrogate id for the past. does not matter. + await using var cmd = new SqlCommand("INSERT INTO dbo.Parameters (Id, Bigint) SELECT @Id, 5105975696064002770"); // surrogate id for the past. does not matter. cmd.Parameters.AddWithValue("@Id", LastCleanedUpTransactionId); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); } private async Task UpdateLastCleanedUpTransactionId(long lastTranId) { - using var cmd = new SqlCommand("UPDATE dbo.Parameters SET Bigint = @LastTranId WHERE Id = @Id"); + await using var cmd = new SqlCommand("UPDATE dbo.Parameters SET Bigint = @LastTranId WHERE Id = @Id"); cmd.Parameters.AddWithValue("@Id", LastCleanedUpTransactionId); cmd.Parameters.AddWithValue("@LastTranId", lastTranId); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs index 7dd6a7d600..c5c75cefdc 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -15,14 +16,12 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - internal class TransactionWatchdog : Watchdog + internal sealed class TransactionWatchdog : Watchdog { private readonly SqlServerFhirDataStore _store; private readonly IResourceWrapperFactory _factory; private readonly ILogger _logger; - private CancellationToken _cancellationToken; - private const double _periodSec = 3; - private const double _leasePeriodSec = 20; + private const string AdvancedVisibilityTemplate = "TransactionWatchdog advanced visibility on {Transactions} transactions."; public TransactionWatchdog(SqlServerFhirDataStore store, IResourceWrapperFactory factory, ISqlRetryService sqlRetryService, ILogger logger) : base(sqlRetryService, logger) @@ -33,49 +32,54 @@ public TransactionWatchdog(SqlServerFhirDataStore store, IResourceWrapperFactory } internal TransactionWatchdog() - : base() { // this is used to get param names for testing } - internal async Task StartAsync(CancellationToken cancellationToken) - { - _cancellationToken = cancellationToken; - await StartAsync(true, _periodSec, _leasePeriodSec, cancellationToken); - } + public override double LeasePeriodSec { get; internal set; } = 20; + + public override bool AllowRebalance { get; internal set; } = true; - protected override async Task ExecuteAsync() + public override double PeriodSec { get; internal set; } = 3; + + protected override async Task RunWorkAsync(CancellationToken cancellationToken) { - _logger.LogInformation("TransactionWatchdog starting..."); - var affectedRows = await _store.StoreClient.MergeResourcesAdvanceTransactionVisibilityAsync(_cancellationToken); - _logger.LogInformation("TransactionWatchdog advanced visibility on {Transactions} transactions.", affectedRows); + var affectedRows = await _store.StoreClient.MergeResourcesAdvanceTransactionVisibilityAsync(cancellationToken); + + _logger.Log( + affectedRows > 0 ? LogLevel.Information : LogLevel.Debug, + AdvancedVisibilityTemplate, + affectedRows); if (affectedRows > 0) { return; } - var timeoutTransactions = await _store.StoreClient.MergeResourcesGetTimeoutTransactionsAsync((int)SqlServerFhirDataStore.MergeResourcesTransactionHeartbeatPeriod.TotalSeconds * 6, _cancellationToken); + IReadOnlyList timeoutTransactions = await _store.StoreClient.MergeResourcesGetTimeoutTransactionsAsync((int)SqlServerFhirDataStore.MergeResourcesTransactionHeartbeatPeriod.TotalSeconds * 6, cancellationToken); if (timeoutTransactions.Count > 0) { _logger.LogWarning("TransactionWatchdog found {Transactions} timed out transactions", timeoutTransactions.Count); - await _store.StoreClient.TryLogEvent("TransactionWatchdog", "Warn", $"found timed out transactions={timeoutTransactions.Count}", null, _cancellationToken); + await _store.StoreClient.TryLogEvent("TransactionWatchdog", "Warn", $"found timed out transactions={timeoutTransactions.Count}", null, cancellationToken); } else { - _logger.LogInformation("TransactionWatchdog found {Transactions} timed out transactions", timeoutTransactions.Count); + _logger.Log( + timeoutTransactions.Count > 0 ? LogLevel.Information : LogLevel.Debug, + "TransactionWatchdog found {Transactions} timed out transactions", + timeoutTransactions.Count); } foreach (var tranId in timeoutTransactions) { var st = DateTime.UtcNow; _logger.LogInformation("TransactionWatchdog found timed out transaction={Transaction}, attempting to roll forward...", tranId); - var resources = await _store.GetResourcesByTransactionIdAsync(tranId, _cancellationToken); + var resources = await _store.GetResourcesByTransactionIdAsync(tranId, cancellationToken); if (resources.Count == 0) { - await _store.StoreClient.MergeResourcesCommitTransactionAsync(tranId, "WD: 0 resources", _cancellationToken); + await _store.StoreClient.MergeResourcesCommitTransactionAsync(tranId, "WD: 0 resources", cancellationToken); _logger.LogWarning("TransactionWatchdog committed transaction={Transaction}, resources=0", tranId); - await _store.StoreClient.TryLogEvent("TransactionWatchdog", "Warn", $"committed transaction={tranId}, resources=0", st, _cancellationToken); + await _store.StoreClient.TryLogEvent("TransactionWatchdog", "Warn", $"committed transaction={tranId}, resources=0", st, cancellationToken); continue; } @@ -84,12 +88,12 @@ protected override async Task ExecuteAsync() _factory.Update(resource); } - await _store.MergeResourcesWrapperAsync(tranId, false, resources.Select(_ => new MergeResourceWrapper(_, true, true)).ToList(), false, 0, _cancellationToken); - await _store.StoreClient.MergeResourcesCommitTransactionAsync(tranId, null, _cancellationToken); + await _store.MergeResourcesWrapperAsync(tranId, false, resources.Select(x => new MergeResourceWrapper(x, true, true)).ToList(), false, 0, cancellationToken); + await _store.StoreClient.MergeResourcesCommitTransactionAsync(tranId, null, cancellationToken); _logger.LogWarning("TransactionWatchdog committed transaction={Transaction}, resources={Resources}", tranId, resources.Count); - await _store.StoreClient.TryLogEvent("TransactionWatchdog", "Warn", $"committed transaction={tranId}, resources={resources.Count}", st, _cancellationToken); + await _store.StoreClient.TryLogEvent("TransactionWatchdog", "Warn", $"committed transaction={tranId}, resources={resources.Count}", st, cancellationToken); - affectedRows = await _store.StoreClient.MergeResourcesAdvanceTransactionVisibilityAsync(_cancellationToken); + affectedRows = await _store.StoreClient.MergeResourcesAdvanceTransactionVisibilityAsync(cancellationToken); _logger.LogInformation("TransactionWatchdog advanced visibility on {Transactions} transactions.", affectedRows); } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs index 95d2917234..19de170bff 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs @@ -10,24 +10,27 @@ using EnsureThat; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; +using Microsoft.Health.Core; using Microsoft.Health.Fhir.SqlServer.Features.Storage; namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - public abstract class Watchdog : FhirTimer + internal abstract class Watchdog + where T : Watchdog { - private ISqlRetryService _sqlRetryService; + private readonly ISqlRetryService _sqlRetryService; private readonly ILogger _logger; private readonly WatchdogLease _watchdogLease; private double _periodSec; private double _leasePeriodSec; + private readonly FhirTimer _fhirTimer; protected Watchdog(ISqlRetryService sqlRetryService, ILogger logger) - : base(logger) { _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); _watchdogLease = new WatchdogLease(_sqlRetryService, _logger); + _fhirTimer = new FhirTimer(_logger); } protected Watchdog() @@ -35,66 +38,83 @@ protected Watchdog() // this is used to get param names for testing } - internal string Name => GetType().Name; + public string Name => GetType().Name; - internal string PeriodSecId => $"{Name}.PeriodSec"; + public string PeriodSecId => $"{Name}.PeriodSec"; - internal string LeasePeriodSecId => $"{Name}.LeasePeriodSec"; + public string LeasePeriodSecId => $"{Name}.LeasePeriodSec"; - internal bool IsLeaseHolder => _watchdogLease.IsLeaseHolder; + public bool IsLeaseHolder => _watchdogLease.IsLeaseHolder; - internal string LeaseWorker => _watchdogLease.Worker; + public string LeaseWorker => _watchdogLease.Worker; - internal double LeasePeriodSec => _watchdogLease.PeriodSec; + public abstract double LeasePeriodSec { get; internal set; } - protected internal async Task StartAsync(bool allowRebalance, double periodSec, double leasePeriodSec, CancellationToken cancellationToken) + public abstract bool AllowRebalance { get; internal set; } + + public abstract double PeriodSec { get; internal set; } + + public bool IsInitialized { get; private set; } + + public async Task ExecuteAsync(CancellationToken cancellationToken) { - _logger.LogInformation($"{Name}.StartAsync: starting..."); - await InitParamsAsync(periodSec, leasePeriodSec); + _logger.LogInformation($"{Name}.ExecuteAsync: starting..."); + + await InitParamsAsync(PeriodSec, LeasePeriodSec); await Task.WhenAll( - StartAsync(_periodSec, cancellationToken), - _watchdogLease.StartAsync(allowRebalance, _leasePeriodSec, cancellationToken)); + _fhirTimer.ExecuteAsync(_periodSec, OnNextTickAsync, cancellationToken), + _watchdogLease.ExecuteAsync(AllowRebalance, _leasePeriodSec, cancellationToken)); - _logger.LogInformation($"{Name}.StartAsync: completed."); + _logger.LogInformation($"{Name}.ExecuteAsync: completed."); } - protected abstract Task ExecuteAsync(); + protected abstract Task RunWorkAsync(CancellationToken cancellationToken); - protected override async Task RunAsync() + private async Task OnNextTickAsync(CancellationToken cancellationToken) { if (!_watchdogLease.IsLeaseHolder) { - _logger.LogInformation($"{Name}.RunAsync: Skipping because watchdog is not a lease holder."); + _logger.LogDebug($"{Name}.OnNextTickAsync: Skipping because watchdog is not a lease holder."); return; } - _logger.LogInformation($"{Name}.RunAsync: Starting..."); - await ExecuteAsync(); - _logger.LogInformation($"{Name}.RunAsync: Completed."); + using (_logger.BeginTimedScope($"{Name}.OnNextTickAsync")) + { + await RunWorkAsync(cancellationToken); + } } private async Task InitParamsAsync(double periodSec, double leasePeriodSec) // No CancellationToken is passed since we shouldn't cancel initialization. { - _logger.LogInformation($"{Name}.InitParamsAsync: starting..."); - - // Offset for other instances running init - await Task.Delay(TimeSpan.FromSeconds(RandomNumberGenerator.GetInt32(10) / 10.0), CancellationToken.None); + using (_logger.BeginTimedScope($"{Name}.InitParamsAsync")) + { + // Offset for other instances running init + await Task.Delay(TimeSpan.FromSeconds(RandomNumberGenerator.GetInt32(10) / 10.0), CancellationToken.None); - using var cmd = new SqlCommand(@" + await using var cmd = new SqlCommand( + @" INSERT INTO dbo.Parameters (Id,Number) SELECT @PeriodSecId, @PeriodSec INSERT INTO dbo.Parameters (Id,Number) SELECT @LeasePeriodSecId, @LeasePeriodSec "); - cmd.Parameters.AddWithValue("@PeriodSecId", PeriodSecId); - cmd.Parameters.AddWithValue("@PeriodSec", periodSec); - cmd.Parameters.AddWithValue("@LeasePeriodSecId", LeasePeriodSecId); - cmd.Parameters.AddWithValue("@LeasePeriodSec", leasePeriodSec); - await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); + cmd.Parameters.AddWithValue("@PeriodSecId", PeriodSecId); + cmd.Parameters.AddWithValue("@PeriodSec", periodSec); + cmd.Parameters.AddWithValue("@LeasePeriodSecId", LeasePeriodSecId); + cmd.Parameters.AddWithValue("@LeasePeriodSec", leasePeriodSec); + await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); + + _periodSec = await GetPeriodAsync(CancellationToken.None); + _leasePeriodSec = await GetLeasePeriodAsync(CancellationToken.None); + + await InitAdditionalParamsAsync(); - _periodSec = await GetPeriodAsync(CancellationToken.None); - _leasePeriodSec = await GetLeasePeriodAsync(CancellationToken.None); + IsInitialized = true; + } + } - _logger.LogInformation($"{Name}.InitParamsAsync: completed."); + protected virtual Task InitAdditionalParamsAsync() + { + return Task.CompletedTask; } private async Task GetPeriodAsync(CancellationToken cancellationToken) @@ -113,7 +133,7 @@ protected async Task GetNumberParameterByIdAsync(string id, Cancellation { EnsureArg.IsNotNullOrEmpty(id, nameof(id)); - using var cmd = new SqlCommand("SELECT Number FROM dbo.Parameters WHERE Id = @Id"); + await using var cmd = new SqlCommand("SELECT Number FROM dbo.Parameters WHERE Id = @Id"); cmd.Parameters.AddWithValue("@Id", id); var value = await cmd.ExecuteScalarAsync(_sqlRetryService, _logger, cancellationToken); @@ -129,7 +149,7 @@ protected async Task GetLongParameterByIdAsync(string id, CancellationToke { EnsureArg.IsNotNullOrEmpty(id, nameof(id)); - using var cmd = new SqlCommand("SELECT Bigint FROM dbo.Parameters WHERE Id = @Id"); + await using var cmd = new SqlCommand("SELECT Bigint FROM dbo.Parameters WHERE Id = @Id"); cmd.Parameters.AddWithValue("@Id", id); var value = await cmd.ExecuteScalarAsync(_sqlRetryService, _logger, cancellationToken); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs index ee4a595cd5..23d4c5bea4 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs @@ -10,81 +10,94 @@ using EnsureThat; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; +using Microsoft.Health.Core; +using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.SqlServer.Features.Storage; namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - internal class WatchdogLease : FhirTimer + internal class WatchdogLease + where T : Watchdog { private const double TimeoutFactor = 0.25; private readonly object _locker = new(); private readonly ISqlRetryService _sqlRetryService; - private readonly ILogger _logger; - private DateTime _leaseEndTime; + private readonly ILogger _logger; + private DateTimeOffset _leaseEndTime; private double _leaseTimeoutSec; private readonly string _worker; - private CancellationToken _cancellationToken; private readonly string _watchdogName; private bool _allowRebalance; + private readonly FhirTimer _fhirTimer; - internal WatchdogLease(ISqlRetryService sqlRetryService, ILogger logger) - : base(logger) + public WatchdogLease(ISqlRetryService sqlRetryService, ILogger logger) { _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); _watchdogName = typeof(T).Name; _worker = $"{Environment.MachineName}.{Environment.ProcessId}"; _logger.LogInformation($"WatchdogLease:Created lease object, worker=[{_worker}]."); + _fhirTimer = new FhirTimer(logger); } - protected internal string Worker => _worker; + public string Worker => _worker; - protected internal bool IsLeaseHolder + public bool IsLeaseHolder { get { lock (_locker) { - return (DateTime.UtcNow - _leaseEndTime).TotalSeconds < _leaseTimeoutSec; + return (Clock.UtcNow - _leaseEndTime).TotalSeconds < _leaseTimeoutSec; } } } - protected internal async Task StartAsync(bool allowRebalance, double periodSec, CancellationToken cancellationToken) + public bool IsRunning => _fhirTimer.IsRunning; + + public double PeriodSec => _fhirTimer.PeriodSec; + + public async Task ExecuteAsync(bool allowRebalance, double periodSec, CancellationToken cancellationToken) { _logger.LogInformation("WatchdogLease.StartAsync: starting..."); + _allowRebalance = allowRebalance; - _cancellationToken = cancellationToken; - _leaseEndTime = DateTime.MinValue; + _leaseEndTime = DateTimeOffset.MinValue; _leaseTimeoutSec = (int)Math.Ceiling(periodSec * TimeoutFactor); // if it is rounded to 0 it causes problems in AcquireResourceLease logic. - await StartAsync(periodSec, cancellationToken); + + await _fhirTimer.ExecuteAsync(periodSec, OnNextTickAsync, cancellationToken); + _logger.LogInformation("WatchdogLease.StartAsync: completed."); } - protected override async Task RunAsync() + protected async Task OnNextTickAsync(CancellationToken cancellationToken) { - _logger.LogInformation($"WatchdogLease.RunAsync: Starting acquire: resource=[{_watchdogName}] worker=[{_worker}] period={PeriodSec} timeout={_leaseTimeoutSec}..."); + _logger.LogInformation($"WatchdogLease.RunAsync: Starting acquire: resource=[{_watchdogName}] worker=[{_worker}] period={_fhirTimer.PeriodSec} timeout={_leaseTimeoutSec}..."); - using var cmd = new SqlCommand("dbo.AcquireWatchdogLease") { CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand("dbo.AcquireWatchdogLease") { CommandType = CommandType.StoredProcedure }; cmd.Parameters.AddWithValue("@Watchdog", _watchdogName); cmd.Parameters.AddWithValue("@Worker", _worker); cmd.Parameters.AddWithValue("@AllowRebalance", _allowRebalance); cmd.Parameters.AddWithValue("@WorkerIsRunning", true); cmd.Parameters.AddWithValue("@ForceAcquire", false); // TODO: Provide ability to set. usefull for tests cmd.Parameters.AddWithValue("@LeasePeriodSec", PeriodSec + _leaseTimeoutSec); - var leaseEndTimePar = cmd.Parameters.AddWithValue("@LeaseEndTime", DateTime.UtcNow); + + SqlParameter leaseEndTimePar = cmd.Parameters.AddWithValue("@LeaseEndTime", DateTime.UtcNow); leaseEndTimePar.Direction = ParameterDirection.Output; - var isAcquiredPar = cmd.Parameters.AddWithValue("@IsAcquired", false); + + SqlParameter isAcquiredPar = cmd.Parameters.AddWithValue("@IsAcquired", false); isAcquiredPar.Direction = ParameterDirection.Output; - var currentHolderPar = cmd.Parameters.Add("@CurrentLeaseHolder", SqlDbType.VarChar, 100); + + SqlParameter currentHolderPar = cmd.Parameters.Add("@CurrentLeaseHolder", SqlDbType.VarChar, 100); currentHolderPar.Direction = ParameterDirection.Output; - await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, _cancellationToken); + await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken); var leaseEndTime = (DateTime)leaseEndTimePar.Value; var isAcquired = (bool)isAcquiredPar.Value; var currentHolder = (string)currentHolderPar.Value; + lock (_locker) { _leaseEndTime = isAcquired ? leaseEndTime : _leaseEndTime; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs index 2e411bd8ac..8d1ceae091 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs @@ -24,20 +24,17 @@ internal class WatchdogsBackgroundService : BackgroundService, INotificationHand private readonly CleanupEventLogWatchdog _cleanupEventLogWatchdog; private readonly IScoped _transactionWatchdog; private readonly InvisibleHistoryCleanupWatchdog _invisibleHistoryCleanupWatchdog; - private readonly SubscriptionProcessorWatchdog _subscriptionsProcessorWatchdog; public WatchdogsBackgroundService( DefragWatchdog defragWatchdog, CleanupEventLogWatchdog cleanupEventLogWatchdog, IScopeProvider transactionWatchdog, - InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog, - SubscriptionProcessorWatchdog eventProcessorWatchdog) + InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog) { _defragWatchdog = EnsureArg.IsNotNull(defragWatchdog, nameof(defragWatchdog)); _cleanupEventLogWatchdog = EnsureArg.IsNotNull(cleanupEventLogWatchdog, nameof(cleanupEventLogWatchdog)); _transactionWatchdog = EnsureArg.IsNotNull(transactionWatchdog, nameof(transactionWatchdog)).Invoke(); _invisibleHistoryCleanupWatchdog = EnsureArg.IsNotNull(invisibleHistoryCleanupWatchdog, nameof(invisibleHistoryCleanupWatchdog)); - _subscriptionsProcessorWatchdog = EnsureArg.IsNotNull(eventProcessorWatchdog, nameof(eventProcessorWatchdog)); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -52,11 +49,10 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) var tasks = new List { - _defragWatchdog.StartAsync(continuationTokenSource.Token), - _cleanupEventLogWatchdog.StartAsync(continuationTokenSource.Token), - _transactionWatchdog.Value.StartAsync(continuationTokenSource.Token), - _invisibleHistoryCleanupWatchdog.StartAsync(continuationTokenSource.Token), - _subscriptionsProcessorWatchdog.StartAsync(continuationTokenSource.Token), + _defragWatchdog.ExecuteAsync(continuationTokenSource.Token), + _cleanupEventLogWatchdog.ExecuteAsync(continuationTokenSource.Token), + _transactionWatchdog.Value.ExecuteAsync(continuationTokenSource.Token), + _invisibleHistoryCleanupWatchdog.ExecuteAsync(continuationTokenSource.Token), }; await Task.WhenAny(tasks); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs index 9655ceaa84..0ae103f3a2 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs @@ -93,7 +93,7 @@ public async Task GivenADatabaseSupportsResourceChangeCapture_WhenUpdatingAResou Assert.NotNull(resourceChanges); Assert.Single(resourceChanges.Where(x => x.ResourceVersion.ToString() == deserialized.VersionId && x.ResourceId == deserialized.Id)); - var resourceChangeData = resourceChanges.Where(x => x.ResourceVersion.ToString() == deserialized.VersionId && x.ResourceId == deserialized.Id).FirstOrDefault(); + var resourceChangeData = resourceChanges.FirstOrDefault(x => x.ResourceVersion.ToString() == deserialized.VersionId && x.ResourceId == deserialized.Id); Assert.NotNull(resourceChangeData); Assert.Equal(ResourceChangeTypeUpdated, resourceChangeData.ResourceChangeTypeId); @@ -141,23 +141,30 @@ public async Task GivenChangeCaptureEnabledAndNoVersionPolicy_AfterUpdating_Invi cts.CancelAfter(TimeSpan.FromSeconds(60)); var storeClient = new SqlStoreClient(_fixture.SqlRetryService, NullLogger.Instance); - var wd = new InvisibleHistoryCleanupWatchdog(storeClient, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)); - await wd.StartAsync(cts.Token, 1, 2, 2.0 / 24 / 3600); // retention 2 seconds + var wd = new InvisibleHistoryCleanupWatchdog(storeClient, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)) + { + PeriodSec = 1, + LeasePeriodSec = 2, + RetentionPeriodDays = 2.0 / 24 / 3600, + }; + + var wdTask = wd.ExecuteAsync(cts.Token); // retention 2 seconds var startTime = DateTime.UtcNow; - while (!wd.IsLeaseHolder && (DateTime.UtcNow - startTime).TotalSeconds < 60) + + while ((!wd.IsLeaseHolder || !wd.IsInitialized) && !cts.IsCancellationRequested) { - await Task.Delay(TimeSpan.FromSeconds(0.2)); + await Task.Delay(TimeSpan.FromSeconds(0.2), cts.Token); } Assert.True(wd.IsLeaseHolder, "Is lease holder"); _testOutputHelper.WriteLine($"Acquired lease in {(DateTime.UtcNow - startTime).TotalSeconds} seconds."); // create 2 records (1 invisible) - var create = await _fixture.Mediator.CreateResourceAsync(Samples.GetDefaultOrganization()); + var create = await _fixture.Mediator.CreateResourceAsync(Samples.GetDefaultOrganization(), cancellationToken: cts.Token); Assert.Equal("1", create.VersionId); var newValue = Samples.GetDefaultOrganization().UpdateId(create.Id); - newValue.ToPoco().Text = new Hl7.Fhir.Model.Narrative { Status = Hl7.Fhir.Model.Narrative.NarrativeStatus.Generated, Div = $"
Whatever
" }; - var update = await _fixture.Mediator.UpsertResourceAsync(newValue); + newValue.ToPoco().Text = new Hl7.Fhir.Model.Narrative { Status = Hl7.Fhir.Model.Narrative.NarrativeStatus.Generated, Div = "
Whatever
" }; + var update = await _fixture.Mediator.UpsertResourceAsync(newValue, cancellationToken: cts.Token); Assert.Equal("2", update.RawResourceElement.VersionId); // check 2 records exist @@ -166,14 +173,15 @@ public async Task GivenChangeCaptureEnabledAndNoVersionPolicy_AfterUpdating_Invi await store.StoreClient.MergeResourcesAdvanceTransactionVisibilityAsync(CancellationToken.None); // this logic is invoked by WD normally // check only 1 record remains - startTime = DateTime.UtcNow; - while (await GetCount() != 1 && (DateTime.UtcNow - startTime).TotalSeconds < 60) + while (await GetCount() != 1 && !cts.IsCancellationRequested) { - await Task.Delay(TimeSpan.FromSeconds(1)); + await Task.Delay(TimeSpan.FromSeconds(1), cts.Token); } Assert.Equal(1, await GetCount()); DisableInvisibleHistory(); + await cts.CancelAsync(); + await wdTask; } [Fact] @@ -189,19 +197,25 @@ public async Task GivenChangeCaptureEnabledAndNoVersionPolicy_AfterHardDeleting_ cts.CancelAfter(TimeSpan.FromSeconds(60)); var storeClient = new SqlStoreClient(_fixture.SqlRetryService, NullLogger.Instance); - var wd = new InvisibleHistoryCleanupWatchdog(storeClient, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)); - await wd.StartAsync(cts.Token, 1, 2, 2.0 / 24 / 3600); // retention 2 seconds + var wd = new InvisibleHistoryCleanupWatchdog(storeClient, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)) + { + PeriodSec = 1, + LeasePeriodSec = 2, + RetentionPeriodDays = 2.0 / 24 / 3600, + }; + + var wdTask = wd.ExecuteAsync(cts.Token); // retention 2 seconds var startTime = DateTime.UtcNow; - while (!wd.IsLeaseHolder && (DateTime.UtcNow - startTime).TotalSeconds < 60) + while ((!wd.IsLeaseHolder || !wd.IsInitialized) && !cts.IsCancellationRequested) { - await Task.Delay(TimeSpan.FromSeconds(0.2)); + await Task.Delay(TimeSpan.FromSeconds(0.2), cts.Token); } Assert.True(wd.IsLeaseHolder, "Is lease holder"); _testOutputHelper.WriteLine($"Acquired lease in {(DateTime.UtcNow - startTime).TotalSeconds} seconds."); // create 1 resource and hard delete it - var create = await _fixture.Mediator.CreateResourceAsync(Samples.GetDefaultOrganization()); + var create = await _fixture.Mediator.CreateResourceAsync(Samples.GetDefaultOrganization(), CancellationToken.None); Assert.Equal("1", create.VersionId); var resource = await store.GetAsync(new ResourceKey("Organization", create.Id, create.VersionId), CancellationToken.None); @@ -218,14 +232,16 @@ public async Task GivenChangeCaptureEnabledAndNoVersionPolicy_AfterHardDeleting_ await store.StoreClient.MergeResourcesAdvanceTransactionVisibilityAsync(CancellationToken.None); // this logic is invoked by WD normally // check no records - startTime = DateTime.UtcNow; - while (await GetCount() > 0 && (DateTime.UtcNow - startTime).TotalSeconds < 60) + while (await GetCount() > 0 && !cts.IsCancellationRequested) { - await Task.Delay(TimeSpan.FromSeconds(1)); + await Task.Delay(TimeSpan.FromSeconds(1), cts.Token); } Assert.Equal(0, await GetCount()); DisableInvisibleHistory(); + + await cts.CancelAsync(); + await wdTask; } [Fact] @@ -364,7 +380,7 @@ public async Task GivenADatabaseSupportsResourceChangeCapture_WhenDeletingAResou Assert.NotNull(resourceChanges); Assert.Single(resourceChanges.Where(x => x.ResourceVersion.ToString() == deletedResourceKey.ResourceKey.VersionId && x.ResourceId == deletedResourceKey.ResourceKey.Id)); - var resourceChangeData = resourceChanges.Where(x => x.ResourceVersion.ToString() == deletedResourceKey.ResourceKey.VersionId && x.ResourceId == deletedResourceKey.ResourceKey.Id).FirstOrDefault(); + var resourceChangeData = resourceChanges.FirstOrDefault(x => x.ResourceVersion.ToString() == deletedResourceKey.ResourceKey.VersionId && x.ResourceId == deletedResourceKey.ResourceKey.Id); Assert.NotNull(resourceChangeData); Assert.Equal(ResourceChangeTypeDeleted, resourceChangeData.ResourceChangeTypeId); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs index 4d94970cb4..edb3758be1 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Net.Http; +using System.Threading; using Hl7.Fhir.ElementModel; using Hl7.Fhir.Model; using Hl7.Fhir.Serialization; @@ -160,7 +161,7 @@ public async Task InitializeAsync() medicationResource.Versioning = CapabilityStatement.ResourceVersionPolicy.VersionedUpdate; ConformanceProvider = Substitute.For(); - ConformanceProvider.GetCapabilityStatementOnStartup().Returns(CapabilityStatement.ToTypedElement().ToResourceElement()); + ConformanceProvider.GetCapabilityStatementOnStartup(Arg.Any()).Returns(CapabilityStatement.ToTypedElement().ToResourceElement()); // TODO: FhirRepository instantiate ResourceDeserializer class directly // which will try to deserialize the raw resource. We should mock it as well. diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs index 127eb7a94b..39d579859c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs @@ -77,12 +77,12 @@ COMMIT TRANSACTION using var cts = new CancellationTokenSource(); cts.CancelAfter(TimeSpan.FromMinutes(10)); - await wd.StartAsync(cts.Token); + Task wsTask = wd.ExecuteAsync(cts.Token); - var startTime = DateTime.UtcNow; + DateTime startTime = DateTime.UtcNow; while (!wd.IsLeaseHolder && (DateTime.UtcNow - startTime).TotalSeconds < 60) { - await Task.Delay(TimeSpan.FromSeconds(1)); + await Task.Delay(TimeSpan.FromSeconds(1), cts.Token); } Assert.True(wd.IsLeaseHolder, "Is lease holder"); @@ -90,7 +90,7 @@ COMMIT TRANSACTION var completed = CheckQueue(current); while (!completed && (DateTime.UtcNow - startTime).TotalSeconds < 120) { - await Task.Delay(TimeSpan.FromSeconds(1)); + await Task.Delay(TimeSpan.FromSeconds(1), cts.Token); completed = CheckQueue(current); } @@ -99,6 +99,9 @@ COMMIT TRANSACTION var sizeAfter = GetSize(); Assert.True(sizeAfter * 9 < sizeBefore, $"{sizeAfter} * 9 < {sizeBefore}"); + + await cts.CancelAsync(); + await wsTask; } [Fact] @@ -122,12 +125,12 @@ WHILE @i < 10000 using var cts = new CancellationTokenSource(); cts.CancelAfter(TimeSpan.FromMinutes(10)); - await wd.StartAsync(cts.Token); + Task wdTask = wd.ExecuteAsync(cts.Token); var startTime = DateTime.UtcNow; while (!wd.IsLeaseHolder && (DateTime.UtcNow - startTime).TotalSeconds < 60) { - await Task.Delay(TimeSpan.FromSeconds(1)); + await Task.Delay(TimeSpan.FromSeconds(1), cts.Token); } Assert.True(wd.IsLeaseHolder, "Is lease holder"); @@ -135,11 +138,14 @@ WHILE @i < 10000 while ((GetCount("EventLog") > 1000) && (DateTime.UtcNow - startTime).TotalSeconds < 120) { - await Task.Delay(TimeSpan.FromSeconds(1)); + await Task.Delay(TimeSpan.FromSeconds(1), cts.Token); } _testOutputHelper.WriteLine($"EventLog.Count={GetCount("EventLog")}."); Assert.True(GetCount("EventLog") <= 1000, "Count is low"); + + await cts.CancelAsync(); + await wdTask; } [Fact] @@ -187,12 +193,18 @@ FOR INSERT ExecuteSql("DROP TRIGGER dbo.tmp_NumberSearchParam"); - var wd = new TransactionWatchdog(_fixture.SqlServerFhirDataStore, factory, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)); - await wd.StartAsync(true, 1, 2, cts.Token); - var startTime = DateTime.UtcNow; + var wd = new TransactionWatchdog(_fixture.SqlServerFhirDataStore, factory, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)) + { + AllowRebalance = true, + PeriodSec = 1, + LeasePeriodSec = 2, + }; + + Task wdTask = wd.ExecuteAsync(cts.Token); + DateTime startTime = DateTime.UtcNow; while (!wd.IsLeaseHolder && (DateTime.UtcNow - startTime).TotalSeconds < 10) { - await Task.Delay(TimeSpan.FromSeconds(0.2)); + await Task.Delay(TimeSpan.FromSeconds(0.2), cts.Token); } Assert.True(wd.IsLeaseHolder, "Is lease holder"); @@ -205,6 +217,9 @@ FOR INSERT } Assert.Equal(1, GetCount("NumberSearchParam")); // wd rolled forward transaction + + await cts.CancelAsync(); + await wdTask; } [Fact] @@ -215,12 +230,18 @@ public async Task AdvanceVisibility() using var cts = new CancellationTokenSource(); cts.CancelAfter(TimeSpan.FromSeconds(60)); - var wd = new TransactionWatchdog(_fixture.SqlServerFhirDataStore, CreateResourceWrapperFactory(), _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)); - await wd.StartAsync(true, 1, 2, cts.Token); + var wd = new TransactionWatchdog(_fixture.SqlServerFhirDataStore, CreateResourceWrapperFactory(), _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)) + { + AllowRebalance = true, + PeriodSec = 1, + LeasePeriodSec = 2, + }; + + Task wdTask = wd.ExecuteAsync(cts.Token); var startTime = DateTime.UtcNow; while (!wd.IsLeaseHolder && (DateTime.UtcNow - startTime).TotalSeconds < 10) { - await Task.Delay(TimeSpan.FromSeconds(0.2)); + await Task.Delay(TimeSpan.FromSeconds(0.2), cts.Token); } Assert.True(wd.IsLeaseHolder, "Is lease holder"); @@ -241,7 +262,7 @@ public async Task AdvanceVisibility() startTime = DateTime.UtcNow; while ((visibility = await _fixture.SqlServerFhirDataStore.StoreClient.MergeResourcesGetTransactionVisibilityAsync(cts.Token)) != tran1.TransactionId && (DateTime.UtcNow - startTime).TotalSeconds < 10) { - await Task.Delay(TimeSpan.FromSeconds(0.1)); + await Task.Delay(TimeSpan.FromSeconds(0.1), cts.Token); } _testOutputHelper.WriteLine($"Visibility={visibility}"); @@ -254,7 +275,7 @@ public async Task AdvanceVisibility() startTime = DateTime.UtcNow; while ((visibility = await _fixture.SqlServerFhirDataStore.StoreClient.MergeResourcesGetTransactionVisibilityAsync(cts.Token)) != tran2.TransactionId && (DateTime.UtcNow - startTime).TotalSeconds < 10) { - await Task.Delay(TimeSpan.FromSeconds(0.1)); + await Task.Delay(TimeSpan.FromSeconds(0.1), cts.Token); } _testOutputHelper.WriteLine($"Visibility={visibility}"); @@ -267,11 +288,14 @@ public async Task AdvanceVisibility() startTime = DateTime.UtcNow; while ((visibility = await _fixture.SqlServerFhirDataStore.StoreClient.MergeResourcesGetTransactionVisibilityAsync(cts.Token)) != tran3.TransactionId && (DateTime.UtcNow - startTime).TotalSeconds < 10) { - await Task.Delay(TimeSpan.FromSeconds(0.1)); + await Task.Delay(TimeSpan.FromSeconds(0.1), cts.Token); } _testOutputHelper.WriteLine($"Visibility={visibility}"); Assert.Equal(tran3.TransactionId, visibility); + + await cts.CancelAsync(); + await wdTask; } private ResourceWrapperFactory CreateResourceWrapperFactory() diff --git a/tools/EventsReader/Program.cs b/tools/EventsReader/Program.cs index d6dcf988d1..32cd8a3dd2 100644 --- a/tools/EventsReader/Program.cs +++ b/tools/EventsReader/Program.cs @@ -16,7 +16,7 @@ public static class Program { private static readonly string _connectionString = ConfigurationManager.ConnectionStrings["Database"].ConnectionString; private static SqlRetryService _sqlRetryService; - private static SqlStoreClient _store; + private static SqlStoreClient _store; private static string _parameterId = "Events.LastProcessedTransactionId"; public static void Main() From c1a084e75d3ce7f51386b045880b8c458a88cdea Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Tue, 16 Jul 2024 16:02:41 -0700 Subject: [PATCH 14/47] Fixes for subscriptionwatchdog --- global.json | 2 +- .../SubscriptionProcessorWatchdog.cs | 27 ++++++++++--------- .../Watchdogs/WatchdogsBackgroundService.cs | 6 ++++- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/global.json b/global.json index 789477d342..7cc48a4e22 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "8.0.204" + "version": "8.0.303" } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs index 8dc1f8e829..5e40f22de3 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs @@ -28,7 +28,6 @@ internal class SubscriptionProcessorWatchdog : Watchdog $"{Name}.{nameof(LastEventProcessedTransactionId)}"; - internal async Task StartAsync(CancellationToken cancellationToken, double? periodSec = null, double? leasePeriodSec = null, double? retentionPeriodDays = null) + public override double LeasePeriodSec { get; internal set; } = 20; + + public override bool AllowRebalance { get; internal set; } = true; + + public override double PeriodSec { get; internal set; } = 3; + + protected override async Task InitAdditionalParamsAsync() { - _cancellationToken = cancellationToken; await InitLastProcessedTransactionId(); - await StartAsync(true, periodSec ?? 3, leasePeriodSec ?? 20, cancellationToken); } - protected override async Task ExecuteAsync() + protected override async Task RunWorkAsync(CancellationToken cancellationToken) { _logger.LogInformation($"{Name}: starting..."); - var lastTranId = await GetLastTransactionId(); - var visibility = await _store.MergeResourcesGetTransactionVisibilityAsync(_cancellationToken); + var lastTranId = await GetLastTransactionId(cancellationToken); + var visibility = await _store.MergeResourcesGetTransactionVisibilityAsync(cancellationToken); _logger.LogInformation($"{Name}: last transaction={lastTranId} visibility={visibility}."); - var transactionsToProcess = await _store.GetTransactionsAsync(lastTranId, visibility, _cancellationToken); + var transactionsToProcess = await _store.GetTransactionsAsync(lastTranId, visibility, cancellationToken); _logger.LogDebug($"{Name}: found transactions={transactionsToProcess.Count}."); if (transactionsToProcess.Count == 0) @@ -88,7 +91,7 @@ protected override async Task ExecuteAsync() transactionsToQueue.Add(jobDefinition); } - await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken: _cancellationToken, definitions: transactionsToQueue.ToArray()); + await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken: cancellationToken, definitions: transactionsToQueue.ToArray()); } await UpdateLastEventProcessedTransactionId(transactionsToProcess.Max(x => x.TransactionId)); @@ -96,16 +99,16 @@ protected override async Task ExecuteAsync() _logger.LogInformation($"{Name}: completed. transactions={transactionsToProcess.Count}"); } - private async Task GetLastTransactionId() + private async Task GetLastTransactionId(CancellationToken cancellationToken) { - return await GetLongParameterByIdAsync(LastEventProcessedTransactionId, _cancellationToken); + return await GetLongParameterByIdAsync(LastEventProcessedTransactionId, cancellationToken); } private async Task InitLastProcessedTransactionId() { using var cmd = new SqlCommand("INSERT INTO dbo.Parameters (Id, Bigint) SELECT @Id, @LastTranId"); cmd.Parameters.AddWithValue("@Id", LastEventProcessedTransactionId); - cmd.Parameters.AddWithValue("@LastTranId", await _store.MergeResourcesGetTransactionVisibilityAsync(_cancellationToken)); + cmd.Parameters.AddWithValue("@LastTranId", await _store.MergeResourcesGetTransactionVisibilityAsync(CancellationToken.None)); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs index 8d1ceae091..bc269ce08a 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs @@ -24,17 +24,20 @@ internal class WatchdogsBackgroundService : BackgroundService, INotificationHand private readonly CleanupEventLogWatchdog _cleanupEventLogWatchdog; private readonly IScoped _transactionWatchdog; private readonly InvisibleHistoryCleanupWatchdog _invisibleHistoryCleanupWatchdog; + private readonly SubscriptionProcessorWatchdog _subscriptionProcessorWatchdog; public WatchdogsBackgroundService( DefragWatchdog defragWatchdog, CleanupEventLogWatchdog cleanupEventLogWatchdog, IScopeProvider transactionWatchdog, - InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog) + InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog, + SubscriptionProcessorWatchdog subscriptionProcessorWatchdog) { _defragWatchdog = EnsureArg.IsNotNull(defragWatchdog, nameof(defragWatchdog)); _cleanupEventLogWatchdog = EnsureArg.IsNotNull(cleanupEventLogWatchdog, nameof(cleanupEventLogWatchdog)); _transactionWatchdog = EnsureArg.IsNotNull(transactionWatchdog, nameof(transactionWatchdog)).Invoke(); _invisibleHistoryCleanupWatchdog = EnsureArg.IsNotNull(invisibleHistoryCleanupWatchdog, nameof(invisibleHistoryCleanupWatchdog)); + _subscriptionProcessorWatchdog = EnsureArg.IsNotNull(subscriptionProcessorWatchdog, nameof(subscriptionProcessorWatchdog)); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -53,6 +56,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) _cleanupEventLogWatchdog.ExecuteAsync(continuationTokenSource.Token), _transactionWatchdog.Value.ExecuteAsync(continuationTokenSource.Token), _invisibleHistoryCleanupWatchdog.ExecuteAsync(continuationTokenSource.Token), + _subscriptionProcessorWatchdog.ExecuteAsync(continuationTokenSource.Token), }; await Task.WhenAny(tasks); From b56fc82bf44f97e6eb5aaa8be6f92d3ae5db0a40 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Tue, 16 Jul 2024 16:48:46 -0700 Subject: [PATCH 15/47] Aligns dotnet sdk version for build --- global.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global.json b/global.json index 7cc48a4e22..789477d342 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "8.0.303" + "version": "8.0.204" } } From 1e5540b0c36cfaa8a12e79278c6f9ac99dcd2500 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Wed, 17 Jul 2024 09:32:46 -0700 Subject: [PATCH 16/47] Adds subscription to docker build --- build/docker/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 9f4d90b776..8971018350 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -49,6 +49,9 @@ COPY ./src/Microsoft.Health.Fhir.CosmosDb.Core/Microsoft.Health.Fhir.CosmosDb.Co COPY ./src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj \ ./src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj +COPY ./src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj \ + ./src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj + COPY ./src/Microsoft.Health.Fhir.${FHIR_VERSION}.Core/Microsoft.Health.Fhir.${FHIR_VERSION}.Core.csproj \ ./src/Microsoft.Health.Fhir.${FHIR_VERSION}.Core/Microsoft.Health.Fhir.${FHIR_VERSION}.Core.csproj From b0cf111cc42be483bf9c2e099e99f4f7a942822c Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Thu, 25 Jul 2024 13:12:15 -0700 Subject: [PATCH 17/47] Adds SearchQueryInterpreter --- R4.slnf | 1 - .../Search/InMemory/ComparisonValueVisitor.cs | 105 ++++++++ .../Features/Search/InMemory/InMemoryIndex.cs | 51 ++++ .../Search/InMemory/SearchQueryInterpreter.cs | 228 ++++++++++++++++++ .../InMemory/SearchQueryInterperaterTests.cs | 110 +++++++++ ...ealth.Fhir.Shared.Core.UnitTests.projitems | 1 + 6 files changed, 495 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs create mode 100644 src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs diff --git a/R4.slnf b/R4.slnf index 7adecaed1e..bb8a55c269 100644 --- a/R4.slnf +++ b/R4.slnf @@ -29,7 +29,6 @@ "src\\Microsoft.Health.Fhir.Shared.Web\\Microsoft.Health.Fhir.Shared.Web.shproj", "src\\Microsoft.Health.Fhir.SqlServer.UnitTests\\Microsoft.Health.Fhir.SqlServer.UnitTests.csproj", "src\\Microsoft.Health.Fhir.SqlServer\\Microsoft.Health.Fhir.SqlServer.csproj", - "src\\Microsoft.Health.Fhir.Subscriptions\\Microsoft.Health.Fhir.Subscriptions.csproj", "src\\Microsoft.Health.Fhir.Tests.Common\\Microsoft.Health.Fhir.Tests.Common.csproj", "src\\Microsoft.Health.TaskManagement\\Microsoft.Health.TaskManagement.csproj", "src\\Microsoft.Health.TaskManagement.UnitTests\\Microsoft.Health.TaskManagement.UnitTests.csproj", diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs new file mode 100644 index 0000000000..86eefd7eb0 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs @@ -0,0 +1,105 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Health.Fhir.Core.Features.Search.Expressions; +using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; + +namespace Microsoft.Health.Fhir.Core.Features.Search.InMemory +{ + internal class ComparisonValueVisitor : ISearchValueVisitor + { + private readonly BinaryOperator _expressionBinaryOperator; + private readonly IComparable _second; + + private readonly List> _comparisonValues = new List>(); + + public ComparisonValueVisitor(BinaryOperator expressionBinaryOperator, IComparable second) + { + _expressionBinaryOperator = expressionBinaryOperator; + _second = second; + } + + public void Visit(CompositeSearchValue composite) + { + foreach (IReadOnlyList c in composite.Components) + { + foreach (ISearchValue inner in c) + { + inner.AcceptVisitor(this); + } + } + } + + public void Visit(DateTimeSearchValue dateTime) + { + AddComparison(_expressionBinaryOperator, dateTime.Start); + } + + public void Visit(NumberSearchValue number) + { + AddComparison(_expressionBinaryOperator, number.High); + } + + public void Visit(QuantitySearchValue quantity) + { + AddComparison(_expressionBinaryOperator, quantity.High); + } + + public void Visit(ReferenceSearchValue reference) + { + AddComparison(_expressionBinaryOperator, reference.ResourceId); + } + + public void Visit(StringSearchValue s) + { + AddComparison(_expressionBinaryOperator, s.String); + } + + public void Visit(TokenSearchValue token) + { + AddComparison(_expressionBinaryOperator, token.Text, token.System, token.Code); + } + + public void Visit(UriSearchValue uri) + { + AddComparison(_expressionBinaryOperator, uri.Uri); + } + + private void AddComparison(BinaryOperator binaryOperator, params IComparable[] first) + { + switch (binaryOperator) + { + case BinaryOperator.Equal: + _comparisonValues.Add(() => first.Any(x => x.CompareTo(_second) == 0)); + break; + case BinaryOperator.GreaterThan: + _comparisonValues.Add(() => first.Any(x => x.CompareTo(_second) > 0)); + break; + case BinaryOperator.LessThan: + _comparisonValues.Add(() => first.Any(x => x.CompareTo(_second) < 0)); + break; + case BinaryOperator.NotEqual: + _comparisonValues.Add(() => first.Any(x => x.CompareTo(_second) != 0)); + break; + case BinaryOperator.GreaterThanOrEqual: + _comparisonValues.Add(() => first.Any(x => x.CompareTo(_second) >= 0)); + break; + case BinaryOperator.LessThanOrEqual: + _comparisonValues.Add(() => first.Any(x => x.CompareTo(_second) <= 0)); + break; + default: + throw new ArgumentOutOfRangeException(nameof(binaryOperator)); + } + } + + public bool Compare() + { + return _comparisonValues.All(x => x.Invoke()); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs new file mode 100644 index 0000000000..00ba7f5d38 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs @@ -0,0 +1,51 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Concurrent; +using System.Collections.Generic; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Core.Features.Search.InMemory +{ + public class InMemoryIndex + { + private readonly ISearchIndexer _searchIndexer; + + public InMemoryIndex(ISearchIndexer searchIndexer) + { + Index = new ConcurrentDictionary)>>(); + _searchIndexer = searchIndexer; + } + + internal ConcurrentDictionary Index)>> Index + { + get; + } + + public void IndexResources(params ResourceElement[] resources) + { + foreach (var resource in resources) + { + var indexEntries = _searchIndexer.Extract(resource); + + Index.AddOrUpdate( + resource.InstanceType, + key => new List<(ResourceKey, IReadOnlyCollection)> { (ToResourceKey(resource), indexEntries) }, + (key, list) => + { + list.Add((ToResourceKey(resource), indexEntries)); + return list; + }); + } + } + + private static ResourceKey ToResourceKey(ResourceElement resource) + { + return new ResourceKey(resource.InstanceType, resource.Id, resource.VersionId); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs new file mode 100644 index 0000000000..91ee4efea8 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs @@ -0,0 +1,228 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Expressions; +using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; + +using SearchPredicate = System.Func< + System.Collections.Generic.IEnumerable<(Microsoft.Health.Fhir.Core.Features.Persistence.ResourceKey Location, System.Collections.Generic.IReadOnlyCollection Index)>, + System.Collections.Generic.IEnumerable<(Microsoft.Health.Fhir.Core.Features.Persistence.ResourceKey Location, System.Collections.Generic.IReadOnlyCollection Index)>>; + +namespace Microsoft.Health.Fhir.Core.Features.Search.InMemory +{ + internal class SearchQueryInterpreter : IExpressionVisitorWithInitialContext + { + Context IExpressionVisitorWithInitialContext.InitialContext => default; + + public SearchPredicate VisitSearchParameter(SearchParameterExpression expression, Context context) + { + return VisitInnerWithContext(expression.Parameter.Name, expression.Expression, context); + } + + public SearchPredicate VisitBinary(BinaryExpression expression, Context context) + { + return VisitBinary( + context.ParameterName, + expression.BinaryOperator, + expression.Value); + } + + private static SearchPredicate VisitBinary(string fieldName, BinaryOperator op, object value) + { + SearchPredicate filter = input => + { + return input.Where(x => x.Index.Any(y => y.SearchParameter.Name == fieldName && + GetMappedValue(op, y.Value, (IComparable)value))); + }; + + return filter; + } + + private static bool GetMappedValue(BinaryOperator expressionBinaryOperator, ISearchValue first, IComparable second) + { + if (first == null || second == null) + { + return false; + } + + var comparisonVisitor = new ComparisonValueVisitor(expressionBinaryOperator, second); + first.AcceptVisitor(comparisonVisitor); + + return comparisonVisitor.Compare(); + } + + public SearchPredicate VisitChained(ChainedExpression expression, Context context) + { + throw new SearchOperationNotSupportedException("ChainedExpression is not supported."); + } + + public SearchPredicate VisitMissingField(MissingFieldExpression expression, Context context) + { + throw new NotImplementedException(); + } + + public SearchPredicate VisitMissingSearchParameter(MissingSearchParameterExpression expression, Context context) + { + throw new NotImplementedException(); + } + + public SearchPredicate VisitMultiary(MultiaryExpression expression, Context context) + { + SearchPredicate filter = input => + { + var results = expression.Expressions.Select(x => x.AcceptVisitor(this, context)) + .Aggregate((x, y) => + { + switch (expression.MultiaryOperation) + { + case MultiaryOperator.And: + return p => x(p).Intersect(y(p)); + case MultiaryOperator.Or: + return p => x(p).Union(y(p)); + default: + throw new NotImplementedException(); + } + }); + + return results(input); + }; + + return filter; + } + + public SearchPredicate VisitString(StringExpression expression, Context context) + { + StringComparison comparison = expression.IgnoreCase + ? StringComparison.OrdinalIgnoreCase + : StringComparison.Ordinal; + + SearchPredicate filter; + + if (context.ParameterName == "_type") + { + filter = input => input.Where(x => x.Location.ResourceType.Equals(expression.Value, comparison)); + } + else + { + switch (expression.StringOperator) + { + case StringOperator.StartsWith: + filter = input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == context.ParameterName && + CompareStringParameter(y, (a, b, c) => a.StartsWith(b, c)))); + break; + case StringOperator.Equals: + filter = input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == context.ParameterName && + CompareStringParameter(y, string.Equals))); + + break; + default: + throw new NotImplementedException(); + } + } + + bool CompareStringParameter(SearchIndexEntry y, Func compareFunc) + { + switch (y.SearchParameter.Type) + { + case ValueSets.SearchParamType.String: + return compareFunc(((StringSearchValue)y.Value).String, expression.Value, comparison); + + case ValueSets.SearchParamType.Token: + return compareFunc(((TokenSearchValue)y.Value).Code, expression.Value, comparison) || + compareFunc(((TokenSearchValue)y.Value).System, expression.Value, comparison); + default: + throw new NotImplementedException(); + } + } + + return filter; + } + + public SearchPredicate VisitCompartment(CompartmentSearchExpression expression, Context context) + { + throw new SearchOperationNotSupportedException("Compartment search is not supported."); + } + + public SearchPredicate VisitInclude(IncludeExpression expression, Context context) + { + throw new NotImplementedException(); + } + + private SearchPredicate VisitInnerWithContext(string parameterName, Expression expression, Context context, bool negate = false) + { + EnsureArg.IsNotNull(parameterName, nameof(parameterName)); + + var newContext = context.WithParameterName(parameterName); + + SearchPredicate filter = input => + { + if (expression != null) + { + return expression.AcceptVisitor(this, newContext)(input); + } + else + { + // :missing will end up here + throw new NotSupportedException("This query is not supported"); + } + }; + + if (negate) + { + SearchPredicate inner = filter; + filter = input => input.Except(inner(input)); + } + + return filter; + } + + public SearchPredicate VisitNotExpression(NotExpression expression, Context context) + { + throw new NotImplementedException(); + } + + public SearchPredicate VisitUnion(UnionExpression expression, Context context) + { + throw new NotImplementedException(); + } + + public SearchPredicate VisitSmartCompartment(SmartCompartmentSearchExpression expression, Context context) + { + throw new NotImplementedException(); + } + + public SearchPredicate VisitSortParameter(SortExpression expression, Context context) + { + throw new NotImplementedException(); + } + + public SearchPredicate VisitIn(InExpression expression, Context context) + { + throw new NotImplementedException(); + } + + /// + /// Context that is passed through the visit. + /// + internal struct Context + { + public string ParameterName { get; set; } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "Internal API")] + public Context WithParameterName(string paramName) + { + return new Context + { + ParameterName = paramName, + }; + } + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs new file mode 100644 index 0000000000..84858a4253 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs @@ -0,0 +1,110 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features.Context; +using Microsoft.Health.Fhir.Core.Features.Definition; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Converters; +using Microsoft.Health.Fhir.Core.Features.Search.Expressions.Parsers; +using Microsoft.Health.Fhir.Core.Features.Search.InMemory; +using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Tests.Common; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.InMemory +{ + public class SearchQueryInterperaterTests : IAsyncLifetime + { + private ExpressionParser _expressionParser; + private InMemoryIndex _memoryIndex; + private SearchQueryInterpreter _searchQueryInterperater; + + public Task DisposeAsync() + { + return Task.CompletedTask; + } + + public async Task InitializeAsync() + { + _searchQueryInterperater = new SearchQueryInterpreter(); + + var fixture = new SearchParameterFixtureData(); + var manager = await fixture.GetSearchDefinitionManagerAsync(); + + var fhirRequestContextAccessor = new FhirRequestContextAccessor(); + var supportedSearchParameterDefinitionManager = new SupportedSearchParameterDefinitionManager(manager); + var searchableSearchParameterDefinitionManager = new SearchableSearchParameterDefinitionManager(manager, fhirRequestContextAccessor); + var typedElementToSearchValueConverterManager = GetTypeConverterAsync().Result; + + var referenceParser = new ReferenceSearchValueParser(fhirRequestContextAccessor); + var referenceToElementResolver = new LightweightReferenceToElementResolver(referenceParser, ModelInfoProvider.Instance); + var modelInfoProvider = ModelInfoProvider.Instance; + var logger = Substitute.For>(); + + var searchIndexer = new TypedElementSearchIndexer(supportedSearchParameterDefinitionManager, typedElementToSearchValueConverterManager, referenceToElementResolver, modelInfoProvider, logger); + _expressionParser = new ExpressionParser(() => searchableSearchParameterDefinitionManager, new SearchParameterExpressionParser(referenceParser)); + _memoryIndex = new InMemoryIndex(searchIndexer); + + _memoryIndex.IndexResources(Samples.GetDefaultPatient(), Samples.GetDefaultObservation().UpdateId("example")); + } + + protected async Task GetTypeConverterAsync() + { + FhirTypedElementToSearchValueConverterManager fhirTypedElementToSearchValueConverterManager = await SearchParameterFixtureData.GetFhirTypedElementToSearchValueConverterManagerAsync(); + return fhirTypedElementToSearchValueConverterManager; + } + + [Fact] + public void GivenASearchQueryInterpreter_WhenSearchingByNameOnPatient_ThenCorrectResultsAreReturned() + { + var expression = _expressionParser.Parse(new[] { "Patient" }, "name", "Jim"); + + var evaluator = expression.AcceptVisitor(_searchQueryInterperater, default); + + var results = evaluator + .Invoke(_memoryIndex.Index.Values.SelectMany(x => x)) + .ToArray(); + + Assert.Single(results); + } + + [Fact] + public void GivenASearchQueryInterpreter_WhenSearchingByDobOnPatient_ThenCorrectResultsAreReturned() + { + var expression = _expressionParser.Parse(new[] { "Patient" }, "birthdate", "gt1950"); + + var evaluator = expression.AcceptVisitor(_searchQueryInterperater, default); + + var results = evaluator + .Invoke(_memoryIndex.Index.Values.SelectMany(x => x)) + .ToArray(); + + Assert.Single(results); + } + + [Fact] + public void GivenASearchQueryInterpreter_WhenSearchingByValueOnObservation_ThenCorrectResultsAreReturned() + { + var expression = _expressionParser.Parse(new[] { "Observation" }, "value-quantity", "lt70"); + + var evaluator = expression.AcceptVisitor(_searchQueryInterperater, default); + + var results = evaluator + .Invoke(_memoryIndex.Index.Values.SelectMany(x => x)) + .ToArray(); + + Assert.Single(results); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems index 163f14e19c..5e192cf24c 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems @@ -46,6 +46,7 @@ + From e1fc4369f8691584fb9caa23e8b73357850cfdd4 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Fri, 26 Jul 2024 11:30:53 -0700 Subject: [PATCH 18/47] Cleanup of SearchQueryInterpreter --- .../Search/InMemory/ComparisonValueVisitor.cs | 14 ++- .../Features/Search/InMemory/InMemoryIndex.cs | 7 +- .../Search/InMemory/SearchQueryInterpreter.cs | 99 ++++++++++++------- .../Properties/AssemblyInfo.cs | 1 + .../InMemory/SearchQueryInterperaterTests.cs | 14 +++ 5 files changed, 98 insertions(+), 37 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs index 86eefd7eb0..3537a16abf 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs @@ -6,6 +6,8 @@ using System; using System.Collections.Generic; using System.Linq; +using EnsureThat; +using Hl7.Fhir.ElementModel.Types; using Microsoft.Health.Fhir.Core.Features.Search.Expressions; using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; @@ -16,12 +18,12 @@ internal class ComparisonValueVisitor : ISearchValueVisitor private readonly BinaryOperator _expressionBinaryOperator; private readonly IComparable _second; - private readonly List> _comparisonValues = new List>(); + private readonly List> _comparisonValues = []; public ComparisonValueVisitor(BinaryOperator expressionBinaryOperator, IComparable second) { _expressionBinaryOperator = expressionBinaryOperator; - _second = second; + _second = EnsureArg.IsNotNull(second, nameof(second)); } public void Visit(CompositeSearchValue composite) @@ -37,41 +39,49 @@ public void Visit(CompositeSearchValue composite) public void Visit(DateTimeSearchValue dateTime) { + EnsureArg.IsNotNull(dateTime, nameof(dateTime)); AddComparison(_expressionBinaryOperator, dateTime.Start); } public void Visit(NumberSearchValue number) { + EnsureArg.IsNotNull(number, nameof(number)); AddComparison(_expressionBinaryOperator, number.High); } public void Visit(QuantitySearchValue quantity) { + EnsureArg.IsNotNull(quantity, nameof(quantity)); AddComparison(_expressionBinaryOperator, quantity.High); } public void Visit(ReferenceSearchValue reference) { + EnsureArg.IsNotNull(reference, nameof(reference)); AddComparison(_expressionBinaryOperator, reference.ResourceId); } public void Visit(StringSearchValue s) { + EnsureArg.IsNotNull(s, nameof(s)); AddComparison(_expressionBinaryOperator, s.String); } public void Visit(TokenSearchValue token) { + EnsureArg.IsNotNull(token, nameof(token)); AddComparison(_expressionBinaryOperator, token.Text, token.System, token.Code); } public void Visit(UriSearchValue uri) { + EnsureArg.IsNotNull(uri, nameof(uri)); AddComparison(_expressionBinaryOperator, uri.Uri); } private void AddComparison(BinaryOperator binaryOperator, params IComparable[] first) { + EnsureArg.IsNotNull(first, nameof(first)); switch (binaryOperator) { case BinaryOperator.Equal: diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs index 00ba7f5d38..f489bfd821 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs @@ -5,6 +5,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; +using EnsureThat; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Models; @@ -17,8 +18,8 @@ public class InMemoryIndex public InMemoryIndex(ISearchIndexer searchIndexer) { + _searchIndexer = EnsureArg.IsNotNull(searchIndexer, nameof(searchIndexer)); Index = new ConcurrentDictionary)>>(); - _searchIndexer = searchIndexer; } internal ConcurrentDictionary Index)>> Index @@ -28,6 +29,8 @@ public InMemoryIndex(ISearchIndexer searchIndexer) public void IndexResources(params ResourceElement[] resources) { + EnsureArg.IsNotNull(resources, nameof(resources)); + foreach (var resource in resources) { var indexEntries = _searchIndexer.Extract(resource); @@ -45,6 +48,8 @@ public void IndexResources(params ResourceElement[] resources) private static ResourceKey ToResourceKey(ResourceElement resource) { + EnsureArg.IsNotNull(resource, nameof(resource)); + return new ResourceKey(resource.InstanceType, resource.Id, resource.VersionId); } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs index 91ee4efea8..da0eebcb90 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs @@ -7,46 +7,48 @@ using System.Collections.Generic; using System.Linq; using EnsureThat; +using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Expressions; using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; -using SearchPredicate = System.Func< - System.Collections.Generic.IEnumerable<(Microsoft.Health.Fhir.Core.Features.Persistence.ResourceKey Location, System.Collections.Generic.IReadOnlyCollection Index)>, - System.Collections.Generic.IEnumerable<(Microsoft.Health.Fhir.Core.Features.Persistence.ResourceKey Location, System.Collections.Generic.IReadOnlyCollection Index)>>; - namespace Microsoft.Health.Fhir.Core.Features.Search.InMemory { + public delegate IEnumerable<(ResourceKey Location, IReadOnlyCollection Index)> SearchPredicate(IEnumerable<(ResourceKey Location, IReadOnlyCollection Index)> input); + internal class SearchQueryInterpreter : IExpressionVisitorWithInitialContext { Context IExpressionVisitorWithInitialContext.InitialContext => default; public SearchPredicate VisitSearchParameter(SearchParameterExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + return VisitInnerWithContext(expression.Parameter.Name, expression.Expression, context); } public SearchPredicate VisitBinary(BinaryExpression expression, Context context) { - return VisitBinary( - context.ParameterName, - expression.BinaryOperator, - expression.Value); + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + + return VisitBinary(context.ParameterName, expression.BinaryOperator, expression.Value); } private static SearchPredicate VisitBinary(string fieldName, BinaryOperator op, object value) { - SearchPredicate filter = input => - { - return input.Where(x => x.Index.Any(y => y.SearchParameter.Name == fieldName && - GetMappedValue(op, y.Value, (IComparable)value))); - }; + EnsureArg.IsNotNull(fieldName, nameof(fieldName)); + EnsureArg.IsNotNull(value, nameof(value)); - return filter; + return input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == fieldName && GetMappedValue(op, y.Value, (IComparable)value))); } private static bool GetMappedValue(BinaryOperator expressionBinaryOperator, ISearchValue first, IComparable second) { + EnsureArg.IsNotNull(first, nameof(first)); + EnsureArg.IsNotNull(second, nameof(second)); + if (first == null || second == null) { return false; @@ -60,24 +62,35 @@ private static bool GetMappedValue(BinaryOperator expressionBinaryOperator, ISea public SearchPredicate VisitChained(ChainedExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new SearchOperationNotSupportedException("ChainedExpression is not supported."); } public SearchPredicate VisitMissingField(MissingFieldExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new NotImplementedException(); } public SearchPredicate VisitMissingSearchParameter(MissingSearchParameterExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new NotImplementedException(); } public SearchPredicate VisitMultiary(MultiaryExpression expression, Context context) { - SearchPredicate filter = input => - { - var results = expression.Expressions.Select(x => x.AcceptVisitor(this, context)) + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(expression.Expressions, nameof(expression.Expressions)); + EnsureArg.IsNotNull(context, nameof(context)); + + return expression.Expressions.Select(x => x.AcceptVisitor(this, context)) .Aggregate((x, y) => { switch (expression.MultiaryOperation) @@ -90,38 +103,32 @@ public SearchPredicate VisitMultiary(MultiaryExpression expression, Context cont throw new NotImplementedException(); } }); - - return results(input); - }; - - return filter; } public SearchPredicate VisitString(StringExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + StringComparison comparison = expression.IgnoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; - SearchPredicate filter; - if (context.ParameterName == "_type") { - filter = input => input.Where(x => x.Location.ResourceType.Equals(expression.Value, comparison)); + return input => input.Where(x => x.Location.ResourceType.Equals(expression.Value, comparison)); } else { switch (expression.StringOperator) { case StringOperator.StartsWith: - filter = input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == context.ParameterName && - CompareStringParameter(y, (a, b, c) => a.StartsWith(b, c)))); - break; + return input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == context.ParameterName && + CompareStringParameter(y, (a, b, c) => a.StartsWith(b, c)))); case StringOperator.Equals: - filter = input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == context.ParameterName && - CompareStringParameter(y, string.Equals))); + return input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == context.ParameterName && + CompareStringParameter(y, string.Equals))); - break; default: throw new NotImplementedException(); } @@ -129,6 +136,8 @@ public SearchPredicate VisitString(StringExpression expression, Context context) bool CompareStringParameter(SearchIndexEntry y, Func compareFunc) { + EnsureArg.IsNotNull(y, nameof(y)); + switch (y.SearchParameter.Type) { case ValueSets.SearchParamType.String: @@ -141,23 +150,29 @@ bool CompareStringParameter(SearchIndexEntry y, Func(context, nameof(context)); + throw new SearchOperationNotSupportedException("Compartment search is not supported."); } public SearchPredicate VisitInclude(IncludeExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new NotImplementedException(); } private SearchPredicate VisitInnerWithContext(string parameterName, Expression expression, Context context, bool negate = false) { EnsureArg.IsNotNull(parameterName, nameof(parameterName)); + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); var newContext = context.WithParameterName(parameterName); @@ -185,27 +200,43 @@ private SearchPredicate VisitInnerWithContext(string parameterName, Expression e public SearchPredicate VisitNotExpression(NotExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new NotImplementedException(); } public SearchPredicate VisitUnion(UnionExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new NotImplementedException(); } public SearchPredicate VisitSmartCompartment(SmartCompartmentSearchExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new NotImplementedException(); } public SearchPredicate VisitSortParameter(SortExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new NotImplementedException(); } public SearchPredicate VisitIn(InExpression expression, Context context) { - throw new NotImplementedException(); + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + + return input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == context.ParameterName && + expression.Values.Contains((T)y.Value))); } /// diff --git a/src/Microsoft.Health.Fhir.Core/Properties/AssemblyInfo.cs b/src/Microsoft.Health.Fhir.Core/Properties/AssemblyInfo.cs index 115d142d47..aa5748585b 100644 --- a/src/Microsoft.Health.Fhir.Core/Properties/AssemblyInfo.cs +++ b/src/Microsoft.Health.Fhir.Core/Properties/AssemblyInfo.cs @@ -49,6 +49,7 @@ [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.R5.Tests.E2E")] [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.Stu3.Tests.E2E")] +[assembly: InternalsVisibleTo("Microsoft.Health.Fhir.Subscriptions")] [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.R4.ResourceParser")] [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.SqlServer.UnitTests")] diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs index 84858a4253..2d96d2494a 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs @@ -93,6 +93,20 @@ public void GivenASearchQueryInterpreter_WhenSearchingByDobOnPatient_ThenCorrect Assert.Single(results); } + [Fact] + public void GivenASearchQueryInterpreter_WhenSearchingByDobOnPatientWithRange_ThenCorrectResultsAreReturned() + { + var expression = _expressionParser.Parse(new[] { "Patient" }, "birthdate", "1974"); + + var evaluator = expression.AcceptVisitor(_searchQueryInterperater, default); + + var results = evaluator + .Invoke(_memoryIndex.Index.Values.SelectMany(x => x)) + .ToArray(); + + Assert.Single(results); + } + [Fact] public void GivenASearchQueryInterpreter_WhenSearchingByValueOnObservation_ThenCorrectResultsAreReturned() { From b9eb3cf0ae3448b3a4cf6499b995b3462740149e Mon Sep 17 00:00:00 2001 From: aponakampalli Date: Wed, 21 Aug 2024 12:51:38 -0700 Subject: [PATCH 19/47] fix sql retry service merge --- .../Features/Storage/SqlRetry/ISqlRetryService.cs | 4 ++-- .../Features/Storage/SqlRetry/SqlRetryService.cs | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/ISqlRetryService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/ISqlRetryService.cs index 9a1928a0c2..ab00060ffd 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/ISqlRetryService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/ISqlRetryService.cs @@ -18,8 +18,8 @@ public interface ISqlRetryService Task ExecuteSql(Func action, ILogger logger, CancellationToken cancellationToken, bool isReadOnly = false); - Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false); + Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false); - Task> ExecuteReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false); + Task> ExecuteReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false); } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs index a544c7b915..b8de373afb 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs @@ -246,6 +246,7 @@ public async Task ExecuteSql(Func + /// Type used for the . /// SQL command to be executed. /// Delegate to be executed by passing as input parameter. /// Logger used on first try error (or retry error) and connection open. @@ -255,7 +256,7 @@ public async Task ExecuteSql(Func"Flag indicating whether retries are disabled." /// A task representing the asynchronous operation. /// When executing this method, if exception is thrown that is not retriable or if last retry fails, then same exception is thrown by this method. - public async Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false) + public async Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false) { EnsureArg.IsNotNull(sqlCommand, nameof(sqlCommand)); EnsureArg.IsNotNull(action, nameof(action)); @@ -307,14 +308,14 @@ public async Task ExecuteSql(SqlCommand sqlCommand, Func> ExecuteSqlDataReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, bool allRows, bool isReadOnly, CancellationToken cancellationToken) + private async Task> ExecuteSqlDataReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, bool allRows, bool isReadOnly, CancellationToken cancellationToken) { EnsureArg.IsNotNull(sqlCommand, nameof(sqlCommand)); EnsureArg.IsNotNull(readerToResult, nameof(readerToResult)); EnsureArg.IsNotNull(logger, nameof(logger)); List results = null; - await ExecuteSql( + await ExecuteSql( sqlCommand, async (sqlCommand, cancellationToken) => { @@ -342,6 +343,7 @@ await ExecuteSql( /// into the data type and returns them. Retries execution of on SQL error or failed /// SQL connection error. In the case if non-retriable exception or if the last retry failed tha same exception is thrown. /// + /// Type used for the . /// Defines data type for the returned SQL rows. /// SQL command to be executed. /// Translation delegate that translates the row returned by execution into the data type. @@ -352,7 +354,7 @@ await ExecuteSql( /// A task representing the asynchronous operation that returns all the rows that result from execution. The rows are translated by delegate /// into data type. /// When executing this method, if exception is thrown that is not retriable or if last retry fails, then same exception is thrown by this method. - public async Task> ExecuteReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false) + public async Task> ExecuteReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false) { return await ExecuteSqlDataReaderAsync(sqlCommand, readerToResult, logger, logMessage, true, isReadOnly, cancellationToken); } From f018f15d38ffd7f1ee2873e427b944e2b172e3d2 Mon Sep 17 00:00:00 2001 From: aponakampalli Date: Wed, 21 Aug 2024 13:14:54 -0700 Subject: [PATCH 20/47] fix sql command extensions merge --- .../Features/Storage/SqlRetry/SqlCommandExtensions.cs | 6 +++--- .../Features/Storage/SqlRetry/SqlRetryService.cs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlCommandExtensions.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlCommandExtensions.cs index 1bc3b13281..809c30f686 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlCommandExtensions.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlCommandExtensions.cs @@ -14,17 +14,17 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage { public static class SqlCommandExtensions { - public static async Task ExecuteNonQueryAsync(this SqlCommand cmd, ISqlRetryService retryService, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false, bool disableRetries = false) + public static async Task ExecuteNonQueryAsync(this SqlCommand cmd, ISqlRetryService retryService, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false, bool disableRetries = false) { await retryService.ExecuteSql(cmd, async (sql, cancel) => await sql.ExecuteNonQueryAsync(cancel), logger, logMessage, cancellationToken, isReadOnly, disableRetries); } - public static async Task> ExecuteReaderAsync(this SqlCommand cmd, ISqlRetryService retryService, Func readerToResult, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false) + public static async Task> ExecuteReaderAsync(this SqlCommand cmd, ISqlRetryService retryService, Func readerToResult, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false) { return await retryService.ExecuteReaderAsync(cmd, readerToResult, logger, logMessage, cancellationToken, isReadOnly); } - public static async Task ExecuteScalarAsync(this SqlCommand cmd, ISqlRetryService retryService, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false, bool disableRetries = false) + public static async Task ExecuteScalarAsync(this SqlCommand cmd, ISqlRetryService retryService, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false, bool disableRetries = false) { object scalar = null; await retryService.ExecuteSql(cmd, async (sql, cancel) => { scalar = await sql.ExecuteScalarAsync(cancel); }, logger, logMessage, cancellationToken, isReadOnly, disableRetries); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs index b8de373afb..1b448c913e 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs @@ -315,7 +315,7 @@ private async Task> ExecuteSqlDataReaderAsync results = null; - await ExecuteSql( + await ExecuteSql( sqlCommand, async (sqlCommand, cancellationToken) => { @@ -343,8 +343,8 @@ await ExecuteSql( /// into the data type and returns them. Retries execution of on SQL error or failed /// SQL connection error. In the case if non-retriable exception or if the last retry failed tha same exception is thrown. /// - /// Type used for the . /// Defines data type for the returned SQL rows. + /// Type used for the . /// SQL command to be executed. /// Translation delegate that translates the row returned by execution into the data type. /// Logger used on first try error or retry error. From 94eb190fceff84108829f793f516dc97c9f4f115 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Wed, 28 Feb 2024 18:06:58 -0800 Subject: [PATCH 21/47] Improve fhirtimer --- .../Features/Watchdogs/DefragWatchdog.cs | 5 +- .../Features/Watchdogs/FhirTimer.cs | 100 ++++-------------- .../Features/Watchdogs/Watchdog.cs | 23 ---- .../Features/Watchdogs/WatchdogLease.cs | 4 +- .../Watchdogs/WatchdogsBackgroundService.cs | 18 +--- .../Persistence/SqlServerWatchdogTests.cs | 8 -- 6 files changed, 33 insertions(+), 125 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/DefragWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/DefragWatchdog.cs index 7cba164b44..191cea4f68 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/DefragWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/DefragWatchdog.cs @@ -57,8 +57,9 @@ internal DefragWatchdog() internal async Task StartAsync(CancellationToken cancellationToken) { _cancellationToken = cancellationToken; - await StartAsync(false, 24 * 3600, 2 * 3600, cancellationToken); - await InitDefragParamsAsync(); + await Task.WhenAll( + StartAsync(false, 24 * 3600, 2 * 3600, cancellationToken), + InitDefragParamsAsync()); } protected override async Task ExecuteAsync() diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs index df33cba028..a5bbc7276a 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs @@ -8,108 +8,54 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.Health.Core; +using Microsoft.Health.Fhir.Core.Extensions; namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - public abstract class FhirTimer : IDisposable + public abstract class FhirTimer(ILogger logger = null) { - private Timer _timer; - private bool _disposed = false; - private bool _isRunning; private bool _isFailing; - private bool _isStarted; - private string _lastException; - private readonly ILogger _logger; - private CancellationToken _cancellationToken; - - protected FhirTimer(ILogger logger = null) - { - _logger = logger; - _isFailing = false; - _lastException = null; - LastRunDateTime = DateTime.Parse("2017-12-01"); - } internal double PeriodSec { get; set; } - internal DateTime LastRunDateTime { get; private set; } - - internal bool IsRunning => _isRunning; + internal DateTimeOffset LastRunDateTime { get; private set; } = DateTime.Parse("2017-12-01"); internal bool IsFailing => _isFailing; - internal bool IsStarted => _isStarted; - - internal string LastException => _lastException; - protected async Task StartAsync(double periodSec, CancellationToken cancellationToken) { PeriodSec = periodSec; - _cancellationToken = cancellationToken; - - // WARNING: Avoid using 'async' lambda when delegate type returns 'void' - _timer = new Timer(async _ => await RunInternalAsync(), null, TimeSpan.FromSeconds(PeriodSec * RandomNumberGenerator.GetInt32(1000) / 1000), TimeSpan.FromSeconds(PeriodSec)); - - _isStarted = true; - await Task.CompletedTask; - } - protected abstract Task RunAsync(); - - private async Task RunInternalAsync() - { - if (_isRunning || _cancellationToken.IsCancellationRequested) - { - return; - } + await Task.Delay(TimeSpan.FromSeconds(PeriodSec * RandomNumberGenerator.GetInt32(1000) / 1000), cancellationToken); + using var periodicTimer = new PeriodicTimer(TimeSpan.FromSeconds(PeriodSec)); - try - { - _isRunning = true; - await RunAsync(); - _isFailing = false; - _lastException = null; - LastRunDateTime = DateTime.UtcNow; - } - catch (Exception e) + while (!cancellationToken.IsCancellationRequested) { try { - _logger.LogWarning(e, "Error executing FHIR Timer"); // exceptions in logger should never bubble up + await periodicTimer.WaitForNextTickAsync(cancellationToken); } - catch + catch (OperationCanceledException) { - // ignored + // Time to exit + break; } - _isFailing = true; - _lastException = e.ToString(); - } - finally - { - _isRunning = false; + try + { + await RunAsync(); + LastRunDateTime = Clock.UtcNow; + _isFailing = false; + } + catch (Exception e) + { + logger.LogWarning(e, "Error executing timer"); + _isFailing = true; + } } } - public void Dispose() - { - Dispose(true); - GC.SuppressFinalize(this); - } - - protected virtual void Dispose(bool disposing) - { - if (_disposed) - { - return; - } - - if (disposing) - { - _timer?.Dispose(); - } - - _disposed = true; - } + protected abstract Task RunAsync(); } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs index 65c4926787..d68b671a44 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs @@ -19,7 +19,6 @@ public abstract class Watchdog : FhirTimer private ISqlRetryService _sqlRetryService; private readonly ILogger _logger; private readonly WatchdogLease _watchdogLease; - private bool _disposed = false; private double _periodSec; private double _leasePeriodSec; @@ -138,27 +137,5 @@ protected async Task GetLongParameterByIdAsync(string id, CancellationToke return (long)value; } - - public new void Dispose() - { - Dispose(true); - } - - protected override void Dispose(bool disposing) - { - if (_disposed) - { - return; - } - - if (disposing) - { - _watchdogLease?.Dispose(); - } - - base.Dispose(disposing); - - _disposed = true; - } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs index cbf92d3f9c..ee4a595cd5 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs @@ -17,7 +17,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs internal class WatchdogLease : FhirTimer { private const double TimeoutFactor = 0.25; - private readonly object _locker = new object(); + private readonly object _locker = new(); private readonly ISqlRetryService _sqlRetryService; private readonly ILogger _logger; @@ -70,7 +70,7 @@ protected override async Task RunAsync() cmd.Parameters.AddWithValue("@Watchdog", _watchdogName); cmd.Parameters.AddWithValue("@Worker", _worker); cmd.Parameters.AddWithValue("@AllowRebalance", _allowRebalance); - cmd.Parameters.AddWithValue("@WorkerIsRunning", IsRunning); + cmd.Parameters.AddWithValue("@WorkerIsRunning", true); cmd.Parameters.AddWithValue("@ForceAcquire", false); // TODO: Provide ability to set. usefull for tests cmd.Parameters.AddWithValue("@LeasePeriodSec", PeriodSec + _leaseTimeoutSec); var leaseEndTimePar = cmd.Parameters.AddWithValue("@LeaseEndTime", DateTime.UtcNow); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs index eea8ded412..85dc260a8d 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs @@ -43,16 +43,11 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); } - await _defragWatchdog.StartAsync(stoppingToken); - await _cleanupEventLogWatchdog.StartAsync(stoppingToken); - await _transactionWatchdog.Value.StartAsync(stoppingToken); - await _invisibleHistoryCleanupWatchdog.StartAsync(stoppingToken); - - while (true) - { - stoppingToken.ThrowIfCancellationRequested(); - await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); - } + await Task.WhenAll( + _defragWatchdog.StartAsync(stoppingToken), + _cleanupEventLogWatchdog.StartAsync(stoppingToken), + _transactionWatchdog.Value.StartAsync(stoppingToken), + _invisibleHistoryCleanupWatchdog.StartAsync(stoppingToken)); } public Task Handle(StorageInitializedNotification notification, CancellationToken cancellationToken) @@ -63,10 +58,7 @@ public Task Handle(StorageInitializedNotification notification, CancellationToke public override void Dispose() { - _defragWatchdog.Dispose(); - _cleanupEventLogWatchdog.Dispose(); _transactionWatchdog.Dispose(); - _invisibleHistoryCleanupWatchdog.Dispose(); base.Dispose(); } } diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs index 1c258a3749..127eb7a94b 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs @@ -99,8 +99,6 @@ COMMIT TRANSACTION var sizeAfter = GetSize(); Assert.True(sizeAfter * 9 < sizeBefore, $"{sizeAfter} * 9 < {sizeBefore}"); - - wd.Dispose(); } [Fact] @@ -142,8 +140,6 @@ WHILE @i < 10000 _testOutputHelper.WriteLine($"EventLog.Count={GetCount("EventLog")}."); Assert.True(GetCount("EventLog") <= 1000, "Count is low"); - - wd.Dispose(); } [Fact] @@ -209,8 +205,6 @@ FOR INSERT } Assert.Equal(1, GetCount("NumberSearchParam")); // wd rolled forward transaction - - wd.Dispose(); } [Fact] @@ -278,8 +272,6 @@ public async Task AdvanceVisibility() _testOutputHelper.WriteLine($"Visibility={visibility}"); Assert.Equal(tran3.TransactionId, visibility); - - wd.Dispose(); } private ResourceWrapperFactory CreateResourceWrapperFactory() From be526273a5d83697e13cc4e7cd4df8690c578405 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Mon, 15 Apr 2024 17:04:04 -0700 Subject: [PATCH 22/47] Subscription infra --- Microsoft.Health.Fhir.sln | 7 ++ .../Microsoft.Health.Fhir.Api.csproj | 1 + .../FhirServerServiceCollectionExtensions.cs | 2 + .../Features/Operations/JobType.cs | 2 + .../Features/Operations/QueueType.cs | 1 + .../Storage/SqlRetry/ISqlRetryService.cs | 4 +- .../Storage/SqlRetry/SqlCommandExtensions.cs | 6 +- .../Storage/SqlRetry/SqlRetryService.cs | 8 +- .../Storage/SqlServerFhirDataStore.cs | 9 +- .../Features/Storage/SqlStoreClient.cs | 7 +- .../Watchdogs/EventProcessorWatchdog.cs | 109 ++++++++++++++++++ .../InvisibleHistoryCleanupWatchdog.cs | 4 +- .../Watchdogs/WatchdogsBackgroundService.cs | 8 +- .../Microsoft.Health.Fhir.SqlServer.csproj | 1 + ...rBuilderSqlServerRegistrationExtensions.cs | 6 +- .../Channels/ISubscriptionChannel.cs | 17 +++ .../Channels/StorageChannel.cs | 17 +++ ...Microsoft.Health.Fhir.Subscriptions.csproj | 12 ++ .../Models/ChannelInfo.cs | 35 ++++++ .../Models/SubscriptionChannelType.cs | 21 ++++ .../Models/SubscriptionContentType.cs | 14 +++ .../Models/SubscriptionInfo.cs | 24 ++++ .../Models/SubscriptionJobDefinition.cs | 31 +++++ .../Operations/SubscriptionProcessingJob.cs | 24 ++++ .../SubscriptionsOrchestratorJob.cs | 51 ++++++++ .../Registration/SubscriptionsModule.cs | 28 +++++ .../SubscriptionManager.cs | 22 ++++ ...erFhirResourceChangeCaptureEnabledTests.cs | 4 +- .../Persistence/SqlRetryServiceTests.cs | 4 +- 29 files changed, 452 insertions(+), 27 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionChannelType.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionContentType.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/SubscriptionManager.cs diff --git a/Microsoft.Health.Fhir.sln b/Microsoft.Health.Fhir.sln index 788977da40..bc333070ea 100644 --- a/Microsoft.Health.Fhir.sln +++ b/Microsoft.Health.Fhir.sln @@ -205,6 +205,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Cosmo EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.CosmosDb.Core", "src\Microsoft.Health.Fhir.CosmosDb.Core\Microsoft.Health.Fhir.CosmosDb.Core.csproj", "{1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Subscriptions", "src\Microsoft.Health.Fhir.Subscriptions\Microsoft.Health.Fhir.Subscriptions.csproj", "{BD8F3137-89F5-4EE5-B269-24D73081E00A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -483,6 +485,10 @@ Global {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75}.Debug|Any CPU.Build.0 = Debug|Any CPU {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75}.Release|Any CPU.ActiveCfg = Release|Any CPU {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75}.Release|Any CPU.Build.0 = Release|Any CPU + {BD8F3137-89F5-4EE5-B269-24D73081E00A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BD8F3137-89F5-4EE5-B269-24D73081E00A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BD8F3137-89F5-4EE5-B269-24D73081E00A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BD8F3137-89F5-4EE5-B269-24D73081E00A}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -578,6 +584,7 @@ Global {10661BC9-01B0-4E35-9751-3B5CE97E25C0} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} {B9AAA11D-8C8C-44C3-AADE-801376EF82F0} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} + {BD8F3137-89F5-4EE5-B269-24D73081E00A} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution RESX_SortFileContentOnSave = True diff --git a/src/Microsoft.Health.Fhir.Api/Microsoft.Health.Fhir.Api.csproj b/src/Microsoft.Health.Fhir.Api/Microsoft.Health.Fhir.Api.csproj index 815c230b38..e91e43ea78 100644 --- a/src/Microsoft.Health.Fhir.Api/Microsoft.Health.Fhir.Api.csproj +++ b/src/Microsoft.Health.Fhir.Api/Microsoft.Health.Fhir.Api.csproj @@ -26,6 +26,7 @@ + diff --git a/src/Microsoft.Health.Fhir.Api/Registration/FhirServerServiceCollectionExtensions.cs b/src/Microsoft.Health.Fhir.Api/Registration/FhirServerServiceCollectionExtensions.cs index 855a5dee9f..d22a5b88dd 100644 --- a/src/Microsoft.Health.Fhir.Api/Registration/FhirServerServiceCollectionExtensions.cs +++ b/src/Microsoft.Health.Fhir.Api/Registration/FhirServerServiceCollectionExtensions.cs @@ -8,6 +8,7 @@ using Microsoft.Health.Api.Modules; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Api.Configs; +using Microsoft.Health.Fhir.Subscriptions.Registration; namespace Microsoft.Extensions.DependencyInjection { @@ -20,6 +21,7 @@ public static IServiceCollection AddFhirServerBase(this IServiceCollection servi services.RegisterAssemblyModules(Assembly.GetExecutingAssembly(), fhirServerConfiguration); services.RegisterAssemblyModules(typeof(InitializationModule).Assembly, fhirServerConfiguration); + services.RegisterAssemblyModules(typeof(SubscriptionsModule).Assembly, fhirServerConfiguration); return services; } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/JobType.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/JobType.cs index f51fc9064d..73dc713b99 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/JobType.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/JobType.cs @@ -14,5 +14,7 @@ public enum JobType : int ExportOrchestrator = 4, BulkDeleteProcessing = 5, BulkDeleteOrchestrator = 6, + SubscriptionsProcessing = 7, + SubscriptionsOrchestrator = 8, } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/QueueType.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/QueueType.cs index 9affedc7c0..cd1529e32f 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/QueueType.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/QueueType.cs @@ -12,6 +12,7 @@ public enum QueueType : byte Import = 2, Defrag = 3, BulkDelete = 4, + Subscriptions = 5, } } #pragma warning restore CA1028 // Enum Storage should be Int32 diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/ISqlRetryService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/ISqlRetryService.cs index ab00060ffd..9a1928a0c2 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/ISqlRetryService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/ISqlRetryService.cs @@ -18,8 +18,8 @@ public interface ISqlRetryService Task ExecuteSql(Func action, ILogger logger, CancellationToken cancellationToken, bool isReadOnly = false); - Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false); + Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false); - Task> ExecuteReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false); + Task> ExecuteReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false); } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlCommandExtensions.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlCommandExtensions.cs index 809c30f686..1bc3b13281 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlCommandExtensions.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlCommandExtensions.cs @@ -14,17 +14,17 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage { public static class SqlCommandExtensions { - public static async Task ExecuteNonQueryAsync(this SqlCommand cmd, ISqlRetryService retryService, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false, bool disableRetries = false) + public static async Task ExecuteNonQueryAsync(this SqlCommand cmd, ISqlRetryService retryService, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false, bool disableRetries = false) { await retryService.ExecuteSql(cmd, async (sql, cancel) => await sql.ExecuteNonQueryAsync(cancel), logger, logMessage, cancellationToken, isReadOnly, disableRetries); } - public static async Task> ExecuteReaderAsync(this SqlCommand cmd, ISqlRetryService retryService, Func readerToResult, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false) + public static async Task> ExecuteReaderAsync(this SqlCommand cmd, ISqlRetryService retryService, Func readerToResult, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false) { return await retryService.ExecuteReaderAsync(cmd, readerToResult, logger, logMessage, cancellationToken, isReadOnly); } - public static async Task ExecuteScalarAsync(this SqlCommand cmd, ISqlRetryService retryService, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false, bool disableRetries = false) + public static async Task ExecuteScalarAsync(this SqlCommand cmd, ISqlRetryService retryService, ILogger logger, CancellationToken cancellationToken, string logMessage = null, bool isReadOnly = false, bool disableRetries = false) { object scalar = null; await retryService.ExecuteSql(cmd, async (sql, cancel) => { scalar = await sql.ExecuteScalarAsync(cancel); }, logger, logMessage, cancellationToken, isReadOnly, disableRetries); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs index c230103a2e..ac2c53d1e4 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs @@ -246,7 +246,6 @@ public async Task ExecuteSql(Func - /// Type used for the . /// SQL command to be executed. /// Delegate to be executed by passing as input parameter. /// Logger used on first try error (or retry error) and connection open. @@ -256,7 +255,7 @@ public async Task ExecuteSql(Func"Flag indicating whether retries are disabled." /// A task representing the asynchronous operation. /// When executing this method, if exception is thrown that is not retriable or if last retry fails, then same exception is thrown by this method. - public async Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false) + public async Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false) { EnsureArg.IsNotNull(sqlCommand, nameof(sqlCommand)); EnsureArg.IsNotNull(action, nameof(action)); @@ -308,7 +307,7 @@ public async Task ExecuteSql(SqlCommand sqlCommand, Func> ExecuteSqlDataReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, bool allRows, bool isReadOnly, CancellationToken cancellationToken) + private async Task> ExecuteSqlDataReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, bool allRows, bool isReadOnly, CancellationToken cancellationToken) { EnsureArg.IsNotNull(sqlCommand, nameof(sqlCommand)); EnsureArg.IsNotNull(readerToResult, nameof(readerToResult)); @@ -344,7 +343,6 @@ await ExecuteSql( /// SQL connection error. In the case if non-retriable exception or if the last retry failed tha same exception is thrown. /// /// Defines data type for the returned SQL rows. - /// Type used for the . /// SQL command to be executed. /// Translation delegate that translates the row returned by execution into the data type. /// Logger used on first try error or retry error. @@ -354,7 +352,7 @@ await ExecuteSql( /// A task representing the asynchronous operation that returns all the rows that result from execution. The rows are translated by delegate /// into data type. /// When executing this method, if exception is thrown that is not retriable or if last retry fails, then same exception is thrown by this method. - public async Task> ExecuteReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false) + public async Task> ExecuteReaderAsync(SqlCommand sqlCommand, Func readerToResult, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false) { return await ExecuteSqlDataReaderAsync(sqlCommand, readerToResult, logger, logMessage, true, isReadOnly, cancellationToken); } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs index 72b116da30..4a7a1767da 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs @@ -52,7 +52,7 @@ internal class SqlServerFhirDataStore : IFhirDataStore, IProvideCapability private readonly IBundleOrchestrator _bundleOrchestrator; private readonly CoreFeatureConfiguration _coreFeatures; private readonly ISqlRetryService _sqlRetryService; - private readonly SqlStoreClient _sqlStoreClient; + private readonly SqlStoreClient _sqlStoreClient; private readonly SqlConnectionWrapperFactory _sqlConnectionWrapperFactory; private readonly ICompressedRawResourceConverter _compressedRawResourceConverter; private readonly ILogger _logger; @@ -76,14 +76,15 @@ public SqlServerFhirDataStore( SchemaInformation schemaInformation, IModelInfoProvider modelInfoProvider, RequestContextAccessor requestContextAccessor, - IImportErrorSerializer importErrorSerializer) + IImportErrorSerializer importErrorSerializer, + SqlStoreClient storeClient) { _model = EnsureArg.IsNotNull(model, nameof(model)); _searchParameterTypeMap = EnsureArg.IsNotNull(searchParameterTypeMap, nameof(searchParameterTypeMap)); _coreFeatures = EnsureArg.IsNotNull(coreFeatures?.Value, nameof(coreFeatures)); _bundleOrchestrator = EnsureArg.IsNotNull(bundleOrchestrator, nameof(bundleOrchestrator)); _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); - _sqlStoreClient = new SqlStoreClient(_sqlRetryService, logger); + _sqlStoreClient = EnsureArg.IsNotNull(storeClient, nameof(storeClient)); _sqlConnectionWrapperFactory = EnsureArg.IsNotNull(sqlConnectionWrapperFactory, nameof(sqlConnectionWrapperFactory)); _compressedRawResourceConverter = EnsureArg.IsNotNull(compressedRawResourceConverter, nameof(compressedRawResourceConverter)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); @@ -119,7 +120,7 @@ public SqlServerFhirDataStore( } } - internal SqlStoreClient StoreClient => _sqlStoreClient; + internal SqlStoreClient StoreClient => _sqlStoreClient; internal static TimeSpan MergeResourcesTransactionHeartbeatPeriod => TimeSpan.FromSeconds(10); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs index 309656f327..5d4daf17b3 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs @@ -25,14 +25,13 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage /// /// Lightweight SQL store client. /// - /// class used in logger - internal class SqlStoreClient + internal class SqlStoreClient { private readonly ISqlRetryService _sqlRetryService; - private readonly ILogger _logger; + private readonly ILogger _logger; private const string _invisibleResource = " "; - public SqlStoreClient(ISqlRetryService sqlRetryService, ILogger logger) + public SqlStoreClient(ISqlRetryService sqlRetryService, ILogger logger) { _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs new file mode 100644 index 0000000000..b8270b7ad7 --- /dev/null +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs @@ -0,0 +1,109 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.SqlServer.Features.Storage; +using Microsoft.Health.Fhir.Subscriptions.Models; +using Microsoft.Health.JobManagement; + +namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs +{ + internal class EventProcessorWatchdog : Watchdog + { + private readonly SqlStoreClient _store; + private readonly ILogger _logger; + private readonly ISqlRetryService _sqlRetryService; + private readonly IQueueClient _queueClient; + private CancellationToken _cancellationToken; + + public EventProcessorWatchdog( + SqlStoreClient store, + ISqlRetryService sqlRetryService, + IQueueClient queueClient, + ILogger logger) + : base(sqlRetryService, logger) + { + _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); + _store = EnsureArg.IsNotNull(store, nameof(store)); + _logger = EnsureArg.IsNotNull(logger, nameof(logger)); + _logger = EnsureArg.IsNotNull(logger, nameof(logger)); + _queueClient = EnsureArg.IsNotNull(queueClient, nameof(queueClient)); + } + + internal string LastEventProcessedTransactionId => $"{Name}.{nameof(LastEventProcessedTransactionId)}"; + + internal async Task StartAsync(CancellationToken cancellationToken, double? periodSec = null, double? leasePeriodSec = null, double? retentionPeriodDays = null) + { + _cancellationToken = cancellationToken; + await InitLastProcessedTransactionId(); + await StartAsync(true, periodSec ?? 3600, leasePeriodSec ?? 2 * 3600, cancellationToken); + } + + protected override async Task ExecuteAsync() + { + _logger.LogInformation($"{Name}: starting..."); + var lastTranId = await GetLastTransactionId(); + var visibility = await _store.MergeResourcesGetTransactionVisibilityAsync(_cancellationToken); + + _logger.LogDebug($"{Name}: last cleaned up transaction={lastTranId} visibility={visibility}."); + + var transactionsToProcess = await _store.GetTransactionsAsync(lastTranId, visibility, _cancellationToken, DateTime.UtcNow.AddDays((-1) * 7)); + _logger.LogDebug($"{Name}: found transactions={transactionsToProcess.Count}."); + + if (transactionsToProcess.Count == 0) + { + _logger.LogDebug($"{Name}: completed. transactions=0."); + return; + } + + var transactionsToQueue = new List(); + var totalRows = 0; + foreach (var tran in transactionsToProcess.Where(x => x.VisibleDate.HasValue).OrderBy(x => x.TransactionId)) + { + var jobDefinition = new SubscriptionJobDefinition(Core.Features.Operations.JobType.SubscriptionsOrchestrator) + { + TransactionId = tran.TransactionId, + VisibleDate = tran.VisibleDate.Value, + }; + + transactionsToQueue.Add(jobDefinition); + } + + await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken: _cancellationToken, definitions: transactionsToQueue.ToArray()); + await UpdateLastEventProcessedTransactionId(transactionsToProcess.Max(x => x.TransactionId)); + + _logger.LogDebug($"{Name}: completed. transactions={transactionsToProcess.Count} removed rows={totalRows}"); + } + + private async Task GetLastTransactionId() + { + return await GetLongParameterByIdAsync(LastEventProcessedTransactionId, _cancellationToken); + } + + private async Task InitLastProcessedTransactionId() + { + using var cmd = new SqlCommand("INSERT INTO dbo.Parameters (Id, Bigint) SELECT @Id, 5105975696064002770"); // surrogate id for the past. does not matter. + cmd.Parameters.AddWithValue("@Id", LastEventProcessedTransactionId); + await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); + } + + private async Task UpdateLastEventProcessedTransactionId(long lastTranId) + { + using var cmd = new SqlCommand("UPDATE dbo.Parameters SET Bigint = @LastTranId WHERE Id = @Id"); + cmd.Parameters.AddWithValue("@Id", LastEventProcessedTransactionId); + cmd.Parameters.AddWithValue("@LastTranId", lastTranId); + await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); + } + } +} diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs index 8e978eb98c..eddcded1d8 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs @@ -16,13 +16,13 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { internal class InvisibleHistoryCleanupWatchdog : Watchdog { - private readonly SqlStoreClient _store; + private readonly SqlStoreClient _store; private readonly ILogger _logger; private readonly ISqlRetryService _sqlRetryService; private CancellationToken _cancellationToken; private double _retentionPeriodDays = 7; - public InvisibleHistoryCleanupWatchdog(SqlStoreClient store, ISqlRetryService sqlRetryService, ILogger logger) + public InvisibleHistoryCleanupWatchdog(SqlStoreClient store, ISqlRetryService sqlRetryService, ILogger logger) : base(sqlRetryService, logger) { _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs index 85dc260a8d..63f3cfb639 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs @@ -22,17 +22,20 @@ internal class WatchdogsBackgroundService : BackgroundService, INotificationHand private readonly CleanupEventLogWatchdog _cleanupEventLogWatchdog; private readonly IScoped _transactionWatchdog; private readonly InvisibleHistoryCleanupWatchdog _invisibleHistoryCleanupWatchdog; + private readonly EventProcessorWatchdog _eventProcessorWatchdog; public WatchdogsBackgroundService( DefragWatchdog defragWatchdog, CleanupEventLogWatchdog cleanupEventLogWatchdog, IScopeProvider transactionWatchdog, - InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog) + InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog, + EventProcessorWatchdog eventProcessorWatchdog) { _defragWatchdog = EnsureArg.IsNotNull(defragWatchdog, nameof(defragWatchdog)); _cleanupEventLogWatchdog = EnsureArg.IsNotNull(cleanupEventLogWatchdog, nameof(cleanupEventLogWatchdog)); _transactionWatchdog = EnsureArg.IsNotNull(transactionWatchdog, nameof(transactionWatchdog)).Invoke(); _invisibleHistoryCleanupWatchdog = EnsureArg.IsNotNull(invisibleHistoryCleanupWatchdog, nameof(invisibleHistoryCleanupWatchdog)); + _eventProcessorWatchdog = EnsureArg.IsNotNull(eventProcessorWatchdog, nameof(eventProcessorWatchdog)); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -47,7 +50,8 @@ await Task.WhenAll( _defragWatchdog.StartAsync(stoppingToken), _cleanupEventLogWatchdog.StartAsync(stoppingToken), _transactionWatchdog.Value.StartAsync(stoppingToken), - _invisibleHistoryCleanupWatchdog.StartAsync(stoppingToken)); + _invisibleHistoryCleanupWatchdog.StartAsync(stoppingToken), + _eventProcessorWatchdog.StartAsync(stoppingToken)); } public Task Handle(StorageInitializedNotification notification, CancellationToken cancellationToken) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj index 062b3dd869..fffb1ebd3a 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj +++ b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj @@ -54,6 +54,7 @@ + diff --git a/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs index c706e47741..eb94941a91 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs @@ -162,7 +162,9 @@ public static IFhirServerBuilder AddSqlServer(this IFhirServerBuilder fhirServer .Singleton() .AsSelf(); - services.Add>().Singleton().AsSelf(); + services.Add() + .Singleton() + .AsSelf(); services.Add().Singleton().AsSelf(); @@ -173,6 +175,8 @@ public static IFhirServerBuilder AddSqlServer(this IFhirServerBuilder fhirServer services.Add().Singleton().AsSelf(); + services.Add().Singleton().AsSelf(); + services.RemoveServiceTypeExact>() // Mediatr registers handlers as Transient by default, this extension ensures these aren't still there, only needed when service != Transient .Add() .Singleton() diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs new file mode 100644 index 0000000000..fc13912923 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Health.Fhir.Subscriptions.Channels +{ + internal interface ISubscriptionChannel + { + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs new file mode 100644 index 0000000000..fa0c938cdc --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Health.Fhir.Subscriptions.Channels +{ + public class StorageChannel : ISubscriptionChannel + { + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj new file mode 100644 index 0000000000..586eb07eae --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj @@ -0,0 +1,12 @@ + + + + enable + enable + + + + + + + diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs new file mode 100644 index 0000000000..a5b9aaf389 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs @@ -0,0 +1,35 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.Health.Fhir.Subscriptions.Models +{ + public class ChannelInfo + { + /// + /// Interval to send 'heartbeat' notification + /// + public TimeSpan HeartBeatPeriod { get; set; } + + /// + /// Timeout to attempt notification delivery + /// + public TimeSpan Timeout { get; set; } + + /// + /// Maximum number of triggering resources included in notification bundles + /// + public int MaxCount { get; set; } + + public SubscriptionChannelType ChannelType { get; set; } + + public SubscriptionContentType ContentType { get; set; } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionChannelType.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionChannelType.cs new file mode 100644 index 0000000000..5b67f98271 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionChannelType.cs @@ -0,0 +1,21 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Subscriptions.Models +{ + public enum SubscriptionChannelType + { + None = 0, + RestHook = 1, + WebSocket = 2, + Email = 3, + FhirMessaging = 4, + + // Custom Channels + EventGrid = 5, + Storage = 6, + DatalakeContract = 7, + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionContentType.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionContentType.cs new file mode 100644 index 0000000000..1eee162f7d --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionContentType.cs @@ -0,0 +1,14 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +namespace Microsoft.Health.Fhir.Subscriptions.Models +{ + public enum SubscriptionContentType + { + Empty, + IdOnly, + FullResource, + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs new file mode 100644 index 0000000000..31d2c782ec --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using EnsureThat; + +namespace Microsoft.Health.Fhir.Subscriptions.Models +{ + public class SubscriptionInfo + { + public SubscriptionInfo(string filterCriteria) + { + FilterCriteria = EnsureArg.IsNotNullOrEmpty(filterCriteria, nameof(filterCriteria)); + } + + public string FilterCriteria { get; set; } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs new file mode 100644 index 0000000000..a5202183e6 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs @@ -0,0 +1,31 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.JobManagement; +using Newtonsoft.Json; + +namespace Microsoft.Health.Fhir.Subscriptions.Models +{ + public class SubscriptionJobDefinition : IJobData + { + public SubscriptionJobDefinition(JobType jobType) + { + TypeId = (int)jobType; + } + + [JsonProperty(JobRecordProperties.TypeId)] + public int TypeId { get; set; } + + public long TransactionId { get; set; } + + public DateTime VisibleDate { get; set; } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs new file mode 100644 index 0000000000..b1b37e5eaf --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -0,0 +1,24 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.JobManagement; + +namespace Microsoft.Health.Fhir.Subscriptions.Operations +{ + [JobTypeId((int)JobType.SubscriptionsProcessing)] + public class SubscriptionProcessingJob : IJob + { + public Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) + { + return Task.FromResult("Done!"); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs new file mode 100644 index 0000000000..c4809bc6eb --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs @@ -0,0 +1,51 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Subscriptions.Models; +using Microsoft.Health.JobManagement; + +namespace Microsoft.Health.Fhir.Subscriptions.Operations +{ + [JobTypeId((int)JobType.SubscriptionsOrchestrator)] + public class SubscriptionsOrchestratorJob : IJob + { + private readonly IQueueClient _queueClient; + private readonly Func> _searchService; + private const string OperationCompleted = "Completed"; + + public SubscriptionsOrchestratorJob( + IQueueClient queueClient, + Func> searchService) + { + EnsureArg.IsNotNull(queueClient, nameof(queueClient)); + EnsureArg.IsNotNull(searchService, nameof(searchService)); + + _queueClient = queueClient; + _searchService = searchService; + } + + public Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) + { + EnsureArg.IsNotNull(jobInfo, nameof(jobInfo)); + + SubscriptionJobDefinition definition = jobInfo.DeserializeDefinition(); + + // Get and evaluate the active subscriptions ... + + // await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken, jobInfo.GroupId, definitions: processingDefinition); + + return Task.FromResult(OperationCompleted); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs new file mode 100644 index 0000000000..69cc743e18 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs @@ -0,0 +1,28 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.JobManagement; + +namespace Microsoft.Health.Fhir.Subscriptions.Registration +{ + public class SubscriptionsModule : IStartupModule + { + public void Load(IServiceCollection services) + { + services.TypesInSameAssemblyAs() + .AssignableTo() + .Transient() + .AsSelf() + .AsService(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/SubscriptionManager.cs new file mode 100644 index 0000000000..b7c03130e0 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/SubscriptionManager.cs @@ -0,0 +1,22 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions +{ + public class SubscriptionManager + { + public Task> GetActiveSubscriptionsAsync() + { + throw new NotImplementedException(); + } + } +} diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs index 0428f93648..9655ceaa84 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs @@ -140,7 +140,7 @@ public async Task GivenChangeCaptureEnabledAndNoVersionPolicy_AfterUpdating_Invi using var cts = new CancellationTokenSource(); cts.CancelAfter(TimeSpan.FromSeconds(60)); - var storeClient = new SqlStoreClient(_fixture.SqlRetryService, NullLogger.Instance); + var storeClient = new SqlStoreClient(_fixture.SqlRetryService, NullLogger.Instance); var wd = new InvisibleHistoryCleanupWatchdog(storeClient, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)); await wd.StartAsync(cts.Token, 1, 2, 2.0 / 24 / 3600); // retention 2 seconds var startTime = DateTime.UtcNow; @@ -188,7 +188,7 @@ public async Task GivenChangeCaptureEnabledAndNoVersionPolicy_AfterHardDeleting_ using var cts = new CancellationTokenSource(); cts.CancelAfter(TimeSpan.FromSeconds(60)); - var storeClient = new SqlStoreClient(_fixture.SqlRetryService, NullLogger.Instance); + var storeClient = new SqlStoreClient(_fixture.SqlRetryService, NullLogger.Instance); var wd = new InvisibleHistoryCleanupWatchdog(storeClient, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)); await wd.StartAsync(cts.Token, 1, 2, 2.0 / 24 / 3600); // retention 2 seconds var startTime = DateTime.UtcNow; diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs index 19e1d124a6..1178d612b1 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlRetryServiceTests.cs @@ -380,7 +380,7 @@ private async Task SingleConnectionRetryTest(Func testStor using var sqlCommand = new SqlCommand(); sqlCommand.CommandText = $"dbo.{storedProcedureName}"; - var result = await sqlRetryService.ExecuteReaderAsync( + var result = await sqlRetryService.ExecuteReaderAsync( sqlCommand, testConnectionInitializationFailure ? ReaderToResult : ReaderToResultAndKillConnection, logger, @@ -420,7 +420,7 @@ private async Task AllConnectionRetriesTest(Func testStore try { _output.WriteLine($"{DateTime.Now:O}: Start executing ExecuteSqlDataReader."); - await sqlRetryService.ExecuteReaderAsync( + await sqlRetryService.ExecuteReaderAsync( sqlCommand, testConnectionInitializationFailure ? ReaderToResult : ReaderToResultAndKillConnection, logger, From 22e1ce22563d2fd205fd5f4f345268f4d869a1b7 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Mon, 15 Apr 2024 20:17:58 -0700 Subject: [PATCH 23/47] Fixes wiring up of Transaction Watchdog => Orchestrator --- .../Features/Persistence/ResourceKey.cs | 6 +++ .../appsettings.json | 4 ++ .../Storage/SqlServerFhirDataStore.cs | 5 ++- .../Watchdogs/EventProcessorWatchdog.cs | 13 +++--- .../Features/Watchdogs/Watchdog.cs | 7 ++- ...Microsoft.Health.Fhir.Subscriptions.csproj | 1 - .../Models/SubscriptionInfo.cs | 5 ++- .../Models/SubscriptionJobDefinition.cs | 16 +++++++ .../Operations/SubscriptionProcessingJob.cs | 2 + .../SubscriptionsOrchestratorJob.cs | 44 +++++++++++++++---- .../Persistence/ISubscriptionManager.cs | 14 ++++++ .../ITransactionDataStore.cs} | 11 ++--- .../Persistence/SubscriptionManager.cs | 34 ++++++++++++++ .../Registration/SubscriptionsModule.cs | 15 ++++++- .../JobHosting.cs | 2 +- .../SqlServerFhirStorageTestsFixture.cs | 3 +- 16 files changed, 151 insertions(+), 31 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs rename src/Microsoft.Health.Fhir.Subscriptions/{SubscriptionManager.cs => Persistence/ITransactionDataStore.cs} (62%) create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs diff --git a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs index 9f27f96022..08d83caa78 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs @@ -7,6 +7,7 @@ using System.Text; using EnsureThat; using Microsoft.Health.Fhir.Core.Models; +using Newtonsoft.Json; namespace Microsoft.Health.Fhir.Core.Features.Persistence { @@ -23,6 +24,11 @@ public ResourceKey(string resourceType, string id, string versionId = null) ResourceType = resourceType; } + [JsonConstructor] + protected ResourceKey() + { + } + public string Id { get; } public string VersionId { get; } diff --git a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json index 2b7fb43427..c91847252a 100644 --- a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json +++ b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json @@ -59,6 +59,10 @@ { "Queue": "BulkDelete", "UpdateProgressOnHeartbeat": false + }, + { + "Queue": "Subscriptions", + "UpdateProgressOnHeartbeat": false } ], "Export": { diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs index 4a7a1767da..20854aec0e 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs @@ -29,6 +29,7 @@ using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model; using Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration; using Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration.Merge; +using Microsoft.Health.Fhir.Subscriptions.Persistence; using Microsoft.Health.Fhir.ValueSets; using Microsoft.Health.SqlServer.Features.Client; using Microsoft.Health.SqlServer.Features.Schema; @@ -41,7 +42,7 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Storage /// /// A SQL Server-backed . /// - internal class SqlServerFhirDataStore : IFhirDataStore, IProvideCapability + internal class SqlServerFhirDataStore : IFhirDataStore, IProvideCapability, ITransactionDataStore { private const string InitialVersion = "1"; @@ -945,7 +946,7 @@ public void Build(ICapabilityStatementBuilder builder) } } - internal async Task> GetResourcesByTransactionIdAsync(long transactionId, CancellationToken cancellationToken) + public async Task> GetResourcesByTransactionIdAsync(long transactionId, CancellationToken cancellationToken) { return await _sqlStoreClient.GetResourcesByTransactionIdAsync(transactionId, _compressedRawResourceConverter.ReadCompressedRawResource, _model.GetResourceTypeName, cancellationToken); } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs index b8270b7ad7..31660110de 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs @@ -47,7 +47,7 @@ internal async Task StartAsync(CancellationToken cancellationToken, double? peri { _cancellationToken = cancellationToken; await InitLastProcessedTransactionId(); - await StartAsync(true, periodSec ?? 3600, leasePeriodSec ?? 2 * 3600, cancellationToken); + await StartAsync(true, periodSec ?? 3, leasePeriodSec ?? 20, cancellationToken); } protected override async Task ExecuteAsync() @@ -56,19 +56,20 @@ protected override async Task ExecuteAsync() var lastTranId = await GetLastTransactionId(); var visibility = await _store.MergeResourcesGetTransactionVisibilityAsync(_cancellationToken); - _logger.LogDebug($"{Name}: last cleaned up transaction={lastTranId} visibility={visibility}."); + _logger.LogInformation($"{Name}: last transaction={lastTranId} visibility={visibility}."); - var transactionsToProcess = await _store.GetTransactionsAsync(lastTranId, visibility, _cancellationToken, DateTime.UtcNow.AddDays((-1) * 7)); + var transactionsToProcess = await _store.GetTransactionsAsync(lastTranId, visibility, _cancellationToken); _logger.LogDebug($"{Name}: found transactions={transactionsToProcess.Count}."); if (transactionsToProcess.Count == 0) { - _logger.LogDebug($"{Name}: completed. transactions=0."); + await UpdateLastEventProcessedTransactionId(visibility); + _logger.LogInformation($"{Name}: completed. transactions=0."); return; } var transactionsToQueue = new List(); - var totalRows = 0; + foreach (var tran in transactionsToProcess.Where(x => x.VisibleDate.HasValue).OrderBy(x => x.TransactionId)) { var jobDefinition = new SubscriptionJobDefinition(Core.Features.Operations.JobType.SubscriptionsOrchestrator) @@ -83,7 +84,7 @@ protected override async Task ExecuteAsync() await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken: _cancellationToken, definitions: transactionsToQueue.ToArray()); await UpdateLastEventProcessedTransactionId(transactionsToProcess.Max(x => x.TransactionId)); - _logger.LogDebug($"{Name}: completed. transactions={transactionsToProcess.Count} removed rows={totalRows}"); + _logger.LogInformation($"{Name}: completed. transactions={transactionsToProcess.Count}"); } private async Task GetLastTransactionId() diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs index d68b671a44..95d2917234 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs @@ -51,8 +51,11 @@ protected internal async Task StartAsync(bool allowRebalance, double periodSec, { _logger.LogInformation($"{Name}.StartAsync: starting..."); await InitParamsAsync(periodSec, leasePeriodSec); - await StartAsync(_periodSec, cancellationToken); - await _watchdogLease.StartAsync(allowRebalance, _leasePeriodSec, cancellationToken); + + await Task.WhenAll( + StartAsync(_periodSec, cancellationToken), + _watchdogLease.StartAsync(allowRebalance, _leasePeriodSec, cancellationToken)); + _logger.LogInformation($"{Name}.StartAsync: completed."); } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj index 586eb07eae..a90c13a333 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj +++ b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj @@ -2,7 +2,6 @@ enable - enable diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs index 31d2c782ec..b2d2e39591 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs @@ -14,11 +14,14 @@ namespace Microsoft.Health.Fhir.Subscriptions.Models { public class SubscriptionInfo { - public SubscriptionInfo(string filterCriteria) + public SubscriptionInfo(string filterCriteria, ChannelInfo channel) { FilterCriteria = EnsureArg.IsNotNullOrEmpty(filterCriteria, nameof(filterCriteria)); + Channel = EnsureArg.IsNotNull(channel, nameof(channel)); } public string FilterCriteria { get; set; } + + public ChannelInfo Channel { get; set; } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs index a5202183e6..7ef7fade06 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs @@ -5,10 +5,12 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Text; using System.Threading.Tasks; using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.JobManagement; using Newtonsoft.Json; @@ -21,11 +23,25 @@ public SubscriptionJobDefinition(JobType jobType) TypeId = (int)jobType; } + [JsonConstructor] + protected SubscriptionJobDefinition() + { + } + [JsonProperty(JobRecordProperties.TypeId)] public int TypeId { get; set; } + [JsonProperty("transactionId")] public long TransactionId { get; set; } + [JsonProperty("visibleDate")] public DateTime VisibleDate { get; set; } + + [JsonProperty("resourceReferences")] + [SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "JSON Poco")] + public IList ResourceReferences { get; set; } + + [JsonProperty("channel")] + public ChannelInfo Channel { get; set; } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs index b1b37e5eaf..d62f584a63 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -18,6 +18,8 @@ public class SubscriptionProcessingJob : IJob { public Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) { + // TODO: Write resource to channel + return Task.FromResult("Done!"); } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs index c4809bc6eb..9c2efab149 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs @@ -10,9 +10,12 @@ using System.Threading.Tasks; using EnsureThat; using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Subscriptions.Models; +using Microsoft.Health.Fhir.Subscriptions.Persistence; using Microsoft.Health.JobManagement; namespace Microsoft.Health.Fhir.Subscriptions.Operations @@ -21,31 +24,56 @@ namespace Microsoft.Health.Fhir.Subscriptions.Operations public class SubscriptionsOrchestratorJob : IJob { private readonly IQueueClient _queueClient; - private readonly Func> _searchService; + private readonly ITransactionDataStore _transactionDataStore; + private readonly ISubscriptionManager _subscriptionManager; private const string OperationCompleted = "Completed"; public SubscriptionsOrchestratorJob( IQueueClient queueClient, - Func> searchService) + ITransactionDataStore transactionDataStore, + ISubscriptionManager subscriptionManager) { EnsureArg.IsNotNull(queueClient, nameof(queueClient)); - EnsureArg.IsNotNull(searchService, nameof(searchService)); + EnsureArg.IsNotNull(transactionDataStore, nameof(transactionDataStore)); + EnsureArg.IsNotNull(subscriptionManager, nameof(subscriptionManager)); _queueClient = queueClient; - _searchService = searchService; + _transactionDataStore = transactionDataStore; + _subscriptionManager = subscriptionManager; } - public Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) + public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) { EnsureArg.IsNotNull(jobInfo, nameof(jobInfo)); SubscriptionJobDefinition definition = jobInfo.DeserializeDefinition(); + var resources = await _transactionDataStore.GetResourcesByTransactionIdAsync(definition.TransactionId, cancellationToken); - // Get and evaluate the active subscriptions ... + var processingDefinition = new List(); - // await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken, jobInfo.GroupId, definitions: processingDefinition); + foreach (var sub in await _subscriptionManager.GetActiveSubscriptionsAsync(cancellationToken)) + { + var chunk = resources + //// TODO: .Where(r => sub.FilterCriteria does something??); + .Chunk(sub.Channel.MaxCount); - return Task.FromResult(OperationCompleted); + foreach (var batch in chunk) + { + var cloneDefinition = jobInfo.DeserializeDefinition(); + cloneDefinition.TypeId = (int)JobType.SubscriptionsProcessing; + cloneDefinition.ResourceReferences = batch.Select(r => new ResourceKey(r.ResourceTypeName, r.ResourceId, r.Version)).ToList(); + cloneDefinition.Channel = sub.Channel; + + processingDefinition.Add(cloneDefinition); + } + } + + if (processingDefinition.Count > 0) + { + await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken, jobInfo.GroupId, definitions: processingDefinition.ToArray()); + } + + return OperationCompleted; } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs new file mode 100644 index 0000000000..a5f2738457 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs @@ -0,0 +1,14 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Persistence +{ + public interface ISubscriptionManager + { + Task> GetActiveSubscriptionsAsync(CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs similarity index 62% rename from src/Microsoft.Health.Fhir.Subscriptions/SubscriptionManager.cs rename to src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs index b7c03130e0..6a1ca37224 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/SubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs @@ -8,15 +8,12 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -using Microsoft.Health.Fhir.Subscriptions.Models; +using Microsoft.Health.Fhir.Core.Features.Persistence; -namespace Microsoft.Health.Fhir.Subscriptions +namespace Microsoft.Health.Fhir.Subscriptions.Persistence { - public class SubscriptionManager + public interface ITransactionDataStore { - public Task> GetActiveSubscriptionsAsync() - { - throw new NotImplementedException(); - } + Task> GetResourcesByTransactionIdAsync(long transactionId, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs new file mode 100644 index 0000000000..b53ef16587 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Persistence +{ + public class SubscriptionManager : ISubscriptionManager + { + public Task> GetActiveSubscriptionsAsync(CancellationToken cancellationToken) + { + IReadOnlyCollection list = new List + { + new SubscriptionInfo( + "Resource", + new ChannelInfo + { + ChannelType = SubscriptionChannelType.Storage, + ContentType = SubscriptionContentType.FullResource, + MaxCount = 100, + }), + }; + + return Task.FromResult(list); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs index 69cc743e18..bbf12b8002 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs @@ -9,7 +9,9 @@ using System.Text; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Subscriptions.Persistence; using Microsoft.Health.JobManagement; namespace Microsoft.Health.Fhir.Subscriptions.Registration @@ -18,11 +20,20 @@ public class SubscriptionsModule : IStartupModule { public void Load(IServiceCollection services) { - services.TypesInSameAssemblyAs() + IEnumerable jobs = services.TypesInSameAssemblyAs() .AssignableTo() .Transient() + .AsSelf(); + + foreach (TypeRegistrationBuilder job in jobs) + { + job.AsDelegate>(); + } + + services.Add() + .Singleton() .AsSelf() - .AsService(); + .AsImplementedInterfaces(); } } } diff --git a/src/Microsoft.Health.TaskManagement/JobHosting.cs b/src/Microsoft.Health.TaskManagement/JobHosting.cs index 7a662d668f..d0d7db83ba 100644 --- a/src/Microsoft.Health.TaskManagement/JobHosting.cs +++ b/src/Microsoft.Health.TaskManagement/JobHosting.cs @@ -61,7 +61,7 @@ public async Task ExecuteAsync(byte queueType, short runningJobCount, string wor { try { - _logger.LogInformation("Dequeuing next job."); + _logger.LogInformation("Dequeuing next job on {QueueType}.", queueType); if (checkTimeoutJobStopwatch.Elapsed.TotalSeconds > 600) { diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs index ad53e6ecef..2610fbf534 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerFhirStorageTestsFixture.cs @@ -244,7 +244,8 @@ public async Task InitializeAsync() SchemaInformation, ModelInfoProvider.Instance, _fhirRequestContextAccessor, - importErrorSerializer); + importErrorSerializer, + new SqlStoreClient(SqlRetryService, NullLogger.Instance)); _fhirOperationDataStore = new SqlServerFhirOperationDataStore(SqlConnectionWrapperFactory, queueClient, NullLogger.Instance, NullLoggerFactory.Instance); From 186ecaf2b35b08aa7976040dada0548a2efb4064 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Tue, 16 Apr 2024 09:03:35 -0700 Subject: [PATCH 24/47] Adding code for writing to storage --- R4.slnf | 3 +- ...og.cs => SubscriptionProcessorWatchdog.cs} | 8 +-- .../Watchdogs/WatchdogsBackgroundService.cs | 32 ++++++--- ...rBuilderSqlServerRegistrationExtensions.cs | 2 +- ...Microsoft.Health.Fhir.Subscriptions.csproj | 1 + .../Operations/SubscriptionProcessingJob.cs | 68 ++++++++++++++++++- 6 files changed, 97 insertions(+), 17 deletions(-) rename src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/{EventProcessorWatchdog.cs => SubscriptionProcessorWatchdog.cs} (94%) diff --git a/R4.slnf b/R4.slnf index f3207945d8..7adecaed1e 100644 --- a/R4.slnf +++ b/R4.slnf @@ -29,6 +29,7 @@ "src\\Microsoft.Health.Fhir.Shared.Web\\Microsoft.Health.Fhir.Shared.Web.shproj", "src\\Microsoft.Health.Fhir.SqlServer.UnitTests\\Microsoft.Health.Fhir.SqlServer.UnitTests.csproj", "src\\Microsoft.Health.Fhir.SqlServer\\Microsoft.Health.Fhir.SqlServer.csproj", + "src\\Microsoft.Health.Fhir.Subscriptions\\Microsoft.Health.Fhir.Subscriptions.csproj", "src\\Microsoft.Health.Fhir.Tests.Common\\Microsoft.Health.Fhir.Tests.Common.csproj", "src\\Microsoft.Health.TaskManagement\\Microsoft.Health.TaskManagement.csproj", "src\\Microsoft.Health.TaskManagement.UnitTests\\Microsoft.Health.TaskManagement.UnitTests.csproj", @@ -43,4 +44,4 @@ "test\\Microsoft.Health.Fhir.Shared.Tests.Integration\\Microsoft.Health.Fhir.Shared.Tests.Integration.shproj" ] } -} +} \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs similarity index 94% rename from src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs rename to src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs index 31660110de..5f31720eda 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/EventProcessorWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs @@ -19,19 +19,19 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - internal class EventProcessorWatchdog : Watchdog + internal class SubscriptionProcessorWatchdog : Watchdog { private readonly SqlStoreClient _store; - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly ISqlRetryService _sqlRetryService; private readonly IQueueClient _queueClient; private CancellationToken _cancellationToken; - public EventProcessorWatchdog( + public SubscriptionProcessorWatchdog( SqlStoreClient store, ISqlRetryService sqlRetryService, IQueueClient queueClient, - ILogger logger) + ILogger logger) : base(sqlRetryService, logger) { _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs index 63f3cfb639..bafa0b1c03 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs @@ -4,12 +4,14 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using EnsureThat; using MediatR; using Microsoft.Extensions.Hosting; using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Messages.Storage; @@ -22,14 +24,14 @@ internal class WatchdogsBackgroundService : BackgroundService, INotificationHand private readonly CleanupEventLogWatchdog _cleanupEventLogWatchdog; private readonly IScoped _transactionWatchdog; private readonly InvisibleHistoryCleanupWatchdog _invisibleHistoryCleanupWatchdog; - private readonly EventProcessorWatchdog _eventProcessorWatchdog; + private readonly SubscriptionProcessorWatchdog _eventProcessorWatchdog; public WatchdogsBackgroundService( DefragWatchdog defragWatchdog, CleanupEventLogWatchdog cleanupEventLogWatchdog, IScopeProvider transactionWatchdog, InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog, - EventProcessorWatchdog eventProcessorWatchdog) + SubscriptionProcessorWatchdog eventProcessorWatchdog) { _defragWatchdog = EnsureArg.IsNotNull(defragWatchdog, nameof(defragWatchdog)); _cleanupEventLogWatchdog = EnsureArg.IsNotNull(cleanupEventLogWatchdog, nameof(cleanupEventLogWatchdog)); @@ -46,12 +48,26 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); } - await Task.WhenAll( - _defragWatchdog.StartAsync(stoppingToken), - _cleanupEventLogWatchdog.StartAsync(stoppingToken), - _transactionWatchdog.Value.StartAsync(stoppingToken), - _invisibleHistoryCleanupWatchdog.StartAsync(stoppingToken), - _eventProcessorWatchdog.StartAsync(stoppingToken)); + using var continuationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); + + var tasks = new List + { + _defragWatchdog.StartAsync(continuationTokenSource.Token), + _cleanupEventLogWatchdog.StartAsync(continuationTokenSource.Token), + _transactionWatchdog.Value.StartAsync(continuationTokenSource.Token), + _invisibleHistoryCleanupWatchdog.StartAsync(continuationTokenSource.Token), + _eventProcessorWatchdog.StartAsync(continuationTokenSource.Token), + }; + + await Task.WhenAny(tasks); + + if (!stoppingToken.IsCancellationRequested) + { + // If any of the watchdogs fail, cancel all the other watchdogs + await continuationTokenSource.CancelAsync(); + } + + await Task.WhenAll(tasks); } public Task Handle(StorageInitializedNotification notification, CancellationToken cancellationToken) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs b/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs index eb94941a91..c980fdde97 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Registration/FhirServerBuilderSqlServerRegistrationExtensions.cs @@ -175,7 +175,7 @@ public static IFhirServerBuilder AddSqlServer(this IFhirServerBuilder fhirServer services.Add().Singleton().AsSelf(); - services.Add().Singleton().AsSelf(); + services.Add().Singleton().AsSelf(); services.RemoveServiceTypeExact>() // Mediatr registers handlers as Transient by default, this extension ensures these aren't still there, only needed when service != Transient .Add() diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj index a90c13a333..50f7da1e86 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj +++ b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj @@ -5,6 +5,7 @@ + diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs index d62f584a63..8600df54f4 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -6,9 +6,16 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net; using System.Text; using System.Threading.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Extensions.Logging; using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Operations.Export; +using Microsoft.Health.Fhir.Core.Features.Operations.Export.ExportDestinationClient; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Subscriptions.Models; using Microsoft.Health.JobManagement; namespace Microsoft.Health.Fhir.Subscriptions.Operations @@ -16,11 +23,66 @@ namespace Microsoft.Health.Fhir.Subscriptions.Operations [JobTypeId((int)JobType.SubscriptionsProcessing)] public class SubscriptionProcessingJob : IJob { - public Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) + private readonly IResourceToByteArraySerializer _resourceToByteArraySerializer; + private readonly IExportDestinationClient _exportDestinationClient; + private readonly IResourceDeserializer _resourceDeserializer; + private readonly IFhirDataStore _dataStore; + private readonly ILogger _logger; + + public SubscriptionProcessingJob( + IResourceToByteArraySerializer resourceToByteArraySerializer, + IExportDestinationClient exportDestinationClient, + IResourceDeserializer resourceDeserializer, + IFhirDataStore dataStore, + ILogger logger) { - // TODO: Write resource to channel + _resourceToByteArraySerializer = resourceToByteArraySerializer; + _exportDestinationClient = exportDestinationClient; + _resourceDeserializer = resourceDeserializer; + _dataStore = dataStore; + _logger = logger; + } + + public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) + { + SubscriptionJobDefinition definition = jobInfo.DeserializeDefinition(); + + if (definition.Channel == null) + { + return HttpStatusCode.BadRequest.ToString(); + } + + if (definition.Channel.ChannelType == SubscriptionChannelType.Storage) + { + try + { + await _exportDestinationClient.ConnectAsync(cancellationToken, "SubscriptionContainer"); + + foreach (var resourceKey in definition.ResourceReferences) + { + var resource = await _dataStore.GetAsync(resourceKey, cancellationToken); + + string fileName = $"{resource.ResourceTypeName}/{resource.ResourceId}/_history/{resource.Version}.json"; + + _exportDestinationClient.WriteFilePart( + fileName, + resource.RawResource.Data); + + _exportDestinationClient.CommitFile(fileName); + } + } + catch (Exception ex) + { + _logger.LogJobError(jobInfo, ex.ToString()); + return HttpStatusCode.InternalServerError.ToString(); + } + } + else + { + return HttpStatusCode.BadRequest.ToString(); + } - return Task.FromResult("Done!"); + return HttpStatusCode.OK.ToString(); } } } From afeab31cac8358b8c8a1614ff60e900ba24175a7 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Tue, 16 Apr 2024 10:02:46 -0700 Subject: [PATCH 25/47] Allow resourceKey to deserialize --- .../Features/Persistence/ResourceKey.cs | 6 +++--- .../Operations/SubscriptionProcessingJob.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs index 08d83caa78..b5316f471a 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs @@ -29,11 +29,11 @@ protected ResourceKey() { } - public string Id { get; } + public string Id { get; protected set; } - public string VersionId { get; } + public string VersionId { get; protected set; } - public string ResourceType { get; } + public string ResourceType { get; protected set; } public bool Equals(ResourceKey other) { diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs index 8600df54f4..40d582d7aa 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -56,7 +56,7 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel { try { - await _exportDestinationClient.ConnectAsync(cancellationToken, "SubscriptionContainer"); + await _exportDestinationClient.ConnectAsync(cancellationToken, "fhirsync"); foreach (var resourceKey in definition.ResourceReferences) { From adf381ac187ae8ac736d174a3b49b8fc087c59f4 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Tue, 16 Apr 2024 13:00:00 -0700 Subject: [PATCH 26/47] Implement basic subscription filtering --- .../Features/Persistence/ResourceKey.cs | 3 + .../Models/ChannelInfo.cs | 3 + .../Models/SubscriptionInfo.cs | 2 +- .../Operations/SubscriptionProcessingJob.cs | 4 +- .../SubscriptionsOrchestratorJob.cs | 58 ++++++++++++++++++- .../Persistence/SubscriptionManager.cs | 20 ++++++- 6 files changed, 83 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs index b5316f471a..b568089315 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ResourceKey.cs @@ -29,10 +29,13 @@ protected ResourceKey() { } + [JsonProperty("id")] public string Id { get; protected set; } + [JsonProperty("versionId")] public string VersionId { get; protected set; } + [JsonProperty("resourceType")] public string ResourceType { get; protected set; } public bool Equals(ResourceKey other) diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs index a5b9aaf389..5d99a57c2e 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs @@ -31,5 +31,8 @@ public class ChannelInfo public SubscriptionChannelType ChannelType { get; set; } public SubscriptionContentType ContentType { get; set; } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "Json Poco")] + public IDictionary Properties { get; set; } = new Dictionary(); } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs index b2d2e39591..038865d938 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs @@ -16,7 +16,7 @@ public class SubscriptionInfo { public SubscriptionInfo(string filterCriteria, ChannelInfo channel) { - FilterCriteria = EnsureArg.IsNotNullOrEmpty(filterCriteria, nameof(filterCriteria)); + FilterCriteria = filterCriteria; Channel = EnsureArg.IsNotNull(channel, nameof(channel)); } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs index 40d582d7aa..277113f818 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -56,13 +56,13 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel { try { - await _exportDestinationClient.ConnectAsync(cancellationToken, "fhirsync"); + await _exportDestinationClient.ConnectAsync(cancellationToken, definition.Channel.Properties["container"]); foreach (var resourceKey in definition.ResourceReferences) { var resource = await _dataStore.GetAsync(resourceKey, cancellationToken); - string fileName = $"{resource.ResourceTypeName}/{resource.ResourceId}/_history/{resource.Version}.json"; + string fileName = $"{resourceKey}.json"; _exportDestinationClient.WriteFilePart( fileName, diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs index 9c2efab149..9cc1fd6dc5 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs @@ -5,14 +5,17 @@ using System; using System.Collections.Generic; +using System.Collections.ObjectModel; using System.Linq; using System.Text; using System.Threading.Tasks; using EnsureThat; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Routing; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Subscriptions.Models; using Microsoft.Health.Fhir.Subscriptions.Persistence; @@ -25,12 +28,16 @@ public class SubscriptionsOrchestratorJob : IJob { private readonly IQueueClient _queueClient; private readonly ITransactionDataStore _transactionDataStore; + private readonly ISearchService _searchService; + private readonly IQueryStringParser _queryStringParser; private readonly ISubscriptionManager _subscriptionManager; private const string OperationCompleted = "Completed"; public SubscriptionsOrchestratorJob( IQueueClient queueClient, ITransactionDataStore transactionDataStore, + ISearchService searchService, + IQueryStringParser queryStringParser, ISubscriptionManager subscriptionManager) { EnsureArg.IsNotNull(queueClient, nameof(queueClient)); @@ -39,6 +46,8 @@ public SubscriptionsOrchestratorJob( _queueClient = queueClient; _transactionDataStore = transactionDataStore; + _searchService = searchService; + _queryStringParser = queryStringParser; _subscriptionManager = subscriptionManager; } @@ -48,20 +57,63 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel SubscriptionJobDefinition definition = jobInfo.DeserializeDefinition(); var resources = await _transactionDataStore.GetResourcesByTransactionIdAsync(definition.TransactionId, cancellationToken); + var resourceKeys = resources.Select(r => new ResourceKey(r.ResourceTypeName, r.ResourceId, r.Version)).ToList().AsReadOnly(); var processingDefinition = new List(); foreach (var sub in await _subscriptionManager.GetActiveSubscriptionsAsync(cancellationToken)) { - var chunk = resources - //// TODO: .Where(r => sub.FilterCriteria does something??); + var channelResources = new List(); + + if (!string.IsNullOrEmpty(sub.FilterCriteria)) + { + var criteriaSegments = sub.FilterCriteria.Split('?'); + + List> query = new List>(); + + if (criteriaSegments.Length > 1) + { + query = _queryStringParser.Parse(criteriaSegments[1]) + .Select(x => new Tuple(x.Key, x.Value)) + .ToList(); + } + + var limitIds = string.Join(",", resourceKeys.Select(x => x.Id)); + var idParam = query.Where(x => x.Item1 == KnownQueryParameterNames.Id).FirstOrDefault(); + if (idParam != null) + { + query.Remove(idParam); + limitIds += "," + idParam.Item2; + } + + query.Add(new Tuple(KnownQueryParameterNames.Id, limitIds)); + + var results = await _searchService.SearchAsync(criteriaSegments[0], new ReadOnlyCollection>(query), cancellationToken, true, ResourceVersionType.Latest, onlyIds: true); + + channelResources.AddRange( + results.Results + .Where(x => x.SearchEntryMode == ValueSets.SearchEntryMode.Match + || x.SearchEntryMode == ValueSets.SearchEntryMode.Include) + .Select(x => x.Resource.ToResourceKey())); + } + else + { + channelResources.AddRange(resourceKeys); + } + + if (channelResources.Count == 0) + { + continue; + } + + var chunk = resourceKeys .Chunk(sub.Channel.MaxCount); foreach (var batch in chunk) { var cloneDefinition = jobInfo.DeserializeDefinition(); cloneDefinition.TypeId = (int)JobType.SubscriptionsProcessing; - cloneDefinition.ResourceReferences = batch.Select(r => new ResourceKey(r.ResourceTypeName, r.ResourceId, r.Version)).ToList(); + cloneDefinition.ResourceReferences = batch.ToList(); cloneDefinition.Channel = sub.Channel; processingDefinition.Add(cloneDefinition); diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs index b53ef16587..be8078dc2a 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs @@ -18,13 +18,31 @@ public Task> GetActiveSubscriptionsAsync(C { IReadOnlyCollection list = new List { + // "reason": "Alert on Diabetes with Complications Diagnosis", + // "criteria": "Condition?code=http://hl7.org/fhir/sid/icd-10|E11.6", new SubscriptionInfo( - "Resource", + null, new ChannelInfo { ChannelType = SubscriptionChannelType.Storage, ContentType = SubscriptionContentType.FullResource, MaxCount = 100, + Properties = new Dictionary + { + { "container", "sync-all" }, + }, + }), + new SubscriptionInfo( + "Patient", + new ChannelInfo + { + ChannelType = SubscriptionChannelType.Storage, + ContentType = SubscriptionContentType.FullResource, + MaxCount = 100, + Properties = new Dictionary + { + { "container", "sync-patient" }, + }, }), }; From 3c49e3ee59f447337bae9035070e7e9f5bada55d Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Tue, 16 Apr 2024 15:02:08 -0700 Subject: [PATCH 27/47] Implements Channel Interface --- .../Configs/CoreFeatureConfiguration.cs | 5 ++ .../appsettings.json | 1 + .../SubscriptionProcessorWatchdog.cs | 30 +++++++---- .../Watchdogs/WatchdogsBackgroundService.cs | 6 +-- .../Channels/ChannelTypeAttribute.cs | 25 +++++++++ .../Channels/ISubscriptionChannel.cs | 5 +- .../Channels/StorageChannel.cs | 29 +++++++++++ .../Channels/StorageChannelFactory.cs | 42 +++++++++++++++ .../Operations/SubscriptionProcessingJob.cs | 51 ++++--------------- .../Registration/SubscriptionsModule.cs | 11 ++++ 10 files changed, 149 insertions(+), 56 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Channels/ChannelTypeAttribute.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannelFactory.cs diff --git a/src/Microsoft.Health.Fhir.Core/Configs/CoreFeatureConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/CoreFeatureConfiguration.cs index 62b585c53a..ca780a0bab 100644 --- a/src/Microsoft.Health.Fhir.Core/Configs/CoreFeatureConfiguration.cs +++ b/src/Microsoft.Health.Fhir.Core/Configs/CoreFeatureConfiguration.cs @@ -81,5 +81,10 @@ public class CoreFeatureConfiguration /// Gets or sets a value indicating whether the server supports the $bulk-delete. /// public bool SupportsBulkDelete { get; set; } + + /// + /// Gets or set a value indicating whether the server supports Subscription processing. + /// + public bool SupportsSubscriptions { get; set; } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json index c91847252a..6a8add7b32 100644 --- a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json +++ b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json @@ -30,6 +30,7 @@ "ProfileValidationOnUpdate": false, "SupportsResourceChangeCapture": false, "SupportsBulkDelete": true, + "SupportsSubscriptions": true, "Versioning": { "Default": "versioned", "ResourceTypeOverrides": null diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs index 5f31720eda..8dc1f8e829 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs @@ -12,6 +12,8 @@ using EnsureThat; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.SqlServer.Features.Storage; using Microsoft.Health.Fhir.Subscriptions.Models; @@ -25,12 +27,14 @@ internal class SubscriptionProcessorWatchdog : Watchdog _logger; private readonly ISqlRetryService _sqlRetryService; private readonly IQueueClient _queueClient; + private readonly CoreFeatureConfiguration _config; private CancellationToken _cancellationToken; public SubscriptionProcessorWatchdog( SqlStoreClient store, ISqlRetryService sqlRetryService, IQueueClient queueClient, + IOptions coreConfiguration, ILogger logger) : base(sqlRetryService, logger) { @@ -39,6 +43,7 @@ public SubscriptionProcessorWatchdog( _logger = EnsureArg.IsNotNull(logger, nameof(logger)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); _queueClient = EnsureArg.IsNotNull(queueClient, nameof(queueClient)); + _config = EnsureArg.IsNotNull(coreConfiguration?.Value, nameof(coreConfiguration)); } internal string LastEventProcessedTransactionId => $"{Name}.{nameof(LastEventProcessedTransactionId)}"; @@ -68,20 +73,24 @@ protected override async Task ExecuteAsync() return; } - var transactionsToQueue = new List(); - - foreach (var tran in transactionsToProcess.Where(x => x.VisibleDate.HasValue).OrderBy(x => x.TransactionId)) + if (_config.SupportsSubscriptions) { - var jobDefinition = new SubscriptionJobDefinition(Core.Features.Operations.JobType.SubscriptionsOrchestrator) + var transactionsToQueue = new List(); + + foreach (var tran in transactionsToProcess.Where(x => x.VisibleDate.HasValue).OrderBy(x => x.TransactionId)) { - TransactionId = tran.TransactionId, - VisibleDate = tran.VisibleDate.Value, - }; + var jobDefinition = new SubscriptionJobDefinition(JobType.SubscriptionsOrchestrator) + { + TransactionId = tran.TransactionId, + VisibleDate = tran.VisibleDate.Value, + }; + + transactionsToQueue.Add(jobDefinition); + } - transactionsToQueue.Add(jobDefinition); + await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken: _cancellationToken, definitions: transactionsToQueue.ToArray()); } - await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken: _cancellationToken, definitions: transactionsToQueue.ToArray()); await UpdateLastEventProcessedTransactionId(transactionsToProcess.Max(x => x.TransactionId)); _logger.LogInformation($"{Name}: completed. transactions={transactionsToProcess.Count}"); @@ -94,8 +103,9 @@ private async Task GetLastTransactionId() private async Task InitLastProcessedTransactionId() { - using var cmd = new SqlCommand("INSERT INTO dbo.Parameters (Id, Bigint) SELECT @Id, 5105975696064002770"); // surrogate id for the past. does not matter. + using var cmd = new SqlCommand("INSERT INTO dbo.Parameters (Id, Bigint) SELECT @Id, @LastTranId"); cmd.Parameters.AddWithValue("@Id", LastEventProcessedTransactionId); + cmd.Parameters.AddWithValue("@LastTranId", await _store.MergeResourcesGetTransactionVisibilityAsync(_cancellationToken)); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs index bafa0b1c03..2e411bd8ac 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs @@ -24,7 +24,7 @@ internal class WatchdogsBackgroundService : BackgroundService, INotificationHand private readonly CleanupEventLogWatchdog _cleanupEventLogWatchdog; private readonly IScoped _transactionWatchdog; private readonly InvisibleHistoryCleanupWatchdog _invisibleHistoryCleanupWatchdog; - private readonly SubscriptionProcessorWatchdog _eventProcessorWatchdog; + private readonly SubscriptionProcessorWatchdog _subscriptionsProcessorWatchdog; public WatchdogsBackgroundService( DefragWatchdog defragWatchdog, @@ -37,7 +37,7 @@ public WatchdogsBackgroundService( _cleanupEventLogWatchdog = EnsureArg.IsNotNull(cleanupEventLogWatchdog, nameof(cleanupEventLogWatchdog)); _transactionWatchdog = EnsureArg.IsNotNull(transactionWatchdog, nameof(transactionWatchdog)).Invoke(); _invisibleHistoryCleanupWatchdog = EnsureArg.IsNotNull(invisibleHistoryCleanupWatchdog, nameof(invisibleHistoryCleanupWatchdog)); - _eventProcessorWatchdog = EnsureArg.IsNotNull(eventProcessorWatchdog, nameof(eventProcessorWatchdog)); + _subscriptionsProcessorWatchdog = EnsureArg.IsNotNull(eventProcessorWatchdog, nameof(eventProcessorWatchdog)); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -56,7 +56,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) _cleanupEventLogWatchdog.StartAsync(continuationTokenSource.Token), _transactionWatchdog.Value.StartAsync(continuationTokenSource.Token), _invisibleHistoryCleanupWatchdog.StartAsync(continuationTokenSource.Token), - _eventProcessorWatchdog.StartAsync(continuationTokenSource.Token), + _subscriptionsProcessorWatchdog.StartAsync(continuationTokenSource.Token), }; await Task.WhenAny(tasks); diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ChannelTypeAttribute.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ChannelTypeAttribute.cs new file mode 100644 index 0000000000..8c52493d98 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ChannelTypeAttribute.cs @@ -0,0 +1,25 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Channels +{ + [AttributeUsage(AttributeTargets.Class)] + public sealed class ChannelTypeAttribute : Attribute + { + public ChannelTypeAttribute(SubscriptionChannelType channelType) + { + ChannelType = channelType; + } + + public SubscriptionChannelType ChannelType { get; } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs index fc13912923..35cb6da2b0 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs @@ -8,10 +8,13 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Subscriptions.Models; namespace Microsoft.Health.Fhir.Subscriptions.Channels { - internal interface ISubscriptionChannel + public interface ISubscriptionChannel { + Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs index fa0c938cdc..3f32ec3a5b 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs @@ -8,10 +8,39 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Health.Fhir.Core.Features.Operations.Export; +using Microsoft.Health.Fhir.Core.Features.Operations.Export.ExportDestinationClient; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Subscriptions.Models; namespace Microsoft.Health.Fhir.Subscriptions.Channels { + [ChannelType(SubscriptionChannelType.Storage)] public class StorageChannel : ISubscriptionChannel { + private readonly IExportDestinationClient _exportDestinationClient; + + public StorageChannel( + IExportDestinationClient exportDestinationClient) + { + _exportDestinationClient = exportDestinationClient; + } + + public async Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) + { + await _exportDestinationClient.ConnectAsync(cancellationToken, channelInfo.Properties["container"]); + + foreach (var resource in resources) + { + string fileName = $"{resource.ToResourceKey()}.json"; + + _exportDestinationClient.WriteFilePart( + fileName, + resource.RawResource.Data); + + _exportDestinationClient.CommitFile(fileName); + } + } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannelFactory.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannelFactory.cs new file mode 100644 index 0000000000..47bb1e1dfd --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannelFactory.cs @@ -0,0 +1,42 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using EnsureThat; +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Channels +{ + public class StorageChannelFactory + { + private IServiceProvider _serviceProvider; + private Dictionary _channelTypeMap; + + public StorageChannelFactory(IServiceProvider serviceProvider) + { + _serviceProvider = EnsureArg.IsNotNull(serviceProvider, nameof(serviceProvider)); + + _channelTypeMap = + typeof(ISubscriptionChannel).Assembly.GetTypes() + .Where(t => typeof(ISubscriptionChannel).IsAssignableFrom(t) && !t.IsInterface && !t.IsAbstract) + .Select(t => new + { + Type = t, + Attribute = t.GetCustomAttributes(typeof(ChannelTypeAttribute), false).FirstOrDefault() as ChannelTypeAttribute, + }) + .Where(t => t.Attribute != null) + .ToDictionary(t => t.Attribute.ChannelType, t => t.Type); + } + + public ISubscriptionChannel Create(SubscriptionChannelType type) + { + return (ISubscriptionChannel)_serviceProvider.GetService(_channelTypeMap[type]); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs index 277113f818..10c4afd4f6 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -15,6 +15,7 @@ using Microsoft.Health.Fhir.Core.Features.Operations.Export; using Microsoft.Health.Fhir.Core.Features.Operations.Export.ExportDestinationClient; using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Subscriptions.Channels; using Microsoft.Health.Fhir.Subscriptions.Models; using Microsoft.Health.JobManagement; @@ -23,24 +24,13 @@ namespace Microsoft.Health.Fhir.Subscriptions.Operations [JobTypeId((int)JobType.SubscriptionsProcessing)] public class SubscriptionProcessingJob : IJob { - private readonly IResourceToByteArraySerializer _resourceToByteArraySerializer; - private readonly IExportDestinationClient _exportDestinationClient; - private readonly IResourceDeserializer _resourceDeserializer; + private readonly StorageChannelFactory _storageChannelFactory; private readonly IFhirDataStore _dataStore; - private readonly ILogger _logger; - public SubscriptionProcessingJob( - IResourceToByteArraySerializer resourceToByteArraySerializer, - IExportDestinationClient exportDestinationClient, - IResourceDeserializer resourceDeserializer, - IFhirDataStore dataStore, - ILogger logger) + public SubscriptionProcessingJob(StorageChannelFactory storageChannelFactory, IFhirDataStore dataStore) { - _resourceToByteArraySerializer = resourceToByteArraySerializer; - _exportDestinationClient = exportDestinationClient; - _resourceDeserializer = resourceDeserializer; + _storageChannelFactory = storageChannelFactory; _dataStore = dataStore; - _logger = logger; } public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) @@ -52,35 +42,12 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel return HttpStatusCode.BadRequest.ToString(); } - if (definition.Channel.ChannelType == SubscriptionChannelType.Storage) - { - try - { - await _exportDestinationClient.ConnectAsync(cancellationToken, definition.Channel.Properties["container"]); - - foreach (var resourceKey in definition.ResourceReferences) - { - var resource = await _dataStore.GetAsync(resourceKey, cancellationToken); - - string fileName = $"{resourceKey}.json"; - - _exportDestinationClient.WriteFilePart( - fileName, - resource.RawResource.Data); + var allResources = await Task.WhenAll( + definition.ResourceReferences + .Select(async x => await _dataStore.GetAsync(x, cancellationToken))); - _exportDestinationClient.CommitFile(fileName); - } - } - catch (Exception ex) - { - _logger.LogJobError(jobInfo, ex.ToString()); - return HttpStatusCode.InternalServerError.ToString(); - } - } - else - { - return HttpStatusCode.BadRequest.ToString(); - } + var channel = _storageChannelFactory.Create(definition.Channel.ChannelType); + await channel.PublishAsync(allResources, definition.Channel, definition.VisibleDate, cancellationToken); return HttpStatusCode.OK.ToString(); } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs index bbf12b8002..d58ff3085e 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Subscriptions.Channels; using Microsoft.Health.Fhir.Subscriptions.Persistence; using Microsoft.Health.JobManagement; @@ -34,6 +35,16 @@ public void Load(IServiceCollection services) .Singleton() .AsSelf() .AsImplementedInterfaces(); + + services.TypesInSameAssemblyAs() + .AssignableTo() + .Transient() + .AsSelf() + .AsImplementedInterfaces(); + + services.Add() + .Singleton() + .AsSelf(); } } } From 7c4a3c7e3eaf1ea9d050c8dc17bb35f290dbd829 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Tue, 16 Apr 2024 17:28:32 -0700 Subject: [PATCH 28/47] Add example subscription --- docs/rest/Subscriptions.http | 72 ++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 docs/rest/Subscriptions.http diff --git a/docs/rest/Subscriptions.http b/docs/rest/Subscriptions.http new file mode 100644 index 0000000000..000ef07cb0 --- /dev/null +++ b/docs/rest/Subscriptions.http @@ -0,0 +1,72 @@ +# # .SUMMARY Sample requests to verify FHIR Conditional Delete +# The assumption for the requests and resources below: +# The FHIR version is R4 + +@hostname = localhost:44348 + +### Get the bearer token, if authentication is enabled +# @name bearer +POST https://{{hostname}}/connect/token +content-type: application/x-www-form-urlencoded + +grant_type=client_credentials +&client_id=globalAdminServicePrincipal +&client_secret=globalAdminServicePrincipal +&scope=fhir-api + +### PUT Subscription for Rest-hook +## More examples: https://build.fhir.org/ig/HL7/fhir-subscription-backport-ig/artifacts.html +PUT https://{{hostname}}/Subscription/example-backport-storage +content-type: application/json +Authorization: Bearer {{bearer.response.body.access_token}} + +{ + "resourceType": "Subscription", + "id": "example-backport-storage", + "meta" : { + "profile": ["http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"] + }, + "status": "requested", + "end": "2031-01-01T12:00:00", + "reason": "Test subscription based on transactions", + "criteria" : "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions", + "_criteria": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-filter-criteria", + "valueString": "Patient" + } + ] + }, + "channel": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-heartbeat-period", + "valueInteger": 120 + } + ], + "type" : "rest-hook", + "_type" : { + "extension" : [ + { + "url" : "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type", + "valueCoding" : { + "system" : "http://azurehealthcareapis.com/data-extentions/subscription-channel-type", + "code" : "azure-storage", + "display" : "Azure Blob Storage" + } + } + ] + }, + "endpoint": "https://127.0.0.1:10000/devstoreaccount1/sync-all", + "payload": "application/fhir+json", + "_payload": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content", + "valueCode": "full-resource" + } + ] + } + } +} From eaeccd79f7a1cf4619d00276b96aa155598b52ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Henrique=20Inoc=C3=AAncio=20Borba=20Ferreira?= Date: Wed, 17 Apr 2024 11:41:20 -0700 Subject: [PATCH 29/47] DataLakeChannel. --- .../Channels/DataLakeChannel.cs | 52 +++++++++++++++++++ .../Persistence/SubscriptionManager.cs | 12 +++++ tools/EventsReader/Program.cs | 2 +- tools/PerfTester/Program.cs | 4 +- 4 files changed, 67 insertions(+), 3 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs new file mode 100644 index 0000000000..efbffbabe9 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs @@ -0,0 +1,52 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Globalization; +using Microsoft.Health.Fhir.Core.Features.Operations.Export.ExportDestinationClient; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Channels +{ + [ChannelType(SubscriptionChannelType.DatalakeContract)] + public class DataLakeChannel : ISubscriptionChannel + { + private readonly IExportDestinationClient _exportDestinationClient; + + public DataLakeChannel(IExportDestinationClient exportDestinationClient) + { + _exportDestinationClient = exportDestinationClient; + } + + public async Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) + { + try + { + await _exportDestinationClient.ConnectAsync(cancellationToken, channelInfo.Properties["container"]); + + IReadOnlyList> resourceGroupedByResourceType = resources.GroupBy(x => x.ResourceTypeName.ToLower(CultureInfo.InvariantCulture)).ToList(); + + foreach (IGrouping groupOfResources in resourceGroupedByResourceType) + { + string blobName = $"{groupOfResources.Key}/{transactionTime.Year:D4}/{transactionTime.Month:D2}/{transactionTime.Day:D2}/{transactionTime.ToUniversalTime().ToString("yyyy-MM-ddTHH.mm.ss.fffZ")}.ndjson"; + + foreach (ResourceWrapper item in groupOfResources) + { + // TODO: implement the soft-delete property addition. + string json = item.RawResource.Data; + + _exportDestinationClient.WriteFilePart(blobName, json); + } + + _exportDestinationClient.Commit(); + } + } + catch (Exception ex) + { + throw new InvalidOperationException("Failure in DatalakeChannel", ex); + } + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs index be8078dc2a..8bbe3bd896 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs @@ -44,6 +44,18 @@ public Task> GetActiveSubscriptionsAsync(C { "container", "sync-patient" }, }, }), + new SubscriptionInfo( + null, + new ChannelInfo + { + ChannelType = SubscriptionChannelType.DatalakeContract, + ContentType = SubscriptionContentType.FullResource, + MaxCount = 100, + Properties = new Dictionary + { + { "container", "lake" }, + }, + }), }; return Task.FromResult(list); diff --git a/tools/EventsReader/Program.cs b/tools/EventsReader/Program.cs index 372de29ffc..d6dcf988d1 100644 --- a/tools/EventsReader/Program.cs +++ b/tools/EventsReader/Program.cs @@ -23,7 +23,7 @@ public static void Main() { ISqlConnectionBuilder iSqlConnectionBuilder = new Sql.SqlConnectionBuilder(_connectionString); _sqlRetryService = SqlRetryService.GetInstance(iSqlConnectionBuilder); - _store = new SqlStoreClient(_sqlRetryService, NullLogger.Instance); + _store = new SqlStoreClient(_sqlRetryService, NullLogger.Instance); ExecuteAsync().Wait(); } diff --git a/tools/PerfTester/Program.cs b/tools/PerfTester/Program.cs index 8f9000ccc8..6abb1bb46f 100644 --- a/tools/PerfTester/Program.cs +++ b/tools/PerfTester/Program.cs @@ -48,14 +48,14 @@ public static class Program private static readonly int _repeat = int.Parse(ConfigurationManager.AppSettings["Repeat"]); private static SqlRetryService _sqlRetryService; - private static SqlStoreClient _store; + private static SqlStoreClient _store; public static void Main() { Console.WriteLine("!!!See App.config for the details!!!"); ISqlConnectionBuilder iSqlConnectionBuilder = new Sql.SqlConnectionBuilder(_connectionString); _sqlRetryService = SqlRetryService.GetInstance(iSqlConnectionBuilder); - _store = new SqlStoreClient(_sqlRetryService, NullLogger.Instance); + _store = new SqlStoreClient(_sqlRetryService, NullLogger.Instance); DumpResourceIds(); From 35c8371ee37a70515764f830b3b349316150ba6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fernando=20Henrique=20Inoc=C3=AAncio=20Borba=20Ferreira?= Date: Wed, 17 Apr 2024 17:57:17 -0700 Subject: [PATCH 30/47] Changes in DataLakeChannel and the project config. --- .../Channels/DataLakeChannel.cs | 22 ++++++++++++++++--- .../Channels/ISubscriptionChannel.cs | 3 +-- .../Channels/StorageChannel.cs | 5 +---- ...Microsoft.Health.Fhir.Subscriptions.csproj | 4 ---- .../Operations/SubscriptionProcessingJob.cs | 8 +------ .../SubscriptionsOrchestratorJob.cs | 4 +--- .../Persistence/ISubscriptionManager.cs | 3 +++ .../Persistence/ITransactionDataStore.cs | 4 +--- .../Persistence/SubscriptionManager.cs | 4 +--- 9 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs index efbffbabe9..a957b88592 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs @@ -3,7 +3,12 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; +using System.Collections.Generic; using System.Globalization; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Health.Fhir.Core.Features.Operations.Export.ExportDestinationClient; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Subscriptions.Models; @@ -14,10 +19,12 @@ namespace Microsoft.Health.Fhir.Subscriptions.Channels public class DataLakeChannel : ISubscriptionChannel { private readonly IExportDestinationClient _exportDestinationClient; + private readonly IResourceDeserializer _resourceDeserializer; - public DataLakeChannel(IExportDestinationClient exportDestinationClient) + public DataLakeChannel(IExportDestinationClient exportDestinationClient, IResourceDeserializer resourceDeserializer) { _exportDestinationClient = exportDestinationClient; + _resourceDeserializer = resourceDeserializer; } public async Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) @@ -28,15 +35,24 @@ public async Task PublishAsync(IReadOnlyCollection resources, C IReadOnlyList> resourceGroupedByResourceType = resources.GroupBy(x => x.ResourceTypeName.ToLower(CultureInfo.InvariantCulture)).ToList(); + DateTimeOffset transactionTimeInUtc = transactionTime.ToUniversalTime(); + foreach (IGrouping groupOfResources in resourceGroupedByResourceType) { - string blobName = $"{groupOfResources.Key}/{transactionTime.Year:D4}/{transactionTime.Month:D2}/{transactionTime.Day:D2}/{transactionTime.ToUniversalTime().ToString("yyyy-MM-ddTHH.mm.ss.fffZ")}.ndjson"; + string blobName = $"{groupOfResources.Key}/{transactionTimeInUtc.Year:D4}/{transactionTimeInUtc.Month:D2}/{transactionTimeInUtc.Day:D2}/{transactionTimeInUtc.ToString("yyyy-MM-ddTHH.mm.ss.fffZ")}.ndjson"; foreach (ResourceWrapper item in groupOfResources) { - // TODO: implement the soft-delete property addition. string json = item.RawResource.Data; + /* + // TODO: Add logic to handle soft-deleted resources. + if (item.IsDeleted) + { + ResourceElement element = _resourceDeserializer.Deserialize(item); + } + */ + _exportDestinationClient.WriteFilePart(blobName, json); } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs index 35cb6da2b0..e98211970b 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs @@ -5,8 +5,7 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; +using System.Threading; using System.Threading.Tasks; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Subscriptions.Models; diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs index 3f32ec3a5b..427e5f7246 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs @@ -5,11 +5,8 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; +using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Logging; -using Microsoft.Health.Fhir.Core.Features.Operations.Export; using Microsoft.Health.Fhir.Core.Features.Operations.Export.ExportDestinationClient; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Subscriptions.Models; diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj index 50f7da1e86..13218ee495 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj +++ b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj @@ -1,9 +1,5 @@  - - enable - - diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs index 10c4afd4f6..7c38327e10 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -3,17 +3,11 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System; -using System.Collections.Generic; using System.Linq; using System.Net; -using System.Text; +using System.Threading; using System.Threading.Tasks; -using Microsoft.Build.Framework; -using Microsoft.Extensions.Logging; using Microsoft.Health.Fhir.Core.Features.Operations; -using Microsoft.Health.Fhir.Core.Features.Operations.Export; -using Microsoft.Health.Fhir.Core.Features.Operations.Export.ExportDestinationClient; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Subscriptions.Channels; using Microsoft.Health.Fhir.Subscriptions.Models; diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs index 9cc1fd6dc5..9c48d1ce72 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs @@ -7,11 +7,9 @@ using System.Collections.Generic; using System.Collections.ObjectModel; using System.Linq; -using System.Text; +using System.Threading; using System.Threading.Tasks; using EnsureThat; -using Microsoft.Health.Extensions.DependencyInjection; -using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Persistence; diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs index a5f2738457..7b1132370e 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs @@ -3,6 +3,9 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Health.Fhir.Subscriptions.Models; namespace Microsoft.Health.Fhir.Subscriptions.Persistence diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs index 6a1ca37224..52d5cf3223 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs @@ -3,10 +3,8 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System; using System.Collections.Generic; -using System.Linq; -using System.Text; +using System.Threading; using System.Threading.Tasks; using Microsoft.Health.Fhir.Core.Features.Persistence; diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs index 8bbe3bd896..cd53fa90db 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs @@ -3,10 +3,8 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System; using System.Collections.Generic; -using System.Linq; -using System.Text; +using System.Threading; using System.Threading.Tasks; using Microsoft.Health.Fhir.Subscriptions.Models; From 5ed896e19345b125413a1bcf88c32911e5bf5908 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Thu, 18 Apr 2024 09:30:37 -0700 Subject: [PATCH 31/47] Load from DB --- Microsoft.Health.Fhir.sln | 7 + docs/rest/Subscriptions.http | 117 +++++++++++- .../Models/KnownResourceTypes.cs | 2 + ...oft.Health.Fhir.Subscriptions.Tests.csproj | 27 +++ .../Peristence/SubscriptionManagerTests.cs | 49 +++++ .../AssemblyInfo.cs | 11 ++ .../Channels/DataLakeChannel.cs | 2 +- .../Channels/StorageChannel.cs | 2 +- .../Models/ChannelInfo.cs | 2 + .../SubscriptionsOrchestratorJob.cs | 7 + .../Persistence/ISubscriptionManager.cs | 2 + .../Persistence/SubscriptionManager.cs | 171 +++++++++++++----- .../Registration/SubscriptionsModule.cs | 7 +- .../CommonSamples.cs | 52 ++++++ .../EmbeddedResourceManager.cs | 11 +- .../Microsoft.Health.Fhir.Tests.Common.csproj | 2 + .../TestFiles/R4/Subscription-Backport.json | 54 ++++++ 17 files changed, 472 insertions(+), 53 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj create mode 100644 src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionManagerTests.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/AssemblyInfo.cs create mode 100644 src/Microsoft.Health.Fhir.Tests.Common/CommonSamples.cs create mode 100644 src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/Subscription-Backport.json diff --git a/Microsoft.Health.Fhir.sln b/Microsoft.Health.Fhir.sln index bc333070ea..0a0b88fedf 100644 --- a/Microsoft.Health.Fhir.sln +++ b/Microsoft.Health.Fhir.sln @@ -207,6 +207,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Cosmo EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Subscriptions", "src\Microsoft.Health.Fhir.Subscriptions\Microsoft.Health.Fhir.Subscriptions.csproj", "{BD8F3137-89F5-4EE5-B269-24D73081E00A}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Subscriptions.Tests", "src\Microsoft.Health.Fhir.Subscriptions.Tests\Microsoft.Health.Fhir.Subscriptions.Tests.csproj", "{AA73AB9D-52EF-4172-9911-3C9D661C8D48}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -489,6 +491,10 @@ Global {BD8F3137-89F5-4EE5-B269-24D73081E00A}.Debug|Any CPU.Build.0 = Debug|Any CPU {BD8F3137-89F5-4EE5-B269-24D73081E00A}.Release|Any CPU.ActiveCfg = Release|Any CPU {BD8F3137-89F5-4EE5-B269-24D73081E00A}.Release|Any CPU.Build.0 = Release|Any CPU + {AA73AB9D-52EF-4172-9911-3C9D661C8D48}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AA73AB9D-52EF-4172-9911-3C9D661C8D48}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AA73AB9D-52EF-4172-9911-3C9D661C8D48}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AA73AB9D-52EF-4172-9911-3C9D661C8D48}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -585,6 +591,7 @@ Global {B9AAA11D-8C8C-44C3-AADE-801376EF82F0} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} {BD8F3137-89F5-4EE5-B269-24D73081E00A} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4} + {AA73AB9D-52EF-4172-9911-3C9D661C8D48} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution RESX_SortFileContentOnSave = True diff --git a/docs/rest/Subscriptions.http b/docs/rest/Subscriptions.http index 000ef07cb0..68bcb743e1 100644 --- a/docs/rest/Subscriptions.http +++ b/docs/rest/Subscriptions.http @@ -14,21 +14,21 @@ grant_type=client_credentials &client_secret=globalAdminServicePrincipal &scope=fhir-api -### PUT Subscription for Rest-hook +### PUT Subscription for Blob Storage ## More examples: https://build.fhir.org/ig/HL7/fhir-subscription-backport-ig/artifacts.html -PUT https://{{hostname}}/Subscription/example-backport-storage +PUT https://{{hostname}}/Subscription/example-backport-storage-patient content-type: application/json Authorization: Bearer {{bearer.response.body.access_token}} { "resourceType": "Subscription", - "id": "example-backport-storage", + "id": "example-backport-storage-patient", "meta" : { "profile": ["http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"] }, "status": "requested", "end": "2031-01-01T12:00:00", - "reason": "Test subscription based on transactions", + "reason": "Test subscription based on transactions, filtered by Patient", "criteria" : "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions", "_criteria": { "extension": [ @@ -58,7 +58,7 @@ Authorization: Bearer {{bearer.response.body.access_token}} } ] }, - "endpoint": "https://127.0.0.1:10000/devstoreaccount1/sync-all", + "endpoint": "sync-patient", "payload": "application/fhir+json", "_payload": { "extension": [ @@ -70,3 +70,110 @@ Authorization: Bearer {{bearer.response.body.access_token}} } } } + +### PUT Subscription for Blob Storage +## More examples: https://build.fhir.org/ig/HL7/fhir-subscription-backport-ig/artifacts.html +PUT https://{{hostname}}/Subscription/example-backport-storage-all +content-type: application/json +Authorization: Bearer {{bearer.response.body.access_token}} + +{ + "resourceType": "Subscription", + "id": "example-backport-storage-all", + "meta" : { + "profile": ["http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"] + }, + "status": "requested", + "end": "2031-01-01T12:00:00", + "reason": "Test subscription based on transactions", + "criteria" : "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions", + "channel": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-heartbeat-period", + "valueInteger": 120 + }, + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-max-count", + "valuePositiveInt": 20 + } + ], + "type" : "rest-hook", + "_type" : { + "extension" : [ + { + "url" : "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type", + "valueCoding" : { + "system" : "http://azurehealthcareapis.com/data-extentions/subscription-channel-type", + "code" : "azure-storage", + "display" : "Azure Blob Storage" + } + } + ] + }, + "endpoint": "sync-all", + "payload": "application/fhir+json", + "_payload": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content", + "valueCode": "full-resource" + } + ] + } + } +} + +### PUT Subscription for Fabric +## More examples: https://build.fhir.org/ig/HL7/fhir-subscription-backport-ig/artifacts.html +PUT https://{{hostname}}/Subscription/example-backport-lake +content-type: application/json +Authorization: Bearer {{bearer.response.body.access_token}} + +{ + "resourceType": "Subscription", + "id": "example-backport-lake", + "meta" : { + "profile": ["http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"] + }, + "status": "requested", + "end": "2031-01-01T12:00:00", + "reason": "Test subscription based on transactions", + "criteria" : "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions", + "channel": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-heartbeat-period", + "valueInteger": 120 + } + ], + "type" : "rest-hook", + "_type" : { + "extension" : [ + { + "url" : "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type", + "valueCoding" : { + "system" : "http://azurehealthcareapis.com/data-extentions/subscription-channel-type", + "code" : "azure-lake-storage", + "display" : "Azure Data Lake Contract Storage" + } + } + ] + }, + "endpoint": "sync-lake", + "payload": "application/fhir+ndjson", + "_payload": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content", + "valueCode": "full-resource" + } + ] + } + } +} + +### +DELETE https://{{hostname}}/Subscription/example-backport-storage-all +content-type: application/json +Authorization: Bearer {{bearer.response.body.access_token}} \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Core/Models/KnownResourceTypes.cs b/src/Microsoft.Health.Fhir.Core/Models/KnownResourceTypes.cs index 2a2708c938..96e4099ce8 100644 --- a/src/Microsoft.Health.Fhir.Core/Models/KnownResourceTypes.cs +++ b/src/Microsoft.Health.Fhir.Core/Models/KnownResourceTypes.cs @@ -55,6 +55,8 @@ public static class KnownResourceTypes public const string SearchParameter = "SearchParameter"; + public const string Subscription = "Subscription"; + public const string Patient = "Patient"; public const string ValueSet = "ValueSet"; diff --git a/src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj new file mode 100644 index 0000000000..f2ca89213d --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj @@ -0,0 +1,27 @@ + + + + enable + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + diff --git a/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionManagerTests.cs b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionManagerTests.cs new file mode 100644 index 0000000000..253e151730 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionManagerTests.cs @@ -0,0 +1,49 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Hl7.Fhir.ElementModel; +using Hl7.Fhir.Model; +using Hl7.Fhir.Serialization; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Subscriptions.Models; +using Microsoft.Health.Fhir.Subscriptions.Persistence; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Subscriptions.Tests.Peristence +{ + public class SubscriptionManagerTests + { + private IModelInfoProvider _modelInfo; + + public SubscriptionManagerTests() + { + _modelInfo = MockModelInfoProviderBuilder + .Create(FhirSpecification.R4) + .AddKnownTypes(KnownResourceTypes.Subscription) + .Build(); + } + + [Fact] + public void GivenAnR4BackportSubscription_WhenConvertingToInfo_ThenTheInformationIsCorrect() + { + var subscription = CommonSamples.GetJsonSample("Subscription-Backport", FhirSpecification.R4, s => s.ToTypedElement(ModelInfo.ModelInspector)); + + var info = SubscriptionManager.ConvertToInfo(subscription); + + Assert.Equal("Patient", info.FilterCriteria); + Assert.Equal("sync-all", info.Channel.Endpoint); + Assert.Equal(20, info.Channel.MaxCount); + Assert.Equal(SubscriptionContentType.FullResource, info.Channel.ContentType); + Assert.Equal(SubscriptionChannelType.Storage, info.Channel.ChannelType); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/AssemblyInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/AssemblyInfo.cs new file mode 100644 index 0000000000..04b1e9fede --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/AssemblyInfo.cs @@ -0,0 +1,11 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Resources; +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Microsoft.Health.Fhir.Subscriptions.Tests")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] +[assembly: NeutralResourcesLanguage("en-us")] diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs index a957b88592..f41a460134 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs @@ -31,7 +31,7 @@ public async Task PublishAsync(IReadOnlyCollection resources, C { try { - await _exportDestinationClient.ConnectAsync(cancellationToken, channelInfo.Properties["container"]); + await _exportDestinationClient.ConnectAsync(cancellationToken, channelInfo.Endpoint); IReadOnlyList> resourceGroupedByResourceType = resources.GroupBy(x => x.ResourceTypeName.ToLower(CultureInfo.InvariantCulture)).ToList(); diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs index 427e5f7246..d7d0c2ad74 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs @@ -26,7 +26,7 @@ public StorageChannel( public async Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) { - await _exportDestinationClient.ConnectAsync(cancellationToken, channelInfo.Properties["container"]); + await _exportDestinationClient.ConnectAsync(cancellationToken, channelInfo.Endpoint); foreach (var resource in resources) { diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs index 5d99a57c2e..863f320d9e 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/ChannelInfo.cs @@ -32,6 +32,8 @@ public class ChannelInfo public SubscriptionContentType ContentType { get; set; } + public string Endpoint { get; set; } + [System.Diagnostics.CodeAnalysis.SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "Json Poco")] public IDictionary Properties { get; set; } = new Dictionary(); } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs index 9c48d1ce72..acca62e009 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs @@ -15,6 +15,7 @@ using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Routing; using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Subscriptions.Models; using Microsoft.Health.Fhir.Subscriptions.Persistence; using Microsoft.Health.JobManagement; @@ -57,6 +58,12 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel var resources = await _transactionDataStore.GetResourcesByTransactionIdAsync(definition.TransactionId, cancellationToken); var resourceKeys = resources.Select(r => new ResourceKey(r.ResourceTypeName, r.ResourceId, r.Version)).ToList().AsReadOnly(); + // Sync subscriptions if a change is detected + if (resources.Any(x => string.Equals(x.ResourceTypeName, KnownResourceTypes.Subscription, StringComparison.Ordinal))) + { + await _subscriptionManager.SyncSubscriptionsAsync(cancellationToken); + } + var processingDefinition = new List(); foreach (var sub in await _subscriptionManager.GetActiveSubscriptionsAsync(cancellationToken)) diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs index 7b1132370e..180df430ba 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs @@ -13,5 +13,7 @@ namespace Microsoft.Health.Fhir.Subscriptions.Persistence public interface ISubscriptionManager { Task> GetActiveSubscriptionsAsync(CancellationToken cancellationToken); + + Task SyncSubscriptionsAsync(CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs index cd53fa90db..915ae83e63 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs @@ -3,60 +3,147 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using EnsureThat; +using MediatR; +using Microsoft.Build.Framework; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Messages.Storage; +using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Subscriptions.Models; namespace Microsoft.Health.Fhir.Subscriptions.Persistence { - public class SubscriptionManager : ISubscriptionManager + public sealed class SubscriptionManager : ISubscriptionManager, INotificationHandler { - public Task> GetActiveSubscriptionsAsync(CancellationToken cancellationToken) + private readonly IScopeProvider _dataStoreProvider; + private readonly IScopeProvider _searchServiceProvider; + private List _subscriptions = new List(); + private readonly IResourceDeserializer _resourceDeserializer; + private readonly ILogger _logger; + private static readonly object _lock = new object(); + private const string MetaString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"; + private const string CriteriaString = "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions"; + private const string CriteriaExtensionString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-filter-criteria"; + private const string ChannelTypeString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type"; + ////private const string AzureChannelTypeString = "http://azurehealthcareapis.com/data-extentions/subscription-channel-type"; + private const string PayloadTypeString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content"; + private const string MaxCountString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-max-count"; + + public SubscriptionManager( + IScopeProvider dataStoreProvider, + IScopeProvider searchServiceProvider, + IResourceDeserializer resourceDeserializer, + ILogger logger) { - IReadOnlyCollection list = new List + _dataStoreProvider = EnsureArg.IsNotNull(dataStoreProvider, nameof(dataStoreProvider)); + _searchServiceProvider = EnsureArg.IsNotNull(searchServiceProvider, nameof(searchServiceProvider)); + _resourceDeserializer = resourceDeserializer; + _logger = logger; + } + + public async Task SyncSubscriptionsAsync(CancellationToken cancellationToken) + { + // requested | active | error | off + + var updatedSubscriptions = new List(); + + using var search = _searchServiceProvider.Invoke(); + + // Get all the active subscriptions + var activeSubscriptions = await search.Value.SearchAsync( + KnownResourceTypes.Subscription, + [ + Tuple.Create("status", "active,requested"), + ], + cancellationToken); + + foreach (var param in activeSubscriptions.Results) { - // "reason": "Alert on Diabetes with Complications Diagnosis", - // "criteria": "Condition?code=http://hl7.org/fhir/sid/icd-10|E11.6", - new SubscriptionInfo( - null, - new ChannelInfo - { - ChannelType = SubscriptionChannelType.Storage, - ContentType = SubscriptionContentType.FullResource, - MaxCount = 100, - Properties = new Dictionary - { - { "container", "sync-all" }, - }, - }), - new SubscriptionInfo( - "Patient", - new ChannelInfo - { - ChannelType = SubscriptionChannelType.Storage, - ContentType = SubscriptionContentType.FullResource, - MaxCount = 100, - Properties = new Dictionary - { - { "container", "sync-patient" }, - }, - }), - new SubscriptionInfo( - null, - new ChannelInfo - { - ChannelType = SubscriptionChannelType.DatalakeContract, - ContentType = SubscriptionContentType.FullResource, - MaxCount = 100, - Properties = new Dictionary - { - { "container", "lake" }, - }, - }), + var resource = _resourceDeserializer.Deserialize(param.Resource); + + SubscriptionInfo info = ConvertToInfo(resource); + + if (info == null) + { + _logger.LogWarning("Subscription with id {SubscriptionId} is valid", resource.Id); + continue; + } + + updatedSubscriptions.Add(info); + } + + lock (_lock) + { + _subscriptions = updatedSubscriptions; + } + } + + internal static SubscriptionInfo ConvertToInfo(ResourceElement resource) + { + var profile = resource.Scalar("Subscription.meta.profile"); + + if (profile != MetaString) + { + return null; + } + + var criteria = resource.Scalar($"Subscription.criteria"); + + if (criteria != CriteriaString) + { + return null; + } + + var criteriaExt = resource.Scalar($"Subscription.criteria.extension.where(url = '{CriteriaExtensionString}').value"); + var channelTypeExt = resource.Scalar($"Subscription.channel.type.extension.where(url = '{ChannelTypeString}').value.code"); + var payloadType = resource.Scalar($"Subscription.channel.payload.extension.where(url = '{PayloadTypeString}').value"); + var maxCount = resource.Scalar($"Subscription.channel.extension.where(url = '{MaxCountString}').value"); + + var channelInfo = new ChannelInfo + { + Endpoint = resource.Scalar($"Subscription.channel.endpoint"), + ChannelType = channelTypeExt switch + { + "azure-storage" => SubscriptionChannelType.Storage, + "azure-lake-storage" => SubscriptionChannelType.DatalakeContract, + _ => SubscriptionChannelType.None, + }, + ContentType = payloadType switch + { + "full-resource" => SubscriptionContentType.FullResource, + "id-only" => SubscriptionContentType.IdOnly, + _ => SubscriptionContentType.Empty, + }, + MaxCount = maxCount ?? 100, }; - return Task.FromResult(list); + var info = new SubscriptionInfo(criteriaExt, channelInfo); + + return info; + } + + public async Task> GetActiveSubscriptionsAsync(CancellationToken cancellationToken) + { + if (_subscriptions.Count == 0) + { + await SyncSubscriptionsAsync(cancellationToken); + } + + return _subscriptions; + } + + public async Task Handle(StorageInitializedNotification notification, CancellationToken cancellationToken) + { + // Preload subscriptions when storage becomes available + await SyncSubscriptionsAsync(cancellationToken); } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs index d58ff3085e..b12ce1f5f7 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs @@ -8,9 +8,12 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using MediatR; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Messages.Storage; using Microsoft.Health.Fhir.Subscriptions.Channels; using Microsoft.Health.Fhir.Subscriptions.Persistence; using Microsoft.Health.JobManagement; @@ -31,7 +34,9 @@ public void Load(IServiceCollection services) job.AsDelegate>(); } - services.Add() + services + .RemoveServiceTypeExact>() + .Add() .Singleton() .AsSelf() .AsImplementedInterfaces(); diff --git a/src/Microsoft.Health.Fhir.Tests.Common/CommonSamples.cs b/src/Microsoft.Health.Fhir.Tests.Common/CommonSamples.cs new file mode 100644 index 0000000000..c89df50779 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Tests.Common/CommonSamples.cs @@ -0,0 +1,52 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using EnsureThat; +using Hl7.Fhir.ElementModel; +using Hl7.Fhir.Serialization; +using Hl7.Fhir.Specification; +using Microsoft.AspNetCore.SignalR; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Tests.Common +{ + public class CommonSamples + { + /// + /// Loads a sample Resource + /// + public static ResourceElement GetJsonSample(string fileName, IModelInfoProvider modelInfoProvider = null) + { + EnsureArg.IsNotNullOrWhiteSpace(fileName, nameof(fileName)); + + if (modelInfoProvider == null) + { + modelInfoProvider = MockModelInfoProviderBuilder + .Create(FhirSpecification.R4) + .Build(); + } + + return GetJsonSample(fileName, modelInfoProvider.Version, node => modelInfoProvider.ToTypedElement(node)); + } + + public static ResourceElement GetJsonSample(string fileName, FhirSpecification fhirSpecification, Func convert) + { + EnsureArg.IsNotNullOrWhiteSpace(fileName, nameof(fileName)); + + var fhirSource = EmbeddedResourceManager.GetStringContent("TestFiles", fileName, "json", fhirSpecification); + + var node = FhirJsonNode.Parse(fhirSource); + + var instance = convert(node); + + return new ResourceElement(instance); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Tests.Common/EmbeddedResourceManager.cs b/src/Microsoft.Health.Fhir.Tests.Common/EmbeddedResourceManager.cs index 44aec6ea0a..bdc47d0606 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/EmbeddedResourceManager.cs +++ b/src/Microsoft.Health.Fhir.Tests.Common/EmbeddedResourceManager.cs @@ -11,13 +11,13 @@ namespace Microsoft.Health.Fhir.Tests.Common { public static class EmbeddedResourceManager { - public static string GetStringContent(string embeddedResourceSubNamespace, string fileName, string extension) + public static string GetStringContent(string embeddedResourceSubNamespace, string fileName, string extension, FhirSpecification version) { - string resourceName = $"{typeof(EmbeddedResourceManager).Namespace}.{embeddedResourceSubNamespace}.{ModelInfoProvider.Version}.{fileName}.{extension}"; + string resourceName = $"{typeof(EmbeddedResourceManager).Namespace}.{embeddedResourceSubNamespace}.{version}.{fileName}.{extension}"; var resourceInfo = Assembly.GetExecutingAssembly().GetManifestResourceInfo(resourceName); - if (resourceInfo == null && ModelInfoProvider.Version == FhirSpecification.R4B) + if (resourceInfo == null && version == FhirSpecification.R4B) { // Try R4 version resourceName = $"{typeof(EmbeddedResourceManager).Namespace}.{embeddedResourceSubNamespace}.R4.{fileName}.{extension}"; @@ -38,5 +38,10 @@ public static string GetStringContent(string embeddedResourceSubNamespace, strin } } } + + public static string GetStringContent(string embeddedResourceSubNamespace, string fileName, string extension) + { + return GetStringContent(embeddedResourceSubNamespace, fileName, extension, ModelInfoProvider.Version); + } } } diff --git a/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj b/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj index 89524374d1..9dc53d5c30 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj +++ b/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj @@ -16,6 +16,7 @@ + @@ -92,6 +93,7 @@ + diff --git a/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/Subscription-Backport.json b/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/Subscription-Backport.json new file mode 100644 index 0000000000..aa41774c65 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/Subscription-Backport.json @@ -0,0 +1,54 @@ +{ + "resourceType": "Subscription", + "id": "example-backport-storage", + "meta": { + "profile": [ "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription" ] + }, + "status": "requested", + "end": "2031-01-01T12:00:00", + "reason": "Test subscription based on transactions, filtered by Patient", + "criteria": "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions", + "_criteria": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-filter-criteria", + "valueString": "Patient" + } + ] + }, + "channel": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-heartbeat-period", + "valueInteger": 120 + }, + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-max-count", + "valuePositiveInt": 20 + } + ], + "type": "rest-hook", + "_type": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type", + "valueCoding": { + "system": "http://azurehealthcareapis.com/data-extentions/subscription-channel-type", + "code": "azure-storage", + "display": "Azure Blob Storage" + } + } + ] + }, + "endpoint": "sync-all", + "payload": "application/fhir+json", + "_payload": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content", + "valueCode": "full-resource" + } + ] + } + } +} From 66fba51e2f2e5321c7c3db4d378595ce1ec693cf Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Mon, 22 Apr 2024 09:29:47 -0700 Subject: [PATCH 32/47] EventGrid WIP --- Directory.Packages.props | 1 + docs/rest/Subscriptions.http | 10 +++ .../Storage/SqlRetry/SqlRetryService.cs | 2 +- .../Channels/EventGridChannel.cs | 81 +++++++++++++++++++ ...Microsoft.Health.Fhir.Subscriptions.csproj | 4 + 5 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 9df8c6869d..7479c7f93d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -31,6 +31,7 @@ + diff --git a/docs/rest/Subscriptions.http b/docs/rest/Subscriptions.http index 68bcb743e1..cab4f1c4ae 100644 --- a/docs/rest/Subscriptions.http +++ b/docs/rest/Subscriptions.http @@ -176,4 +176,14 @@ Authorization: Bearer {{bearer.response.body.access_token}} ### DELETE https://{{hostname}}/Subscription/example-backport-storage-all content-type: application/json +Authorization: Bearer {{bearer.response.body.access_token}} + +### +DELETE https://{{hostname}}/Subscription/example-backport-storage-patient +content-type: application/json +Authorization: Bearer {{bearer.response.body.access_token}} + +### +DELETE https://{{hostname}}/Subscription/example-backport-lake +content-type: application/json Authorization: Bearer {{bearer.response.body.access_token}} \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs index ac2c53d1e4..a4f4d6c8ba 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs @@ -255,7 +255,7 @@ public async Task ExecuteSql(Func"Flag indicating whether retries are disabled." /// A task representing the asynchronous operation. /// When executing this method, if exception is thrown that is not retriable or if last retry fails, then same exception is thrown by this method. - public async Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false) + public async Task ExecuteSql(SqlCommand sqlCommand, Func action, ILogger logger, string logMessage, CancellationToken cancellationToken, bool isReadOnly = false, bool disableRetries = false) { EnsureArg.IsNotNull(sqlCommand, nameof(sqlCommand)); EnsureArg.IsNotNull(action, nameof(action)); diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs new file mode 100644 index 0000000000..9476649097 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs @@ -0,0 +1,81 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure.Messaging.EventGrid; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Channels +{ + [ChannelType(SubscriptionChannelType.EventGrid)] + public class EventGridChannel : ISubscriptionChannel + { + public EventGridChannel() + { + } + + public Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) + { + throw new NotImplementedException(); + } + + /* + public EventGridEvent CreateEventGridEvent(ResourceWrapper rcd) + { + EnsureArg.IsNotNull(rcd); + + string resourceId = rcd.ResourceId; + string resourceTypeName = rcd.ResourceTypeName; + string resourceVersion = rcd.Version; + string dataVersion = resourceVersion.ToString(CultureInfo.InvariantCulture); + string fhirAccountDomainName = _workerConfiguration.FhirAccount; + + string eventSubject = GetEventSubject(rcd); + string eventType = _workerConfiguration.ResourceChangeTypeMap[rcd.ResourceChangeTypeId]; + string eventGuid = rcd.GetSha256BasedGuid(); + + // The swagger specification requires the response JSON to have all properties use camelcasing + // and hence the dataPayload properties below have to use camelcase. + var dataPayload = new BinaryData(new + { + resourceType = resourceTypeName, + resourceFhirAccount = fhirAccountDomainName, + resourceFhirId = resourceId, + resourceVersionId = resourceVersion, + }); + + return new EventGridEvent( + subject: eventSubject, + eventType: eventType, + dataVersion: dataVersion, + data: dataPayload) + { + Topic = _workerConfiguration.EventGridTopic, + Id = eventGuid, + EventTime = rcd.Timestamp, + }; + } + + public string GetEventSubject(ResourceChangeData rcd) + { + EnsureArg.IsNotNull(rcd); + + // Example: "myfhirserver.contoso.com/Observation/cb875194-1195-4617-b2e9-0966bd6b8a10" + var fhirAccountDomainName = "fhirevents"; + var subjectSegements = new string[] { fhirAccountDomainName, rcd.ResourceTypeName, rcd.ResourceId }; + var subject = string.Join("/", subjectSegements); + return subject; + } + */ + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj index 13218ee495..7ec3054f0c 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj +++ b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj @@ -1,5 +1,9 @@  + + + + From d51bde3d4642818e1317342ee32cb3838117bd94 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Wed, 28 Feb 2024 18:06:58 -0800 Subject: [PATCH 33/47] Improve fhirtimer --- .../Storage/SqlRetry/SqlRetryService.cs | 2 +- .../Features/Storage/SqlStoreClient.cs | 24 ++--- .../Watchdogs/CleanupEventLogWatchdog.cs | 25 ++--- .../Features/Watchdogs/DefragWatchdog.cs | 28 +++--- .../Features/Watchdogs/FhirTimer.cs | 73 +++++++++------ .../InvisibleHistoryCleanupWatchdog.cs | 54 ++++++----- .../Features/Watchdogs/TransactionWatchdog.cs | 52 ++++++----- .../Features/Watchdogs/Watchdog.cs | 92 +++++++++++-------- .../Features/Watchdogs/WatchdogLease.cs | 53 +++++++---- .../Watchdogs/WatchdogsBackgroundService.cs | 14 +-- ...erFhirResourceChangeCaptureEnabledTests.cs | 56 +++++++---- .../Persistence/FhirStorageTestsFixture.cs | 3 +- .../Persistence/SqlServerWatchdogTests.cs | 58 ++++++++---- tools/EventsReader/Program.cs | 2 +- 14 files changed, 310 insertions(+), 226 deletions(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs index a4f4d6c8ba..a544c7b915 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs @@ -369,7 +369,7 @@ public async Task TryLogEvent(string process, string status, string text, DateTi { try { - using var cmd = new SqlCommand() { CommandType = CommandType.StoredProcedure, CommandText = "dbo.LogEvent" }; + await using var cmd = new SqlCommand { CommandType = CommandType.StoredProcedure, CommandText = "dbo.LogEvent" }; cmd.Parameters.AddWithValue("@Process", process); cmd.Parameters.AddWithValue("@Status", status); cmd.Parameters.AddWithValue("@Text", text); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs index 5d4daf17b3..f6b27e3848 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlStoreClient.cs @@ -146,7 +146,7 @@ private static Lazy ReadRawResource(SqlDataReader reader, Func> GetResourcesByTransactionIdAsync(long transactionId, Func decompress, Func getResourceTypeName, CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.GetResourcesByTransactionId", CommandType = CommandType.StoredProcedure, CommandTimeout = 600 }; + await using var cmd = new SqlCommand() { CommandText = "dbo.GetResourcesByTransactionId", CommandType = CommandType.StoredProcedure, CommandTimeout = 600 }; cmd.Parameters.AddWithValue("@TransactionId", transactionId); //// ignore invisible resources return (await cmd.ExecuteReaderAsync(_sqlRetryService, (reader) => { return ReadResourceWrapper(reader, true, decompress, getResourceTypeName); }, _logger, cancellationToken)).Where(_ => _.RawResource.Data != _invisibleResource).ToList(); @@ -186,7 +186,7 @@ internal async Task MergeResourcesPutTransactionHeartbeatAsync(long transactionI { try { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesPutTransactionHeartbeat", CommandType = CommandType.StoredProcedure, CommandTimeout = (heartbeatPeriod.Seconds / 3) + 1 }; // +1 to avoid = SQL default timeout value + await using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesPutTransactionHeartbeat", CommandType = CommandType.StoredProcedure, CommandTimeout = (heartbeatPeriod.Seconds / 3) + 1 }; // +1 to avoid = SQL default timeout value cmd.Parameters.AddWithValue("@TransactionId", transactionId); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken); } @@ -209,7 +209,7 @@ private ResourceDateKey ReadResourceDateKeyWrapper(SqlDataReader reader) internal async Task MergeResourcesGetTransactionVisibilityAsync(CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesGetTransactionVisibility", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesGetTransactionVisibility", CommandType = CommandType.StoredProcedure }; var transactionIdParam = new SqlParameter("@TransactionId", SqlDbType.BigInt) { Direction = ParameterDirection.Output }; cmd.Parameters.Add(transactionIdParam); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken); @@ -218,7 +218,7 @@ internal async Task MergeResourcesGetTransactionVisibilityAsync(Cancellati internal async Task<(long TransactionId, int Sequence)> MergeResourcesBeginTransactionAsync(int resourceVersionCount, CancellationToken cancellationToken, DateTime? heartbeatDate = null) { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesBeginTransaction", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesBeginTransaction", CommandType = CommandType.StoredProcedure }; cmd.Parameters.AddWithValue("@Count", resourceVersionCount); var transactionIdParam = new SqlParameter("@TransactionId", SqlDbType.BigInt) { Direction = ParameterDirection.Output }; cmd.Parameters.Add(transactionIdParam); @@ -258,7 +258,7 @@ internal async Task MergeResourcesGetTransactionVisibilityAsync(Cancellati internal async Task MergeResourcesDeleteInvisibleHistory(long transactionId, CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesDeleteInvisibleHistory", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesDeleteInvisibleHistory", CommandType = CommandType.StoredProcedure }; cmd.Parameters.AddWithValue("@TransactionId", transactionId); var affectedRowsParam = new SqlParameter("@affectedRows", SqlDbType.Int) { Direction = ParameterDirection.Output }; cmd.Parameters.Add(affectedRowsParam); @@ -268,7 +268,7 @@ internal async Task MergeResourcesDeleteInvisibleHistory(long transactionId internal async Task MergeResourcesCommitTransactionAsync(long transactionId, string failureReason, CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesCommitTransaction", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesCommitTransaction", CommandType = CommandType.StoredProcedure }; cmd.Parameters.AddWithValue("@TransactionId", transactionId); if (failureReason != null) { @@ -280,14 +280,14 @@ internal async Task MergeResourcesCommitTransactionAsync(long transactionId, str internal async Task MergeResourcesPutTransactionInvisibleHistoryAsync(long transactionId, CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesPutTransactionInvisibleHistory", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesPutTransactionInvisibleHistory", CommandType = CommandType.StoredProcedure }; cmd.Parameters.AddWithValue("@TransactionId", transactionId); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken); } internal async Task MergeResourcesAdvanceTransactionVisibilityAsync(CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesAdvanceTransactionVisibility", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand { CommandText = "dbo.MergeResourcesAdvanceTransactionVisibility", CommandType = CommandType.StoredProcedure }; var affectedRowsParam = new SqlParameter("@AffectedRows", SqlDbType.Int) { Direction = ParameterDirection.Output }; cmd.Parameters.Add(affectedRowsParam); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken); @@ -297,14 +297,14 @@ internal async Task MergeResourcesAdvanceTransactionVisibilityAsync(Cancell internal async Task> MergeResourcesGetTimeoutTransactionsAsync(int timeoutSec, CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.MergeResourcesGetTimeoutTransactions", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand { CommandText = "dbo.MergeResourcesGetTimeoutTransactions", CommandType = CommandType.StoredProcedure }; cmd.Parameters.AddWithValue("@TimeoutSec", timeoutSec); - return await cmd.ExecuteReaderAsync(_sqlRetryService, (reader) => { return reader.GetInt64(0); }, _logger, cancellationToken); + return await cmd.ExecuteReaderAsync(_sqlRetryService, reader => reader.GetInt64(0), _logger, cancellationToken); } internal async Task> GetTransactionsAsync(long startNotInclusiveTranId, long endInclusiveTranId, CancellationToken cancellationToken, DateTime? endDate = null) { - using var cmd = new SqlCommand() { CommandText = "dbo.GetTransactions", CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand { CommandText = "dbo.GetTransactions", CommandType = CommandType.StoredProcedure }; cmd.Parameters.AddWithValue("@StartNotInclusiveTranId", startNotInclusiveTranId); cmd.Parameters.AddWithValue("@EndInclusiveTranId", endInclusiveTranId); if (endDate.HasValue) @@ -326,7 +326,7 @@ internal async Task> MergeResourcesGetTimeoutTransactionsAsy internal async Task> GetResourceDateKeysByTransactionIdAsync(long transactionId, CancellationToken cancellationToken) { - using var cmd = new SqlCommand() { CommandText = "dbo.GetResourcesByTransactionId", CommandType = CommandType.StoredProcedure, CommandTimeout = 600 }; + await using var cmd = new SqlCommand { CommandText = "dbo.GetResourcesByTransactionId", CommandType = CommandType.StoredProcedure, CommandTimeout = 600 }; cmd.Parameters.AddWithValue("@TransactionId", transactionId); cmd.Parameters.AddWithValue("@IncludeHistory", true); cmd.Parameters.AddWithValue("@ReturnResourceKeysOnly", true); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/CleanupEventLogWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/CleanupEventLogWatchdog.cs index 23d1e1b0e5..1b01cafdd7 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/CleanupEventLogWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/CleanupEventLogWatchdog.cs @@ -13,13 +13,10 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - public class CleanupEventLogWatchdog : Watchdog + internal sealed class CleanupEventLogWatchdog : Watchdog { private readonly ISqlRetryService _sqlRetryService; private readonly ILogger _logger; - private CancellationToken _cancellationToken; - private const double _periodSec = 12 * 3600; - private const double _leasePeriodSec = 3600; public CleanupEventLogWatchdog(ISqlRetryService sqlRetryService, ILogger logger) : base(sqlRetryService, logger) @@ -29,25 +26,23 @@ public CleanupEventLogWatchdog(ISqlRetryService sqlRetryService, ILogger + internal sealed class DefragWatchdog : Watchdog { private const byte QueueType = (byte)Core.Features.Operations.QueueType.Defrag; private int _threads; private int _heartbeatPeriodSec; private int _heartbeatTimeoutSec; - private CancellationToken _cancellationToken; private static readonly string[] Definitions = { "Defrag" }; private readonly ISqlRetryService _sqlRetryService; @@ -41,7 +40,6 @@ public DefragWatchdog( } internal DefragWatchdog() - : base() { // this is used to get param names for testing } @@ -54,24 +52,22 @@ internal DefragWatchdog() internal string IsEnabledId => $"{Name}.IsEnabled"; - internal async Task StartAsync(CancellationToken cancellationToken) - { - _cancellationToken = cancellationToken; - await Task.WhenAll( - StartAsync(false, 24 * 3600, 2 * 3600, cancellationToken), - InitDefragParamsAsync()); - } + public override double LeasePeriodSec { get; internal set; } = 2 * 3600; + + public override bool AllowRebalance { get; internal set; } = false; + + public override double PeriodSec { get; internal set; } = 24 * 3600; - protected override async Task ExecuteAsync() + protected override async Task RunWorkAsync(CancellationToken cancellationToken) { - if (!await IsEnabledAsync(_cancellationToken)) + if (!await IsEnabledAsync(cancellationToken)) { _logger.LogInformation("Watchdog is not enabled. Exiting..."); return; } - using var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(_cancellationToken); - var job = await GetCoordinatorJobAsync(_cancellationToken); + using var cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + (long groupId, long jobId, long version, int activeDefragItems) job = await GetCoordinatorJobAsync(cancellationToken); if (job.jobId == -1) { @@ -123,7 +119,7 @@ await JobHosting.ExecuteJobWithHeartbeatsAsync( TimeSpan.FromSeconds(_heartbeatPeriodSec), cancellationTokenSource); - await CompleteJobAsync(job.jobId, job.version, false, _cancellationToken); + await CompleteJobAsync(job.jobId, job.version, false, cancellationToken); } private async Task ChangeDatabaseSettingsAsync(bool isOn, CancellationToken cancellationToken) @@ -301,7 +297,7 @@ private async Task GetHeartbeatTimeoutAsync(CancellationToken cancellationT return (int)value; } - private async Task InitDefragParamsAsync() // No CancellationToken is passed since we shouldn't cancel initialization. + protected override async Task InitAdditionalParamsAsync() { _logger.LogInformation("InitDefragParamsAsync starting..."); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs index a5bbc7276a..a85499c997 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/FhirTimer.cs @@ -7,55 +7,76 @@ using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; +using EnsureThat; using Microsoft.Extensions.Logging; using Microsoft.Health.Core; using Microsoft.Health.Fhir.Core.Extensions; namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - public abstract class FhirTimer(ILogger logger = null) + public class FhirTimer(ILogger logger = null) { + private bool _active; + private bool _isFailing; - internal double PeriodSec { get; set; } + public double PeriodSec { get; private set; } + + public DateTimeOffset LastRunDateTime { get; private set; } = DateTimeOffset.Parse("2017-12-01"); - internal DateTimeOffset LastRunDateTime { get; private set; } = DateTime.Parse("2017-12-01"); + public bool IsFailing => _isFailing; - internal bool IsFailing => _isFailing; + public bool IsRunning { get; private set; } - protected async Task StartAsync(double periodSec, CancellationToken cancellationToken) + /// + /// Runs the execution of the timer until the is cancelled. + /// + public async Task ExecuteAsync(double periodSec, Func onNextTick, CancellationToken cancellationToken) { + EnsureArg.IsNotNull(onNextTick, nameof(onNextTick)); PeriodSec = periodSec; + if (_active) + { + throw new InvalidOperationException("Timer is already running"); + } + + _active = true; await Task.Delay(TimeSpan.FromSeconds(PeriodSec * RandomNumberGenerator.GetInt32(1000) / 1000), cancellationToken); using var periodicTimer = new PeriodicTimer(TimeSpan.FromSeconds(PeriodSec)); - while (!cancellationToken.IsCancellationRequested) + try { - try - { - await periodicTimer.WaitForNextTickAsync(cancellationToken); - } - catch (OperationCanceledException) + while (!cancellationToken.IsCancellationRequested) { - // Time to exit - break; - } + try + { + await periodicTimer.WaitForNextTickAsync(cancellationToken); + } + catch (OperationCanceledException) + { + break; + } - try - { - await RunAsync(); - LastRunDateTime = Clock.UtcNow; - _isFailing = false; - } - catch (Exception e) - { - logger.LogWarning(e, "Error executing timer"); - _isFailing = true; + try + { + IsRunning = true; + await onNextTick(cancellationToken); + LastRunDateTime = Clock.UtcNow; + _isFailing = false; + } + catch (Exception e) + { + logger?.LogWarning(e, "Error executing timer"); + _isFailing = true; + } } } + finally + { + _active = false; + IsRunning = false; + } } - - protected abstract Task RunAsync(); } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs index eddcded1d8..7efa16a6c7 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/InvisibleHistoryCleanupWatchdog.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -14,13 +15,11 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - internal class InvisibleHistoryCleanupWatchdog : Watchdog + internal sealed class InvisibleHistoryCleanupWatchdog : Watchdog { private readonly SqlStoreClient _store; private readonly ILogger _logger; private readonly ISqlRetryService _sqlRetryService; - private CancellationToken _cancellationToken; - private double _retentionPeriodDays = 7; public InvisibleHistoryCleanupWatchdog(SqlStoreClient store, ISqlRetryService sqlRetryService, ILogger logger) : base(sqlRetryService, logger) @@ -31,48 +30,47 @@ public InvisibleHistoryCleanupWatchdog(SqlStoreClient store, ISqlRetryService sq } internal InvisibleHistoryCleanupWatchdog() - : base() { // this is used to get param names for testing } - internal string LastCleanedUpTransactionId => $"{Name}.LastCleanedUpTransactionId"; + public string LastCleanedUpTransactionId => $"{Name}.LastCleanedUpTransactionId"; - internal async Task StartAsync(CancellationToken cancellationToken, double? periodSec = null, double? leasePeriodSec = null, double? retentionPeriodDays = null) - { - _cancellationToken = cancellationToken; - await InitLastCleanedUpTransactionId(); - await StartAsync(true, periodSec ?? 3600, leasePeriodSec ?? 2 * 3600, cancellationToken); - if (retentionPeriodDays.HasValue) - { - _retentionPeriodDays = retentionPeriodDays.Value; - } - } + public override double LeasePeriodSec { get; internal set; } = 2 * 3600; - protected override async Task ExecuteAsync() + public override bool AllowRebalance { get; internal set; } = true; + + public override double PeriodSec { get; internal set; } = 3600; + + public double RetentionPeriodDays { get; internal set; } = 7; + + protected override async Task RunWorkAsync(CancellationToken cancellationToken) { _logger.LogInformation($"{Name}: starting..."); - var lastTranId = await GetLastCleanedUpTransactionId(); - var visibility = await _store.MergeResourcesGetTransactionVisibilityAsync(_cancellationToken); + var lastTranId = await GetLastCleanedUpTransactionIdAsync(cancellationToken); + var visibility = await _store.MergeResourcesGetTransactionVisibilityAsync(cancellationToken); _logger.LogInformation($"{Name}: last cleaned up transaction={lastTranId} visibility={visibility}."); - var transToClean = await _store.GetTransactionsAsync(lastTranId, visibility, _cancellationToken, DateTime.UtcNow.AddDays((-1) * _retentionPeriodDays)); + IReadOnlyList<(long TransactionId, DateTime? VisibleDate, DateTime? InvisibleHistoryRemovedDate)> transToClean = + await _store.GetTransactionsAsync(lastTranId, visibility, cancellationToken, DateTime.UtcNow.AddDays(-1 * RetentionPeriodDays)); + _logger.LogInformation($"{Name}: found transactions={transToClean.Count}."); if (transToClean.Count == 0) { - _logger.LogInformation($"{Name}: completed. transactions=0."); + _logger.LogDebug($"{Name}: completed. transactions=0."); return; } var totalRows = 0; - foreach (var tran in transToClean.Where(_ => !_.InvisibleHistoryRemovedDate.HasValue).OrderBy(_ => _.TransactionId)) + foreach ((long TransactionId, DateTime? VisibleDate, DateTime? InvisibleHistoryRemovedDate) tran in + transToClean.Where(x => !x.InvisibleHistoryRemovedDate.HasValue).OrderBy(x => x.TransactionId)) { - var rows = await _store.MergeResourcesDeleteInvisibleHistory(tran.TransactionId, _cancellationToken); + var rows = await _store.MergeResourcesDeleteInvisibleHistory(tran.TransactionId, cancellationToken); _logger.LogInformation($"{Name}: transaction={tran.TransactionId} removed rows={rows}."); totalRows += rows; - await _store.MergeResourcesPutTransactionInvisibleHistoryAsync(tran.TransactionId, _cancellationToken); + await _store.MergeResourcesPutTransactionInvisibleHistoryAsync(tran.TransactionId, cancellationToken); } await UpdateLastCleanedUpTransactionId(transToClean.Max(_ => _.TransactionId)); @@ -80,21 +78,21 @@ protected override async Task ExecuteAsync() _logger.LogInformation($"{Name}: completed. transactions={transToClean.Count} removed rows={totalRows}"); } - private async Task GetLastCleanedUpTransactionId() + private async Task GetLastCleanedUpTransactionIdAsync(CancellationToken cancellationToken) { - return await GetLongParameterByIdAsync(LastCleanedUpTransactionId, _cancellationToken); + return await GetLongParameterByIdAsync(LastCleanedUpTransactionId, cancellationToken); } - private async Task InitLastCleanedUpTransactionId() + protected override async Task InitAdditionalParamsAsync() { - using var cmd = new SqlCommand("INSERT INTO dbo.Parameters (Id, Bigint) SELECT @Id, 5105975696064002770"); // surrogate id for the past. does not matter. + await using var cmd = new SqlCommand("INSERT INTO dbo.Parameters (Id, Bigint) SELECT @Id, 5105975696064002770"); // surrogate id for the past. does not matter. cmd.Parameters.AddWithValue("@Id", LastCleanedUpTransactionId); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); } private async Task UpdateLastCleanedUpTransactionId(long lastTranId) { - using var cmd = new SqlCommand("UPDATE dbo.Parameters SET Bigint = @LastTranId WHERE Id = @Id"); + await using var cmd = new SqlCommand("UPDATE dbo.Parameters SET Bigint = @LastTranId WHERE Id = @Id"); cmd.Parameters.AddWithValue("@Id", LastCleanedUpTransactionId); cmd.Parameters.AddWithValue("@LastTranId", lastTranId); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs index 7dd6a7d600..c5c75cefdc 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/TransactionWatchdog.cs @@ -4,6 +4,7 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -15,14 +16,12 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - internal class TransactionWatchdog : Watchdog + internal sealed class TransactionWatchdog : Watchdog { private readonly SqlServerFhirDataStore _store; private readonly IResourceWrapperFactory _factory; private readonly ILogger _logger; - private CancellationToken _cancellationToken; - private const double _periodSec = 3; - private const double _leasePeriodSec = 20; + private const string AdvancedVisibilityTemplate = "TransactionWatchdog advanced visibility on {Transactions} transactions."; public TransactionWatchdog(SqlServerFhirDataStore store, IResourceWrapperFactory factory, ISqlRetryService sqlRetryService, ILogger logger) : base(sqlRetryService, logger) @@ -33,49 +32,54 @@ public TransactionWatchdog(SqlServerFhirDataStore store, IResourceWrapperFactory } internal TransactionWatchdog() - : base() { // this is used to get param names for testing } - internal async Task StartAsync(CancellationToken cancellationToken) - { - _cancellationToken = cancellationToken; - await StartAsync(true, _periodSec, _leasePeriodSec, cancellationToken); - } + public override double LeasePeriodSec { get; internal set; } = 20; + + public override bool AllowRebalance { get; internal set; } = true; - protected override async Task ExecuteAsync() + public override double PeriodSec { get; internal set; } = 3; + + protected override async Task RunWorkAsync(CancellationToken cancellationToken) { - _logger.LogInformation("TransactionWatchdog starting..."); - var affectedRows = await _store.StoreClient.MergeResourcesAdvanceTransactionVisibilityAsync(_cancellationToken); - _logger.LogInformation("TransactionWatchdog advanced visibility on {Transactions} transactions.", affectedRows); + var affectedRows = await _store.StoreClient.MergeResourcesAdvanceTransactionVisibilityAsync(cancellationToken); + + _logger.Log( + affectedRows > 0 ? LogLevel.Information : LogLevel.Debug, + AdvancedVisibilityTemplate, + affectedRows); if (affectedRows > 0) { return; } - var timeoutTransactions = await _store.StoreClient.MergeResourcesGetTimeoutTransactionsAsync((int)SqlServerFhirDataStore.MergeResourcesTransactionHeartbeatPeriod.TotalSeconds * 6, _cancellationToken); + IReadOnlyList timeoutTransactions = await _store.StoreClient.MergeResourcesGetTimeoutTransactionsAsync((int)SqlServerFhirDataStore.MergeResourcesTransactionHeartbeatPeriod.TotalSeconds * 6, cancellationToken); if (timeoutTransactions.Count > 0) { _logger.LogWarning("TransactionWatchdog found {Transactions} timed out transactions", timeoutTransactions.Count); - await _store.StoreClient.TryLogEvent("TransactionWatchdog", "Warn", $"found timed out transactions={timeoutTransactions.Count}", null, _cancellationToken); + await _store.StoreClient.TryLogEvent("TransactionWatchdog", "Warn", $"found timed out transactions={timeoutTransactions.Count}", null, cancellationToken); } else { - _logger.LogInformation("TransactionWatchdog found {Transactions} timed out transactions", timeoutTransactions.Count); + _logger.Log( + timeoutTransactions.Count > 0 ? LogLevel.Information : LogLevel.Debug, + "TransactionWatchdog found {Transactions} timed out transactions", + timeoutTransactions.Count); } foreach (var tranId in timeoutTransactions) { var st = DateTime.UtcNow; _logger.LogInformation("TransactionWatchdog found timed out transaction={Transaction}, attempting to roll forward...", tranId); - var resources = await _store.GetResourcesByTransactionIdAsync(tranId, _cancellationToken); + var resources = await _store.GetResourcesByTransactionIdAsync(tranId, cancellationToken); if (resources.Count == 0) { - await _store.StoreClient.MergeResourcesCommitTransactionAsync(tranId, "WD: 0 resources", _cancellationToken); + await _store.StoreClient.MergeResourcesCommitTransactionAsync(tranId, "WD: 0 resources", cancellationToken); _logger.LogWarning("TransactionWatchdog committed transaction={Transaction}, resources=0", tranId); - await _store.StoreClient.TryLogEvent("TransactionWatchdog", "Warn", $"committed transaction={tranId}, resources=0", st, _cancellationToken); + await _store.StoreClient.TryLogEvent("TransactionWatchdog", "Warn", $"committed transaction={tranId}, resources=0", st, cancellationToken); continue; } @@ -84,12 +88,12 @@ protected override async Task ExecuteAsync() _factory.Update(resource); } - await _store.MergeResourcesWrapperAsync(tranId, false, resources.Select(_ => new MergeResourceWrapper(_, true, true)).ToList(), false, 0, _cancellationToken); - await _store.StoreClient.MergeResourcesCommitTransactionAsync(tranId, null, _cancellationToken); + await _store.MergeResourcesWrapperAsync(tranId, false, resources.Select(x => new MergeResourceWrapper(x, true, true)).ToList(), false, 0, cancellationToken); + await _store.StoreClient.MergeResourcesCommitTransactionAsync(tranId, null, cancellationToken); _logger.LogWarning("TransactionWatchdog committed transaction={Transaction}, resources={Resources}", tranId, resources.Count); - await _store.StoreClient.TryLogEvent("TransactionWatchdog", "Warn", $"committed transaction={tranId}, resources={resources.Count}", st, _cancellationToken); + await _store.StoreClient.TryLogEvent("TransactionWatchdog", "Warn", $"committed transaction={tranId}, resources={resources.Count}", st, cancellationToken); - affectedRows = await _store.StoreClient.MergeResourcesAdvanceTransactionVisibilityAsync(_cancellationToken); + affectedRows = await _store.StoreClient.MergeResourcesAdvanceTransactionVisibilityAsync(cancellationToken); _logger.LogInformation("TransactionWatchdog advanced visibility on {Transactions} transactions.", affectedRows); } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs index 95d2917234..19de170bff 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/Watchdog.cs @@ -10,24 +10,27 @@ using EnsureThat; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; +using Microsoft.Health.Core; using Microsoft.Health.Fhir.SqlServer.Features.Storage; namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - public abstract class Watchdog : FhirTimer + internal abstract class Watchdog + where T : Watchdog { - private ISqlRetryService _sqlRetryService; + private readonly ISqlRetryService _sqlRetryService; private readonly ILogger _logger; private readonly WatchdogLease _watchdogLease; private double _periodSec; private double _leasePeriodSec; + private readonly FhirTimer _fhirTimer; protected Watchdog(ISqlRetryService sqlRetryService, ILogger logger) - : base(logger) { _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); _watchdogLease = new WatchdogLease(_sqlRetryService, _logger); + _fhirTimer = new FhirTimer(_logger); } protected Watchdog() @@ -35,66 +38,83 @@ protected Watchdog() // this is used to get param names for testing } - internal string Name => GetType().Name; + public string Name => GetType().Name; - internal string PeriodSecId => $"{Name}.PeriodSec"; + public string PeriodSecId => $"{Name}.PeriodSec"; - internal string LeasePeriodSecId => $"{Name}.LeasePeriodSec"; + public string LeasePeriodSecId => $"{Name}.LeasePeriodSec"; - internal bool IsLeaseHolder => _watchdogLease.IsLeaseHolder; + public bool IsLeaseHolder => _watchdogLease.IsLeaseHolder; - internal string LeaseWorker => _watchdogLease.Worker; + public string LeaseWorker => _watchdogLease.Worker; - internal double LeasePeriodSec => _watchdogLease.PeriodSec; + public abstract double LeasePeriodSec { get; internal set; } - protected internal async Task StartAsync(bool allowRebalance, double periodSec, double leasePeriodSec, CancellationToken cancellationToken) + public abstract bool AllowRebalance { get; internal set; } + + public abstract double PeriodSec { get; internal set; } + + public bool IsInitialized { get; private set; } + + public async Task ExecuteAsync(CancellationToken cancellationToken) { - _logger.LogInformation($"{Name}.StartAsync: starting..."); - await InitParamsAsync(periodSec, leasePeriodSec); + _logger.LogInformation($"{Name}.ExecuteAsync: starting..."); + + await InitParamsAsync(PeriodSec, LeasePeriodSec); await Task.WhenAll( - StartAsync(_periodSec, cancellationToken), - _watchdogLease.StartAsync(allowRebalance, _leasePeriodSec, cancellationToken)); + _fhirTimer.ExecuteAsync(_periodSec, OnNextTickAsync, cancellationToken), + _watchdogLease.ExecuteAsync(AllowRebalance, _leasePeriodSec, cancellationToken)); - _logger.LogInformation($"{Name}.StartAsync: completed."); + _logger.LogInformation($"{Name}.ExecuteAsync: completed."); } - protected abstract Task ExecuteAsync(); + protected abstract Task RunWorkAsync(CancellationToken cancellationToken); - protected override async Task RunAsync() + private async Task OnNextTickAsync(CancellationToken cancellationToken) { if (!_watchdogLease.IsLeaseHolder) { - _logger.LogInformation($"{Name}.RunAsync: Skipping because watchdog is not a lease holder."); + _logger.LogDebug($"{Name}.OnNextTickAsync: Skipping because watchdog is not a lease holder."); return; } - _logger.LogInformation($"{Name}.RunAsync: Starting..."); - await ExecuteAsync(); - _logger.LogInformation($"{Name}.RunAsync: Completed."); + using (_logger.BeginTimedScope($"{Name}.OnNextTickAsync")) + { + await RunWorkAsync(cancellationToken); + } } private async Task InitParamsAsync(double periodSec, double leasePeriodSec) // No CancellationToken is passed since we shouldn't cancel initialization. { - _logger.LogInformation($"{Name}.InitParamsAsync: starting..."); - - // Offset for other instances running init - await Task.Delay(TimeSpan.FromSeconds(RandomNumberGenerator.GetInt32(10) / 10.0), CancellationToken.None); + using (_logger.BeginTimedScope($"{Name}.InitParamsAsync")) + { + // Offset for other instances running init + await Task.Delay(TimeSpan.FromSeconds(RandomNumberGenerator.GetInt32(10) / 10.0), CancellationToken.None); - using var cmd = new SqlCommand(@" + await using var cmd = new SqlCommand( + @" INSERT INTO dbo.Parameters (Id,Number) SELECT @PeriodSecId, @PeriodSec INSERT INTO dbo.Parameters (Id,Number) SELECT @LeasePeriodSecId, @LeasePeriodSec "); - cmd.Parameters.AddWithValue("@PeriodSecId", PeriodSecId); - cmd.Parameters.AddWithValue("@PeriodSec", periodSec); - cmd.Parameters.AddWithValue("@LeasePeriodSecId", LeasePeriodSecId); - cmd.Parameters.AddWithValue("@LeasePeriodSec", leasePeriodSec); - await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); + cmd.Parameters.AddWithValue("@PeriodSecId", PeriodSecId); + cmd.Parameters.AddWithValue("@PeriodSec", periodSec); + cmd.Parameters.AddWithValue("@LeasePeriodSecId", LeasePeriodSecId); + cmd.Parameters.AddWithValue("@LeasePeriodSec", leasePeriodSec); + await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); + + _periodSec = await GetPeriodAsync(CancellationToken.None); + _leasePeriodSec = await GetLeasePeriodAsync(CancellationToken.None); + + await InitAdditionalParamsAsync(); - _periodSec = await GetPeriodAsync(CancellationToken.None); - _leasePeriodSec = await GetLeasePeriodAsync(CancellationToken.None); + IsInitialized = true; + } + } - _logger.LogInformation($"{Name}.InitParamsAsync: completed."); + protected virtual Task InitAdditionalParamsAsync() + { + return Task.CompletedTask; } private async Task GetPeriodAsync(CancellationToken cancellationToken) @@ -113,7 +133,7 @@ protected async Task GetNumberParameterByIdAsync(string id, Cancellation { EnsureArg.IsNotNullOrEmpty(id, nameof(id)); - using var cmd = new SqlCommand("SELECT Number FROM dbo.Parameters WHERE Id = @Id"); + await using var cmd = new SqlCommand("SELECT Number FROM dbo.Parameters WHERE Id = @Id"); cmd.Parameters.AddWithValue("@Id", id); var value = await cmd.ExecuteScalarAsync(_sqlRetryService, _logger, cancellationToken); @@ -129,7 +149,7 @@ protected async Task GetLongParameterByIdAsync(string id, CancellationToke { EnsureArg.IsNotNullOrEmpty(id, nameof(id)); - using var cmd = new SqlCommand("SELECT Bigint FROM dbo.Parameters WHERE Id = @Id"); + await using var cmd = new SqlCommand("SELECT Bigint FROM dbo.Parameters WHERE Id = @Id"); cmd.Parameters.AddWithValue("@Id", id); var value = await cmd.ExecuteScalarAsync(_sqlRetryService, _logger, cancellationToken); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs index ee4a595cd5..23d4c5bea4 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogLease.cs @@ -10,81 +10,94 @@ using EnsureThat; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; +using Microsoft.Health.Core; +using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.SqlServer.Features.Storage; namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { - internal class WatchdogLease : FhirTimer + internal class WatchdogLease + where T : Watchdog { private const double TimeoutFactor = 0.25; private readonly object _locker = new(); private readonly ISqlRetryService _sqlRetryService; - private readonly ILogger _logger; - private DateTime _leaseEndTime; + private readonly ILogger _logger; + private DateTimeOffset _leaseEndTime; private double _leaseTimeoutSec; private readonly string _worker; - private CancellationToken _cancellationToken; private readonly string _watchdogName; private bool _allowRebalance; + private readonly FhirTimer _fhirTimer; - internal WatchdogLease(ISqlRetryService sqlRetryService, ILogger logger) - : base(logger) + public WatchdogLease(ISqlRetryService sqlRetryService, ILogger logger) { _sqlRetryService = EnsureArg.IsNotNull(sqlRetryService, nameof(sqlRetryService)); _logger = EnsureArg.IsNotNull(logger, nameof(logger)); _watchdogName = typeof(T).Name; _worker = $"{Environment.MachineName}.{Environment.ProcessId}"; _logger.LogInformation($"WatchdogLease:Created lease object, worker=[{_worker}]."); + _fhirTimer = new FhirTimer(logger); } - protected internal string Worker => _worker; + public string Worker => _worker; - protected internal bool IsLeaseHolder + public bool IsLeaseHolder { get { lock (_locker) { - return (DateTime.UtcNow - _leaseEndTime).TotalSeconds < _leaseTimeoutSec; + return (Clock.UtcNow - _leaseEndTime).TotalSeconds < _leaseTimeoutSec; } } } - protected internal async Task StartAsync(bool allowRebalance, double periodSec, CancellationToken cancellationToken) + public bool IsRunning => _fhirTimer.IsRunning; + + public double PeriodSec => _fhirTimer.PeriodSec; + + public async Task ExecuteAsync(bool allowRebalance, double periodSec, CancellationToken cancellationToken) { _logger.LogInformation("WatchdogLease.StartAsync: starting..."); + _allowRebalance = allowRebalance; - _cancellationToken = cancellationToken; - _leaseEndTime = DateTime.MinValue; + _leaseEndTime = DateTimeOffset.MinValue; _leaseTimeoutSec = (int)Math.Ceiling(periodSec * TimeoutFactor); // if it is rounded to 0 it causes problems in AcquireResourceLease logic. - await StartAsync(periodSec, cancellationToken); + + await _fhirTimer.ExecuteAsync(periodSec, OnNextTickAsync, cancellationToken); + _logger.LogInformation("WatchdogLease.StartAsync: completed."); } - protected override async Task RunAsync() + protected async Task OnNextTickAsync(CancellationToken cancellationToken) { - _logger.LogInformation($"WatchdogLease.RunAsync: Starting acquire: resource=[{_watchdogName}] worker=[{_worker}] period={PeriodSec} timeout={_leaseTimeoutSec}..."); + _logger.LogInformation($"WatchdogLease.RunAsync: Starting acquire: resource=[{_watchdogName}] worker=[{_worker}] period={_fhirTimer.PeriodSec} timeout={_leaseTimeoutSec}..."); - using var cmd = new SqlCommand("dbo.AcquireWatchdogLease") { CommandType = CommandType.StoredProcedure }; + await using var cmd = new SqlCommand("dbo.AcquireWatchdogLease") { CommandType = CommandType.StoredProcedure }; cmd.Parameters.AddWithValue("@Watchdog", _watchdogName); cmd.Parameters.AddWithValue("@Worker", _worker); cmd.Parameters.AddWithValue("@AllowRebalance", _allowRebalance); cmd.Parameters.AddWithValue("@WorkerIsRunning", true); cmd.Parameters.AddWithValue("@ForceAcquire", false); // TODO: Provide ability to set. usefull for tests cmd.Parameters.AddWithValue("@LeasePeriodSec", PeriodSec + _leaseTimeoutSec); - var leaseEndTimePar = cmd.Parameters.AddWithValue("@LeaseEndTime", DateTime.UtcNow); + + SqlParameter leaseEndTimePar = cmd.Parameters.AddWithValue("@LeaseEndTime", DateTime.UtcNow); leaseEndTimePar.Direction = ParameterDirection.Output; - var isAcquiredPar = cmd.Parameters.AddWithValue("@IsAcquired", false); + + SqlParameter isAcquiredPar = cmd.Parameters.AddWithValue("@IsAcquired", false); isAcquiredPar.Direction = ParameterDirection.Output; - var currentHolderPar = cmd.Parameters.Add("@CurrentLeaseHolder", SqlDbType.VarChar, 100); + + SqlParameter currentHolderPar = cmd.Parameters.Add("@CurrentLeaseHolder", SqlDbType.VarChar, 100); currentHolderPar.Direction = ParameterDirection.Output; - await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, _cancellationToken); + await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, cancellationToken); var leaseEndTime = (DateTime)leaseEndTimePar.Value; var isAcquired = (bool)isAcquiredPar.Value; var currentHolder = (string)currentHolderPar.Value; + lock (_locker) { _leaseEndTime = isAcquired ? leaseEndTime : _leaseEndTime; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs index 2e411bd8ac..8d1ceae091 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs @@ -24,20 +24,17 @@ internal class WatchdogsBackgroundService : BackgroundService, INotificationHand private readonly CleanupEventLogWatchdog _cleanupEventLogWatchdog; private readonly IScoped _transactionWatchdog; private readonly InvisibleHistoryCleanupWatchdog _invisibleHistoryCleanupWatchdog; - private readonly SubscriptionProcessorWatchdog _subscriptionsProcessorWatchdog; public WatchdogsBackgroundService( DefragWatchdog defragWatchdog, CleanupEventLogWatchdog cleanupEventLogWatchdog, IScopeProvider transactionWatchdog, - InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog, - SubscriptionProcessorWatchdog eventProcessorWatchdog) + InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog) { _defragWatchdog = EnsureArg.IsNotNull(defragWatchdog, nameof(defragWatchdog)); _cleanupEventLogWatchdog = EnsureArg.IsNotNull(cleanupEventLogWatchdog, nameof(cleanupEventLogWatchdog)); _transactionWatchdog = EnsureArg.IsNotNull(transactionWatchdog, nameof(transactionWatchdog)).Invoke(); _invisibleHistoryCleanupWatchdog = EnsureArg.IsNotNull(invisibleHistoryCleanupWatchdog, nameof(invisibleHistoryCleanupWatchdog)); - _subscriptionsProcessorWatchdog = EnsureArg.IsNotNull(eventProcessorWatchdog, nameof(eventProcessorWatchdog)); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -52,11 +49,10 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) var tasks = new List { - _defragWatchdog.StartAsync(continuationTokenSource.Token), - _cleanupEventLogWatchdog.StartAsync(continuationTokenSource.Token), - _transactionWatchdog.Value.StartAsync(continuationTokenSource.Token), - _invisibleHistoryCleanupWatchdog.StartAsync(continuationTokenSource.Token), - _subscriptionsProcessorWatchdog.StartAsync(continuationTokenSource.Token), + _defragWatchdog.ExecuteAsync(continuationTokenSource.Token), + _cleanupEventLogWatchdog.ExecuteAsync(continuationTokenSource.Token), + _transactionWatchdog.Value.ExecuteAsync(continuationTokenSource.Token), + _invisibleHistoryCleanupWatchdog.ExecuteAsync(continuationTokenSource.Token), }; await Task.WhenAny(tasks); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs index 9655ceaa84..0ae103f3a2 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/ChangeFeed/SqlServerFhirResourceChangeCaptureEnabledTests.cs @@ -93,7 +93,7 @@ public async Task GivenADatabaseSupportsResourceChangeCapture_WhenUpdatingAResou Assert.NotNull(resourceChanges); Assert.Single(resourceChanges.Where(x => x.ResourceVersion.ToString() == deserialized.VersionId && x.ResourceId == deserialized.Id)); - var resourceChangeData = resourceChanges.Where(x => x.ResourceVersion.ToString() == deserialized.VersionId && x.ResourceId == deserialized.Id).FirstOrDefault(); + var resourceChangeData = resourceChanges.FirstOrDefault(x => x.ResourceVersion.ToString() == deserialized.VersionId && x.ResourceId == deserialized.Id); Assert.NotNull(resourceChangeData); Assert.Equal(ResourceChangeTypeUpdated, resourceChangeData.ResourceChangeTypeId); @@ -141,23 +141,30 @@ public async Task GivenChangeCaptureEnabledAndNoVersionPolicy_AfterUpdating_Invi cts.CancelAfter(TimeSpan.FromSeconds(60)); var storeClient = new SqlStoreClient(_fixture.SqlRetryService, NullLogger.Instance); - var wd = new InvisibleHistoryCleanupWatchdog(storeClient, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)); - await wd.StartAsync(cts.Token, 1, 2, 2.0 / 24 / 3600); // retention 2 seconds + var wd = new InvisibleHistoryCleanupWatchdog(storeClient, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)) + { + PeriodSec = 1, + LeasePeriodSec = 2, + RetentionPeriodDays = 2.0 / 24 / 3600, + }; + + var wdTask = wd.ExecuteAsync(cts.Token); // retention 2 seconds var startTime = DateTime.UtcNow; - while (!wd.IsLeaseHolder && (DateTime.UtcNow - startTime).TotalSeconds < 60) + + while ((!wd.IsLeaseHolder || !wd.IsInitialized) && !cts.IsCancellationRequested) { - await Task.Delay(TimeSpan.FromSeconds(0.2)); + await Task.Delay(TimeSpan.FromSeconds(0.2), cts.Token); } Assert.True(wd.IsLeaseHolder, "Is lease holder"); _testOutputHelper.WriteLine($"Acquired lease in {(DateTime.UtcNow - startTime).TotalSeconds} seconds."); // create 2 records (1 invisible) - var create = await _fixture.Mediator.CreateResourceAsync(Samples.GetDefaultOrganization()); + var create = await _fixture.Mediator.CreateResourceAsync(Samples.GetDefaultOrganization(), cancellationToken: cts.Token); Assert.Equal("1", create.VersionId); var newValue = Samples.GetDefaultOrganization().UpdateId(create.Id); - newValue.ToPoco().Text = new Hl7.Fhir.Model.Narrative { Status = Hl7.Fhir.Model.Narrative.NarrativeStatus.Generated, Div = $"
Whatever
" }; - var update = await _fixture.Mediator.UpsertResourceAsync(newValue); + newValue.ToPoco().Text = new Hl7.Fhir.Model.Narrative { Status = Hl7.Fhir.Model.Narrative.NarrativeStatus.Generated, Div = "
Whatever
" }; + var update = await _fixture.Mediator.UpsertResourceAsync(newValue, cancellationToken: cts.Token); Assert.Equal("2", update.RawResourceElement.VersionId); // check 2 records exist @@ -166,14 +173,15 @@ public async Task GivenChangeCaptureEnabledAndNoVersionPolicy_AfterUpdating_Invi await store.StoreClient.MergeResourcesAdvanceTransactionVisibilityAsync(CancellationToken.None); // this logic is invoked by WD normally // check only 1 record remains - startTime = DateTime.UtcNow; - while (await GetCount() != 1 && (DateTime.UtcNow - startTime).TotalSeconds < 60) + while (await GetCount() != 1 && !cts.IsCancellationRequested) { - await Task.Delay(TimeSpan.FromSeconds(1)); + await Task.Delay(TimeSpan.FromSeconds(1), cts.Token); } Assert.Equal(1, await GetCount()); DisableInvisibleHistory(); + await cts.CancelAsync(); + await wdTask; } [Fact] @@ -189,19 +197,25 @@ public async Task GivenChangeCaptureEnabledAndNoVersionPolicy_AfterHardDeleting_ cts.CancelAfter(TimeSpan.FromSeconds(60)); var storeClient = new SqlStoreClient(_fixture.SqlRetryService, NullLogger.Instance); - var wd = new InvisibleHistoryCleanupWatchdog(storeClient, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)); - await wd.StartAsync(cts.Token, 1, 2, 2.0 / 24 / 3600); // retention 2 seconds + var wd = new InvisibleHistoryCleanupWatchdog(storeClient, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)) + { + PeriodSec = 1, + LeasePeriodSec = 2, + RetentionPeriodDays = 2.0 / 24 / 3600, + }; + + var wdTask = wd.ExecuteAsync(cts.Token); // retention 2 seconds var startTime = DateTime.UtcNow; - while (!wd.IsLeaseHolder && (DateTime.UtcNow - startTime).TotalSeconds < 60) + while ((!wd.IsLeaseHolder || !wd.IsInitialized) && !cts.IsCancellationRequested) { - await Task.Delay(TimeSpan.FromSeconds(0.2)); + await Task.Delay(TimeSpan.FromSeconds(0.2), cts.Token); } Assert.True(wd.IsLeaseHolder, "Is lease holder"); _testOutputHelper.WriteLine($"Acquired lease in {(DateTime.UtcNow - startTime).TotalSeconds} seconds."); // create 1 resource and hard delete it - var create = await _fixture.Mediator.CreateResourceAsync(Samples.GetDefaultOrganization()); + var create = await _fixture.Mediator.CreateResourceAsync(Samples.GetDefaultOrganization(), CancellationToken.None); Assert.Equal("1", create.VersionId); var resource = await store.GetAsync(new ResourceKey("Organization", create.Id, create.VersionId), CancellationToken.None); @@ -218,14 +232,16 @@ public async Task GivenChangeCaptureEnabledAndNoVersionPolicy_AfterHardDeleting_ await store.StoreClient.MergeResourcesAdvanceTransactionVisibilityAsync(CancellationToken.None); // this logic is invoked by WD normally // check no records - startTime = DateTime.UtcNow; - while (await GetCount() > 0 && (DateTime.UtcNow - startTime).TotalSeconds < 60) + while (await GetCount() > 0 && !cts.IsCancellationRequested) { - await Task.Delay(TimeSpan.FromSeconds(1)); + await Task.Delay(TimeSpan.FromSeconds(1), cts.Token); } Assert.Equal(0, await GetCount()); DisableInvisibleHistory(); + + await cts.CancelAsync(); + await wdTask; } [Fact] @@ -364,7 +380,7 @@ public async Task GivenADatabaseSupportsResourceChangeCapture_WhenDeletingAResou Assert.NotNull(resourceChanges); Assert.Single(resourceChanges.Where(x => x.ResourceVersion.ToString() == deletedResourceKey.ResourceKey.VersionId && x.ResourceId == deletedResourceKey.ResourceKey.Id)); - var resourceChangeData = resourceChanges.Where(x => x.ResourceVersion.ToString() == deletedResourceKey.ResourceKey.VersionId && x.ResourceId == deletedResourceKey.ResourceKey.Id).FirstOrDefault(); + var resourceChangeData = resourceChanges.FirstOrDefault(x => x.ResourceVersion.ToString() == deletedResourceKey.ResourceKey.VersionId && x.ResourceId == deletedResourceKey.ResourceKey.Id); Assert.NotNull(resourceChangeData); Assert.Equal(ResourceChangeTypeDeleted, resourceChangeData.ResourceChangeTypeId); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs index 4d94970cb4..edb3758be1 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/FhirStorageTestsFixture.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.Net.Http; +using System.Threading; using Hl7.Fhir.ElementModel; using Hl7.Fhir.Model; using Hl7.Fhir.Serialization; @@ -160,7 +161,7 @@ public async Task InitializeAsync() medicationResource.Versioning = CapabilityStatement.ResourceVersionPolicy.VersionedUpdate; ConformanceProvider = Substitute.For(); - ConformanceProvider.GetCapabilityStatementOnStartup().Returns(CapabilityStatement.ToTypedElement().ToResourceElement()); + ConformanceProvider.GetCapabilityStatementOnStartup(Arg.Any()).Returns(CapabilityStatement.ToTypedElement().ToResourceElement()); // TODO: FhirRepository instantiate ResourceDeserializer class directly // which will try to deserialize the raw resource. We should mock it as well. diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs index 127eb7a94b..39d579859c 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/SqlServerWatchdogTests.cs @@ -77,12 +77,12 @@ COMMIT TRANSACTION using var cts = new CancellationTokenSource(); cts.CancelAfter(TimeSpan.FromMinutes(10)); - await wd.StartAsync(cts.Token); + Task wsTask = wd.ExecuteAsync(cts.Token); - var startTime = DateTime.UtcNow; + DateTime startTime = DateTime.UtcNow; while (!wd.IsLeaseHolder && (DateTime.UtcNow - startTime).TotalSeconds < 60) { - await Task.Delay(TimeSpan.FromSeconds(1)); + await Task.Delay(TimeSpan.FromSeconds(1), cts.Token); } Assert.True(wd.IsLeaseHolder, "Is lease holder"); @@ -90,7 +90,7 @@ COMMIT TRANSACTION var completed = CheckQueue(current); while (!completed && (DateTime.UtcNow - startTime).TotalSeconds < 120) { - await Task.Delay(TimeSpan.FromSeconds(1)); + await Task.Delay(TimeSpan.FromSeconds(1), cts.Token); completed = CheckQueue(current); } @@ -99,6 +99,9 @@ COMMIT TRANSACTION var sizeAfter = GetSize(); Assert.True(sizeAfter * 9 < sizeBefore, $"{sizeAfter} * 9 < {sizeBefore}"); + + await cts.CancelAsync(); + await wsTask; } [Fact] @@ -122,12 +125,12 @@ WHILE @i < 10000 using var cts = new CancellationTokenSource(); cts.CancelAfter(TimeSpan.FromMinutes(10)); - await wd.StartAsync(cts.Token); + Task wdTask = wd.ExecuteAsync(cts.Token); var startTime = DateTime.UtcNow; while (!wd.IsLeaseHolder && (DateTime.UtcNow - startTime).TotalSeconds < 60) { - await Task.Delay(TimeSpan.FromSeconds(1)); + await Task.Delay(TimeSpan.FromSeconds(1), cts.Token); } Assert.True(wd.IsLeaseHolder, "Is lease holder"); @@ -135,11 +138,14 @@ WHILE @i < 10000 while ((GetCount("EventLog") > 1000) && (DateTime.UtcNow - startTime).TotalSeconds < 120) { - await Task.Delay(TimeSpan.FromSeconds(1)); + await Task.Delay(TimeSpan.FromSeconds(1), cts.Token); } _testOutputHelper.WriteLine($"EventLog.Count={GetCount("EventLog")}."); Assert.True(GetCount("EventLog") <= 1000, "Count is low"); + + await cts.CancelAsync(); + await wdTask; } [Fact] @@ -187,12 +193,18 @@ FOR INSERT ExecuteSql("DROP TRIGGER dbo.tmp_NumberSearchParam"); - var wd = new TransactionWatchdog(_fixture.SqlServerFhirDataStore, factory, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)); - await wd.StartAsync(true, 1, 2, cts.Token); - var startTime = DateTime.UtcNow; + var wd = new TransactionWatchdog(_fixture.SqlServerFhirDataStore, factory, _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)) + { + AllowRebalance = true, + PeriodSec = 1, + LeasePeriodSec = 2, + }; + + Task wdTask = wd.ExecuteAsync(cts.Token); + DateTime startTime = DateTime.UtcNow; while (!wd.IsLeaseHolder && (DateTime.UtcNow - startTime).TotalSeconds < 10) { - await Task.Delay(TimeSpan.FromSeconds(0.2)); + await Task.Delay(TimeSpan.FromSeconds(0.2), cts.Token); } Assert.True(wd.IsLeaseHolder, "Is lease holder"); @@ -205,6 +217,9 @@ FOR INSERT } Assert.Equal(1, GetCount("NumberSearchParam")); // wd rolled forward transaction + + await cts.CancelAsync(); + await wdTask; } [Fact] @@ -215,12 +230,18 @@ public async Task AdvanceVisibility() using var cts = new CancellationTokenSource(); cts.CancelAfter(TimeSpan.FromSeconds(60)); - var wd = new TransactionWatchdog(_fixture.SqlServerFhirDataStore, CreateResourceWrapperFactory(), _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)); - await wd.StartAsync(true, 1, 2, cts.Token); + var wd = new TransactionWatchdog(_fixture.SqlServerFhirDataStore, CreateResourceWrapperFactory(), _fixture.SqlRetryService, XUnitLogger.Create(_testOutputHelper)) + { + AllowRebalance = true, + PeriodSec = 1, + LeasePeriodSec = 2, + }; + + Task wdTask = wd.ExecuteAsync(cts.Token); var startTime = DateTime.UtcNow; while (!wd.IsLeaseHolder && (DateTime.UtcNow - startTime).TotalSeconds < 10) { - await Task.Delay(TimeSpan.FromSeconds(0.2)); + await Task.Delay(TimeSpan.FromSeconds(0.2), cts.Token); } Assert.True(wd.IsLeaseHolder, "Is lease holder"); @@ -241,7 +262,7 @@ public async Task AdvanceVisibility() startTime = DateTime.UtcNow; while ((visibility = await _fixture.SqlServerFhirDataStore.StoreClient.MergeResourcesGetTransactionVisibilityAsync(cts.Token)) != tran1.TransactionId && (DateTime.UtcNow - startTime).TotalSeconds < 10) { - await Task.Delay(TimeSpan.FromSeconds(0.1)); + await Task.Delay(TimeSpan.FromSeconds(0.1), cts.Token); } _testOutputHelper.WriteLine($"Visibility={visibility}"); @@ -254,7 +275,7 @@ public async Task AdvanceVisibility() startTime = DateTime.UtcNow; while ((visibility = await _fixture.SqlServerFhirDataStore.StoreClient.MergeResourcesGetTransactionVisibilityAsync(cts.Token)) != tran2.TransactionId && (DateTime.UtcNow - startTime).TotalSeconds < 10) { - await Task.Delay(TimeSpan.FromSeconds(0.1)); + await Task.Delay(TimeSpan.FromSeconds(0.1), cts.Token); } _testOutputHelper.WriteLine($"Visibility={visibility}"); @@ -267,11 +288,14 @@ public async Task AdvanceVisibility() startTime = DateTime.UtcNow; while ((visibility = await _fixture.SqlServerFhirDataStore.StoreClient.MergeResourcesGetTransactionVisibilityAsync(cts.Token)) != tran3.TransactionId && (DateTime.UtcNow - startTime).TotalSeconds < 10) { - await Task.Delay(TimeSpan.FromSeconds(0.1)); + await Task.Delay(TimeSpan.FromSeconds(0.1), cts.Token); } _testOutputHelper.WriteLine($"Visibility={visibility}"); Assert.Equal(tran3.TransactionId, visibility); + + await cts.CancelAsync(); + await wdTask; } private ResourceWrapperFactory CreateResourceWrapperFactory() diff --git a/tools/EventsReader/Program.cs b/tools/EventsReader/Program.cs index d6dcf988d1..32cd8a3dd2 100644 --- a/tools/EventsReader/Program.cs +++ b/tools/EventsReader/Program.cs @@ -16,7 +16,7 @@ public static class Program { private static readonly string _connectionString = ConfigurationManager.ConnectionStrings["Database"].ConnectionString; private static SqlRetryService _sqlRetryService; - private static SqlStoreClient _store; + private static SqlStoreClient _store; private static string _parameterId = "Events.LastProcessedTransactionId"; public static void Main() From 656d4d662023e8f1ccc5605b60944320da70002d Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Tue, 16 Jul 2024 16:02:41 -0700 Subject: [PATCH 34/47] Fixes for subscriptionwatchdog --- global.json | 2 +- .../SubscriptionProcessorWatchdog.cs | 27 ++++++++++--------- .../Watchdogs/WatchdogsBackgroundService.cs | 6 ++++- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/global.json b/global.json index 789477d342..7cc48a4e22 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "8.0.204" + "version": "8.0.303" } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs index 8dc1f8e829..5e40f22de3 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs @@ -28,7 +28,6 @@ internal class SubscriptionProcessorWatchdog : Watchdog $"{Name}.{nameof(LastEventProcessedTransactionId)}"; - internal async Task StartAsync(CancellationToken cancellationToken, double? periodSec = null, double? leasePeriodSec = null, double? retentionPeriodDays = null) + public override double LeasePeriodSec { get; internal set; } = 20; + + public override bool AllowRebalance { get; internal set; } = true; + + public override double PeriodSec { get; internal set; } = 3; + + protected override async Task InitAdditionalParamsAsync() { - _cancellationToken = cancellationToken; await InitLastProcessedTransactionId(); - await StartAsync(true, periodSec ?? 3, leasePeriodSec ?? 20, cancellationToken); } - protected override async Task ExecuteAsync() + protected override async Task RunWorkAsync(CancellationToken cancellationToken) { _logger.LogInformation($"{Name}: starting..."); - var lastTranId = await GetLastTransactionId(); - var visibility = await _store.MergeResourcesGetTransactionVisibilityAsync(_cancellationToken); + var lastTranId = await GetLastTransactionId(cancellationToken); + var visibility = await _store.MergeResourcesGetTransactionVisibilityAsync(cancellationToken); _logger.LogInformation($"{Name}: last transaction={lastTranId} visibility={visibility}."); - var transactionsToProcess = await _store.GetTransactionsAsync(lastTranId, visibility, _cancellationToken); + var transactionsToProcess = await _store.GetTransactionsAsync(lastTranId, visibility, cancellationToken); _logger.LogDebug($"{Name}: found transactions={transactionsToProcess.Count}."); if (transactionsToProcess.Count == 0) @@ -88,7 +91,7 @@ protected override async Task ExecuteAsync() transactionsToQueue.Add(jobDefinition); } - await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken: _cancellationToken, definitions: transactionsToQueue.ToArray()); + await _queueClient.EnqueueAsync(QueueType.Subscriptions, cancellationToken: cancellationToken, definitions: transactionsToQueue.ToArray()); } await UpdateLastEventProcessedTransactionId(transactionsToProcess.Max(x => x.TransactionId)); @@ -96,16 +99,16 @@ protected override async Task ExecuteAsync() _logger.LogInformation($"{Name}: completed. transactions={transactionsToProcess.Count}"); } - private async Task GetLastTransactionId() + private async Task GetLastTransactionId(CancellationToken cancellationToken) { - return await GetLongParameterByIdAsync(LastEventProcessedTransactionId, _cancellationToken); + return await GetLongParameterByIdAsync(LastEventProcessedTransactionId, cancellationToken); } private async Task InitLastProcessedTransactionId() { using var cmd = new SqlCommand("INSERT INTO dbo.Parameters (Id, Bigint) SELECT @Id, @LastTranId"); cmd.Parameters.AddWithValue("@Id", LastEventProcessedTransactionId); - cmd.Parameters.AddWithValue("@LastTranId", await _store.MergeResourcesGetTransactionVisibilityAsync(_cancellationToken)); + cmd.Parameters.AddWithValue("@LastTranId", await _store.MergeResourcesGetTransactionVisibilityAsync(CancellationToken.None)); await cmd.ExecuteNonQueryAsync(_sqlRetryService, _logger, CancellationToken.None); } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs index 8d1ceae091..bc269ce08a 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs @@ -24,17 +24,20 @@ internal class WatchdogsBackgroundService : BackgroundService, INotificationHand private readonly CleanupEventLogWatchdog _cleanupEventLogWatchdog; private readonly IScoped _transactionWatchdog; private readonly InvisibleHistoryCleanupWatchdog _invisibleHistoryCleanupWatchdog; + private readonly SubscriptionProcessorWatchdog _subscriptionProcessorWatchdog; public WatchdogsBackgroundService( DefragWatchdog defragWatchdog, CleanupEventLogWatchdog cleanupEventLogWatchdog, IScopeProvider transactionWatchdog, - InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog) + InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog, + SubscriptionProcessorWatchdog subscriptionProcessorWatchdog) { _defragWatchdog = EnsureArg.IsNotNull(defragWatchdog, nameof(defragWatchdog)); _cleanupEventLogWatchdog = EnsureArg.IsNotNull(cleanupEventLogWatchdog, nameof(cleanupEventLogWatchdog)); _transactionWatchdog = EnsureArg.IsNotNull(transactionWatchdog, nameof(transactionWatchdog)).Invoke(); _invisibleHistoryCleanupWatchdog = EnsureArg.IsNotNull(invisibleHistoryCleanupWatchdog, nameof(invisibleHistoryCleanupWatchdog)); + _subscriptionProcessorWatchdog = EnsureArg.IsNotNull(subscriptionProcessorWatchdog, nameof(subscriptionProcessorWatchdog)); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -53,6 +56,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) _cleanupEventLogWatchdog.ExecuteAsync(continuationTokenSource.Token), _transactionWatchdog.Value.ExecuteAsync(continuationTokenSource.Token), _invisibleHistoryCleanupWatchdog.ExecuteAsync(continuationTokenSource.Token), + _subscriptionProcessorWatchdog.ExecuteAsync(continuationTokenSource.Token), }; await Task.WhenAny(tasks); From 09acb1e850b12a619d82459d4eee9597eee3feaf Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Tue, 16 Jul 2024 16:48:46 -0700 Subject: [PATCH 35/47] Aligns dotnet sdk version for build --- global.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/global.json b/global.json index 7cc48a4e22..789477d342 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "8.0.303" + "version": "8.0.204" } } From eb3b2bf00b7a601dd465a2eb9b9392e19e71dfa5 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Wed, 17 Jul 2024 09:32:46 -0700 Subject: [PATCH 36/47] Adds subscription to docker build --- build/docker/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 9f4d90b776..8971018350 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -49,6 +49,9 @@ COPY ./src/Microsoft.Health.Fhir.CosmosDb.Core/Microsoft.Health.Fhir.CosmosDb.Co COPY ./src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj \ ./src/Microsoft.Health.Fhir.CosmosDb.Initialization/Microsoft.Health.Fhir.CosmosDb.Initialization.csproj +COPY ./src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj \ + ./src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj + COPY ./src/Microsoft.Health.Fhir.${FHIR_VERSION}.Core/Microsoft.Health.Fhir.${FHIR_VERSION}.Core.csproj \ ./src/Microsoft.Health.Fhir.${FHIR_VERSION}.Core/Microsoft.Health.Fhir.${FHIR_VERSION}.Core.csproj From c9cb5666421bfcfa7ffac860a2216cf7b1d3ef91 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Thu, 25 Jul 2024 13:12:15 -0700 Subject: [PATCH 37/47] Adds SearchQueryInterpreter --- R4.slnf | 1 - .../Search/InMemory/ComparisonValueVisitor.cs | 105 ++++++++ .../Features/Search/InMemory/InMemoryIndex.cs | 51 ++++ .../Search/InMemory/SearchQueryInterpreter.cs | 228 ++++++++++++++++++ .../InMemory/SearchQueryInterperaterTests.cs | 110 +++++++++ ...ealth.Fhir.Shared.Core.UnitTests.projitems | 1 + 6 files changed, 495 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs create mode 100644 src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs create mode 100644 src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs diff --git a/R4.slnf b/R4.slnf index 7adecaed1e..bb8a55c269 100644 --- a/R4.slnf +++ b/R4.slnf @@ -29,7 +29,6 @@ "src\\Microsoft.Health.Fhir.Shared.Web\\Microsoft.Health.Fhir.Shared.Web.shproj", "src\\Microsoft.Health.Fhir.SqlServer.UnitTests\\Microsoft.Health.Fhir.SqlServer.UnitTests.csproj", "src\\Microsoft.Health.Fhir.SqlServer\\Microsoft.Health.Fhir.SqlServer.csproj", - "src\\Microsoft.Health.Fhir.Subscriptions\\Microsoft.Health.Fhir.Subscriptions.csproj", "src\\Microsoft.Health.Fhir.Tests.Common\\Microsoft.Health.Fhir.Tests.Common.csproj", "src\\Microsoft.Health.TaskManagement\\Microsoft.Health.TaskManagement.csproj", "src\\Microsoft.Health.TaskManagement.UnitTests\\Microsoft.Health.TaskManagement.UnitTests.csproj", diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs new file mode 100644 index 0000000000..86eefd7eb0 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs @@ -0,0 +1,105 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Health.Fhir.Core.Features.Search.Expressions; +using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; + +namespace Microsoft.Health.Fhir.Core.Features.Search.InMemory +{ + internal class ComparisonValueVisitor : ISearchValueVisitor + { + private readonly BinaryOperator _expressionBinaryOperator; + private readonly IComparable _second; + + private readonly List> _comparisonValues = new List>(); + + public ComparisonValueVisitor(BinaryOperator expressionBinaryOperator, IComparable second) + { + _expressionBinaryOperator = expressionBinaryOperator; + _second = second; + } + + public void Visit(CompositeSearchValue composite) + { + foreach (IReadOnlyList c in composite.Components) + { + foreach (ISearchValue inner in c) + { + inner.AcceptVisitor(this); + } + } + } + + public void Visit(DateTimeSearchValue dateTime) + { + AddComparison(_expressionBinaryOperator, dateTime.Start); + } + + public void Visit(NumberSearchValue number) + { + AddComparison(_expressionBinaryOperator, number.High); + } + + public void Visit(QuantitySearchValue quantity) + { + AddComparison(_expressionBinaryOperator, quantity.High); + } + + public void Visit(ReferenceSearchValue reference) + { + AddComparison(_expressionBinaryOperator, reference.ResourceId); + } + + public void Visit(StringSearchValue s) + { + AddComparison(_expressionBinaryOperator, s.String); + } + + public void Visit(TokenSearchValue token) + { + AddComparison(_expressionBinaryOperator, token.Text, token.System, token.Code); + } + + public void Visit(UriSearchValue uri) + { + AddComparison(_expressionBinaryOperator, uri.Uri); + } + + private void AddComparison(BinaryOperator binaryOperator, params IComparable[] first) + { + switch (binaryOperator) + { + case BinaryOperator.Equal: + _comparisonValues.Add(() => first.Any(x => x.CompareTo(_second) == 0)); + break; + case BinaryOperator.GreaterThan: + _comparisonValues.Add(() => first.Any(x => x.CompareTo(_second) > 0)); + break; + case BinaryOperator.LessThan: + _comparisonValues.Add(() => first.Any(x => x.CompareTo(_second) < 0)); + break; + case BinaryOperator.NotEqual: + _comparisonValues.Add(() => first.Any(x => x.CompareTo(_second) != 0)); + break; + case BinaryOperator.GreaterThanOrEqual: + _comparisonValues.Add(() => first.Any(x => x.CompareTo(_second) >= 0)); + break; + case BinaryOperator.LessThanOrEqual: + _comparisonValues.Add(() => first.Any(x => x.CompareTo(_second) <= 0)); + break; + default: + throw new ArgumentOutOfRangeException(nameof(binaryOperator)); + } + } + + public bool Compare() + { + return _comparisonValues.All(x => x.Invoke()); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs new file mode 100644 index 0000000000..00ba7f5d38 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs @@ -0,0 +1,51 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Collections.Concurrent; +using System.Collections.Generic; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Core.Features.Search.InMemory +{ + public class InMemoryIndex + { + private readonly ISearchIndexer _searchIndexer; + + public InMemoryIndex(ISearchIndexer searchIndexer) + { + Index = new ConcurrentDictionary)>>(); + _searchIndexer = searchIndexer; + } + + internal ConcurrentDictionary Index)>> Index + { + get; + } + + public void IndexResources(params ResourceElement[] resources) + { + foreach (var resource in resources) + { + var indexEntries = _searchIndexer.Extract(resource); + + Index.AddOrUpdate( + resource.InstanceType, + key => new List<(ResourceKey, IReadOnlyCollection)> { (ToResourceKey(resource), indexEntries) }, + (key, list) => + { + list.Add((ToResourceKey(resource), indexEntries)); + return list; + }); + } + } + + private static ResourceKey ToResourceKey(ResourceElement resource) + { + return new ResourceKey(resource.InstanceType, resource.Id, resource.VersionId); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs new file mode 100644 index 0000000000..91ee4efea8 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs @@ -0,0 +1,228 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using EnsureThat; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Expressions; +using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; + +using SearchPredicate = System.Func< + System.Collections.Generic.IEnumerable<(Microsoft.Health.Fhir.Core.Features.Persistence.ResourceKey Location, System.Collections.Generic.IReadOnlyCollection Index)>, + System.Collections.Generic.IEnumerable<(Microsoft.Health.Fhir.Core.Features.Persistence.ResourceKey Location, System.Collections.Generic.IReadOnlyCollection Index)>>; + +namespace Microsoft.Health.Fhir.Core.Features.Search.InMemory +{ + internal class SearchQueryInterpreter : IExpressionVisitorWithInitialContext + { + Context IExpressionVisitorWithInitialContext.InitialContext => default; + + public SearchPredicate VisitSearchParameter(SearchParameterExpression expression, Context context) + { + return VisitInnerWithContext(expression.Parameter.Name, expression.Expression, context); + } + + public SearchPredicate VisitBinary(BinaryExpression expression, Context context) + { + return VisitBinary( + context.ParameterName, + expression.BinaryOperator, + expression.Value); + } + + private static SearchPredicate VisitBinary(string fieldName, BinaryOperator op, object value) + { + SearchPredicate filter = input => + { + return input.Where(x => x.Index.Any(y => y.SearchParameter.Name == fieldName && + GetMappedValue(op, y.Value, (IComparable)value))); + }; + + return filter; + } + + private static bool GetMappedValue(BinaryOperator expressionBinaryOperator, ISearchValue first, IComparable second) + { + if (first == null || second == null) + { + return false; + } + + var comparisonVisitor = new ComparisonValueVisitor(expressionBinaryOperator, second); + first.AcceptVisitor(comparisonVisitor); + + return comparisonVisitor.Compare(); + } + + public SearchPredicate VisitChained(ChainedExpression expression, Context context) + { + throw new SearchOperationNotSupportedException("ChainedExpression is not supported."); + } + + public SearchPredicate VisitMissingField(MissingFieldExpression expression, Context context) + { + throw new NotImplementedException(); + } + + public SearchPredicate VisitMissingSearchParameter(MissingSearchParameterExpression expression, Context context) + { + throw new NotImplementedException(); + } + + public SearchPredicate VisitMultiary(MultiaryExpression expression, Context context) + { + SearchPredicate filter = input => + { + var results = expression.Expressions.Select(x => x.AcceptVisitor(this, context)) + .Aggregate((x, y) => + { + switch (expression.MultiaryOperation) + { + case MultiaryOperator.And: + return p => x(p).Intersect(y(p)); + case MultiaryOperator.Or: + return p => x(p).Union(y(p)); + default: + throw new NotImplementedException(); + } + }); + + return results(input); + }; + + return filter; + } + + public SearchPredicate VisitString(StringExpression expression, Context context) + { + StringComparison comparison = expression.IgnoreCase + ? StringComparison.OrdinalIgnoreCase + : StringComparison.Ordinal; + + SearchPredicate filter; + + if (context.ParameterName == "_type") + { + filter = input => input.Where(x => x.Location.ResourceType.Equals(expression.Value, comparison)); + } + else + { + switch (expression.StringOperator) + { + case StringOperator.StartsWith: + filter = input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == context.ParameterName && + CompareStringParameter(y, (a, b, c) => a.StartsWith(b, c)))); + break; + case StringOperator.Equals: + filter = input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == context.ParameterName && + CompareStringParameter(y, string.Equals))); + + break; + default: + throw new NotImplementedException(); + } + } + + bool CompareStringParameter(SearchIndexEntry y, Func compareFunc) + { + switch (y.SearchParameter.Type) + { + case ValueSets.SearchParamType.String: + return compareFunc(((StringSearchValue)y.Value).String, expression.Value, comparison); + + case ValueSets.SearchParamType.Token: + return compareFunc(((TokenSearchValue)y.Value).Code, expression.Value, comparison) || + compareFunc(((TokenSearchValue)y.Value).System, expression.Value, comparison); + default: + throw new NotImplementedException(); + } + } + + return filter; + } + + public SearchPredicate VisitCompartment(CompartmentSearchExpression expression, Context context) + { + throw new SearchOperationNotSupportedException("Compartment search is not supported."); + } + + public SearchPredicate VisitInclude(IncludeExpression expression, Context context) + { + throw new NotImplementedException(); + } + + private SearchPredicate VisitInnerWithContext(string parameterName, Expression expression, Context context, bool negate = false) + { + EnsureArg.IsNotNull(parameterName, nameof(parameterName)); + + var newContext = context.WithParameterName(parameterName); + + SearchPredicate filter = input => + { + if (expression != null) + { + return expression.AcceptVisitor(this, newContext)(input); + } + else + { + // :missing will end up here + throw new NotSupportedException("This query is not supported"); + } + }; + + if (negate) + { + SearchPredicate inner = filter; + filter = input => input.Except(inner(input)); + } + + return filter; + } + + public SearchPredicate VisitNotExpression(NotExpression expression, Context context) + { + throw new NotImplementedException(); + } + + public SearchPredicate VisitUnion(UnionExpression expression, Context context) + { + throw new NotImplementedException(); + } + + public SearchPredicate VisitSmartCompartment(SmartCompartmentSearchExpression expression, Context context) + { + throw new NotImplementedException(); + } + + public SearchPredicate VisitSortParameter(SortExpression expression, Context context) + { + throw new NotImplementedException(); + } + + public SearchPredicate VisitIn(InExpression expression, Context context) + { + throw new NotImplementedException(); + } + + /// + /// Context that is passed through the visit. + /// + internal struct Context + { + public string ParameterName { get; set; } + + [System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "Internal API")] + public Context WithParameterName(string paramName) + { + return new Context + { + ParameterName = paramName, + }; + } + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs new file mode 100644 index 0000000000..84858a4253 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs @@ -0,0 +1,110 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features.Context; +using Microsoft.Health.Fhir.Core.Features.Definition; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Converters; +using Microsoft.Health.Fhir.Core.Features.Search.Expressions.Parsers; +using Microsoft.Health.Fhir.Core.Features.Search.InMemory; +using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Tests.Common; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.InMemory +{ + public class SearchQueryInterperaterTests : IAsyncLifetime + { + private ExpressionParser _expressionParser; + private InMemoryIndex _memoryIndex; + private SearchQueryInterpreter _searchQueryInterperater; + + public Task DisposeAsync() + { + return Task.CompletedTask; + } + + public async Task InitializeAsync() + { + _searchQueryInterperater = new SearchQueryInterpreter(); + + var fixture = new SearchParameterFixtureData(); + var manager = await fixture.GetSearchDefinitionManagerAsync(); + + var fhirRequestContextAccessor = new FhirRequestContextAccessor(); + var supportedSearchParameterDefinitionManager = new SupportedSearchParameterDefinitionManager(manager); + var searchableSearchParameterDefinitionManager = new SearchableSearchParameterDefinitionManager(manager, fhirRequestContextAccessor); + var typedElementToSearchValueConverterManager = GetTypeConverterAsync().Result; + + var referenceParser = new ReferenceSearchValueParser(fhirRequestContextAccessor); + var referenceToElementResolver = new LightweightReferenceToElementResolver(referenceParser, ModelInfoProvider.Instance); + var modelInfoProvider = ModelInfoProvider.Instance; + var logger = Substitute.For>(); + + var searchIndexer = new TypedElementSearchIndexer(supportedSearchParameterDefinitionManager, typedElementToSearchValueConverterManager, referenceToElementResolver, modelInfoProvider, logger); + _expressionParser = new ExpressionParser(() => searchableSearchParameterDefinitionManager, new SearchParameterExpressionParser(referenceParser)); + _memoryIndex = new InMemoryIndex(searchIndexer); + + _memoryIndex.IndexResources(Samples.GetDefaultPatient(), Samples.GetDefaultObservation().UpdateId("example")); + } + + protected async Task GetTypeConverterAsync() + { + FhirTypedElementToSearchValueConverterManager fhirTypedElementToSearchValueConverterManager = await SearchParameterFixtureData.GetFhirTypedElementToSearchValueConverterManagerAsync(); + return fhirTypedElementToSearchValueConverterManager; + } + + [Fact] + public void GivenASearchQueryInterpreter_WhenSearchingByNameOnPatient_ThenCorrectResultsAreReturned() + { + var expression = _expressionParser.Parse(new[] { "Patient" }, "name", "Jim"); + + var evaluator = expression.AcceptVisitor(_searchQueryInterperater, default); + + var results = evaluator + .Invoke(_memoryIndex.Index.Values.SelectMany(x => x)) + .ToArray(); + + Assert.Single(results); + } + + [Fact] + public void GivenASearchQueryInterpreter_WhenSearchingByDobOnPatient_ThenCorrectResultsAreReturned() + { + var expression = _expressionParser.Parse(new[] { "Patient" }, "birthdate", "gt1950"); + + var evaluator = expression.AcceptVisitor(_searchQueryInterperater, default); + + var results = evaluator + .Invoke(_memoryIndex.Index.Values.SelectMany(x => x)) + .ToArray(); + + Assert.Single(results); + } + + [Fact] + public void GivenASearchQueryInterpreter_WhenSearchingByValueOnObservation_ThenCorrectResultsAreReturned() + { + var expression = _expressionParser.Parse(new[] { "Observation" }, "value-quantity", "lt70"); + + var evaluator = expression.AcceptVisitor(_searchQueryInterperater, default); + + var results = evaluator + .Invoke(_memoryIndex.Index.Values.SelectMany(x => x)) + .ToArray(); + + Assert.Single(results); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems index 163f14e19c..5e192cf24c 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems @@ -46,6 +46,7 @@ + From 6971bec839d25547128faa19430a2e8e4a281059 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Fri, 26 Jul 2024 11:30:53 -0700 Subject: [PATCH 38/47] Cleanup of SearchQueryInterpreter --- .../Search/InMemory/ComparisonValueVisitor.cs | 14 ++- .../Features/Search/InMemory/InMemoryIndex.cs | 7 +- .../Search/InMemory/SearchQueryInterpreter.cs | 99 ++++++++++++------- .../Properties/AssemblyInfo.cs | 1 + .../InMemory/SearchQueryInterperaterTests.cs | 14 +++ 5 files changed, 98 insertions(+), 37 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs index 86eefd7eb0..3537a16abf 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/ComparisonValueVisitor.cs @@ -6,6 +6,8 @@ using System; using System.Collections.Generic; using System.Linq; +using EnsureThat; +using Hl7.Fhir.ElementModel.Types; using Microsoft.Health.Fhir.Core.Features.Search.Expressions; using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; @@ -16,12 +18,12 @@ internal class ComparisonValueVisitor : ISearchValueVisitor private readonly BinaryOperator _expressionBinaryOperator; private readonly IComparable _second; - private readonly List> _comparisonValues = new List>(); + private readonly List> _comparisonValues = []; public ComparisonValueVisitor(BinaryOperator expressionBinaryOperator, IComparable second) { _expressionBinaryOperator = expressionBinaryOperator; - _second = second; + _second = EnsureArg.IsNotNull(second, nameof(second)); } public void Visit(CompositeSearchValue composite) @@ -37,41 +39,49 @@ public void Visit(CompositeSearchValue composite) public void Visit(DateTimeSearchValue dateTime) { + EnsureArg.IsNotNull(dateTime, nameof(dateTime)); AddComparison(_expressionBinaryOperator, dateTime.Start); } public void Visit(NumberSearchValue number) { + EnsureArg.IsNotNull(number, nameof(number)); AddComparison(_expressionBinaryOperator, number.High); } public void Visit(QuantitySearchValue quantity) { + EnsureArg.IsNotNull(quantity, nameof(quantity)); AddComparison(_expressionBinaryOperator, quantity.High); } public void Visit(ReferenceSearchValue reference) { + EnsureArg.IsNotNull(reference, nameof(reference)); AddComparison(_expressionBinaryOperator, reference.ResourceId); } public void Visit(StringSearchValue s) { + EnsureArg.IsNotNull(s, nameof(s)); AddComparison(_expressionBinaryOperator, s.String); } public void Visit(TokenSearchValue token) { + EnsureArg.IsNotNull(token, nameof(token)); AddComparison(_expressionBinaryOperator, token.Text, token.System, token.Code); } public void Visit(UriSearchValue uri) { + EnsureArg.IsNotNull(uri, nameof(uri)); AddComparison(_expressionBinaryOperator, uri.Uri); } private void AddComparison(BinaryOperator binaryOperator, params IComparable[] first) { + EnsureArg.IsNotNull(first, nameof(first)); switch (binaryOperator) { case BinaryOperator.Equal: diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs index 00ba7f5d38..f489bfd821 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/InMemoryIndex.cs @@ -5,6 +5,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; +using EnsureThat; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Models; @@ -17,8 +18,8 @@ public class InMemoryIndex public InMemoryIndex(ISearchIndexer searchIndexer) { + _searchIndexer = EnsureArg.IsNotNull(searchIndexer, nameof(searchIndexer)); Index = new ConcurrentDictionary)>>(); - _searchIndexer = searchIndexer; } internal ConcurrentDictionary Index)>> Index @@ -28,6 +29,8 @@ public InMemoryIndex(ISearchIndexer searchIndexer) public void IndexResources(params ResourceElement[] resources) { + EnsureArg.IsNotNull(resources, nameof(resources)); + foreach (var resource in resources) { var indexEntries = _searchIndexer.Extract(resource); @@ -45,6 +48,8 @@ public void IndexResources(params ResourceElement[] resources) private static ResourceKey ToResourceKey(ResourceElement resource) { + EnsureArg.IsNotNull(resource, nameof(resource)); + return new ResourceKey(resource.InstanceType, resource.Id, resource.VersionId); } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs index 91ee4efea8..da0eebcb90 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/InMemory/SearchQueryInterpreter.cs @@ -7,46 +7,48 @@ using System.Collections.Generic; using System.Linq; using EnsureThat; +using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search; using Microsoft.Health.Fhir.Core.Features.Search.Expressions; using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; -using SearchPredicate = System.Func< - System.Collections.Generic.IEnumerable<(Microsoft.Health.Fhir.Core.Features.Persistence.ResourceKey Location, System.Collections.Generic.IReadOnlyCollection Index)>, - System.Collections.Generic.IEnumerable<(Microsoft.Health.Fhir.Core.Features.Persistence.ResourceKey Location, System.Collections.Generic.IReadOnlyCollection Index)>>; - namespace Microsoft.Health.Fhir.Core.Features.Search.InMemory { + public delegate IEnumerable<(ResourceKey Location, IReadOnlyCollection Index)> SearchPredicate(IEnumerable<(ResourceKey Location, IReadOnlyCollection Index)> input); + internal class SearchQueryInterpreter : IExpressionVisitorWithInitialContext { Context IExpressionVisitorWithInitialContext.InitialContext => default; public SearchPredicate VisitSearchParameter(SearchParameterExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + return VisitInnerWithContext(expression.Parameter.Name, expression.Expression, context); } public SearchPredicate VisitBinary(BinaryExpression expression, Context context) { - return VisitBinary( - context.ParameterName, - expression.BinaryOperator, - expression.Value); + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + + return VisitBinary(context.ParameterName, expression.BinaryOperator, expression.Value); } private static SearchPredicate VisitBinary(string fieldName, BinaryOperator op, object value) { - SearchPredicate filter = input => - { - return input.Where(x => x.Index.Any(y => y.SearchParameter.Name == fieldName && - GetMappedValue(op, y.Value, (IComparable)value))); - }; + EnsureArg.IsNotNull(fieldName, nameof(fieldName)); + EnsureArg.IsNotNull(value, nameof(value)); - return filter; + return input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == fieldName && GetMappedValue(op, y.Value, (IComparable)value))); } private static bool GetMappedValue(BinaryOperator expressionBinaryOperator, ISearchValue first, IComparable second) { + EnsureArg.IsNotNull(first, nameof(first)); + EnsureArg.IsNotNull(second, nameof(second)); + if (first == null || second == null) { return false; @@ -60,24 +62,35 @@ private static bool GetMappedValue(BinaryOperator expressionBinaryOperator, ISea public SearchPredicate VisitChained(ChainedExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new SearchOperationNotSupportedException("ChainedExpression is not supported."); } public SearchPredicate VisitMissingField(MissingFieldExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new NotImplementedException(); } public SearchPredicate VisitMissingSearchParameter(MissingSearchParameterExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new NotImplementedException(); } public SearchPredicate VisitMultiary(MultiaryExpression expression, Context context) { - SearchPredicate filter = input => - { - var results = expression.Expressions.Select(x => x.AcceptVisitor(this, context)) + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(expression.Expressions, nameof(expression.Expressions)); + EnsureArg.IsNotNull(context, nameof(context)); + + return expression.Expressions.Select(x => x.AcceptVisitor(this, context)) .Aggregate((x, y) => { switch (expression.MultiaryOperation) @@ -90,38 +103,32 @@ public SearchPredicate VisitMultiary(MultiaryExpression expression, Context cont throw new NotImplementedException(); } }); - - return results(input); - }; - - return filter; } public SearchPredicate VisitString(StringExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + StringComparison comparison = expression.IgnoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; - SearchPredicate filter; - if (context.ParameterName == "_type") { - filter = input => input.Where(x => x.Location.ResourceType.Equals(expression.Value, comparison)); + return input => input.Where(x => x.Location.ResourceType.Equals(expression.Value, comparison)); } else { switch (expression.StringOperator) { case StringOperator.StartsWith: - filter = input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == context.ParameterName && - CompareStringParameter(y, (a, b, c) => a.StartsWith(b, c)))); - break; + return input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == context.ParameterName && + CompareStringParameter(y, (a, b, c) => a.StartsWith(b, c)))); case StringOperator.Equals: - filter = input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == context.ParameterName && - CompareStringParameter(y, string.Equals))); + return input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == context.ParameterName && + CompareStringParameter(y, string.Equals))); - break; default: throw new NotImplementedException(); } @@ -129,6 +136,8 @@ public SearchPredicate VisitString(StringExpression expression, Context context) bool CompareStringParameter(SearchIndexEntry y, Func compareFunc) { + EnsureArg.IsNotNull(y, nameof(y)); + switch (y.SearchParameter.Type) { case ValueSets.SearchParamType.String: @@ -141,23 +150,29 @@ bool CompareStringParameter(SearchIndexEntry y, Func(context, nameof(context)); + throw new SearchOperationNotSupportedException("Compartment search is not supported."); } public SearchPredicate VisitInclude(IncludeExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new NotImplementedException(); } private SearchPredicate VisitInnerWithContext(string parameterName, Expression expression, Context context, bool negate = false) { EnsureArg.IsNotNull(parameterName, nameof(parameterName)); + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); var newContext = context.WithParameterName(parameterName); @@ -185,27 +200,43 @@ private SearchPredicate VisitInnerWithContext(string parameterName, Expression e public SearchPredicate VisitNotExpression(NotExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new NotImplementedException(); } public SearchPredicate VisitUnion(UnionExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new NotImplementedException(); } public SearchPredicate VisitSmartCompartment(SmartCompartmentSearchExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new NotImplementedException(); } public SearchPredicate VisitSortParameter(SortExpression expression, Context context) { + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + throw new NotImplementedException(); } public SearchPredicate VisitIn(InExpression expression, Context context) { - throw new NotImplementedException(); + EnsureArg.IsNotNull(expression, nameof(expression)); + EnsureArg.IsNotNull(context, nameof(context)); + + return input => input.Where(x => x.Index.Any(y => y.SearchParameter.Name == context.ParameterName && + expression.Values.Contains((T)y.Value))); } /// diff --git a/src/Microsoft.Health.Fhir.Core/Properties/AssemblyInfo.cs b/src/Microsoft.Health.Fhir.Core/Properties/AssemblyInfo.cs index 115d142d47..aa5748585b 100644 --- a/src/Microsoft.Health.Fhir.Core/Properties/AssemblyInfo.cs +++ b/src/Microsoft.Health.Fhir.Core/Properties/AssemblyInfo.cs @@ -49,6 +49,7 @@ [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.R5.Tests.E2E")] [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.Stu3.Tests.E2E")] +[assembly: InternalsVisibleTo("Microsoft.Health.Fhir.Subscriptions")] [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.R4.ResourceParser")] [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.SqlServer.UnitTests")] diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs index 84858a4253..2d96d2494a 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs @@ -93,6 +93,20 @@ public void GivenASearchQueryInterpreter_WhenSearchingByDobOnPatient_ThenCorrect Assert.Single(results); } + [Fact] + public void GivenASearchQueryInterpreter_WhenSearchingByDobOnPatientWithRange_ThenCorrectResultsAreReturned() + { + var expression = _expressionParser.Parse(new[] { "Patient" }, "birthdate", "1974"); + + var evaluator = expression.AcceptVisitor(_searchQueryInterperater, default); + + var results = evaluator + .Invoke(_memoryIndex.Index.Values.SelectMany(x => x)) + .ToArray(); + + Assert.Single(results); + } + [Fact] public void GivenASearchQueryInterpreter_WhenSearchingByValueOnObservation_ThenCorrectResultsAreReturned() { From b83c201c5a7ac39356f08c4d1789a8da0755616b Mon Sep 17 00:00:00 2001 From: feordin Date: Wed, 21 Aug 2024 17:31:29 -0700 Subject: [PATCH 39/47] Remove --- .../Features/Storage/SqlRetry/SqlRetryService.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs index a544c7b915..272476e5f0 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlRetry/SqlRetryService.cs @@ -435,7 +435,7 @@ public ReplicaHandler() { } - public async Task GetConnection(ISqlConnectionBuilder sqlConnectionBuilder, bool isReadOnly, ILogger logger, CancellationToken cancel) + public async Task GetConnection(ISqlConnectionBuilder sqlConnectionBuilder, bool isReadOnly, ILogger logger, CancellationToken cancel) { SqlConnection conn; var sw = Stopwatch.StartNew(); @@ -474,6 +474,7 @@ public async Task GetConnection(ISqlConnectionBuilder sq // Connection is never opened by the _sqlConnectionBuilder but RetryLogicProvider is set to the old, deprecated retry implementation. According to the .NET spec, RetryLogicProvider // must be set before opening connection to take effect. Therefore we must reset it to null here before opening the connection. conn.RetryLogicProvider = null; // To remove this line _sqlConnectionBuilder in healthcare-shared-components must be modified. + logger.LogInformation($"Retrieved {isReadOnlyConnection}connection to the database in {sw.Elapsed.TotalSeconds} seconds."); sw = Stopwatch.StartNew(); From d0eb1dd52e837d79aa6c5a54310b20e654b74310 Mon Sep 17 00:00:00 2001 From: Adithi Ponakampalli <120080886+aponakampalli@users.noreply.github.com> Date: Thu, 22 Aug 2024 00:59:02 -0400 Subject: [PATCH 40/47] In Memory Search Filter For Subscriptions (#3971) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Improve fhirtimer * Subscription infra * Fixes wiring up of Transaction Watchdog => Orchestrator * Adding code for writing to storage * Allow resourceKey to deserialize * Implement basic subscription filtering * Implements Channel Interface * Add example subscription * DataLakeChannel. * Changes in DataLakeChannel and the project config. * Load from DB * EventGrid WIP * Improve fhirtimer * Fixes for subscriptionwatchdog * Aligns dotnet sdk version for build * Adds subscription to docker build * Adds SearchQueryInterpreter * set up notification manager * Cleanup of SearchQueryInterpreter * integrate in memory search with filter search for subscriptions * resolve notification manager build errors * set up notification manager * integrate in memory search with filter search for subscriptions * resolve notification manager build errors * remove build error in webhook channel * rename webhook channel name * set up notification manager * integrate in memory search with filter search for subscriptions * resolve notification manager build errors * set up notification manager * resolve notification manager build errors * remove build error in webhook channel * rename webhook channel name * in memory search test set up * subscription orchestrator job test checks for id * subscription orchestator job only sends filtered resources instead of all resources * new test for patient name filter * add a failing test * rename tests for subscription orchestrator job * remove web hook code * move test to r4 folder * Improve fhirtimer * Subscription infra * Fixes wiring up of Transaction Watchdog => Orchestrator * Adding code for writing to storage * Allow resourceKey to deserialize * Implement basic subscription filtering * Implements Channel Interface * Add example subscription * DataLakeChannel. * Changes in DataLakeChannel and the project config. * Load from DB * EventGrid WIP * Improve fhirtimer * Fixes for subscriptionwatchdog * Aligns dotnet sdk version for build * Adds subscription to docker build * Adds SearchQueryInterpreter * Cleanup of SearchQueryInterpreter * clean up merge * fix merge conflict params in orchestra job --------- Co-authored-by: Brendan Kowitz Co-authored-by: Fernando Henrique Inocêncio Borba Ferreira --- .../SubscriptionsOrchestratorJobTests.cs | 238 ++++++++++++++++++ ...ealth.Fhir.Shared.Core.UnitTests.projitems | 1 + ...oft.Health.Fhir.Subscriptions.Tests.csproj | 2 + .../AssemblyInfo.cs | 4 + .../SubscriptionsOrchestratorJob.cs | 41 +-- .../Microsoft.Health.Fhir.Tests.Common.csproj | 4 + .../R4/SubscriptionForEncounter.json | 50 ++++ ...ptionForObservationReferenceToPatient.json | 50 ++++ .../TestFiles/R4/SubscriptionForPatient.json | 50 ++++ .../R4/SubscriptionForPatientName.json | 50 ++++ 10 files changed, 470 insertions(+), 20 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs create mode 100644 src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForEncounter.json create mode 100644 src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForObservationReferenceToPatient.json create mode 100644 src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForPatient.json create mode 100644 src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForPatientName.json diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs new file mode 100644 index 0000000000..0974e43880 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs @@ -0,0 +1,238 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Hl7.Fhir.Serialization; +using Hl7.Fhir.Utility; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features.Context; +using Microsoft.Health.Fhir.Core.Features.Definition; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Routing; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.Access; +using Microsoft.Health.Fhir.Core.Features.Search.Converters; +using Microsoft.Health.Fhir.Core.Features.Search.Expressions.Parsers; +using Microsoft.Health.Fhir.Core.Features.Search.InMemory; +using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Subscriptions.Models; +using Microsoft.Health.Fhir.Subscriptions.Operations; +using Microsoft.Health.Fhir.Subscriptions.Persistence; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.JobManagement; +using Newtonsoft.Json; +using NSubstitute; +using NSubstitute.ReceivedExtensions; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.InMemory +{ + public class SubscriptionsOrchestratorJobTests : IAsyncLifetime + { + private ISearchIndexer _searchIndexer; + private IQueueClient _mockQueueClient = Substitute.For(); + private ITransactionDataStore _transactionDataStore = Substitute.For(); + private ISearchOptionsFactory _searchOptionsFactory; + private IQueryStringParser _queryStringParser; + private ISubscriptionManager _subscriptionManager = Substitute.For(); + private IResourceDeserializer _resourceDeserializer; + private IExpressionParser _expressionParser; + + public Task DisposeAsync() + { + return Task.CompletedTask; + } + + public async Task InitializeAsync() + { + var searchQueryInterpreter = new SearchQueryInterpreter(); + + var fixture = new SearchParameterFixtureData(); + var manager = await fixture.GetSearchDefinitionManagerAsync(); + + var fhirRequestContextAccessor = new FhirRequestContextAccessor(); + var supportedSearchParameterDefinitionManager = new SupportedSearchParameterDefinitionManager(manager); + var searchableSearchParameterDefinitionManager = new SearchableSearchParameterDefinitionManager(manager, fhirRequestContextAccessor); + var typedElementToSearchValueConverterManager = GetTypeConverterAsync().Result; + + var referenceParser = new ReferenceSearchValueParser(fhirRequestContextAccessor); + var referenceToElementResolver = new LightweightReferenceToElementResolver(referenceParser, ModelInfoProvider.Instance); + var modelInfoProvider = ModelInfoProvider.Instance; + var logger = Substitute.For>(); + + _searchIndexer = new TypedElementSearchIndexer(supportedSearchParameterDefinitionManager, typedElementToSearchValueConverterManager, referenceToElementResolver, modelInfoProvider, logger); + + _transactionDataStore.GetResourcesByTransactionIdAsync(Arg.Any(), Arg.Any()).Returns(x => + { + var resourceWrappers = new List(); + var allResources = new List + { + Samples.GetJsonSample("Patient").UpdateId("1"), + Samples.GetJsonSample("Patient-f001").UpdateId("2"), + Samples.GetJsonSample("Observation-For-Patient-f001").UpdateId("3"), + Samples.GetJsonSample("Practitioner").UpdateId("4"), + }; + + foreach (var resource in allResources) + { + var rawResourceFactory = new RawResourceFactory(new FhirJsonSerializer()); + var resourceWrapper = new ResourceWrapper(resource, rawResourceFactory.Create(resource, keepMeta: true), new ResourceRequest(HttpMethod.Post, "http://fhir"), false, null, null, null); + resourceWrappers.Add(resourceWrapper); + } + + return resourceWrappers.AsReadOnly(); + }); + _queryStringParser = new TestQueryStringParser(); + var options = new OptionsWrapper(new CoreFeatureConfiguration()); + _expressionParser = new ExpressionParser(() => searchableSearchParameterDefinitionManager, new SearchParameterExpressionParser(referenceParser)); + + _searchOptionsFactory = new SearchOptionsFactory( + _expressionParser, () => manager, options, fhirRequestContextAccessor, Substitute.For(), new ExpressionAccessControl(fhirRequestContextAccessor), NullLogger.Instance); + + var fhirJsonParser = new FhirJsonParser(); + _resourceDeserializer = Deserializers.ResourceDeserializer; + } + + protected async Task GetTypeConverterAsync() + { + FhirTypedElementToSearchValueConverterManager fhirTypedElementToSearchValueConverterManager = await SearchParameterFixtureData.GetFhirTypedElementToSearchValueConverterManagerAsync(); + return fhirTypedElementToSearchValueConverterManager; + } + + private bool ContainsResourcesWithIds(string[] definitions, string[] expectedIds) + { + var deserializedDefinitions = definitions.Select(r => JsonConvert.DeserializeObject(r)).ToArray(); + var resources = deserializedDefinitions.SelectMany(x => x.ResourceReferences).ToArray(); + return resources.Length == expectedIds.Length && expectedIds.All(id => resources.Any(x => x.Id == id)); + } + + [SkippableFact] + public async Task GivenASubscriptionOrchestrator_WhenPatientResourceRecieved_ThenCorrectResourcesQueued() + { + Skip.If(ModelInfoProvider.Version != FhirSpecification.R4); + _subscriptionManager.GetActiveSubscriptionsAsync(Arg.Any()).Returns(x => + { + var subscriptionInfo = SubscriptionManager.ConvertToInfo(Samples.GetJsonSample("SubscriptionForPatient")); + var subscriptionInfoList = new List + { + subscriptionInfo, + }; + return subscriptionInfoList.AsReadOnly(); + }); + + var orchestrator = new SubscriptionsOrchestratorJob(_mockQueueClient, _transactionDataStore, _searchOptionsFactory, _queryStringParser, _subscriptionManager, _resourceDeserializer, _searchIndexer); + var definition = new SubscriptionJobDefinition(JobType.SubscriptionsOrchestrator) { TransactionId = 1, TypeId = 1 }; + var jobInfo = new JobInfo() { Status = JobStatus.Created, Definition = JsonConvert.SerializeObject(definition), GroupId = 1 }; + await orchestrator.ExecuteAsync(jobInfo, default); + + var expectedIds = new[] { "1", "2" }; + await _mockQueueClient.Received().EnqueueAsync( + (byte)QueueType.Subscriptions, + Arg.Is(resources => ContainsResourcesWithIds(resources, expectedIds)), + 1, + Arg.Any(), + Arg.Any(), + Arg.Any()); + } + + [SkippableFact] + public async Task GivenANameFilterSubscription_WhenResourcesPosted_ThenCorrectResourcesQueued() + { + Skip.If(ModelInfoProvider.Version != FhirSpecification.R4); + _subscriptionManager.GetActiveSubscriptionsAsync(Arg.Any()).Returns(x => + { + var subscriptionInfo = SubscriptionManager.ConvertToInfo(Samples.GetJsonSample("SubscriptionForPatientName")); + var subscriptionInfoList = new List + { + subscriptionInfo, + }; + return subscriptionInfoList.AsReadOnly(); + }); + + var orchestrator = new SubscriptionsOrchestratorJob(_mockQueueClient, _transactionDataStore, _searchOptionsFactory, _queryStringParser, _subscriptionManager, _resourceDeserializer, _searchIndexer); + var definition = new SubscriptionJobDefinition(JobType.SubscriptionsOrchestrator) { TransactionId = 1, TypeId = 1 }; + var jobInfo = new JobInfo() { Status = JobStatus.Created, Definition = JsonConvert.SerializeObject(definition), GroupId = 1 }; + await orchestrator.ExecuteAsync(jobInfo, default); + + var expectedIds = new[] { "1"}; + await _mockQueueClient.Received().EnqueueAsync( + (byte)QueueType.Subscriptions, + Arg.Is(resources => ContainsResourcesWithIds(resources, expectedIds)), + 1, + Arg.Any(), + Arg.Any(), + Arg.Any()); + } + + [SkippableFact] + public async Task GivenAReferenceFilterSubscription_WhenResourcesPosted_ThenCorrectResourcesQueued() + { + Skip.If(ModelInfoProvider.Version != FhirSpecification.R4); + _subscriptionManager.GetActiveSubscriptionsAsync(Arg.Any()).Returns(x => + { + var subscriptionInfo = SubscriptionManager.ConvertToInfo(Samples.GetJsonSample("SubscriptionForObservationReferenceToPatient")); + var subscriptionInfoList = new List + { + subscriptionInfo, + }; + return subscriptionInfoList.AsReadOnly(); + }); + + var orchestrator = new SubscriptionsOrchestratorJob(_mockQueueClient, _transactionDataStore, _searchOptionsFactory, _queryStringParser, _subscriptionManager, _resourceDeserializer, _searchIndexer); + var definition = new SubscriptionJobDefinition(JobType.SubscriptionsOrchestrator) { TransactionId = 1, TypeId = 1 }; + var jobInfo = new JobInfo() { Status = JobStatus.Created, Definition = JsonConvert.SerializeObject(definition), GroupId = 1 }; + await orchestrator.ExecuteAsync(jobInfo, default); + + var expectedIds = new[] { "3" }; + await _mockQueueClient.Received().EnqueueAsync( + (byte)QueueType.Subscriptions, + Arg.Is(resources => ContainsResourcesWithIds(resources, expectedIds)), + 1, + Arg.Any(), + Arg.Any(), + Arg.Any()); + } + + [SkippableFact] + public async Task GivenEncounterFilterSubscription_WhenNonEncounterResourcesPosted_ThenNoResourcesQueued() + { + Skip.If(ModelInfoProvider.Version != FhirSpecification.R4); + _subscriptionManager.GetActiveSubscriptionsAsync(Arg.Any()).Returns(x => + { + var subscriptionInfo = SubscriptionManager.ConvertToInfo(Samples.GetJsonSample("SubscriptionForEncounter")); + var subscriptionInfoList = new List + { + subscriptionInfo, + }; + return subscriptionInfoList.AsReadOnly(); + }); + + var orchestrator = new SubscriptionsOrchestratorJob(_mockQueueClient, _transactionDataStore, _searchOptionsFactory, _queryStringParser, _subscriptionManager, _resourceDeserializer, _searchIndexer); + var definition = new SubscriptionJobDefinition(JobType.SubscriptionsOrchestrator) { TransactionId = 1, TypeId = 1 }; + var jobInfo = new JobInfo() { Status = JobStatus.Created, Definition = JsonConvert.SerializeObject(definition), GroupId = 1 }; + await orchestrator.ExecuteAsync(jobInfo, default); + + await _mockQueueClient.DidNotReceive().EnqueueAsync( + (byte)QueueType.Subscriptions, + Arg.Any(), + 1, + Arg.Any(), + Arg.Any(), + Arg.Any()); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems index 5e192cf24c..8b8de99b38 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems @@ -47,6 +47,7 @@ + diff --git a/src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj index f2ca89213d..814d4639d3 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj +++ b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj @@ -21,6 +21,8 @@ + + diff --git a/src/Microsoft.Health.Fhir.Subscriptions/AssemblyInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/AssemblyInfo.cs index 04b1e9fede..99aa9de280 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/AssemblyInfo.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/AssemblyInfo.cs @@ -7,5 +7,9 @@ using System.Runtime.CompilerServices; [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.Subscriptions.Tests")] +[assembly: InternalsVisibleTo("Microsoft.Health.Fhir.R4.Core.UnitTests")] +[assembly: InternalsVisibleTo("Microsoft.Health.Fhir.R4B.Core.UnitTests")] +[assembly: InternalsVisibleTo("Microsoft.Health.Fhir.R5.Core.UnitTests")] +[assembly: InternalsVisibleTo("Microsoft.Health.Fhir.Stu3.Core.UnitTests")] [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] [assembly: NeutralResourcesLanguage("en-us")] diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs index acca62e009..1dd5de4d92 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs @@ -15,6 +15,7 @@ using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Routing; using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Search.InMemory; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Subscriptions.Models; using Microsoft.Health.Fhir.Subscriptions.Persistence; @@ -27,17 +28,21 @@ public class SubscriptionsOrchestratorJob : IJob { private readonly IQueueClient _queueClient; private readonly ITransactionDataStore _transactionDataStore; - private readonly ISearchService _searchService; + private readonly ISearchOptionsFactory _searchOptionsFactory; private readonly IQueryStringParser _queryStringParser; private readonly ISubscriptionManager _subscriptionManager; + private readonly IResourceDeserializer _resourceDeserializer; + private readonly ISearchIndexer _searchIndexer; private const string OperationCompleted = "Completed"; public SubscriptionsOrchestratorJob( IQueueClient queueClient, ITransactionDataStore transactionDataStore, - ISearchService searchService, + ISearchOptionsFactory searchOptionsFactory, IQueryStringParser queryStringParser, - ISubscriptionManager subscriptionManager) + ISubscriptionManager subscriptionManager, + IResourceDeserializer resourceDeserializer, + ISearchIndexer searchIndexer) { EnsureArg.IsNotNull(queueClient, nameof(queueClient)); EnsureArg.IsNotNull(transactionDataStore, nameof(transactionDataStore)); @@ -45,9 +50,11 @@ public SubscriptionsOrchestratorJob( _queueClient = queueClient; _transactionDataStore = transactionDataStore; - _searchService = searchService; + _searchOptionsFactory = searchOptionsFactory; _queryStringParser = queryStringParser; _subscriptionManager = subscriptionManager; + _resourceDeserializer = resourceDeserializer; + _searchIndexer = searchIndexer; } public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancellationToken) @@ -83,23 +90,17 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel .ToList(); } - var limitIds = string.Join(",", resourceKeys.Select(x => x.Id)); - var idParam = query.Where(x => x.Item1 == KnownQueryParameterNames.Id).FirstOrDefault(); - if (idParam != null) + var searchOptions = _searchOptionsFactory.Create(criteriaSegments[0], query); + var searchInterpreter = new SearchQueryInterpreter(); + var memoryIndex = new InMemoryIndex(_searchIndexer); + memoryIndex.IndexResources(resources.Select(x => _resourceDeserializer.Deserialize(x)).ToArray()); + var expression = searchOptions.Expression; + var evaluator = expression.AcceptVisitor(searchInterpreter, default); + if (memoryIndex.Index.TryGetValue(criteriaSegments[0], out List<(ResourceKey Location, IReadOnlyCollection Index)> value)) { - query.Remove(idParam); - limitIds += "," + idParam.Item2; + var results = evaluator.Invoke(value).ToArray(); + channelResources.AddRange(results.Select(x => x.Location)); } - - query.Add(new Tuple(KnownQueryParameterNames.Id, limitIds)); - - var results = await _searchService.SearchAsync(criteriaSegments[0], new ReadOnlyCollection>(query), cancellationToken, true, ResourceVersionType.Latest, onlyIds: true); - - channelResources.AddRange( - results.Results - .Where(x => x.SearchEntryMode == ValueSets.SearchEntryMode.Match - || x.SearchEntryMode == ValueSets.SearchEntryMode.Include) - .Select(x => x.Resource.ToResourceKey())); } else { @@ -111,7 +112,7 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel continue; } - var chunk = resourceKeys + var chunk = channelResources .Chunk(sub.Channel.MaxCount); foreach (var batch in chunk) diff --git a/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj b/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj index 9dc53d5c30..31069ced39 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj +++ b/src/Microsoft.Health.Fhir.Tests.Common/Microsoft.Health.Fhir.Tests.Common.csproj @@ -37,6 +37,10 @@ + + + + diff --git a/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForEncounter.json b/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForEncounter.json new file mode 100644 index 0000000000..13c8e56135 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForEncounter.json @@ -0,0 +1,50 @@ +{ + "resourceType": "Subscription", + "id": "example-backport-storage-patient", + "meta" : { + "profile": ["http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"] + }, + "status": "requested", + "end": "2031-01-01T12:00:00", + "reason": "Test subscription based on transactions, filtered by Encounter", + "criteria" : "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions", + "_criteria": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-filter-criteria", + "valueString": "Encounter" + } + ] + }, + "channel": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-heartbeat-period", + "valueInteger": 120 + } + ], + "type" : "rest-hook", + "_type" : { + "extension" : [ + { + "url" : "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type", + "valueCoding" : { + "system" : "http://azurehealthcareapis.com/data-extentions/subscription-channel-type", + "code" : "azure-storage", + "display" : "Azure Blob Storage" + } + } + ] + }, + "endpoint": "sync-patient", + "payload": "application/fhir+json", + "_payload": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content", + "valueCode": "full-resource" + } + ] + } + } + } diff --git a/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForObservationReferenceToPatient.json b/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForObservationReferenceToPatient.json new file mode 100644 index 0000000000..ac56da96f6 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForObservationReferenceToPatient.json @@ -0,0 +1,50 @@ +{ + "resourceType": "Subscription", + "id": "example-backport-storage-patient", + "meta" : { + "profile": ["http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"] + }, + "status": "requested", + "end": "2031-01-01T12:00:00", + "reason": "Test subscription based on transactions, filtered by Observation with reference to specific patient", + "criteria" : "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions", + "_criteria": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-filter-criteria", + "valueString": "Observation?reference=Patient/f001" + } + ] + }, + "channel": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-heartbeat-period", + "valueInteger": 120 + } + ], + "type" : "rest-hook", + "_type" : { + "extension" : [ + { + "url" : "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type", + "valueCoding" : { + "system" : "http://azurehealthcareapis.com/data-extentions/subscription-channel-type", + "code" : "azure-storage", + "display" : "Azure Blob Storage" + } + } + ] + }, + "endpoint": "sync-patient", + "payload": "application/fhir+json", + "_payload": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content", + "valueCode": "full-resource" + } + ] + } + } + } diff --git a/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForPatient.json b/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForPatient.json new file mode 100644 index 0000000000..37c5ab8278 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForPatient.json @@ -0,0 +1,50 @@ +{ + "resourceType": "Subscription", + "id": "example-backport-storage-patient", + "meta" : { + "profile": ["http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"] + }, + "status": "requested", + "end": "2031-01-01T12:00:00", + "reason": "Test subscription based on transactions, filtered by Patient", + "criteria" : "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions", + "_criteria": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-filter-criteria", + "valueString": "Patient" + } + ] + }, + "channel": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-heartbeat-period", + "valueInteger": 120 + } + ], + "type" : "rest-hook", + "_type" : { + "extension" : [ + { + "url" : "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type", + "valueCoding" : { + "system" : "http://azurehealthcareapis.com/data-extentions/subscription-channel-type", + "code" : "azure-storage", + "display" : "Azure Blob Storage" + } + } + ] + }, + "endpoint": "sync-patient", + "payload": "application/fhir+json", + "_payload": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content", + "valueCode": "full-resource" + } + ] + } + } + } diff --git a/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForPatientName.json b/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForPatientName.json new file mode 100644 index 0000000000..730d8c7a87 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Tests.Common/TestFiles/R4/SubscriptionForPatientName.json @@ -0,0 +1,50 @@ +{ + "resourceType": "Subscription", + "id": "example-backport-storage-patient", + "meta" : { + "profile": ["http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"] + }, + "status": "requested", + "end": "2031-01-01T12:00:00", + "reason": "Test subscription based on transactions, filtered by Patient name", + "criteria" : "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions", + "_criteria": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-filter-criteria", + "valueString": "Patient?name=Peter" + } + ] + }, + "channel": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-heartbeat-period", + "valueInteger": 120 + } + ], + "type" : "rest-hook", + "_type" : { + "extension" : [ + { + "url" : "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type", + "valueCoding" : { + "system" : "http://azurehealthcareapis.com/data-extentions/subscription-channel-type", + "code" : "azure-storage", + "display" : "Azure Blob Storage" + } + } + ] + }, + "endpoint": "sync-patient", + "payload": "application/fhir+json", + "_payload": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content", + "valueCode": "full-resource" + } + ] + } + } + } From 082e1056c3620672e98aac0614c77f217002e3ba Mon Sep 17 00:00:00 2001 From: Adithi Ponakampalli <120080886+aponakampalli@users.noreply.github.com> Date: Wed, 28 Aug 2024 16:31:04 -0400 Subject: [PATCH 41/47] Rest Hook Channel for Subscriptions (#4008) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Improve fhirtimer * Subscription infra * Fixes wiring up of Transaction Watchdog => Orchestrator * Adding code for writing to storage * Allow resourceKey to deserialize * Implement basic subscription filtering * Implements Channel Interface * Add example subscription * DataLakeChannel. * Changes in DataLakeChannel and the project config. * Load from DB * EventGrid WIP * Improve fhirtimer * Fixes for subscriptionwatchdog * Aligns dotnet sdk version for build * Adds subscription to docker build * Adds SearchQueryInterpreter * set up notification manager * Cleanup of SearchQueryInterpreter * integrate in memory search with filter search for subscriptions * resolve notification manager build errors * set up notification manager * integrate in memory search with filter search for subscriptions * resolve notification manager build errors * remove build error in webhook channel * rename webhook channel name * set up notification manager * integrate in memory search with filter search for subscriptions * resolve notification manager build errors * set up notification manager * resolve notification manager build errors * remove build error in webhook channel * rename webhook channel name * in memory search test set up * subscription orchestrator job test checks for id * subscription orchestator job only sends filtered resources instead of all resources * new test for patient name filter * add a failing test * rename tests for subscription orchestrator job * remove web hook code * set up validation for subscriptions * remove rest hook channel temporarily * add generic notify method * setup for creating bundles * create subscription bundle for notification events * set up heartbeat and handshake * add notification bundle for rest hook channel and pass in additional fields for subscriptions * handshake method for subscriptions * set up heartbeat * set up heartbeat background service and validation * modify subscriptionUpdator to convert resource into element node to edit * build contect for url resolver * cleanup handshake and heatbeat implementation * cleanup duplicated test files * rename fhir context and tests --------- Co-authored-by: Brendan Kowitz Co-authored-by: Fernando Henrique Inocêncio Borba Ferreira --- Directory.Packages.props | 4 +- .../Features/Routing/UrlResolver.cs | 6 +- .../Features/Search/IBundleFactory.cs | 5 + .../CreateOrUpdateSearchParameterBehavior.cs | 4 +- .../Subscriptions/ISubscriptionUpdator.cs | 19 ++ .../Messages/Create/CreateResourceRequest.cs | 2 +- .../Messages/Upsert/UpsertResourceRequest.cs | 2 +- .../Models/IModelInfoProvider.cs | 3 + .../SubscriptionsOrchestratorJobTests.cs | 11 +- .../Features/Search/BundleFactory.cs | 38 +++ .../VersionSpecificModelInfoProvider.cs | 8 + ...oft.Health.Fhir.Subscriptions.Tests.csproj | 1 - ...s.cs => SubscriptionModelConverterTest.cs} | 10 +- .../Peristence/SubscriptionUpdatorTest.cs | 34 +++ .../Channels/DataLakeChannel.cs | 15 +- .../Channels/EventGridChannel.cs | 14 +- .../Channels/ISubscriptionChannel.cs | 6 +- .../Channels/RestHookChannel.cs | 222 ++++++++++++++++++ .../Channels/StorageChannel.cs | 14 +- .../HeartBeats/HeartBeatBackgroundService.cs | 108 +++++++++ ...Microsoft.Health.Fhir.Subscriptions.csproj | 2 + .../Models/ISubscriptionModelConverter.cs | 21 ++ .../Models/SubscriptionInfo.cs | 18 +- .../Models/SubscriptionJobDefinition.cs | 4 +- .../Models/SubscriptionModelConverterR4.cs | 88 +++++++ .../Models/SubscriptionStatus.cs | 27 +++ .../Operations/SubscriptionProcessingJob.cs | 6 +- .../SubscriptionsOrchestratorJob.cs | 2 +- .../Persistence/ISubscriptionManager.cs | 2 + .../Persistence/SubscriptionManager.cs | 88 +++---- .../Persistence/SubscriptionUpdator.cs | 29 +++ .../Registration/SubscriptionsModule.cs | 42 ++++ .../CreateOrUpdateSubscriptionBehavior.cs | 62 +++++ .../Validation/ISubscriptionValidator.cs | 17 ++ .../Validation/SubscriptionException.cs | 27 +++ .../Validation/SubscriptionValidator.cs | 81 +++++++ 36 files changed, 955 insertions(+), 87 deletions(-) create mode 100644 src/Microsoft.Health.Fhir.Core/Features/Subscriptions/ISubscriptionUpdator.cs rename src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/{SubscriptionManagerTests.cs => SubscriptionModelConverterTest.cs} (84%) create mode 100644 src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionUpdatorTest.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Channels/RestHookChannel.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/HeartBeats/HeartBeatBackgroundService.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Models/ISubscriptionModelConverter.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionModelConverterR4.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionStatus.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionUpdator.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Validation/CreateOrUpdateSubscriptionBehavior.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Validation/ISubscriptionValidator.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionException.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionValidator.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 7479c7f93d..8bffb46c01 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -6,6 +6,8 @@ + + @@ -125,4 +127,4 @@ - + \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Api/Features/Routing/UrlResolver.cs b/src/Microsoft.Health.Fhir.Api/Features/Routing/UrlResolver.cs index c2ea11f4b9..6e4e4dc7a1 100644 --- a/src/Microsoft.Health.Fhir.Api/Features/Routing/UrlResolver.cs +++ b/src/Microsoft.Health.Fhir.Api/Features/Routing/UrlResolver.cs @@ -138,11 +138,11 @@ private Uri ResolveResourceUrl(string resourceId, string resourceTypeName, strin } return GetRouteUri( - ActionContext.HttpContext, + ActionContext?.HttpContext, routeName, routeValues, - Request.Scheme, - Request.Host.Value); + Request?.Scheme, + Request?.Host.Value); } public Uri ResolveRouteUrl(IReadOnlyCollection> unsupportedSearchParams = null, IReadOnlyList<(SearchParameterInfo searchParameterInfo, SortOrder sortOrder)> resultSortOrder = null, string continuationToken = null, bool removeTotalParameter = false) diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/IBundleFactory.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/IBundleFactory.cs index d9969e84e0..80c198d148 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/IBundleFactory.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/IBundleFactory.cs @@ -4,7 +4,10 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Collections; +using System.Collections.Generic; using Hl7.Fhir.Model; +using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Models; namespace Microsoft.Health.Fhir.Core.Features.Search @@ -15,6 +18,8 @@ public interface IBundleFactory ResourceElement CreateHistoryBundle(SearchResult result); + System.Threading.Tasks.Task CreateSubscriptionBundleAsync(params ResourceWrapper[] resources); + Resource CreateDeletedResourcesBundle(string bundleId, DateTimeOffset lastUpdated, params ResourceReference[] resourceReferences); } } diff --git a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs index bf68edf11a..e9fa8646a2 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Search/Parameters/CreateOrUpdateSearchParameterBehavior.cs @@ -18,8 +18,8 @@ namespace Microsoft.Health.Fhir.Core.Features.Search.Parameters public class CreateOrUpdateSearchParameterBehavior : IPipelineBehavior, IPipelineBehavior { - private ISearchParameterOperations _searchParameterOperations; - private IFhirDataStore _fhirDataStore; + private readonly ISearchParameterOperations _searchParameterOperations; + private readonly IFhirDataStore _fhirDataStore; public CreateOrUpdateSearchParameterBehavior(ISearchParameterOperations searchParameterOperations, IFhirDataStore fhirDataStore) { diff --git a/src/Microsoft.Health.Fhir.Core/Features/Subscriptions/ISubscriptionUpdator.cs b/src/Microsoft.Health.Fhir.Core/Features/Subscriptions/ISubscriptionUpdator.cs new file mode 100644 index 0000000000..ea2d79ca7e --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Features/Subscriptions/ISubscriptionUpdator.cs @@ -0,0 +1,19 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Core.Features.Subscriptions +{ + public interface ISubscriptionUpdator + { + ResourceElement UpdateStatus(ResourceElement subscription, string status); + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Messages/Create/CreateResourceRequest.cs b/src/Microsoft.Health.Fhir.Core/Messages/Create/CreateResourceRequest.cs index 1d76c7c3bd..0d081d4fcb 100644 --- a/src/Microsoft.Health.Fhir.Core/Messages/Create/CreateResourceRequest.cs +++ b/src/Microsoft.Health.Fhir.Core/Messages/Create/CreateResourceRequest.cs @@ -24,7 +24,7 @@ public CreateResourceRequest(ResourceElement resource, BundleResourceContext bun public BundleResourceContext BundleResourceContext { get; } - public ResourceElement Resource { get; } + public ResourceElement Resource { get; set; } public IEnumerable RequiredCapabilities() { diff --git a/src/Microsoft.Health.Fhir.Core/Messages/Upsert/UpsertResourceRequest.cs b/src/Microsoft.Health.Fhir.Core/Messages/Upsert/UpsertResourceRequest.cs index 767181cd7c..602d45ca7b 100644 --- a/src/Microsoft.Health.Fhir.Core/Messages/Upsert/UpsertResourceRequest.cs +++ b/src/Microsoft.Health.Fhir.Core/Messages/Upsert/UpsertResourceRequest.cs @@ -23,7 +23,7 @@ public UpsertResourceRequest(ResourceElement resource, BundleResourceContext bun WeakETag = weakETag; } - public ResourceElement Resource { get; } + public ResourceElement Resource { get; set; } public BundleResourceContext BundleResourceContext { get; } diff --git a/src/Microsoft.Health.Fhir.Core/Models/IModelInfoProvider.cs b/src/Microsoft.Health.Fhir.Core/Models/IModelInfoProvider.cs index 75ceff698f..ce0135bccb 100644 --- a/src/Microsoft.Health.Fhir.Core/Models/IModelInfoProvider.cs +++ b/src/Microsoft.Health.Fhir.Core/Models/IModelInfoProvider.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using Hl7.Fhir.ElementModel; +using Hl7.Fhir.Model; using Hl7.Fhir.Specification; using Hl7.FhirPath; using Microsoft.Health.Fhir.Core.Features.Persistence; @@ -37,5 +38,7 @@ public interface IModelInfoProvider ITypedElement ToTypedElement(ISourceNode sourceNode); ITypedElement ToTypedElement(RawResource rawResource); + + ResourceElement ToResourceElement(Resource resource); } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs index 0974e43880..7ee968cede 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs @@ -51,6 +51,7 @@ public class SubscriptionsOrchestratorJobTests : IAsyncLifetime private ISubscriptionManager _subscriptionManager = Substitute.For(); private IResourceDeserializer _resourceDeserializer; private IExpressionParser _expressionParser; + private ISubscriptionModelConverter _subscriptionModelConverter; public Task DisposeAsync() { @@ -73,7 +74,7 @@ public async Task InitializeAsync() var referenceToElementResolver = new LightweightReferenceToElementResolver(referenceParser, ModelInfoProvider.Instance); var modelInfoProvider = ModelInfoProvider.Instance; var logger = Substitute.For>(); - + _subscriptionModelConverter = new SubscriptionModelConverterR4(); _searchIndexer = new TypedElementSearchIndexer(supportedSearchParameterDefinitionManager, typedElementToSearchValueConverterManager, referenceToElementResolver, modelInfoProvider, logger); _transactionDataStore.GetResourcesByTransactionIdAsync(Arg.Any(), Arg.Any()).Returns(x => @@ -126,7 +127,7 @@ public async Task GivenASubscriptionOrchestrator_WhenPatientResourceRecieved_The Skip.If(ModelInfoProvider.Version != FhirSpecification.R4); _subscriptionManager.GetActiveSubscriptionsAsync(Arg.Any()).Returns(x => { - var subscriptionInfo = SubscriptionManager.ConvertToInfo(Samples.GetJsonSample("SubscriptionForPatient")); + var subscriptionInfo = _subscriptionModelConverter.Convert(Samples.GetJsonSample("SubscriptionForPatient")); var subscriptionInfoList = new List { subscriptionInfo, @@ -155,7 +156,7 @@ public async Task GivenANameFilterSubscription_WhenResourcesPosted_ThenCorrectRe Skip.If(ModelInfoProvider.Version != FhirSpecification.R4); _subscriptionManager.GetActiveSubscriptionsAsync(Arg.Any()).Returns(x => { - var subscriptionInfo = SubscriptionManager.ConvertToInfo(Samples.GetJsonSample("SubscriptionForPatientName")); + var subscriptionInfo = _subscriptionModelConverter.Convert(Samples.GetJsonSample("SubscriptionForPatientName")); var subscriptionInfoList = new List { subscriptionInfo, @@ -184,7 +185,7 @@ public async Task GivenAReferenceFilterSubscription_WhenResourcesPosted_ThenCorr Skip.If(ModelInfoProvider.Version != FhirSpecification.R4); _subscriptionManager.GetActiveSubscriptionsAsync(Arg.Any()).Returns(x => { - var subscriptionInfo = SubscriptionManager.ConvertToInfo(Samples.GetJsonSample("SubscriptionForObservationReferenceToPatient")); + var subscriptionInfo = _subscriptionModelConverter.Convert(Samples.GetJsonSample("SubscriptionForObservationReferenceToPatient")); var subscriptionInfoList = new List { subscriptionInfo, @@ -213,7 +214,7 @@ public async Task GivenEncounterFilterSubscription_WhenNonEncounterResourcesPost Skip.If(ModelInfoProvider.Version != FhirSpecification.R4); _subscriptionManager.GetActiveSubscriptionsAsync(Arg.Any()).Returns(x => { - var subscriptionInfo = SubscriptionManager.ConvertToInfo(Samples.GetJsonSample("SubscriptionForEncounter")); + var subscriptionInfo = _subscriptionModelConverter.Convert(Samples.GetJsonSample("SubscriptionForEncounter")); var subscriptionInfoList = new List { subscriptionInfo, diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/BundleFactory.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/BundleFactory.cs index d4c34b4992..49144ab5da 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/BundleFactory.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/BundleFactory.cs @@ -7,8 +7,11 @@ using System.Collections.Generic; using System.Linq; using System.Net; +using DotLiquid.Util; using EnsureThat; +using Hl7.Fhir.ElementModel; using Hl7.Fhir.Model; +using Hl7.Fhir.Serialization; using Microsoft.Extensions.Logging; using Microsoft.Health.Core; using Microsoft.Health.Core.Features.Context; @@ -16,6 +19,7 @@ using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Routing; +using Microsoft.Health.Fhir.Core.Features.Search.Converters; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Shared.Core.Features.Search; using Microsoft.Health.Fhir.ValueSets; @@ -27,6 +31,7 @@ public class BundleFactory : IBundleFactory private readonly IUrlResolver _urlResolver; private readonly RequestContextAccessor _fhirRequestContextAccessor; private readonly ILogger _logger; + private readonly FhirJsonParser _fhirJsonParser = new Hl7.Fhir.Serialization.FhirJsonParser(); public BundleFactory(IUrlResolver urlResolver, RequestContextAccessor fhirRequestContextAccessor, ILogger logger) { @@ -235,5 +240,38 @@ private ResourceElement CreateBundle(SearchResult result, Bundle.BundleType type return bundle.ToResourceElement(); } + + public async System.Threading.Tasks.Task CreateSubscriptionBundleAsync(params ResourceWrapper[] resources) + { + EnsureArg.HasItems(resources, nameof(resources)); + + Bundle bundle = new() + { + Type = Bundle.BundleType.Transaction, + Entry = new(), + }; + + foreach (ResourceWrapper rw in resources) + { + var rawResource = rw.RawResource.Data; + var resource = await _fhirJsonParser.ParseAsync(rawResource); + Enum.TryParse(rw.Request?.Method, true, out Bundle.HTTPVerb httpVerb); + + var resourcesEntry = new Bundle.EntryComponent + { + Resource = resource, + FullUrlElement = new FhirUri(_urlResolver.ResolveResourceWrapperUrl(rw)), + Request = new Bundle.RequestComponent + { + Method = httpVerb, + Url = $"{rw.ResourceTypeName}/{(httpVerb == Bundle.HTTPVerb.POST ? null : rw.ResourceId)}", + }, + }; + + bundle.Entry.Add(resourcesEntry); + } + + return await bundle.ToJsonAsync(); + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Core/VersionSpecificModelInfoProvider.cs b/src/Microsoft.Health.Fhir.Shared.Core/VersionSpecificModelInfoProvider.cs index 11e5eeb8c9..a07eae16bd 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/VersionSpecificModelInfoProvider.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/VersionSpecificModelInfoProvider.cs @@ -14,6 +14,7 @@ using Hl7.Fhir.Serialization; using Hl7.Fhir.Specification; using Hl7.FhirPath; +using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Validation; using Microsoft.Health.Fhir.Core.Models; @@ -103,5 +104,12 @@ public ITypedElement ToTypedElement(RawResource rawResource) throw new ResourceNotValidException(new List() { issue }); } } + + public ResourceElement ToResourceElement(Resource resource) + { + EnsureArg.IsNotNull(resource); + + return resource.ToResourceElement(); + } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj index 814d4639d3..6c49ca4100 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj +++ b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Microsoft.Health.Fhir.Subscriptions.Tests.csproj @@ -22,7 +22,6 @@ - diff --git a/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionManagerTests.cs b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionModelConverterTest.cs similarity index 84% rename from src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionManagerTests.cs rename to src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionModelConverterTest.cs index 253e151730..9c92a81855 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionManagerTests.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionModelConverterTest.cs @@ -20,24 +20,26 @@ namespace Microsoft.Health.Fhir.Subscriptions.Tests.Peristence { - public class SubscriptionManagerTests + public class SubscriptionModelConverterTest { private IModelInfoProvider _modelInfo; + private ISubscriptionModelConverter _subscriptionModelConverter; - public SubscriptionManagerTests() + public SubscriptionModelConverterTest() { _modelInfo = MockModelInfoProviderBuilder .Create(FhirSpecification.R4) .AddKnownTypes(KnownResourceTypes.Subscription) .Build(); + + _subscriptionModelConverter = new SubscriptionModelConverterR4(); } [Fact] public void GivenAnR4BackportSubscription_WhenConvertingToInfo_ThenTheInformationIsCorrect() { var subscription = CommonSamples.GetJsonSample("Subscription-Backport", FhirSpecification.R4, s => s.ToTypedElement(ModelInfo.ModelInspector)); - - var info = SubscriptionManager.ConvertToInfo(subscription); + var info = _subscriptionModelConverter.Convert(subscription); Assert.Equal("Patient", info.FilterCriteria); Assert.Equal("sync-all", info.Channel.Endpoint); diff --git a/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionUpdatorTest.cs b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionUpdatorTest.cs new file mode 100644 index 0000000000..b1c393e4b7 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionUpdatorTest.cs @@ -0,0 +1,34 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Hl7.Fhir.ElementModel; +using Hl7.Fhir.Model; +using Hl7.FhirPath; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Subscriptions.Persistence; +using Microsoft.Health.Fhir.Tests.Common; +using Xunit; + +namespace Microsoft.Health.Fhir.Subscriptions.Tests.Peristence +{ + public class SubscriptionUpdatorTest + { + [Fact] + public void GivenAnR4BackportSubscription_WhenUpdatingStatusToActive_ThenTheInformationIsCorrect() + { + var subscription = CommonSamples.GetJsonSample("Subscription-Backport", FhirSpecification.R4, s => s.ToTypedElement(ModelInfo.ModelInspector)); + var updator = new SubscriptionUpdator(); + ModelInfoProvider.SetProvider(MockModelInfoProviderBuilder.Create(FhirSpecification.R4).Build()); + var updatedSubscription = updator.UpdateStatus(subscription, "active"); + + Assert.Equal("active", updatedSubscription.Scalar("Subscription.status")); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs index f41a460134..1e7a21a80c 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs @@ -27,14 +27,13 @@ public DataLakeChannel(IExportDestinationClient exportDestinationClient, IResour _resourceDeserializer = resourceDeserializer; } - public async Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) + public async Task PublishAsync(IReadOnlyCollection resources, SubscriptionInfo subscriptionInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) { try { - await _exportDestinationClient.ConnectAsync(cancellationToken, channelInfo.Endpoint); + await _exportDestinationClient.ConnectAsync(cancellationToken, subscriptionInfo.Channel.Endpoint); IReadOnlyList> resourceGroupedByResourceType = resources.GroupBy(x => x.ResourceTypeName.ToLower(CultureInfo.InvariantCulture)).ToList(); - DateTimeOffset transactionTimeInUtc = transactionTime.ToUniversalTime(); foreach (IGrouping groupOfResources in resourceGroupedByResourceType) @@ -64,5 +63,15 @@ public async Task PublishAsync(IReadOnlyCollection resources, C throw new InvalidOperationException("Failure in DatalakeChannel", ex); } } + + public Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo) + { + return Task.CompletedTask; + } + + public Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo) + { + return Task.CompletedTask; + } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs index 9476649097..6abe4d773c 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs @@ -24,9 +24,19 @@ public EventGridChannel() { } - public Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) + public Task PublishAsync(IReadOnlyCollection resources, SubscriptionInfo subscriptionInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) { - throw new NotImplementedException(); + return Task.CompletedTask; + } + + public Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo) + { + return Task.CompletedTask; + } + + public Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo) + { + return Task.CompletedTask; } /* diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs index e98211970b..6d61284b94 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs @@ -14,6 +14,10 @@ namespace Microsoft.Health.Fhir.Subscriptions.Channels { public interface ISubscriptionChannel { - Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken); + Task PublishAsync(IReadOnlyCollection resources, SubscriptionInfo subscriptionInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken); + + Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo); + + Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo); } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/RestHookChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/RestHookChannel.cs new file mode 100644 index 0000000000..68d06e6320 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/RestHookChannel.cs @@ -0,0 +1,222 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Hl7.Fhir.Model; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc.Infrastructure; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Primitives; +using Microsoft.Health.Core.Features.Context; +using Microsoft.Health.Fhir.Core.Features.Context; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Routing; +using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Subscriptions.Models; +using Microsoft.Health.Fhir.Subscriptions.Validation; + +namespace Microsoft.Health.Fhir.Subscriptions.Channels +{ + [ChannelType(SubscriptionChannelType.RestHook)] + + #pragma warning disable CA1001 // Types that own disposable fields should be disposable + public class RestHookChannel : ISubscriptionChannel + #pragma warning restore CA1001 // Types that own disposable fields should be disposable + { + private readonly ILogger _logger; + private readonly IBundleFactory _bundleFactory; + private readonly IRawResourceFactory _rawResourceFactory; + private readonly HttpClient _httpClient; + private readonly IModelInfoProvider _modelInfoProvider; + private readonly IUrlResolver _urlResolver; + private RequestContextAccessor _contextAccessor; + private readonly IHttpContextAccessor _httpContextAccessor; + private readonly IActionContextAccessor _actionContextAccessor; + + public RestHookChannel(ILogger logger, HttpClient httpClient, IBundleFactory bundleFactory, IRawResourceFactory rawResourceFactory, IModelInfoProvider modelInfoProvider, IUrlResolver urlResolver, RequestContextAccessor contextAccessor, IHttpContextAccessor httpContextAccessor, IActionContextAccessor actionContextAccessor) + { + _logger = logger; +#pragma warning disable CA2000 // Dispose objects before losing scope + _httpClient = new HttpClient(new HttpClientHandler() { CheckCertificateRevocationList = true }, disposeHandler: true); +#pragma warning restore CA2000 // Dispose objects before losing scope + _bundleFactory = bundleFactory; + _rawResourceFactory = rawResourceFactory; + _modelInfoProvider = modelInfoProvider; + _urlResolver = urlResolver; + + var fhirRequestContext = new FhirRequestContext( + method: null, + uriString: string.Empty, + baseUriString: string.Empty, + correlationId: string.Empty, + requestHeaders: new Dictionary(), + responseHeaders: new Dictionary()) + { + IsBackgroundTask = true, + AuditEventType = OperationsConstants.Reindex, + }; + + _contextAccessor = contextAccessor; + _contextAccessor.RequestContext = fhirRequestContext; + _httpContextAccessor = httpContextAccessor; + _actionContextAccessor = actionContextAccessor; + + var httpContext = new DefaultHttpContext(); + httpContext.Request.Scheme = "https"; + httpContext.Request.Host = new HostString("fhir.com", 433); + _httpContextAccessor.HttpContext = httpContext; + + _actionContextAccessor.ActionContext = new AspNetCore.Mvc.ActionContext(); + _actionContextAccessor.ActionContext.HttpContext = httpContext; + } + + public async Task PublishAsync(IReadOnlyCollection resources, SubscriptionInfo subscriptionInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) + { + List resourceWrappers = new List(); + var parameter = new Parameters + { + { "subscription", new ResourceReference(subscriptionInfo.ResourceId) }, + { "topic", new FhirUri(subscriptionInfo.Topic) }, + { "status", new Code(subscriptionInfo.Status.ToString()) }, + { "type", new Code("event-notification") }, + }; + + if (!subscriptionInfo.Channel.ContentType.Equals(SubscriptionContentType.Empty)) + { + // add new fields to parameter object from subscription data + foreach (ResourceWrapper rw in resources) + { + var notificationEvent = new Parameters.ParameterComponent + { + Name = "notification-event", + Part = new List + { + new Parameters.ParameterComponent + { + Name = "focus", + Value = new FhirUri(_urlResolver.ResolveResourceWrapperUrl(rw)), + }, + new Parameters.ParameterComponent + { + Name = "timestamp", + Value = new FhirDateTime(transactionTime), + }, + }, + }; + parameter.Parameter.Add(notificationEvent); + } + } + + parameter.Id = Guid.NewGuid().ToString(); + var parameterResourceElement = _modelInfoProvider.ToResourceElement(parameter); + var parameterResourceWrapper = new ResourceWrapper(parameterResourceElement, _rawResourceFactory.Create(parameterResourceElement, keepMeta: true), new ResourceRequest(HttpMethod.Post, "http://fhir"), false, new List().AsReadOnly(), new CompartmentIndices(), new List>().AsReadOnly()); + resourceWrappers.Add(parameterResourceWrapper); + + if (subscriptionInfo.Channel.ContentType.Equals(SubscriptionContentType.FullResource)) + { + resourceWrappers.AddRange(resources); + } + + string bundle = await _bundleFactory.CreateSubscriptionBundleAsync(resourceWrappers.ToArray()); + + await SendPayload(subscriptionInfo.Channel, bundle); + } + + public async Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo) + { + List resourceWrappers = new List(); + var parameter = new Parameters + { + { "subscription", new ResourceReference(subscriptionInfo.ResourceId) }, + { "topic", new FhirUri(subscriptionInfo.Topic) }, + { "status", new Code(subscriptionInfo.Status.ToString()) }, + { "type", new Code("handshake") }, + }; + + parameter.Id = Guid.NewGuid().ToString(); + var parameterResourceElement = _modelInfoProvider.ToResourceElement(parameter); + var parameterResourceWrapper = new ResourceWrapper(parameterResourceElement, _rawResourceFactory.Create(parameterResourceElement, keepMeta: true), new ResourceRequest(HttpMethod.Post, "http://fhir"), false, null, null, null); + resourceWrappers.Add(parameterResourceWrapper); + + string bundle = await _bundleFactory.CreateSubscriptionBundleAsync(resourceWrappers.ToArray()); + + await SendPayload(subscriptionInfo.Channel, bundle); + } + + public async Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo) + { + List resourceWrappers = new List(); + var parameter = new Parameters + { + { "subscription", new ResourceReference(subscriptionInfo.ResourceId) }, + { "topic", new FhirUri(subscriptionInfo.Topic) }, + { "status", new Code(subscriptionInfo.Status.ToString()) }, + { "type", new Code("heartbeat") }, + }; + + parameter.Id = Guid.NewGuid().ToString(); + var parameterResourceElement = _modelInfoProvider.ToResourceElement(parameter); + var parameterResourceWrapper = new ResourceWrapper(parameterResourceElement, _rawResourceFactory.Create(parameterResourceElement, keepMeta: true), new ResourceRequest(HttpMethod.Post, "http://fhir"), false, null, null, null); + resourceWrappers.Add(parameterResourceWrapper); + + string bundle = await _bundleFactory.CreateSubscriptionBundleAsync(resourceWrappers.ToArray()); + + await SendPayload(subscriptionInfo.Channel, bundle); + } + + private async Task SendPayload( + ChannelInfo chanelInfo, + string contents) + { + HttpRequestMessage request = null!; + + // send the request to the endpoint + try + { + // build our request + request = new HttpRequestMessage() + { + Method = HttpMethod.Post, + RequestUri = new Uri(chanelInfo.Endpoint), + Content = new StringContent(contents), + }; + + // send our request + HttpResponseMessage response = await _httpClient.SendAsync(request); + + // check the status code + if ((response.StatusCode != System.Net.HttpStatusCode.OK) && + (response.StatusCode != System.Net.HttpStatusCode.Accepted)) + { + // failure + _logger.LogError($"REST POST to {chanelInfo.Endpoint} failed: {response.StatusCode}"); + throw new SubscriptionException("Subscription message invalid."); + } + else + { + _logger.LogError($"REST POST to {chanelInfo.Endpoint} succeeded: {response.StatusCode}"); + } + } + catch (Exception ex) + { + _logger.LogError($"REST POST {chanelInfo.ChannelType} to {chanelInfo.Endpoint} failed: {ex.Message}"); + } + finally + { + if (request != null) + { + request.Dispose(); + } + } + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs index d7d0c2ad74..68fe68d423 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs @@ -24,9 +24,9 @@ public StorageChannel( _exportDestinationClient = exportDestinationClient; } - public async Task PublishAsync(IReadOnlyCollection resources, ChannelInfo channelInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) + public async Task PublishAsync(IReadOnlyCollection resources, SubscriptionInfo subscriptionInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) { - await _exportDestinationClient.ConnectAsync(cancellationToken, channelInfo.Endpoint); + await _exportDestinationClient.ConnectAsync(cancellationToken, subscriptionInfo.Channel.Endpoint); foreach (var resource in resources) { @@ -39,5 +39,15 @@ public async Task PublishAsync(IReadOnlyCollection resources, C _exportDestinationClient.CommitFile(fileName); } } + + public Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo) + { + return Task.CompletedTask; + } + + public Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo) + { + return Task.CompletedTask; + } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/HeartBeats/HeartBeatBackgroundService.cs b/src/Microsoft.Health.Fhir.Subscriptions/HeartBeats/HeartBeatBackgroundService.cs new file mode 100644 index 0000000000..d0dc9ec26a --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/HeartBeats/HeartBeatBackgroundService.cs @@ -0,0 +1,108 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using FluentValidation.Results; +using MediatR; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Core.Messages.Storage; +using Microsoft.Health.Fhir.Subscriptions.Channels; +using Microsoft.Health.Fhir.Subscriptions.Models; +using Microsoft.Health.Fhir.Subscriptions.Persistence; +using Microsoft.Health.Fhir.Subscriptions.Validation; + +namespace Microsoft.Health.Fhir.Subscriptions.HeartBeats +{ + public class HeartBeatBackgroundService : BackgroundService, INotificationHandler + { + private bool _storageReady = false; + private readonly ILogger _logger; + private readonly IScopeProvider _subscriptionManager; + private readonly StorageChannelFactory _storageChannelFactory; + + public HeartBeatBackgroundService(ILogger logger, IScopeProvider subscriptionManager, StorageChannelFactory storageChannelFactory) + { + _logger = logger; + _subscriptionManager = subscriptionManager; + _storageChannelFactory = storageChannelFactory; + } + + public Task Handle(StorageInitializedNotification notification, CancellationToken cancellationToken) + { + _storageReady = true; + return Task.CompletedTask; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!_storageReady) + { + stoppingToken.ThrowIfCancellationRequested(); + await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); + } + + using var periodicTimer = new PeriodicTimer(TimeSpan.FromSeconds(60)); + var nextHeartBeat = new Dictionary(); + + while (!stoppingToken.IsCancellationRequested) + { + try + { + await periodicTimer.WaitForNextTickAsync(stoppingToken); + } + catch (OperationCanceledException) + { + break; + } + + try + { + // our logic + using IScoped subscriptionManager = _subscriptionManager.Invoke(); + await subscriptionManager.Value.SyncSubscriptionsAsync(stoppingToken); + + var activeSubscriptions = await subscriptionManager.Value.GetActiveSubscriptionsAsync(stoppingToken); + var subscriptionsWithHeartbeat = activeSubscriptions.Where(subscription => !subscription.Channel.HeartBeatPeriod.Equals(null)); + + foreach (var subscription in subscriptionsWithHeartbeat) + { + if (!nextHeartBeat.ContainsKey(subscription.ResourceId)) + { + nextHeartBeat[subscription.ResourceId] = DateTime.Now; + } + + // checks if datetime is after current time + if (nextHeartBeat.GetValueOrDefault(subscription.ResourceId) <= DateTime.Now) + { + var channel = _storageChannelFactory.Create(subscription.Channel.ChannelType); + try + { + await channel.PublishHeartBeatAsync(subscription); + nextHeartBeat[subscription.ResourceId] = nextHeartBeat.GetValueOrDefault(subscription.ResourceId).Add(subscription.Channel.HeartBeatPeriod); + } + catch (SubscriptionException) + { + await subscriptionManager.Value.MarkAsError(subscription, stoppingToken); + } + } + } + } + catch (Exception e) + { + _logger.LogWarning(e, "Error executing timer"); + } + } + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj index 7ec3054f0c..e5b76c0940 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj +++ b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj @@ -2,6 +2,8 @@ + + diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/ISubscriptionModelConverter.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/ISubscriptionModelConverter.cs new file mode 100644 index 0000000000..3c0e007f9a --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/ISubscriptionModelConverter.cs @@ -0,0 +1,21 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Hl7.Fhir.Model; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Models +{ + public interface ISubscriptionModelConverter + { + SubscriptionInfo Convert(ResourceElement resource); + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs index 038865d938..ba92430d4b 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionInfo.cs @@ -6,7 +6,9 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Security.Policy; using System.Text; +using System.Text.Json.Serialization; using System.Threading.Tasks; using EnsureThat; @@ -14,14 +16,28 @@ namespace Microsoft.Health.Fhir.Subscriptions.Models { public class SubscriptionInfo { - public SubscriptionInfo(string filterCriteria, ChannelInfo channel) + public SubscriptionInfo(string filterCriteria, ChannelInfo channel, Uri topic, string resourceId, SubscriptionStatus status) { FilterCriteria = filterCriteria; Channel = EnsureArg.IsNotNull(channel, nameof(channel)); + Topic = topic; + ResourceId = resourceId; + Status = status; + } + + [JsonConstructor] + protected SubscriptionInfo() + { } public string FilterCriteria { get; set; } public ChannelInfo Channel { get; set; } + + public Uri Topic { get; set; } + + public string ResourceId { get; set; } + + public SubscriptionStatus Status { get; set; } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs index 7ef7fade06..198e9f9a92 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionJobDefinition.cs @@ -41,7 +41,7 @@ protected SubscriptionJobDefinition() [SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "JSON Poco")] public IList ResourceReferences { get; set; } - [JsonProperty("channel")] - public ChannelInfo Channel { get; set; } + [JsonProperty("subscriptionInfo")] + public SubscriptionInfo SubscriptionInfo { get; set; } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionModelConverterR4.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionModelConverterR4.cs new file mode 100644 index 0000000000..fdc5542a9a --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionModelConverterR4.cs @@ -0,0 +1,88 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Models +{ + public class SubscriptionModelConverterR4 : ISubscriptionModelConverter + { + private const string MetaString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"; + private const string CriteriaString = "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions"; + private const string CriteriaExtensionString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-filter-criteria"; + private const string ChannelTypeString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type"; + + // private const string AzureChannelTypeString = "http://azurehealthcareapis.com/data-extentions/subscription-channel-type"; + private const string PayloadTypeString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content"; + private const string MaxCountString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-max-count"; + private const string HeartBeatString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-heartbeat-period"; + + public SubscriptionInfo Convert(ResourceElement resource) + { + var profile = resource.Scalar("Subscription.meta.profile"); + + if (profile != MetaString) + { + return null; + } + + var criteria = resource.Scalar($"Subscription.criteria"); + + if (criteria != CriteriaString) + { + return null; + } + + var criteriaExt = resource.Scalar($"Subscription.criteria.extension.where(url = '{CriteriaExtensionString}').value"); + var channelTypeExt = resource.Scalar($"Subscription.channel.type.extension.where(url = '{ChannelTypeString}').value.code"); + var payloadType = resource.Scalar($"Subscription.channel.payload.extension.where(url = '{PayloadTypeString}').value"); + var maxCount = resource.Scalar($"Subscription.channel.extension.where(url = '{MaxCountString}').value"); + var heartBeatSpan = resource.Scalar($"Subscription.channel.extension.where(url = '{HeartBeatString}').value"); + var resourceId = resource.Scalar("Subscription.id"); + var status = resource.Scalar("Subscription.status") switch + { + "active" => SubscriptionStatus.Active, + "requested" => SubscriptionStatus.Requested, + "error" => SubscriptionStatus.Error, + "off" => SubscriptionStatus.Off, + _ => SubscriptionStatus.None, + }; + var topic = new Uri(resource.Scalar("Subscription.criteria")); + + var channelInfo = new ChannelInfo + { + Endpoint = resource.Scalar($"Subscription.channel.endpoint"), + ChannelType = channelTypeExt switch + { + "azure-storage" => SubscriptionChannelType.Storage, + "azure-lake-storage" => SubscriptionChannelType.DatalakeContract, + "rest-hook" => SubscriptionChannelType.RestHook, + _ => SubscriptionChannelType.None, + }, + ContentType = payloadType switch + { + "full-resource" => SubscriptionContentType.FullResource, + "id-only" => SubscriptionContentType.IdOnly, + _ => SubscriptionContentType.Empty, + }, + MaxCount = maxCount ?? 100, + }; + + if (heartBeatSpan.HasValue) + { + channelInfo.HeartBeatPeriod = TimeSpan.FromSeconds(heartBeatSpan.Value); + } + + var info = new SubscriptionInfo(criteriaExt, channelInfo, topic, resourceId, status); + + return info; + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionStatus.cs b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionStatus.cs new file mode 100644 index 0000000000..5100ccf593 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Models/SubscriptionStatus.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Hl7.Fhir.Utility; + +namespace Microsoft.Health.Fhir.Subscriptions.Models +{ + public enum SubscriptionStatus + { + [EnumLiteral("requested")] + Requested, + [EnumLiteral("active")] + Active, + [EnumLiteral("error")] + Error, + [EnumLiteral("off")] + Off, + None, + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs index 7c38327e10..4431cc14b2 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -31,7 +31,7 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel { SubscriptionJobDefinition definition = jobInfo.DeserializeDefinition(); - if (definition.Channel == null) + if (definition.SubscriptionInfo == null) { return HttpStatusCode.BadRequest.ToString(); } @@ -40,8 +40,8 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel definition.ResourceReferences .Select(async x => await _dataStore.GetAsync(x, cancellationToken))); - var channel = _storageChannelFactory.Create(definition.Channel.ChannelType); - await channel.PublishAsync(allResources, definition.Channel, definition.VisibleDate, cancellationToken); + var channel = _storageChannelFactory.Create(definition.SubscriptionInfo.Channel.ChannelType); + await channel.PublishAsync(allResources, definition.SubscriptionInfo, definition.VisibleDate, cancellationToken); return HttpStatusCode.OK.ToString(); } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs index 1dd5de4d92..292eb7818c 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionsOrchestratorJob.cs @@ -120,7 +120,7 @@ public async Task ExecuteAsync(JobInfo jobInfo, CancellationToken cancel var cloneDefinition = jobInfo.DeserializeDefinition(); cloneDefinition.TypeId = (int)JobType.SubscriptionsProcessing; cloneDefinition.ResourceReferences = batch.ToList(); - cloneDefinition.Channel = sub.Channel; + cloneDefinition.SubscriptionInfo = sub; processingDefinition.Add(cloneDefinition); } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs index 180df430ba..e71104ab34 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ISubscriptionManager.cs @@ -15,5 +15,7 @@ public interface ISubscriptionManager Task> GetActiveSubscriptionsAsync(CancellationToken cancellationToken); Task SyncSubscriptionsAsync(CancellationToken cancellationToken); + + Task MarkAsError(SubscriptionInfo subscriptionInfo, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs index 915ae83e63..efd26c6f7d 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs @@ -5,9 +5,12 @@ using System; using System.Collections.Generic; +using System.Linq; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using EnsureThat; +using Hl7.Fhir.Utility; using MediatR; using Microsoft.Build.Framework; using Microsoft.Extensions.Hosting; @@ -15,6 +18,7 @@ using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search; +using Microsoft.Health.Fhir.Core.Features.Subscriptions; using Microsoft.Health.Fhir.Core.Messages.Storage; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Subscriptions.Models; @@ -28,25 +32,27 @@ public sealed class SubscriptionManager : ISubscriptionManager, INotificationHan private List _subscriptions = new List(); private readonly IResourceDeserializer _resourceDeserializer; private readonly ILogger _logger; + private readonly ISubscriptionModelConverter _subscriptionModelConverter; private static readonly object _lock = new object(); - private const string MetaString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"; - private const string CriteriaString = "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions"; - private const string CriteriaExtensionString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-filter-criteria"; - private const string ChannelTypeString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type"; - ////private const string AzureChannelTypeString = "http://azurehealthcareapis.com/data-extentions/subscription-channel-type"; - private const string PayloadTypeString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content"; - private const string MaxCountString = "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-max-count"; + private readonly ISubscriptionUpdator _subscriptionUpdator; + private readonly IRawResourceFactory _rawResourceFactory; public SubscriptionManager( IScopeProvider dataStoreProvider, IScopeProvider searchServiceProvider, IResourceDeserializer resourceDeserializer, - ILogger logger) + ILogger logger, + ISubscriptionModelConverter subscriptionModelConverter, + ISubscriptionUpdator subscriptionUpdator, + IRawResourceFactory rawResourceFactory) { _dataStoreProvider = EnsureArg.IsNotNull(dataStoreProvider, nameof(dataStoreProvider)); _searchServiceProvider = EnsureArg.IsNotNull(searchServiceProvider, nameof(searchServiceProvider)); _resourceDeserializer = resourceDeserializer; _logger = logger; + _subscriptionModelConverter = subscriptionModelConverter; + _subscriptionUpdator = subscriptionUpdator; + _rawResourceFactory = rawResourceFactory; } public async Task SyncSubscriptionsAsync(CancellationToken cancellationToken) @@ -68,8 +74,7 @@ public async Task SyncSubscriptionsAsync(CancellationToken cancellationToken) foreach (var param in activeSubscriptions.Results) { var resource = _resourceDeserializer.Deserialize(param.Resource); - - SubscriptionInfo info = ConvertToInfo(resource); + SubscriptionInfo info = _subscriptionModelConverter.Convert(resource); if (info == null) { @@ -86,50 +91,6 @@ public async Task SyncSubscriptionsAsync(CancellationToken cancellationToken) } } - internal static SubscriptionInfo ConvertToInfo(ResourceElement resource) - { - var profile = resource.Scalar("Subscription.meta.profile"); - - if (profile != MetaString) - { - return null; - } - - var criteria = resource.Scalar($"Subscription.criteria"); - - if (criteria != CriteriaString) - { - return null; - } - - var criteriaExt = resource.Scalar($"Subscription.criteria.extension.where(url = '{CriteriaExtensionString}').value"); - var channelTypeExt = resource.Scalar($"Subscription.channel.type.extension.where(url = '{ChannelTypeString}').value.code"); - var payloadType = resource.Scalar($"Subscription.channel.payload.extension.where(url = '{PayloadTypeString}').value"); - var maxCount = resource.Scalar($"Subscription.channel.extension.where(url = '{MaxCountString}').value"); - - var channelInfo = new ChannelInfo - { - Endpoint = resource.Scalar($"Subscription.channel.endpoint"), - ChannelType = channelTypeExt switch - { - "azure-storage" => SubscriptionChannelType.Storage, - "azure-lake-storage" => SubscriptionChannelType.DatalakeContract, - _ => SubscriptionChannelType.None, - }, - ContentType = payloadType switch - { - "full-resource" => SubscriptionContentType.FullResource, - "id-only" => SubscriptionContentType.IdOnly, - _ => SubscriptionContentType.Empty, - }, - MaxCount = maxCount ?? 100, - }; - - var info = new SubscriptionInfo(criteriaExt, channelInfo); - - return info; - } - public async Task> GetActiveSubscriptionsAsync(CancellationToken cancellationToken) { if (_subscriptions.Count == 0) @@ -145,5 +106,24 @@ public async Task Handle(StorageInitializedNotification notification, Cancellati // Preload subscriptions when storage becomes available await SyncSubscriptionsAsync(cancellationToken); } + + public async Task MarkAsError(SubscriptionInfo subscriptionInfo, CancellationToken cancellationToken) + { + using var search = _searchServiceProvider.Invoke(); + using var datastore = _dataStoreProvider.Invoke(); + + var getSubscriptionsWithId = await datastore.Value.GetAsync( + new List() + { + new ResourceKey("Subscription", subscriptionInfo.ResourceId), + }, + cancellationToken); + + var resourceElement = _resourceDeserializer.Deserialize(getSubscriptionsWithId.ToList()[0]); + var updatedStatusResource = _subscriptionUpdator.UpdateStatus(resourceElement, SubscriptionStatus.Error.GetLiteral()); + var resourceWrapper = new ResourceWrapper(updatedStatusResource, _rawResourceFactory.Create(updatedStatusResource, keepMeta: true), new ResourceRequest(HttpMethod.Post, "http://fhir"), false, null, null, null); + + await datastore.Value.UpsertAsync(new ResourceWrapperOperation(resourceWrapper, false, true, null, false, true, null), cancellationToken); + } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionUpdator.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionUpdator.cs new file mode 100644 index 0000000000..7067433a99 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionUpdator.cs @@ -0,0 +1,29 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Linq; +using Hl7.Fhir.ElementModel; +using Hl7.Fhir.Specification; +using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features.Subscriptions; +using Microsoft.Health.Fhir.Core.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Persistence +{ + public class SubscriptionUpdator : ISubscriptionUpdator + { + public ResourceElement UpdateStatus(ResourceElement subscription, string status) + { + var subscriptionElementNode = ElementNode.FromElement(subscription.Instance); + var oldStatusNode = (ElementNode)subscriptionElementNode.Children("status").FirstOrDefault(); + var newStatus = ElementNode.FromElement(oldStatusNode); + newStatus.Value = status; + subscriptionElementNode.Replace(ModelInfoProvider.Instance.StructureDefinitionSummaryProvider, oldStatusNode, newStatus); + + return subscriptionElementNode.ToResourceElement(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs index b12ce1f5f7..69088c03f8 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs @@ -11,11 +11,20 @@ using MediatR; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Extensions; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Features.Subscriptions; +using Microsoft.Health.Fhir.Core.Messages.Create; using Microsoft.Health.Fhir.Core.Messages.Storage; +using Microsoft.Health.Fhir.Core.Messages.Upsert; +using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Subscriptions.Channels; +using Microsoft.Health.Fhir.Subscriptions.HeartBeats; +using Microsoft.Health.Fhir.Subscriptions.Models; using Microsoft.Health.Fhir.Subscriptions.Persistence; +using Microsoft.Health.Fhir.Subscriptions.Validation; using Microsoft.Health.JobManagement; namespace Microsoft.Health.Fhir.Subscriptions.Registration @@ -50,6 +59,39 @@ public void Load(IServiceCollection services) services.Add() .Singleton() .AsSelf(); + + services.Add(c => + { + switch (c.GetService().Version) + { + case FhirSpecification.R4: + return new SubscriptionModelConverterR4(); + default: + throw new BadRequestException("Version not supported"); + } + }) + .Singleton() + .AsSelf() + .AsImplementedInterfaces(); + + services.Add() + .Singleton() + .AsSelf() + .AsImplementedInterfaces(); + + services.Add() + .Singleton() + .AsSelf() + .AsImplementedInterfaces(); + + services.RemoveServiceTypeExact>() // Mediatr registers handlers as Transient by default, this extension ensures these aren't still there, only needed when service != Transient + .Add() + .Singleton() + .AsSelf() + .AsImplementedInterfaces(); + + services.AddTransient(typeof(IPipelineBehavior), typeof(CreateOrUpdateSubscriptionBehavior)); + services.AddTransient(typeof(IPipelineBehavior), typeof(CreateOrUpdateSubscriptionBehavior)); } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Validation/CreateOrUpdateSubscriptionBehavior.cs b/src/Microsoft.Health.Fhir.Subscriptions/Validation/CreateOrUpdateSubscriptionBehavior.cs new file mode 100644 index 0000000000..8cdac279c6 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Validation/CreateOrUpdateSubscriptionBehavior.cs @@ -0,0 +1,62 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using EnsureThat; +using MediatR; +using Microsoft.Health.Fhir.Core.Features.Persistence; +using Microsoft.Health.Fhir.Core.Messages.Create; +using Microsoft.Health.Fhir.Core.Messages.Upsert; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Subscriptions.Validation; + +namespace Microsoft.Health.Fhir.Core.Features.Subscriptions +{ + public class CreateOrUpdateSubscriptionBehavior : IPipelineBehavior, + IPipelineBehavior + { + private ISubscriptionValidator _subscriptionValidator; + private IFhirDataStore _fhirDataStore; + + public CreateOrUpdateSubscriptionBehavior(ISubscriptionValidator subscriptionValidator, IFhirDataStore fhirDataStore) + { + EnsureArg.IsNotNull(subscriptionValidator, nameof(subscriptionValidator)); + EnsureArg.IsNotNull(fhirDataStore, nameof(fhirDataStore)); + + _subscriptionValidator = subscriptionValidator; + _fhirDataStore = fhirDataStore; + } + + public async Task Handle(CreateResourceRequest request, RequestHandlerDelegate next, CancellationToken cancellationToken) + { + if (request.Resource.InstanceType.Equals(KnownResourceTypes.Subscription, StringComparison.Ordinal)) + { + request.Resource = await _subscriptionValidator.ValidateSubscriptionInput(request.Resource, cancellationToken); + } + + // Allow the resource to be updated with the normal handler + return await next(); + } + + public async Task Handle(UpsertResourceRequest request, RequestHandlerDelegate next, CancellationToken cancellationToken) + { + // if the resource type being updated is a SearchParameter, then we want to query the previous version before it is changed + // because we will need to the Url property to update the definition in the SearchParameterDefinitionManager + // and the user could be changing the Url as part of this update + if (request.Resource.InstanceType.Equals(KnownResourceTypes.Subscription, StringComparison.Ordinal)) + { + request.Resource = await _subscriptionValidator.ValidateSubscriptionInput(request.Resource, cancellationToken); + } + + // Now allow the resource to updated per the normal behavior + return await next(); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Validation/ISubscriptionValidator.cs b/src/Microsoft.Health.Fhir.Subscriptions/Validation/ISubscriptionValidator.cs new file mode 100644 index 0000000000..3f330beaf0 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Validation/ISubscriptionValidator.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Validation +{ + public interface ISubscriptionValidator + { + Task ValidateSubscriptionInput(ResourceElement subscription, CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionException.cs b/src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionException.cs new file mode 100644 index 0000000000..d9396cc0d8 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionException.cs @@ -0,0 +1,27 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.Health.Abstractions.Exceptions; + +namespace Microsoft.Health.Fhir.Subscriptions.Validation +{ + public class SubscriptionException : MicrosoftHealthException + { + public SubscriptionException(string message) + : base(message) + { + } + + public SubscriptionException(string message, Exception innerException) + : base(message, innerException) + { + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionValidator.cs b/src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionValidator.cs new file mode 100644 index 0000000000..323ec30b87 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionValidator.cs @@ -0,0 +1,81 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FluentValidation.Results; +using Hl7.Fhir.Model; +using Hl7.Fhir.Rest; +using Hl7.Fhir.Utility; +using Microsoft.Extensions.Logging; +using Microsoft.Health.Core.Features.Security.Authorization; +using Microsoft.Health.Fhir.Core; +using Microsoft.Health.Fhir.Core.Exceptions; +using Microsoft.Health.Fhir.Core.Features.Security; +using Microsoft.Health.Fhir.Core.Features.Subscriptions; +using Microsoft.Health.Fhir.Core.Features.Validation; +using Microsoft.Health.Fhir.Core.Models; +using Microsoft.Health.Fhir.Subscriptions.Channels; +using Microsoft.Health.Fhir.Subscriptions.Models; + +namespace Microsoft.Health.Fhir.Subscriptions.Validation +{ + public class SubscriptionValidator : ISubscriptionValidator + { + private readonly ILogger _logger; + private readonly ISubscriptionModelConverter _subscriptionModelConverter; + private readonly StorageChannelFactory _subscriptionChannelFactory; + private readonly ISubscriptionUpdator _subscriptionUpdator; + + public SubscriptionValidator(ILogger logger, ISubscriptionModelConverter subscriptionModelConverter, StorageChannelFactory subscriptionChannelFactory, ISubscriptionUpdator subscriptionUpdator) + { + _logger = logger; + _subscriptionModelConverter = subscriptionModelConverter; + _subscriptionChannelFactory = subscriptionChannelFactory; + _subscriptionUpdator = subscriptionUpdator; + } + + public async Task ValidateSubscriptionInput(ResourceElement subscription, CancellationToken cancellationToken) + { + SubscriptionInfo subscriptionInfo = _subscriptionModelConverter.Convert(subscription); + + var validationFailures = new List(); + + if (subscriptionInfo.Channel.ChannelType.Equals(SubscriptionChannelType.None)) + { + _logger.LogInformation("Subscription channel type is not valid."); + validationFailures.Add( + new ValidationFailure(nameof(subscriptionInfo.Channel.ChannelType), "Subscription channel type is not valid.")); + } + + if (!subscriptionInfo.Status.Equals(SubscriptionStatus.Off)) + { + try + { + var subscriptionChannel = _subscriptionChannelFactory.Create(subscriptionInfo.Channel.ChannelType); + await subscriptionChannel.PublishHandShakeAsync(subscriptionInfo); + subscription = _subscriptionUpdator.UpdateStatus(subscription, SubscriptionStatus.Active.GetLiteral()); + } + catch (SubscriptionException) + { + _logger.LogInformation("Subscription endpoint is not valid."); + validationFailures.Add( + new ValidationFailure(nameof(subscriptionInfo.Channel.Endpoint), "Subscription endpoint is not valid.")); + subscription = _subscriptionUpdator.UpdateStatus(subscription, SubscriptionStatus.Error.GetLiteral()); + } + } + + if (validationFailures.Any()) + { + throw new ResourceNotValidException(validationFailures); + } + + return subscription; + } + } +} From bd063b89eaddd266d1159965ebf8762979c9f4ef Mon Sep 17 00:00:00 2001 From: aponakampalli Date: Wed, 28 Aug 2024 14:55:05 -0700 Subject: [PATCH 42/47] remove duplicate tests --- .../Peristence/SubscriptionManagerTests.cs | 49 ------------------- 1 file changed, 49 deletions(-) delete mode 100644 src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionManagerTests.cs diff --git a/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionManagerTests.cs b/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionManagerTests.cs deleted file mode 100644 index 253e151730..0000000000 --- a/src/Microsoft.Health.Fhir.Subscriptions.Tests/Peristence/SubscriptionManagerTests.cs +++ /dev/null @@ -1,49 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Hl7.Fhir.ElementModel; -using Hl7.Fhir.Model; -using Hl7.Fhir.Serialization; -using Microsoft.Health.Fhir.Core.Models; -using Microsoft.Health.Fhir.Subscriptions.Models; -using Microsoft.Health.Fhir.Subscriptions.Persistence; -using Microsoft.Health.Fhir.Tests.Common; -using Microsoft.Health.Test.Utilities; -using Xunit; - -namespace Microsoft.Health.Fhir.Subscriptions.Tests.Peristence -{ - public class SubscriptionManagerTests - { - private IModelInfoProvider _modelInfo; - - public SubscriptionManagerTests() - { - _modelInfo = MockModelInfoProviderBuilder - .Create(FhirSpecification.R4) - .AddKnownTypes(KnownResourceTypes.Subscription) - .Build(); - } - - [Fact] - public void GivenAnR4BackportSubscription_WhenConvertingToInfo_ThenTheInformationIsCorrect() - { - var subscription = CommonSamples.GetJsonSample("Subscription-Backport", FhirSpecification.R4, s => s.ToTypedElement(ModelInfo.ModelInspector)); - - var info = SubscriptionManager.ConvertToInfo(subscription); - - Assert.Equal("Patient", info.FilterCriteria); - Assert.Equal("sync-all", info.Channel.Endpoint); - Assert.Equal(20, info.Channel.MaxCount); - Assert.Equal(SubscriptionContentType.FullResource, info.Channel.ContentType); - Assert.Equal(SubscriptionChannelType.Storage, info.Channel.ChannelType); - } - } -} From 42438f2d18ad5e8e09e26a20531e1b455fb8990a Mon Sep 17 00:00:00 2001 From: aponakampalli Date: Thu, 29 Aug 2024 10:18:26 -0700 Subject: [PATCH 43/47] remove duplicates in share core project items --- .../Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems index 7a41e79291..042a6bd5f6 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Microsoft.Health.Fhir.Shared.Core.UnitTests.projitems @@ -48,13 +48,12 @@ - - + @@ -91,7 +90,7 @@ - + From 4e4f3d7e92dd8e769a1405cbcd047ab3b74802e6 Mon Sep 17 00:00:00 2001 From: aponakampalli Date: Thu, 29 Aug 2024 11:58:16 -0700 Subject: [PATCH 44/47] add feature flag to subscription module --- .../Features/Search/BundleFactory.cs | 2 +- .../Registration/SubscriptionsModule.cs | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/BundleFactory.cs b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/BundleFactory.cs index 49144ab5da..498535c64d 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/BundleFactory.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core/Features/Search/BundleFactory.cs @@ -264,7 +264,7 @@ public async System.Threading.Tasks.Task CreateSubscriptionBundleAsync(p Request = new Bundle.RequestComponent { Method = httpVerb, - Url = $"{rw.ResourceTypeName}/{(httpVerb == Bundle.HTTPVerb.POST ? null : rw.ResourceId)}", + Url = $"{rw.ResourceTypeName}/{(httpVerb == Bundle.HTTPVerb.POST ? rw.ResourceId : null)}", }, }; diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs index 69088c03f8..55178f90a3 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs @@ -8,11 +8,13 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using EnsureThat; using MediatR; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Hosting; using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Subscriptions; @@ -31,8 +33,20 @@ namespace Microsoft.Health.Fhir.Subscriptions.Registration { public class SubscriptionsModule : IStartupModule { + private readonly CoreFeatureConfiguration _coreFeatureConfiguration; + + public SubscriptionsModule(CoreFeatureConfiguration coreFeatureConfiguration) + { + _coreFeatureConfiguration = EnsureArg.IsNotNull(coreFeatureConfiguration, nameof(coreFeatureConfiguration)); + } + public void Load(IServiceCollection services) { + if (!_coreFeatureConfiguration.SupportsSubscriptions) + { + return; + } + IEnumerable jobs = services.TypesInSameAssemblyAs() .AssignableTo() .Transient() From 08ff8b2b11aab8940cb65e6e4a78b5b484303e5f Mon Sep 17 00:00:00 2001 From: aponakampalli Date: Fri, 30 Aug 2024 10:43:03 -0700 Subject: [PATCH 45/47] set optional http context on publish notification and heartbeat --- docs/rest/Subscriptions.http | 61 +++++++++++++++++++ .../FhirServerServiceCollectionExtensions.cs | 2 +- .../appsettings.json | 2 +- .../Channels/RestHookChannel.cs | 50 ++++++++------- 4 files changed, 92 insertions(+), 23 deletions(-) diff --git a/docs/rest/Subscriptions.http b/docs/rest/Subscriptions.http index cab4f1c4ae..577de722df 100644 --- a/docs/rest/Subscriptions.http +++ b/docs/rest/Subscriptions.http @@ -71,6 +71,62 @@ Authorization: Bearer {{bearer.response.body.access_token}} } } +### PUT Subscription for REST Hook +PUT https://{{hostname}}/Subscription/example-rest-hook-patient +content-type: application/json +Authorization: Bearer {{bearer.response.body.access_token}} + +{ + "resourceType": "Subscription", + "id": "example-rest-hook-patient", + "meta" : { + "profile": ["http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-subscription"] + }, + "status": "requested", + "end": "2031-01-01T12:00:00", + "reason": "Test subscription based on transactions, filtered by Patient", + "criteria" : "http://azurehealthcareapis.com/data-extentions/SubscriptionTopics/transactions", + "_criteria": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-filter-criteria", + "valueString": "Patient" + } + ] + }, + "channel": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-heartbeat-period", + "valueInteger": 120 + } + ], + "type" : "rest-hook", + "endpoint": "https://subscriptions.argo.run/fhir/r4/$subscription-hook", + "_type" : { + "extension" : [ + { + "url" : "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-channel-type", + "valueCoding" : { + "system" : "http://azurehealthcareapis.com/data-extentions/subscription-channel-type", + "code" : "rest-hook", + "display" : "Rest Hook" + } + } + ] + }, + "payload": "application/fhir+json", + "_payload": { + "extension": [ + { + "url": "http://hl7.org/fhir/uv/subscriptions-backport/StructureDefinition/backport-payload-content", + "valueCode": "full-resource" + } + ] + } + } +} + ### PUT Subscription for Blob Storage ## More examples: https://build.fhir.org/ig/HL7/fhir-subscription-backport-ig/artifacts.html PUT https://{{hostname}}/Subscription/example-backport-storage-all @@ -186,4 +242,9 @@ Authorization: Bearer {{bearer.response.body.access_token}} ### DELETE https://{{hostname}}/Subscription/example-backport-lake content-type: application/json +Authorization: Bearer {{bearer.response.body.access_token}} + +### +DELETE https://{{hostname}}/Subscription/example-rest-hook-patient +content-type: application/json Authorization: Bearer {{bearer.response.body.access_token}} \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Api/Registration/FhirServerServiceCollectionExtensions.cs b/src/Microsoft.Health.Fhir.Api/Registration/FhirServerServiceCollectionExtensions.cs index d22a5b88dd..bbc239478d 100644 --- a/src/Microsoft.Health.Fhir.Api/Registration/FhirServerServiceCollectionExtensions.cs +++ b/src/Microsoft.Health.Fhir.Api/Registration/FhirServerServiceCollectionExtensions.cs @@ -21,7 +21,7 @@ public static IServiceCollection AddFhirServerBase(this IServiceCollection servi services.RegisterAssemblyModules(Assembly.GetExecutingAssembly(), fhirServerConfiguration); services.RegisterAssemblyModules(typeof(InitializationModule).Assembly, fhirServerConfiguration); - services.RegisterAssemblyModules(typeof(SubscriptionsModule).Assembly, fhirServerConfiguration); + services.RegisterAssemblyModules(typeof(SubscriptionsModule).Assembly, fhirServerConfiguration.CoreFeatures); return services; } diff --git a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json index 6a8add7b32..78e631471c 100644 --- a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json +++ b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json @@ -30,7 +30,7 @@ "ProfileValidationOnUpdate": false, "SupportsResourceChangeCapture": false, "SupportsBulkDelete": true, - "SupportsSubscriptions": true, + "SupportsSubscriptions": false, "Versioning": { "Default": "versioned", "ResourceTypeOverrides": null diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/RestHookChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/RestHookChannel.cs index 68d06e6320..e4b2b12306 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/RestHookChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/RestHookChannel.cs @@ -53,34 +53,14 @@ public RestHookChannel(ILogger logger, HttpClient httpClient, I _modelInfoProvider = modelInfoProvider; _urlResolver = urlResolver; - var fhirRequestContext = new FhirRequestContext( - method: null, - uriString: string.Empty, - baseUriString: string.Empty, - correlationId: string.Empty, - requestHeaders: new Dictionary(), - responseHeaders: new Dictionary()) - { - IsBackgroundTask = true, - AuditEventType = OperationsConstants.Reindex, - }; - _contextAccessor = contextAccessor; - _contextAccessor.RequestContext = fhirRequestContext; _httpContextAccessor = httpContextAccessor; _actionContextAccessor = actionContextAccessor; - - var httpContext = new DefaultHttpContext(); - httpContext.Request.Scheme = "https"; - httpContext.Request.Host = new HostString("fhir.com", 433); - _httpContextAccessor.HttpContext = httpContext; - - _actionContextAccessor.ActionContext = new AspNetCore.Mvc.ActionContext(); - _actionContextAccessor.ActionContext.HttpContext = httpContext; } public async Task PublishAsync(IReadOnlyCollection resources, SubscriptionInfo subscriptionInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) { + SetContext(); List resourceWrappers = new List(); var parameter = new Parameters { @@ -154,6 +134,7 @@ public async Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo) public async Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo) { + SetContext(); List resourceWrappers = new List(); var parameter = new Parameters { @@ -173,6 +154,33 @@ public async Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo) await SendPayload(subscriptionInfo.Channel, bundle); } + private void SetContext() + { + if (_httpContextAccessor.HttpContext == null) + { + var fhirRequestContext = new FhirRequestContext( + method: "subscription", + uriString: "subscription", + baseUriString: "subscription", + correlationId: "subscription", + requestHeaders: new Dictionary(), + responseHeaders: new Dictionary()) + { + IsBackgroundTask = true, + AuditEventType = OperationsConstants.Reindex, + }; + _contextAccessor.RequestContext = fhirRequestContext; + + var httpContext = new DefaultHttpContext(); + httpContext.Request.Scheme = "https"; + httpContext.Request.Host = new HostString("fhir.com", 433); + _httpContextAccessor.HttpContext = httpContext; + + _actionContextAccessor.ActionContext = new AspNetCore.Mvc.ActionContext(); + _actionContextAccessor.ActionContext.HttpContext = httpContext; + } + } + private async Task SendPayload( ChannelInfo chanelInfo, string contents) From dc0d6c99b5125ae9dc18c7257d848f66bb50a94f Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Wed, 18 Sep 2024 17:28:35 -0700 Subject: [PATCH 46/47] Small cleanup --- Microsoft.Health.Fhir.sln | 16 +-- R4.slnf | 4 +- .../Operations/OperationsConstants.cs | 2 + .../Persistence/ITransactionDataStore.cs | 3 +- .../InMemory/SearchQueryInterperaterTests.cs | 3 + .../SubscriptionsOrchestratorJobTests.cs | 3 + .../Storage/SqlServerFhirDataStore.cs | 2 - .../SubscriptionProcessorWatchdog.cs | 2 - .../Watchdogs/WatchdogsBackgroundService.cs | 28 ++-- .../Microsoft.Health.Fhir.SqlServer.csproj | 2 +- .../Channels/DataLakeChannel.cs | 10 +- .../Channels/EventGridChannel.cs | 91 ------------ .../Channels/ISubscriptionChannel.cs | 4 +- .../Channels/RestHookChannel.cs | 69 ++++++---- .../Channels/StorageChannel.cs | 14 +- ...ctory.cs => SubscriptionChannelFactory.cs} | 4 +- .../HeartBeats/HeartBeatBackgroundService.cs | 18 +-- ...Microsoft.Health.Fhir.Subscriptions.csproj | 15 ++ .../Operations/SubscriptionProcessingJob.cs | 4 +- .../Persistence/SubscriptionManager.cs | 21 ++- .../Persistence/SubscriptionUpdator.cs | 3 + .../Registration/SubscriptionsModule.cs | 11 +- .../Resources.Designer.cs | 90 ++++++++++++ .../Resources.resx | 129 ++++++++++++++++++ .../CreateOrUpdateSubscriptionBehavior.cs | 4 + .../Validation/SubscriptionException.cs | 3 + .../Validation/SubscriptionValidator.cs | 21 +-- .../Categories.cs | 2 + 28 files changed, 377 insertions(+), 201 deletions(-) rename src/{Microsoft.Health.Fhir.Subscriptions => Microsoft.Health.Fhir.Core/Features}/Persistence/ITransactionDataStore.cs (85%) delete mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs rename src/Microsoft.Health.Fhir.Subscriptions/Channels/{StorageChannelFactory.cs => SubscriptionChannelFactory.cs} (93%) create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Resources.Designer.cs create mode 100644 src/Microsoft.Health.Fhir.Subscriptions/Resources.resx diff --git a/Microsoft.Health.Fhir.sln b/Microsoft.Health.Fhir.sln index 0a0b88fedf..2a45776d6a 100644 --- a/Microsoft.Health.Fhir.sln +++ b/Microsoft.Health.Fhir.sln @@ -199,15 +199,15 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PerfTester", "tools\PerfTes EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SqlScriptRunner", "tools\SqlScriptRunner\SqlScriptRunner.csproj", "{76C29222-8D35-43A2-89C5-43114D113C10}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.CosmosDb.Initialization", "src\Microsoft.Health.Fhir.CosmosDb.Initialization\Microsoft.Health.Fhir.CosmosDb.Initialization.csproj", "{10661BC9-01B0-4E35-9751-3B5CE97E25C0}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.CosmosDb.Initialization", "src\Microsoft.Health.Fhir.CosmosDb.Initialization\Microsoft.Health.Fhir.CosmosDb.Initialization.csproj", "{10661BC9-01B0-4E35-9751-3B5CE97E25C0}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests", "src\Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests\Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj", "{B9AAA11D-8C8C-44C3-AADE-801376EF82F0}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests", "src\Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests\Microsoft.Health.Fhir.CosmosDb.Initialization.UnitTests.csproj", "{B9AAA11D-8C8C-44C3-AADE-801376EF82F0}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.CosmosDb.Core", "src\Microsoft.Health.Fhir.CosmosDb.Core\Microsoft.Health.Fhir.CosmosDb.Core.csproj", "{1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.CosmosDb.Core", "src\Microsoft.Health.Fhir.CosmosDb.Core\Microsoft.Health.Fhir.CosmosDb.Core.csproj", "{1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Subscriptions", "src\Microsoft.Health.Fhir.Subscriptions\Microsoft.Health.Fhir.Subscriptions.csproj", "{BD8F3137-89F5-4EE5-B269-24D73081E00A}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.Subscriptions", "src\Microsoft.Health.Fhir.Subscriptions\Microsoft.Health.Fhir.Subscriptions.csproj", "{BD8F3137-89F5-4EE5-B269-24D73081E00A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Health.Fhir.Subscriptions.Tests", "src\Microsoft.Health.Fhir.Subscriptions.Tests\Microsoft.Health.Fhir.Subscriptions.Tests.csproj", "{AA73AB9D-52EF-4172-9911-3C9D661C8D48}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Health.Fhir.Subscriptions.Tests", "src\Microsoft.Health.Fhir.Subscriptions.Tests\Microsoft.Health.Fhir.Subscriptions.Tests.csproj", "{AA73AB9D-52EF-4172-9911-3C9D661C8D48}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -590,12 +590,12 @@ Global {10661BC9-01B0-4E35-9751-3B5CE97E25C0} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} {B9AAA11D-8C8C-44C3-AADE-801376EF82F0} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} {1CD46DC5-6022-4BBE-9A1C-6B13C3CEFC75} = {DC5A2CB1-8995-4D39-97FE-3CE80E892C69} - {BD8F3137-89F5-4EE5-B269-24D73081E00A} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4} - {AA73AB9D-52EF-4172-9911-3C9D661C8D48} = {38B3BA4A-3510-4615-BCC4-4C9B96A486C4} + {BD8F3137-89F5-4EE5-B269-24D73081E00A} = {7457B218-2651-49B5-BED8-22233889516A} + {AA73AB9D-52EF-4172-9911-3C9D661C8D48} = {7457B218-2651-49B5-BED8-22233889516A} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution - RESX_SortFileContentOnSave = True SolutionGuid = {E370FB31-CF95-47D1-B1E1-863A77973FF8} + RESX_SortFileContentOnSave = True EndGlobalSection GlobalSection(SharedMSBuildProjectFiles) = preSolution test\Microsoft.Health.Fhir.Shared.Tests.E2E.Common\Microsoft.Health.Fhir.Shared.Tests.E2E.Common.projitems*{0478b687-7105-40f6-a2dc-81057890e944}*SharedItemsImports = 13 diff --git a/R4.slnf b/R4.slnf index bb8a55c269..a8658c313a 100644 --- a/R4.slnf +++ b/R4.slnf @@ -29,12 +29,12 @@ "src\\Microsoft.Health.Fhir.Shared.Web\\Microsoft.Health.Fhir.Shared.Web.shproj", "src\\Microsoft.Health.Fhir.SqlServer.UnitTests\\Microsoft.Health.Fhir.SqlServer.UnitTests.csproj", "src\\Microsoft.Health.Fhir.SqlServer\\Microsoft.Health.Fhir.SqlServer.csproj", + "src\\Microsoft.Health.Fhir.Subscriptions.Tests\\Microsoft.Health.Fhir.Subscriptions.Tests.csproj", + "src\\Microsoft.Health.Fhir.Subscriptions\\Microsoft.Health.Fhir.Subscriptions.csproj", "src\\Microsoft.Health.Fhir.Tests.Common\\Microsoft.Health.Fhir.Tests.Common.csproj", "src\\Microsoft.Health.TaskManagement\\Microsoft.Health.TaskManagement.csproj", "src\\Microsoft.Health.TaskManagement.UnitTests\\Microsoft.Health.TaskManagement.UnitTests.csproj", "src\\Microsoft.Health.Fhir.ValueSets\\Microsoft.Health.Fhir.ValueSets.csproj", - "src\\Microsoft.Health.TaskManagement.UnitTests\\Microsoft.Health.TaskManagement.UnitTests.csproj", - "src\\Microsoft.Health.TaskManagement\\Microsoft.Health.TaskManagement.csproj", "test\\Microsoft.Health.Fhir.R4.Tests.E2E\\Microsoft.Health.Fhir.R4.Tests.E2E.csproj", "test\\Microsoft.Health.Fhir.R4.Tests.Integration\\Microsoft.Health.Fhir.R4.Tests.Integration.csproj", "test\\Microsoft.Health.Fhir.Shared.Tests.Crucible\\Microsoft.Health.Fhir.Shared.Tests.Crucible.shproj", diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/OperationsConstants.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/OperationsConstants.cs index af6b2d1858..1801df33ee 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/OperationsConstants.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/OperationsConstants.cs @@ -46,5 +46,7 @@ public static class OperationsConstants public const string ResourceTypeBulkDelete = "resource-type-bulk-delete"; public const string BulkDeleteSoftDeleted = "bulk-delete-soft-deleted"; + + public const string Subscription = "subscription"; } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ITransactionDataStore.cs similarity index 85% rename from src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs rename to src/Microsoft.Health.Fhir.Core/Features/Persistence/ITransactionDataStore.cs index 52d5cf3223..24d65ead13 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/ITransactionDataStore.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Persistence/ITransactionDataStore.cs @@ -6,9 +6,8 @@ using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using Microsoft.Health.Fhir.Core.Features.Persistence; -namespace Microsoft.Health.Fhir.Subscriptions.Persistence +namespace Microsoft.Health.Fhir.Core.Features.Persistence { public interface ITransactionDataStore { diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs index 2d96d2494a..2b2e638d7d 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SearchQueryInterperaterTests.cs @@ -19,11 +19,14 @@ using Microsoft.Health.Fhir.Core.Features.Search.SearchValues; using Microsoft.Health.Fhir.Core.Models; using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; using NSubstitute; using Xunit; namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.InMemory { + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Subscriptions)] public class SearchQueryInterperaterTests : IAsyncLifetime { private ExpressionParser _expressionParser; diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs index 7ee968cede..75fa5984ed 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs @@ -34,6 +34,7 @@ using Microsoft.Health.Fhir.Subscriptions.Persistence; using Microsoft.Health.Fhir.Tests.Common; using Microsoft.Health.JobManagement; +using Microsoft.Health.Test.Utilities; using Newtonsoft.Json; using NSubstitute; using NSubstitute.ReceivedExtensions; @@ -41,6 +42,8 @@ namespace Microsoft.Health.Fhir.Core.UnitTests.Features.Search.InMemory { + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Subscriptions)] public class SubscriptionsOrchestratorJobTests : IAsyncLifetime { private ISearchIndexer _searchIndexer; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs index d2352e63de..2ca971401f 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs @@ -12,7 +12,6 @@ using System.Threading; using System.Threading.Tasks; using EnsureThat; -using Hl7.FhirPath.Sprache; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -29,7 +28,6 @@ using Microsoft.Health.Fhir.SqlServer.Features.Schema.Model; using Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration; using Microsoft.Health.Fhir.SqlServer.Features.Storage.TvpRowGeneration.Merge; -using Microsoft.Health.Fhir.Subscriptions.Persistence; using Microsoft.Health.Fhir.ValueSets; using Microsoft.Health.SqlServer.Features.Client; using Microsoft.Health.SqlServer.Features.Schema; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs index 5e40f22de3..1335d94e31 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/SubscriptionProcessorWatchdog.cs @@ -3,10 +3,8 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System; using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using EnsureThat; diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs index bc269ce08a..463db519d4 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/WatchdogsBackgroundService.cs @@ -10,7 +10,9 @@ using EnsureThat; using MediatR; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Configs; using Microsoft.Health.Fhir.Core.Extensions; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Messages.Storage; @@ -19,25 +21,28 @@ namespace Microsoft.Health.Fhir.SqlServer.Features.Watchdogs { internal class WatchdogsBackgroundService : BackgroundService, INotificationHandler { + private readonly CoreFeatureConfiguration _featureConfig; private bool _storageReady = false; private readonly DefragWatchdog _defragWatchdog; private readonly CleanupEventLogWatchdog _cleanupEventLogWatchdog; - private readonly IScoped _transactionWatchdog; + private readonly IScopeProvider _transactionWatchdogProvider; private readonly InvisibleHistoryCleanupWatchdog _invisibleHistoryCleanupWatchdog; - private readonly SubscriptionProcessorWatchdog _subscriptionProcessorWatchdog; + private readonly Lazy _subscriptionProcessorWatchdog; public WatchdogsBackgroundService( DefragWatchdog defragWatchdog, CleanupEventLogWatchdog cleanupEventLogWatchdog, IScopeProvider transactionWatchdog, InvisibleHistoryCleanupWatchdog invisibleHistoryCleanupWatchdog, - SubscriptionProcessorWatchdog subscriptionProcessorWatchdog) + Lazy subscriptionProcessorWatchdog, + IOptions featureConfig) { _defragWatchdog = EnsureArg.IsNotNull(defragWatchdog, nameof(defragWatchdog)); _cleanupEventLogWatchdog = EnsureArg.IsNotNull(cleanupEventLogWatchdog, nameof(cleanupEventLogWatchdog)); - _transactionWatchdog = EnsureArg.IsNotNull(transactionWatchdog, nameof(transactionWatchdog)).Invoke(); + _transactionWatchdogProvider = EnsureArg.IsNotNull(transactionWatchdog, nameof(transactionWatchdog)); _invisibleHistoryCleanupWatchdog = EnsureArg.IsNotNull(invisibleHistoryCleanupWatchdog, nameof(invisibleHistoryCleanupWatchdog)); _subscriptionProcessorWatchdog = EnsureArg.IsNotNull(subscriptionProcessorWatchdog, nameof(subscriptionProcessorWatchdog)); + _featureConfig = EnsureArg.IsNotNull(featureConfig?.Value, nameof(featureConfig)); } protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -49,16 +54,21 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } using var continuationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(stoppingToken); + using IScoped transactionWatchdog = _transactionWatchdogProvider.Invoke(); var tasks = new List { _defragWatchdog.ExecuteAsync(continuationTokenSource.Token), _cleanupEventLogWatchdog.ExecuteAsync(continuationTokenSource.Token), - _transactionWatchdog.Value.ExecuteAsync(continuationTokenSource.Token), + transactionWatchdog.Value.ExecuteAsync(continuationTokenSource.Token), _invisibleHistoryCleanupWatchdog.ExecuteAsync(continuationTokenSource.Token), - _subscriptionProcessorWatchdog.ExecuteAsync(continuationTokenSource.Token), }; + if (_featureConfig.SupportsSubscriptions) + { + tasks.Add(_subscriptionProcessorWatchdog.Value.ExecuteAsync(continuationTokenSource.Token)); + } + await Task.WhenAny(tasks); if (!stoppingToken.IsCancellationRequested) @@ -75,11 +85,5 @@ public Task Handle(StorageInitializedNotification notification, CancellationToke _storageReady = true; return Task.CompletedTask; } - - public override void Dispose() - { - _transactionWatchdog.Dispose(); - base.Dispose(); - } } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj index fffb1ebd3a..c51a65788b 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj +++ b/src/Microsoft.Health.Fhir.SqlServer/Microsoft.Health.Fhir.SqlServer.csproj @@ -75,7 +75,7 @@ - + diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs index 1e7a21a80c..0868074f9b 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/DataLakeChannel.cs @@ -44,13 +44,7 @@ public async Task PublishAsync(IReadOnlyCollection resources, S { string json = item.RawResource.Data; - /* // TODO: Add logic to handle soft-deleted resources. - if (item.IsDeleted) - { - ResourceElement element = _resourceDeserializer.Deserialize(item); - } - */ _exportDestinationClient.WriteFilePart(blobName, json); } @@ -64,12 +58,12 @@ public async Task PublishAsync(IReadOnlyCollection resources, S } } - public Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo) + public Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo, CancellationToken cancellationToken) { return Task.CompletedTask; } - public Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo) + public Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo, CancellationToken cancellationToken) { return Task.CompletedTask; } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs deleted file mode 100644 index 6abe4d773c..0000000000 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/EventGridChannel.cs +++ /dev/null @@ -1,91 +0,0 @@ -// ------------------------------------------------------------------------------------------------- -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. -// ------------------------------------------------------------------------------------------------- - -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Azure.Messaging.EventGrid; -using EnsureThat; -using Microsoft.Health.Fhir.Core.Features.Persistence; -using Microsoft.Health.Fhir.Subscriptions.Models; - -namespace Microsoft.Health.Fhir.Subscriptions.Channels -{ - [ChannelType(SubscriptionChannelType.EventGrid)] - public class EventGridChannel : ISubscriptionChannel - { - public EventGridChannel() - { - } - - public Task PublishAsync(IReadOnlyCollection resources, SubscriptionInfo subscriptionInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) - { - return Task.CompletedTask; - } - - public Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo) - { - return Task.CompletedTask; - } - - public Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo) - { - return Task.CompletedTask; - } - - /* - public EventGridEvent CreateEventGridEvent(ResourceWrapper rcd) - { - EnsureArg.IsNotNull(rcd); - - string resourceId = rcd.ResourceId; - string resourceTypeName = rcd.ResourceTypeName; - string resourceVersion = rcd.Version; - string dataVersion = resourceVersion.ToString(CultureInfo.InvariantCulture); - string fhirAccountDomainName = _workerConfiguration.FhirAccount; - - string eventSubject = GetEventSubject(rcd); - string eventType = _workerConfiguration.ResourceChangeTypeMap[rcd.ResourceChangeTypeId]; - string eventGuid = rcd.GetSha256BasedGuid(); - - // The swagger specification requires the response JSON to have all properties use camelcasing - // and hence the dataPayload properties below have to use camelcase. - var dataPayload = new BinaryData(new - { - resourceType = resourceTypeName, - resourceFhirAccount = fhirAccountDomainName, - resourceFhirId = resourceId, - resourceVersionId = resourceVersion, - }); - - return new EventGridEvent( - subject: eventSubject, - eventType: eventType, - dataVersion: dataVersion, - data: dataPayload) - { - Topic = _workerConfiguration.EventGridTopic, - Id = eventGuid, - EventTime = rcd.Timestamp, - }; - } - - public string GetEventSubject(ResourceChangeData rcd) - { - EnsureArg.IsNotNull(rcd); - - // Example: "myfhirserver.contoso.com/Observation/cb875194-1195-4617-b2e9-0966bd6b8a10" - var fhirAccountDomainName = "fhirevents"; - var subjectSegements = new string[] { fhirAccountDomainName, rcd.ResourceTypeName, rcd.ResourceId }; - var subject = string.Join("/", subjectSegements); - return subject; - } - */ - } -} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs index 6d61284b94..5ea6383d37 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/ISubscriptionChannel.cs @@ -16,8 +16,8 @@ public interface ISubscriptionChannel { Task PublishAsync(IReadOnlyCollection resources, SubscriptionInfo subscriptionInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken); - Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo); + Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo, CancellationToken cancellationToken); - Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo); + Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo, CancellationToken cancellationToken); } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/RestHookChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/RestHookChannel.cs index e4b2b12306..98f7fe45a5 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/RestHookChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/RestHookChannel.cs @@ -9,6 +9,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using EnsureThat; using Hl7.Fhir.Model; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc.Infrastructure; @@ -27,10 +28,7 @@ namespace Microsoft.Health.Fhir.Subscriptions.Channels { [ChannelType(SubscriptionChannelType.RestHook)] - - #pragma warning disable CA1001 // Types that own disposable fields should be disposable - public class RestHookChannel : ISubscriptionChannel - #pragma warning restore CA1001 // Types that own disposable fields should be disposable + public sealed class RestHookChannel : ISubscriptionChannel, IDisposable { private readonly ILogger _logger; private readonly IBundleFactory _bundleFactory; @@ -42,20 +40,29 @@ public class RestHookChannel : ISubscriptionChannel private readonly IHttpContextAccessor _httpContextAccessor; private readonly IActionContextAccessor _actionContextAccessor; - public RestHookChannel(ILogger logger, HttpClient httpClient, IBundleFactory bundleFactory, IRawResourceFactory rawResourceFactory, IModelInfoProvider modelInfoProvider, IUrlResolver urlResolver, RequestContextAccessor contextAccessor, IHttpContextAccessor httpContextAccessor, IActionContextAccessor actionContextAccessor) + [System.Diagnostics.CodeAnalysis.SuppressMessage("Reliability", "CA2000:Dispose objects before losing scope", Justification = "HttpClientHandler is disposed by HttpClient.")] + public RestHookChannel( + ILogger logger, + IBundleFactory bundleFactory, + IRawResourceFactory rawResourceFactory, + IModelInfoProvider modelInfoProvider, + IUrlResolver urlResolver, + RequestContextAccessor contextAccessor, + IHttpContextAccessor httpContextAccessor, + IActionContextAccessor actionContextAccessor) { - _logger = logger; -#pragma warning disable CA2000 // Dispose objects before losing scope - _httpClient = new HttpClient(new HttpClientHandler() { CheckCertificateRevocationList = true }, disposeHandler: true); -#pragma warning restore CA2000 // Dispose objects before losing scope - _bundleFactory = bundleFactory; - _rawResourceFactory = rawResourceFactory; - _modelInfoProvider = modelInfoProvider; - _urlResolver = urlResolver; - - _contextAccessor = contextAccessor; - _httpContextAccessor = httpContextAccessor; - _actionContextAccessor = actionContextAccessor; + _logger = EnsureArg.IsNotNull(logger, nameof(logger)); + _bundleFactory = EnsureArg.IsNotNull(bundleFactory, nameof(bundleFactory)); + _rawResourceFactory = EnsureArg.IsNotNull(rawResourceFactory, nameof(rawResourceFactory)); + _modelInfoProvider = EnsureArg.IsNotNull(modelInfoProvider, nameof(modelInfoProvider)); + _urlResolver = EnsureArg.IsNotNull(urlResolver, nameof(urlResolver)); + + _contextAccessor = EnsureArg.IsNotNull(contextAccessor, nameof(contextAccessor)); + _httpContextAccessor = EnsureArg.IsNotNull(httpContextAccessor, nameof(httpContextAccessor)); + _actionContextAccessor = EnsureArg.IsNotNull(actionContextAccessor, nameof(actionContextAccessor)); + + var handler = new HttpClientHandler() { CheckCertificateRevocationList = true }; + _httpClient = new HttpClient(handler, disposeHandler: true); } public async Task PublishAsync(IReadOnlyCollection resources, SubscriptionInfo subscriptionInfo, DateTimeOffset transactionTime, CancellationToken cancellationToken) @@ -70,7 +77,7 @@ public async Task PublishAsync(IReadOnlyCollection resources, S { "type", new Code("event-notification") }, }; - if (!subscriptionInfo.Channel.ContentType.Equals(SubscriptionContentType.Empty)) + if (subscriptionInfo.Channel.ContentType != SubscriptionContentType.Empty) { // add new fields to parameter object from subscription data foreach (ResourceWrapper rw in resources) @@ -108,10 +115,10 @@ public async Task PublishAsync(IReadOnlyCollection resources, S string bundle = await _bundleFactory.CreateSubscriptionBundleAsync(resourceWrappers.ToArray()); - await SendPayload(subscriptionInfo.Channel, bundle); + await SendPayload(subscriptionInfo.Channel, bundle, cancellationToken); } - public async Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo) + public async Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo, CancellationToken cancellationToken) { List resourceWrappers = new List(); var parameter = new Parameters @@ -129,10 +136,10 @@ public async Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo) string bundle = await _bundleFactory.CreateSubscriptionBundleAsync(resourceWrappers.ToArray()); - await SendPayload(subscriptionInfo.Channel, bundle); + await SendPayload(subscriptionInfo.Channel, bundle, cancellationToken); } - public async Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo) + public async Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo, CancellationToken cancellationToken) { SetContext(); List resourceWrappers = new List(); @@ -151,7 +158,7 @@ public async Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo) string bundle = await _bundleFactory.CreateSubscriptionBundleAsync(resourceWrappers.ToArray()); - await SendPayload(subscriptionInfo.Channel, bundle); + await SendPayload(subscriptionInfo.Channel, bundle, cancellationToken); } private void SetContext() @@ -167,13 +174,13 @@ private void SetContext() responseHeaders: new Dictionary()) { IsBackgroundTask = true, - AuditEventType = OperationsConstants.Reindex, + AuditEventType = OperationsConstants.Subscription, }; _contextAccessor.RequestContext = fhirRequestContext; var httpContext = new DefaultHttpContext(); httpContext.Request.Scheme = "https"; - httpContext.Request.Host = new HostString("fhir.com", 433); + httpContext.Request.Host = new HostString("fhir.azurehealthcareapis.com", 433); _httpContextAccessor.HttpContext = httpContext; _actionContextAccessor.ActionContext = new AspNetCore.Mvc.ActionContext(); @@ -183,7 +190,8 @@ private void SetContext() private async Task SendPayload( ChannelInfo chanelInfo, - string contents) + string contents, + CancellationToken cancellationToken) { HttpRequestMessage request = null!; @@ -199,7 +207,7 @@ private async Task SendPayload( }; // send our request - HttpResponseMessage response = await _httpClient.SendAsync(request); + HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); // check the status code if ((response.StatusCode != System.Net.HttpStatusCode.OK) && @@ -207,7 +215,7 @@ private async Task SendPayload( { // failure _logger.LogError($"REST POST to {chanelInfo.Endpoint} failed: {response.StatusCode}"); - throw new SubscriptionException("Subscription message invalid."); + throw new SubscriptionException(Resources.SubscriptionInvalid); } else { @@ -226,5 +234,10 @@ private async Task SendPayload( } } } + + public void Dispose() + { + _httpClient?.Dispose(); + } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs index 68fe68d423..a13a6fcea3 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannel.cs @@ -10,6 +10,7 @@ using Microsoft.Health.Fhir.Core.Features.Operations.Export.ExportDestinationClient; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Subscriptions.Models; +using Microsoft.Health.Fhir.Subscriptions.Validation; namespace Microsoft.Health.Fhir.Subscriptions.Channels { @@ -40,12 +41,19 @@ public async Task PublishAsync(IReadOnlyCollection resources, S } } - public Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo) + public async Task PublishHandShakeAsync(SubscriptionInfo subscriptionInfo, CancellationToken cancellationToken) { - return Task.CompletedTask; + try + { + await _exportDestinationClient.ConnectAsync(cancellationToken, subscriptionInfo.Channel.Endpoint); + } + catch (DestinationConnectionException ex) + { + throw new SubscriptionException(Resources.SubscriptionEndpointNotValid, ex); + } } - public Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo) + public Task PublishHeartBeatAsync(SubscriptionInfo subscriptionInfo, CancellationToken cancellationToken) { return Task.CompletedTask; } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannelFactory.cs b/src/Microsoft.Health.Fhir.Subscriptions/Channels/SubscriptionChannelFactory.cs similarity index 93% rename from src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannelFactory.cs rename to src/Microsoft.Health.Fhir.Subscriptions/Channels/SubscriptionChannelFactory.cs index 47bb1e1dfd..52b6095880 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Channels/StorageChannelFactory.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Channels/SubscriptionChannelFactory.cs @@ -13,12 +13,12 @@ namespace Microsoft.Health.Fhir.Subscriptions.Channels { - public class StorageChannelFactory + public class SubscriptionChannelFactory { private IServiceProvider _serviceProvider; private Dictionary _channelTypeMap; - public StorageChannelFactory(IServiceProvider serviceProvider) + public SubscriptionChannelFactory(IServiceProvider serviceProvider) { _serviceProvider = EnsureArg.IsNotNull(serviceProvider, nameof(serviceProvider)); diff --git a/src/Microsoft.Health.Fhir.Subscriptions/HeartBeats/HeartBeatBackgroundService.cs b/src/Microsoft.Health.Fhir.Subscriptions/HeartBeats/HeartBeatBackgroundService.cs index d0dc9ec26a..6891fbd7cb 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/HeartBeats/HeartBeatBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/HeartBeats/HeartBeatBackgroundService.cs @@ -9,6 +9,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using EnsureThat; using FluentValidation.Results; using MediatR; using Microsoft.Extensions.Hosting; @@ -29,13 +30,13 @@ public class HeartBeatBackgroundService : BackgroundService, INotificationHandle private bool _storageReady = false; private readonly ILogger _logger; private readonly IScopeProvider _subscriptionManager; - private readonly StorageChannelFactory _storageChannelFactory; + private readonly SubscriptionChannelFactory _storageChannelFactory; - public HeartBeatBackgroundService(ILogger logger, IScopeProvider subscriptionManager, StorageChannelFactory storageChannelFactory) + public HeartBeatBackgroundService(ILogger logger, IScopeProvider subscriptionManager, SubscriptionChannelFactory storageChannelFactory) { - _logger = logger; - _subscriptionManager = subscriptionManager; - _storageChannelFactory = storageChannelFactory; + _logger = EnsureArg.IsNotNull(logger, nameof(logger)); + _subscriptionManager = EnsureArg.IsNotNull(subscriptionManager, nameof(subscriptionManager)); + _storageChannelFactory = EnsureArg.IsNotNull(storageChannelFactory, nameof(storageChannelFactory)); } public Task Handle(StorageInitializedNotification notification, CancellationToken cancellationToken) @@ -52,6 +53,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); } + // TODO: Multi instance sync + using var periodicTimer = new PeriodicTimer(TimeSpan.FromSeconds(60)); var nextHeartBeat = new Dictionary(); @@ -68,7 +71,6 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) try { - // our logic using IScoped subscriptionManager = _subscriptionManager.Invoke(); await subscriptionManager.Value.SyncSubscriptionsAsync(stoppingToken); @@ -88,7 +90,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) var channel = _storageChannelFactory.Create(subscription.Channel.ChannelType); try { - await channel.PublishHeartBeatAsync(subscription); + await channel.PublishHeartBeatAsync(subscription, stoppingToken); nextHeartBeat[subscription.ResourceId] = nextHeartBeat.GetValueOrDefault(subscription.ResourceId).Add(subscription.Channel.HeartBeatPeriod); } catch (SubscriptionException) @@ -100,7 +102,7 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) } catch (Exception e) { - _logger.LogWarning(e, "Error executing timer"); + _logger.LogWarning(e, "Error executing subscription heartbeat timer"); } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj index e5b76c0940..5d0cc1f7f8 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj +++ b/src/Microsoft.Health.Fhir.Subscriptions/Microsoft.Health.Fhir.Subscriptions.csproj @@ -11,4 +11,19 @@ + + + ResXFileCodeGenerator + Resources.Designer.cs + + + + + + True + True + Resources.resx + + + diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs index 4431cc14b2..71ff6a18ca 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Operations/SubscriptionProcessingJob.cs @@ -18,10 +18,10 @@ namespace Microsoft.Health.Fhir.Subscriptions.Operations [JobTypeId((int)JobType.SubscriptionsProcessing)] public class SubscriptionProcessingJob : IJob { - private readonly StorageChannelFactory _storageChannelFactory; + private readonly SubscriptionChannelFactory _storageChannelFactory; private readonly IFhirDataStore _dataStore; - public SubscriptionProcessingJob(StorageChannelFactory storageChannelFactory, IFhirDataStore dataStore) + public SubscriptionProcessingJob(SubscriptionChannelFactory storageChannelFactory, IFhirDataStore dataStore) { _storageChannelFactory = storageChannelFactory; _dataStore = dataStore; diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs index efd26c6f7d..6d94e37258 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionManager.cs @@ -12,9 +12,8 @@ using EnsureThat; using Hl7.Fhir.Utility; using MediatR; -using Microsoft.Build.Framework; -using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; +using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Persistence; using Microsoft.Health.Fhir.Core.Features.Search; @@ -29,11 +28,11 @@ public sealed class SubscriptionManager : ISubscriptionManager, INotificationHan { private readonly IScopeProvider _dataStoreProvider; private readonly IScopeProvider _searchServiceProvider; - private List _subscriptions = new List(); + private List _subscriptions = new(); private readonly IResourceDeserializer _resourceDeserializer; private readonly ILogger _logger; private readonly ISubscriptionModelConverter _subscriptionModelConverter; - private static readonly object _lock = new object(); + private static readonly object _lock = new(); private readonly ISubscriptionUpdator _subscriptionUpdator; private readonly IRawResourceFactory _rawResourceFactory; @@ -48,11 +47,11 @@ public SubscriptionManager( { _dataStoreProvider = EnsureArg.IsNotNull(dataStoreProvider, nameof(dataStoreProvider)); _searchServiceProvider = EnsureArg.IsNotNull(searchServiceProvider, nameof(searchServiceProvider)); - _resourceDeserializer = resourceDeserializer; - _logger = logger; - _subscriptionModelConverter = subscriptionModelConverter; - _subscriptionUpdator = subscriptionUpdator; - _rawResourceFactory = rawResourceFactory; + _resourceDeserializer = EnsureArg.IsNotNull(resourceDeserializer, nameof(resourceDeserializer)); + _logger = EnsureArg.IsNotNull(logger, nameof(logger)); + _subscriptionModelConverter = EnsureArg.IsNotNull(subscriptionModelConverter, nameof(subscriptionModelConverter)); + _subscriptionUpdator = EnsureArg.IsNotNull(subscriptionUpdator, nameof(subscriptionUpdator)); + _rawResourceFactory = EnsureArg.IsNotNull(rawResourceFactory, nameof(rawResourceFactory)); } public async Task SyncSubscriptionsAsync(CancellationToken cancellationToken) @@ -61,10 +60,10 @@ public async Task SyncSubscriptionsAsync(CancellationToken cancellationToken) var updatedSubscriptions = new List(); - using var search = _searchServiceProvider.Invoke(); + using IScoped search = _searchServiceProvider.Invoke(); // Get all the active subscriptions - var activeSubscriptions = await search.Value.SearchAsync( + SearchResult activeSubscriptions = await search.Value.SearchAsync( KnownResourceTypes.Subscription, [ Tuple.Create("status", "active,requested"), diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionUpdator.cs b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionUpdator.cs index 7067433a99..5735f9e637 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionUpdator.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Persistence/SubscriptionUpdator.cs @@ -5,6 +5,7 @@ using System; using System.Linq; +using EnsureThat; using Hl7.Fhir.ElementModel; using Hl7.Fhir.Specification; using Microsoft.Health.Fhir.Core.Extensions; @@ -17,6 +18,8 @@ public class SubscriptionUpdator : ISubscriptionUpdator { public ResourceElement UpdateStatus(ResourceElement subscription, string status) { + EnsureArg.IsNotNull(subscription, nameof(subscription)); + var subscriptionElementNode = ElementNode.FromElement(subscription.Instance); var oldStatusNode = (ElementNode)subscriptionElementNode.Children("status").FirstOrDefault(); var newStatus = ElementNode.FromElement(oldStatusNode); diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs index 55178f90a3..7a730ea987 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Registration/SubscriptionsModule.cs @@ -31,14 +31,9 @@ namespace Microsoft.Health.Fhir.Subscriptions.Registration { - public class SubscriptionsModule : IStartupModule + public class SubscriptionsModule(CoreFeatureConfiguration coreFeatureConfiguration) : IStartupModule { - private readonly CoreFeatureConfiguration _coreFeatureConfiguration; - - public SubscriptionsModule(CoreFeatureConfiguration coreFeatureConfiguration) - { - _coreFeatureConfiguration = EnsureArg.IsNotNull(coreFeatureConfiguration, nameof(coreFeatureConfiguration)); - } + private readonly CoreFeatureConfiguration _coreFeatureConfiguration = EnsureArg.IsNotNull(coreFeatureConfiguration, nameof(coreFeatureConfiguration)); public void Load(IServiceCollection services) { @@ -70,7 +65,7 @@ public void Load(IServiceCollection services) .AsSelf() .AsImplementedInterfaces(); - services.Add() + services.Add() .Singleton() .AsSelf(); diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Resources.Designer.cs b/src/Microsoft.Health.Fhir.Subscriptions/Resources.Designer.cs new file mode 100644 index 0000000000..b0f0f33285 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Resources.Designer.cs @@ -0,0 +1,90 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace Microsoft.Health.Fhir.Subscriptions { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "17.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + internal class Resources { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Resources() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("Microsoft.Health.Fhir.Subscriptions.Resources", typeof(Resources).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to Subscription endpoint is not valid.. + /// + internal static string SubscriptionEndpointNotValid { + get { + return ResourceManager.GetString("SubscriptionEndpointNotValid", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Subscription is invalid.. + /// + internal static string SubscriptionInvalid { + get { + return ResourceManager.GetString("SubscriptionInvalid", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Subscription channel type is not valid.. + /// + internal static string SubscriptionTypeIsNotValid { + get { + return ResourceManager.GetString("SubscriptionTypeIsNotValid", resourceCulture); + } + } + } +} diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Resources.resx b/src/Microsoft.Health.Fhir.Subscriptions/Resources.resx new file mode 100644 index 0000000000..7bd0d3c445 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Subscriptions/Resources.resx @@ -0,0 +1,129 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Subscription is invalid. + + + Subscription endpoint is not valid. + + + Subscription channel type is not valid. + + \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Validation/CreateOrUpdateSubscriptionBehavior.cs b/src/Microsoft.Health.Fhir.Subscriptions/Validation/CreateOrUpdateSubscriptionBehavior.cs index 8cdac279c6..842beb2f19 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Validation/CreateOrUpdateSubscriptionBehavior.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Validation/CreateOrUpdateSubscriptionBehavior.cs @@ -36,6 +36,8 @@ public CreateOrUpdateSubscriptionBehavior(ISubscriptionValidator subscriptionVal public async Task Handle(CreateResourceRequest request, RequestHandlerDelegate next, CancellationToken cancellationToken) { + EnsureArg.IsNotNull(request, nameof(request)); + if (request.Resource.InstanceType.Equals(KnownResourceTypes.Subscription, StringComparison.Ordinal)) { request.Resource = await _subscriptionValidator.ValidateSubscriptionInput(request.Resource, cancellationToken); @@ -47,6 +49,8 @@ public async Task Handle(CreateResourceRequest request, public async Task Handle(UpsertResourceRequest request, RequestHandlerDelegate next, CancellationToken cancellationToken) { + EnsureArg.IsNotNull(request, nameof(request)); + // if the resource type being updated is a SearchParameter, then we want to query the previous version before it is changed // because we will need to the Url property to update the definition in the SearchParameterDefinitionManager // and the user could be changing the Url as part of this update diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionException.cs b/src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionException.cs index d9396cc0d8..3acefa06fe 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionException.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionException.cs @@ -5,6 +5,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -17,11 +18,13 @@ public class SubscriptionException : MicrosoftHealthException public SubscriptionException(string message) : base(message) { + Debug.Assert(message != null); } public SubscriptionException(string message, Exception innerException) : base(message, innerException) { + Debug.Assert(message != null); } } } diff --git a/src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionValidator.cs b/src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionValidator.cs index 323ec30b87..5bba819011 100644 --- a/src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionValidator.cs +++ b/src/Microsoft.Health.Fhir.Subscriptions/Validation/SubscriptionValidator.cs @@ -8,6 +8,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using EnsureThat; using FluentValidation.Results; using Hl7.Fhir.Model; using Hl7.Fhir.Rest; @@ -29,19 +30,21 @@ public class SubscriptionValidator : ISubscriptionValidator { private readonly ILogger _logger; private readonly ISubscriptionModelConverter _subscriptionModelConverter; - private readonly StorageChannelFactory _subscriptionChannelFactory; + private readonly SubscriptionChannelFactory _subscriptionChannelFactory; private readonly ISubscriptionUpdator _subscriptionUpdator; - public SubscriptionValidator(ILogger logger, ISubscriptionModelConverter subscriptionModelConverter, StorageChannelFactory subscriptionChannelFactory, ISubscriptionUpdator subscriptionUpdator) + public SubscriptionValidator(ILogger logger, ISubscriptionModelConverter subscriptionModelConverter, SubscriptionChannelFactory subscriptionChannelFactory, ISubscriptionUpdator subscriptionUpdator) { - _logger = logger; - _subscriptionModelConverter = subscriptionModelConverter; - _subscriptionChannelFactory = subscriptionChannelFactory; - _subscriptionUpdator = subscriptionUpdator; + _logger = EnsureArg.IsNotNull(logger, nameof(logger)); + _subscriptionModelConverter = EnsureArg.IsNotNull(subscriptionModelConverter, nameof(subscriptionModelConverter)); + _subscriptionChannelFactory = EnsureArg.IsNotNull(subscriptionChannelFactory, nameof(subscriptionChannelFactory)); + _subscriptionUpdator = EnsureArg.IsNotNull(subscriptionUpdator, nameof(subscriptionUpdator)); } public async Task ValidateSubscriptionInput(ResourceElement subscription, CancellationToken cancellationToken) { + EnsureArg.IsNotNull(subscription, nameof(subscription)); + SubscriptionInfo subscriptionInfo = _subscriptionModelConverter.Convert(subscription); var validationFailures = new List(); @@ -50,7 +53,7 @@ public async Task ValidateSubscriptionInput(ResourceElement sub { _logger.LogInformation("Subscription channel type is not valid."); validationFailures.Add( - new ValidationFailure(nameof(subscriptionInfo.Channel.ChannelType), "Subscription channel type is not valid.")); + new ValidationFailure(nameof(subscriptionInfo.Channel.ChannelType), Resources.SubscriptionTypeIsNotValid)); } if (!subscriptionInfo.Status.Equals(SubscriptionStatus.Off)) @@ -58,14 +61,14 @@ public async Task ValidateSubscriptionInput(ResourceElement sub try { var subscriptionChannel = _subscriptionChannelFactory.Create(subscriptionInfo.Channel.ChannelType); - await subscriptionChannel.PublishHandShakeAsync(subscriptionInfo); + await subscriptionChannel.PublishHandShakeAsync(subscriptionInfo, cancellationToken); subscription = _subscriptionUpdator.UpdateStatus(subscription, SubscriptionStatus.Active.GetLiteral()); } catch (SubscriptionException) { _logger.LogInformation("Subscription endpoint is not valid."); validationFailures.Add( - new ValidationFailure(nameof(subscriptionInfo.Channel.Endpoint), "Subscription endpoint is not valid.")); + new ValidationFailure(nameof(subscriptionInfo.Channel.Endpoint), Resources.SubscriptionEndpointNotValid)); subscription = _subscriptionUpdator.UpdateStatus(subscription, SubscriptionStatus.Error.GetLiteral()); } } diff --git a/src/Microsoft.Health.Fhir.Tests.Common/Categories.cs b/src/Microsoft.Health.Fhir.Tests.Common/Categories.cs index 9f3f65e8a7..bb6cae3334 100644 --- a/src/Microsoft.Health.Fhir.Tests.Common/Categories.cs +++ b/src/Microsoft.Health.Fhir.Tests.Common/Categories.cs @@ -90,6 +90,8 @@ public static class Categories public const string Sort = nameof(Sort); + public const string Subscriptions = nameof(Subscriptions); + public const string Transaction = nameof(Transaction); public const string Throttling = nameof(Throttling); From e048ce047366ef0bc71f1d9802bb746252506d26 Mon Sep 17 00:00:00 2001 From: Brendan Kowitz Date: Wed, 9 Oct 2024 10:59:20 -0700 Subject: [PATCH 47/47] Fixes from merge --- .../Search/InMemory/SubscriptionsOrchestratorJobTests.cs | 4 ---- .../Features/Watchdogs/DefragWatchdog.cs | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs index 75fa5984ed..7be150b83e 100644 --- a/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Core.UnitTests/Features/Search/InMemory/SubscriptionsOrchestratorJobTests.cs @@ -149,7 +149,6 @@ await _mockQueueClient.Received().EnqueueAsync( Arg.Is(resources => ContainsResourcesWithIds(resources, expectedIds)), 1, Arg.Any(), - Arg.Any(), Arg.Any()); } @@ -178,7 +177,6 @@ await _mockQueueClient.Received().EnqueueAsync( Arg.Is(resources => ContainsResourcesWithIds(resources, expectedIds)), 1, Arg.Any(), - Arg.Any(), Arg.Any()); } @@ -207,7 +205,6 @@ await _mockQueueClient.Received().EnqueueAsync( Arg.Is(resources => ContainsResourcesWithIds(resources, expectedIds)), 1, Arg.Any(), - Arg.Any(), Arg.Any()); } @@ -235,7 +232,6 @@ await _mockQueueClient.DidNotReceive().EnqueueAsync( Arg.Any(), 1, Arg.Any(), - Arg.Any(), Arg.Any()); } } diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/DefragWatchdog.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/DefragWatchdog.cs index f53af961e6..30f337ec04 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/DefragWatchdog.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Watchdogs/DefragWatchdog.cs @@ -206,7 +206,7 @@ private async Task InitDefragAsync(long groupId, CancellationToken cancella (long groupId, long jobId, long version) id = (-1, -1, -1); try { - var jobs = await _sqlQueueClient.EnqueueAsync(QueueType, Definitions, null, true, false, cancellationToken); + var jobs = await _sqlQueueClient.EnqueueAsync(QueueType, Definitions, null, true, cancellationToken); if (jobs.Count > 0) {