diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs index 9aeb6c7f99..0e09133625 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Storage/SqlServerFhirDataStore.cs @@ -434,7 +434,7 @@ internal async Task> ImportResourcesAsync(IReadOnlyList resources, ImportMode importMode) { - return importMode == ImportMode.IncrementalLoad && resources.Any(_ => _.KeepLastUpdated) ? 80000 / resources.Count : 30; // 80K is id sequence rollover + return importMode == ImportMode.IncrementalLoad && resources.Any(_ => _.KeepLastUpdated) ? Math.Min(8000, 80000 / resources.Count) : 30; // 80K is id sequence rollover } List GetErrors(IReadOnlyCollection dups, IReadOnlyCollection conflicts) diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTests.cs index fa1e5b07d9..d61861ca67 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.E2E/Rest/Import/ImportTests.cs @@ -217,7 +217,7 @@ EXECUTE dbo.MergeResourcesCommitTransaction @TransactionId public async Task GivenIncrementalLoad_80KSurrogateIds_BadRequestIsReturned() { var ndJson = new StringBuilder(); - for (int i = 0; i < 80001; i++) + for (int i = 0; i < 79001; i++) { var id = Guid.NewGuid().ToString("N"); var str = CreateTestPatient(id, DateTimeOffset.Parse("1900-01-01Z00:00")); // make sure this date is not used by other tests. @@ -227,7 +227,21 @@ public async Task GivenIncrementalLoad_80KSurrogateIds_BadRequestIsReturned() var location = (await ImportTestHelper.UploadFileAsync(ndJson.ToString(), _fixture.StorageAccount)).location; var request = CreateImportRequest(location, ImportMode.IncrementalLoad); var checkLocation = await ImportTestHelper.CreateImportTaskAsync(_client, request); - var message = await ImportWaitAsync(checkLocation, false); + await ImportWaitAsync(checkLocation, true); + + ndJson = new StringBuilder(); + for (int i = 0; i < 1000; i++) // load violators in nlarge batch to avoid large number of retries + { + var id = Guid.NewGuid().ToString("N"); + var str = CreateTestPatient(id, DateTimeOffset.Parse("1900-01-01Z00:00")); // make sure this date is not used by other tests. + ndJson.Append(str); + } + + location = (await ImportTestHelper.UploadFileAsync(ndJson.ToString(), _fixture.StorageAccount)).location; + request = CreateImportRequest(location, ImportMode.IncrementalLoad); + checkLocation = await ImportTestHelper.CreateImportTaskAsync(_client, request); + var message = await ImportWaitAsync(checkLocation, true); + Assert.Equal(HttpStatusCode.BadRequest, message.StatusCode); Assert.Contains(ImportProcessingJob.SurrogateIdsErrorMessage, await message.Content.ReadAsStringAsync()); }