From 6d692ccecb2e15c6c1ec462d42a67eae374e4e0b Mon Sep 17 00:00:00 2001 From: Rob Gordon Date: Thu, 23 May 2024 14:37:37 -0700 Subject: [PATCH 1/6] Treat datetime strings as Strings during json deserialization in JsonParser --- data/SampleData/Json/ExamplePatient.json | 3 ++- data/Templates/Json/ExamplePatient.liquid | 3 ++- .../ExamplePatient-expected.json | 3 ++- .../Parsers/JsonDataParser.cs | 23 ++++++++++++++++++- .../Processors/JsonProcessor.cs | 9 +++++++- 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/data/SampleData/Json/ExamplePatient.json b/data/SampleData/Json/ExamplePatient.json index e8870e2f8..6883781e5 100644 --- a/data/SampleData/Json/ExamplePatient.json +++ b/data/SampleData/Json/ExamplePatient.json @@ -8,5 +8,6 @@ "1234-5679" ], "Gender": "M", - "DOB": "20010110" + "DOB": "20010110", + "TimeOfDeath": "2023-07-28T01:59:23.388-05:00" } \ No newline at end of file diff --git a/data/Templates/Json/ExamplePatient.liquid b/data/Templates/Json/ExamplePatient.liquid index 660d75fa5..2bc697c4e 100644 --- a/data/Templates/Json/ExamplePatient.liquid +++ b/data/Templates/Json/ExamplePatient.liquid @@ -50,5 +50,6 @@ "managingOrganization": { "reference": "Organization/2.16.840.1.113883.19.5", "display": "Good Health Clinic" - } + }, + "deceasedDateTime": "{{ msg.TimeOfDeath }}" } \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Json/ExamplePatient/ExamplePatient-expected.json b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Json/ExamplePatient/ExamplePatient-expected.json index 53addab2f..28d940a49 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Json/ExamplePatient/ExamplePatient-expected.json +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.FunctionalTests/TestData/Expected/Json/ExamplePatient/ExamplePatient-expected.json @@ -40,5 +40,6 @@ "managingOrganization": { "reference": "Organization/2.16.840.1.113883.19.5", "display": "Good Health Clinic" - } + }, + "deceasedDateTime": "2023-07-28T01:59:23.388-05:00" } \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Parsers/JsonDataParser.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Parsers/JsonDataParser.cs index c1be2cf78..684d05e65 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Parsers/JsonDataParser.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Parsers/JsonDataParser.cs @@ -4,15 +4,35 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.IO; +using EnsureThat; using Microsoft.Health.Fhir.Liquid.Converter.Exceptions; using Microsoft.Health.Fhir.Liquid.Converter.Extensions; using Microsoft.Health.Fhir.Liquid.Converter.Models; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; namespace Microsoft.Health.Fhir.Liquid.Converter.Parsers { public class JsonDataParser : IDataParser { + private static Func _defaultJsonReaderGenerator = (json) => new JsonTextReader(new StringReader(json)) + { + DateParseHandling = DateParseHandling.None, + }; + + public JsonDataParser() + : this(_defaultJsonReaderGenerator) + { + } + + public JsonDataParser(Func jsonReaderGenerator) + { + this.JsonReaderGenerator = EnsureArg.IsNotNull(jsonReaderGenerator, nameof(jsonReaderGenerator)); + } + + protected Func JsonReaderGenerator { get; } + public object Parse(string json) { if (string.IsNullOrWhiteSpace(json)) @@ -22,7 +42,8 @@ public object Parse(string json) try { - return JToken.Parse(json).ToObject(); + using var reader = JsonReaderGenerator.Invoke(json); + return JToken.ReadFrom(reader).ToObject(); } catch (Exception ex) { diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Processors/JsonProcessor.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Processors/JsonProcessor.cs index 18c56ee32..933e6eaef 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Processors/JsonProcessor.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Processors/JsonProcessor.cs @@ -7,6 +7,7 @@ using System.Globalization; using System.Threading; using DotLiquid; +using EnsureThat; using Microsoft.Extensions.Logging; using Microsoft.Health.Fhir.Liquid.Converter.Extensions; using Microsoft.Health.Fhir.Liquid.Converter.Models; @@ -20,11 +21,17 @@ namespace Microsoft.Health.Fhir.Liquid.Converter.Processors { public class JsonProcessor : BaseProcessor { - private readonly IDataParser _parser = new JsonDataParser(); + private readonly IDataParser _parser; public JsonProcessor(ProcessorSettings processorSettings, ILogger logger) + : this(processorSettings, new JsonDataParser(), logger) + { + } + + public JsonProcessor(ProcessorSettings processorSettings, IDataParser parser, ILogger logger) : base(processorSettings, logger) { + _parser = EnsureArg.IsNotNull(parser, nameof(parser)); } protected override DataType DataType { get; set; } = DataType.Json; From 104b068e27c6ca9ecf63c0a24b166316e3f3dee3 Mon Sep 17 00:00:00 2001 From: Rob Gordon Date: Thu, 23 May 2024 15:44:54 -0700 Subject: [PATCH 2/6] Updating unit test --- .../Processors/ProcessorTests.cs | 6 +++--- .../TestData/Expected/ExamplePatient.json | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Processors/ProcessorTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Processors/ProcessorTests.cs index 3678c1549..81d65bc64 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Processors/ProcessorTests.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Processors/ProcessorTests.cs @@ -5,7 +5,6 @@ using System; using System.Collections.Generic; -using System.Data.Common; using System.IO; using System.Text.RegularExpressions; using System.Threading; @@ -13,6 +12,7 @@ using Microsoft.Health.Fhir.Liquid.Converter.Exceptions; using Microsoft.Health.Fhir.Liquid.Converter.Models; using Microsoft.Health.Fhir.Liquid.Converter.Processors; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Xunit; @@ -362,8 +362,8 @@ public void GivenJObjectInput_WhenConvertWithJsonProcessor_CorrectResultShouldBe { var processor = _jsonProcessor; var templateProvider = new TemplateProvider(TestConstants.JsonTemplateDirectory, DataType.Json); - var testData = JObject.Parse(_jsonTestData); - var result = processor.Convert(testData, "ExamplePatient", templateProvider); + var testData = JObject.ReadFrom(new JsonTextReader(new StringReader(_jsonTestData)) { DateParseHandling = DateParseHandling.None }); + var result = processor.Convert(testData as JObject, "ExamplePatient", templateProvider); Assert.True(JToken.DeepEquals(JObject.Parse(_jsonExpectData), JToken.Parse(result))); } diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/TestData/Expected/ExamplePatient.json b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/TestData/Expected/ExamplePatient.json index 1e65aca0c..bb93fa811 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/TestData/Expected/ExamplePatient.json +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/TestData/Expected/ExamplePatient.json @@ -40,5 +40,6 @@ "managingOrganization": { "reference": "Organization/2.16.840.1.113883.19.5", "display": "Good Health Clinic" - } + }, + "deceasedDateTime": "2023-07-28T01:59:23.388-05:00" } \ No newline at end of file From e7b83326497a98527296d84457cc0a8fab759c8f Mon Sep 17 00:00:00 2001 From: Rob Gordon Date: Wed, 29 May 2024 13:37:13 -0700 Subject: [PATCH 3/6] Using JsonConvert to deserialize json string --- .../Parsers/JsonDataParser.cs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Parsers/JsonDataParser.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Parsers/JsonDataParser.cs index 684d05e65..64dd3fc02 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Parsers/JsonDataParser.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Parsers/JsonDataParser.cs @@ -4,7 +4,6 @@ // ------------------------------------------------------------------------------------------------- using System; -using System.IO; using EnsureThat; using Microsoft.Health.Fhir.Liquid.Converter.Exceptions; using Microsoft.Health.Fhir.Liquid.Converter.Extensions; @@ -16,22 +15,22 @@ namespace Microsoft.Health.Fhir.Liquid.Converter.Parsers { public class JsonDataParser : IDataParser { - private static Func _defaultJsonReaderGenerator = (json) => new JsonTextReader(new StringReader(json)) + private static readonly JsonSerializerSettings DefaultSerializerSettings = new JsonSerializerSettings() { DateParseHandling = DateParseHandling.None, }; public JsonDataParser() - : this(_defaultJsonReaderGenerator) + : this(DefaultSerializerSettings) { } - public JsonDataParser(Func jsonReaderGenerator) + public JsonDataParser(JsonSerializerSettings jsonSerializerSettings) { - this.JsonReaderGenerator = EnsureArg.IsNotNull(jsonReaderGenerator, nameof(jsonReaderGenerator)); + JsonSerializerSettings = EnsureArg.IsNotNull(jsonSerializerSettings, nameof(jsonSerializerSettings)); } - protected Func JsonReaderGenerator { get; } + protected JsonSerializerSettings JsonSerializerSettings { get; private set; } public object Parse(string json) { @@ -42,8 +41,7 @@ public object Parse(string json) try { - using var reader = JsonReaderGenerator.Invoke(json); - return JToken.ReadFrom(reader).ToObject(); + return JsonConvert.DeserializeObject(json, JsonSerializerSettings).ToObject(); } catch (Exception ex) { From 3fe3915d13e3b7f53cf18d4a79ad7f259246e9b3 Mon Sep 17 00:00:00 2001 From: Rob Gordon Date: Thu, 30 May 2024 10:08:43 -0700 Subject: [PATCH 4/6] Converting to JToken to support deserialization of all types of Json components --- .../Parsers/JsonDataParser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Parsers/JsonDataParser.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Parsers/JsonDataParser.cs index 64dd3fc02..dea353920 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Parsers/JsonDataParser.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Parsers/JsonDataParser.cs @@ -41,7 +41,7 @@ public object Parse(string json) try { - return JsonConvert.DeserializeObject(json, JsonSerializerSettings).ToObject(); + return JsonConvert.DeserializeObject(json, JsonSerializerSettings).ToObject(); } catch (Exception ex) { From f2bad8a024fade602daf7f9038ab2d9be44311d0 Mon Sep 17 00:00:00 2001 From: Rob Gordon Date: Mon, 3 Jun 2024 09:08:09 -0700 Subject: [PATCH 5/6] Updating json post processing to treat dates as strings --- .../Processors/JsonToHl7v2Processor.cs | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter/Processors/JsonToHl7v2Processor.cs b/src/Microsoft.Health.Fhir.Liquid.Converter/Processors/JsonToHl7v2Processor.cs index 8dd82e90d..14e8afacc 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter/Processors/JsonToHl7v2Processor.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter/Processors/JsonToHl7v2Processor.cs @@ -10,6 +10,7 @@ using System.Text; using System.Threading; using DotLiquid; +using EnsureThat; using Microsoft.Extensions.Logging; using Microsoft.Health.Fhir.Liquid.Converter.Exceptions; using Microsoft.Health.Fhir.Liquid.Converter.Extensions; @@ -18,6 +19,7 @@ using Microsoft.Health.Fhir.Liquid.Converter.Models.Json; using Microsoft.Health.Fhir.Liquid.Converter.Parsers; using Microsoft.Health.MeasurementUtility; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NJsonSchema; @@ -25,13 +27,24 @@ namespace Microsoft.Health.Fhir.Liquid.Converter.Processors { public class JsonToHl7v2Processor : BaseProcessor { - private readonly IDataParser _parser = new JsonDataParser(); + private static readonly JsonSerializerSettings DefaultSerializerSettings = new JsonSerializerSettings() + { + DateParseHandling = DateParseHandling.None, + }; + + private readonly IDataParser _parser; private string[] _segmentsWithFieldSeparator = new string[] { "MSH", "BHS", "FHS" }; public JsonToHl7v2Processor(ProcessorSettings processorSettings, ILogger logger) + : this(processorSettings, new JsonDataParser(), logger) + { + } + + public JsonToHl7v2Processor(ProcessorSettings processorSettings, IDataParser parser, ILogger logger) : base(processorSettings, logger) { + _parser = EnsureArg.IsNotNull(parser, nameof(parser)); } protected override string InternalConvert(string data, string rootTemplate, ITemplateProvider templateProvider, TraceInfo traceInfo = null) @@ -45,7 +58,7 @@ protected override string InternalConvert(string data, string rootTemplate, ITem var result = InternalConvertFromObject(jsonData, rootTemplate, templateProvider, traceInfo); - var hl7Message = GenerateHL7Message(JObject.Parse(result)); + var hl7Message = GenerateHL7Message(ConvertToJObject(result)); var hl7String = ConvertHl7MessageToString(hl7Message); @@ -56,7 +69,7 @@ public string Convert(JObject data, string rootTemplate, ITemplateProvider templ { var jsonData = data.ToObject(); var result = InternalConvertFromObject(jsonData, rootTemplate, templateProvider, traceInfo); - var hl7Message = GenerateHL7Message(JObject.Parse(result)); + var hl7Message = GenerateHL7Message(ConvertToJObject(result)); var hl7String = ConvertHl7MessageToString(hl7Message); return hl7String; @@ -239,5 +252,10 @@ protected override void CreateTraceInfo(object data, Context context, TraceInfo jsonTraceInfo.ValidateSchemas = jsonContext.ValidateSchemas; } } + + private JObject ConvertToJObject(string input) + { + return JsonConvert.DeserializeObject(input, DefaultSerializerSettings); + } } } From 9a4205e982c32d2fe537035af0cdcd419f50c3b8 Mon Sep 17 00:00:00 2001 From: Rob Gordon Date: Mon, 3 Jun 2024 11:42:33 -0700 Subject: [PATCH 6/6] Adjusting due to pr comment --- .../Processors/ProcessorTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Processors/ProcessorTests.cs b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Processors/ProcessorTests.cs index 81d65bc64..4fa9faeba 100644 --- a/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Processors/ProcessorTests.cs +++ b/src/Microsoft.Health.Fhir.Liquid.Converter.UnitTests/Processors/ProcessorTests.cs @@ -362,8 +362,8 @@ public void GivenJObjectInput_WhenConvertWithJsonProcessor_CorrectResultShouldBe { var processor = _jsonProcessor; var templateProvider = new TemplateProvider(TestConstants.JsonTemplateDirectory, DataType.Json); - var testData = JObject.ReadFrom(new JsonTextReader(new StringReader(_jsonTestData)) { DateParseHandling = DateParseHandling.None }); - var result = processor.Convert(testData as JObject, "ExamplePatient", templateProvider); + var testData = JsonConvert.DeserializeObject(_jsonTestData, new JsonSerializerSettings() { DateParseHandling = DateParseHandling.None }); + var result = processor.Convert(testData, "ExamplePatient", templateProvider); Assert.True(JToken.DeepEquals(JObject.Parse(_jsonExpectData), JToken.Parse(result))); }