Improved N-1 join query performance for DW SQL (#2631)

## Why make this change?
- Back 1 year ago, as DW SQL does not support `JSON PATH` for converting
the execution to json format. Hence we had to use `STRING_AGG` as the
workaround.

- Recently, we've noticed `JSON PATH` is now supported for outer query
in DW, and we can use `JSON_OBJECT` + `JSON_PATH` to address the json
conversion for N-to-1 relations, which can optimize the performance.

- For N-N relations, we're still looking into any resolutions as
`JSON_ARRAYAGG` does not provide much performance improvements.
- 
- For other scenarios when joins are not needed for a simple SELECT, we
will `JSON PATH` instead of `STRING_AGG` for better performance.


## What is this change?

This PR covers
1. Introduced a feature flag to safeguard the changes, the feature flag
is default as False when not provided to avoid any regressions. It will
be removed once the changes are validated in production with scoped
audiences.
2. For DW query builder, use `JSON_OBJECT` to generate the columns for
sub-queries and applied `JSON PATH` to handle outer query, which fully
replace the need of `STRING_AGG`.
3. Also, for non-join queries (in which we don't need to handle the
relations), used `JSON PATH` to replace the need of `STRING_AGG` for
better performance as well. This will have impact on aggregations,
non-join queries and pagination.
4. Added some helper functions into the unit tests module, which aims to
compare the results from GraphQL & DB engine easily for deeply nested
queries.

## How was this tested?

- [x] Unit Tests
- As this change does not introduce new scenarios, so mostly added some
new test cases to get more coverage when M-M / M-1 join queries are
needed.

- [x] Integration Tests

### Manual Testing - Join Scenarios
#### Query 1-1 relation - As expected, optimization applied

![image](https://github.com/user-attachments/assets/514626e6-bcf5-4c37-a82e-0de13d947fd7)

![image](https://github.com/user-attachments/assets/a1921dfb-689b-496d-9f2e-520944c44ac8)

#### Query N-1 relation - As expected, optimization applied

![image](https://github.com/user-attachments/assets/86b6e25d-6f1a-48de-8ac5-f0e0ea6dd555)

![image](https://github.com/user-attachments/assets/8f9d47ff-6d4a-4207-843f-fdcf7d45a3a3)


#### Query 1-N Relation - As expected, optimization not applied

![image](https://github.com/user-attachments/assets/bea0d845-2e74-4a37-9410-9133f52fcdc5)

![image](https://github.com/user-attachments/assets/bd8d3a78-3d12-4909-abcc-4f6957e84564)

#### Query N-N Relation - As expected, optimization not applied

![image](https://github.com/user-attachments/assets/8e5777c1-34ba-433e-a7b2-4c0a806533ea)

![image](https://github.com/user-attachments/assets/5660ec06-e63b-4db7-be4a-9dfbd828c95a)

### Other Scenarios
We've applied the `JSON PATH` when there is no join in the query to
replace the `STRING_AGG` for better performance.

#### Aggregation

![image](https://github.com/user-attachments/assets/0adc0f75-92a6-418e-a756-22d349baec94)

#### Non-Join Query

![image](https://github.com/user-attachments/assets/8b0f799a-a424-44cb-975d-d346eb1ae5b0)


#### Pagination
- N to 1, total items: 3

![image](https://github.com/user-attachments/assets/b7ac611b-af4f-46de-b023-24d9f77acdae)

![image](https://github.com/user-attachments/assets/4b7a028d-255b-41da-b085-b7197bfef181)

- N to N

![image](https://github.com/user-attachments/assets/ec611346-5fcf-47a4-894b-454547fc99a6)

![image](https://github.com/user-attachments/assets/9714bdac-6467-492b-b15f-02a2cb2939c4)

![image](https://github.com/user-attachments/assets/a193b95d-d37e-4879-b43b-6402d758e3bf)

Author: lingxiao-microsoft

src/Cli.Tests/ModuleInitializer.cs

@@ -79,6 +79,10 @@ public static void Init()
         VerifierSettings.IgnoreMember<RuntimeConfig>(options => options.EnableAggregation);
         // Ignore the EnableAggregation as that's unimportant from a test standpoint.
         VerifierSettings.IgnoreMember<GraphQLRuntimeOptions>(options => options.EnableAggregation);
+        // Ignore the EnableDwNto1JoinOpt as that's unimportant from a test standpoint.
+        VerifierSettings.IgnoreMember<RuntimeConfig>(options => options.EnableDwNto1JoinOpt);
+        // Ignore the FeatureFlags as that's unimportant from a test standpoint.
+        VerifierSettings.IgnoreMember<GraphQLRuntimeOptions>(options => options.FeatureFlags);
         // Ignore the JSON schema path as that's unimportant from a test standpoint.
         VerifierSettings.IgnoreMember<RuntimeConfig>(config => config.Schema);
         // Ignore the message as that's not serialized in our config file anyway.

src/Config/ObjectModel/FeatureFlags.cs

@@ -0,0 +1,19 @@
+namespace Azure.DataApiBuilder.Config.ObjectModel
+{
+    /// <summary>
+    /// The class is used for ephemeral feature flags to turn on/off features in development
+    /// </summary>
+    public class FeatureFlags
+    {
+        /// <summary>
+        /// By default EnableDwNto1JoinQueryOptimization is disabled
+        /// We should change the default as True once got more confidence with the fix
+        /// </summary>
+        public bool EnableDwNto1JoinQueryOptimization { get; set; }
+
+        public FeatureFlags()
+        {
+            this.EnableDwNto1JoinQueryOptimization = false;
+        }
+    }
+}

src/Config/ObjectModel/GraphQLRuntimeOptions.cs

@@ -11,7 +11,8 @@ public record GraphQLRuntimeOptions(bool Enabled = true,
                                     bool AllowIntrospection = true,
                                     int? DepthLimit = null,
                                     MultipleMutationOptions? MultipleMutationOptions = null,
-                                    bool EnableAggregation = true)
+                                    bool EnableAggregation = true,
+                                    FeatureFlags? FeatureFlags = null)
 {
     public const string DEFAULT_PATH = "/graphql";
 
@@ -24,4 +25,10 @@ public record GraphQLRuntimeOptions(bool Enabled = true,
     [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
     [MemberNotNullWhen(true, nameof(DepthLimit))]
     public bool UserProvidedDepthLimit { get; init; } = false;
+
+    /// <summary>
+    /// Feature flag contains ephemeral flags passed in to init the runtime options 
+    /// </summary>
+    [JsonIgnore(Condition = JsonIgnoreCondition.Always)]
+    public FeatureFlags FeatureFlags { get; init; } = FeatureFlags ?? new FeatureFlags();
 }

src/Config/ObjectModel/RuntimeConfig.cs

@@ -150,6 +150,16 @@ Runtime is not null &&
         Runtime.GraphQL is not null &&
         Runtime.GraphQL.EnableAggregation;
 
+    /// <summary>
+    /// Retrieves the value of runtime.graphql.dwnto1joinopt.enabled property if present, default is false.
+    /// </summary>
+    [JsonIgnore]
+    public bool EnableDwNto1JoinOpt =>
+        Runtime is not null &&
+        Runtime.GraphQL is not null &&
+        Runtime.GraphQL.FeatureFlags is not null &&
+        Runtime.GraphQL.FeatureFlags.EnableDwNto1JoinQueryOptimization;
+
     private Dictionary<string, DataSource> _dataSourceNameToDataSource;
 
     private Dictionary<string, string> _entityNameToDataSourceName = new();

src/Core/Resolvers/BaseTSqlQueryBuilder.cs

@@ -0,0 +1,132 @@
+using Azure.DataApiBuilder.Config.ObjectModel;
+using Azure.DataApiBuilder.Core.Models;
+
+namespace Azure.DataApiBuilder.Core.Resolvers
+{
+    /// <summary>
+    /// Base query builder class for T-SQL engine
+    /// Can be used by dwsql and mssql
+    /// </summary>
+    public abstract class BaseTSqlQueryBuilder : BaseSqlQueryBuilder
+    {
+        protected const string FOR_JSON_SUFFIX = " FOR JSON PATH, INCLUDE_NULL_VALUES";
+        protected const string WITHOUT_ARRAY_WRAPPER_SUFFIX = "WITHOUT_ARRAY_WRAPPER";
+
+        /// <summary>
+        /// Build the Json Path query needed to append to the main query
+        /// </summary>
+        /// <param name="structure">Sql query structure to build query on</param>
+        /// <returns>SQL query with JSON PATH format</returns>
+        protected virtual string BuildJsonPath(SqlQueryStructure structure)
+        {
+            string query = string.Empty;
+            query += FOR_JSON_SUFFIX;
+            if (!structure.IsListQuery)
+            {
+                query += "," + WITHOUT_ARRAY_WRAPPER_SUFFIX;
+            }
+
+            return query;
+        }
+
+        /// <summary>
+        /// Build the predicates query needed to append to the main query
+        /// </summary>
+        /// <param name="structure">Sql query structure to build query on</param>
+        /// <returns>SQL query with predicates</returns>
+        protected virtual string BuildPredicates(SqlQueryStructure structure)
+        {
+            return JoinPredicateStrings(
+                          structure.GetDbPolicyForOperation(EntityActionOperation.Read),
+                          structure.FilterPredicates,
+                          Build(structure.Predicates),
+                          Build(structure.PaginationMetadata.PaginationPredicate));
+        }
+
+        /// <summary>
+        /// Build the Group By Clause needed to append to the main query
+        /// </summary>
+        /// <param name="structure">Sql query structure to build query on</param>
+        /// <returns>SQL query with group-by clause</returns>
+        protected virtual string BuildGroupBy(SqlQueryStructure structure)
+        {
+            // Add GROUP BY clause if there are any group by columns
+            if (structure.GroupByMetadata.Fields.Any())
+            {
+                return $" GROUP BY {string.Join(", ", structure.GroupByMetadata.Fields.Values.Select(c => Build(c)))}";
+            }
+
+            return string.Empty;
+        }
+
+        /// <summary>
+        /// Build the Having clause needed to append to the main query
+        /// </summary>
+        /// <param name="structure">Sql query structure to build query on</param>
+        /// <returns>SQL query with having clause</returns>
+        protected virtual string BuildHaving(SqlQueryStructure structure)
+        {
+            if (structure.GroupByMetadata.Aggregations.Count > 0)
+            {
+                List<Predicate>? havingPredicates = structure.GroupByMetadata.Aggregations
+                      .SelectMany(aggregation => aggregation.HavingPredicates ?? new List<Predicate>())
+                      .ToList();
+
+                if (havingPredicates.Any())
+                {
+                    return $" HAVING {Build(havingPredicates)}";
+                }
+            }
+
+            return string.Empty;
+        }
+
+        /// <summary>
+        /// Build the Order By clause needed to append to the main query
+        /// </summary>
+        /// <param name="structure">Sql query structure to build query on</param>
+        /// <returns>SQL query with order-by clause</returns>
+        protected virtual string BuildOrderBy(SqlQueryStructure structure)
+        {
+            if (structure.OrderByColumns.Any())
+            {
+                return $" ORDER BY {Build(structure.OrderByColumns)}";
+            }
+
+            return string.Empty;
+        }
+
+        /// <summary>
+        /// Build the aggregation columns needed to append to the main query
+        /// </summary>
+        /// <param name="structure">Sql query structure to build query on</param>
+        /// <returns>SQL query with aggregation columns</returns>
+        protected virtual string BuildAggregationColumns(SqlQueryStructure structure)
+        {
+            string aggregations = string.Empty;
+            if (structure.GroupByMetadata.Aggregations.Count > 0)
+            {
+                if (structure.Columns.Any())
+                {
+                    aggregations = $",{BuildAggregationColumns(structure.GroupByMetadata)}";
+                }
+                else
+                {
+                    aggregations = $"{BuildAggregationColumns(structure.GroupByMetadata)}";
+                }
+            }
+
+            return aggregations;
+        }
+
+        /// <summary>
+        /// Build the aggregation columns needed to append to the main query
+        /// </summary>
+        /// <param name="metadata">GroupByMetadata</param>
+        /// <returns>SQL query with aggregation columns</returns>
+        protected virtual string BuildAggregationColumns(GroupByMetadata metadata)
+        {
+            return string.Join(", ", metadata.Aggregations.Select(aggregation => Build(aggregation.Column, useAlias: true)));
+        }
+    }
+}