Skip to content

Commit

Permalink
O365 connector validation & code updates (#2112)
Browse files Browse the repository at this point in the history 
- Full validation of O365 Groups connector
- Full validation of O365 Users connector
- Fix in connector code around internal parameters (discovered a
variation w/ PA Client which is confirmed as a bug on their end)
- add logic in HttpFunctionInvoker to trace requests (disabled by
default)
- add support in test code for providing JPG response files as byte
array
  • Loading branch information
@LucGenetier
LucGenetier authored Dec 19, 2023
1 parent 0aa6860 commit 39456d6
Show file tree
Hide file tree
Showing 53 changed files with 19,168 additions and 94 deletions.
20 changes: 10 additions & 10 deletions src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,12 +134,12 @@ public string NotSupportedReason
/// <summary>
/// Return type of the function.
/// </summary>
public FormulaType ReturnType => Operation.GetReturnType();
public FormulaType ReturnType => Operation.GetReturnType(ConnectorSettings.Compatibility);

/// <summary>
/// Connector return type of the function (contains inner "x-ms-summary" and descriptions).
/// </summary>
internal ConnectorType ConnectorReturnType => Operation.GetConnectorReturnType().ConnectorType;
internal ConnectorType ConnectorReturnType => Operation.GetConnectorReturnType(ConnectorSettings.Compatibility).ConnectorType;

/// <summary>
/// Minimum number of arguments.
Expand Down Expand Up @@ -685,7 +685,7 @@ private async Task<ConnectorType> GetConnectorSuggestionsFromDynamicSchemaAsync(

JsonElement je = ExtractFromJson(sv, cds.ValuePath);
OpenApiSchema schema = new OpenApiStringReader().ReadFragment<OpenApiSchema>(je.ToString(), Microsoft.OpenApi.OpenApiSpecVersion.OpenApi2_0, out OpenApiDiagnostic diag);
ConnectorType connectorType = new ConnectorType(schema);
ConnectorType connectorType = new ConnectorType(schema, ConnectorSettings.Compatibility);

runtimeContext.ExecutionLogger?.LogDebug($"Exiting {this.LogFunction(nameof(GetConnectorSuggestionsFromDynamicSchemaAsync))}, with {LogConnectorType(connectorType)}");
return connectorType;
Expand All @@ -711,7 +711,7 @@ private async Task<ConnectorType> GetConnectorSuggestionsFromDynamicPropertyAsyn

JsonElement je = ExtractFromJson(sv, cdp.ItemValuePath);
OpenApiSchema schema = new OpenApiStringReader().ReadFragment<OpenApiSchema>(je.ToString(), Microsoft.OpenApi.OpenApiSpecVersion.OpenApi2_0, out OpenApiDiagnostic diag);
ConnectorType connectorType = new ConnectorType(schema);
ConnectorType connectorType = new ConnectorType(schema, ConnectorSettings.Compatibility);

runtimeContext.ExecutionLogger?.LogDebug($"Exiting {this.LogFunction(nameof(GetConnectorSuggestionsFromDynamicPropertyAsync))}, with {LogConnectorType(connectorType)}");
return connectorType;
Expand Down Expand Up @@ -957,7 +957,7 @@ private ConnectorParameterInternals Initialize()
return null;
}

ConnectorParameter connectorParameter = new ConnectorParameter(parameter);
ConnectorParameter connectorParameter = new ConnectorParameter(parameter, ConnectorSettings.Compatibility);

if (connectorParameter.HiddenRecordType != null)
{
Expand Down Expand Up @@ -1022,11 +1022,11 @@ private ConnectorParameterInternals Initialize()

OpenApiParameter bodyParameter = new OpenApiParameter() { Name = bodyPropertyName, Schema = bodyPropertySchema, Description = requestBody.Description, Required = bodyPropertyRequired, Extensions = bodyPropertySchema.Extensions };
openApiBodyParameters.Add(bodyParameter);
ConnectorParameter bodyConnectorParameter2 = new ConnectorParameter(bodyParameter, requestBody);
ConnectorParameter bodyConnectorParameter2 = new ConnectorParameter(bodyParameter, requestBody, ConnectorSettings.Compatibility);

if (bodyConnectorParameter2.HiddenRecordType != null)
{
hiddenRequiredParameters.Add(new ConnectorParameter(bodyParameter, true));
hiddenRequiredParameters.Add(new ConnectorParameter(bodyParameter, true, ConnectorSettings.Compatibility));
}

List<ConnectorParameter> parameterList = !bodyPropertyRequired ? optionalParameters : bodyPropertyHiddenRequired ? hiddenRequiredParameters : requiredParameters;
Expand All @@ -1040,7 +1040,7 @@ private ConnectorParameterInternals Initialize()
OpenApiParameter bodyParameter2 = new OpenApiParameter() { Name = bodyName, Schema = bodySchema, Description = requestBody.Description, Required = requestBody.Required, Extensions = bodySchema.Extensions };
openApiBodyParameters.Add(bodyParameter2);

ConnectorParameter bodyConnectorParameter3 = new ConnectorParameter(bodyParameter2, requestBody);
ConnectorParameter bodyConnectorParameter3 = new ConnectorParameter(bodyParameter2, requestBody, ConnectorSettings.Compatibility);

if (bodyConnectorParameter3.HiddenRecordType != null)
{
Expand All @@ -1061,7 +1061,7 @@ private ConnectorParameterInternals Initialize()
OpenApiParameter bodyParameter3 = new OpenApiParameter() { Name = bodyName, Schema = bodyParameterSchema, Description = "Body", Required = requestBody.Required };
openApiBodyParameters.Add(bodyParameter3);

ConnectorParameter bodyParameter = new ConnectorParameter(bodyParameter3, requestBody);
ConnectorParameter bodyParameter = new ConnectorParameter(bodyParameter3, requestBody, ConnectorSettings.Compatibility);

List<ConnectorParameter> parameterList = requestBody.Required ? requiredParameters : optionalParameters;
parameterList.Add(bodyParameter);
Expand Down Expand Up @@ -1101,7 +1101,7 @@ private ConnectorParameterInternals Initialize()
_arityMin = _requiredParameters.Length;
_arityMax = _arityMin + (_optionalParameters.Length == 0 ? 0 : 1);

(ConnectorType connectorType, string unsupportedReason) = Operation.GetConnectorReturnType();
(ConnectorType connectorType, string unsupportedReason) = Operation.GetConnectorReturnType(ConnectorSettings.Compatibility);
_returnParameterType = connectorType;

if (!string.IsNullOrEmpty(unsupportedReason))
Expand Down
37 changes: 36 additions & 1 deletion src/libraries/Microsoft.PowerFx.Connectors/Execution/HttpFunctionInvoker.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,13 +303,48 @@ private HttpContent GetBody(string referenceId, bool schemaLessBody, Dictionary<
}

public async Task<FormulaValue> DecodeResponseAsync(HttpResponseMessage response, bool throwOnError = false)
{
{
var text = response?.Content == null
? string.Empty
: await response.Content.ReadAsStringAsync().ConfigureAwait(false);

var statusCode = (int)response.StatusCode;

#if RECORD_RESULTS
if (response.RequestMessage.Headers.TryGetValues("x-ms-request-url", out IEnumerable<string> urlHeader) &&
response.RequestMessage.Headers.TryGetValues("x-ms-request-method", out IEnumerable<string> verbHeader))
{
string url = urlHeader.FirstOrDefault();
string verb = verbHeader.FirstOrDefault();
string ext = _returnRawResults ? "raw" : "json";

if (!string.IsNullOrEmpty(url))
{
string u2 = url.Replace('/', '_').Replace('?', '_').Replace('+', ' ').Replace('%', '_');
u2 = u2.Substring(0, Math.Min(u2.Length, 100));

int i = 0;
string file = $@"C:\Temp\Response_{verb}_{(int)statusCode}_{u2}.{ext}";

// Paging, when multiple result pages are returned, or when same request is run multiple times
while (System.IO.File.Exists(file))
{
i++;
file = $@"C:\Temp\Response_{verb}_{(int)statusCode}_{u2}_#{i:00}.{ext}";
}

if (!_returnRawResults)
{
System.IO.File.WriteAllText(file, text);
}
else
{
System.IO.File.WriteAllBytes(file, await response.Content.ReadAsByteArrayAsync().ConfigureAwait(false));
}
}
}
#endif

if (statusCode < 300)
{
return string.IsNullOrWhiteSpace(text)
Expand Down
37 changes: 21 additions & 16 deletions src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ internal static void WhenPresent(this OpenApiObject apiObj, string str, Action<O
}

// See https://swagger.io/docs/specification/data-models/data-types/
internal static ConnectorType ToConnectorType(this OpenApiParameter openApiParameter, Stack<string> chain = null, int level = 0)
internal static ConnectorType ToConnectorType(this OpenApiParameter openApiParameter, ConnectorCompatibility compatibility, Stack<string> chain = null, int level = 0)
{
chain ??= new Stack<string>();

Expand Down Expand Up @@ -456,7 +456,7 @@ internal static ConnectorType ToConnectorType(this OpenApiParameter openApiParam
}

chain.Push(innerA);
ConnectorType arrayType = new OpenApiParameter() { Name = "Array", Required = true, Schema = schema.Items, Extensions = schema.Items.Extensions }.ToConnectorType(chain, level + 1);
ConnectorType arrayType = new OpenApiParameter() { Name = "Array", Required = true, Schema = schema.Items, Extensions = schema.Items.Extensions }.ToConnectorType(compatibility, chain, level + 1);

chain.Pop();

Expand Down Expand Up @@ -504,13 +504,18 @@ internal static ConnectorType ToConnectorType(this OpenApiParameter openApiParam

if (kv.Value.IsInternal())
{
if (schema.Required.Contains(kv.Key) && kv.Value.Default != null)
{
hiddenRequired = true;
}
else
{
continue;
if (schema.Required.Contains(kv.Key))
{
if (kv.Value.Default == null)
{
continue;
}

hiddenRequired = true;
}
else if (compatibility == ConnectorCompatibility.SwaggerCompatibility)
{
continue;
}
}

Expand All @@ -524,7 +529,7 @@ internal static ConnectorType ToConnectorType(this OpenApiParameter openApiParam
}

chain.Push(innerO);
ConnectorType propertyType = new OpenApiParameter() { Name = propName, Required = schema.Required.Contains(propName), Schema = kv.Value, Extensions = kv.Value.Extensions }.ToConnectorType(chain, level + 1);
ConnectorType propertyType = new OpenApiParameter() { Name = propName, Required = schema.Required.Contains(propName), Schema = kv.Value, Extensions = kv.Value.Extensions }.ToConnectorType(compatibility, chain, level + 1);
chain.Pop();

if (propertyType.HiddenRecordType != null)
Expand Down Expand Up @@ -573,9 +578,9 @@ public static HttpMethod ToHttpMethod(this OperationType key)
};
}

public static FormulaType GetReturnType(this OpenApiOperation openApiOperation)
public static FormulaType GetReturnType(this OpenApiOperation openApiOperation, ConnectorCompatibility compatibility)
{
(ConnectorType connectorType, string unsupportedReason) = openApiOperation.GetConnectorReturnType();
(ConnectorType connectorType, string unsupportedReason) = openApiOperation.GetConnectorReturnType(compatibility);
FormulaType ft = connectorType?.FormulaType ?? new BlankType();
return ft;
}
Expand All @@ -585,7 +590,7 @@ public static bool GetRequiresUserConfirmation(this OpenApiOperation op)
return op.Extensions.TryGetValue(XMsRequireUserConfirmation, out IOpenApiExtension openExt) && openExt is OpenApiBoolean b && b.Value;
}

internal static (ConnectorType ConnectorType, string UnsupportedReason) GetConnectorReturnType(this OpenApiOperation openApiOperation)
internal static (ConnectorType ConnectorType, string UnsupportedReason) GetConnectorReturnType(this OpenApiOperation openApiOperation, ConnectorCompatibility compatibility)
{
OpenApiResponses responses = openApiOperation.Responses;
OpenApiResponse response = responses.Where(kvp => kvp.Key?.Length == 3 && kvp.Key.StartsWith("2", StringComparison.Ordinal)).OrderBy(kvp => kvp.Key).FirstOrDefault().Value;
Expand All @@ -609,7 +614,7 @@ internal static (ConnectorType ConnectorType, string UnsupportedReason) GetConne
if (response.Content.Count == 0)
{
OpenApiSchema schema = new OpenApiSchema() { Type = "string", Format = "binary" };
ConnectorType connectorType = new OpenApiParameter() { Name = "response", Required = true, Schema = schema, Extensions = response.Extensions }.ToConnectorType();
ConnectorType connectorType = new OpenApiParameter() { Name = "response", Required = true, Schema = schema, Extensions = response.Extensions }.ToConnectorType(compatibility);
return (connectorType, null);
}

Expand All @@ -630,11 +635,11 @@ internal static (ConnectorType ConnectorType, string UnsupportedReason) GetConne
{
// Treat as void.
OpenApiSchema schema = new OpenApiSchema() { Type = "string", Format = "binary" };
ConnectorType cType = new OpenApiParameter() { Name = "response", Required = true, Schema = schema, Extensions = response.Extensions }.ToConnectorType();
ConnectorType cType = new OpenApiParameter() { Name = "response", Required = true, Schema = schema, Extensions = response.Extensions }.ToConnectorType(compatibility);
return (cType, null);
}

ConnectorType connectorType = new OpenApiParameter() { Name = "response", Required = true, Schema = openApiMediaType.Schema, Extensions = openApiMediaType.Schema.Extensions }.ToConnectorType();
ConnectorType connectorType = new OpenApiParameter() { Name = "response", Required = true, Schema = openApiMediaType.Schema, Extensions = openApiMediaType.Schema.Extensions }.ToConnectorType(compatibility);

return (connectorType, null);
}
Expand Down
16 changes: 8 additions & 8 deletions src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorParameter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,24 +22,24 @@ public class ConnectorParameter : ConnectorSchema

internal bool IsBodyParameter = false;

internal ConnectorParameter(OpenApiParameter openApiParameter)
: this(openApiParameter, null, false)
internal ConnectorParameter(OpenApiParameter openApiParameter, ConnectorCompatibility compatibility)
: this(openApiParameter, null, false, compatibility)
{
}

internal ConnectorParameter(OpenApiParameter openApiParameter, bool useHiddenTypes)
: this(openApiParameter, null, useHiddenTypes)
internal ConnectorParameter(OpenApiParameter openApiParameter, bool useHiddenTypes, ConnectorCompatibility compatibility)
: this(openApiParameter, null, useHiddenTypes, compatibility)
{
}

internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions)
: this(openApiParameter, bodyExtensions, false)
internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, ConnectorCompatibility compatibility)
: this(openApiParameter, bodyExtensions, false, compatibility)
{
IsBodyParameter = true;
}

internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, bool useHiddenTypes)
: base(openApiParameter, bodyExtensions, useHiddenTypes)
internal ConnectorParameter(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, bool useHiddenTypes, ConnectorCompatibility compatibility)
: base(openApiParameter, bodyExtensions, useHiddenTypes, compatibility)
{
Name = openApiParameter.Name;
Description = openApiParameter.Description;
Expand Down
4 changes: 2 additions & 2 deletions src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorSchema.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ public class ConnectorSchema

public bool SupportsDynamicIntellisense => ConnectorType.SupportsDynamicIntellisense;

internal ConnectorSchema(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, bool useHiddenTypes)
internal ConnectorSchema(OpenApiParameter openApiParameter, IOpenApiExtensible bodyExtensions, bool useHiddenTypes, ConnectorCompatibility compatibility)
{
Schema = openApiParameter.Schema;
UseHiddenTypes = useHiddenTypes;
ConnectorType = openApiParameter.ToConnectorType();
ConnectorType = openApiParameter.ToConnectorType(compatibility);
DefaultValue = openApiParameter.Schema.TryGetDefaultValue(FormulaType, out FormulaValue defaultValue) && defaultValue is not BlankValue ? defaultValue : null;
ConnectorExtensions = new ConnectorExtensions(openApiParameter, bodyExtensions);
}
Expand Down
8 changes: 4 additions & 4 deletions src/libraries/Microsoft.PowerFx.Connectors/Public/ConnectorType.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ public class ConnectorType

public Visibility Visibility { get; internal set; }

internal RecordType HiddenRecordType { get; }
internal RecordType HiddenRecordType { get; }

public bool SupportsDynamicValuesOrList => DynamicValues != null || DynamicList != null;

Expand Down Expand Up @@ -105,7 +105,7 @@ internal ConnectorType(OpenApiSchema schema, OpenApiParameter openApiParameter,
{
EnumValues = Array.Empty<FormulaValue>();
EnumDisplayNames = Array.Empty<string>();
}
}
}

DynamicSchema = openApiParameter.GetDynamicSchema();
Expand All @@ -119,8 +119,8 @@ internal ConnectorType()
FormulaType = new BlankType();
}

internal ConnectorType(OpenApiSchema schema)
: this(schema, null, new OpenApiParameter() { Schema = schema }.ToConnectorType())
internal ConnectorType(OpenApiSchema schema, ConnectorCompatibility compatibility)
: this(schema, null, new OpenApiParameter() { Schema = schema }.ToConnectorType(compatibility))
{
}

Expand Down
Loading

0 comments on commit 39456d6

Please sign in to comment.