Refactor CDP implementation and add Capabilities (#2636)

Refactor CDP implementation
Use AssociatedDataSources for CDP record types
Make all CDP records lazy
Add x-ms-enum management
New TabularRecordType (lazy)
Make CdpTableResolver return ConnectorType
Remove CdpDType hack
Remove CdpTableType hack
Remove CdpTableDescriptor
Remove ExternalCdpDataSource
Remove ICdpTableResolver.GenerateADS hack
Rework Connector service capabilities
Add Relationships to CdpService
Add TableParameters to CdpTable
New InternalTableMetadata and related classes in Pfx Core
New InternalTableParameters in Pfx Core (implementing
IExternalTabularDataSource)
Fix DType ToRecord, ToTable

src/libraries/Microsoft.PowerFx.Connectors/ConnectorConstants.cs

@@ -10,6 +10,7 @@ public static class Constants
         public const string XMsDynamicProperties = "x-ms-dynamic-properties";
         public const string XMsDynamicSchema = "x-ms-dynamic-schema";
         public const string XMsDynamicValues = "x-ms-dynamic-values";
+        public const string XMsEnum = "x-ms-enum";
         public const string XMsEnumDisplayName = "x-ms-enum-display-name";
         public const string XMsEnumValues = "x-ms-enum-values";
         public const string XMsExplicitInput = "x-ms-explicit-input";

src/libraries/Microsoft.PowerFx.Connectors/ConnectorFunction.cs

@@ -17,6 +17,7 @@
 using Microsoft.OpenApi.Readers;
 using Microsoft.OpenApi.Validations;
 using Microsoft.PowerFx.Connectors.Localization;
+using Microsoft.PowerFx.Core.Entities;
 using Microsoft.PowerFx.Core.Errors;
 using Microsoft.PowerFx.Core.Localization;
 using Microsoft.PowerFx.Core.Types;
@@ -867,7 +868,7 @@ private async Task<FormulaValue> PostProcessResultAsync(FormulaValue result, Bas
                 ExpressionError newError = er is HttpExpressionError her
                     ? new HttpExpressionError(her.StatusCode) { Kind = er.Kind, Severity = er.Severity, Message = $"{DPath.Root.Append(new DName(Namespace)).ToDottedSyntax()}.{Name} failed: {er.Message}" }
                     : new ExpressionError() { Kind = er.Kind, Severity = er.Severity, Message = $"{DPath.Root.Append(new DName(Namespace)).ToDottedSyntax()}.{Name} failed: {er.Message}" };
-                result = FormulaValue.NewError(newError, ev.Type); 
+                result = FormulaValue.NewError(newError, ev.Type);
             }
 
             if (IsPageable && result is RecordValue rv)
@@ -1005,33 +1006,23 @@ internal static ConnectorType GetConnectorType(string valuePath, StringValue sv,
         }
 
         // Only called by ConnectorTable.GetSchema
-        internal static ConnectorType GetConnectorTypeAndTableCapabilities(ICdpTableResolver tableResolver, string connectorName, string valuePath, StringValue sv, List<SqlRelationship> sqlRelationships, ConnectorCompatibility compatibility, string datasetName, out string name, out string displayName, out ServiceCapabilities tableCapabilities)
+        // Returns a FormulaType with AssociatedDataSources set (done in AddTabularDataSource)
+        internal static ConnectorType GetCdpTableType(ICdpTableResolver tableResolver, string connectorName, string valuePath, StringValue stringValue, List<SqlRelationship> sqlRelationships, ConnectorCompatibility compatibility, string datasetName, out string name, out string displayName, out TableDelegationInfo delegationInfo)
         {
             // There are some errors when parsing this Json payload but that's not a problem here as we only need x-ms-capabilities parsing to work
             OpenApiReaderSettings oars = new OpenApiReaderSettings() { RuleSet = DefaultValidationRuleSet };
-            ISwaggerSchema tableSchema = SwaggerSchema.New(new OpenApiStringReader(oars).ReadFragment<OpenApiSchema>(sv.Value, OpenApi.OpenApiSpecVersion.OpenApi2_0, out OpenApiDiagnostic _));
-            tableCapabilities = tableSchema.GetTableCapabilities();
-
-            JsonElement je = ExtractFromJson(sv, valuePath, out name, out displayName);
-
-            // Json version to be able to read SalesForce unique properties
-            ConnectorType connectorType = GetJsonConnectorTypeInternal(compatibility, je, sqlRelationships);
-            connectorType.Name = name;
-            IList<ReferencedEntity> referencedEntities = GetReferenceEntities(connectorName, sv);
+            ISwaggerSchema tableSchema = SwaggerSchema.New(new OpenApiStringReader(oars).ReadFragment<OpenApiSchema>(stringValue.Value, OpenApi.OpenApiSpecVersion.OpenApi2_0, out OpenApiDiagnostic _));
 
+            ServiceCapabilities serviceCapabilities = tableSchema.GetTableCapabilities();
             ConnectorPermission tablePermission = tableSchema.GetPermission();
-            bool isTableReadOnly = tablePermission == ConnectorPermission.PermissionReadOnly;
-
-            List<ConnectorType> primaryKeyParts = connectorType.Fields.Where(f => f.KeyType == ConnectorKeyType.Primary).OrderBy(f => f.KeyOrder).ToList();
-
-            if (primaryKeyParts.Count == 0)
-            {
-                // $$$ need to check what triggers RO for SQL
-                //isTableReadOnly = true;
-            }
-
-            connectorType.AddTabularDataSource(tableResolver, referencedEntities, sqlRelationships, new DName(name), datasetName, connectorType, tableCapabilities, isTableReadOnly);
-
+
+            JsonElement jsonElement = ExtractFromJson(stringValue, valuePath, out name, out displayName);
+            bool isTableReadOnly = tablePermission == ConnectorPermission.PermissionReadOnly;
+            IList<ReferencedEntity> referencedEntities = GetReferenceEntities(connectorName, stringValue);
+
+            ConnectorType connectorType = new ConnectorType(jsonElement, compatibility, sqlRelationships, referencedEntities, datasetName, name, connectorName, tableResolver, serviceCapabilities, isTableReadOnly);
+            delegationInfo = ((DataSourceInfo)connectorType.FormulaType._type.AssociatedDataSources.First()).DelegationInfo;
+
             return connectorType;
         }
 

src/libraries/Microsoft.PowerFx.Connectors/OpenApiExtensions.cs

@@ -333,13 +333,22 @@ internal static bool TryGetOpenApiValue(IOpenApiAny openApiAny, FormulaType form
         internal static bool IsInternal(this IOpenApiExtensible oae) => SwaggerExtensions.New(oae)?.IsInternal() ?? false;
 
         internal static string GetVisibility(this IOpenApiExtensible oae) => SwaggerExtensions.New(oae)?.GetVisibility();
+
+        internal static string GetEnumName(this IOpenApiExtensible oae) => SwaggerExtensions.New(oae)?.GetEnumName();
 
         // Internal parameters are not showen to the user.
         // They can have a default value or be special cased by the infrastructure (like "connectionId").
         internal static bool IsInternal(this ISwaggerExtensions schema) => string.Equals(schema.GetVisibility(), "internal", StringComparison.OrdinalIgnoreCase);
 
         internal static string GetVisibility(this ISwaggerExtensions schema) => schema.Extensions.TryGetValue(XMsVisibility, out IOpenApiExtension openApiExt) && openApiExt is OpenApiString openApiStr ? openApiStr.Value : null;
 
+        internal static string GetEnumName(this ISwaggerExtensions schema) => schema.Extensions.TryGetValue(XMsEnum, out IOpenApiExtension openApiExt) && 
+                                                                              openApiExt is SwaggerJsonObject jsonObject &&
+                                                                              jsonObject.TryGetValue("name", out IOpenApiAny enumName) &&
+                                                                              enumName is OpenApiString enumNameStr 
+                                                                            ? enumNameStr.Value 
+                                                                            : null;
+
         internal static string GetMediaKind(this ISwaggerExtensions schema) => schema.Extensions.TryGetValue(XMsMediaKind, out IOpenApiExtension openApiExt) && openApiExt is OpenApiString openApiStr ? openApiStr.Value : null;
 
         internal static (bool IsPresent, string Value) GetString(this IDictionary<string, IOpenApiAny> apiObj, string str) => apiObj.TryGetValue(str, out IOpenApiAny openApiAny) && openApiAny is OpenApiString openApiStr ? (true, openApiStr.Value) : (false, null);
@@ -438,7 +447,8 @@ internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiPar
                     {
                         if (schema.Enum.All(e => e is OpenApiString))
                         {
-                            OptionSet optionSet = new OptionSet("enum", schema.Enum.Select(e => new DName((e as OpenApiString).Value)).ToDictionary(k => k, e => e).ToImmutableDictionary());
+                            string enumName = schema.GetEnumName() ?? "enum";
+                            OptionSet optionSet = new OptionSet(enumName, schema.Enum.Select(e => new DName((e as OpenApiString).Value)).ToDictionary(k => k, e => e).ToImmutableDictionary());
                             return new ConnectorType(schema, openApiParameter, optionSet.FormulaType);
                         }
                         else
@@ -466,14 +476,19 @@ internal static ConnectorType GetConnectorType(this ISwaggerParameter openApiPar
                         case null:
                         case "decimal":
                         case "currency":
-                            return new ConnectorType(schema, openApiParameter, FormulaType.Decimal);
+                            return new ConnectorType(schema, openApiParameter, FormulaType.Decimal);                       
 
                         default:
                             return new ConnectorType(error: $"Unsupported type of number: {schema.Format}");
                     }
 
+                // For testing only
+                case "fxnumber":
+                    return new ConnectorType(schema, openApiParameter, FormulaType.Number);
+
                 // Always a boolean (Format not used)
-                case "boolean": return new ConnectorType(schema, openApiParameter, FormulaType.Boolean);
+                case "boolean": 
+                    return new ConnectorType(schema, openApiParameter, FormulaType.Boolean);
 
                 // OpenAPI spec: Format could be <null>, int32, int64
                 case "integer":

src/libraries/Microsoft.PowerFx.Connectors/Public/CdpDelegationInfo.cs

@@ -0,0 +1,15 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT license.
+
+namespace Microsoft.PowerFx.Core.Entities
+{
+    // Used by ServiceCapabilities.ToDelegationInfo for managing CDP x-ms-capabilities
+    internal class CdpDelegationInfo : TableDelegationInfo
+    {
+        public override ColumnCapabilitiesDefinition GetColumnCapability(string fieldName)
+        {
+            // We should never reach that point in CDP case
+            throw new System.NotImplementedException();
+        }
+    }
+}

src/libraries/Microsoft.PowerFx.Connectors/Public/CdpRecordType.cs

@@ -5,7 +5,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading;
-using Microsoft.PowerFx.Core.Types;
+using Microsoft.PowerFx.Core.Entities;
 using Microsoft.PowerFx.Types;
 
 namespace Microsoft.PowerFx.Connectors
@@ -14,34 +14,23 @@ internal class CdpRecordType : RecordType
     {
         internal ConnectorType ConnectorType { get; }
 
-        internal IList<ReferencedEntity> ReferencedEntities { get; }
-
-        internal IList<SqlRelationship> SqlRelationships { get; }
-
         internal ICdpTableResolver TableResolver { get; }
 
-        internal CdpRecordType(ConnectorType connectorType, DType recordType, ICdpTableResolver tableResolver, IList<ReferencedEntity> referencedEntities, IList<SqlRelationship> sqlRelationships)
-            : base(recordType)
+        internal CdpRecordType(ConnectorType connectorType, ICdpTableResolver tableResolver, TableDelegationInfo delegationInfo)
+            : base(connectorType.DisplayNameProvider, delegationInfo)
         {
             ConnectorType = connectorType;
             TableResolver = tableResolver;
-            ReferencedEntities = referencedEntities;
-            SqlRelationships = sqlRelationships;
         }
 
         public bool TryGetFieldExternalTableName(string fieldName, out string tableName, out string foreignKey)
         {
             tableName = null;
             foreignKey = null;
 
-            if (!base.TryGetBackingDType(fieldName, out _))
-            {
-                return false;
-            }
-
             ConnectorType connectorType = ConnectorType.Fields.First(ct => ct.Name == fieldName);
 
-            if (connectorType.ExternalTables?.Any() != true)
+            if (connectorType == null || connectorType.ExternalTables?.Any() != true)
             {
                 return false;
             }
@@ -51,28 +40,33 @@ public bool TryGetFieldExternalTableName(string fieldName, out string tableName,
             return true;
         }
 
-        public override bool TryGetFieldType(string fieldName, out FormulaType type)
+        public override bool TryGetUnderlyingFieldType(string name, out FormulaType type) => TryGetFieldType(name, true, out type);
+
+        public override bool TryGetFieldType(string name, out FormulaType type) => TryGetFieldType(name, false, out type);
+
+        private bool TryGetFieldType(string fieldName, bool ignorelationship, out FormulaType type)
         {
-            if (!base.TryGetBackingDType(fieldName, out _))
+            ConnectorType field = ConnectorType.Fields.FirstOrDefault(ct => ct.Name == fieldName);
+
+            if (field == null)
             {
                 type = null;
                 return false;
             }
 
-            ConnectorType ct = ConnectorType.Fields.First(ct => ct.Name == fieldName);
-
-            if (ct.ExternalTables?.Any() != true)
+            if (field.ExternalTables?.Any() != true || ignorelationship)
             {
-                return base.TryGetFieldType(fieldName, out type);
+                type = field.FormulaType;
+                return true;
             }
 
-            string tableName = ct.ExternalTables.First();
+            string tableName = field.ExternalTables.First();
 
             try
             {
-                CdpTableDescriptor ttd = TableResolver.ResolveTableAsync(tableName, CancellationToken.None).ConfigureAwait(false).GetAwaiter().GetResult();
+                ConnectorType connectorType = TableResolver.ResolveTableAsync(tableName, CancellationToken.None).ConfigureAwait(false).GetAwaiter().GetResult();
 
-                type = ttd.ConnectorType.FormulaType;
+                type = connectorType.FormulaType;
                 return true;
             }
             catch (Exception ex)
@@ -84,7 +78,17 @@ public override bool TryGetFieldType(string fieldName, out FormulaType type)
 
         public override bool Equals(object other)
         {
-            throw new NotImplementedException();
+            if (object.ReferenceEquals(this, other))
+            {
+                return true;
+            }
+
+            if (other is not CdpRecordType otherRecordType)
+            {
+                return false;
+            }
+
+            return ConnectorType.Equals(otherRecordType.ConnectorType);
         }
 
         public override int GetHashCode()
@@ -94,6 +98,6 @@ public override int GetHashCode()
 
         public override string TableSymbolName => ConnectorType.Name;
 
-        public override IEnumerable<string> FieldNames => _type.GetRootFieldNames().Select(name => name.Value);
+        public override IEnumerable<string> FieldNames => ConnectorType.Fields.Select(field => field.Name);
     }
 }

src/libraries/Microsoft.PowerFx.Connectors/Public/CdpTableValue.cs

@@ -6,7 +6,7 @@
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
-using Microsoft.PowerFx.Core.Entities; 
+using Microsoft.PowerFx.Core.Entities;
 using Microsoft.PowerFx.Core.IR;
 using Microsoft.PowerFx.Types;
 
@@ -18,22 +18,21 @@ public class CdpTableValue : TableValue, IRefreshable, IDelegatableTableValue
     {
         public bool IsDelegable => _tabularService.IsDelegable;
 
-        protected internal readonly CdpService _tabularService;
+        protected internal readonly CdpService _tabularService;        
 
-        protected internal readonly ConnectorType _connectorType;
+        internal readonly IReadOnlyDictionary<string, Relationship> Relationships;
 
         private IReadOnlyCollection<DValue<RecordValue>> _cachedRows;
 
         internal readonly HttpClient HttpClient;
 
-        public RecordType TabularRecordType => _tabularService?.TabularRecordType;
+        public RecordType RecordType => _tabularService?.RecordType;
 
-        public CdpTableValue(CdpService tabularService, ConnectorType connectorType)
-            : base(IRContext.NotInSource(new CdpTableType(tabularService.TableType)))
+        internal CdpTableValue(CdpService tabularService, IReadOnlyDictionary<string, Relationship> relationships)
+            : base(IRContext.NotInSource(tabularService.TableType))
         {
             _tabularService = tabularService;
-            _connectorType = connectorType;
-
+            Relationships = relationships;                        
             HttpClient = tabularService.HttpClient;
         }