Merge branch 'datahub-project:master' into master

acryldata · Apr 23, 2024 · 3a439e0 · 3a439e0
2 parents 670967e + b9a34fe
commit 3a439e0
Show file tree

Hide file tree

Showing 10 changed files with 4,360 additions and 18 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/salesforce.py b/metadata-ingestion/src/datahub/ingestion/source/salesforce.py
@@ -537,11 +537,21 @@ def get_profile_workunit(
 
     # Here field description is created from label, description and inlineHelpText
     def _get_field_description(self, field: dict, customField: dict) -> str:
-        desc = field["Label"]
-        if field.get("FieldDefinition", {}).get("Description"):
-            desc = "{0}\n\n{1}".format(desc, field["FieldDefinition"]["Description"])
-        if field.get("InlineHelpText"):
-            desc = "{0}\n\n{1}".format(desc, field["InlineHelpText"])
+        desc = (
+            "\\" + field["Label"] if field["Label"].startswith("#") else field["Label"]
+        )
+
+        for key in ["FieldDefinition", "InlineHelpText"]:
+            text: Optional[str] = ""
+            if isinstance(field.get(key), dict):
+                text = field[key].get("Description")
+            else:
+                text = field.get(key)
+
+            if text:
+                prefix = "\\" if text.startswith("#") else ""
+                desc += f"\n\n{prefix}{text}"
+
         return desc
 
     # Here jsonProps is used to add additional salesforce field level properties.
@@ -575,13 +585,6 @@ def _get_schema_field(
 
         description = self._get_field_description(field, customField)
 
-        # escaping string starting with `#`
-        description = (
-            "\\" + description
-            if description and description.startswith("#")
-            else description
-        )
-
         schemaField = SchemaFieldClass(
             fieldPath=fieldPath,
             type=SchemaFieldDataTypeClass(type=TypeClass()),  # type:ignore

diff --git a/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py b/metadata-ingestion/src/datahub/ingestion/source/sigma/sigma.py
@@ -253,7 +253,7 @@ def _gen_entity_browsepath_aspect(
             urn = builder.make_dataplatform_instance_urn(
                 self.platform, self.config.platform_instance
             )
-        entries = [BrowsePathEntryClass(id=urn, urn=urn)] + entries
+            entries = [BrowsePathEntryClass(id=urn, urn=urn)] + entries
         return MetadataChangeProposalWrapper(
             entityUrn=entity_urn,
             aspect=BrowsePathsV2Class(entries),

diff --git a/metadata-ingestion/src/datahub/ingestion/source/usage/starburst_trino_usage.py b/metadata-ingestion/src/datahub/ingestion/source/usage/starburst_trino_usage.py
@@ -191,6 +191,8 @@ def _convert_str_to_datetime(self, v):
         if isinstance(v, str):
             isodate = parser.parse(v)  # compatible with Python 3.6+
             return isodate
+        if isinstance(v, datetime):
+            return v
 
     def _get_joined_access_event(self, events):
         joined_access_events = []

diff --git a/metadata-ingestion/tests/integration/salesforce/mock_files/account_fields_soql_response.json b/metadata-ingestion/tests/integration/salesforce/mock_files/account_fields_soql_response.json
@@ -2517,6 +2517,7 @@
             "QualifiedApiName": "Unique_Account",
             "DeveloperName": "Unique_Account",
             "Label": "# Unique_Account",
+            "InlineHelpText": "# Help Text",
             "FieldDefinition": {
                 "attributes": {
                     "type": "FieldDefinition",
@@ -2527,7 +2528,8 @@
                 "LastModifiedBy": null,
                 "IsIndexed": false,
                 "ComplianceGroup": null,
-                "SecurityClassification": null
+                "SecurityClassification": null,
+                "Description": "This is the # description"
             },
             "DataType": "string",
             "Precision": 0,

diff --git a/metadata-ingestion/tests/integration/salesforce/salesforce_mces_golden.json b/metadata-ingestion/tests/integration/salesforce/salesforce_mces_golden.json
@@ -1354,7 +1354,7 @@
                 {
                     "fieldPath": "Unique_Account",
                     "nullable": true,
-                    "description": "\\# Unique_Account",
+                    "description": "\\# Unique_Account\n\nThis is the # description\n\n\\# Help Text",
                     "type": {
                         "type": {
                             "com.linkedin.schema.StringType": {}