Merge branch 'datahub-project:master' into master

datahub-web-react/src/app/entity/shared/components/styled/ERModelRelationship/ERModelRelationUtils.tsx

@@ -68,6 +68,6 @@ export function getDatasetName(datainput: any): string {
         datainput?.editableProperties?.name ||
         datainput?.properties?.name ||
         datainput?.name ||
-        datainput?.urn?.split(',').at(1)
+        datainput?.urn?.split(',')?.at(1)
     );
 }

datahub-web-react/src/app/entity/shared/tabs/Dataset/Queries/utils/filterQueries.ts

@@ -10,9 +10,9 @@ export const filterQueries = (filterText, queries: Query[]) => {
     const lowerFilterText = filterText.toLowerCase();
     return queries.filter((query) => {
         return (
-            query.title?.toLowerCase().includes(lowerFilterText) ||
-            query.description?.toLowerCase().includes(lowerFilterText) ||
-            query.query?.toLowerCase().includes(lowerFilterText)
+            query.title?.toLowerCase()?.includes(lowerFilterText) ||
+            query.description?.toLowerCase()?.includes(lowerFilterText) ||
+            query.query?.toLowerCase()?.includes(lowerFilterText)
         );
     });
 };

datahub-web-react/src/app/entity/shared/tabs/Dataset/Schema/utils/filterSchemaRows.ts

@@ -12,7 +12,7 @@ function matchesTagsOrTermsOrDescription(field: SchemaField, filterText: string,
                 .toLocaleLowerCase()
                 .includes(filterText),
         ) ||
-        field.description?.toLocaleLowerCase().includes(filterText)
+        field.description?.toLocaleLowerCase()?.includes(filterText)
     );
 }
 

datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx

@@ -129,7 +129,7 @@ export const ExecutionDetailsModal = ({ urn, open, onClose }: Props) => {
         downloadFile(output, `exec-${urn}.log`);
     };
 
-    const logs = (showExpandedLogs && output) || output?.split('\n').slice(0, 5).join('\n');
+    const logs = (showExpandedLogs && output) || output?.split('\n')?.slice(0, 5)?.join('\n');
     const result = data?.executionRequest?.result as Partial<ExecutionRequestResult>;
     const status = getIngestionSourceStatus(result);
 
@@ -163,7 +163,7 @@ export const ExecutionDetailsModal = ({ urn, open, onClose }: Props) => {
     } catch (e) {
         recipeYaml = '';
     }
-    const recipe = showExpandedRecipe ? recipeYaml : recipeYaml?.split('\n').slice(0, 5).join('\n');
+    const recipe = showExpandedRecipe ? recipeYaml : recipeYaml?.split('\n')?.slice(0, 5)?.join('\n');
 
     const areLogsExpandable = output?.split(/\r\n|\r|\n/)?.length > 5;
     const isRecipeExpandable = recipeYaml?.split(/\r\n|\r|\n/)?.length > 5;

datahub-web-react/src/app/lineage/utils/titleUtils.ts

@@ -124,10 +124,10 @@ function truncate(input, length) {
 function getLastTokenOfTitle(title?: string): string {
     if (!title) return '';
 
-    const lastToken = title?.split('.').slice(-1)[0];
+    const lastToken = title?.split('.')?.slice(-1)?.[0];
 
     // if the last token does not contain any content, the string should not be tokenized on `.`
-    if (lastToken.replace(/\s/g, '').length === 0) {
+    if (lastToken?.replace(/\s/g, '')?.length === 0) {
         return title;
     }
 

datahub-web-react/src/app/search/context/SearchResultContext.tsx

@@ -40,7 +40,7 @@ export const useSearchResult = () => {
 };
 
 export const useEntityType = () => {
-    return useSearchResultContext()?.searchResult.entity.type;
+    return useSearchResultContext()?.searchResult?.entity?.type;
 };
 
 export const useMatchedFields = () => {

datahub-web-react/src/app/search/matches/MatchedFieldList.tsx

@@ -42,7 +42,7 @@ const RenderedField = ({
     field: MatchedField;
 }) => {
     const entityRegistry = useEntityRegistry();
-    const query = useSearchQuery()?.trim().toLowerCase();
+    const query = useSearchQuery()?.trim()?.toLowerCase();
     const customRenderedField = customFieldRenderer?.(field);
     if (customRenderedField) return <b>{customRenderedField}</b>;
     if (isHighlightableEntityField(field)) {

datahub-web-react/src/app/search/matches/SearchTextHighlighter.tsx

@@ -23,7 +23,7 @@ const SearchTextHighlighter = ({ field, text, enableFullHighlight = false }: Pro
     const enableNameHighlight = appConfig.config.visualConfig.searchResult?.enableNameHighlight;
     const matchedFields = useMatchedFieldsByGroup(field);
     const hasMatchedField = !!matchedFields?.length;
-    const normalizedSearchQuery = useSearchQuery()?.trim().toLowerCase();
+    const normalizedSearchQuery = useSearchQuery()?.trim()?.toLowerCase();
     const normalizedText = text.trim().toLowerCase();
     const hasSubstring = hasMatchedField && !!normalizedSearchQuery && normalizedText.includes(normalizedSearchQuery);
     const pattern = enableFullHighlight ? HIGHLIGHT_ALL_PATTERN : undefined;

metadata-ingestion/src/datahub/ingestion/graph/client.py

@@ -188,9 +188,12 @@ def from_emitter(cls, emitter: DatahubRestEmitter) -> "DataHubGraph":
                 retry_max_times=emitter._retry_max_times,
                 extra_headers=emitter._session.headers,
                 disable_ssl_verification=emitter._session.verify is False,
-                # TODO: Support these headers.
-                # ca_certificate_path=emitter._ca_certificate_path,
-                # client_certificate_path=emitter._client_certificate_path,
+                ca_certificate_path=(
+                    emitter._session.verify
+                    if isinstance(emitter._session.verify, str)
+                    else None
+                ),
+                client_certificate_path=emitter._session.cert,
             )
         )
 

metadata-ingestion/src/datahub/specific/dataproduct.py

@@ -131,7 +131,7 @@ def add_asset(self, asset_urn: str) -> "DataProductPatchBuilder":
         self._add_patch(
             DataProductProperties.ASPECT_NAME,
             "add",
-            path=f"/assets/{asset_urn}",
+            path=f"/assets/{self.quote(asset_urn)}",
             value=DataProductAssociation(destinationUrn=asset_urn),
         )
         return self
@@ -140,7 +140,7 @@ def remove_asset(self, asset_urn: str) -> "DataProductPatchBuilder":
         self._add_patch(
             DataProductProperties.ASPECT_NAME,
             "remove",
-            path=f"/assets/{asset_urn}",
+            path=f"/assets/{self.quote(asset_urn)}",
             value={},
         )
         return self

metadata-ingestion/src/datahub/sql_parsing/sqlglot_lineage.py

@@ -66,6 +66,7 @@
     "SQL_LINEAGE_TIMEOUT_ENABLED", True
 )
 SQL_LINEAGE_TIMEOUT_SECONDS = 10
+SQL_PARSER_TRACE = get_boolean_env_variable("DATAHUB_SQL_PARSER_TRACE", False)
 
 
 # These rules are a subset of the rules in sqlglot.optimizer.optimizer.RULES.
@@ -365,10 +366,11 @@ def _sqlglot_force_column_normalizer(
 
             return node
 
-        # logger.debug(
-        #     "Prior to case normalization sql %s",
-        #     statement.sql(pretty=True, dialect=dialect),
-        # )
+        if SQL_PARSER_TRACE:
+            logger.debug(
+                "Prior to case normalization sql %s",
+                statement.sql(pretty=True, dialect=dialect),
+            )
         statement = statement.transform(_sqlglot_force_column_normalizer, copy=False)
         # logger.debug(
         #     "Sql after casing normalization %s",
@@ -562,7 +564,7 @@ def _select_statement_cll(  # noqa: C901
                 )
             )
 
-        # TODO: Also extract referenced columns (aka auxillary / non-SELECT lineage)
+        # TODO: Also extract referenced columns (aka auxiliary / non-SELECT lineage)
     except (sqlglot.errors.OptimizeError, ValueError, IndexError) as e:
         raise SqlUnderstandingError(
             f"sqlglot failed to compute some lineage: {e}"
@@ -1022,6 +1024,14 @@ def _sqlglot_lineage_inner(
     logger.debug(
         f"Resolved {total_schemas_resolved} of {total_tables_discovered} table schemas"
     )
+    if SQL_PARSER_TRACE:
+        for qualified_table, schema_info in table_name_schema_mapping.items():
+            logger.debug(
+                "Table name %s resolved to %s with schema %s",
+                qualified_table,
+                table_name_urn_mapping[qualified_table],
+                schema_info,
+            )
 
     column_lineage: Optional[List[_ColumnLineageInfo]] = None
     try: