Merge branch 'datahub-project:master' into master

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java

@@ -636,48 +636,49 @@ public GmsGraphQLEngine(final GmsGraphQLEngineArgs args) {
     this.businessAttributeType = new BusinessAttributeType(entityClient);
     // Init Lists
     this.entityTypes =
-        ImmutableList.of(
-            datasetType,
-            roleType,
-            corpUserType,
-            corpGroupType,
-            dataPlatformType,
-            chartType,
-            dashboardType,
-            tagType,
-            mlModelType,
-            mlModelGroupType,
-            mlFeatureType,
-            mlFeatureTableType,
-            mlPrimaryKeyType,
-            dataFlowType,
-            dataJobType,
-            glossaryTermType,
-            glossaryNodeType,
-            connectionType,
-            containerType,
-            notebookType,
-            domainType,
-            assertionType,
-            versionedDatasetType,
-            dataPlatformInstanceType,
-            accessTokenMetadataType,
-            testType,
-            dataHubPolicyType,
-            dataHubRoleType,
-            schemaFieldType,
-            erModelRelationshipType,
-            dataHubViewType,
-            queryType,
-            dataProductType,
-            ownershipType,
-            structuredPropertyType,
-            dataTypeType,
-            entityTypeType,
-            formType,
-            incidentType,
-            restrictedType,
-            businessAttributeType);
+        new ArrayList<>(
+            ImmutableList.of(
+                datasetType,
+                roleType,
+                corpUserType,
+                corpGroupType,
+                dataPlatformType,
+                chartType,
+                dashboardType,
+                tagType,
+                mlModelType,
+                mlModelGroupType,
+                mlFeatureType,
+                mlFeatureTableType,
+                mlPrimaryKeyType,
+                dataFlowType,
+                dataJobType,
+                glossaryTermType,
+                glossaryNodeType,
+                connectionType,
+                containerType,
+                notebookType,
+                domainType,
+                assertionType,
+                versionedDatasetType,
+                dataPlatformInstanceType,
+                accessTokenMetadataType,
+                testType,
+                dataHubPolicyType,
+                dataHubRoleType,
+                schemaFieldType,
+                erModelRelationshipType,
+                dataHubViewType,
+                queryType,
+                dataProductType,
+                ownershipType,
+                structuredPropertyType,
+                dataTypeType,
+                entityTypeType,
+                formType,
+                incidentType,
+                restrictedType,
+                businessAttributeType));
     this.loadableTypes = new ArrayList<>(entityTypes);
     // Extend loadable types with types from the plugins
     // This allows us to offer search and browse capabilities out of the box for those types

datahub-graphql-core/src/main/resources/assertions.graphql

@@ -213,6 +213,11 @@ enum FreshnessAssertionScheduleType {
   A scheduled based on a recurring fixed schedule which is used to compute the expected operation window. E.g. "every 24 hours".
   """
   FIXED_INTERVAL
+
+  """
+  A schedule computed based on when the assertion was last evaluated, to the current moment in time.
+  """
+  SINCE_THE_LAST_CHECK
 }
 
 """

datahub-graphql-core/src/main/resources/entity.graphql

@@ -7658,6 +7658,16 @@ enum AssertionStdOperator {
     """
     EQUAL_TO
 
+    """
+    Value being asserted is not equal to value
+    """
+    NOT_EQUAL_TO
+
+    """
+    Value being asserted is null
+    """
+    NULL
+
     """
     Value being asserted is not null
     """
@@ -7694,12 +7704,12 @@ enum AssertionStdOperator {
     NOT_IN
 
     """
-    Value being asserted is true.
+    Value being asserted is true
     """
     IS_TRUE
 
     """
-    Value being asserted is false.
+    Value being asserted is false
     """
     IS_FALSE
 

datahub-web-react/src/app/entity/shared/ExternalUrlButton.tsx

@@ -5,6 +5,8 @@ import UrlButton from './UrlButton';
 
 const GITHUB_LINK = 'github.com';
 const GITHUB = 'GitHub';
+const GITLAB_LINK = 'gitlab.com';
+const GITLAB = 'GitLab';
 
 interface Props {
     externalUrl: string;
@@ -26,6 +28,8 @@ export default function ExternalUrlButton({ externalUrl, platformName, entityTyp
     let displayedName = platformName;
     if (externalUrl.toLocaleLowerCase().includes(GITHUB_LINK)) {
         displayedName = GITHUB;
+    } else if (externalUrl.toLocaleLowerCase().includes(GITLAB_LINK)) {
+        displayedName = GITLAB;
     }
 
     return (

datahub-web-react/src/app/ingest/source/builder/NameSourceStep.tsx

@@ -178,10 +178,12 @@ export const NameSourceStep = ({ state, updateState, prev, submit }: StepProps)
                 </Form.Item>
                 <Collapse ghost>
                     <Collapse.Panel header={<Typography.Text type="secondary">Advanced</Typography.Text>} key="1">
-                        <Form.Item label={<Typography.Text strong>Executor Id</Typography.Text>}>
+                        <Form.Item label={<Typography.Text strong>Executor ID</Typography.Text>}>
                             <Typography.Paragraph>
-                                Provide the executor id to route execution requests to. The built-in DataHub executor id
-                                is &apos;default&apos;. Do not change this unless you have configured a custom executor.
+                                Provide the ID of the executor that should execute this ingestion recipe. This ID is used
+                                to route execution requests of the recipe to the executor of the same ID. The built-in
+                                DataHub executor ID is &apos;default&apos;. Do not change this unless you have configured
+                                a remote or custom executor.
                             </Typography.Paragraph>
                             <Input
                                 placeholder="default"

datahub-web-react/src/app/lineage/__tests__/manageLineage.test.tsx

@@ -8,15 +8,15 @@ describe('existsInEntitiesToAdd', () => {
     it('should return false if the search result is not in entitiesAlreadyAdded', () => {
         const result = { entity: { urn: 'urn:li:test' } } as any;
         const entitiesAlreadyAdded = [{ urn: 'urn:li:testing123' }] as any;
-        const exists = existsInEntitiesToAdd(result, entitiesAlreadyAdded);
+        const exists = existsInEntitiesToAdd(result.entity, entitiesAlreadyAdded);
 
         expect(exists).toBe(false);
     });
 
     it('should return true if the search result is in entitiesAlreadyAdded', () => {
         const result = { entity: { urn: 'urn:li:test' } } as any;
         const entitiesAlreadyAdded = [{ urn: 'urn:li:testing123' }, { urn: 'urn:li:test' }] as any;
-        const exists = existsInEntitiesToAdd(result, entitiesAlreadyAdded);
+        const exists = existsInEntitiesToAdd(result.entity, entitiesAlreadyAdded);
 
         expect(exists).toBe(true);
     });

datahub-web-react/src/app/lineage/manage/AddEntityEdge.tsx

@@ -3,8 +3,8 @@ import { AutoComplete, Empty } from 'antd';
 import React, { useState } from 'react';
 import styled from 'styled-components/macro';
 import { useEntityRegistry } from '../../useEntityRegistry';
-import { useGetSearchResultsForMultipleLazyQuery } from '../../../graphql/search.generated';
-import { Entity, EntityType, SearchResult } from '../../../types.generated';
+import { useGetAutoCompleteMultipleResultsLazyQuery } from '../../../graphql/search.generated';
+import { Entity, EntityType } from '../../../types.generated';
 import { Direction } from '../types';
 import { getValidEntityTypes } from '../utils/manageLineageUtils';
 import LineageEntityView from './LineageEntityView';
@@ -62,8 +62,8 @@ function getPlaceholderText(validEntityTypes: EntityType[], entityRegistry: Enti
     return placeholderText;
 }
 
-export function existsInEntitiesToAdd(result: SearchResult, entitiesAlreadyAdded: Entity[]) {
-    return !!entitiesAlreadyAdded.find((entity) => entity.urn === result.entity.urn);
+export function existsInEntitiesToAdd(result: Entity, entitiesAlreadyAdded: Entity[]) {
+    return !!entitiesAlreadyAdded.find((entity) => entity.urn === result.urn);
 }
 
 interface Props {
@@ -82,33 +82,34 @@ export default function AddEntityEdge({
     entityType,
 }: Props) {
     const entityRegistry = useEntityRegistry();
-    const [search, { data: searchData, loading }] = useGetSearchResultsForMultipleLazyQuery();
+    const [getAutoCompleteResults, { data: autoCompleteResults, loading }] =
+        useGetAutoCompleteMultipleResultsLazyQuery();
     const [queryText, setQueryText] = useState<string>('');
 
     const validEntityTypes = getValidEntityTypes(lineageDirection, entityType);
 
     function handleSearch(text: string) {
         setQueryText(text);
         if (text !== '') {
-            search({
+            getAutoCompleteResults({
                 variables: {
                     input: {
                         types: validEntityTypes,
                         query: text,
-                        start: 0,
-                        count: 15,
+                        limit: 15,
                     },
                 },
             });
         }
     }
 
     function selectEntity(urn: string) {
-        const selectedEntity = searchData?.searchAcrossEntities?.searchResults.find(
-            (result) => result.entity.urn === urn,
+        const resultEntities = autoCompleteResults?.autoCompleteForMultiple?.suggestions.flatMap(
+            (suggestion) => suggestion.entities || [],
         );
+        const selectedEntity = resultEntities?.find((entity) => entity.urn === urn);
         if (selectedEntity) {
-            setEntitiesToAdd((existingEntities) => [...existingEntities, selectedEntity.entity]);
+            setEntitiesToAdd((existingEntities) => [...existingEntities, selectedEntity]);
         }
     }
 
@@ -120,9 +121,10 @@ export default function AddEntityEdge({
         );
     };
 
-    const searchResults = searchData?.searchAcrossEntities?.searchResults
-        .filter((result) => !existsInEntitiesToAdd(result, entitiesToAdd) && result.entity.urn !== entityUrn)
-        .map((result) => renderSearchResult(result.entity));
+    const searchResults = autoCompleteResults?.autoCompleteForMultiple?.suggestions
+        .flatMap((suggestion) => suggestion.entities || [])
+        .filter((entity) => entity && !existsInEntitiesToAdd(entity, entitiesToAdd) && entity.urn !== entityUrn)
+        .map((entity) => renderSearchResult(entity));
 
     const placeholderText = getPlaceholderText(validEntityTypes, entityRegistry);
 
@@ -142,7 +144,7 @@ export default function AddEntityEdge({
                 filterOption={false}
                 notFoundContent={(queryText.length > 3 && <Empty description="No Assets Found" />) || undefined}
             >
-                {!searchData && loading && (
+                {loading && (
                     <AutoComplete.Option value="loading">
                         <LoadingWrapper>
                             <LoadingOutlined />

docs-website/sidebars.js

@@ -79,6 +79,18 @@ module.exports = {
               id: "docs/managed-datahub/observe/volume-assertions",
               className: "saasOnly",
             },
+            {
+              label: "Open Assertions Specification",
+              type: "category",
+              link: { type: "doc", id: "docs/assertions/open-assertions-spec" },
+              items: [
+                {
+                  label: "Snowflake",
+                  type: "doc",
+                  id: "docs/assertions/snowflake/snowflake_dmfs",
+                },
+              ],
+            },
           ],
         },
         {

docs/api/tutorials/structured-properties.md

@@ -252,7 +252,7 @@ If successful, you should see `Update succeeded for urn:li:dataset:...`
 </TabItem>
 <TabItem value="OpenAPI" label="OpenAPI">
 
-Following command will set structured properties `retentionTime` as `90` to a dataset `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)`.
+Following command will set structured properties `retentionTime` as `60.0` to a dataset `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)`.
 Please note that the structured property and the dataset must exist before executing this command. (You can create sample datasets using the `datahub docker ingest-sample-data`)
 
 ```commandline
@@ -265,7 +265,7 @@ curl -X 'POST' -v \
     {
       "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime",
       "values": [
-        {"string": "90"}
+        {"double": 60.0}
       ]
     }
   ]
@@ -331,7 +331,7 @@ curl -X 'POST' -v \
 ```
 
 This command will attach one of each of the two properties to our test dataset `urn:li:dataset:(urn:li:dataPlatform:hive,SampleHiveDataset,PROD)`
-Specically, this will set `io.acryl.privacy.retentionTime` as `90` and `io.acryl.privacy.retentionTime02` as `bar2`.
+Specically, this will set `io.acryl.privacy.retentionTime` as `60.0` and `io.acryl.privacy.retentionTime02` as `bar2`.
 
 
 ```
@@ -344,7 +344,7 @@ curl -X 'POST' -v \
     {
       "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime",
       "values": [
-        {"string": "90"}
+        {"double": 60.0}
       ]
     },
     {
@@ -452,7 +452,7 @@ curl -X 'PATCH' -v \
                     "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime",
                     "values": [
                         {
-                            "string": "365"
+                            "double": 365.0
                         }
                     ]
                 }
@@ -485,7 +485,7 @@ Below is the expected response:
           {
             "values": [
               {
-                "string": "365"
+                "double": 365.0
               }
             ],
             "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime"