From 7ca9fc328ded391cd9d4a9c7396ef250cddf1825 Mon Sep 17 00:00:00 2001 From: scalvanese452 <62960930+scalvanese452@users.noreply.github.com> Date: Tue, 6 Feb 2024 10:49:33 -0500 Subject: [PATCH 1/4] docs(aws): Update aws.md (#9726) Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com> --- docs/deploy/aws.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/deploy/aws.md b/docs/deploy/aws.md index 6598b93c25e9a1..d060eddd9acc88 100644 --- a/docs/deploy/aws.md +++ b/docs/deploy/aws.md @@ -53,6 +53,8 @@ ip-192-168-64-56.us-west-2.compute.internal Ready 3h v1.18.9-ek ip-192-168-8-126.us-west-2.compute.internal Ready 3h v1.18.9-eks-d1db3c ``` +Once your cluster is running, make sure to install the EBS CSI driver, Core DNS, and VPC CNI plugin for Kubernetes. [add-ons](https://docs.aws.amazon.com/eks/latest/userguide/eks-add-ons.html) + ## Setup DataHub using Helm Once the kubernetes cluster has been set up, you can deploy DataHub and it’s prerequisites using helm. Please follow the From 82035437be4fe32d5ca946531c2abaf095d997e4 Mon Sep 17 00:00:00 2001 From: siladitya <68184387+siladitya2@users.noreply.github.com> Date: Tue, 6 Feb 2024 16:53:51 +0100 Subject: [PATCH 2/4] fix(search): Fix missing mapping of DOUBLE searchable type (#9651) Co-authored-by: si-chakraborty --- .../main/java/com/linkedin/metadata/search/utils/ESUtils.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java index 4d74bfb66b8dbc..86d411e9b5b928 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java @@ -261,6 +261,8 @@ public static String getElasticTypeForFieldType(SearchableAnnotation.FieldType f return DATE_FIELD_TYPE; } else if (fieldType == SearchableAnnotation.FieldType.OBJECT) { return OBJECT_FIELD_TYPE; + } else if (fieldType == SearchableAnnotation.FieldType.DOUBLE) { + return DOUBLE_FIELD_TYPE; } else { log.warn("FieldType {} has no mappings implemented", fieldType); return null; From 11f7804b1ea2f47495c336e8d91d7dc380ccd228 Mon Sep 17 00:00:00 2001 From: david-leifker <114954101+david-leifker@users.noreply.github.com> Date: Tue, 6 Feb 2024 11:09:54 -0600 Subject: [PATCH 3/4] chore(lint): apply spotless (#9789) --- .../java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index 8d9b9a5ad82c84..e3eef0688c2692 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -1643,7 +1643,8 @@ private void configureResolvedAuditStampResolvers(final RuntimeWiring.Builder bu typeWiring.dataFetcher( "actor", new LoadableTypeResolver<>( - corpUserType, (env) -> ((ResolvedAuditStamp) env.getSource()).getActor().getUrn()))); + corpUserType, + (env) -> ((ResolvedAuditStamp) env.getSource()).getActor().getUrn()))); } /** From e1c8ac732098b1b05c381bb33c9c4c2043510d34 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Wed, 7 Feb 2024 06:58:09 +0530 Subject: [PATCH 4/4] docs(observe): add upsert assertion monitor graphql examples (#9766) --- .../observe/column-assertions.md | 85 +++++++++++++++++++ .../observe/custom-sql-assertions.md | 59 +++++++++++++ .../observe/freshness-assertions.md | 53 ++++++++++++ .../observe/volume-assertions.md | 73 ++++++++++++++++ 4 files changed, 270 insertions(+) diff --git a/docs/managed-datahub/observe/column-assertions.md b/docs/managed-datahub/observe/column-assertions.md index 8ef32e73b4b729..a5dcb02c87badd 100644 --- a/docs/managed-datahub/observe/column-assertions.md +++ b/docs/managed-datahub/observe/column-assertions.md @@ -340,6 +340,91 @@ This entity defines _when_ to run the check (Using CRON format - every 8th hour) After creating the monitor, the new assertion will start to be evaluated every 8 hours in your selected timezone. +Alternatively you can use `upsertDatasetFieldAssertionMonitor` graphql endpoint for creating a Column Assertion and corresponding Monitor for a dataset. + +```json +mutation upsertDatasetFieldAssertionMonitor { + upsertDatasetFieldAssertionMonitor( + input: { + entityUrn: "" + type: FIELD_VALUES, + fieldValuesAssertion: { + field: { + path: "", + type: "NUMBER", + nativeType: "NUMBER(38,0)" + }, + operator: GREATER_THAN, + parameters: { + value: { + type: NUMBER, + value: "10" + } + }, + failThreshold: { + type: COUNT, + value: 0 + }, + excludeNulls: true + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + evaluationParameters: { + sourceType: ALL_ROWS_QUERY + } + mode: ACTIVE + } + ){ + urn + } +} +``` + +You can use same endpoint with assertion urn input to update an existing Column Assertion and corresponding Monitor. + +```json +mutation upsertDatasetFieldAssertionMonitor { + upsertDatasetFieldAssertionMonitor( + assertionUrn: "" + input: { + entityUrn: "" + type: FIELD_VALUES, + fieldValuesAssertion: { + field: { + path: "", + type: "NUMBER", + nativeType: "NUMBER(38,0)" + }, + operator: GREATER_THAN_OR_EQUAL_TO, + parameters: { + value: { + type: NUMBER, + value: "10" + } + }, + failThreshold: { + type: COUNT, + value: 0 + }, + excludeNulls: true + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + evaluationParameters: { + sourceType: ALL_ROWS_QUERY + } + mode: ACTIVE + } + ){ + urn + } +} +``` + You can delete assertions along with their monitors using GraphQL mutations: `deleteAssertion` and `deleteMonitor`. ### Tips diff --git a/docs/managed-datahub/observe/custom-sql-assertions.md b/docs/managed-datahub/observe/custom-sql-assertions.md index 581b5426881347..434788ff45a6fd 100644 --- a/docs/managed-datahub/observe/custom-sql-assertions.md +++ b/docs/managed-datahub/observe/custom-sql-assertions.md @@ -297,6 +297,65 @@ This entity defines _when_ to run the check (Using CRON format - every 8th hour) After creating the monitor, the new assertion will start to be evaluated every 8 hours in your selected timezone. +Alternatively you can use `upsertDatasetSqlAssertionMonitor` graphql endpoint for creating a Custom SQL Assertion and corresponding Monitor for a dataset. + +```json +mutation upsertDatasetSqlAssertionMonitor { + upsertDatasetSqlAssertionMonitor( + input: { + entityUrn: "" + type: METRIC, + description: "", + statement: "", + operator: GREATER_THAN_OR_EQUAL_TO, + parameters: { + value: { + value: "100", + type: NUMBER + } + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + mode: ACTIVE + } + ) { + urn + } +} +``` + +You can use same endpoint with assertion urn input to update an existing Custom SQL Assertion and corresponding Monitor. + +```json +mutation upsertDatasetSqlAssertionMonitor { + upsertDatasetSqlAssertionMonitor( + assertionUrn: "" + input: { + entityUrn: "" + type: METRIC, + description: "", + statement: "", + operator: GREATER_THAN_OR_EQUAL_TO, + parameters: { + value: { + value: "100", + type: NUMBER + } + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */6 * * *" + } + mode: ACTIVE + } + ) { + urn + } +} +``` + You can delete assertions along with their monitors using GraphQL mutations: `deleteAssertion` and `deleteMonitor`. ### Tips diff --git a/docs/managed-datahub/observe/freshness-assertions.md b/docs/managed-datahub/observe/freshness-assertions.md index 9704f475b15873..50cae1567f93a6 100644 --- a/docs/managed-datahub/observe/freshness-assertions.md +++ b/docs/managed-datahub/observe/freshness-assertions.md @@ -346,6 +346,59 @@ This entity defines _when_ to run the check (Using CRON format - every 8th hour) After creating the monitor, the new assertion will start to be evaluated every 8 hours in your selected timezone. +Alternatively you can use `upsertDatasetFreshnessAssertionMonitor` graphql endpoint for creating a Freshness Assertion and corresponding Monitor for a dataset. + +```json +mutation upsertDatasetFreshnessAssertionMonitor { + upsertDatasetFreshnessAssertionMonitor( + input: { + entityUrn: "", + schedule: { + type: FIXED_INTERVAL, + fixedInterval: { unit: HOUR, multiple: 8 } + } + evaluationSchedule: { + timezone: "America/Los_Angeles", + cron: "0 */8 * * *" + } + evaluationParameters: { + sourceType: INFORMATION_SCHEMA + } + mode: ACTIVE + } + ){ + urn + } +} +``` + +You can use same endpoint with assertion urn input to update an existing Freshness Assertion and corresponding Monitor. + +```json +mutation upsertDatasetFreshnessAssertionMonitor { + upsertDatasetFreshnessAssertionMonitor( + assertionUrn: "" + input: { + entityUrn: "", + schedule: { + type: FIXED_INTERVAL, + fixedInterval: { unit: HOUR, multiple: 6 } + } + evaluationSchedule: { + timezone: "America/Los_Angeles", + cron: "0 */6 * * *" + } + evaluationParameters: { + sourceType: INFORMATION_SCHEMA + } + mode: ACTIVE + } + ){ + urn + } +} +``` + You can delete assertions along with their monitors using GraphQL mutations: `deleteAssertion` and `deleteMonitor`. ### Reporting Operations via API diff --git a/docs/managed-datahub/observe/volume-assertions.md b/docs/managed-datahub/observe/volume-assertions.md index 67971d0c20037a..f925920a0c248f 100644 --- a/docs/managed-datahub/observe/volume-assertions.md +++ b/docs/managed-datahub/observe/volume-assertions.md @@ -337,6 +337,79 @@ This entity defines _when_ to run the check (Using CRON format - every 8th hour) After creating the monitor, the new assertion will start to be evaluated every 8 hours in your selected timezone. +Alternatively you can use `upsertDatasetVolumeAssertionMonitor` graphql endpoint for creating a Volume Assertion and corresponding Monitor. + +```json +mutation upsertDatasetVolumeAssertionMonitor { + upsertDatasetVolumeAssertionMonitor( + input: { + entityUrn: "" + type: ROW_COUNT_TOTAL + rowCountTotal: { + operator: BETWEEN + parameters: { + minValue: { + value: "10" + type: NUMBER + } + maxValue: { + value: "20" + type: NUMBER + } + } + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */8 * * *" + } + evaluationParameters: { + sourceType: INFORMATION_SCHEMA + } + mode: ACTIVE + } + ) { + urn + } +} +``` + +You can use same endpoint with assertion urn input to update an existing Volume Assertion and corresponding Monitor. + +```json +mutation upsertDatasetVolumeAssertionMonitor { + upsertDatasetVolumeAssertionMonitor( + assertionUrn: "" + input: { + entityUrn: "" + type: ROW_COUNT_TOTAL + rowCountTotal: { + operator: BETWEEN + parameters: { + minValue: { + value: "10" + type: NUMBER + } + maxValue: { + value: "20" + type: NUMBER + } + } + } + evaluationSchedule: { + timezone: "America/Los_Angeles" + cron: "0 */6 * * *" + } + evaluationParameters: { + sourceType: INFORMATION_SCHEMA + } + mode: ACTIVE + } + ) { + urn + } +} +``` + You can delete assertions along with their monitors using GraphQL mutations: `deleteAssertion` and `deleteMonitor`. ### Tips