Merge branch 'datahub-project:master' into master

docs-website/sidebars.js

@@ -67,6 +67,12 @@ module.exports = {
               id: "docs/managed-datahub/observe/freshness-assertions",
               className: "saasOnly",
             },
+            {
+              label: "Schema Assertions",
+              type: "doc",
+              id: "docs/managed-datahub/observe/schema-assertions",
+              className: "saasOnly",
+            },
             {
               label: "Volume Assertions",
               type: "doc",
@@ -256,6 +262,7 @@ module.exports = {
         },
         {
           "Managed DataHub Release History": [
+            "docs/managed-datahub/release-notes/v_0_3_2",
             "docs/managed-datahub/release-notes/v_0_3_1",
             "docs/managed-datahub/release-notes/v_0_2_16",
             "docs/managed-datahub/release-notes/v_0_2_15",

docs/managed-datahub/release-notes/v_0_3_2.md

@@ -0,0 +1,34 @@
+# v0.3.2
+---
+
+Release Availability Date
+---
+15-May-2024
+
+Recommended CLI/SDK
+---
+- `v0.13.2.2` with release notes at https://github.com/acryldata/datahub/releases/tag/v0.13.2.2
+
+If you are using an older CLI/SDK version then please upgrade it. This applies for all CLI/SDK usages, if you are using it through your terminal, github actions, airflow, in python SDK somewhere, Java SKD etc. This is a strong recommendation to upgrade as we keep on pushing fixes in the CLI and it helps us support you better.
+
+## Release Changelog
+---
+- Since `v0.3.1` these changes from OSS DataHub https://github.com/datahub-project/datahub/compare/57de905c66b6992aefb2051708fa83898fa82cec...6ed21bd1bc70a3ceb7dddb43ea7db4ca56874547 have been pulled in.
+- OpenAPI Entitites v3 (Entity-registry generated endpoints)
+- Business Attributes Support (disabled by default)
+- Graph Retriever (validators, mutators, side-effects)
+- New Lineage Graphql Endpoints (optimizations for the UI)
+- Metadata Tests Tracks Unique Hash
+- SCIM Support
+- V3 CLL Across DataJob Nodes Upgrade
+- Subscribe to assertion errors: Get notified when assertion errors occur
+- Group owners are labeled in slack incident notifications
+- Field assertions now raise incidents when they go in error (if configured to do so)
+- Assertion timeline viz handles missing data more gracefully
+- Freshness assertions on Snowflake using the AUDIT_LOG will no longer consider no-op queries as valid 'updates' to the dataset
+- [NEW] Schema Assertions: Allows users to configure a set of columns and expected types and verify that they actually exist in the tabe schema. Disabled by default. Please reach out to Acryl team if you wish to have this enabled.
+- Adding polish and fixing bugs throughout the new UI
+- Ability to add and remove lineage manually in the new UI
+- Ability to configure the default time filter on lineage
+- Ability to collapse lineage in the lineage graph (new UI)
+- Support rich text on metadata form descriptions

metadata-ingestion/src/datahub/ingestion/source/sql/hive_metastore.py

@@ -130,8 +130,8 @@ class HiveMetastore(BasicSQLAlchemyConfig):
     )
 
     enable_properties_merge: bool = Field(
-        default=False,
-        description="By default, the connector overwrites properties every time. Set this to True to enable merging of properties with what exists on the server.",
+        default=True,
+        description="By default, the connector enables merging of properties with what exists on the server. Set this to False to enable the default connector behavior of overwriting properties on each ingestion.",
     )
 
     simplify_nested_field_paths: bool = Field(

metadata-ingestion/src/datahub/testing/compare_metadata_json.py

@@ -5,6 +5,7 @@
 import os
 import pathlib
 import pprint
+import re
 import shutil
 import tempfile
 from typing import Any, Dict, List, Sequence, Union
@@ -40,6 +41,7 @@ def assert_metadata_files_equal(
     update_golden: bool,
     copy_output: bool,
     ignore_paths: Sequence[str] = (),
+    ignore_paths_v2: Sequence[str] = (),
     ignore_order: bool = True,
 ) -> None:
     golden_exists = os.path.isfile(golden_path)
@@ -70,6 +72,16 @@ def assert_metadata_files_equal(
                 logger.info(f"Error reformatting golden file as MCP/MCEs: {e}")
                 golden = load_json_file(golden_path)
 
+    if ignore_paths_v2:
+        golden_json = load_json_file(golden_path)
+        for i, obj in enumerate(golden_json):
+            aspect_json = obj.get("aspect", {}).get("json", [])
+            for j, item in enumerate(aspect_json):
+                if isinstance(item, dict):
+                    if item.get("path") in ignore_paths_v2:
+                        json_path = f"root[{i}]['aspect']['json'][{j}]['value']"
+                        ignore_paths = (*ignore_paths, re.escape(json_path))
+
     diff = diff_metadata_json(output, golden, ignore_paths, ignore_order=ignore_order)
     if diff and update_golden:
         if isinstance(diff, MCPDiff):