feat(ingestion): adds env property in ContainerProperties (datahub-pr…

…oject#11214) Co-authored-by: siladitya2 <[email protected]>
acryldata · Sep 18, 2024 · 31edb46 · 31edb46
1 parent e7a3890
commit 31edb46
Show file tree

Hide file tree

Showing 103 changed files with 3,537 additions and 3,524 deletions.
diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md
@@ -38,6 +38,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 - #11313 - `datahub get` will no longer return a key aspect for entities that don't exist.
 - #11369 - The default datahub-rest sink mode has been changed to `ASYNC_BATCH`. This requires a server with version 0.14.0+.
+- #11214 Container properties aspect will produce an additional field that will require a corresponding upgrade of server. Otherwise server can reject the aspects.
 
 ### Potential Downtime
 

diff --git a/metadata-ingestion/src/datahub/emitter/mcp_builder.py b/metadata-ingestion/src/datahub/emitter/mcp_builder.py
@@ -23,6 +23,7 @@
     ContainerClass,
     DomainsClass,
     EmbedClass,
+    FabricTypeClass,
     GlobalTagsClass,
     MetadataChangeEventClass,
     OwnerClass,
@@ -190,6 +191,12 @@ def gen_containers(
     created: Optional[int] = None,
     last_modified: Optional[int] = None,
 ) -> Iterable[MetadataWorkUnit]:
+    # because of backwards compatibility with a past issue, container_key.env may be a valid env or an instance name
+    env = (
+        container_key.env
+        if container_key.env in vars(FabricTypeClass).values()
+        else None
+    )
     container_urn = container_key.as_urn()
     yield MetadataChangeProposalWrapper(
         entityUrn=f"{container_urn}",
@@ -207,6 +214,7 @@ def gen_containers(
             lastModified=(
                 TimeStamp(time=last_modified) if last_modified is not None else None
             ),
+            env=env if env is not None else None,
         ),
     ).as_workunit()
 

diff --git a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_mcp_golden.json
@@ -11,7 +11,8 @@
                 "env": "PROD",
                 "project_id": "project-id-1"
             },
-            "name": "project-id-1"
+            "name": "project-id-1",
+            "env": "PROD"
         }
     },
     "systemMetadata": {
@@ -100,7 +101,8 @@
                 "dataset_id": "bigquery-dataset-1"
             },
             "externalUrl": "https://console.cloud.google.com/bigquery?project=project-id-1&ws=!1m4!1m3!3m2!1sproject-id-1!2sbigquery-dataset-1",
-            "name": "bigquery-dataset-1"
+            "name": "bigquery-dataset-1",
+            "env": "PROD"
         }
     },
     "systemMetadata": {

diff --git a/metadata-ingestion/tests/integration/bigquery_v2/bigquery_project_label_mcp_golden.json b/metadata-ingestion/tests/integration/bigquery_v2/bigquery_project_label_mcp_golden.json
@@ -11,7 +11,8 @@
                 "env": "PROD",
                 "project_id": "dev"
             },
-            "name": "dev"
+            "name": "dev",
+            "env": "PROD"
         }
     },
     "systemMetadata": {
@@ -100,7 +101,8 @@
                 "dataset_id": "bigquery-dataset-1"
             },
             "externalUrl": "https://console.cloud.google.com/bigquery?project=dev&ws=!1m4!1m3!3m2!1sdev!2sbigquery-dataset-1",
-            "name": "bigquery-dataset-1"
+            "name": "bigquery-dataset-1",
+            "env": "PROD"
         }
     },
     "systemMetadata": {