Merge branch 'datahub-project:master' into master

acryldata · Dec 2, 2024 · 23988df · 23988df
2 parents 5f06dfa + dc87b51
commit 23988df
Show file tree

Hide file tree

Showing 41 changed files with 1,505 additions and 369 deletions.
diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts
@@ -38,6 +38,7 @@ import sigmaLogo from '../../../../images/sigmalogo.png';
 import sacLogo from '../../../../images/saclogo.svg';
 import cassandraLogo from '../../../../images/cassandralogo.png';
 import datahubLogo from '../../../../images/datahublogo.png';
+import neo4j from '../../../../images/neo4j.png';
 
 export const ATHENA = 'athena';
 export const ATHENA_URN = `urn:li:dataPlatform:${ATHENA}`;
@@ -137,6 +138,8 @@ export const DATAHUB_GC = 'datahub-gc';
 export const DATAHUB_LINEAGE_FILE = 'datahub-lineage-file';
 export const DATAHUB_BUSINESS_GLOSSARY = 'datahub-business-glossary';
 export const DATAHUB_URN = `urn:li:dataPlatform:${DATAHUB}`;
+export const NEO4J = 'neo4j';
+export const NEO4J_URN = `urn:li:dataPlatform:${NEO4J}`;
 
 export const PLATFORM_URN_TO_LOGO = {
     [ATHENA_URN]: athenaLogo,
@@ -180,6 +183,7 @@ export const PLATFORM_URN_TO_LOGO = {
     [SAC_URN]: sacLogo,
     [CASSANDRA_URN]: cassandraLogo,
     [DATAHUB_URN]: datahubLogo,
+    [NEO4J_URN]: neo4j,
 };
 
 export const SOURCE_TO_PLATFORM_URN = {

diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json
@@ -309,7 +309,7 @@
         "displayName": "Dremio",
         "description": "Import Spaces, Sources, Tables and statistics from Dremio.",
         "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/",
-        "recipe": "source:\n    type: dremio\n    config:\n        # Coordinates\n        hostname: null\n        port: null\n        #true if https, otherwise false\n        tls: true\n\n        #For cloud instance\n        #is_dremio_cloud: True\n        #dremio_cloud_project_id: <project_id>\n\n        #Credentials with personal access token\n        authentication_method: PAT\n        password: pass\n\n        #Or Credentials with basic auth\n        #authentication_method: password\n        #username: null\n        #password: null\n\n        stateful_ingestion:\n            enabled: true"
+        "recipe": "source:\n    type: dremio\n    config:\n        # Coordinates\n        hostname: null\n        port: null\n        #true if https, otherwise false\n        tls: true\n\n        #For cloud instance\n        #is_dremio_cloud: True\n        #dremio_cloud_project_id: <project_id>\n\n        #Credentials with personal access token\n        authentication_method: PAT\n        password: pass\n\n        #Or Credentials with basic auth\n        #authentication_method: password\n        #username: null\n        #password: null\n\n       ingest_owner: true\n\n        stateful_ingestion:\n            enabled: true"
     },
     {
         "urn": "urn:li:dataPlatform:cassandra",
@@ -325,5 +325,13 @@
         "description": "Ingest databases and tables from any Iceberg catalog implementation",
         "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/iceberg",
         "recipe": "source:\n type: \"iceberg\"\n config:\n   env: dev\n   # each thread will open internet connections to fetch manifest files independently, \n   # this value needs to be adjusted with ulimit\n   processing_threads: 1 \n   # a single catalog definition with a form of a dictionary\n   catalog: \n     demo: # name of the catalog\n       type: \"rest\" # other types are available\n       uri: \"uri\"\n       s3.access-key-id: \"access-key\"\n       s3.secret-access-key: \"secret-access-key\"\n       s3.region: \"aws-region\"\n   profiling:\n     enabled: false\n"
+    },
+    {
+        "urn": "urn:li:dataPlatform:neo4j",
+        "name": "neo4j",
+        "displayName": "Neo4j",
+        "description": "Import Nodes and Relationships from Neo4j.",
+        "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/neo4j/",
+        "recipe": "source:\n    type: 'neo4j'\n    config:\n        uri: 'neo4j+ssc://host:7687'\n        username: 'neo4j'\n        password: 'password'\n        env: 'PROD'\n\nsink:\n  type: \"datahub-rest\"\n  config:\n    server: 'http://localhost:8080'"
     }
 ]
diff --git a/datahub-web-react/src/images/neo4j.png b/datahub-web-react/src/images/neo4j.png
diff --git a/...rc/main/java/com/linkedin/metadata/aspect/patch/builder/CustomPropertiesPatchBuilder.java b/...rc/main/java/com/linkedin/metadata/aspect/patch/builder/CustomPropertiesPatchBuilder.java
@@ -17,10 +17,16 @@ public class CustomPropertiesPatchBuilder<T extends AbstractMultiFieldPatchBuild
   public static final String CUSTOM_PROPERTIES_BASE_PATH = "/customProperties";
 
   private final T parent;
-  private final List<ImmutableTriple<String, String, JsonNode>> operations = new ArrayList<>();
+  private final List<ImmutableTriple<String, String, JsonNode>> operations;
 
   public CustomPropertiesPatchBuilder(T parentBuilder) {
     this.parent = parentBuilder;
+    if (parentBuilder != null) {
+      // If a parent builder is provided, we use the same path operations list.
+      this.operations = parentBuilder.getPathValues();
+    } else {
+      this.operations = new ArrayList<>();
+    }
   }
 
   /**
@@ -72,9 +78,4 @@ public CustomPropertiesPatchBuilder<T> setProperties(Map<String, String> propert
   public T getParent() {
     return parent;
   }
-
-  @Override
-  public List<ImmutableTriple<String, String, JsonNode>> getSubPaths() {
-    return operations;
-  }
 }
diff --git a/...ry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilder.java b/...ry/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataFlowInfoPatchBuilder.java
@@ -4,12 +4,10 @@
 import static com.linkedin.metadata.Constants.DATA_FLOW_ENTITY_NAME;
 import static com.linkedin.metadata.Constants.DATA_FLOW_INFO_ASPECT_NAME;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.linkedin.common.TimeStamp;
 import com.linkedin.metadata.aspect.patch.PatchOperationType;
 import com.linkedin.metadata.aspect.patch.builder.subtypesupport.CustomPropertiesPatchBuilderSupport;
-import java.util.List;
 import java.util.Map;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -87,28 +85,23 @@ public DataFlowInfoPatchBuilder setCreated(@Nullable TimeStamp created) {
   }
 
   public DataFlowInfoPatchBuilder setLastModified(@Nullable TimeStamp lastModified) {
+    ObjectNode lastModifiedNode = instance.objectNode();
     if (lastModified == null) {
       pathValues.add(
           ImmutableTriple.of(
               PatchOperationType.REMOVE.getValue(), BASE_PATH + LAST_MODIFIED_KEY, null));
+    } else {
+      lastModifiedNode.put(TIME_KEY, lastModified.getTime());
+      if (lastModified.getActor() != null) {
+        lastModifiedNode.put(ACTOR_KEY, lastModified.getActor().toString());
+      }
+      pathValues.add(
+          ImmutableTriple.of(
+              PatchOperationType.ADD.getValue(), BASE_PATH + LAST_MODIFIED_KEY, lastModifiedNode));
     }
-    ObjectNode lastModifiedNode = instance.objectNode();
-    lastModifiedNode.put(TIME_KEY, lastModified.getTime());
-    if (lastModified.getActor() != null) {
-      lastModifiedNode.put(ACTOR_KEY, lastModified.getActor().toString());
-    }
-    pathValues.add(
-        ImmutableTriple.of(
-            PatchOperationType.ADD.getValue(), BASE_PATH + LAST_MODIFIED_KEY, lastModifiedNode));
     return this;
   }
 
-  @Override
-  protected List<ImmutableTriple<String, String, JsonNode>> getPathValues() {
-    pathValues.addAll(customPropertiesPatchBuilder.getSubPaths());
-    return pathValues;
-  }
-
   @Override
   protected String getAspectName() {
     return DATA_FLOW_INFO_ASPECT_NAME;

diff --git a/...try/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInfoPatchBuilder.java b/...try/src/main/java/com/linkedin/metadata/aspect/patch/builder/DataJobInfoPatchBuilder.java
@@ -4,13 +4,11 @@
 import static com.linkedin.metadata.Constants.DATA_JOB_ENTITY_NAME;
 import static com.linkedin.metadata.Constants.DATA_JOB_INFO_ASPECT_NAME;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.linkedin.common.TimeStamp;
 import com.linkedin.common.urn.DataFlowUrn;
 import com.linkedin.metadata.aspect.patch.PatchOperationType;
 import com.linkedin.metadata.aspect.patch.builder.subtypesupport.CustomPropertiesPatchBuilderSupport;
-import java.util.List;
 import java.util.Map;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -113,12 +111,6 @@ public DataJobInfoPatchBuilder setLastModified(@Nullable TimeStamp lastModified)
     return this;
   }
 
-  @Override
-  protected List<ImmutableTriple<String, String, JsonNode>> getPathValues() {
-    pathValues.addAll(customPropertiesPatchBuilder.getSubPaths());
-    return pathValues;
-  }
-
   @Override
   protected String getAspectName() {
     return DATA_JOB_INFO_ASPECT_NAME;

diff --git a/...c/main/java/com/linkedin/metadata/aspect/patch/builder/DatasetPropertiesPatchBuilder.java b/...c/main/java/com/linkedin/metadata/aspect/patch/builder/DatasetPropertiesPatchBuilder.java
@@ -4,10 +4,8 @@
 import static com.linkedin.metadata.Constants.DATASET_ENTITY_NAME;
 import static com.linkedin.metadata.Constants.DATASET_PROPERTIES_ASPECT_NAME;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import com.linkedin.metadata.aspect.patch.PatchOperationType;
 import com.linkedin.metadata.aspect.patch.builder.subtypesupport.CustomPropertiesPatchBuilderSupport;
-import java.util.List;
 import java.util.Map;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -116,12 +114,6 @@ public DatasetPropertiesPatchBuilder setCustomProperties(Map<String, String> pro
     return this;
   }
 
-  @Override
-  protected List<ImmutableTriple<String, String, JsonNode>> getPathValues() {
-    pathValues.addAll(customPropertiesPatchBuilder.getSubPaths());
-    return pathValues;
-  }
-
   @Override
   protected String getAspectName() {
     return DATASET_PROPERTIES_ASPECT_NAME;

diff --git a/...a/com/linkedin/metadata/aspect/patch/builder/subtypesupport/IntermediatePatchBuilder.java b/...a/com/linkedin/metadata/aspect/patch/builder/subtypesupport/IntermediatePatchBuilder.java
@@ -1,9 +1,6 @@
 package com.linkedin.metadata.aspect.patch.builder.subtypesupport;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import com.linkedin.metadata.aspect.patch.builder.AbstractMultiFieldPatchBuilder;
-import java.util.List;
-import org.apache.commons.lang3.tuple.ImmutableTriple;
 
 /**
  * Used for supporting intermediate subtypes when constructing a patch for an aspect that includes
@@ -15,10 +12,4 @@ public interface IntermediatePatchBuilder<T extends AbstractMultiFieldPatchBuild
 
   /** Convenience method to return parent patch builder in functional callstack */
   T getParent();
-
-  /**
-   * Exposes subpath values to parent patch builder in Op, Path, Value triples. Should usually only
-   * be called by the parent patch builder class when constructing the path values.
-   */
-  List<ImmutableTriple<String, String, JsonNode>> getSubPaths();
 }