Merge branch 'datahub-project:master' into master

.github/workflows/docker-unified.yml

@@ -64,7 +64,7 @@ jobs:
           echo "full_tag=$(get_tag)-full" >> $GITHUB_OUTPUT
           echo "unique_tag=$(get_unique_tag)" >> $GITHUB_OUTPUT
           echo "unique_slim_tag=$(get_unique_tag)-slim" >> $GITHUB_OUTPUT
-          echo "unique_full_tag=$(get_unique_tag)-full" >> $GITHUB_OUTPUT
+          echo "unique_full_tag=$(get_unique_tag)" >> $GITHUB_OUTPUT
           echo "python_release_version=$(get_python_docker_release_v)" >> $GITHUB_OUTPUT
       - name: Check whether publishing enabled
         id: publish
@@ -507,7 +507,7 @@ jobs:
           platforms: linux/amd64,linux/arm64/v8
       - name: Compute DataHub Ingestion (Base-Slim) Tag
         id: tag
-        run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head' }}" >> $GITHUB_OUTPUT
+        run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT
   datahub_ingestion_base_full_build:
     name: Build and Push DataHub Ingestion (Base-Full) Docker Image
     runs-on: ubuntu-latest
@@ -573,13 +573,13 @@ jobs:
             datahub-ingestion:
               - 'docker/datahub-ingestion/**'
       - name: Build codegen
-        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' }}
+        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
         run: ./gradlew :metadata-ingestion:codegen
       - name: Download Base Image
         uses: ishworkh/docker-image-artifact-download@v1
         if: ${{ needs.setup.outputs.publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }}
         with:
-          image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head' }}
+          image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}
       - name: Build and push Slim Image
         if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
         uses: ./.github/actions/docker-custom-build-and-push
@@ -589,7 +589,7 @@ jobs:
             ${{ env.DATAHUB_INGESTION_IMAGE }}
           build-args: |
             BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
-            DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head' }}
+            DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}
             RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }}
             APP_ENV=slim
           tags: ${{ needs.setup.outputs.slim_tag }}
@@ -601,7 +601,7 @@ jobs:
           platforms: linux/amd64,linux/arm64/v8
       - name: Compute Tag
         id: tag
-        run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_slim_tag || 'head' }}" >> $GITHUB_OUTPUT
+        run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT
   datahub_ingestion_slim_scan:
     permissions:
       contents: read # for actions/checkout to fetch code
@@ -656,7 +656,7 @@ jobs:
             datahub-ingestion:
               - 'docker/datahub-ingestion/**'
       - name: Build codegen
-        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' }}
+        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
         run: ./gradlew :metadata-ingestion:codegen
       - name: Download Base Image
         uses: ishworkh/docker-image-artifact-download@v1

build.gradle

@@ -289,6 +289,11 @@ subprojects {
     }
     // https://docs.gradle.org/current/userguide/performance.html
     maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
+
+    if (project.configurations.getByName("testImplementation").getDependencies()
+            .any{ it.getName() == "testng" }) {
+      useTestNG()
+    }
   }
 
   afterEvaluate {

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/auth/ListAccessTokensResolverTest.java

@@ -1,5 +1,6 @@
 package com.linkedin.datahub.graphql.resolvers.auth;
 
+import com.datahub.authentication.Authentication;
 import com.google.common.collect.ImmutableList;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.TestUtils;
@@ -8,6 +9,10 @@
 import com.linkedin.datahub.graphql.generated.ListAccessTokenResult;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.Constants;
+import com.linkedin.metadata.query.SearchFlags;
+import com.linkedin.metadata.query.filter.SortCriterion;
+import com.linkedin.metadata.search.SearchEntityArray;
+import com.linkedin.metadata.search.SearchResult;
 import graphql.schema.DataFetchingEnvironment;
 import java.util.Collections;
 import org.mockito.Mockito;
@@ -36,14 +41,17 @@ public void testGetSuccess() throws Exception {
     Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(input);
 
     final EntityClient mockClient = Mockito.mock(EntityClient.class);
-    Mockito.when(Mockito.eq(mockClient.filter(
-        Mockito.eq(Constants.ACCESS_TOKEN_ENTITY_NAME),
-            Mockito.eq(buildFilter(filters, Collections.emptyList())),
-            Mockito.notNull(),
-            Mockito.eq(input.getStart()),
-            Mockito.eq(input.getCount()),
-            Mockito.eq(getAuthentication(mockEnv)))))
-        .thenReturn(null);
+    final Authentication testAuth = getAuthentication(mockEnv);
+    Mockito.when(mockClient.search(
+                    Mockito.eq(Constants.ACCESS_TOKEN_ENTITY_NAME),
+                    Mockito.eq(""),
+                    Mockito.eq(buildFilter(filters, Collections.emptyList())),
+                    Mockito.any(SortCriterion.class),
+                    Mockito.eq(input.getStart()),
+                    Mockito.eq(input.getCount()),
+                    Mockito.eq(testAuth),
+                    Mockito.any(SearchFlags.class)))
+            .thenReturn(new SearchResult().setFrom(0).setNumEntities(0).setPageSize(0).setEntities(new SearchEntityArray()));
 
     final ListAccessTokensResolver resolver = new ListAccessTokensResolver(mockClient);
     final ListAccessTokenResult listAccessTokenResult = resolver.get(mockEnv).get();

datahub-web-react/src/app/ingest/source/builder/constants.ts

@@ -27,6 +27,7 @@ import powerbiLogo from '../../../../images/powerbilogo.png';
 import modeLogo from '../../../../images/modelogo.png';
 import databricksLogo from '../../../../images/databrickslogo.png';
 import verticaLogo from '../../../../images/verticalogo.png';
+import mlflowLogo from '../../../../images/mlflowlogo.png';
 import dynamodbLogo from '../../../../images/dynamodblogo.png';
 
 export const ATHENA = 'athena';
@@ -64,6 +65,8 @@ export const MARIA_DB = 'mariadb';
 export const MARIA_DB_URN = `urn:li:dataPlatform:${MARIA_DB}`;
 export const METABASE = 'metabase';
 export const METABASE_URN = `urn:li:dataPlatform:${METABASE}`;
+export const MLFLOW = 'mlflow';
+export const MLFLOW_URN = `urn:li:dataPlatform:${MLFLOW}`;
 export const MODE = 'mode';
 export const MODE_URN = `urn:li:dataPlatform:${MODE}`;
 export const MONGO_DB = 'mongodb';
@@ -119,6 +122,7 @@ export const PLATFORM_URN_TO_LOGO = {
     [LOOKER_URN]: lookerLogo,
     [MARIA_DB_URN]: mariadbLogo,
     [METABASE_URN]: metabaseLogo,
+    [MLFLOW_URN]: mlflowLogo,
     [MODE_URN]: modeLogo,
     [MONGO_DB_URN]: mongodbLogo,
     [MSSQL_URN]: mssqlLogo,

datahub-web-react/src/app/ingest/source/builder/sources.json

@@ -181,6 +181,13 @@
         "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/metabase/",
         "recipe": "source:\n  type: metabase\n  config:\n    # Coordinates\n    connect_uri:\n\n    # Credentials\n    username: root\n    password: example"
     },
+    {
+        "urn": "urn:li:dataPlatform:mlflow",
+        "name": "mlflow",
+        "displayName": "MLflow",
+        "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mlflow/",
+        "recipe": "source:\n  type: mlflow\n  config:\n    tracking_uri: tracking_uri"
+    },
     {
         "urn": "urn:li:dataPlatform:mode",
         "name": "mode",

docker/datahub-ingestion-base/Dockerfile

@@ -1,7 +1,7 @@
 ARG APP_ENV=full
 ARG BASE_IMAGE=base
 
-FROM golang:1-alpine3.17 AS binary
+FROM golang:1-alpine3.17 AS dockerize-binary
 
 ENV DOCKERIZE_VERSION v0.6.1
 WORKDIR /go/src/github.com/jwilder
@@ -41,7 +41,7 @@ RUN apt-get update && apt-get install -y -qq \
     && rm -rf /var/lib/apt/lists/* /var/cache/apk/*
 
 # compiled against newer golang for security fixes
-COPY --from=binary /go/bin/dockerize /usr/local/bin
+COPY --from=dockerize-binary /go/bin/dockerize /usr/local/bin
 
 COPY ./docker/datahub-ingestion-base/base-requirements.txt requirements.txt
 COPY ./docker/datahub-ingestion-base/entrypoint.sh /entrypoint.sh

docker/datahub-ingestion/Dockerfile

@@ -1,7 +1,7 @@
 # Defining environment
 ARG APP_ENV=full
 ARG BASE_IMAGE=acryldata/datahub-ingestion-base
-ARG DOCKER_VERSION=latest
+ARG DOCKER_VERSION=head
 
 FROM $BASE_IMAGE:$DOCKER_VERSION as base
 USER 0

docker/datahub-ingestion/Dockerfile-slim-only

@@ -1,6 +1,6 @@
 # Defining environment
 ARG BASE_IMAGE=acryldata/datahub-ingestion-base
-ARG DOCKER_VERSION=latest
+ARG DOCKER_VERSION=head-slim
 
 FROM $BASE_IMAGE:$DOCKER_VERSION as base
 USER 0

docker/postgres-setup/init.sh

@@ -1,8 +1,13 @@
 #!/bin/sh
 export PGPASSWORD=$POSTGRES_PASSWORD
 
+POSTGRES_CREATE_DB=${POSTGRES_CREATE_DB:-true}
+POSTGRES_CREATE_DB_CONNECTION_DB=${POSTGRES_CREATE_DB_CONNECTION_DB:-postgres}
+
 # workaround create database if not exists, check https://stackoverflow.com/a/36591842
-psql -U $POSTGRES_USERNAME -h $POSTGRES_HOST -p $POSTGRES_PORT -tc "SELECT 1 FROM pg_database WHERE datname = '${DATAHUB_DB_NAME}'" | grep -q 1 || psql -U $POSTGRES_USERNAME -h $POSTGRES_HOST -p $POSTGRES_PORT -c "CREATE DATABASE ${DATAHUB_DB_NAME}"
+if [ "$POSTGRES_CREATE_DB" = true ]; then
+    psql -d "$POSTGRES_CREATE_DB_CONNECTION_DB" -U "$POSTGRES_USERNAME" -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -tc "SELECT 1 FROM pg_database WHERE datname = '${DATAHUB_DB_NAME}'" | grep -q 1 || psql -d "$POSTGRES_CREATE_DB_CONNECTION_DB" -U "$POSTGRES_USERNAME" -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -c "CREATE DATABASE ${DATAHUB_DB_NAME}"
+fi
 
 sed -e "s/DATAHUB_DB_NAME/${DATAHUB_DB_NAME}/g" /init.sql | tee -a /tmp/init-final.sql
-psql -d $DATAHUB_DB_NAME -U $POSTGRES_USERNAME -h $POSTGRES_HOST -p $POSTGRES_PORT < /tmp/init-final.sql
+psql -d "$DATAHUB_DB_NAME" -U "$POSTGRES_USERNAME" -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" < /tmp/init-final.sql

docs-website/docusaurus.config.js

@@ -12,13 +12,11 @@ module.exports = {
   organizationName: "datahub-project", // Usually your GitHub org/user name.
   projectName: "datahub", // Usually your repo name.
   staticDirectories: ["static", "genStatic"],
-  stylesheets: [
-    "https://fonts.googleapis.com/css2?family=Manrope:wght@400;600&display=swap",
-  ],
+  stylesheets: ["https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;700&display=swap"],
   noIndex: isSaas,
   customFields: {
     isSaas: isSaas,
-    markpromptProjectKey: process.env.DOCUSAURUS_MARKPROMPT_PROJECT_KEY || 'IeF3CUFCUQWuouZ8MP5Np9nES52QAtaA',
+    markpromptProjectKey: process.env.DOCUSAURUS_MARKPROMPT_PROJECT_KEY || "IeF3CUFCUQWuouZ8MP5Np9nES52QAtaA",
   },
   themeConfig: {
     ...(!isSaas && {
@@ -35,12 +33,8 @@ module.exports = {
       title: null,
       logo: {
         alt: "DataHub Logo",
-        src: `img/${
-          isSaas ? "acryl" : "datahub"
-        }-logo-color-light-horizontal.svg`,
-        srcDark: `img/${
-          isSaas ? "acryl" : "datahub"
-        }-logo-color-dark-horizontal.svg`,
+        src: `img/${isSaas ? "acryl" : "datahub"}-logo-color-light-horizontal.svg`,
+        srcDark: `img/${isSaas ? "acryl" : "datahub"}-logo-color-dark-horizontal.svg`,
       },
       items: [
         {
@@ -50,7 +44,8 @@ module.exports = {
           position: "right",
         },
         {
-          href: "/integrations",
+          to: "/integrations",
+          activeBasePath: "integrations",
           label: "Integrations",
           position: "right",
         },
@@ -70,8 +65,8 @@ module.exports = {
           position: "right",
         },
         {
-          type: 'docsVersionDropdown',
-          position: 'right',
+          type: "docsVersionDropdown",
+          position: "right",
           dropdownActiveClassDisabled: true,
         },
         {
@@ -201,9 +196,7 @@ module.exports = {
         blog: false,
         theme: {
           customCss: [
-            isSaas
-              ? require.resolve("./src/styles/acryl.scss")
-              : require.resolve("./src/styles/datahub.scss"),
+            isSaas ? require.resolve("./src/styles/acryl.scss") : require.resolve("./src/styles/datahub.scss"),
             require.resolve("./src/styles/global.scss"),
             require.resolve("./src/styles/sphinx.scss"),
             require.resolve("./src/styles/config-table.scss"),
@@ -220,10 +213,7 @@ module.exports = {
     ],
   ],
   plugins: [
-    [
-      "@docusaurus/plugin-ideal-image",
-      { quality: 100, sizes: [320, 640, 1280, 1440, 1600] },
-    ],
+    ["@docusaurus/plugin-ideal-image", { quality: 100, sizes: [320, 640, 1280, 1440, 1600] }],
     "docusaurus-plugin-sass",
     [
       "docusaurus-graphql-plugin",

docs-website/sidebars.js

@@ -81,6 +81,13 @@ module.exports = {
                 "docs/quick-ingestion-guides/powerbi/configuration",
               ],
             },
+            {
+              Looker: [
+                "docs/quick-ingestion-guides/looker/overview",
+                "docs/quick-ingestion-guides/looker/setup",
+                "docs/quick-ingestion-guides/looker/configuration",
+              ],
+            },
           ],
         },
         {
@@ -437,7 +444,7 @@ module.exports = {
           Observability: [
             "docs/managed-datahub/observe/freshness-assertions",
             "docs/managed-datahub/observe/volume-assertions",
-            "docs/managed-datahub/observe/custom-assertions",
+            "docs/managed-datahub/observe/custom-sql-assertions",
           ],
         },
       ],

docs-website/src/components/Feedback/styles.module.scss

@@ -37,11 +37,11 @@
 }
 
 .feedbackText {
+  font-family: var(--ifm-font-family-base);
   width: 100%;
   border: var(--ifm-hr-border-color) 1px solid;
   border-radius: 0.4rem;
   padding: 0.4rem;
-  font-family: "Manrope", sans-serif;
 }
 
 .feedbackButton {

docs-website/src/components/MarkpromptHelp/markprompthelp.module.scss

@@ -325,7 +325,6 @@ button {
   padding-left: 1.5714286em;
 }
 .MarkpromptAnswer ol > li::marker {
-  font-weight: 400;
   color: var(--markprompt-foreground);
 }
 .MarkpromptAnswer ul > li::marker {
@@ -454,7 +453,6 @@ button {
   background-color: var(--markprompt-muted);
   border: 1px solid var(--markprompt-border);
   overflow-x: auto;
-  font-weight: 400;
   font-size: 0.8571429em;
   line-height: 1.6666667;
   margin-top: 1.6666667em;

docs-website/src/pages/_components/Hero/hero.module.scss

@@ -74,3 +74,26 @@
     margin-right: 0.5rem;
   }
 }
+
+.quickstartContent {
+  text-align: center;
+  padding: 2rem 0;
+  height: 100%;
+  margin: 2rem 0;
+  background: #34394d;
+  border-radius: var(--ifm-card-border-radius);
+}
+
+.quickstartTitle {
+  color: #fafafa;
+}
+
+.quickstartSubtitle {
+  font-size: 1.1rem;
+  color: gray;
+}
+
+.quickstartCodeblock {
+  text-align: left;
+  padding: 0 20vh;
+}

docs-website/src/pages/_components/Hero/index.js

@@ -34,12 +34,11 @@ const Hero = ({}) => {
               complexity of your data ecosystem.
             </p>
             <p className="hero__subtitle">
-              Built with ❤️ by{" "}
-              <img src="https://datahubproject.io/img/acryl-logo-light-mark.png" style={{ "vertical-align": "text-top;" }} width="25" />{" "}
+              Built with ❤️ by <img src={useBaseUrl("/img/acryl-logo-transparent-mark.svg")} width="25" />{" "}
               <a href="https://acryldata.io" target="blank" rel="noopener noreferrer">
                 Acryl Data
               </a>{" "}
-              and <img src="https://datahubproject.io/img/LI-In-Bug.png" width="25" /> LinkedIn.
+              and <img src={useBaseUrl("img/LI-In-Bug.png")} width="25" /> LinkedIn.
             </p>
             <Link className="button button--primary button--md" to={useBaseUrl("docs/")}>
               Get Started →
@@ -51,10 +50,10 @@ const Hero = ({}) => {
         </div>
         <CardCTAs />
         <Image className="hero__image" img={require(`/img/diagrams/datahub-flow-diagram-${colorMode}.png`)} alt="DataHub Flow Diagram" />
-        <div className="quickstart__content">
-          <h1 className="quickstart__title">Get Started Now</h1>
-          <p className="quickstart__subtitle">Run the following command to get started with DataHub.</p>
-          <div className="quickstart__codeblock">
+        <div className={styles.quickstartContent}>
+          <h1 className={styles.quickstartTitle}>Get Started Now</h1>
+          <p className={styles.quickstartSubtitle}>Run the following command to get started with DataHub.</p>
+          <div className={styles.quickstartCodeblock}>
             <CodeBlock className={"language-shell"}>
               python3 -m pip install --upgrade pip wheel setuptools <br />
               python3 -m pip install --upgrade acryl-datahub <br />

docs-website/src/pages/_components/Section/section.module.scss

@@ -9,7 +9,6 @@
 .sectionTitle {
   font-size: 2.5rem;
   margin-bottom: 3rem;
-  font-weight: normal;
   text-align: center;
 }