Merge branch 'master' into CUS-903-dbt-meta-mapping

ethan-cartwright · Sep 26, 2023 · e81bbc0 · e81bbc0
2 parents c2f3b5e + f95d1ae
commit e81bbc0
Show file tree

Hide file tree

Showing 38 changed files with 1,498 additions and 169 deletions.
diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -58,7 +58,7 @@ jobs:
           echo "full_tag=$(get_tag)-full" >> $GITHUB_OUTPUT
           echo "unique_tag=$(get_unique_tag)" >> $GITHUB_OUTPUT
           echo "unique_slim_tag=$(get_unique_tag)-slim" >> $GITHUB_OUTPUT
-          echo "unique_full_tag=$(get_unique_tag)-full" >> $GITHUB_OUTPUT
+          echo "unique_full_tag=$(get_unique_tag)" >> $GITHUB_OUTPUT
           echo "python_release_version=$(get_python_docker_release_v)" >> $GITHUB_OUTPUT
       - name: Check whether publishing enabled
         id: publish
@@ -501,7 +501,7 @@ jobs:
           platforms: linux/amd64,linux/arm64/v8
       - name: Compute DataHub Ingestion (Base-Slim) Tag
         id: tag
-        run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head' }}" >> $GITHUB_OUTPUT
+        run: echo "tag=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT
   datahub_ingestion_base_full_build:
     name: Build and Push DataHub Ingestion (Base-Full) Docker Image
     runs-on: ubuntu-latest
@@ -567,13 +567,13 @@ jobs:
             datahub-ingestion:
               - 'docker/datahub-ingestion/**'
       - name: Build codegen
-        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' }}
+        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
         run: ./gradlew :metadata-ingestion:codegen
       - name: Download Base Image
         uses: ishworkh/docker-image-artifact-download@v1
         if: ${{ needs.setup.outputs.publish != 'true' && steps.filter.outputs.datahub-ingestion-base == 'true' }}
         with:
-          image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head' }}
+          image: ${{ env.DATAHUB_INGESTION_BASE_IMAGE }}:${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}
       - name: Build and push Slim Image
         if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
         uses: ./.github/actions/docker-custom-build-and-push
@@ -583,7 +583,7 @@ jobs:
             ${{ env.DATAHUB_INGESTION_IMAGE }}
           build-args: |
             BASE_IMAGE=${{ env.DATAHUB_INGESTION_BASE_IMAGE }}
-            DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head' }}
+            DOCKER_VERSION=${{ steps.filter.outputs.datahub-ingestion-base == 'true' && needs.setup.outputs.unique_slim_tag || 'head-slim' }}
             RELEASE_VERSION=${{ needs.setup.outputs.python_release_version }}
             APP_ENV=slim
           tags: ${{ needs.setup.outputs.slim_tag }}
@@ -595,7 +595,7 @@ jobs:
           platforms: linux/amd64,linux/arm64/v8
       - name: Compute Tag
         id: tag
-        run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_slim_tag || 'head' }}" >> $GITHUB_OUTPUT
+        run: echo "tag=${{ (steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true') && needs.setup.outputs.unique_slim_tag || 'head-slim' }}" >> $GITHUB_OUTPUT
   datahub_ingestion_slim_scan:
     permissions:
       contents: read # for actions/checkout to fetch code
@@ -650,7 +650,7 @@ jobs:
             datahub-ingestion:
               - 'docker/datahub-ingestion/**'
       - name: Build codegen
-        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' }}
+        if: ${{ steps.filter.outputs.datahub-ingestion-base == 'true' || steps.filter.outputs.datahub-ingestion == 'true' || needs.setup.outputs.publish }}
         run: ./gradlew :metadata-ingestion:codegen
       - name: Download Base Image
         uses: ishworkh/docker-image-artifact-download@v1

diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts
@@ -27,6 +27,7 @@ import powerbiLogo from '../../../../images/powerbilogo.png';
 import modeLogo from '../../../../images/modelogo.png';
 import databricksLogo from '../../../../images/databrickslogo.png';
 import verticaLogo from '../../../../images/verticalogo.png';
+import mlflowLogo from '../../../../images/mlflowlogo.png';
 import dynamodbLogo from '../../../../images/dynamodblogo.png';
 
 export const ATHENA = 'athena';
@@ -64,6 +65,8 @@ export const MARIA_DB = 'mariadb';
 export const MARIA_DB_URN = `urn:li:dataPlatform:${MARIA_DB}`;
 export const METABASE = 'metabase';
 export const METABASE_URN = `urn:li:dataPlatform:${METABASE}`;
+export const MLFLOW = 'mlflow';
+export const MLFLOW_URN = `urn:li:dataPlatform:${MLFLOW}`;
 export const MODE = 'mode';
 export const MODE_URN = `urn:li:dataPlatform:${MODE}`;
 export const MONGO_DB = 'mongodb';
@@ -119,6 +122,7 @@ export const PLATFORM_URN_TO_LOGO = {
     [LOOKER_URN]: lookerLogo,
     [MARIA_DB_URN]: mariadbLogo,
     [METABASE_URN]: metabaseLogo,
+    [MLFLOW_URN]: mlflowLogo,
     [MODE_URN]: modeLogo,
     [MONGO_DB_URN]: mongodbLogo,
     [MSSQL_URN]: mssqlLogo,

diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json
@@ -181,6 +181,13 @@
         "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/metabase/",
         "recipe": "source:\n  type: metabase\n  config:\n    # Coordinates\n    connect_uri:\n\n    # Credentials\n    username: root\n    password: example"
     },
+    {
+        "urn": "urn:li:dataPlatform:mlflow",
+        "name": "mlflow",
+        "displayName": "MLflow",
+        "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mlflow/",
+        "recipe": "source:\n  type: mlflow\n  config:\n    tracking_uri: tracking_uri"
+    },
     {
         "urn": "urn:li:dataPlatform:mode",
         "name": "mode",

diff --git a/datahub-web-react/src/images/mlflowlogo.png b/datahub-web-react/src/images/mlflowlogo.png
diff --git a/docker/datahub-ingestion-base/Dockerfile b/docker/datahub-ingestion-base/Dockerfile
@@ -1,7 +1,7 @@
 ARG APP_ENV=full
 ARG BASE_IMAGE=base
 
-FROM golang:1-alpine3.17 AS binary
+FROM golang:1-alpine3.17 AS dockerize-binary
 
 ENV DOCKERIZE_VERSION v0.6.1
 WORKDIR /go/src/github.com/jwilder
@@ -41,7 +41,7 @@ RUN apt-get update && apt-get install -y -qq \
     && rm -rf /var/lib/apt/lists/* /var/cache/apk/*
 
 # compiled against newer golang for security fixes
-COPY --from=binary /go/bin/dockerize /usr/local/bin
+COPY --from=dockerize-binary /go/bin/dockerize /usr/local/bin
 
 COPY ./docker/datahub-ingestion-base/base-requirements.txt requirements.txt
 COPY ./docker/datahub-ingestion-base/entrypoint.sh /entrypoint.sh

diff --git a/docker/datahub-ingestion/Dockerfile b/docker/datahub-ingestion/Dockerfile
@@ -1,7 +1,7 @@
 # Defining environment
 ARG APP_ENV=full
 ARG BASE_IMAGE=acryldata/datahub-ingestion-base
-ARG DOCKER_VERSION=latest
+ARG DOCKER_VERSION=head
 
 FROM $BASE_IMAGE:$DOCKER_VERSION as base
 USER 0

diff --git a/docker/datahub-ingestion/Dockerfile-slim-only b/docker/datahub-ingestion/Dockerfile-slim-only
@@ -1,6 +1,6 @@
 # Defining environment
 ARG BASE_IMAGE=acryldata/datahub-ingestion-base
-ARG DOCKER_VERSION=latest
+ARG DOCKER_VERSION=head-slim
 
 FROM $BASE_IMAGE:$DOCKER_VERSION as base
 USER 0

diff --git a/docs-website/docusaurus.config.js b/docs-website/docusaurus.config.js
@@ -12,13 +12,11 @@ module.exports = {
   organizationName: "datahub-project", // Usually your GitHub org/user name.
   projectName: "datahub", // Usually your repo name.
   staticDirectories: ["static", "genStatic"],
-  stylesheets: [
-    "https://fonts.googleapis.com/css2?family=Manrope:wght@400;600&display=swap",
-  ],
+  stylesheets: ["https://fonts.googleapis.com/css2?family=Manrope:wght@400;500;700&display=swap"],
   noIndex: isSaas,
   customFields: {
     isSaas: isSaas,
-    markpromptProjectKey: process.env.DOCUSAURUS_MARKPROMPT_PROJECT_KEY || 'IeF3CUFCUQWuouZ8MP5Np9nES52QAtaA',
+    markpromptProjectKey: process.env.DOCUSAURUS_MARKPROMPT_PROJECT_KEY || "IeF3CUFCUQWuouZ8MP5Np9nES52QAtaA",
   },
   themeConfig: {
     ...(!isSaas && {
@@ -35,12 +33,8 @@ module.exports = {
       title: null,
       logo: {
         alt: "DataHub Logo",
-        src: `img/${
-          isSaas ? "acryl" : "datahub"
-        }-logo-color-light-horizontal.svg`,
-        srcDark: `img/${
-          isSaas ? "acryl" : "datahub"
-        }-logo-color-dark-horizontal.svg`,
+        src: `img/${isSaas ? "acryl" : "datahub"}-logo-color-light-horizontal.svg`,
+        srcDark: `img/${isSaas ? "acryl" : "datahub"}-logo-color-dark-horizontal.svg`,
       },
       items: [
         {
@@ -50,7 +44,8 @@ module.exports = {
           position: "right",
         },
         {
-          href: "/integrations",
+          to: "/integrations",
+          activeBasePath: "integrations",
           label: "Integrations",
           position: "right",
         },
@@ -70,8 +65,8 @@ module.exports = {
           position: "right",
         },
         {
-          type: 'docsVersionDropdown',
-          position: 'right',
+          type: "docsVersionDropdown",
+          position: "right",
           dropdownActiveClassDisabled: true,
         },
         {
@@ -201,9 +196,7 @@ module.exports = {
         blog: false,
         theme: {
           customCss: [
-            isSaas
-              ? require.resolve("./src/styles/acryl.scss")
-              : require.resolve("./src/styles/datahub.scss"),
+            isSaas ? require.resolve("./src/styles/acryl.scss") : require.resolve("./src/styles/datahub.scss"),
             require.resolve("./src/styles/global.scss"),
             require.resolve("./src/styles/sphinx.scss"),
             require.resolve("./src/styles/config-table.scss"),
@@ -217,10 +210,7 @@ module.exports = {
     ],
   ],
   plugins: [
-    [
-      "@docusaurus/plugin-ideal-image",
-      { quality: 100, sizes: [320, 640, 1280, 1440, 1600] },
-    ],
+    ["@docusaurus/plugin-ideal-image", { quality: 100, sizes: [320, 640, 1280, 1440, 1600] }],
     "docusaurus-plugin-sass",
     [
       "docusaurus-graphql-plugin",

diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js
@@ -81,6 +81,13 @@ module.exports = {
                 "docs/quick-ingestion-guides/powerbi/configuration",
               ],
             },
+            {
+              Looker: [
+                "docs/quick-ingestion-guides/looker/overview",
+                "docs/quick-ingestion-guides/looker/setup",
+                "docs/quick-ingestion-guides/looker/configuration",
+              ],
+            },
           ],
         },
         {
@@ -437,7 +444,7 @@ module.exports = {
           Observability: [
             "docs/managed-datahub/observe/freshness-assertions",
             "docs/managed-datahub/observe/volume-assertions",
-            "docs/managed-datahub/observe/custom-assertions",
+            "docs/managed-datahub/observe/custom-sql-assertions",
           ],
         },
       ],

diff --git a/docs-website/src/components/Feedback/styles.module.scss b/docs-website/src/components/Feedback/styles.module.scss
@@ -37,11 +37,11 @@
 }
 
 .feedbackText {
+  font-family: var(--ifm-font-family-base);
   width: 100%;
   border: var(--ifm-hr-border-color) 1px solid;
   border-radius: 0.4rem;
   padding: 0.4rem;
-  font-family: "Manrope", sans-serif;
 }
 
 .feedbackButton {

diff --git a/docs-website/src/components/MarkpromptHelp/markprompthelp.module.scss b/docs-website/src/components/MarkpromptHelp/markprompthelp.module.scss
@@ -325,7 +325,6 @@ button {
   padding-left: 1.5714286em;
 }
 .MarkpromptAnswer ol > li::marker {
-  font-weight: 400;
   color: var(--markprompt-foreground);
 }
 .MarkpromptAnswer ul > li::marker {
@@ -454,7 +453,6 @@ button {
   background-color: var(--markprompt-muted);
   border: 1px solid var(--markprompt-border);
   overflow-x: auto;
-  font-weight: 400;
   font-size: 0.8571429em;
   line-height: 1.6666667;
   margin-top: 1.6666667em;

diff --git a/docs-website/src/pages/_components/Hero/hero.module.scss b/docs-website/src/pages/_components/Hero/hero.module.scss
@@ -74,3 +74,26 @@
     margin-right: 0.5rem;
   }
 }
+
+.quickstartContent {
+  text-align: center;
+  padding: 2rem 0;
+  height: 100%;
+  margin: 2rem 0;
+  background: #34394d;
+  border-radius: var(--ifm-card-border-radius);
+}
+
+.quickstartTitle {
+  color: #fafafa;
+}
+
+.quickstartSubtitle {
+  font-size: 1.1rem;
+  color: gray;
+}
+
+.quickstartCodeblock {
+  text-align: left;
+  padding: 0 20vh;
+}
diff --git a/docs-website/src/pages/_components/Hero/index.js b/docs-website/src/pages/_components/Hero/index.js
@@ -34,12 +34,11 @@ const Hero = ({}) => {
               complexity of your data ecosystem.
             </p>
             <p className="hero__subtitle">
-              Built with ❤️ by{" "}
-              <img src="https://datahubproject.io/img/acryl-logo-light-mark.png" style={{ "vertical-align": "text-top;" }} width="25" />{" "}
+              Built with ❤️ by <img src={useBaseUrl("/img/acryl-logo-transparent-mark.svg")} width="25" />{" "}
               <a href="https://acryldata.io" target="blank" rel="noopener noreferrer">
                 Acryl Data
               </a>{" "}
-              and <img src="https://datahubproject.io/img/LI-In-Bug.png" width="25" /> LinkedIn.
+              and <img src={useBaseUrl("img/LI-In-Bug.png")} width="25" /> LinkedIn.
             </p>
             <Link className="button button--primary button--md" to={useBaseUrl("docs/")}>
               Get Started →
@@ -51,10 +50,10 @@ const Hero = ({}) => {
         </div>
         <CardCTAs />
         <Image className="hero__image" img={require(`/img/diagrams/datahub-flow-diagram-${colorMode}.png`)} alt="DataHub Flow Diagram" />
-        <div className="quickstart__content">
-          <h1 className="quickstart__title">Get Started Now</h1>
-          <p className="quickstart__subtitle">Run the following command to get started with DataHub.</p>
-          <div className="quickstart__codeblock">
+        <div className={styles.quickstartContent}>
+          <h1 className={styles.quickstartTitle}>Get Started Now</h1>
+          <p className={styles.quickstartSubtitle}>Run the following command to get started with DataHub.</p>
+          <div className={styles.quickstartCodeblock}>
             <CodeBlock className={"language-shell"}>
               python3 -m pip install --upgrade pip wheel setuptools <br />
               python3 -m pip install --upgrade acryl-datahub <br />

diff --git a/docs-website/src/pages/_components/Section/section.module.scss b/docs-website/src/pages/_components/Section/section.module.scss
@@ -9,7 +9,6 @@
 .sectionTitle {
   font-size: 2.5rem;
   margin-bottom: 3rem;
-  font-weight: normal;
   text-align: center;
 }
 

diff --git a/docs-website/src/pages/docs/_components/DropDownFilter/search.module.scss b/docs-website/src/pages/docs/_components/DropDownFilter/search.module.scss
@@ -74,7 +74,6 @@
 }
 
 .searchResultItemHeading {
-  font-weight: 400;
   margin-bottom: 0;
 }
 

diff --git a/docs-website/src/pages/docs/_components/GuideList/guidelist.module.scss b/docs-website/src/pages/docs/_components/GuideList/guidelist.module.scss
@@ -16,6 +16,9 @@
     display: block;
     margin-bottom: 0.25rem;
   }
+  strong {
+    font-weight: 600;
+  }
   span {
     font-size: 0.875rem;
     line-height: 1.25em;

diff --git a/docs-website/src/pages/docs/_components/GuideList/index.jsx b/docs-website/src/pages/docs/_components/GuideList/index.jsx
@@ -19,7 +19,7 @@ const GuideList = ({ title, content, seeMoreLink }) =>
   content?.length > 0 ? (
     <div style={{ padding: "2vh 0" }}>
       <div className="container">
-        <h2 style={{ fontWeight: "normal" }}>{title}</h2>
+        <h2>{title}</h2>
         <div className="row row--no-gutters">
           {content.map((props, idx) => (
             <ListItem key={idx} {...props} />

diff --git a/docs-website/src/pages/docs/_components/QuickLinkCard/quicklinkcard.module.scss b/docs-website/src/pages/docs/_components/QuickLinkCard/quicklinkcard.module.scss
@@ -20,6 +20,10 @@
     display: block;
     margin-bottom: 0.25rem;
   }
+  strong {
+    font-weight: 600;
+  }
+
   span {
     font-size: 0.875rem;
     line-height: 1.25em;

diff --git a/docs-website/src/pages/docs/_components/SearchBar/search.module.scss b/docs-website/src/pages/docs/_components/SearchBar/search.module.scss
@@ -91,7 +91,6 @@
 }
 
 .searchResultItemHeading {
-  font-weight: 400;
   margin-bottom: 0;
 }
 

diff --git a/docs-website/src/styles/acryl.scss b/docs-website/src/styles/acryl.scss
@@ -7,4 +7,7 @@
   --ifm-color-primary-light: #13beb0;
   --ifm-color-primary-lighter: #14c7b8;
   --ifm-color-primary-lightest: #16e1d0;
+
+  // Custom
+  --ifm-color-primary-opaque: rgba(17, 173, 160, 0.1);
 }
diff --git a/docs-website/src/styles/datahub.scss b/docs-website/src/styles/datahub.scss
@@ -7,4 +7,6 @@
   --ifm-color-primary-light: #349dff;
   --ifm-color-primary-lighter: #42a4ff;
   --ifm-color-primary-lightest: #6cb8ff;
+
+  --ifm-color-primary-opaque: rgba(24, 144, 255, 0.1);
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -74,7 +74,6 @@ @@
     }
     .searchResultItemHeading {
-      font-weight: 400;
       margin-bottom: 0;
     }
@@ Expand Down @@