From fd129c7d5d7efd205a0d1ddf3544a9367fb0fe80 Mon Sep 17 00:00:00 2001 From: Hyejin Yoon <0327jane@gmail.com> Date: Wed, 22 Nov 2023 19:00:34 +0900 Subject: [PATCH 1/4] fix: Revamp features page (#8839) --- .../FeatureCard/featurecard.module.scss | 38 ++++++ .../docs/_components/FeatureCard/index.jsx | 21 +++ .../featurecardsection.module.scss | 36 +++++ .../_components/FeatureCardSection/index.jsx | 65 +++++++++ .../docs/_components/QuickstartCard/index.jsx | 22 +++ .../QuickstartCard/quickstartcard.module.scss | 48 +++++++ .../_components/QuickstartCards/index.jsx | 35 +++++ .../quickstartcards.module.scss | 34 +++++ docs/features.md | 127 ++++-------------- 9 files changed, 324 insertions(+), 102 deletions(-) create mode 100644 docs-website/src/pages/docs/_components/FeatureCard/featurecard.module.scss create mode 100644 docs-website/src/pages/docs/_components/FeatureCard/index.jsx create mode 100644 docs-website/src/pages/docs/_components/FeatureCardSection/featurecardsection.module.scss create mode 100644 docs-website/src/pages/docs/_components/FeatureCardSection/index.jsx create mode 100644 docs-website/src/pages/docs/_components/QuickstartCard/index.jsx create mode 100644 docs-website/src/pages/docs/_components/QuickstartCard/quickstartcard.module.scss create mode 100644 docs-website/src/pages/docs/_components/QuickstartCards/index.jsx create mode 100644 docs-website/src/pages/docs/_components/QuickstartCards/quickstartcards.module.scss diff --git a/docs-website/src/pages/docs/_components/FeatureCard/featurecard.module.scss b/docs-website/src/pages/docs/_components/FeatureCard/featurecard.module.scss new file mode 100644 index 00000000000000..61739d5b6922cb --- /dev/null +++ b/docs-website/src/pages/docs/_components/FeatureCard/featurecard.module.scss @@ -0,0 +1,38 @@ +.feature { + flex-direction: row; + padding: 1.75rem; + color: var(--ifm-hero-text-color); + margin: 0rem 2rem 1rem 0rem; + min-height: 14rem; + max-height: 15rem; + overflow: hidden; + text-decoration: none !important; + + img { + width: 1.5rem; + height: 1.5rem; + margin-right: 0.75rem; + } + svg { + width: 1.5rem; + height: 1.5rem; + margin-right: 0.75rem; + } + strong, + span { + display: block; + margin-bottom: 0.75rem; + } + strong { + font-weight: 600; + font-size: 1.1rem; + } + + span { + font-size: 1rem; + line-height: 1.25em; + } + &:hover { + border-color: var(--ifm-color-primary); + } +} diff --git a/docs-website/src/pages/docs/_components/FeatureCard/index.jsx b/docs-website/src/pages/docs/_components/FeatureCard/index.jsx new file mode 100644 index 00000000000000..407e8eb4019878 --- /dev/null +++ b/docs-website/src/pages/docs/_components/FeatureCard/index.jsx @@ -0,0 +1,21 @@ +import React from "react"; +import clsx from "clsx"; +import styles from "./featurecard.module.scss"; +import useBaseUrl from "@docusaurus/useBaseUrl"; +import Link from "@docusaurus/Link"; + +const FeatureCard = ({icon, title, description, to}) => { +return ( +
+ +
+ {icon} + {title} → + {description} +
+ +
+ ); +}; + +export default FeatureCard; diff --git a/docs-website/src/pages/docs/_components/FeatureCardSection/featurecardsection.module.scss b/docs-website/src/pages/docs/_components/FeatureCardSection/featurecardsection.module.scss new file mode 100644 index 00000000000000..9e08c789c9068b --- /dev/null +++ b/docs-website/src/pages/docs/_components/FeatureCardSection/featurecardsection.module.scss @@ -0,0 +1,36 @@ + + +.feature { + flex-direction: row; + padding: 0.675rem; + color: var(--ifm-text-color); + margin: 0.5rem; + min-height: calc(100% - 1rem); + text-decoration: none !important; + img { + width: 1.5rem; + height: 1.5rem; + margin-right: 0.75rem; + } + svg { + width: 1.5rem; + height: 1.5rem; + margin-right: 0.75rem; + } + strong, + span { + display: block; + margin-bottom: 0.25rem; + } + strong { + font-weight: 600; + } + + span { + font-size: 0.875rem; + line-height: 1.25em; + } + &:hover { + border-color: var(--ifm-color-primary); + } +} diff --git a/docs-website/src/pages/docs/_components/FeatureCardSection/index.jsx b/docs-website/src/pages/docs/_components/FeatureCardSection/index.jsx new file mode 100644 index 00000000000000..bac97a805d2d7f --- /dev/null +++ b/docs-website/src/pages/docs/_components/FeatureCardSection/index.jsx @@ -0,0 +1,65 @@ +import React from "react"; +import FeatureCard from '../FeatureCard' +import { + EyeTwoTone, + HeartTwoTone, + ApiTwoTone, + AlertTwoTone, + CompassTwoTone, + ProfileTwoTone, +} from "@ant-design/icons"; + +const featureCardContent = [ +{ + title: "Data Discovery", + description: "Search your entire data ecosystem, including dashboards, datasets, ML models, and raw files.", + to: "docs/how/search", + icon: + }, +{ + title: "Data Governance", + description: "Define ownership and track PII.", + to: "https://www.acryldata.io/blog/the-3-must-haves-of-metadata-management-part-2?utm_source=datahub&utm_medium=referral&utm_content=blog", + icon: + }, +{ + title: "Data Quality Control", + description: "Improve data quality through metadata tests, assertions, data freshness checks, and data contracts.", + to: "https://www.acryldata.io/blog/data-contracts-in-datahub-combining-verifiability-with-holistic-data-management?utm_source=datahub&utm_medium=referral&utm_content=blog", + icon: + }, +{ + title: "UI-based Ingestion", + description: "Easily set up integrations in minutes using DataHub's intuitive UI-based ingestion feature.", + to: "docs/ui-ingestion", + icon: +}, +{ + title: "APIs and SDKs", + description: "For users who prefer programmatic control, DataHub offers a comprehensive set of APIs and SDKs.", + to: "docs/api/datahub-apis", + icon: +}, +{ + title: "Vibrant Community", + description: "Our community provides support through office hours, workshops, and a Slack channel.", + to: "docs/slack", + icon: +} +] + +const FeatureCards = () => { +return ( +
+
+
+ {featureCardContent.map((props, idx) => ( + + ))} +
+
+
+ ); +}; + +export default FeatureCards; diff --git a/docs-website/src/pages/docs/_components/QuickstartCard/index.jsx b/docs-website/src/pages/docs/_components/QuickstartCard/index.jsx new file mode 100644 index 00000000000000..b4e3895fa40e77 --- /dev/null +++ b/docs-website/src/pages/docs/_components/QuickstartCard/index.jsx @@ -0,0 +1,22 @@ +import React from "react"; +import clsx from "clsx"; +import styles from "./quickstartcard.module.scss"; +import Link from "@docusaurus/Link"; +import useBaseUrl from "@docusaurus/useBaseUrl"; + + +const QuickstartCard = ({ icon, title, to, color, fontColor }) => { + return ( +
+ + +
+ {title} → +
+ +
+ ); +}; + + +export default QuickstartCard; diff --git a/docs-website/src/pages/docs/_components/QuickstartCard/quickstartcard.module.scss b/docs-website/src/pages/docs/_components/QuickstartCard/quickstartcard.module.scss new file mode 100644 index 00000000000000..fd35a4b777c99c --- /dev/null +++ b/docs-website/src/pages/docs/_components/QuickstartCard/quickstartcard.module.scss @@ -0,0 +1,48 @@ +.feature { + flex-direction: row; + height: 10rem; + flex-shrink: 0; + padding: 3rem; + color: var(--ifm-text-color); + margin: 0rem 2rem 1rem 0rem; + min-height: calc(100% - 1rem); + text-decoration: none !important; + + + img { + width: 3rem; + height: 3rem; + margin: auto 1rem; + } + svg { + width: 1.5rem; + height: 1.5rem; + margin-right: 0.75rem; + } + strong, + span { + display: block; + margin-bottom: 0.25rem; + } + strong { + font-weight: 600; + padding: auto 0; + } + + span { + font-size: 0.875rem; + line-height: 1.25em; + } + &:hover { + border-color: var(--ifm-color-primary); + } + + .quickstart-text { + margin: auto 0; + } + +} + +.quickstart-text { + margin: auto 0; +} \ No newline at end of file diff --git a/docs-website/src/pages/docs/_components/QuickstartCards/index.jsx b/docs-website/src/pages/docs/_components/QuickstartCards/index.jsx new file mode 100644 index 00000000000000..bcb77c043f1d0b --- /dev/null +++ b/docs-website/src/pages/docs/_components/QuickstartCards/index.jsx @@ -0,0 +1,35 @@ +import React from "react"; +import QuickstartCard from '../QuickstartCard' + +const quickstartContent = [ +{ + title: "Quickstart with DataHub", + icon: "datahub-logo-color-mark", + to: "quickstart", + color: '#FFF', + fontColor: '#091013', + }, +{ + title: "Learn about Managed DataHub", + icon: "acryl-logo-transparent-mark", + to: "managed-datahub/managed-datahub-overview", + color: '#091013', + fontColor: '#FFF', +} +] + +const QuickstartCards = () => { +return ( +
+
+
+ {quickstartContent.map((props, idx) => ( + + ))} +
+
+
+ ); +}; + +export default QuickstartCards; diff --git a/docs-website/src/pages/docs/_components/QuickstartCards/quickstartcards.module.scss b/docs-website/src/pages/docs/_components/QuickstartCards/quickstartcards.module.scss new file mode 100644 index 00000000000000..4fbbc4583d6629 --- /dev/null +++ b/docs-website/src/pages/docs/_components/QuickstartCards/quickstartcards.module.scss @@ -0,0 +1,34 @@ +.feature { + flex-direction: row; + padding: 0.675rem; + color: var(--ifm-text-color); + margin: 0.5rem; + min-height: calc(100% - 1rem); + text-decoration: none !important; + img { + width: 1.5rem; + height: 1.5rem; + margin-right: 0.75rem; + } + svg { + width: 1.5rem; + height: 1.5rem; + margin-right: 0.75rem; + } + strong, + span { + display: block; + margin-bottom: 0.25rem; + } + strong { + font-weight: 600; + } + + span { + font-size: 0.875rem; + line-height: 1.25em; + } + &:hover { + border-color: var(--ifm-color-primary); + } +} diff --git a/docs/features.md b/docs/features.md index a06789f28f82bd..9ce85d83ee54a2 100644 --- a/docs/features.md +++ b/docs/features.md @@ -1,118 +1,41 @@ ---- -title: "Features" ---- +import QuickstartCards from '@site/src/pages/docs/_components/QuickstartCards'; +import FeatureCardSection from '@site/src/pages/docs/_components/FeatureCardSection'; -# DataHub Features Overview +# What is DataHub? -DataHub is a modern data catalog built to enable end-to-end data discovery, data observability, and data governance. This extensible metadata platform is built for developers to tame the complexity of their rapidly evolving data ecosystems and for data practitioners to leverage the total value of data within their organization. +DataHub is a modern data catalog built to enable end-to-end data discovery, data observability, and data governance. +This extensible metadata platform is built for developers to tame the complexity of their rapidly evolving data ecosystems and for data practitioners to leverage the total value of data within their organization. -Here’s an overview of DataHub’s current functionality. Check out our [roadmap](https://feature-requests.datahubproject.io/roadmap) to see what's to come. +## Quickstart ---- + -## Search and Discovery +## Key Features -### **Search All Corners of Your Data Stack** + -DataHub's unified search experience surfaces results across databases, data lakes, BI platforms, ML feature stores, orchestration tools, and more. +## Get Started -

- -

+### Deployment -### **Trace End-to-End Lineage** +To get started with DataHub, you can use a simple CLI command. Alternatively, you can deploy the instance to production using Docker or Helm charts. -Quickly understand the end-to-end journey of data by tracing lineage across platforms, datasets, ETL/ELT pipelines, charts, dashboards, and beyond. +- [Quickstart](quickstart.md) +- [Self-hosted DataHub](deploy/kubernetes.md) +- [Managed DataHub](managed-datahub/managed-datahub-overview.md) -

- -

+### Ingestion -### **Understand the Impact of Breaking Changes on Downstream Dependencies** +DataHub supports ingestion by UI and CLI. -Proactively identify which entities may be impacted by a breaking change using Impact Analysis. +- [UI-based Ingestion](ui-ingestion.md) +- [CLI-based Ingestion](../metadata-ingestion/cli-ingestion.md) -

- -

+## Join the Community -### **View Metadata 360 at a Glance** +For additional information and assistance, feel free to visit one of these channels! -Combine *technical* and *logical* metadata to provide a 360º view of your data entities. - -Generate **Dataset Stats** to understand the shape & distribution of the data - -

- -

- -Capture historical **Data Validation Outcomes** from tools like Great Expectations - -

- -

- -Leverage DataHub's **Schema Version History** to track changes to the physical structure of data over time - -

- -

- ---- - -## Modern Data Governance - -### **Govern in Real Time** - -[The Actions Framework](./actions/README.md) powers the following real-time use cases: - -* **Notifications:** Generate organization-specific notifications when a change is made on DataHub. For example, send an email to the governance team when a "PII" tag is added to any data asset. -* **Workflow Integration:** Integrate DataHub into your organization's internal workflows. For example, create a Jira ticket when specific Tags or Terms are proposed on a Dataset. -* **Synchronization:** Sync changes made in DataHub into a 3rd party system. For example, reflect Tag additions in DataHub into Snowflake. -* **Auditing:** Audit who is making what changes on DataHub through time. - -

- -

- -### **Manage Entity Ownership** -Quickly and easily assign entity ownership to users and user groups. - -

- -

- -### **Govern with Tags, Glossary Terms, and Domains** -Empower data owners to govern their data entities with: - -1. **Tags:** Informal, loosely controlled labels that serve as a tool for search & discovery. No formal, central management. -2. **Glossary Terms:** A controlled vocabulary with optional hierarchy, commonly used to describe core business concepts and measurements. -3. **Domains:** Curated, top-level folders or categories, widely used in Data Mesh to organize entities by department (i.e., Finance, Marketing) or Data Products. - -

- -

- ---- -## DataHub Administration - -### **Create Users, Groups, & Access Policies** - -DataHub admins can create Policies to define who can perform what action against which resource(s). When you create a new Policy, you will be able to define the following: - -* **Policy Type** - Platform (top-level DataHub Platform privileges, i.e., managing users, groups, and policies) or Metadata (ability to manipulate ownership, tags, documentation, and more) -* **Resource Type** - Specify the type of resources, such as Datasets, Dashboards, Pipelines, and beyond -* **Privileges** - Choose the set of permissions, such as Edit Owners, Edit Documentation, Edit Links -* **Users and/or Groups** - Assign relevant Users and Groups; you can also assign the Policy to Resource Owners, regardless of which Group they belong - -

- -

- -### **Ingest Metadata from the UI** - -Create, configure, schedule, & execute batch metadata ingestion using the DataHub user interface. This makes getting metadata into DataHub easier by minimizing the overhead required to operate custom integration pipelines. - -

- -

\ No newline at end of file +- [Slack](https://datahubspace.slack.com) +- [Blog](https://blog.datahubproject.io/) +- [LinkedIn](https://www.linkedin.com/company/acryl-data/) +- Our champions - [Data Practitioners Guild](https://datahubproject.io/guild/?_gl=1*11cd6n0*_gcl_au*ODQyMTk0NTI5LjE2OTQ2NjI1MjM.) From cda980bfb279a70ba85786e4dae8eadffbb2cde9 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Wed, 22 Nov 2023 21:58:44 +0530 Subject: [PATCH 2/4] docs(acryl cloud): release notes 0.2.13 (#9291) --- docs-website/sidebars.js | 1 + .../managed-datahub/release-notes/v_0_2_13.md | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 docs/managed-datahub/release-notes/v_0_2_13.md diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index be12aa3a827f2f..c70a609a4cc4b9 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -630,6 +630,7 @@ module.exports = { }, { "Managed DataHub Release History": [ + "docs/managed-datahub/release-notes/v_0_2_13", "docs/managed-datahub/release-notes/v_0_2_12", "docs/managed-datahub/release-notes/v_0_2_11", "docs/managed-datahub/release-notes/v_0_2_10", diff --git a/docs/managed-datahub/release-notes/v_0_2_13.md b/docs/managed-datahub/release-notes/v_0_2_13.md new file mode 100644 index 00000000000000..65cea863d9714d --- /dev/null +++ b/docs/managed-datahub/release-notes/v_0_2_13.md @@ -0,0 +1,19 @@ +# v0.2.13 +--- + +Release Availability Date +--- +22-Nov-2023 + +Recommended CLI/SDK +--- +- `v0.12.0.2` with release notes at https://github.com/acryldata/datahub/releases/tag/v0.12.0.2= + +If you are using an older CLI/SDK version then please upgrade it. This applies for all CLI/SDK usages, if you are using it through your terminal, github actions, airflow, in python SDK somewhere, Java SKD etc. This is a strong recommendation to upgrade as we keep on pushing fixes in the CLI and it helps us support you better. + +## Release Changelog +--- +- Since `v0.2.12` these changes from OSS DataHub https://github.com/datahub-project/datahub/compare/75252a3d9f6a576904be5a0790d644b9ae2df6ac...d9de854d276c118afc55264ecc9e2712b91b4ab2 have been pulled in. + +## Some notable features in this SaaS release +- Data Contract support added. This is currently disabled by default. If you wish to use this feature please reach out to your rep. From 308de491200b01caa7a3bd314bf5b2086af72138 Mon Sep 17 00:00:00 2001 From: gaurav2733 <77378510+gaurav2733@users.noreply.github.com> Date: Thu, 23 Nov 2023 00:55:32 +0530 Subject: [PATCH 3/4] fix(): stats are spaced out too far (#9292) --- .../src/app/entity/dashboard/shared/DashboardStatsSummary.tsx | 1 - .../src/app/entity/dataset/shared/DatasetStatsSummary.tsx | 1 - 2 files changed, 2 deletions(-) diff --git a/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx b/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx index fb6364cffac8b6..a359d658d27f78 100644 --- a/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx +++ b/datahub-web-react/src/app/entity/dashboard/shared/DashboardStatsSummary.tsx @@ -12,7 +12,6 @@ import ExpandingStat from '../../dataset/shared/ExpandingStat'; const StatText = styled.span` color: ${ANTD_GRAY[8]}; @media (min-width: 1024px) { - width: 100%; white-space: nowrap; `; diff --git a/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx b/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx index 3dcd41a3f8a412..1a5c01df5bde21 100644 --- a/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx +++ b/datahub-web-react/src/app/entity/dataset/shared/DatasetStatsSummary.tsx @@ -13,7 +13,6 @@ import ExpandingStat from './ExpandingStat'; const StatText = styled.span<{ color: string }>` color: ${(props) => props.color}; @media (min-width: 1160px) { - width: 100%; white-space: nowrap; `; From a62d52e6b19599e4a4e0a0a9c4fbaf578933ff35 Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Wed, 22 Nov 2023 13:54:12 -0600 Subject: [PATCH 4/4] feat(mysql): upgrade to version 8.2 for quickstart (#9241) Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com> --- .../docker-compose-without-neo4j.override.yml | 2 +- docker/docker-compose.override.yml | 2 +- docker/mysql/docker-compose.mysql.yml | 2 +- ...ocker-compose-without-neo4j.quickstart.yml | 2 +- .../quickstart/docker-compose.quickstart.yml | 2 +- .../quickstart_version_mapping.yaml | 9 ++++++ docs/how/extract-container-logs.md | 2 +- docs/how/updating-datahub.md | 1 + docs/troubleshooting/quickstart.md | 2 +- .../src/datahub/cli/docker_cli.py | 4 +++ .../src/datahub/cli/quickstart_versioning.py | 14 +++++--- .../cli/test_quickstart_version_mapping.py | 32 +++++++++++++++---- 12 files changed, 57 insertions(+), 17 deletions(-) diff --git a/docker/docker-compose-without-neo4j.override.yml b/docker/docker-compose-without-neo4j.override.yml index 24f83301351ba5..36f3c974b93afc 100644 --- a/docker/docker-compose-without-neo4j.override.yml +++ b/docker/docker-compose-without-neo4j.override.yml @@ -48,7 +48,7 @@ services: mysql: container_name: mysql hostname: mysql - image: mysql:5.7 + image: mysql:${DATAHUB_MYSQL_VERSION:-5.7} command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password ports: - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml index 0907f47d70c3ce..ef13b86a3d1514 100644 --- a/docker/docker-compose.override.yml +++ b/docker/docker-compose.override.yml @@ -32,7 +32,7 @@ services: mysql: container_name: mysql hostname: mysql - image: mysql:5.7 + image: mysql:${DATAHUB_MYSQL_VERSION:-5.7} command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password ports: - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 diff --git a/docker/mysql/docker-compose.mysql.yml b/docker/mysql/docker-compose.mysql.yml index 70287433bb5413..853d0c425ea61a 100644 --- a/docker/mysql/docker-compose.mysql.yml +++ b/docker/mysql/docker-compose.mysql.yml @@ -5,7 +5,7 @@ services: mysql: container_name: mysql hostname: mysql - image: mysql:5.7 + image: mysql:${DATAHUB_MYSQL_VERSION:-5.7} env_file: env/docker.env command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin ports: diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index ab5182bf98ae50..6eac53229e82ab 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -218,7 +218,7 @@ services: test: mysqladmin ping -h mysql -u $$MYSQL_USER --password=$$MYSQL_PASSWORD timeout: 5s hostname: mysql - image: mysql:5.7 + image: mysql:${DATAHUB_MYSQL_VERSION:-5.7} ports: - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 restart: on-failure diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 29c980532d46f6..86d70abd2b8157 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -225,7 +225,7 @@ services: test: mysqladmin ping -h mysql -u $$MYSQL_USER --password=$$MYSQL_PASSWORD timeout: 5s hostname: mysql - image: mysql:5.7 + image: mysql:${DATAHUB_MYSQL_VERSION:-5.7} ports: - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 restart: on-failure diff --git a/docker/quickstart/quickstart_version_mapping.yaml b/docker/quickstart/quickstart_version_mapping.yaml index 824e1f8237453f..9948bd55fdc0b6 100644 --- a/docker/quickstart/quickstart_version_mapping.yaml +++ b/docker/quickstart/quickstart_version_mapping.yaml @@ -23,6 +23,7 @@ quickstart_version_map: default: composefile_git_ref: master docker_tag: head + mysql_tag: 5.7 # default: # Use this to pin default to a specific version. # composefile_git_ref: fd1bd51541a132017a648f4a2f037eec8f70ba26 # v0.10.0 + quickstart compose file fixes # docker_tag: v0.10.0 @@ -30,11 +31,19 @@ quickstart_version_map: head: composefile_git_ref: master docker_tag: head + mysql_tag: 5.7 + + # v0.13.0 we upgraded MySQL image for EOL + v0.13.0: + composefile_git_ref: master + docker_tag: head + mysql_tag: 8.2 # v0.9.6 images contain security vulnerabilities v0.9.6: composefile_git_ref: v0.9.6.1 docker_tag: v0.9.6.1 + mysql_tag: 5.7 # If stable is not defined the latest released version will be used. # stable: diff --git a/docs/how/extract-container-logs.md b/docs/how/extract-container-logs.md index b93c077eb21f5c..9251d0665c02cf 100644 --- a/docs/how/extract-container-logs.md +++ b/docs/how/extract-container-logs.md @@ -21,7 +21,7 @@ CONTAINER ID IMAGE COMMAND 3680fcaef3ed confluentinc/cp-kafka:5.4.0 "/etc/confluent/dock…" 5 days ago Up 5 days 0.0.0.0:9092->9092/tcp, 0.0.0.0:29092->29092/tcp broker 9d6730ddd4c4 neo4j:4.0.6 "/sbin/tini -g -- /d…" 5 days ago Up 5 days 0.0.0.0:7474->7474/tcp, 7473/tcp, 0.0.0.0:7687->7687/tcp neo4j c97edec663af confluentinc/cp-zookeeper:5.4.0 "/etc/confluent/dock…" 5 days ago Up 5 days 2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp zookeeper -150ba161cf26 mysql:5.7 "docker-entrypoint.s…" 5 days ago Up 5 days 0.0.0.0:3306->3306/tcp, 33060/tcp mysql +150ba161cf26 mysql:8.2 "docker-entrypoint.s…" 5 days ago Up 5 days 0.0.0.0:3306->3306/tcp, 33060/tcp mysql 4b72a3eab73f elasticsearch:7.9.3 "/tini -- /usr/local…" 5 days ago Up 5 days (healthy) 0.0.0.0:9200->9200/tcp, 9300/tcp elasticsearch ``` diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 54f5775d8331f8..21c4cef2e848b7 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -6,6 +6,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ### Breaking Changes +- Updating MySQL version for quickstarts to 8.2, may cause quickstart issues for existing instances. - #9244: The `redshift-legacy` and `redshift-legacy-usage` sources, which have been deprecated for >6 months, have been removed. The new `redshift` source is a superset of the functionality provided by those legacy sources. ### Potential Downtime diff --git a/docs/troubleshooting/quickstart.md b/docs/troubleshooting/quickstart.md index 64684c5ffa36c7..0392ffc426a6cb 100644 --- a/docs/troubleshooting/quickstart.md +++ b/docs/troubleshooting/quickstart.md @@ -99,7 +99,7 @@ c267c287a235 landoop/schema-registry-ui:latest "/run. 943e60f9b4d0 neo4j:4.0.6 "/sbin/tini -g -- /d…" 10 hours ago Up 10 hours 0.0.0.0:7474->7474/tcp, 7473/tcp, 0.0.0.0:7687->7687/tcp neo4j 6d79b6f02735 confluentinc/cp-zookeeper:5.2.1 "/etc/confluent/dock…" 10 hours ago Up 10 hours 2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp zookeeper 491d9f2b2e9e docker.elastic.co/elasticsearch/elasticsearch:5.6.8 "/bin/bash bin/es-do…" 10 hours ago Up 10 hours 0.0.0.0:9200->9200/tcp, 9300/tcp elasticsearch -ce14b9758eb3 mysql:5.7 +ce14b9758eb3 mysql:8.2 ``` Also you can check individual Docker container logs by running `docker logs <>`. For `datahub-gms`, you should see a log similar to this at the end of the initialization: diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py index 08f3faae8abb2a..0e0bc37c61573d 100644 --- a/metadata-ingestion/src/datahub/cli/docker_cli.py +++ b/metadata-ingestion/src/datahub/cli/docker_cli.py @@ -158,6 +158,7 @@ def should_use_neo4j_for_graph_service(graph_service_override: Optional[str]) -> def _set_environment_variables( version: Optional[str], + mysql_version: Optional[str], mysql_port: Optional[pydantic.PositiveInt], zk_port: Optional[pydantic.PositiveInt], kafka_broker_port: Optional[pydantic.PositiveInt], @@ -172,6 +173,8 @@ def _set_environment_variables( ) version = f"v{version}" os.environ["DATAHUB_VERSION"] = version + if mysql_version is not None: + os.environ["DATAHUB_MYSQL_VERSION"] = mysql_version if mysql_port is not None: os.environ["DATAHUB_MAPPED_MYSQL_PORT"] = str(mysql_port) @@ -675,6 +678,7 @@ def quickstart( # noqa: C901 # set version _set_environment_variables( version=quickstart_execution_plan.docker_tag, + mysql_version=quickstart_execution_plan.mysql_tag, mysql_port=mysql_port, zk_port=zk_port, kafka_broker_port=kafka_broker_port, diff --git a/metadata-ingestion/src/datahub/cli/quickstart_versioning.py b/metadata-ingestion/src/datahub/cli/quickstart_versioning.py index dc7bd1ecb4a89d..be7439f330dfb6 100644 --- a/metadata-ingestion/src/datahub/cli/quickstart_versioning.py +++ b/metadata-ingestion/src/datahub/cli/quickstart_versioning.py @@ -21,6 +21,7 @@ class QuickstartExecutionPlan(BaseModel): composefile_git_ref: str docker_tag: str + mysql_tag: Optional[str] def _is_it_a_version(version: str) -> bool: @@ -81,7 +82,7 @@ def fetch_quickstart_config(cls) -> "QuickstartVersionMappingConfig": return QuickstartVersionMappingConfig( quickstart_version_map={ "default": QuickstartExecutionPlan( - composefile_git_ref="master", docker_tag="head" + composefile_git_ref="master", docker_tag="head", mysql_tag="5.7" ), } ) @@ -93,7 +94,7 @@ def fetch_quickstart_config(cls) -> "QuickstartVersionMappingConfig": try: release = cls._fetch_latest_version() config.quickstart_version_map["stable"] = QuickstartExecutionPlan( - composefile_git_ref=release, docker_tag=release + composefile_git_ref=release, docker_tag=release, mysql_tag=release ) except Exception: click.echo( @@ -103,7 +104,8 @@ def fetch_quickstart_config(cls) -> "QuickstartVersionMappingConfig": return config def get_quickstart_execution_plan( - self, requested_version: Optional[str] + self, + requested_version: Optional[str], ) -> QuickstartExecutionPlan: """ From the requested version and stable flag, returns the execution plan for the quickstart. @@ -114,10 +116,14 @@ def get_quickstart_execution_plan( requested_version = "default" composefile_git_ref = requested_version docker_tag = requested_version + # Default to 5.7 if not specified in version map + mysql_tag = "5.7" result = self.quickstart_version_map.get( requested_version, QuickstartExecutionPlan( - composefile_git_ref=composefile_git_ref, docker_tag=docker_tag + composefile_git_ref=composefile_git_ref, + docker_tag=docker_tag, + mysql_tag=mysql_tag, ), ) # new CLI version is downloading the composefile corresponding to the requested version diff --git a/metadata-ingestion/tests/unit/cli/test_quickstart_version_mapping.py b/metadata-ingestion/tests/unit/cli/test_quickstart_version_mapping.py index 2fd56560c933da..3b06e48522955d 100644 --- a/metadata-ingestion/tests/unit/cli/test_quickstart_version_mapping.py +++ b/metadata-ingestion/tests/unit/cli/test_quickstart_version_mapping.py @@ -6,16 +6,30 @@ example_version_mapper = QuickstartVersionMappingConfig.parse_obj( { "quickstart_version_map": { - "default": {"composefile_git_ref": "master", "docker_tag": "latest"}, + "default": { + "composefile_git_ref": "master", + "docker_tag": "latest", + "mysql_tag": "5.7", + }, "v0.9.6": { "composefile_git_ref": "v0.9.6.1", "docker_tag": "v0.9.6.1", + "mysql_tag": "5.7", + }, + "v2.0.0": { + "composefile_git_ref": "v2.0.1", + "docker_tag": "v2.0.0", + "mysql_tag": "5.7", + }, + "v1.0.0": { + "composefile_git_ref": "v1.0.0", + "docker_tag": "v1.0.0", + "mysql_tag": "5.7", }, - "v2.0.0": {"composefile_git_ref": "v2.0.1", "docker_tag": "v2.0.0"}, - "v1.0.0": {"composefile_git_ref": "v1.0.0", "docker_tag": "v1.0.0"}, "stable": { "composefile_git_ref": "v1.0.1", "docker_tag": "latest", + "mysql_tag": "5.7", }, }, } @@ -27,6 +41,7 @@ def test_quickstart_version_config(): expected = QuickstartExecutionPlan( docker_tag="v1.0.0", composefile_git_ref="v1.0.0", + mysql_tag="5.7", ) assert execution_plan == expected @@ -36,6 +51,7 @@ def test_quickstart_version_config_default(): expected = QuickstartExecutionPlan( docker_tag="v2.0.0", composefile_git_ref="v2.0.1", + mysql_tag="5.7", ) assert execution_plan == expected @@ -43,20 +59,22 @@ def test_quickstart_version_config_default(): def test_quickstart_version_config_stable(): execution_plan = example_version_mapper.get_quickstart_execution_plan("stable") expected = QuickstartExecutionPlan( - docker_tag="latest", - composefile_git_ref="v1.0.1", + docker_tag="latest", composefile_git_ref="v1.0.1", mysql_tag="5.7" ) assert execution_plan == expected def test_quickstart_forced_stable(): example_version_mapper.quickstart_version_map["default"] = QuickstartExecutionPlan( - composefile_git_ref="v1.0.1", docker_tag="latest" + composefile_git_ref="v1.0.1", + docker_tag="latest", + mysql_tag="5.7", ) execution_plan = example_version_mapper.get_quickstart_execution_plan(None) expected = QuickstartExecutionPlan( docker_tag="latest", composefile_git_ref="v1.0.1", + mysql_tag="5.7", ) assert execution_plan == expected @@ -74,6 +92,7 @@ def test_quickstart_forced_not_a_version_tag(): expected = QuickstartExecutionPlan( docker_tag="NOT A VERSION", composefile_git_ref="NOT A VERSION", + mysql_tag="5.7", ) assert execution_plan == expected @@ -83,5 +102,6 @@ def test_quickstart_get_older_version(): expected = QuickstartExecutionPlan( docker_tag="v0.9.6.1", composefile_git_ref="v0.9.6.1", + mysql_tag="5.7", ) assert execution_plan == expected