From a62d52e6b19599e4a4e0a0a9c4fbaf578933ff35 Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Wed, 22 Nov 2023 13:54:12 -0600 Subject: [PATCH] feat(mysql): upgrade to version 8.2 for quickstart (#9241) Co-authored-by: david-leifker <114954101+david-leifker@users.noreply.github.com> --- .../docker-compose-without-neo4j.override.yml | 2 +- docker/docker-compose.override.yml | 2 +- docker/mysql/docker-compose.mysql.yml | 2 +- ...ocker-compose-without-neo4j.quickstart.yml | 2 +- .../quickstart/docker-compose.quickstart.yml | 2 +- .../quickstart_version_mapping.yaml | 9 ++++++ docs/how/extract-container-logs.md | 2 +- docs/how/updating-datahub.md | 1 + docs/troubleshooting/quickstart.md | 2 +- .../src/datahub/cli/docker_cli.py | 4 +++ .../src/datahub/cli/quickstart_versioning.py | 14 +++++--- .../cli/test_quickstart_version_mapping.py | 32 +++++++++++++++---- 12 files changed, 57 insertions(+), 17 deletions(-) diff --git a/docker/docker-compose-without-neo4j.override.yml b/docker/docker-compose-without-neo4j.override.yml index 24f83301351ba5..36f3c974b93afc 100644 --- a/docker/docker-compose-without-neo4j.override.yml +++ b/docker/docker-compose-without-neo4j.override.yml @@ -48,7 +48,7 @@ services: mysql: container_name: mysql hostname: mysql - image: mysql:5.7 + image: mysql:${DATAHUB_MYSQL_VERSION:-5.7} command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password ports: - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 diff --git a/docker/docker-compose.override.yml b/docker/docker-compose.override.yml index 0907f47d70c3ce..ef13b86a3d1514 100644 --- a/docker/docker-compose.override.yml +++ b/docker/docker-compose.override.yml @@ -32,7 +32,7 @@ services: mysql: container_name: mysql hostname: mysql - image: mysql:5.7 + image: mysql:${DATAHUB_MYSQL_VERSION:-5.7} command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password ports: - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 diff --git a/docker/mysql/docker-compose.mysql.yml b/docker/mysql/docker-compose.mysql.yml index 70287433bb5413..853d0c425ea61a 100644 --- a/docker/mysql/docker-compose.mysql.yml +++ b/docker/mysql/docker-compose.mysql.yml @@ -5,7 +5,7 @@ services: mysql: container_name: mysql hostname: mysql - image: mysql:5.7 + image: mysql:${DATAHUB_MYSQL_VERSION:-5.7} env_file: env/docker.env command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin ports: diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index ab5182bf98ae50..6eac53229e82ab 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -218,7 +218,7 @@ services: test: mysqladmin ping -h mysql -u $$MYSQL_USER --password=$$MYSQL_PASSWORD timeout: 5s hostname: mysql - image: mysql:5.7 + image: mysql:${DATAHUB_MYSQL_VERSION:-5.7} ports: - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 restart: on-failure diff --git a/docker/quickstart/docker-compose.quickstart.yml b/docker/quickstart/docker-compose.quickstart.yml index 29c980532d46f6..86d70abd2b8157 100644 --- a/docker/quickstart/docker-compose.quickstart.yml +++ b/docker/quickstart/docker-compose.quickstart.yml @@ -225,7 +225,7 @@ services: test: mysqladmin ping -h mysql -u $$MYSQL_USER --password=$$MYSQL_PASSWORD timeout: 5s hostname: mysql - image: mysql:5.7 + image: mysql:${DATAHUB_MYSQL_VERSION:-5.7} ports: - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 restart: on-failure diff --git a/docker/quickstart/quickstart_version_mapping.yaml b/docker/quickstart/quickstart_version_mapping.yaml index 824e1f8237453f..9948bd55fdc0b6 100644 --- a/docker/quickstart/quickstart_version_mapping.yaml +++ b/docker/quickstart/quickstart_version_mapping.yaml @@ -23,6 +23,7 @@ quickstart_version_map: default: composefile_git_ref: master docker_tag: head + mysql_tag: 5.7 # default: # Use this to pin default to a specific version. # composefile_git_ref: fd1bd51541a132017a648f4a2f037eec8f70ba26 # v0.10.0 + quickstart compose file fixes # docker_tag: v0.10.0 @@ -30,11 +31,19 @@ quickstart_version_map: head: composefile_git_ref: master docker_tag: head + mysql_tag: 5.7 + + # v0.13.0 we upgraded MySQL image for EOL + v0.13.0: + composefile_git_ref: master + docker_tag: head + mysql_tag: 8.2 # v0.9.6 images contain security vulnerabilities v0.9.6: composefile_git_ref: v0.9.6.1 docker_tag: v0.9.6.1 + mysql_tag: 5.7 # If stable is not defined the latest released version will be used. # stable: diff --git a/docs/how/extract-container-logs.md b/docs/how/extract-container-logs.md index b93c077eb21f5c..9251d0665c02cf 100644 --- a/docs/how/extract-container-logs.md +++ b/docs/how/extract-container-logs.md @@ -21,7 +21,7 @@ CONTAINER ID IMAGE COMMAND 3680fcaef3ed confluentinc/cp-kafka:5.4.0 "/etc/confluent/dock…" 5 days ago Up 5 days 0.0.0.0:9092->9092/tcp, 0.0.0.0:29092->29092/tcp broker 9d6730ddd4c4 neo4j:4.0.6 "/sbin/tini -g -- /d…" 5 days ago Up 5 days 0.0.0.0:7474->7474/tcp, 7473/tcp, 0.0.0.0:7687->7687/tcp neo4j c97edec663af confluentinc/cp-zookeeper:5.4.0 "/etc/confluent/dock…" 5 days ago Up 5 days 2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp zookeeper -150ba161cf26 mysql:5.7 "docker-entrypoint.s…" 5 days ago Up 5 days 0.0.0.0:3306->3306/tcp, 33060/tcp mysql +150ba161cf26 mysql:8.2 "docker-entrypoint.s…" 5 days ago Up 5 days 0.0.0.0:3306->3306/tcp, 33060/tcp mysql 4b72a3eab73f elasticsearch:7.9.3 "/tini -- /usr/local…" 5 days ago Up 5 days (healthy) 0.0.0.0:9200->9200/tcp, 9300/tcp elasticsearch ``` diff --git a/docs/how/updating-datahub.md b/docs/how/updating-datahub.md index 54f5775d8331f8..21c4cef2e848b7 100644 --- a/docs/how/updating-datahub.md +++ b/docs/how/updating-datahub.md @@ -6,6 +6,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe ### Breaking Changes +- Updating MySQL version for quickstarts to 8.2, may cause quickstart issues for existing instances. - #9244: The `redshift-legacy` and `redshift-legacy-usage` sources, which have been deprecated for >6 months, have been removed. The new `redshift` source is a superset of the functionality provided by those legacy sources. ### Potential Downtime diff --git a/docs/troubleshooting/quickstart.md b/docs/troubleshooting/quickstart.md index 64684c5ffa36c7..0392ffc426a6cb 100644 --- a/docs/troubleshooting/quickstart.md +++ b/docs/troubleshooting/quickstart.md @@ -99,7 +99,7 @@ c267c287a235 landoop/schema-registry-ui:latest "/run. 943e60f9b4d0 neo4j:4.0.6 "/sbin/tini -g -- /d…" 10 hours ago Up 10 hours 0.0.0.0:7474->7474/tcp, 7473/tcp, 0.0.0.0:7687->7687/tcp neo4j 6d79b6f02735 confluentinc/cp-zookeeper:5.2.1 "/etc/confluent/dock…" 10 hours ago Up 10 hours 2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp zookeeper 491d9f2b2e9e docker.elastic.co/elasticsearch/elasticsearch:5.6.8 "/bin/bash bin/es-do…" 10 hours ago Up 10 hours 0.0.0.0:9200->9200/tcp, 9300/tcp elasticsearch -ce14b9758eb3 mysql:5.7 +ce14b9758eb3 mysql:8.2 ``` Also you can check individual Docker container logs by running `docker logs <>`. For `datahub-gms`, you should see a log similar to this at the end of the initialization: diff --git a/metadata-ingestion/src/datahub/cli/docker_cli.py b/metadata-ingestion/src/datahub/cli/docker_cli.py index 08f3faae8abb2a..0e0bc37c61573d 100644 --- a/metadata-ingestion/src/datahub/cli/docker_cli.py +++ b/metadata-ingestion/src/datahub/cli/docker_cli.py @@ -158,6 +158,7 @@ def should_use_neo4j_for_graph_service(graph_service_override: Optional[str]) -> def _set_environment_variables( version: Optional[str], + mysql_version: Optional[str], mysql_port: Optional[pydantic.PositiveInt], zk_port: Optional[pydantic.PositiveInt], kafka_broker_port: Optional[pydantic.PositiveInt], @@ -172,6 +173,8 @@ def _set_environment_variables( ) version = f"v{version}" os.environ["DATAHUB_VERSION"] = version + if mysql_version is not None: + os.environ["DATAHUB_MYSQL_VERSION"] = mysql_version if mysql_port is not None: os.environ["DATAHUB_MAPPED_MYSQL_PORT"] = str(mysql_port) @@ -675,6 +678,7 @@ def quickstart( # noqa: C901 # set version _set_environment_variables( version=quickstart_execution_plan.docker_tag, + mysql_version=quickstart_execution_plan.mysql_tag, mysql_port=mysql_port, zk_port=zk_port, kafka_broker_port=kafka_broker_port, diff --git a/metadata-ingestion/src/datahub/cli/quickstart_versioning.py b/metadata-ingestion/src/datahub/cli/quickstart_versioning.py index dc7bd1ecb4a89d..be7439f330dfb6 100644 --- a/metadata-ingestion/src/datahub/cli/quickstart_versioning.py +++ b/metadata-ingestion/src/datahub/cli/quickstart_versioning.py @@ -21,6 +21,7 @@ class QuickstartExecutionPlan(BaseModel): composefile_git_ref: str docker_tag: str + mysql_tag: Optional[str] def _is_it_a_version(version: str) -> bool: @@ -81,7 +82,7 @@ def fetch_quickstart_config(cls) -> "QuickstartVersionMappingConfig": return QuickstartVersionMappingConfig( quickstart_version_map={ "default": QuickstartExecutionPlan( - composefile_git_ref="master", docker_tag="head" + composefile_git_ref="master", docker_tag="head", mysql_tag="5.7" ), } ) @@ -93,7 +94,7 @@ def fetch_quickstart_config(cls) -> "QuickstartVersionMappingConfig": try: release = cls._fetch_latest_version() config.quickstart_version_map["stable"] = QuickstartExecutionPlan( - composefile_git_ref=release, docker_tag=release + composefile_git_ref=release, docker_tag=release, mysql_tag=release ) except Exception: click.echo( @@ -103,7 +104,8 @@ def fetch_quickstart_config(cls) -> "QuickstartVersionMappingConfig": return config def get_quickstart_execution_plan( - self, requested_version: Optional[str] + self, + requested_version: Optional[str], ) -> QuickstartExecutionPlan: """ From the requested version and stable flag, returns the execution plan for the quickstart. @@ -114,10 +116,14 @@ def get_quickstart_execution_plan( requested_version = "default" composefile_git_ref = requested_version docker_tag = requested_version + # Default to 5.7 if not specified in version map + mysql_tag = "5.7" result = self.quickstart_version_map.get( requested_version, QuickstartExecutionPlan( - composefile_git_ref=composefile_git_ref, docker_tag=docker_tag + composefile_git_ref=composefile_git_ref, + docker_tag=docker_tag, + mysql_tag=mysql_tag, ), ) # new CLI version is downloading the composefile corresponding to the requested version diff --git a/metadata-ingestion/tests/unit/cli/test_quickstart_version_mapping.py b/metadata-ingestion/tests/unit/cli/test_quickstart_version_mapping.py index 2fd56560c933da..3b06e48522955d 100644 --- a/metadata-ingestion/tests/unit/cli/test_quickstart_version_mapping.py +++ b/metadata-ingestion/tests/unit/cli/test_quickstart_version_mapping.py @@ -6,16 +6,30 @@ example_version_mapper = QuickstartVersionMappingConfig.parse_obj( { "quickstart_version_map": { - "default": {"composefile_git_ref": "master", "docker_tag": "latest"}, + "default": { + "composefile_git_ref": "master", + "docker_tag": "latest", + "mysql_tag": "5.7", + }, "v0.9.6": { "composefile_git_ref": "v0.9.6.1", "docker_tag": "v0.9.6.1", + "mysql_tag": "5.7", + }, + "v2.0.0": { + "composefile_git_ref": "v2.0.1", + "docker_tag": "v2.0.0", + "mysql_tag": "5.7", + }, + "v1.0.0": { + "composefile_git_ref": "v1.0.0", + "docker_tag": "v1.0.0", + "mysql_tag": "5.7", }, - "v2.0.0": {"composefile_git_ref": "v2.0.1", "docker_tag": "v2.0.0"}, - "v1.0.0": {"composefile_git_ref": "v1.0.0", "docker_tag": "v1.0.0"}, "stable": { "composefile_git_ref": "v1.0.1", "docker_tag": "latest", + "mysql_tag": "5.7", }, }, } @@ -27,6 +41,7 @@ def test_quickstart_version_config(): expected = QuickstartExecutionPlan( docker_tag="v1.0.0", composefile_git_ref="v1.0.0", + mysql_tag="5.7", ) assert execution_plan == expected @@ -36,6 +51,7 @@ def test_quickstart_version_config_default(): expected = QuickstartExecutionPlan( docker_tag="v2.0.0", composefile_git_ref="v2.0.1", + mysql_tag="5.7", ) assert execution_plan == expected @@ -43,20 +59,22 @@ def test_quickstart_version_config_default(): def test_quickstart_version_config_stable(): execution_plan = example_version_mapper.get_quickstart_execution_plan("stable") expected = QuickstartExecutionPlan( - docker_tag="latest", - composefile_git_ref="v1.0.1", + docker_tag="latest", composefile_git_ref="v1.0.1", mysql_tag="5.7" ) assert execution_plan == expected def test_quickstart_forced_stable(): example_version_mapper.quickstart_version_map["default"] = QuickstartExecutionPlan( - composefile_git_ref="v1.0.1", docker_tag="latest" + composefile_git_ref="v1.0.1", + docker_tag="latest", + mysql_tag="5.7", ) execution_plan = example_version_mapper.get_quickstart_execution_plan(None) expected = QuickstartExecutionPlan( docker_tag="latest", composefile_git_ref="v1.0.1", + mysql_tag="5.7", ) assert execution_plan == expected @@ -74,6 +92,7 @@ def test_quickstart_forced_not_a_version_tag(): expected = QuickstartExecutionPlan( docker_tag="NOT A VERSION", composefile_git_ref="NOT A VERSION", + mysql_tag="5.7", ) assert execution_plan == expected @@ -83,5 +102,6 @@ def test_quickstart_get_older_version(): expected = QuickstartExecutionPlan( docker_tag="v0.9.6.1", composefile_git_ref="v0.9.6.1", + mysql_tag="5.7", ) assert execution_plan == expected