feat(mysql): upgrade to version 8.2 for quickstart (datahub-project#9241

)

Co-authored-by: david-leifker <[email protected]>

docker/docker-compose-without-neo4j.override.yml

@@ -48,7 +48,7 @@ services:
   mysql:
     container_name: mysql
     hostname: mysql
-    image: mysql:5.7
+    image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
     command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
     ports:
       - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306

docker/docker-compose.override.yml

@@ -32,7 +32,7 @@ services:
   mysql:
     container_name: mysql
     hostname: mysql
-    image: mysql:5.7
+    image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
     command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password
     ports:
     - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306

docker/mysql/docker-compose.mysql.yml

@@ -5,7 +5,7 @@ services:
   mysql:
     container_name: mysql
     hostname: mysql
-    image: mysql:5.7
+    image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
     env_file: env/docker.env
     command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin
     ports:

docker/quickstart/docker-compose-without-neo4j.quickstart.yml

@@ -218,7 +218,7 @@ services:
       test: mysqladmin ping -h mysql -u $$MYSQL_USER --password=$$MYSQL_PASSWORD
       timeout: 5s
     hostname: mysql
-    image: mysql:5.7
+    image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
     ports:
     - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
     restart: on-failure

docker/quickstart/docker-compose.quickstart.yml

@@ -225,7 +225,7 @@ services:
       test: mysqladmin ping -h mysql -u $$MYSQL_USER --password=$$MYSQL_PASSWORD
       timeout: 5s
     hostname: mysql
-    image: mysql:5.7
+    image: mysql:${DATAHUB_MYSQL_VERSION:-5.7}
     ports:
     - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306
     restart: on-failure

docker/quickstart/quickstart_version_mapping.yaml

@@ -23,18 +23,27 @@ quickstart_version_map:
   default:
     composefile_git_ref: master
     docker_tag: head
+    mysql_tag: 5.7
   # default:  # Use this to pin default to a specific version.
   #   composefile_git_ref: fd1bd51541a132017a648f4a2f037eec8f70ba26 # v0.10.0 + quickstart compose file fixes
   #   docker_tag: v0.10.0
 
   head:
     composefile_git_ref: master
     docker_tag: head
+    mysql_tag: 5.7
+
+  # v0.13.0 we upgraded MySQL image for EOL
+  v0.13.0:
+    composefile_git_ref: master
+    docker_tag: head
+    mysql_tag: 8.2
 
   # v0.9.6 images contain security vulnerabilities
   v0.9.6:
     composefile_git_ref: v0.9.6.1
     docker_tag: v0.9.6.1
+    mysql_tag: 5.7
 
   # If stable is not defined the latest released version will be used.
   # stable:

docs/how/extract-container-logs.md

@@ -21,7 +21,7 @@ CONTAINER ID   IMAGE                                   COMMAND
 3680fcaef3ed   confluentinc/cp-kafka:5.4.0             "/etc/confluent/dock…"   5 days ago   Up 5 days               0.0.0.0:9092->9092/tcp, 0.0.0.0:29092->29092/tcp           broker
 9d6730ddd4c4   neo4j:4.0.6                             "/sbin/tini -g -- /d…"   5 days ago   Up 5 days               0.0.0.0:7474->7474/tcp, 7473/tcp, 0.0.0.0:7687->7687/tcp   neo4j
 c97edec663af   confluentinc/cp-zookeeper:5.4.0         "/etc/confluent/dock…"   5 days ago   Up 5 days               2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp                 zookeeper
-150ba161cf26   mysql:5.7                               "docker-entrypoint.s…"   5 days ago   Up 5 days               0.0.0.0:3306->3306/tcp, 33060/tcp                          mysql
+150ba161cf26   mysql:8.2                               "docker-entrypoint.s…"   5 days ago   Up 5 days               0.0.0.0:3306->3306/tcp, 33060/tcp                          mysql
 4b72a3eab73f   elasticsearch:7.9.3                     "/tini -- /usr/local…"   5 days ago   Up 5 days (healthy)     0.0.0.0:9200->9200/tcp, 9300/tcp                           elasticsearch
 ```
 

docs/how/updating-datahub.md

@@ -6,6 +6,7 @@ This file documents any backwards-incompatible changes in DataHub and assists pe
 
 ### Breaking Changes
 
+- Updating MySQL version for quickstarts to 8.2, may cause quickstart issues for existing instances.
 - #9244: The `redshift-legacy` and `redshift-legacy-usage` sources, which have been deprecated for >6 months, have been removed. The new `redshift` source is a superset of the functionality provided by those legacy sources.
 
 ### Potential Downtime

docs/troubleshooting/quickstart.md

@@ -99,7 +99,7 @@ c267c287a235        landoop/schema-registry-ui:latest                     "/run.
 943e60f9b4d0        neo4j:4.0.6                                           "/sbin/tini -g -- /d…"   10 hours ago        Up 10 hours         0.0.0.0:7474->7474/tcp, 7473/tcp, 0.0.0.0:7687->7687/tcp   neo4j
 6d79b6f02735        confluentinc/cp-zookeeper:5.2.1                       "/etc/confluent/dock…"   10 hours ago        Up 10 hours         2888/tcp, 0.0.0.0:2181->2181/tcp, 3888/tcp                 zookeeper
 491d9f2b2e9e        docker.elastic.co/elasticsearch/elasticsearch:5.6.8   "/bin/bash bin/es-do…"   10 hours ago        Up 10 hours         0.0.0.0:9200->9200/tcp, 9300/tcp                           elasticsearch
-ce14b9758eb3        mysql:5.7
+ce14b9758eb3        mysql:8.2
 ```
 
 Also you can check individual Docker container logs by running `docker logs <<container_name>>`. For `datahub-gms`, you should see a log similar to this at the end of the initialization:

metadata-ingestion/src/datahub/cli/docker_cli.py

@@ -158,6 +158,7 @@ def should_use_neo4j_for_graph_service(graph_service_override: Optional[str]) ->
 
 def _set_environment_variables(
     version: Optional[str],
+    mysql_version: Optional[str],
     mysql_port: Optional[pydantic.PositiveInt],
     zk_port: Optional[pydantic.PositiveInt],
     kafka_broker_port: Optional[pydantic.PositiveInt],
@@ -172,6 +173,8 @@ def _set_environment_variables(
             )
             version = f"v{version}"
         os.environ["DATAHUB_VERSION"] = version
+    if mysql_version is not None:
+        os.environ["DATAHUB_MYSQL_VERSION"] = mysql_version
     if mysql_port is not None:
         os.environ["DATAHUB_MAPPED_MYSQL_PORT"] = str(mysql_port)
 
@@ -675,6 +678,7 @@ def quickstart(  # noqa: C901
     # set version
     _set_environment_variables(
         version=quickstart_execution_plan.docker_tag,
+        mysql_version=quickstart_execution_plan.mysql_tag,
         mysql_port=mysql_port,
         zk_port=zk_port,
         kafka_broker_port=kafka_broker_port,

metadata-ingestion/src/datahub/cli/quickstart_versioning.py

@@ -21,6 +21,7 @@
 class QuickstartExecutionPlan(BaseModel):
     composefile_git_ref: str
     docker_tag: str
+    mysql_tag: Optional[str]
 
 
 def _is_it_a_version(version: str) -> bool:
@@ -81,7 +82,7 @@ def fetch_quickstart_config(cls) -> "QuickstartVersionMappingConfig":
             return QuickstartVersionMappingConfig(
                 quickstart_version_map={
                     "default": QuickstartExecutionPlan(
-                        composefile_git_ref="master", docker_tag="head"
+                        composefile_git_ref="master", docker_tag="head", mysql_tag="5.7"
                     ),
                 }
             )
@@ -93,7 +94,7 @@ def fetch_quickstart_config(cls) -> "QuickstartVersionMappingConfig":
             try:
                 release = cls._fetch_latest_version()
                 config.quickstart_version_map["stable"] = QuickstartExecutionPlan(
-                    composefile_git_ref=release, docker_tag=release
+                    composefile_git_ref=release, docker_tag=release, mysql_tag=release
                 )
             except Exception:
                 click.echo(
@@ -103,7 +104,8 @@ def fetch_quickstart_config(cls) -> "QuickstartVersionMappingConfig":
         return config
 
     def get_quickstart_execution_plan(
-        self, requested_version: Optional[str]
+        self,
+        requested_version: Optional[str],
     ) -> QuickstartExecutionPlan:
         """
         From the requested version and stable flag, returns the execution plan for the quickstart.
@@ -114,10 +116,14 @@ def get_quickstart_execution_plan(
             requested_version = "default"
         composefile_git_ref = requested_version
         docker_tag = requested_version
+        # Default to 5.7 if not specified in version map
+        mysql_tag = "5.7"
         result = self.quickstart_version_map.get(
             requested_version,
             QuickstartExecutionPlan(
-                composefile_git_ref=composefile_git_ref, docker_tag=docker_tag
+                composefile_git_ref=composefile_git_ref,
+                docker_tag=docker_tag,
+                mysql_tag=mysql_tag,
             ),
         )
         # new CLI version is downloading the composefile corresponding to the requested version

metadata-ingestion/tests/unit/cli/test_quickstart_version_mapping.py

@@ -6,16 +6,30 @@
 example_version_mapper = QuickstartVersionMappingConfig.parse_obj(
     {
         "quickstart_version_map": {
-            "default": {"composefile_git_ref": "master", "docker_tag": "latest"},
+            "default": {
+                "composefile_git_ref": "master",
+                "docker_tag": "latest",
+                "mysql_tag": "5.7",
+            },
             "v0.9.6": {
                 "composefile_git_ref": "v0.9.6.1",
                 "docker_tag": "v0.9.6.1",
+                "mysql_tag": "5.7",
+            },
+            "v2.0.0": {
+                "composefile_git_ref": "v2.0.1",
+                "docker_tag": "v2.0.0",
+                "mysql_tag": "5.7",
+            },
+            "v1.0.0": {
+                "composefile_git_ref": "v1.0.0",
+                "docker_tag": "v1.0.0",
+                "mysql_tag": "5.7",
             },
-            "v2.0.0": {"composefile_git_ref": "v2.0.1", "docker_tag": "v2.0.0"},
-            "v1.0.0": {"composefile_git_ref": "v1.0.0", "docker_tag": "v1.0.0"},
             "stable": {
                 "composefile_git_ref": "v1.0.1",
                 "docker_tag": "latest",
+                "mysql_tag": "5.7",
             },
         },
     }
@@ -27,6 +41,7 @@ def test_quickstart_version_config():
     expected = QuickstartExecutionPlan(
         docker_tag="v1.0.0",
         composefile_git_ref="v1.0.0",
+        mysql_tag="5.7",
     )
     assert execution_plan == expected
 
@@ -36,27 +51,30 @@ def test_quickstart_version_config_default():
     expected = QuickstartExecutionPlan(
         docker_tag="v2.0.0",
         composefile_git_ref="v2.0.1",
+        mysql_tag="5.7",
     )
     assert execution_plan == expected
 
 
 def test_quickstart_version_config_stable():
     execution_plan = example_version_mapper.get_quickstart_execution_plan("stable")
     expected = QuickstartExecutionPlan(
-        docker_tag="latest",
-        composefile_git_ref="v1.0.1",
+        docker_tag="latest", composefile_git_ref="v1.0.1", mysql_tag="5.7"
     )
     assert execution_plan == expected
 
 
 def test_quickstart_forced_stable():
     example_version_mapper.quickstart_version_map["default"] = QuickstartExecutionPlan(
-        composefile_git_ref="v1.0.1", docker_tag="latest"
+        composefile_git_ref="v1.0.1",
+        docker_tag="latest",
+        mysql_tag="5.7",
     )
     execution_plan = example_version_mapper.get_quickstart_execution_plan(None)
     expected = QuickstartExecutionPlan(
         docker_tag="latest",
         composefile_git_ref="v1.0.1",
+        mysql_tag="5.7",
     )
     assert execution_plan == expected
 
@@ -74,6 +92,7 @@ def test_quickstart_forced_not_a_version_tag():
     expected = QuickstartExecutionPlan(
         docker_tag="NOT A VERSION",
         composefile_git_ref="NOT A VERSION",
+        mysql_tag="5.7",
     )
     assert execution_plan == expected
 
@@ -83,5 +102,6 @@ def test_quickstart_get_older_version():
     expected = QuickstartExecutionPlan(
         docker_tag="v0.9.6.1",
         composefile_git_ref="v0.9.6.1",
+        mysql_tag="5.7",
     )
     assert execution_plan == expected