From 64080312e77833ccc439c151c0f1baf818aeb1f1 Mon Sep 17 00:00:00 2001 From: Gero Posmyk-Leinemann <32448529+geropl@users.noreply.github.com> Date: Wed, 6 Sep 2023 15:28:03 +0200 Subject: [PATCH] [db] Run test against the same (bitnami/)mysql:8.0.33 version (#18373) --- .github/workflows/build.yml | 10 +++++---- components/gitpod-db/BUILD.yaml | 3 ++- .../pkg/components/database/incluster/helm.go | 22 ++++++++++++++++++- install/installer/pkg/helm/common.go | 10 ++++++++- install/installer/pkg/helm/helm.go | 3 +-- 5 files changed, 39 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bfcc219c4fbf97..41d402d5c73413 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -145,10 +145,12 @@ jobs: cancel-in-progress: ${{ needs.configuration.outputs.is_main_branch == 'false' }} services: mysql: - image: mysql:5.7 + image: bitnami/mysql:8.0.33-debian-11-r24 env: MYSQL_ROOT_PASSWORD: test - MYSQL_TCP_PORT: 23306 + #MYSQL_TCP_PORT: 23306 bitnami/mysql does not honor this, but has it's own: + MYSQL_PORT_NUMBER: 23306 + MYSQL_AUTHENTICATION_PLUGIN: mysql_native_password ports: - 23306:23306 redis: @@ -161,6 +163,8 @@ jobs: DB_HOST: "mysql" DB_PORT: "23306" REDIS_HOST: "redis" + # GitHub action + MySQL 8.0 need longer to initialize + DB_RETRIES: 5 steps: - uses: actions/checkout@v3 - uses: ./.github/actions/setup-environment @@ -222,8 +226,6 @@ jobs: id: leeway shell: bash env: - DB_HOST: "mysql" - DB_PORT: "23306" NODE_OPTIONS: "--max_old_space_size=4096" JAVA_HOME: /home/gitpod/.sdkman/candidates/java/current VERSION: ${{needs.configuration.outputs.version}} diff --git a/components/gitpod-db/BUILD.yaml b/components/gitpod-db/BUILD.yaml index 7f7686c4e96fde..8a614a0133af1a 100644 --- a/components/gitpod-db/BUILD.yaml +++ b/components/gitpod-db/BUILD.yaml @@ -76,7 +76,8 @@ packages: # Check if a DB is present. If not: start one and wait until it's up # Note: In CI there is a DB running as sidecar; in workspaces we're starting it once. # Re-use of the instance because of the init scripts (cmp. next step). - - ["sh", "-c", "mysqladmin ping -h $DB_HOST --port $DB_PORT -p$DB_PASSWORD -u$DB_USER --silent || (docker run --name test-mysql -d -e MYSQL_ROOT_PASSWORD=$DB_PASSWORD -e MYSQL_TCP_PORT=$DB_PORT -p $DB_PORT:$DB_PORT mysql:5.7; while ! mysqladmin ping -h \"$DB_HOST\" -P \"$DB_PORT\" -p$DB_PASSWORD -u$DB_USER --silent; do echo \"waiting for DB...\"; sleep 2; done)"] + # (gpl): It would be nice to use bitnami/mysql here as we do in previews. However the container does not start in Gitpod workspaces due to some docker/kernel/namespace issue. + - ["sh", "-c", "mysqladmin ping --wait=${DB_RETRIES:-1} -h $DB_HOST --port $DB_PORT -p$DB_PASSWORD -u$DB_USER --default-auth=mysql_native_password --silent || (docker run --name test-mysql -d -e MYSQL_ROOT_PASSWORD=$DB_PASSWORD -e MYSQL_TCP_PORT=$DB_PORT -p $DB_PORT:$DB_PORT mysql:8.0.33 --default-authentication-plugin=mysql_native_password; while ! mysqladmin ping -h \"$DB_HOST\" -P \"$DB_PORT\" -p$DB_PASSWORD -u$DB_USER --default-auth=mysql_native_password --silent; do echo \"waiting for DB...\"; sleep 2; done)"] # Apply the DB initialization scripts (re-creates the "gitpod" DB if already there) - ["mkdir", "-p", "init-scripts"] - ["sh", "-c", "find . -name \"*.sql\" | sort | xargs -I file cp file init-scripts"] diff --git a/install/installer/pkg/components/database/incluster/helm.go b/install/installer/pkg/components/database/incluster/helm.go index 4261ec005de564..e6cbdb9a089720 100644 --- a/install/installer/pkg/components/database/incluster/helm.go +++ b/install/installer/pkg/components/database/incluster/helm.go @@ -10,6 +10,7 @@ import ( "github.com/gitpod-io/gitpod/installer/pkg/helm" "github.com/gitpod-io/gitpod/installer/third_party/charts" "helm.sh/helm/v3/pkg/cli/values" + "sigs.k8s.io/yaml" ) var Helm = common.CompositeHelmFunc( @@ -26,10 +27,28 @@ var Helm = common.CompositeHelmFunc( imageRegistry := common.ThirdPartyContainerRepo(cfg.Config.Repository, common.DockerRegistryURL) - // We switched to specific tags because we got subtle broken versions with just specifying major versions + type EnvVar struct { + // json because: https://pkg.go.dev/sigs.k8s.io/yaml@v1.3.0#Marshal + Name string `json:"name,omitempty"` + Value string `json:"value,omitempty"` + } + extraEnvVars := []EnvVar{} + // MySQL 5.7: We switched to specific tags because we got subtle broken versions with just specifying major versions mysqlBitnamiImageTag := "5.7.34-debian-10-r55" if cfg.Config.Database.InClusterMysSQL_8_0 { mysqlBitnamiImageTag = "8.0.33-debian-11-r24" + extraEnvVars = append(extraEnvVars, EnvVar{ + Name: "MYSQL_AUTHENTICATION_PLUGIN", + Value: "mysql_native_password", + }) + } + extraEnvVarsBytes, err := yaml.Marshal(extraEnvVars) + if err != nil { + return nil, err + } + extraEnvVarsTemplate, err := helm.KeyFileValue("mysql.primary.extraEnvVars", extraEnvVarsBytes) + if err != nil { + return nil, err } return &common.HelmConfig{ @@ -57,6 +76,7 @@ var Helm = common.CompositeHelmFunc( // This is too complex to be sent as a string FileValues: []string{ primaryAffinityTemplate, + extraEnvVarsTemplate, }, }, }, nil diff --git a/install/installer/pkg/helm/common.go b/install/installer/pkg/helm/common.go index 082057b9412c90..b915c793e431ee 100644 --- a/install/installer/pkg/helm/common.go +++ b/install/installer/pkg/helm/common.go @@ -6,8 +6,10 @@ package helm import ( "fmt" - "github.com/gitpod-io/gitpod/installer/pkg/common" "os" + "strings" + + "github.com/gitpod-io/gitpod/installer/pkg/common" ) // KeyValue ensure that a key/value pair is correctly formatted for Values @@ -15,6 +17,12 @@ func KeyValue(key string, value string) string { return fmt.Sprintf("%s=%s", key, value) } +// KeyValueArray ensure that a key/value pair is correctly formatted for Arrays +func KeyValueArray(key string, arr []string) string { + // Helm array nomenclature + return KeyValue(key, fmt.Sprintf("{%s}", strings.Join(arr, ","))) +} + // KeyFileValue ensure that a key/value pair is correctly formatted for FileValues func KeyFileValue(key string, data []byte) (string, error) { dir, err := os.MkdirTemp("", "helm") diff --git a/install/installer/pkg/helm/helm.go b/install/installer/pkg/helm/helm.go index 5c3fb1729a3ab9..829cab9df5876f 100644 --- a/install/installer/pkg/helm/helm.go +++ b/install/installer/pkg/helm/helm.go @@ -154,8 +154,7 @@ func ImagePullSecrets(key string, ctx *common.RenderContext) string { pullSecrets = append(pullSecrets, i.Name) } - // Helm array nomenclature - return KeyValue(key, fmt.Sprintf("{%s}", strings.Join(pullSecrets, ","))) + return KeyValueArray(key, pullSecrets) } // Nothing to be set