diff --git a/.github/workflows/code-nightly.yml b/.github/workflows/code-nightly.yml index 18ea00da754866..5a943ce6d1cd92 100644 --- a/.github/workflows/code-nightly.yml +++ b/.github/workflows/code-nightly.yml @@ -45,7 +45,7 @@ jobs: -DcodeCommit=$codeHeadCommit \ -DcodeVersion=$codeVersion \ -DcodeQuality=insider \ - .:docker + .:docker-nightly - name: Get previous job's status id: lastrun uses: filiptronicek/get-last-job-status@main diff --git a/components/ide/code/BUILD.yaml b/components/ide/code/BUILD.yaml index c5e74172d5ddc2..06ac7a7db3a056 100644 --- a/components/ide/code/BUILD.yaml +++ b/components/ide/code/BUILD.yaml @@ -17,3 +17,20 @@ packages: image: - ${imageRepoBase}/ide/code:${version} - ${imageRepoBase}/ide/code:commit-${__git_commit} + - name: docker-nightly + type: docker + argdeps: + - imageRepoBase + - codeCommit + - codeQuality + - codeVersion + config: + dockerfile: leeway.Dockerfile + metadata: + helm-component: workspace.codeImage + buildArgs: + CODE_COMMIT: ${codeCommit} + CODE_QUALITY: ${codeQuality} + CODE_VERSION: ${codeVersion} + image: + - ${imageRepoBase}/ide/code:nightly diff --git a/components/ide/code/leeway.Dockerfile b/components/ide/code/leeway.Dockerfile index 776c5cab1ef5cb..de6ec55b43c136 100644 --- a/components/ide/code/leeway.Dockerfile +++ b/components/ide/code/leeway.Dockerfile @@ -3,6 +3,8 @@ # See License.AGPL.txt in the project root for license information. FROM gitpod/openvscode-server-linux-build-agent:centos7-devtoolset8-x64 as dependencies_builder +ENV TRIGGER_REBUILD 1 + ARG CODE_COMMIT RUN mkdir /gp-code \ diff --git a/components/ide/gha-update-image/lib/code-pin-version.ts b/components/ide/gha-update-image/lib/code-pin-version.ts index 2f909eaa191165..b2cd6b5d3a9779 100644 --- a/components/ide/gha-update-image/lib/code-pin-version.ts +++ b/components/ide/gha-update-image/lib/code-pin-version.ts @@ -9,6 +9,7 @@ import { pathToConfigmap, readIDEConfigmapJson, readWorkspaceYaml, + renderInstallerIDEConfigMap, } from "./common"; $.nothrow(); @@ -50,13 +51,15 @@ export async function updateCodeIDEConfigMapJson() { newJson.ideOptions.options.code = updateImages(newJson.ideOptions.options.code); // try append new pin versions - const installationCodeVersion = await getIDEVersionOfImage(`ide/code:${latestBuildImage.code}`); - if (installationCodeVersion.trim() === "") { - throw new Error("installation code version can't be empty"); + const previousCodeVersion = await getIDEVersionOfImage("eu.gcr.io/gitpod-core-dev/build/" + ideConfigmapJson.ideOptions.options.code.image.replace("{{.Repository}}/", "")); + const installerIDEConfigMap = await renderInstallerIDEConfigMap(undefined); + const installationCodeVersion = await getIDEVersionOfImage(installerIDEConfigMap.ideOptions.options.code.image); + if (installationCodeVersion.trim() === "" || previousCodeVersion.trim() === "") { + throw new Error("installation or previous code version can't be empty"); } let appendNewVersion = false; - console.log("installation code version", installationCodeVersion); - if (installationCodeVersion === workspaceYaml.defaultArgs.codeVersion) { + console.log("code versions", { installationCodeVersion, previousCodeVersion }); + if (installationCodeVersion === previousCodeVersion) { console.log("code version is the same, no need to update (ide-service will do it)", installationCodeVersion); } else { const hasPinned = firstPinnedInfo.version === installationCodeVersion; diff --git a/components/ide/gha-update-image/lib/common.ts b/components/ide/gha-update-image/lib/common.ts index 40229d82e8ee1c..d10eb91c15d5e7 100644 --- a/components/ide/gha-update-image/lib/common.ts +++ b/components/ide/gha-update-image/lib/common.ts @@ -58,21 +58,32 @@ export const pathToConfigmap = path.resolve( pathToProjectRoot, "install/installer/pkg/components/ide-service/ide-configmap.json", ); + +const IDEOptionSchema = z.object({ + image: z.string(), + imageLayers: z.array(z.string()), + versions: z.array( + z.object({ + version: z.string(), + image: z.string(), + imageLayers: z.array(z.string()), + }), + ), +}); const ideConfigmapJsonSchema = z.object({ supervisorImage: z.string(), ideOptions: z.object({ options: z.object({ - code: z.object({ - image: z.string(), - imageLayers: z.array(z.string()), - versions: z.array( - z.object({ - version: z.string(), - image: z.string(), - imageLayers: z.array(z.string()), - }), - ), - }), + code: IDEOptionSchema, + intellij: IDEOptionSchema, + goland: IDEOptionSchema, + pycharm: IDEOptionSchema, + phpstorm: IDEOptionSchema, + rubymine: IDEOptionSchema, + webstorm: IDEOptionSchema, + rider: IDEOptionSchema, + clion: IDEOptionSchema, + rustrover: IDEOptionSchema, }), }), }); @@ -88,8 +99,7 @@ export const readIDEConfigmapJson = async () => { }; }; -// installer versions -export const getLatestInstallerVersions = async (version?: string) => { +const getInstallerVersion = async (version: string | undefined) => { const v = version ? version : "main-gha."; let tagInfo: string; try { @@ -108,17 +118,16 @@ export const getLatestInstallerVersions = async (version?: string) => { .catch((e) => { throw new Error("Failed to parse installer version from git tag", e); }); - console.log("Fetching installer version for", installationVersion); - // exec command below to see results - // ``` - // $ docker run --rm eu.gcr.io/gitpod-core-dev/build/versions:main-gha.25759 cat /versions.yaml | yq r - - // ``` - const versionData = - await $`docker run --rm eu.gcr.io/gitpod-core-dev/build/versions:${installationVersion.trim()} cat /versions.yaml` - .text() - .catch((e) => { - throw new Error("Failed to fetch versions.yaml from latest installer", e); - }); + return installationVersion; +} + +// installer versions +export const getLatestInstallerVersions = async (version?: string) => { + const installationVersion = await getInstallerVersion(version); + console.log("Fetching installer versions for", installationVersion); + const versionData = await $`docker run --rm eu.gcr.io/gitpod-core-dev/build/versions:${installationVersion} cat /versions.yaml`.text().catch((e) => { + throw new Error(`Failed to get installer versions`, e); + }); const versionObj = z.object({ version: z.string() }); return z @@ -161,10 +170,24 @@ export const getLatestInstallerVersions = async (version?: string) => { .parse(yaml.parse(versionData)); }; +export const renderInstallerIDEConfigMap = async (version?: string) => { + const installationVersion = await getInstallerVersion(version); + await $`docker run --rm -v /tmp:/tmp eu.gcr.io/gitpod-core-dev/build/installer:${installationVersion} config init --overwrite --log-level=error -c /tmp/gitpod.config.yaml`.catch((e) => { + throw new Error("Failed to render gitpod.config.yaml", e); + }) + const ideConfigMapStr = await $`cat /tmp/gitpod.config.yaml | docker run -i --rm eu.gcr.io/gitpod-core-dev/build/installer:${installationVersion} ide-configmap -c -`.text().catch((e) => { + throw new Error(`Failed to render ide-configmap`, e); + }); + const ideConfigmapJsonObj = JSON.parse(ideConfigMapStr); + const ideConfigmapJson = ideConfigmapJsonSchema.parse(ideConfigmapJsonObj); + return ideConfigmapJson; +} export const getIDEVersionOfImage = async (img: string) => { - console.log("Fetching IDE version in image:", `oci-tool fetch image eu.gcr.io/gitpod-core-dev/build/${img} | jq -r '.config.Labels["io.gitpod.ide.version"]'`) - return await $`oci-tool fetch image eu.gcr.io/gitpod-core-dev/build/${img} | jq -r '.config.Labels["io.gitpod.ide.version"]'`.text().catch((e) => { + console.log("Fetching IDE version in image:", `oci-tool fetch image execInstaller${img} | jq -r '.config.Labels["io.gitpod.ide.version"]'`) + const version = await $`oci-tool fetch image execInstaller${img} | jq -r '.config.Labels["io.gitpod.ide.version"]'`.text().catch((e) => { throw new Error("Failed to fetch ide version in image", e); }).then(str => str.replaceAll("\n", "")); + console.log("IDE version in image:", version); + return version; } diff --git a/components/ide/gha-update-image/lib/jb-pin-version.ts b/components/ide/gha-update-image/lib/jb-pin-version.ts index b62368b1415be2..4af1bb31b9575a 100644 --- a/components/ide/gha-update-image/lib/jb-pin-version.ts +++ b/components/ide/gha-update-image/lib/jb-pin-version.ts @@ -8,6 +8,8 @@ import { readIDEConfigmapJson, readWorkspaceYaml, IWorkspaceYaml, + getIDEVersionOfImage, + renderInstallerIDEConfigMap, } from "./common"; const configmap = await readIDEConfigmapJson().then((d) => d.rawObj); @@ -41,16 +43,16 @@ export const appendPinVersionsIntoIDEConfigMap = async (updatedIDEs: string[] | console.debug(`Processing ${ide} ${ideVersion}...`); + const ideConfigMap = await renderInstallerIDEConfigMap(undefined) + if (Object.keys(configmap.ideOptions.options).includes(ide)) { const { version: installerImageVersion } = versionObject; - const installerIdeVersion = { - version: ideVersion, - image: `{{.Repository}}/ide/${ide}:${installerImageVersion}`, - imageLayers: [ - `{{.Repository}}/ide/jb-backend-plugin:${latestInstallerVersions.components.workspace.desktopIdeImages.jbBackendPlugin.version}`, - `{{.Repository}}/ide/jb-launcher:${latestInstallerVersions.components.workspace.desktopIdeImages.jbLauncher.version}`, - ], + const previousVersion = await getIDEVersionOfImage(ideConfigMap.ideOptions.options[ide].image); + const previousInfo = { + version: previousVersion, + image: ideConfigMap.ideOptions.options[ide].image, + imageLayers: ideConfigMap.ideOptions.options[ide].imageLayers, }; if (!updatedIDEs || !updatedIDEs.includes(ide)) { @@ -60,8 +62,8 @@ export const appendPinVersionsIntoIDEConfigMap = async (updatedIDEs: string[] | if (!configmap.ideOptions.options[ide].versions) { configmap.ideOptions.options[ide].versions = [] } - console.log(`Added ${ide} (new ${installerIdeVersion.image})`); - configmap.ideOptions.options[ide].versions.unshift(installerIdeVersion); + console.log(`Added ${ide} (new ${previousInfo.image})`); + configmap.ideOptions.options[ide].versions.unshift(previousInfo); } } diff --git a/install/installer/cmd/ideconfigmap.go b/install/installer/cmd/ideconfigmap.go new file mode 100644 index 00000000000000..27057220ef383f --- /dev/null +++ b/install/installer/cmd/ideconfigmap.go @@ -0,0 +1,83 @@ +// Copyright (c) 2024 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License.AGPL.txt in the project root for license information. + +package cmd + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/gitpod-io/gitpod/common-go/log" + "github.com/gitpod-io/gitpod/installer/pkg/common" + ide_service "github.com/gitpod-io/gitpod/installer/pkg/components/ide-service" + + "github.com/spf13/cobra" +) + +var ideOpts struct { + ConfigFN string + Namespace string + UseExperimentalConfig bool +} + +// ideConfigmapCmd generates ide-configmap.json +var ideConfigmapCmd = &cobra.Command{ + Use: "ide-configmap", + Hidden: true, + Short: "render ide-configmap.json", + RunE: func(cmd *cobra.Command, args []string) error { + renderCtx, err := getRenderCtx(ideOpts.ConfigFN, ideOpts.Namespace, ideOpts.UseExperimentalConfig) + if err != nil { + return err + } + + ideConfig, err := ide_service.GenerateIDEConfigmap(renderCtx) + if err != nil { + return err + } + + fc, err := common.ToJSONString(ideConfig) + if err != nil { + return fmt.Errorf("failed to marshal ide-config config: %w", err) + } + + fmt.Println(string(fc)) + return nil + }, +} + +func getRenderCtx(configFN, namespace string, useExperimentalConfig bool) (*common.RenderContext, error) { + _, _, cfg, err := loadConfig(configFN) + if err != nil { + return nil, err + } + + if cfg.Experimental != nil { + if useExperimentalConfig { + fmt.Fprintf(os.Stderr, "rendering using experimental config\n") + } else { + fmt.Fprintf(os.Stderr, "ignoring experimental config. Use `--use-experimental-config` to include the experimental section in config\n") + cfg.Experimental = nil + } + } + versionMF, err := getVersionManifest() + if err != nil { + return nil, err + } + return common.NewRenderContext(*cfg, *versionMF, namespace) +} + +func init() { + rootCmd.AddCommand(ideConfigmapCmd) + + dir, err := os.Getwd() + if err != nil { + log.WithError(err).Fatal("Failed to get working directory") + } + + ideConfigmapCmd.PersistentFlags().StringVarP(&ideOpts.ConfigFN, "config", "c", getEnvvar("GITPOD_INSTALLER_CONFIG", filepath.Join(dir, "gitpod.config.yaml")), "path to the config file, use - for stdin") + ideConfigmapCmd.PersistentFlags().StringVarP(&ideOpts.Namespace, "namespace", "n", getEnvvar("NAMESPACE", "default"), "namespace to deploy to") + ideConfigmapCmd.Flags().BoolVar(&ideOpts.UseExperimentalConfig, "use-experimental-config", false, "enable the use of experimental config that is prone to be changed") +} diff --git a/install/installer/pkg/components/ide-service/ide_config_configmap.go b/install/installer/pkg/components/ide-service/ide_config_configmap.go index 3993243bb49031..8977b0a997a2fe 100644 --- a/install/installer/pkg/components/ide-service/ide_config_configmap.go +++ b/install/installer/pkg/components/ide-service/ide_config_configmap.go @@ -25,7 +25,7 @@ import ( //go:embed ide-configmap.json var ideConfigFile string -func ideConfigConfigmap(ctx *common.RenderContext) ([]runtime.Object, error) { +func GenerateIDEConfigmap(ctx *common.RenderContext) (*ide_config.IDEConfig, error) { resolveLatestImage := func(name string, tag string, bundledLatest versions.Versioned) string { resolveLatest := true if ctx.Config.Components != nil && ctx.Config.Components.IDE != nil && ctx.Config.Components.IDE.ResolveLatest != nil { @@ -119,7 +119,14 @@ func ideConfigConfigmap(ctx *common.RenderContext) ([]runtime.Object, error) { if idecfg.IdeOptions.Options[idecfg.IdeOptions.DefaultDesktopIde].Type != ide_config.IDETypeDesktop { return nil, fmt.Errorf("default desktop IDE '%s' does not point to a desktop IDE option", idecfg.IdeOptions.DefaultIde) } + return &idecfg, nil +} +func ideConfigConfigmap(ctx *common.RenderContext) ([]runtime.Object, error) { + idecfg, err := GenerateIDEConfigmap(ctx) + if err != nil { + return nil, fmt.Errorf("failed to generate ide-config config: %w", err) + } fc, err := common.ToJSONString(idecfg) if err != nil { return nil, fmt.Errorf("failed to marshal ide-config config: %w", err) diff --git a/test/run.sh b/test/run.sh index a1cad7c5a7d919..21b5702de55b48 100755 --- a/test/run.sh +++ b/test/run.sh @@ -146,11 +146,7 @@ else cd "${TEST_PATH}" set +e - if [ "$TEST_SUITE" == "jetbrains" ]; then - go test -parallel=3 -v ./... "${args[@]}" 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "${RESULTS_DIR}/TEST-${TEST_NAME}.xml" -iocopy - else - go test -v ./... "${args[@]}" 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "${RESULTS_DIR}/TEST-${TEST_NAME}.xml" -iocopy - fi + go test -parallel=3 -v ./... "${args[@]}" 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "${RESULTS_DIR}/TEST-${TEST_NAME}.xml" -iocopy RC=${PIPESTATUS[0]} set -e cd -