Skip to content

Commit

Permalink
Fix nightly VS Code Browser rewrite stable version images (#19940)
Browse files Browse the repository at this point in the history
* Fix nightly VS Code Browser rewrite stable version images

* fix gha

* Add installer cmd

* use rendered ide-configmap

* fixup

* Address

* write gitpod.config.yaml in /tmp
  • Loading branch information
mustard-mh authored Jun 20, 2024
1 parent dfb6b6d commit 9380449
Show file tree
Hide file tree
Showing 9 changed files with 180 additions and 47 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/code-nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs:
-DcodeCommit=$codeHeadCommit \
-DcodeVersion=$codeVersion \
-DcodeQuality=insider \
.:docker
.:docker-nightly
- name: Get previous job's status
id: lastrun
uses: filiptronicek/get-last-job-status@main
Expand Down
17 changes: 17 additions & 0 deletions components/ide/code/BUILD.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,20 @@ packages:
image:
- ${imageRepoBase}/ide/code:${version}
- ${imageRepoBase}/ide/code:commit-${__git_commit}
- name: docker-nightly
type: docker
argdeps:
- imageRepoBase
- codeCommit
- codeQuality
- codeVersion
config:
dockerfile: leeway.Dockerfile
metadata:
helm-component: workspace.codeImage
buildArgs:
CODE_COMMIT: ${codeCommit}
CODE_QUALITY: ${codeQuality}
CODE_VERSION: ${codeVersion}
image:
- ${imageRepoBase}/ide/code:nightly
2 changes: 2 additions & 0 deletions components/ide/code/leeway.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
# See License.AGPL.txt in the project root for license information.
FROM gitpod/openvscode-server-linux-build-agent:centos7-devtoolset8-x64 as dependencies_builder

ENV TRIGGER_REBUILD 1

ARG CODE_COMMIT

RUN mkdir /gp-code \
Expand Down
13 changes: 8 additions & 5 deletions components/ide/gha-update-image/lib/code-pin-version.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
pathToConfigmap,
readIDEConfigmapJson,
readWorkspaceYaml,
renderInstallerIDEConfigMap,
} from "./common";

$.nothrow();
Expand Down Expand Up @@ -50,13 +51,15 @@ export async function updateCodeIDEConfigMapJson() {
newJson.ideOptions.options.code = updateImages(newJson.ideOptions.options.code);

// try append new pin versions
const installationCodeVersion = await getIDEVersionOfImage(`ide/code:${latestBuildImage.code}`);
if (installationCodeVersion.trim() === "") {
throw new Error("installation code version can't be empty");
const previousCodeVersion = await getIDEVersionOfImage("eu.gcr.io/gitpod-core-dev/build/" + ideConfigmapJson.ideOptions.options.code.image.replace("{{.Repository}}/", ""));
const installerIDEConfigMap = await renderInstallerIDEConfigMap(undefined);
const installationCodeVersion = await getIDEVersionOfImage(installerIDEConfigMap.ideOptions.options.code.image);
if (installationCodeVersion.trim() === "" || previousCodeVersion.trim() === "") {
throw new Error("installation or previous code version can't be empty");
}
let appendNewVersion = false;
console.log("installation code version", installationCodeVersion);
if (installationCodeVersion === workspaceYaml.defaultArgs.codeVersion) {
console.log("code versions", { installationCodeVersion, previousCodeVersion });
if (installationCodeVersion === previousCodeVersion) {
console.log("code version is the same, no need to update (ide-service will do it)", installationCodeVersion);
} else {
const hasPinned = firstPinnedInfo.version === installationCodeVersion;
Expand Down
75 changes: 49 additions & 26 deletions components/ide/gha-update-image/lib/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -58,21 +58,32 @@ export const pathToConfigmap = path.resolve(
pathToProjectRoot,
"install/installer/pkg/components/ide-service/ide-configmap.json",
);

const IDEOptionSchema = z.object({
image: z.string(),
imageLayers: z.array(z.string()),
versions: z.array(
z.object({
version: z.string(),
image: z.string(),
imageLayers: z.array(z.string()),
}),
),
});
const ideConfigmapJsonSchema = z.object({
supervisorImage: z.string(),
ideOptions: z.object({
options: z.object({
code: z.object({
image: z.string(),
imageLayers: z.array(z.string()),
versions: z.array(
z.object({
version: z.string(),
image: z.string(),
imageLayers: z.array(z.string()),
}),
),
}),
code: IDEOptionSchema,
intellij: IDEOptionSchema,
goland: IDEOptionSchema,
pycharm: IDEOptionSchema,
phpstorm: IDEOptionSchema,
rubymine: IDEOptionSchema,
webstorm: IDEOptionSchema,
rider: IDEOptionSchema,
clion: IDEOptionSchema,
rustrover: IDEOptionSchema,
}),
}),
});
Expand All @@ -88,8 +99,7 @@ export const readIDEConfigmapJson = async () => {
};
};

// installer versions
export const getLatestInstallerVersions = async (version?: string) => {
const getInstallerVersion = async (version: string | undefined) => {
const v = version ? version : "main-gha.";
let tagInfo: string;
try {
Expand All @@ -108,17 +118,16 @@ export const getLatestInstallerVersions = async (version?: string) => {
.catch((e) => {
throw new Error("Failed to parse installer version from git tag", e);
});
console.log("Fetching installer version for", installationVersion);
// exec command below to see results
// ```
// $ docker run --rm eu.gcr.io/gitpod-core-dev/build/versions:main-gha.25759 cat /versions.yaml | yq r -
// ```
const versionData =
await $`docker run --rm eu.gcr.io/gitpod-core-dev/build/versions:${installationVersion.trim()} cat /versions.yaml`
.text()
.catch((e) => {
throw new Error("Failed to fetch versions.yaml from latest installer", e);
});
return installationVersion;
}

// installer versions
export const getLatestInstallerVersions = async (version?: string) => {
const installationVersion = await getInstallerVersion(version);
console.log("Fetching installer versions for", installationVersion);
const versionData = await $`docker run --rm eu.gcr.io/gitpod-core-dev/build/versions:${installationVersion} cat /versions.yaml`.text().catch((e) => {
throw new Error(`Failed to get installer versions`, e);
});

const versionObj = z.object({ version: z.string() });
return z
Expand Down Expand Up @@ -161,10 +170,24 @@ export const getLatestInstallerVersions = async (version?: string) => {
.parse(yaml.parse(versionData));
};

export const renderInstallerIDEConfigMap = async (version?: string) => {
const installationVersion = await getInstallerVersion(version);
await $`docker run --rm -v /tmp:/tmp eu.gcr.io/gitpod-core-dev/build/installer:${installationVersion} config init --overwrite --log-level=error -c /tmp/gitpod.config.yaml`.catch((e) => {
throw new Error("Failed to render gitpod.config.yaml", e);
})
const ideConfigMapStr = await $`cat /tmp/gitpod.config.yaml | docker run -i --rm eu.gcr.io/gitpod-core-dev/build/installer:${installationVersion} ide-configmap -c -`.text().catch((e) => {
throw new Error(`Failed to render ide-configmap`, e);
});
const ideConfigmapJsonObj = JSON.parse(ideConfigMapStr);
const ideConfigmapJson = ideConfigmapJsonSchema.parse(ideConfigmapJsonObj);
return ideConfigmapJson;
}

export const getIDEVersionOfImage = async (img: string) => {
console.log("Fetching IDE version in image:", `oci-tool fetch image eu.gcr.io/gitpod-core-dev/build/${img} | jq -r '.config.Labels["io.gitpod.ide.version"]'`)
return await $`oci-tool fetch image eu.gcr.io/gitpod-core-dev/build/${img} | jq -r '.config.Labels["io.gitpod.ide.version"]'`.text().catch((e) => {
console.log("Fetching IDE version in image:", `oci-tool fetch image execInstaller${img} | jq -r '.config.Labels["io.gitpod.ide.version"]'`)
const version = await $`oci-tool fetch image execInstaller${img} | jq -r '.config.Labels["io.gitpod.ide.version"]'`.text().catch((e) => {
throw new Error("Failed to fetch ide version in image", e);
}).then(str => str.replaceAll("\n", ""));
console.log("IDE version in image:", version);
return version;
}
20 changes: 11 additions & 9 deletions components/ide/gha-update-image/lib/jb-pin-version.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import {
readIDEConfigmapJson,
readWorkspaceYaml,
IWorkspaceYaml,
getIDEVersionOfImage,
renderInstallerIDEConfigMap,
} from "./common";

const configmap = await readIDEConfigmapJson().then((d) => d.rawObj);
Expand Down Expand Up @@ -41,16 +43,16 @@ export const appendPinVersionsIntoIDEConfigMap = async (updatedIDEs: string[] |

console.debug(`Processing ${ide} ${ideVersion}...`);

const ideConfigMap = await renderInstallerIDEConfigMap(undefined)

if (Object.keys(configmap.ideOptions.options).includes(ide)) {
const { version: installerImageVersion } = versionObject;

const installerIdeVersion = {
version: ideVersion,
image: `{{.Repository}}/ide/${ide}:${installerImageVersion}`,
imageLayers: [
`{{.Repository}}/ide/jb-backend-plugin:${latestInstallerVersions.components.workspace.desktopIdeImages.jbBackendPlugin.version}`,
`{{.Repository}}/ide/jb-launcher:${latestInstallerVersions.components.workspace.desktopIdeImages.jbLauncher.version}`,
],
const previousVersion = await getIDEVersionOfImage(ideConfigMap.ideOptions.options[ide].image);
const previousInfo = {
version: previousVersion,
image: ideConfigMap.ideOptions.options[ide].image,
imageLayers: ideConfigMap.ideOptions.options[ide].imageLayers,
};

if (!updatedIDEs || !updatedIDEs.includes(ide)) {
Expand All @@ -60,8 +62,8 @@ export const appendPinVersionsIntoIDEConfigMap = async (updatedIDEs: string[] |
if (!configmap.ideOptions.options[ide].versions) {
configmap.ideOptions.options[ide].versions = []
}
console.log(`Added ${ide} (new ${installerIdeVersion.image})`);
configmap.ideOptions.options[ide].versions.unshift(installerIdeVersion);
console.log(`Added ${ide} (new ${previousInfo.image})`);
configmap.ideOptions.options[ide].versions.unshift(previousInfo);
}
}

Expand Down
83 changes: 83 additions & 0 deletions install/installer/cmd/ideconfigmap.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// Copyright (c) 2024 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL).
// See License.AGPL.txt in the project root for license information.

package cmd

import (
"fmt"
"os"
"path/filepath"

"github.com/gitpod-io/gitpod/common-go/log"
"github.com/gitpod-io/gitpod/installer/pkg/common"
ide_service "github.com/gitpod-io/gitpod/installer/pkg/components/ide-service"

"github.com/spf13/cobra"
)

var ideOpts struct {
ConfigFN string
Namespace string
UseExperimentalConfig bool
}

// ideConfigmapCmd generates ide-configmap.json
var ideConfigmapCmd = &cobra.Command{
Use: "ide-configmap",
Hidden: true,
Short: "render ide-configmap.json",
RunE: func(cmd *cobra.Command, args []string) error {
renderCtx, err := getRenderCtx(ideOpts.ConfigFN, ideOpts.Namespace, ideOpts.UseExperimentalConfig)
if err != nil {
return err
}

ideConfig, err := ide_service.GenerateIDEConfigmap(renderCtx)
if err != nil {
return err
}

fc, err := common.ToJSONString(ideConfig)
if err != nil {
return fmt.Errorf("failed to marshal ide-config config: %w", err)
}

fmt.Println(string(fc))
return nil
},
}

func getRenderCtx(configFN, namespace string, useExperimentalConfig bool) (*common.RenderContext, error) {
_, _, cfg, err := loadConfig(configFN)
if err != nil {
return nil, err
}

if cfg.Experimental != nil {
if useExperimentalConfig {
fmt.Fprintf(os.Stderr, "rendering using experimental config\n")
} else {
fmt.Fprintf(os.Stderr, "ignoring experimental config. Use `--use-experimental-config` to include the experimental section in config\n")
cfg.Experimental = nil
}
}
versionMF, err := getVersionManifest()
if err != nil {
return nil, err
}
return common.NewRenderContext(*cfg, *versionMF, namespace)
}

func init() {
rootCmd.AddCommand(ideConfigmapCmd)

dir, err := os.Getwd()
if err != nil {
log.WithError(err).Fatal("Failed to get working directory")
}

ideConfigmapCmd.PersistentFlags().StringVarP(&ideOpts.ConfigFN, "config", "c", getEnvvar("GITPOD_INSTALLER_CONFIG", filepath.Join(dir, "gitpod.config.yaml")), "path to the config file, use - for stdin")
ideConfigmapCmd.PersistentFlags().StringVarP(&ideOpts.Namespace, "namespace", "n", getEnvvar("NAMESPACE", "default"), "namespace to deploy to")
ideConfigmapCmd.Flags().BoolVar(&ideOpts.UseExperimentalConfig, "use-experimental-config", false, "enable the use of experimental config that is prone to be changed")
}
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ import (
//go:embed ide-configmap.json
var ideConfigFile string

func ideConfigConfigmap(ctx *common.RenderContext) ([]runtime.Object, error) {
func GenerateIDEConfigmap(ctx *common.RenderContext) (*ide_config.IDEConfig, error) {
resolveLatestImage := func(name string, tag string, bundledLatest versions.Versioned) string {
resolveLatest := true
if ctx.Config.Components != nil && ctx.Config.Components.IDE != nil && ctx.Config.Components.IDE.ResolveLatest != nil {
Expand Down Expand Up @@ -119,7 +119,14 @@ func ideConfigConfigmap(ctx *common.RenderContext) ([]runtime.Object, error) {
if idecfg.IdeOptions.Options[idecfg.IdeOptions.DefaultDesktopIde].Type != ide_config.IDETypeDesktop {
return nil, fmt.Errorf("default desktop IDE '%s' does not point to a desktop IDE option", idecfg.IdeOptions.DefaultIde)
}
return &idecfg, nil
}

func ideConfigConfigmap(ctx *common.RenderContext) ([]runtime.Object, error) {
idecfg, err := GenerateIDEConfigmap(ctx)
if err != nil {
return nil, fmt.Errorf("failed to generate ide-config config: %w", err)
}
fc, err := common.ToJSONString(idecfg)
if err != nil {
return nil, fmt.Errorf("failed to marshal ide-config config: %w", err)
Expand Down
6 changes: 1 addition & 5 deletions test/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -146,11 +146,7 @@ else

cd "${TEST_PATH}"
set +e
if [ "$TEST_SUITE" == "jetbrains" ]; then
go test -parallel=3 -v ./... "${args[@]}" 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "${RESULTS_DIR}/TEST-${TEST_NAME}.xml" -iocopy
else
go test -v ./... "${args[@]}" 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "${RESULTS_DIR}/TEST-${TEST_NAME}.xml" -iocopy
fi
go test -parallel=3 -v ./... "${args[@]}" 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "${RESULTS_DIR}/TEST-${TEST_NAME}.xml" -iocopy
RC=${PIPESTATUS[0]}
set -e
cd -
Expand Down

0 comments on commit 9380449

Please sign in to comment.