Build

Add default image get to PAPI and improve Dashboard #17417

Workflow file for this run

	name: Build
	on:
	pull_request:
	types: [ opened, edited ]
	push:
	workflow_dispatch:
	inputs:
	recreate_vm:
	required: true
	type: string
	description: "Whether to recreate the VM"
	default: "false"

	jobs:
	create-runner:
	uses: gitpod-io/gce-github-runner/.github/workflows/create-vm.yml@main
	secrets:
	runner_token: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_TOKEN }}
	gcp_credentials: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_GCP_CREDENTIALS }}
	concurrency:
	group: ${{ github.ref == 'refs/heads/main' && github.run_id \|\| github.sha }}-create-runner
	cancel-in-progress: false

	configuration:
	name: Configure job parameters
	runs-on: ${{ needs.create-runner.outputs.label }}
	needs: [ create-runner ]
	concurrency:
	group: ${{ github.ref == 'refs/heads/main' && github.run_id \|\| github.sha }}-configuration
	cancel-in-progress: true
	outputs:
	is_main_branch: ${{ (github.head_ref \|\| github.ref) == 'refs/heads/main' }}
	version: ${{ steps.branches.outputs.sanitized-branch-name }}-gha.${{github.run_number}}
	preview_enable: ${{ contains( steps.pr-details.outputs.pr_body, '[x] /werft with-preview') \|\| (steps.output.outputs.with_integration_tests != '') }}
	preview_name: ${{ github.head_ref \|\| github.ref_name }}
	preview_infra_provider: ${{ contains( steps.pr-details.outputs.pr_body, '[x] /werft with-gce-vm') && 'gce' \|\| 'harvester' }}
	build_no_cache: ${{ contains( steps.pr-details.outputs.pr_body, '[x] leeway-no-cache') }}
	build_no_test: ${{ contains( steps.pr-details.outputs.pr_body, '[x] /werft no-test') }}
	with_large_vm: ${{ contains( steps.pr-details.outputs.pr_body, '[X] /werft with-large-vm') \|\| (steps.output.outputs.with_integration_tests != '') }}
	publish_to_npm: ${{ contains( steps.pr-details.outputs.pr_body, '[X] /werft publish-to-npm') }}
	publish_to_jbmp: ${{ contains( steps.pr-details.outputs.pr_body, '[X] /werft publish-to-jb-marketplace') }}
	with_dedicated_emulation: ${{ contains( steps.pr-details.outputs.pr_body, '[X] with-dedicated-emulation') }}
	analytics: ${{ steps.output.outputs.analytics }}
	workspace_feature_flags: ${{ steps.output.outputs.workspace_feature_flags }}
	pr_no_diff_skip: ${{ steps.pr-diff.outputs.pr_no_diff_skip }}
	with_werft: ${{ steps.output.outputs.with-werft }}
	with_integration_tests: ${{ steps.output.outputs.with_integration_tests }}
	with_monitoring: ${{ contains( steps.pr-details.outputs.pr_body, '[x] with-monitoring') }}
	latest_ide_version: ${{ contains( steps.pr-details.outputs.pr_body, '[x] latest-ide-version=true') }}
	leeway_cache_bucket: ${{ steps.output.outputs.leeway_cache_bucket }}
	pr_number: ${{ steps.pr-details.outputs.number }}
	pr_body: ${{ steps.pr-details.outputs.pr_body }}
	steps:
	- name: "Determine Branch"
	id: branches
	uses: transferwise/sanitize-branch-name@v1
	# Since we trigger this worklow on other event types, besides pull_request
	# We use this action to help us get the pr body, as it's not included in push/workflow_dispatch events
	- uses: 8BitJonny/[email protected]
	id: pr-details
	with:
	sha: ${{ github.event.pull_request.head.sha }}
	# If the PR got edited, but no checkbox changes occurred, we don't want to run the whole build
	# Therefore we set a flag and use it to skip the next jobs
	- name: "Check PR diff"
	id: pr-diff
	if: (github.event_name == 'pull_request' && github.event.action == 'edited')
	env:
	PR_DESC: "${{ steps.pr-details.outputs.pr_body }}"
	OLD_BODY: "${{ github.event.changes.body.from }}"
	run: \|
	if ! diff <(echo "$OLD_BODY") <(echo "$PR_DESC") \| grep -e '\[x\]' -e '\[X\]'; then
	echo "pr_no_diff_skip=true" >> $GITHUB_OUTPUT
	fi
	- name: "Set outputs"
	id: output
	env:
	PR_DESC: "${{ steps.pr-details.outputs.pr_body }}"
	MAIN_BRANCH: ${{ (github.head_ref \|\| github.ref) == 'refs/heads/main' }}
	shell: bash
	run: \|
	{
	echo "workspace_feature_flags=$(echo "$PR_DESC" \| grep -oiP '(?<=\[x\] workspace-feature-flags=).?(?=\s$)')"
	echo "with_integration_tests=$(echo "$PR_DESC" \| grep -oiP '(?<=\[x\] with-integration-tests=).?(?=\s$)')"
	echo "analytics=$(echo "$PR_DESC" \| grep -oiP '(?<=\[x\] analytics=).?(?=\s$)')"
	echo "leeway_cache_bucket=$([[ "$MAIN_BRANCH" = "true" ]] && echo "gitpod-core-leeway-cache-main" \|\| echo "gitpod-core-leeway-cache-branch")"
	} >> $GITHUB_OUTPUT

	build-previewctl:
	name: Build previewctl
	if: \|
	(needs.configuration.outputs.pr_no_diff_skip != 'true') &&
	(needs.configuration.outputs.preview_enable == 'true')
	needs: [ configuration, create-runner ]
	concurrency:
	group: ${{ github.ref == 'refs/heads/main' && github.run_id \|\| github.sha }}-build-previewctl
	cancel-in-progress: ${{ needs.configuration.outputs.is_main_branch == 'false' }}
	runs-on: ${{ needs.create-runner.outputs.label }}
	container:
	image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:at-node-18-17-1-gha.15610
	outputs:
	previewctl_hash: ${{ steps.build.outputs.previewctl_hash }}
	steps:
	- uses: actions/checkout@v3
	- uses: ./.github/actions/setup-environment
	with:
	sa_key: ${{ secrets.GCP_CREDENTIALS }}
	leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }}
	- name: Build previewctl
	id: build
	shell: bash
	run: \|
	leeway build dev/preview/previewctl:docker -Dversion="${{needs.configuration.outputs.version}}"
	echo "previewctl_hash=$(leeway describe dev/preview/previewctl:docker -Dversion="${{needs.configuration.outputs.version}}" -t '{{ .Metadata.Version }}')" >> $GITHUB_OUTPUT

	infrastructure:
	needs: [ configuration, build-previewctl, create-runner ]
	if: \|
	(needs.configuration.outputs.pr_no_diff_skip != 'true') &&
	(needs.configuration.outputs.preview_enable == 'true') &&
	(needs.configuration.outputs.is_main_branch != 'true')
	runs-on: ${{ needs.create-runner.outputs.label }}
	concurrency:
	group: ${{ github.ref == 'refs/heads/main' && github.run_id \|\| github.sha }}-infrastructure
	cancel-in-progress: true
	steps:
	- uses: actions/checkout@v3
	- name: Create preview environment infrastructure
	id: create
	uses: ./.github/actions/preview-create
	with:
	name: ${{ needs.configuration.outputs.preview_name }}
	sa_key: ${{ secrets.GCP_CREDENTIALS }}
	infrastructure_provider: ${{ needs.configuration.outputs.preview_infra_provider }}
	previewctl_hash: ${{ needs.build-previewctl.outputs.previewctl_hash }}
	large_vm: ${{ needs.configuration.outputs.with_large_vm }}
	recreate_vm: ${{ inputs.recreate_vm }}

	build-gitpod:
	name: Build Gitpod
	needs: [ configuration, create-runner ]
	runs-on: ${{ needs.create-runner.outputs.label }}
	concurrency:
	group: ${{ github.ref == 'refs/heads/main' && github.run_id \|\| github.sha }}-build-gitpod
	cancel-in-progress: ${{ needs.configuration.outputs.is_main_branch == 'false' }}
	services:
	mysql:
	image: bitnami/mysql:8.0.33-debian-11-r24
	env:
	MYSQL_ROOT_PASSWORD: test
	#MYSQL_TCP_PORT: 23306 bitnami/mysql does not honor this, but has it's own:
	MYSQL_PORT_NUMBER: 23306
	MYSQL_AUTHENTICATION_PLUGIN: mysql_native_password
	ports:
	- 23306:23306
	redis:
	image: redis
	ports:
	- 6379:6379
	container:
	image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:at-node-18-17-1-gha.15610
	env:
	DB_HOST: "mysql"
	DB_PORT: "23306"
	REDIS_HOST: "redis"
	# GitHub action + MySQL 8.0 need longer to initialize
	DB_RETRIES: 5
	steps:
	- uses: actions/checkout@v3
	- uses: ./.github/actions/setup-environment
	with:
	sa_key: ${{ secrets.GCP_CREDENTIALS }}
	leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }}
	- name: Leeway Vet
	shell: bash
	run: \|
	leeway vet --ignore-warnings
	- name: Pre-Commit Checks
	shell: bash
	run: \|
	RESULT=0
	pre-commit run --show-diff-on-failure \|\| RESULT=$?
	if [ $RESULT -ne 0 ]; then
	echo "::error title=The Pre-Commit Checks Failed.::Please run 'pre-commit run --show-diff-on-failure'"
	fi
	exit "$RESULT"
	- name: Get Secrets from GCP
	id: "secrets"
	uses: "google-github-actions/get-secretmanager-secrets@v1"
	with:
	secrets: \|-
	segment-io-token:gitpod-core-dev/segment-io-token
	npm-auth-token:gitpod-core-dev/npm-auth-token
	jb-marketplace-publish-token:gitpod-core-dev/jb-marketplace-publish-token
	codecov-token:gitpod-core-dev/codecov
	- name: Dev Build
	id: dev-build
	env:
	JAVA_HOME: /home/gitpod/.sdkman/candidates/java/current
	VERSION: ${{needs.configuration.outputs.version}}
	shell: bash
	run: \|
	RESULT=0
	set -x

	leeway build dev:all \
	--docker-build-options network=host \
	--cache remote \
	-Dversion=$VERSION \
	-DlocalAppVersion=$VERSION \
	-DimageRepoBase=eu.gcr.io/gitpod-core-dev/dev \
	--report large-report.html \|\| RESULT=$?

	set +x

	# capture logs for the dev/image build, they're too big for GITHUB_STEP_SUMMARY
	pup -p -f large-report.html '[id="dev/image:docker"] + details + details' > remove.txt

	# remove logs for the dev/image build output from the report file
	grep -vxFf remove.txt large-report.html > report.html

	cat report.html >> $GITHUB_STEP_SUMMARY

	exit $RESULT
	- name: Leeway Build
	id: leeway
	shell: bash
	env:
	NODE_OPTIONS: "--max_old_space_size=4096"
	JAVA_HOME: /home/gitpod/.sdkman/candidates/java/current
	VERSION: ${{needs.configuration.outputs.version}}
	SEGMENT_IO_TOKEN: "${{ steps.secrets.outputs.segment-io-token }}"
	PR_NO_CACHE: ${{needs.configuration.outputs.build_no_cache}}
	PR_NO_TEST: ${{needs.configuration.outputs.build_no_test}}
	NPM_AUTH_TOKEN: "${{ steps.secrets.outputs.npm-auth-token }}"
	PUBLISH_TO_NPM: ${{ needs.configuration.outputs.publish_to_npm == 'true' \|\| needs.configuration.outputs.is_main_branch == 'true' }}
	JB_MARKETPLACE_PUBLISH_TOKEN: "${{ steps.secrets.outputs.jb-marketplace-publish-token }}"
	PUBLISH_TO_JBPM: ${{ needs.configuration.outputs.publish_to_jbmp == 'true' \|\| needs.configuration.outputs.is_main_branch == 'true' }}
	CODECOV_TOKEN: "${{ steps.secrets.outputs.codecov-token }}"
	run: \|
	[[ "$PR_NO_CACHE" = "true" ]] && CACHE="none" \|\| CACHE="remote"
	[[ "$PR_NO_TEST" = "true" ]] && TEST="--dont-test" \|\| TEST=""
	[[ "${PUBLISH_TO_NPM}" = 'true' ]] && NPM_PUBLISH_TRIGGER=$(date +%s%3N) \|\| NPM_PUBLISH_TRIGGER="false"

	RESULT=0
	set -x
	# CI=true is a var set by GHA. Unsetting it for the build, as yarn builds treat warnings as errors if that var is set to true
	CI= leeway build \
	--cache $CACHE \
	$TEST \
	-Dversion=$VERSION \
	--docker-build-options network=host \
	--max-concurrent-tasks 1 \
	-DlocalAppVersion=$VERSION \
	-DSEGMENT_IO_TOKEN=$SEGMENT_IO_TOKEN \
	-DpublishToNPM="${PUBLISH_TO_NPM}" \
	-DnpmPublishTrigger="${NPM_PUBLISH_TRIGGER}" \
	-DpublishToJBMarketplace="${PUBLISH_TO_JBPM}" \
	-DimageRepoBase=eu.gcr.io/gitpod-core-dev/build \
	--report report.html \|\| RESULT=$?
	set +x

	cat report.html >> $GITHUB_STEP_SUMMARY

	SUMMARY_SIZE=$(wc -c < $GITHUB_STEP_SUMMARY)
	if [[ $SUMMARY_SIZE -gt 1000000 ]]; then
	head -c 1000000 $GITHUB_STEP_SUMMARY > $GITHUB_STEP_SUMMARY.tmp
	mv $GITHUB_STEP_SUMMARY.tmp $GITHUB_STEP_SUMMARY
	(
	echo '.....'
	echo ''
	echo ':x: WARNING: Summary is too large and has been truncated.'
	echo ''
	) >> $GITHUB_STEP_SUMMARY
	fi

	exit $RESULT
	- name: Tag the release
	if: github.ref == 'refs/heads/main'
	run: \|
	git config --global user.name $GITHUB_USER
	git config --global user.email $GITHUB_EMAIL
	git tag -a ${VERSION} -m "Release ${VERSION}"
	git push origin ${VERSION}
	env:
	GITHUB_USER: roboquat
	GITHUB_EMAIL: [email protected]
	VERSION: ${{ needs.configuration.outputs.version }}

	install-app:
	runs-on: ${{ needs.create-runner.outputs.label }}
	needs: [ configuration, build-gitpod, create-runner ]
	if: ${{ needs.configuration.outputs.is_main_branch == 'true' }}
	strategy:
	fail-fast: false
	matrix:
	env: [ staging, production ]
	steps:
	- uses: gitpod-io/[email protected]
	id: auth
	with:
	private-key: ${{ secrets.ACTIONS_APP_PKEY }}
	app-id: 308947
	installation-id: 35574470
	- name: trigger installation
	uses: actions/github-script@v6
	with:
	github-token: ${{ steps.auth.outputs.token }}
	script: \|
	const result = await github.rest.actions.createWorkflowDispatch({
	owner: context.repo.owner,
	repo: 'gitpod-dedicated',
	workflow_id: 'install-app.yaml',
	ref: 'main',
	inputs: {
	"version": '${{ needs.configuration.outputs.version }}',
	"repo": "https://github.com/gitpod-io/gitpod",
	"commit": "${{github.sha}}",
	"release": "${{ matrix.env }}",
	"env": "${{ matrix.env }}"
	}
	})

	install:
	name: "Install Gitpod"
	needs:
	- configuration
	- build-previewctl
	- build-gitpod
	- infrastructure
	- create-runner
	runs-on: ${{ needs.create-runner.outputs.label }}
	concurrency:
	group: ${{ github.ref == 'refs/heads/main' && github.run_id \|\| github.sha }}-install
	cancel-in-progress: ${{ needs.configuration.outputs.is_main_branch == 'false' }}
	steps:
	- uses: actions/checkout@v3
	- name: Deploy Gitpod to the preview environment
	id: deploy-gitpod
	uses: ./.github/actions/deploy-gitpod
	with:
	name: ${{ needs.configuration.outputs.preview_name }}
	version: ${{needs.configuration.outputs.version}}
	sa_key: ${{ secrets.GCP_CREDENTIALS }}
	previewctl_hash: ${{ needs.build-previewctl.outputs.previewctl_hash }}
	with_dedicated_emu: ${{needs.configuration.outputs.with_dedicated_emulation}}
	analytics: ${{needs.configuration.outputs.analytics}}
	workspace_feature_flags: ${{needs.configuration.outputs.workspace_feature_flags}}
	- uses: actions/github-script@v6
	if: needs.configuration.outputs.pr_number != '' && contains(needs.configuration.outputs.pr_body, 'gitpod:summary')
	with:
	script: \|
	const prNumber = ${{ needs.configuration.outputs.pr_number }};

	const summaryEncoded = `${{ steps.deploy-gitpod.outputs.report }}`;
	const summary = Buffer.from(summaryEncoded, 'base64').toString('utf8');

	const { data: pr } = await github.rest.pulls.get({
	...context.repo,
	pull_number: prNumber,
	});

	const prBody = pr.body;

	const newBody = prBody.replace('gitpod:summary', summary);

	await github.rest.pulls.update({
	...context.repo,
	pull_number: prNumber,
	body: newBody,
	});

	monitoring:
	name: "Install Monitoring Satellite"
	needs: [ infrastructure, build-previewctl, create-runner ]
	runs-on: ${{ needs.create-runner.outputs.label }}
	if: needs.configuration.outputs.with_monitoring == 'true'
	concurrency:
	group: ${{ github.ref == 'refs/heads/main' && github.run_id \|\| github.sha }}-monitoring
	cancel-in-progress: true
	steps:
	- uses: actions/checkout@v3
	- name: Deploy monitoring satellite to the preview environment
	id: deploy-monitoring-satellite
	uses: ./.github/actions/deploy-monitoring-satellite
	with:
	sa_key: ${{ secrets.GCP_CREDENTIALS }}
	previewctl_hash: ${{ needs.build-previewctl.outputs.previewctl_hash }}

	integration-test:
	name: "Run integration test"
	needs:
	- configuration
	- build-previewctl
	- build-gitpod
	- infrastructure
	- install
	- create-runner
	runs-on: ${{ needs.create-runner.outputs.label }}
	container:
	image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:at-node-18-17-1-gha.15610
	if: needs.configuration.outputs.with_integration_tests != ''
	concurrency:
	group: ${{ needs.configuration.outputs.preview_name }}-integration-test
	cancel-in-progress: true
	steps:
	- uses: actions/checkout@v3
	- uses: ./.github/actions/setup-environment
	with:
	sa_key: ${{ secrets.GCP_CREDENTIALS }}
	leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }}

	- name: Run integration test
	id: integration-test
	uses: ./.github/actions/integration-tests
	with:
	preview_name: ${{ needs.configuration.outputs.preview_name }}
	test_suite: ${{ needs.configuration.outputs.with_integration_tests }}
	notify_slack_webhook: ''
	sa_key: ${{ secrets.GCP_CREDENTIALS }}
	github_token: ${{ secrets.GITHUB_TOKEN }}
	latest_ide_version: ${{ needs.configuration.outputs.latest_ide_version }}
	test_build_id: ${{ github.run_id }}
	test_build_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
	test_build_ref: ${{ github.head_ref \|\| github.ref }}

	workspace-integration-tests-main:
	name: "Run workspace integration tests on main branch"
	needs:
	- configuration
	- build-gitpod
	- create-runner
	if: needs.configuration.outputs.is_main_branch == 'true'
	uses: ./.github/workflows/workspace-integration-tests.yml
	with:
	version: ${{ needs.configuration.outputs.version }}
	secrets: inherit

	delete-runner:
	if: always()
	needs:
	- create-runner
	- configuration
	- build-previewctl
	- infrastructure
	- build-gitpod
	- install-app
	- install
	- monitoring
	- integration-test
	uses: gitpod-io/gce-github-runner/.github/workflows/delete-vm.yml@main
	secrets:
	gcp_credentials: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_GCP_CREDENTIALS }}
	with:
	runner-label: ${{ needs.create-runner.outputs.label }}
	machine-zone: ${{ needs.create-runner.outputs.machine-zone }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add default image get to PAPI and improve Dashboard #17417

Workflow file

Add default image get to PAPI and improve Dashboard #17417

Jobs

Run details

Workflow file for this run