From 0ef0df19277fbc2246c67471b0250bbf5494336b Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Tue, 19 Mar 2024 14:01:09 -0400 Subject: [PATCH] ci: build and publish development images on commit to main --- .drone/drone.yml | 45 +++++++++++++++++++++++- .drone/pipelines/publish.jsonnet | 60 +++++++++++++++++++++++++++++++- tools/ci/docker-containers | 24 +++++++++---- 3 files changed, 120 insertions(+), 9 deletions(-) diff --git a/.drone/drone.yml b/.drone/drone.yml index fae58d3202..59fc5df03c 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -303,6 +303,49 @@ trigger: type: docker --- kind: pipeline +name: Publish development Linux agent container +platform: + arch: amd64 + os: linux +steps: +- commands: + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + failure: ignore + image: grafana/agent-build-image:0.40.2 + name: Configure QEMU + volumes: + - name: docker + path: /var/run/docker.sock +- commands: + - mkdir -p $HOME/.docker + - printenv GCR_CREDS > $HOME/.docker/config.json + - docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD + - docker buildx create --name multiarch-agent-agent-${DRONE_COMMIT_SHA} --driver + docker-container --use + - DEVELOPMENT=1 ./tools/ci/docker-containers agent + - docker buildx rm multiarch-agent-agent-${DRONE_COMMIT_SHA} + environment: + DOCKER_LOGIN: + from_secret: docker_login + DOCKER_PASSWORD: + from_secret: docker_password + GCR_CREDS: + from_secret: gcr_admin + image: grafana/agent-build-image:0.40.2 + name: Publish container + volumes: + - name: docker + path: /var/run/docker.sock +trigger: + ref: + - refs/heads/main +type: docker +volumes: +- host: + path: /var/run/docker.sock + name: docker +--- +kind: pipeline name: Test Linux system packages platform: arch: amd64 @@ -407,6 +450,6 @@ kind: secret name: updater_private_key --- kind: signature -hmac: 59c741cd4e3cd3f555cbf0165da386b269a7f54987fe5a2aba621edc6ebb09a5 +hmac: eb4c87d4abc880513c7c2977c46910fa96041461aa2edea16a7970f5c145dd01 ... diff --git a/.drone/pipelines/publish.jsonnet b/.drone/pipelines/publish.jsonnet index e3117042a3..7501017ac3 100644 --- a/.drone/pipelines/publish.jsonnet +++ b/.drone/pipelines/publish.jsonnet @@ -7,6 +7,61 @@ local ghTokenFilename = '/drone/src/gh-token.txt'; local job_names = function(jobs) std.map(function(job) job.name, jobs); local linux_containers = ['agent', 'agent-boringcrypto']; +local dev_linux_containers = ['agent']; // TODO(rfratto): add boringcrypto after figuring out what to do with it + +local linux_containers_dev_jobs = std.map(function(container) ( + pipelines.linux('Publish development Linux %s container' % container) { + trigger: { + ref: [ + 'refs/heads/main', + ], + }, + steps: [{ + // We only need to run this once per machine, so it's OK if it fails. It + // is also likely to fail when run in parallel on the same machine. + name: 'Configure QEMU', + image: build_image.linux, + failure: 'ignore', + volumes: [{ + name: 'docker', + path: '/var/run/docker.sock', + }], + commands: [ + 'docker run --rm --privileged multiarch/qemu-user-static --reset -p yes', + ], + }, { + name: 'Publish container', + image: build_image.linux, + volumes: [{ + name: 'docker', + path: '/var/run/docker.sock', + }], + environment: { + DOCKER_LOGIN: secrets.docker_login.fromSecret, + DOCKER_PASSWORD: secrets.docker_password.fromSecret, + GCR_CREDS: secrets.gcr_admin.fromSecret, + }, + commands: [ + 'mkdir -p $HOME/.docker', + 'printenv GCR_CREDS > $HOME/.docker/config.json', + 'docker login -u $DOCKER_LOGIN -p $DOCKER_PASSWORD', + + // Create a buildx worker for our cross platform builds. + 'docker buildx create --name multiarch-agent-%s-${DRONE_COMMIT_SHA} --driver docker-container --use' % container, + + 'DEVELOPMENT=1 ./tools/ci/docker-containers %s' % container, + + 'docker buildx rm multiarch-agent-%s-${DRONE_COMMIT_SHA}' % container, + ], + }], + volumes: [{ + name: 'docker', + host: { path: '/var/run/docker.sock' }, + }], + } +), dev_linux_containers); + + local linux_containers_jobs = std.map(function(container) ( pipelines.linux('Publish Linux %s container' % container) { trigger: { @@ -94,7 +149,10 @@ local windows_containers_jobs = std.map(function(container) ( // TODO(rfratto): The following are TEMPORARILY disabled as grafana/alloy gets // set up. Remove the line below in favor of the comment block to reenable the // publish jobs. -[] +// +// This file must be refactored in the future after development has fully +// shifted. +linux_containers_dev_jobs /* linux_containers_jobs + windows_containers_jobs + [ diff --git a/tools/ci/docker-containers b/tools/ci/docker-containers index e81de10834..76f5c1d19f 100755 --- a/tools/ci/docker-containers +++ b/tools/ci/docker-containers @@ -7,6 +7,14 @@ # from a Drone trigger. set -euxo pipefail +RELEASE_AGENT_IMAGE=grafana/agent +RELEASE_AGENTBORINGCRYPTO_IMAGE=grafana/agent-boringcrypto +DEVELOPMENT_AGENT_IMAGE=us-docker.pkg.dev/grafanalabs-dev/docker-alloy-dev +DEVELOPMENT_AGENTBORINGCRYPTO_IMAGE=us-docker.pkg.dev/grafanalabs-dev/docker-alloy-boringcrypto-dev + +DEFAULT_AGENT_IMAGE=${RELEASE_AGENT_IMAGE} +DEFAULT_AGENTBORINGCRYPTO_IMAGE=${RELEASE_AGENTBORINGCRYPTO_IMAGE} + # Environment variables used throughout this script. These must be set # otherwise bash will fail with an "unbound variable" error because of the `set # -u` flag on the above line. @@ -15,10 +23,15 @@ set -euxo pipefail # empty string. export TARGET_CONTAINER=${1:-} export DRONE_TAG=${DRONE_TAG:-} -export DRONE_BRANCH=${DRONE_BRANCH:-} +export DEVELOPMENT=${DEVELOPMENT:-} + +if [ -n "$DEVELOPMENT" ]; then + DEFAULT_AGENT_IMAGE=${DEVELOPMENT_AGENT_IMAGE} + DEFAULT_AGENTBORINGCRYPTO_IMAGE=${DEVELOPMENT_AGENTBORINGCRYPTO_IMAGE} +fi -export AGENT_IMAGE=grafana/agent -export AGENT_BORINGCRYPTO_IMAGE=grafana/agent-boringcrypto +export AGENT_IMAGE=${DEFAULT_AGENT_IMAGE} +export AGENT_BORINGCRYPTO_IMAGE=${DEFAULT_AGENTBORINGCRYPTO_IMAGE} # We need to determine what version to assign to built binaries. If containers # are being built from a Drone tag trigger, we force the version to come from the @@ -33,16 +46,13 @@ fi # We also need to know which "branch tag" to update. Branch tags are used as a # secondary tag for Docker containers. The branch tag is "latest" when being -# tagged from a stable release (i.e., not a release candidate) or the Drone -# branch when coming from a Drone job. +# tagged from a stable release (i.e., not a release candidate). # # If we're not running from drone, we'll set the branch tag to match the # version. This effectively acts as a no-op because it will tag the same Docker # image twice. if [ -n "$DRONE_TAG" ] && [[ "$DRONE_TAG" != *"-rc."* ]]; then BRANCH_TAG=latest -elif [ -n "$DRONE_BRANCH" ]; then - BRANCH_TAG=$DRONE_BRANCH else BRANCH_TAG=$VERSION fi