From c152a6614a0d8143bc0f906526ebe8b2afcce7e3 Mon Sep 17 00:00:00 2001 From: Sylvain <35365065+sanderegg@users.noreply.github.com> Date: Tue, 1 Sep 2020 10:32:46 +0200 Subject: [PATCH] Simplify release process (#1748) use tagging instead of staging branch --- .github/workflows/ci-hotfix.yml | 36 ++ .github/workflows/ci-release.yml | 7 +- .github/workflows/ci-staging.yml | 35 ++ .github/workflows/ci-testing-deploy.yml | 27 +- .travis.yml | 17 +- Makefile | 41 ++ ci/README.md | 19 +- ci/deploy/dockerhub-deploy.bash | 27 +- ci/deploy/dockerhub-release.bash | 38 -- ci/deploy/dockerhub-tag-version.bash | 49 +++ .../find_docker_image_tag_from_git_sha.bash | 37 ++ ci/helpers/find_staging_version.bash | 33 -- ci/helpers/install_pylint.bash | 0 docs/img/git-hotfix-workflow.svg | 1 + docs/img/git-release-workflow.md | 110 +++--- docs/img/git-release-workflow.svg | 361 +----------------- docs/production-instructions.md | 17 - docs/releasing-workflow-instructions.md | 148 +++++++ docs/staging-instructions.md | 28 -- scripts/helpers/logger.bash | 32 ++ scripts/url-encoder.bash | 5 + 21 files changed, 497 insertions(+), 571 deletions(-) create mode 100644 .github/workflows/ci-hotfix.yml create mode 100644 .github/workflows/ci-staging.yml delete mode 100755 ci/deploy/dockerhub-release.bash create mode 100755 ci/deploy/dockerhub-tag-version.bash create mode 100755 ci/helpers/find_docker_image_tag_from_git_sha.bash delete mode 100755 ci/helpers/find_staging_version.bash mode change 100644 => 100755 ci/helpers/install_pylint.bash create mode 100644 docs/img/git-hotfix-workflow.svg delete mode 100644 docs/production-instructions.md create mode 100644 docs/releasing-workflow-instructions.md delete mode 100644 docs/staging-instructions.md create mode 100644 scripts/helpers/logger.bash diff --git a/.github/workflows/ci-hotfix.yml b/.github/workflows/ci-hotfix.yml new file mode 100644 index 00000000000..a594ef3004d --- /dev/null +++ b/.github/workflows/ci-hotfix.yml @@ -0,0 +1,36 @@ +name: Github-CI-Hotfix-Release + +on: + push: + tags: + - v[0-9]+.[0-9]+.[0-9]+ + +env: + # secrets can be set in settings/secrets on github + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + +jobs: + deploy: + if: contains(github.event.base_ref, 'refs/heads/hotfix_v') + # only run on staging branch + name: deploy release + runs-on: ubuntu-latest + env: + FROM_TAG_PREFIX: hotfix-github + TO_TAG_PREFIX: release-github + steps: + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 + - name: set owner variable + run: echo ::set-env name=OWNER::${GITHUB_REPOSITORY%/*} + - name: set git tag + run: echo ::set-env name=GIT_TAG::${GITHUB_REF##*/} + - name: deploy + run: ./ci/deploy/dockerhub-tag-version.bash diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml index aec9103a72f..e66936ca41b 100644 --- a/.github/workflows/ci-release.yml +++ b/.github/workflows/ci-release.yml @@ -13,12 +13,13 @@ env: jobs: deploy: + if: github.event.base_ref == 'refs/heads/master' # only run on staging branch - if: github.event.base_ref == 'refs/heads/staging' name: deploy release runs-on: ubuntu-latest env: - TAG_PREFIX: staging-github + FROM_TAG_PREFIX: staging-github + TO_TAG_PREFIX: release-github steps: - uses: actions/checkout@v2 - name: setup docker @@ -32,4 +33,4 @@ jobs: - name: set git tag run: echo ::set-env name=GIT_TAG::${GITHUB_REF##*/} - name: deploy - run: ./ci/deploy/dockerhub-release.bash + run: ./ci/deploy/dockerhub-tag-version.bash diff --git a/.github/workflows/ci-staging.yml b/.github/workflows/ci-staging.yml new file mode 100644 index 00000000000..82626103db1 --- /dev/null +++ b/.github/workflows/ci-staging.yml @@ -0,0 +1,35 @@ +name: Github-CI-Staging + +on: + push: + tags: + - staging_[a-zA-Z]+[0-9]+ + +env: + # secrets can be set in settings/secrets on github + DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + +jobs: + deploy: + if: github.event.base_ref == 'refs/heads/master' + name: deploy staging + runs-on: ubuntu-latest + env: + FROM_TAG_PREFIX: master-github + TO_TAG_PREFIX: staging-github + steps: + - uses: actions/checkout@v2 + - name: setup docker + run: | + sudo ./ci/github/helpers/setup_docker_compose.bash + ./ci/github/helpers/setup_docker_experimental.bash + ./ci/github/helpers/setup_docker_buildx.bash + echo ::set-env name=DOCKER_BUILDX::1 + - name: set owner variable + run: echo ::set-env name=OWNER::${GITHUB_REPOSITORY%/*} + - name: set git tag + run: echo ::set-env name=GIT_TAG::${GITHUB_REF##*/} + - name: deploy + run: ./ci/deploy/dockerhub-tag-version.bash diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml index ee8a923d677..1138218c52f 100644 --- a/.github/workflows/ci-testing-deploy.yml +++ b/.github/workflows/ci-testing-deploy.yml @@ -2,7 +2,19 @@ name: Github-CI Push/PR on: push: + branches: + - "*" + tags-ignore: + - "*" + paths-ignore: + - "*.md" + - "*.png" + - "*.svg" + - "docs/**" + - ".vscode/**" pull_request: + branches: + - "*" paths-ignore: - "*.md" - "*.png" @@ -1176,17 +1188,8 @@ jobs: env: TAG_PREFIX: master-github run: ./ci/deploy/dockerhub-deploy.bash - - name: deploy staging - if: github.ref == 'refs/heads/staging' + - name: deploy hotfix + if: contains(github.ref, 'refs/heads/hotfix_v') env: - TAG_PREFIX: staging-github + TAG_PREFIX: hotfix-github run: ./ci/deploy/dockerhub-deploy.bash - # TODO: here we need a incoming webhook for mattermost (ask admin for it) (https://docs.mattermost.com/developer/webhooks-incoming.html) - # - name: mattermost notification - # uses: tferreira/matterfy@releases/v1 - # if: always() - # with: - # type: ${{ job.status }} - # job_name: '*Simcore build state*' - # channel: '${{ secrets.MATTERMOST_CHANNEL }}' - # url: ${{ secrets.MATTERMOST_URL }} diff --git a/.travis.yml b/.travis.yml index 327c83d123c..8827d5f6cd4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -404,32 +404,35 @@ jobs: # deployment to staging environ ---------------------------------------------------------------- - stage: deployment name: staging - if: branch = staging env: - OWNER=${TRAVIS_REPO_SLUG%/*} - - TAG_PREFIX=staging + - FROM_TAG_PREFIX=master-travis + - TO_TAG_PREFIX=staging-travis + - GIT_TAG=${TRAVIS_TAG} git: depth: false script: echo "Deploy staging" deploy: - provider: script - script: unbuffer bash ci/deploy/dockerhub-deploy.bash + script: unbuffer bash ci/deploy/dockerhub-tag-version.bash on: - branch: staging - + all_branches: true + tags: true + condition: $TRAVIS_TAG =~ ^FREEZE_[a-zA-Z]+[0-9]+$ # deployment to production environ ---------------------------------------------------------------- - stage: deployment name: production/release env: - OWNER=${TRAVIS_REPO_SLUG%/*} - - TAG_PREFIX=staging + - FROM_TAG_PREFIX=staging-travis + - TO_TAG_PREFIX=release-travis - GIT_TAG=${TRAVIS_TAG} git: depth: false script: echo "Deploy production version $TRAVIS_TAG" deploy: - provider: script - script: unbuffer bash ci/deploy/dockerhub-release.bash + script: unbuffer bash ci/deploy/dockerhub-tag-version.bash on: all_branches: true tags: true diff --git a/Makefile b/Makefile index 3124dea1207..7c982e8bbe2 100644 --- a/Makefile +++ b/Makefile @@ -523,3 +523,44 @@ clean-all: clean clean-more clean-images # Deep clean including .venv and produc .PHONY: reset reset: ## restart docker daemon (LINUX ONLY) sudo systemctl restart docker + + +# RELEASE -------------------------------------------------------------------------------------------------------------------------------------------- + +staging_prefix := staging_ +prod_prefix := v +_git_get_current_branch = $(shell git rev-parse --abbrev-ref HEAD) +# NOTE: be careful that GNU Make replaces newlines with space which is why this command cannot work using a Make function +_url_encoded_title = $(if $(findstring -staging, $@),Staging%20$(name),)$(version) +_url_encoded_tag = $(if $(findstring -staging, $@),$(staging_prefix)$(name),$(prod_prefix))$(version) +_url_encoded_target = $(if $(git_sha),$(git_sha),$(if $(findstring -hotfix, $@),$(_git_get_current_branch),master)) +define _url_encoded_logs +$(shell \ + scripts/url-encoder.bash \ + "$$(git log \ + $$(git describe --match="$(if $(findstring -staging, $@),$(staging_prefix),$(prod_prefix))*" --abbrev=0 --tags)..$(if $(git_sha),$(git_sha),HEAD) \ + --pretty=format:"- %s")"\ +) +endef +_git_get_repo_orga_name = $(shell git config --get remote.origin.url | \ + grep --perl-regexp --only-matching "((?<=git@github\.com:)|(?<=https:\/\/github\.com\/))(.*?)(?=.git)") + +.PHONY: .check-master-branch +.check-master-branch: + @if [ "$(_git_get_current_branch)" != "master" ]; then\ + echo -e "\e[91mcurrent branch is not master branch."; exit 1;\ + fi + +.PHONY: release-staging release-prod +release-staging release-prod: .check-master-branch ## Helper to create a staging or production release in Github (usage: make release-staging name=sprint version=1 git_sha=optional or make release-prod version=1.2.3 git_sha=optional) + # ensure tags are uptodate + @git pull --tags + @echo -e "\e[33mOpen the following link to create the $(if $(findstring -staging, $@),staging,production) release:"; + @echo -e "\e[32mhttps://github.com/$(_git_get_repo_orga_name)/releases/new?prerelease=$(if $(findstring -staging, $@),1,0)&target=$(_url_encoded_target)&tag=$(_url_encoded_tag)&title=$(_url_encoded_title)&body=$(_url_encoded_logs)"; + +.PHONY: release-hotfix +release-hotfix: ## Helper to create a hotfix release in Github (usage: make release-hotfix version=1.2.4 git_sha=optional) + # ensure tags are uptodate + @git pull --tags + @echo -e "\e[33mOpen the following link to create the $(if $(findstring -staging, $@),staging,production) release:"; + @echo -e "\e[32mhttps://github.com/$(_git_get_repo_orga_name)/releases/new?prerelease=$(if $(findstring -staging, $@),1,0)&target=$(_url_encoded_target)&tag=$(_url_encoded_tag)&title=$(_url_encoded_title)&body=$(_url_encoded_logs)"; diff --git a/ci/README.md b/ci/README.md index b7af802a79a..917f66531bb 100644 --- a/ci/README.md +++ b/ci/README.md @@ -1,23 +1,20 @@ # Continuous Integration -## Travis / Dockerhub setup +## CI / Dockerhub setup -The osparc-simcore repository provides a Travis-CI [recipe](.travis.yml) that fullfills the [Release Workflow](docs/img/git-release-workflow.svg). +The osparc-simcore repository provides a Travis-CI [recipe](.travis.yml) and Gihub-CI [workflows](.github/workflows) that fullfills the [Release Workflow](docs/img/git-release-workflow.svg) and the [Hotfix Workflow](docs/img/git-hotfix-workflow.svg). -To this end Travis should be enabled for the repository in each fork and a [Dockerhub](https://hub.docker.com/) account is recommended to push the docker images generated by Travis. +### Configuration +Github Actions and/or Travis-CI should be enabled for the repository in each fork and a [Dockerhub](https://hub.docker.com/) account is recommended to push the docker images generated by the CI. -Configuring your travis settings will speed up the travis CI process by making use of Dockerhub (or another docker registry) to move docker images between stages. +Configuring your CI settings will speed up the CI process by making use of Dockerhub (or another docker registry) to move docker images between stages. -### Travis configuration +### Secrets configuration Define the following secure environment variables in your fork: ```bash DOCKER_REGISTRY # this shall be set to your own dockerhub repository account for example: itisfoundation -DOCKER_USERNAME # the docker username (!beware this should be a [secure env variable](https://docs.travis-ci.com/user/environment-variables/#defining-encrypted-variables-in-travisyml)) -DOCKER_PASSWORD # the docker password (!beware this should be a [secure env variable](https://docs.travis-ci.com/user/environment-variables/#defining-encrypted-variables-in-travisyml)) +DOCKER_USERNAME # the docker username +DOCKER_PASSWORD # the docker password ``` - -### Release workflow - -![Git release workflow](../docs/img/git-release-workflow.svg) diff --git a/ci/deploy/dockerhub-deploy.bash b/ci/deploy/dockerhub-deploy.bash index 07fb5dc656a..9670a3c081f 100755 --- a/ci/deploy/dockerhub-deploy.bash +++ b/ci/deploy/dockerhub-deploy.bash @@ -1,35 +1,40 @@ -#!/bin/bash -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail +#!/usr/bin/env bash +# strict mode +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes IFS=$'\n\t' -bash ci/helpers/dockerhub_login.bash +my_dir="$(dirname "$0")" +source "$my_dir/../../scripts/helpers/logger.bash" # check script needed variables if [ ! -v TAG_PREFIX ]; then - echo "## ERROR: incorrect use of script. TAG_PREFIX (e.g. master, staging) not defined!" - exit 1 + error_exit "$LINENO" "incorrect use of script. TAG_PREFIX (e.g. master, staging) not defined!" fi +log_info "logging in dockerhub..." +bash ci/helpers/dockerhub_login.bash + # pull the current tested build + DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash) export DOCKER_IMAGE_TAG +log_info "pulling build ${DOCKER_IMAGE_TAG}" make pull-version tag-local # show current images on system -echo "## Before push" +log_info "Before push" make info-images # re-tag build DOCKER_IMAGE_TAG="$TAG_PREFIX-latest" export DOCKER_IMAGE_TAG +log_info "pushing images ${DOCKER_IMAGE_TAG} to ${DOCKER_REGISTRY}" make push-version # re-tag build to master-github-DATE.GIT_SHA DOCKER_IMAGE_TAG=$TAG_PREFIX-$(date --utc +"%Y-%m-%d--%H-%M").$(git rev-parse HEAD) export DOCKER_IMAGE_TAG +log_info "pushing images ${DOCKER_IMAGE_TAG} to ${DOCKER_REGISTRY}" make push-version - -# show update of images on system -echo "## After push" -make info-images diff --git a/ci/deploy/dockerhub-release.bash b/ci/deploy/dockerhub-release.bash deleted file mode 100755 index cd36f86cd93..00000000000 --- a/ci/deploy/dockerhub-release.bash +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail -IFS=$'\n\t' - -bash ci/helpers/dockerhub_login.bash - -# check script needed variables -if [ ! -v TAG_PREFIX ] || [ ! -v GIT_TAG ]; then - echo "## ERROR: incorrect use of script. OWNER (e.g. itisfoundation or user) and/or TAG_PREFIX (e.g. master, staging), and/or GIT_TAG not defined!" - exit 1 -fi - -# pull the tagged staging build -# find the docker image tag -export ORG=${DOCKER_REGISTRY} -export REPO="webserver" -# staging-github-DATE.GIT_SHA -export TAG_PATTERN="^${TAG_PREFIX}-.+\..+" -DOCKER_IMAGE_TAG=$(./ci/helpers/find_staging_version.bash | awk 'END{print}') || exit $? -export DOCKER_IMAGE_TAG -make pull-version tag-local - -# show current images on system -echo "## Before push" -make info-images - -# re-tag staging to {GIT_TAG}-DATE.GIT_SHA -DOCKER_IMAGE_TAG=${GIT_TAG}-$(date --utc +"%Y-%m-%d--%H-%M").$(git rev-parse HEAD) -export DOCKER_IMAGE_TAG -make push-version -if [ -v TRAVIS ] && [ "$TRAVIS" = "true" ]; then - # currently TRAVIS is still the master - make push-latest -fi - -echo "## After push" -make info-images diff --git a/ci/deploy/dockerhub-tag-version.bash b/ci/deploy/dockerhub-tag-version.bash new file mode 100755 index 00000000000..b82279903ea --- /dev/null +++ b/ci/deploy/dockerhub-tag-version.bash @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# strict mode +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes +IFS=$'\n\t' + +my_dir="$(dirname "$0")" +source "$my_dir/../../scripts/helpers/logger.bash" + +# check script needed variables +if [ ! -v FROM_TAG_PREFIX ] || [ ! -v TO_TAG_PREFIX ] || [ ! -v GIT_TAG ]; then + error_exit "$LINENO" "incorrect use of script. FROM_TAG_PREFIX/TO_TAG_PREFIX (e.g. master, staging), and/or GIT_TAG not defined!" +fi + +log_info "logging in dockerhub..." +bash ci/helpers/dockerhub_login.bash + +log_info "finding the version in the docker hub registry..." +# find and pull the tagged build +# find the docker image tag +export ORG=${DOCKER_REGISTRY} +export REPO="webserver" +# FROM_TAG_PREFIX-DATE.GIT_SHA +export TAG_PATTERN="^${FROM_TAG_PREFIX}-.+\..+" +DOCKER_IMAGE_TAG=$(./ci/helpers/find_docker_image_tag_from_git_sha.bash | awk 'END{print}') || exit $? +log_info "found image ${DOCKER_IMAGE_TAG}" +export DOCKER_IMAGE_TAG +log_info "pulling images ${DOCKER_IMAGE_TAG} from ${DOCKER_REGISTRY}" +make pull-version tag-local + +# show current images on system +log_info "Before push" +make info-images + +# re-tag images to ${TO_TAG_PREFIX}-{GIT_TAG}-DATE.GIT_SHA +readonly GIT_COMMIT_SHA=$(git show-ref -s "${GIT_TAG}") +DOCKER_IMAGE_TAG="${TO_TAG_PREFIX}-${GIT_TAG}"-$(date --utc +"%Y-%m-%d--%H-%M")."${GIT_COMMIT_SHA}" +export DOCKER_IMAGE_TAG +log_info "pushing images ${DOCKER_IMAGE_TAG} to ${DOCKER_REGISTRY}" +make push-version + +# push latest image ${TO_TAG_PREFIX}-latest +DOCKER_IMAGE_TAG="${TO_TAG_PREFIX}-latest" +export DOCKER_IMAGE_TAG +log_info "pushing images ${DOCKER_IMAGE_TAG} to ${DOCKER_REGISTRY}" +make push-version + +log_info "complete!" diff --git a/ci/helpers/find_docker_image_tag_from_git_sha.bash b/ci/helpers/find_docker_image_tag_from_git_sha.bash new file mode 100755 index 00000000000..b1e37ffe53a --- /dev/null +++ b/ci/helpers/find_docker_image_tag_from_git_sha.bash @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Usage: find_docker_image_tag_from_git_sha.bash +# +# returns the full image tag corresponding to the git tag name that shall be used + +# strict mode +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes +IFS=$'\n\t' + +my_dir="$(dirname "$0")" +source "$my_dir/../../scripts/helpers/logger.bash" + +log_info "Retrieving SHA for tag ${GIT_TAG}" +readonly GIT_COMMIT_SHA=$(git show-ref -s "${GIT_TAG}") +log_info "Found SHA for tag ${GIT_COMMIT_SHA}" + +# get token +log_info "Retrieving token ..." +readonly TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username": "'"${DOCKER_USERNAME}"'", "password": "'"${DOCKER_PASSWORD}"'"}' https://hub.docker.com/v2/users/login/ | jq -r .token) + +# output images & tags + +log_info "Images and tags for organization: ${ORG} in repo ${REPO}" +readonly IMAGE_TAGS=$(curl -s -H "Authorization: JWT ${TOKEN}" https://hub.docker.com/v2/repositories/"${ORG}"/"${REPO}"/tags/?page_size=100 | jq -r '.results|.[]|.name') +for j in ${IMAGE_TAGS} +do + if [[ ${j} =~ ${TAG_PATTERN} ]]; then + if [[ ${j} =~ ${GIT_COMMIT_SHA} ]]; then + echo "${j}" + exit 0 + fi + fi +done +# not found +error_exit "$LINENO" "could not find image:${GIT_TAG} with ${GIT_COMMIT_SHA}. Is the image available in the registry?" diff --git a/ci/helpers/find_staging_version.bash b/ci/helpers/find_staging_version.bash deleted file mode 100755 index 768a69d6f57..00000000000 --- a/ci/helpers/find_staging_version.bash +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -# Usage: find_staging_version.bash -# -# returns the full image tag corresponding to the git tag name that shall be used - -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail -IFS=$'\n\t' - -echo "Retrieving SHA for tag ${GIT_TAG}" -GIT_COMMIT_SHA=$(git show-ref -s "${GIT_TAG}") -echo "Found SHA for tag ${GIT_COMMIT_SHA}" - -# get token -echo "Retrieving token ..." -TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username": "'"${DOCKER_USERNAME}"'", "password": "'"${DOCKER_PASSWORD}"'"}' https://hub.docker.com/v2/users/login/ | jq -r .token) - -# output images & tags -echo -echo "Images and tags for organization: ${ORG}" -echo "in repo ${REPO}" -IMAGE_TAGS=$(curl -s -H "Authorization: JWT ${TOKEN}" https://hub.docker.com/v2/repositories/"${ORG}"/"${REPO}"/tags/?page_size=100 | jq -r '.results|.[]|.name') -for j in ${IMAGE_TAGS} -do - if [[ ${j} =~ ${TAG_PATTERN} ]]; then - if [[ ${j} =~ ${GIT_COMMIT_SHA} ]]; then - echo "${j}" - exit 0 - fi - fi -done -# not found -exit 1 diff --git a/ci/helpers/install_pylint.bash b/ci/helpers/install_pylint.bash old mode 100644 new mode 100755 diff --git a/docs/img/git-hotfix-workflow.svg b/docs/img/git-hotfix-workflow.svg new file mode 100644 index 00000000000..79d5a2f870f --- /dev/null +++ b/docs/img/git-hotfix-workflow.svg @@ -0,0 +1 @@ +ReleasedVersionHotfixDockerhubcreate hotfix_v1.0.x branch1fix issue...2Pull request hotfix13loop[Hotfix]CI: build/test docker images [hotfix-latest]4ready for release?RELEASE: Tag v1.0.15CI: Tag docker images [release-latest]6loop[Releasing]ReleasedVersionHotfixDockerhub \ No newline at end of file diff --git a/docs/img/git-release-workflow.md b/docs/img/git-release-workflow.md index ca8ca2999f8..66152da2143 100644 --- a/docs/img/git-release-workflow.md +++ b/docs/img/git-release-workflow.md @@ -2,68 +2,76 @@ [mermaid live editor](https://mermaidjs.github.io/mermaid-live-editor) +Release process ```mermaid sequenceDiagram +autonumber participant Feature participant Master -participant Hotfix -participant Staging +participant Dockerhub -Master->>Feature: create feature1 branch -Note over Feature: develop feature1... -Feature-->>Master: Pull Request feature1 -Master->Master: CI: build docker images - tests -> Dockerhub -Master->Master: CD -> master.dev +Loop FeatureDev + Master-->>Feature: create feature1 branch + Feature->Feature: develop feature1... + Feature-->>Master: Pull Request feature1 +end +Master-->>Dockerhub: CI: build/test docker images [master-latest] -Master->>Feature: create feature2 branch -Note over Feature: develop feature2... -Feature-->>Master: Pull Request feature2 -Master->Master: CI: build docker images - tests -> Dockerhub -Master->Master: CD -> master.dev +Loop FeatureDev + Master-->>Feature: create feature2 branch + Feature->Feature: develop feature2... + Feature-->>Master: Pull Request feature2 +end +Master-->>Dockerhub: CI: build/test docker images [master-latest] -Master-->>Staging: Pull Request staging1 -Staging->Staging: CI: build docker images - tests -> Dockerhub -Staging->Staging: CD -> staging.io -Note over Staging: ready for release? +Loop Staging + Master->Master: STAGING: Tag staging_Sprint1 + Master-->>Dockerhub: CI: Tag docker images [staging-latest] +end +Note over Master: ready for release? -Master->>Feature: create feature3 branch -Note over Feature: develop feature3... -Feature-->>Master: Pull Request feature3 -Master->Master: CI: build docker images - tests -> Dockerhub -Master->Master: CD -> master.dev +Loop FeatureDev + Master-->>Feature: create feature3 branch + Feature->Feature: develop feature3... + Feature-->>Master: Pull Request feature3 +end +Master-->>Dockerhub: CI: build/test docker images [master-latest] -Master->>Feature: create feature4 branch -Note over Feature: develop feature4... -Feature-->>Master: Pull Request feature4 -Master->Master: CI: build docker images - tests -> Dockerhub -Master->Master: CD -> master.dev +Loop FeatureDev + Master-->>Feature: create feature4 branch + Feature->Feature: develop feature4... + Feature-->>Master: Pull Request feature4 +end +Master-->>Dockerhub: CI: build/test docker images [master-latest] -Master-->>Staging: Pull Request staging2 -Staging->Staging: CI: build docker images - tests -> Dockerhub -Staging->Staging: CD -> staging.io -Note over Staging: ready for release? -Staging->Staging: RELEASE: Tag v1.0.0 - CD -> osparc.io +Loop Staging + Master->Master: STAGING: Tag staging_Sprint2 + Master-->>Dockerhub: CI: Tag docker images [staging-latest] +end +Note over Master: ready for release? -Staging->>Hotfix: create hotfix1 branch -Note over Hotfix: fix issue... -Hotfix-->>Staging: Pull request hotfix1 -Staging->Staging: CI: build docker images - tests -> Dockerhub -Staging->Staging: CD -> staging.io -Note over Staging: ready for release? -Staging->Staging: RELEASE: Tag v1.0.1 - CD -> osparc.io -Hotfix-->>Master: Pull request hotfix1 -Master->Master: CI: build docker images - tests -> Dockerhub -Master->Master: CD -> master.dev +Loop Releasing + Master->Master: RELEASE: Tag v1.0.0 + Master-->>Dockerhub: CI: Tag docker images [release-latest] +end +``` -Master->>Feature: create feature10 branch -Note over Feature: develop feature10... -Feature-->>Master: Pull Request feature10 -Master->Master: CI: build docker images - tests -> Dockerhub -Master->Master: CD -> master.dev +Hotfix process +```mermaid +sequenceDiagram +autonumber +participant ReleasedVersion +participant Hotfix -Master->>Feature: create featureN branch -Note over Feature: develop featureN... -Feature-->>Master: Pull Request featureN -Master->Master: CI: build docker images - tests -> Dockerhub -Master->Master: CD -> master.dev +Loop Hotfix + ReleasedVersion->>Hotfix: create hotfix_v1.0.x branch + Hotfix->Hotfix: fix issue... + Hotfix-->>ReleasedVersion: Pull request hotfix1 +end +Hotfix-->>Dockerhub: CI: build/test docker images [hotfix-latest] +Note over Hotfix: ready for release? +Loop Releasing + Hotfix->Hotfix: RELEASE: Tag v1.0.1 + Hotfix-->>Dockerhub: CI: Tag docker images [release-latest] +end ``` diff --git a/docs/img/git-release-workflow.svg b/docs/img/git-release-workflow.svg index 8958e1e6f4d..5c30a7c878f 100644 --- a/docs/img/git-release-workflow.svg +++ b/docs/img/git-release-workflow.svg @@ -1,360 +1 @@ -FeatureMasterHotfixStagingcreate feature1 branchdevelop feature1...Pull Request feature1CI: build docker images - tests -> DockerhubCD -> master.devcreate feature2 branchdevelop feature2...Pull Request feature2CI: build docker images - tests -> DockerhubCD -> master.devPull Request staging1CI: build docker images - tests -> DockerhubCD -> staging.ioready for release?create feature3 branchdevelop feature3...Pull Request feature3CI: build docker images - tests -> DockerhubCD -> master.devcreate feature4 branchdevelop feature4...Pull Request feature4CI: build docker images - tests -> DockerhubCD -> master.devPull Request staging2CI: build docker images - tests -> DockerhubCD -> staging.ioready for release?RELEASE: Tag v1.0.0 - CD -> osparc.iocreate hotfix1 branchfix issue...Pull request hotfix1CI: build docker images - tests -> DockerhubCD -> staging.ioready for release?RELEASE: Tag v1.0.1 - CD -> osparc.ioPull request hotfix1CI: build docker images - tests -> DockerhubCD -> master.devcreate feature10 branchdevelop feature10...Pull Request feature10CI: build docker images - tests -> DockerhubCD -> master.devcreate featureN branchdevelop featureN...Pull Request featureNCI: build docker images - tests -> DockerhubCD -> master.devFeatureMasterHotfixStaging \ No newline at end of file +FeatureMasterDockerhubcreate feature1 branch1develop feature1...2Pull Request feature13loop[FeatureDev]CI: build/test docker images [master-latest]4create feature2 branch5develop feature2...6Pull Request feature27loop[FeatureDev]CI: build/test docker images [master-latest]8STAGING: Tag staging_Sprint19CI: Tag docker images [staging-latest]10loop[Staging]ready for release?create feature3 branch11develop feature3...12Pull Request feature313loop[FeatureDev]CI: build/test docker images [master-latest]14create feature4 branch15develop feature4...16Pull Request feature417loop[FeatureDev]CI: build/test docker images [master-latest]18STAGING: Tag staging_Sprint219CI: Tag docker images [staging-latest]20loop[Staging]ready for release?RELEASE: Tag v1.0.021CI: Tag docker images [release-latest]22loop[Releasing]FeatureMasterDockerhub \ No newline at end of file diff --git a/docs/production-instructions.md b/docs/production-instructions.md deleted file mode 100644 index 5334638537f..00000000000 --- a/docs/production-instructions.md +++ /dev/null @@ -1,17 +0,0 @@ -# Deployment of production environment - - -The production environment is automaticaly built and deployed upon creation of a new [release](https://github.com/ITISFoundation/osparc-simcore/releases/). - - -- A new release adds a tag to [staging branch](https://github.com/ITISFoundation/osparc-simcore/branches) as ``v1.2.3``. -- Release notes shall be human readable and shall summarize briefly the commits in staging since last release, i.e. -``` -git log --pretty=oneline --abbrev-commit v1.1.55..HEAD -``` -where ``v1.1.55` is supposed to be the last release. - - -## Magic behind the scene - -TODO: deployment workflow: who and when is involved in deploying production diff --git a/docs/releasing-workflow-instructions.md b/docs/releasing-workflow-instructions.md new file mode 100644 index 00000000000..e64c7247377 --- /dev/null +++ b/docs/releasing-workflow-instructions.md @@ -0,0 +1,148 @@ +# Management of releases of simcore + +The process of creating staging/release/hotfix versions of code from [Master](https://github.com/ITISFoundation/osparc-simcore/tree/master) is described here. + +## Description + +Each commit on the master branch triggers a CI workflow that builds the simcore platform docker images. +These docker images are then stored on [dockerhub](https://hub.docker.com/repositories/itisfoundation) registry in the itisfoundation repository. +Each docker image are named following this pattern: + +- itisfoundation/[image_name]:master-[CINAME]-latest +- itisfoundation/[image_name]:master-[CINAME]-[BUILD_DATE]--[BUILD_TIME].[GIT_SHA] + +with: + +- CINAME the name of the CI +- BUILD_DATE is the build date +- BUILD_TIME is the build time +- GIT_SHA is the git SHA corresponding to the code used for building + +see [docs/img/git-release-workflow.svg](docs/img/git-release-workflow.svg) + +### Example + +The ``webserver`` service in master branch with commit ``752ef...`` will be built by ``github`` actions CI on ``2020/08/31`` at ``12:36`` and therefore the image is named as: + +- ``itisfoundation/webserver:master-github-v2.0.1-2020-08-31--12-36.752ef50f3babb6537580c0e03b85b9a8209bbf10`` +- ``itisfoundation/webserver:master-github-latest`` + +## Staging process + +A staging version of simcore is like a pre-released version, that is marked as such on the master branch by leveraging Github pre-release tagging mechanism. The CI is triggered and will pull the marked docker images (by using the given git SHA or latest master build), and tag them as staging images. +Each docker image are renamed as: + +- itisfoundation/[image_name]:staging-[CINAME]-latest +- itisfoundation/[image_name]:staging-[CINAME]-staging_[BUILD_NAMEVERSION]-[BUILD_DATE]--[BUILD_TIME].[GIT_SHA] + +with: + +- CINAME the name of the CI +- BUILD_NAMEVERSION is a one word name and a number (e.g. DaJia0, Dajia1, ...) +- BUILD_DATE is the build date +- BUILD_TIME is the build time +- GIT_SHA is the git SHA corresponding to the code used for building + +see [docs/img/git-release-workflow.svg](docs/img/git-release-workflow.svg) + +### Staging example + +Just before the review of the ``DAJIA`` sprint we release a staging version ``v1.0.0``, the commit ``752ef...`` from above is tagged as stage and the ``github`` actions CI on ``2020/09/01`` at ``17:30``. + +- ``itisfoundation/webserver:staging-github-stage_DAJIA1-2020-09-01--17-30.752ef50f3babb6537580c0e03b85b9a8209bbf10`` +- ``itisfoundation/webserver:staging-github-latest`` + +then after the review we do a couple of additions and re-release staging ``DAJIA`` sprint as `v1.1.0` + +- ``itisfoundation/webserver:staging-github-stage_DAJIA2-2020-09-01--20-30.560eq50f3babb6537580c0e03b85b9a8209bbf10`` +- ``itisfoundation/webserver:staging-github-latest`` + +### Instructions to generate a staging release + +1. Generate Github release tag + + ```bash + git clone https://github.com/ITISFoundation/osparc-simcore.git + cd osparc-simcore + make release-staging name=SPRINTNAME version=VERSION (git_sha=OPTIONAL) + ``` + +2. Adjust the list of changes if needed +3. Press the **Publish release** button +4. The CI will be automatically triggered and will deploy the staging release + +## Release process + +A released version of simcore, that is marked as such on the master branch by leveraging Github release tagging mechanism. The CI is triggered and will pull the marked staging docker images (by using the given git SHA or the latest staging images), and tag them as release images. +**NOTE:** A release version is ALWAYS preceded by a staging version. The CI will fail if it does not find the corresponding staging version. +Each docker build marked as released are tagged as: + +- itisfoundation/[image_name]:release-[CINAME]-latest +- itisfoundation/[image_name]:release-[CINAME]-v[RELEASE_VERSION]-[BUILD_DATE]--[BUILD_TIME].GIT_SHA + +with: + +- CINAME the name of the CI +- RELEASE_VERSION is a version number following semantic versioning (e.g. 1.0.0, 1.2.0, 1.2.1, ...) +- BUILD_DATE is the build date +- BUILD_TIME is the build time +- GIT_SHA is the git SHA corresponding to the code used for building + +see [docs/img/git-release-workflow.svg](docs/img/git-release-workflow.svg) + +### Release example + +The team decides to release to production the lastest staging version of ``DAJIA`` sprint. Next release version, following semantic versioning and previous releases, is `v5.6.0`. The images will be retaged by github actions CI as: + +- ``itisfoundation/webserver:release-github-v5.6.0-2020-09-02--19-30.560eq50f3babb6537580c0e03b85b9a8209bbf10`` +- ``itisfoundation/webserver:release-github-latest`` + +### Instructions to generate a release + +1. Generate Github release tag + + ```bash + git clone https://github.com/ITISFoundation/osparc-simcore.git + cd osparc-simcore + make release-prod version=MAJ.MIN.PATCH (git_sha=OPTIONAL) + ``` + +2. Adjust the list of changes if needed +3. Press the **Publish release** button +4. The CI will be automatically triggered and will deploy the staging release + +## Hotfix process + +A hotfix is **ALWAYS made from an already released version**. A branch is created from the tagged release version (must be named hotfix_v*) and the fix is implemented there following usual best practices. On each commit in that branch the CI is triggered and will generate the following images in Dockerhub: + +- itisfoundation/[image_name]:hotfix-[CINAME]-latest +- itisfoundation/[image_name]:hotfix-[CINAME]-[BUILD_DATE]--[BUILD_TIME].[GIT_SHA] + +Once ready, the release-hotfix process starts by leveraging Github release mechanism. The CI will trigger again and pull the docker images (based on git SHA or latest hotfix image) and push a new release version. + +Each docker build marked as released are tagged as described in the Release process. + +see [docs/img/git-hotfix-workflow.svg](docs/img/git-hotfix-workflow.svg) + +### Hotfix example + +A bug was found in version 1.2.0 of the simcore stack. The team decides to fix it as it is a fairly simple one. Therefore a hotfix is created and a new release will be created by the CI: + +- ``itisfoundation/webserver:release-github-v1.2.1-2020-09-02--19-30.560eq50f3babb6537580c0e03b85b9a8209bbf10`` +- ``itisfoundation/webserver:release-github-latest`` + +### Instructions to generate a hotfix release + +1. Generate Github release tag + + ```bash + git clone https://github.com/ITISFoundation/osparc-simcore.git + cd osparc-simcore + git checkout VERSION_TAG_FOR_HOTFIXING + # make the fix through usual PR process + make release-hotfix version=MAJ.MIN.PATCH (git_sha=OPTIONAL) + ``` + +2. Adjust the list of changes if needed +3. Press the **Publish release** button +4. The CI will be automatically triggered and will deploy the staging release diff --git a/docs/staging-instructions.md b/docs/staging-instructions.md deleted file mode 100644 index 4a54fcfca04..00000000000 --- a/docs/staging-instructions.md +++ /dev/null @@ -1,28 +0,0 @@ -# Management of creating staging release - -The process of moving code from [Master](https://github.com/ITISFoundation/osparc-simcore/tree/master) branch to [Staging](https://github.com/ITISFoundation/osparc-simcore/tree/staging) branch is described here. - -## Process - -1. Create a PR from master to staging - - ```bash - export SPRINT_NAME=TheNameOfTheSprint - export GIT_SHA=TheGitSHAOfTheLatestSprintCommit - # clone the repo - git clone git@github.com:ITISFoundation/osparc-simcore.git - # create a branch from master at defined git SHA - git checkout -b FREEZE_${SPRINT_NAME} ${GIT_SHA} - # push the branch to git origin repo - git push --set-upstream origin FREEZE_${SPRINT_NAME} - # create the log entries to be copied into the pull request - #body=scripts/url-encoder.sh "$(git log --oneline staging..HEAD --no-decorate)" - body=$(scripts/url-encoder.bash "$(git log origin/staging..HEAD --pretty="format:- %s")") - # open the PR on github website - echo "https://github.com/ITISFoundation/osparc-simcore/compare/staging...FREEZE_${SPRINT_NAME}?expand=1&title=FREEZE_${SPRINT_NAME}&body=$body" - - ``` - -2. Adjust the PR list of changes -3. Once the PR is "green", do a **merge pull request** (NOTE: DO NOT SQUASH!!!) -4. The CI will be automatically triggered and will build/test/deploy diff --git a/scripts/helpers/logger.bash b/scripts/helpers/logger.bash new file mode 100644 index 00000000000..02cd0a1778b --- /dev/null +++ b/scripts/helpers/logger.bash @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# strict mode +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes +IFS=$'\n\t' + +readonly NC='\033[0m' # no color +readonly BLACK='\033[0;30m' +readonly RED='\033[0;31m' +readonly GREEN='\033[0;32m' +readonly ORANGE='\033[0;33m' +readonly YELLOW='\033[1;33m' +readonly WHITE='\033[1;37m' + +function error_exit { + readonly line=$1 + shift 1 + echo + echo -e "${RED}[ERROR]:$line: ${1:-"Unknown Error"}" 1>&2 + exit 1 +} + +function log_info { + echo + echo -e "${WHITE}[INFO]: ${1:-"Unknown message"}${NC}" +} + +function log_warning { + echo + echo -e "${YELLOW}[WARNING]: ${1:-"Unknown message"}${NC}" +} diff --git a/scripts/url-encoder.bash b/scripts/url-encoder.bash index 8cbf7ceb477..fce18e18c0d 100755 --- a/scripts/url-encoder.bash +++ b/scripts/url-encoder.bash @@ -1,4 +1,9 @@ #!/bin/bash +# strict mode +set -o errexit # abort on nonzero exitstatus +set -o nounset # abort on unbound variable +set -o pipefail # don't hide errors within pipes +IFS=$'\n\t' # https://stackoverflow.com/questions/296536/how-to-urlencode-data-for-curl-command