Simplify release process (#1748)

use tagging instead of staging branch

.github/workflows/ci-hotfix.yml

@@ -0,0 +1,36 @@
+name: Github-CI-Hotfix-Release
+
+on:
+  push:
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
+
+env:
+  # secrets can be set in settings/secrets on github
+  DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
+  DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+  DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+
+jobs:
+  deploy:
+    if: contains(github.event.base_ref, 'refs/heads/hotfix_v')
+    # only run on staging branch
+    name: deploy release
+    runs-on: ubuntu-latest
+    env:
+      FROM_TAG_PREFIX: hotfix-github
+      TO_TAG_PREFIX: release-github
+    steps:
+      - uses: actions/checkout@v2
+      - name: setup docker
+        run: |
+          sudo ./ci/github/helpers/setup_docker_compose.bash
+          ./ci/github/helpers/setup_docker_experimental.bash
+          ./ci/github/helpers/setup_docker_buildx.bash
+          echo ::set-env name=DOCKER_BUILDX::1
+      - name: set owner variable
+        run: echo ::set-env name=OWNER::${GITHUB_REPOSITORY%/*}
+      - name: set git tag
+        run: echo ::set-env name=GIT_TAG::${GITHUB_REF##*/}
+      - name: deploy
+        run: ./ci/deploy/dockerhub-tag-version.bash

.github/workflows/ci-release.yml

@@ -13,12 +13,13 @@ env:
 
 jobs:
   deploy:
+    if: github.event.base_ref == 'refs/heads/master'
     # only run on staging branch
-    if: github.event.base_ref == 'refs/heads/staging'
     name: deploy release
     runs-on: ubuntu-latest
     env:
-      TAG_PREFIX: staging-github
+      FROM_TAG_PREFIX: staging-github
+      TO_TAG_PREFIX: release-github
     steps:
       - uses: actions/checkout@v2
       - name: setup docker
@@ -32,4 +33,4 @@ jobs:
       - name: set git tag
         run: echo ::set-env name=GIT_TAG::${GITHUB_REF##*/}
       - name: deploy
-        run: ./ci/deploy/dockerhub-release.bash
+        run: ./ci/deploy/dockerhub-tag-version.bash

.github/workflows/ci-staging.yml

@@ -0,0 +1,35 @@
+name: Github-CI-Staging
+
+on:
+  push:
+    tags:
+      - staging_[a-zA-Z]+[0-9]+
+
+env:
+  # secrets can be set in settings/secrets on github
+  DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }}
+  DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+  DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+
+jobs:
+  deploy:
+    if: github.event.base_ref == 'refs/heads/master'
+    name: deploy staging
+    runs-on: ubuntu-latest
+    env:
+      FROM_TAG_PREFIX: master-github
+      TO_TAG_PREFIX: staging-github
+    steps:
+      - uses: actions/checkout@v2
+      - name: setup docker
+        run: |
+          sudo ./ci/github/helpers/setup_docker_compose.bash
+          ./ci/github/helpers/setup_docker_experimental.bash
+          ./ci/github/helpers/setup_docker_buildx.bash
+          echo ::set-env name=DOCKER_BUILDX::1
+      - name: set owner variable
+        run: echo ::set-env name=OWNER::${GITHUB_REPOSITORY%/*}
+      - name: set git tag
+        run: echo ::set-env name=GIT_TAG::${GITHUB_REF##*/}
+      - name: deploy
+        run: ./ci/deploy/dockerhub-tag-version.bash

.github/workflows/ci-testing-deploy.yml

@@ -2,7 +2,19 @@ name: Github-CI Push/PR
 
 on:
   push:
+    branches:
+      - "*"
+    tags-ignore:
+      - "*"
+    paths-ignore:
+      - "*.md"
+      - "*.png"
+      - "*.svg"
+      - "docs/**"
+      - ".vscode/**"
   pull_request:
+    branches:
+      - "*"
     paths-ignore:
       - "*.md"
       - "*.png"
@@ -1176,17 +1188,8 @@ jobs:
         env:
           TAG_PREFIX: master-github
         run: ./ci/deploy/dockerhub-deploy.bash
-      - name: deploy staging
-        if: github.ref == 'refs/heads/staging'
+      - name: deploy hotfix
+        if: contains(github.ref, 'refs/heads/hotfix_v')
         env:
-          TAG_PREFIX: staging-github
+          TAG_PREFIX: hotfix-github
         run: ./ci/deploy/dockerhub-deploy.bash
-      # TODO: here we need a incoming webhook for mattermost (ask admin for it) (https://docs.mattermost.com/developer/webhooks-incoming.html)
-      # - name: mattermost notification
-      #   uses: tferreira/matterfy@releases/v1
-      #   if: always()
-      #   with:
-      #     type: ${{ job.status }}
-      #     job_name: '*Simcore build state*'
-      #     channel: '${{ secrets.MATTERMOST_CHANNEL }}'
-      #     url: ${{ secrets.MATTERMOST_URL }}

.travis.yml

@@ -404,32 +404,35 @@ jobs:
     # deployment to staging environ ----------------------------------------------------------------
     - stage: deployment
       name: staging
-      if: branch = staging
       env:
         - OWNER=${TRAVIS_REPO_SLUG%/*}
-        - TAG_PREFIX=staging
+        - FROM_TAG_PREFIX=master-travis
+        - TO_TAG_PREFIX=staging-travis
+        - GIT_TAG=${TRAVIS_TAG}
       git:
         depth: false
       script: echo "Deploy staging"
       deploy:
         - provider: script
-          script: unbuffer bash ci/deploy/dockerhub-deploy.bash
+          script: unbuffer bash ci/deploy/dockerhub-tag-version.bash
           on:
-            branch: staging
-
+            all_branches: true
+            tags: true
+            condition: $TRAVIS_TAG =~ ^FREEZE_[a-zA-Z]+[0-9]+$
     # deployment to production environ ----------------------------------------------------------------
     - stage: deployment
       name: production/release
       env:
         - OWNER=${TRAVIS_REPO_SLUG%/*}
-        - TAG_PREFIX=staging
+        - FROM_TAG_PREFIX=staging-travis
+        - TO_TAG_PREFIX=release-travis
         - GIT_TAG=${TRAVIS_TAG}
       git:
         depth: false
       script: echo "Deploy production version $TRAVIS_TAG"
       deploy:
         - provider: script
-          script: unbuffer bash ci/deploy/dockerhub-release.bash
+          script: unbuffer bash ci/deploy/dockerhub-tag-version.bash
           on:
             all_branches: true
             tags: true

Makefile

@@ -523,3 +523,44 @@ clean-all: clean clean-more clean-images # Deep clean including .venv and produc
 .PHONY: reset
 reset: ## restart docker daemon (LINUX ONLY)
 	sudo systemctl restart docker
+
+
+# RELEASE --------------------------------------------------------------------------------------------------------------------------------------------
+
+staging_prefix := staging_
+prod_prefix := v
+_git_get_current_branch = $(shell git rev-parse --abbrev-ref HEAD)
+# NOTE: be careful that GNU Make replaces newlines with space which is why this command cannot work using a Make function
+_url_encoded_title = $(if $(findstring -staging, $@),Staging%20$(name),)$(version)
+_url_encoded_tag = $(if $(findstring -staging, $@),$(staging_prefix)$(name),$(prod_prefix))$(version)
+_url_encoded_target = $(if $(git_sha),$(git_sha),$(if $(findstring -hotfix, $@),$(_git_get_current_branch),master))
+define _url_encoded_logs
+$(shell \
+	scripts/url-encoder.bash \
+	"$$(git log \
+		$$(git describe --match="$(if $(findstring -staging, $@),$(staging_prefix),$(prod_prefix))*" --abbrev=0 --tags)..$(if $(git_sha),$(git_sha),HEAD) \
+		--pretty=format:"- %s")"\
+)
+endef
+_git_get_repo_orga_name = $(shell git config --get remote.origin.url | \
+							grep --perl-regexp --only-matching "((?<=git@github\.com:)|(?<=https:\/\/github\.com\/))(.*?)(?=.git)")
+
+.PHONY: .check-master-branch
+.check-master-branch:
+	@if [ "$(_git_get_current_branch)" != "master" ]; then\
+		echo -e "\e[91mcurrent branch is not master branch."; exit 1;\
+	fi
+
+.PHONY: release-staging release-prod
+release-staging release-prod: .check-master-branch ## Helper to create a staging or production release in Github (usage: make release-staging name=sprint version=1 git_sha=optional or make release-prod version=1.2.3 git_sha=optional)
+	# ensure tags are uptodate
+	@git pull --tags
+	@echo -e "\e[33mOpen the following link to create the $(if $(findstring -staging, $@),staging,production) release:";
+	@echo -e "\e[32mhttps://github.com/$(_git_get_repo_orga_name)/releases/new?prerelease=$(if $(findstring -staging, $@),1,0)&target=$(_url_encoded_target)&tag=$(_url_encoded_tag)&title=$(_url_encoded_title)&body=$(_url_encoded_logs)";
+
+.PHONY: release-hotfix
+release-hotfix: ## Helper to create a hotfix release in Github (usage: make release-hotfix version=1.2.4 git_sha=optional)
+	# ensure tags are uptodate
+	@git pull --tags
+	@echo -e "\e[33mOpen the following link to create the $(if $(findstring -staging, $@),staging,production) release:";
+	@echo -e "\e[32mhttps://github.com/$(_git_get_repo_orga_name)/releases/new?prerelease=$(if $(findstring -staging, $@),1,0)&target=$(_url_encoded_target)&tag=$(_url_encoded_tag)&title=$(_url_encoded_title)&body=$(_url_encoded_logs)";

ci/README.md

@@ -1,23 +1,20 @@
 # Continuous Integration
 
-## Travis / Dockerhub setup
+## CI / Dockerhub setup
 
-The osparc-simcore repository provides a Travis-CI [recipe](.travis.yml) that fullfills the [Release Workflow](docs/img/git-release-workflow.svg).
+The osparc-simcore repository provides a Travis-CI [recipe](.travis.yml) and Gihub-CI [workflows](.github/workflows) that fullfills the [Release Workflow](docs/img/git-release-workflow.svg) and the [Hotfix Workflow](docs/img/git-hotfix-workflow.svg).
 
-To this end Travis should be enabled for the repository in each fork and a [Dockerhub](https://hub.docker.com/) account is recommended to push the docker images generated by Travis.
+### Configuration
+Github Actions and/or Travis-CI should be enabled for the repository in each fork and a [Dockerhub](https://hub.docker.com/) account is recommended to push the docker images generated by the CI.
 
-Configuring your travis settings will speed up the travis CI process by making use of Dockerhub (or another docker registry) to move docker images between stages.
+Configuring your CI settings will speed up the CI process by making use of Dockerhub (or another docker registry) to move docker images between stages.
 
-### Travis configuration
+### Secrets configuration
 
 Define the following secure environment variables in your fork:
 
 ```bash
 DOCKER_REGISTRY # this shall be set to your own dockerhub repository account for example: itisfoundation
-DOCKER_USERNAME # the docker username (!beware this should be a [secure env variable](https://docs.travis-ci.com/user/environment-variables/#defining-encrypted-variables-in-travisyml))
-DOCKER_PASSWORD # the docker password (!beware this should be a [secure env variable](https://docs.travis-ci.com/user/environment-variables/#defining-encrypted-variables-in-travisyml))
+DOCKER_USERNAME # the docker username
+DOCKER_PASSWORD # the docker password
 ```
-
-### Release workflow
-
-![Git release workflow](../docs/img/git-release-workflow.svg)

ci/deploy/dockerhub-deploy.bash

@@ -1,35 +1,40 @@
-#!/bin/bash
-# http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -euo pipefail
+#!/usr/bin/env bash
+# strict mode
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
-bash ci/helpers/dockerhub_login.bash
+my_dir="$(dirname "$0")"
+source "$my_dir/../../scripts/helpers/logger.bash"
 
 # check script needed variables
 if [ ! -v TAG_PREFIX ]; then
-    echo "## ERROR: incorrect use of script. TAG_PREFIX (e.g. master, staging) not defined!"
-    exit 1
+    error_exit "$LINENO" "incorrect use of script. TAG_PREFIX (e.g. master, staging) not defined!"
 fi
 
+log_info "logging in dockerhub..."
+bash ci/helpers/dockerhub_login.bash
+
 # pull the current tested build
+
 DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)
 export DOCKER_IMAGE_TAG
+log_info "pulling build ${DOCKER_IMAGE_TAG}"
 make pull-version tag-local
 
 # show current images on system
-echo "## Before push"
+log_info "Before push"
 make info-images
 
 # re-tag build
 DOCKER_IMAGE_TAG="$TAG_PREFIX-latest"
 export DOCKER_IMAGE_TAG
+log_info "pushing images ${DOCKER_IMAGE_TAG} to ${DOCKER_REGISTRY}"
 make push-version
 
 # re-tag build to master-github-DATE.GIT_SHA
 DOCKER_IMAGE_TAG=$TAG_PREFIX-$(date --utc +"%Y-%m-%d--%H-%M").$(git rev-parse HEAD)
 export DOCKER_IMAGE_TAG
+log_info "pushing images ${DOCKER_IMAGE_TAG} to ${DOCKER_REGISTRY}"
 make push-version
-
-# show update of images on system
-echo "## After push"
-make info-images

ci/deploy/dockerhub-tag-version.bash

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# strict mode
+set -o errexit   # abort on nonzero exitstatus
+set -o nounset   # abort on unbound variable
+set -o pipefail  # don't hide errors within pipes
+IFS=$'\n\t'
+
+my_dir="$(dirname "$0")"
+source "$my_dir/../../scripts/helpers/logger.bash"
+
+# check script needed variables
+if [ ! -v FROM_TAG_PREFIX ] || [ ! -v TO_TAG_PREFIX ] || [ ! -v GIT_TAG ]; then
+    error_exit "$LINENO" "incorrect use of script. FROM_TAG_PREFIX/TO_TAG_PREFIX (e.g. master, staging), and/or GIT_TAG not defined!"
+fi
+
+log_info "logging in dockerhub..."
+bash ci/helpers/dockerhub_login.bash
+
+log_info "finding the version in the docker hub registry..."
+# find and pull the tagged build
+# find the docker image tag
+export ORG=${DOCKER_REGISTRY}
+export REPO="webserver"
+# FROM_TAG_PREFIX-DATE.GIT_SHA
+export TAG_PATTERN="^${FROM_TAG_PREFIX}-.+\..+"
+DOCKER_IMAGE_TAG=$(./ci/helpers/find_docker_image_tag_from_git_sha.bash | awk 'END{print}') || exit $?
+log_info "found image ${DOCKER_IMAGE_TAG}"
+export DOCKER_IMAGE_TAG
+log_info "pulling images ${DOCKER_IMAGE_TAG} from ${DOCKER_REGISTRY}"
+make pull-version tag-local
+
+# show current images on system
+log_info "Before push"
+make info-images
+
+# re-tag images to ${TO_TAG_PREFIX}-{GIT_TAG}-DATE.GIT_SHA
+readonly GIT_COMMIT_SHA=$(git show-ref -s "${GIT_TAG}")
+DOCKER_IMAGE_TAG="${TO_TAG_PREFIX}-${GIT_TAG}"-$(date --utc +"%Y-%m-%d--%H-%M")."${GIT_COMMIT_SHA}"
+export DOCKER_IMAGE_TAG
+log_info "pushing images ${DOCKER_IMAGE_TAG} to ${DOCKER_REGISTRY}"
+make push-version
+
+# push latest image ${TO_TAG_PREFIX}-latest
+DOCKER_IMAGE_TAG="${TO_TAG_PREFIX}-latest"
+export DOCKER_IMAGE_TAG
+log_info "pushing images ${DOCKER_IMAGE_TAG} to ${DOCKER_REGISTRY}"
+make push-version
+
+log_info "complete!"