Is1877/allow constraints on director services (#1878)

* added DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS variable
ITISFoundation · Oct 14, 2020 · a9cf515 · a9cf515
1 parent 217aac9
commit a9cf515
Show file tree

Hide file tree

Showing 12 changed files with 85 additions and 56 deletions.
diff --git a/.github/workflows/ci-release.yml b/.github/workflows/ci-release.yml
@@ -24,17 +24,17 @@ jobs:
       - name: set target variable
         run: |
           target=$(git name-rev --refs="refs/remotes/origin/*" --name-only ${GITHUB_SHA})
-          echo ::set-env name=TARGET::${target}
+          echo "TARGET=${target}" >> $GITHUB_ENV
       - name: setup docker
         run: |
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: set owner variable
-        run: echo ::set-env name=OWNER::${GITHUB_REPOSITORY%/*}
+        run: echo "OWNER=${GITHUB_REPOSITORY%/*}" >> $GITHUB_ENV
       - name: set git tag
-        run: echo ::set-env name=GIT_TAG::${GITHUB_REF##*/}
+        run: echo "GIT_TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV
       - if: env.TARGET == 'remotes/origin/master'
         env:
           FROM_TAG_PREFIX: staging-github

diff --git a/.github/workflows/ci-staging.yml b/.github/workflows/ci-staging.yml
@@ -26,10 +26,10 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: set owner variable
-        run: echo ::set-env name=OWNER::${GITHUB_REPOSITORY%/*}
+        run: echo "OWNER=${GITHUB_REPOSITORY%/*}" >> $GITHUB_ENV
       - name: set git tag
-        run: echo ::set-env name=GIT_TAG::${GITHUB_REF##*/}
+        run: echo "GIT_TAG=${GITHUB_REF##*/}" >> $GITHUB_ENV
       - name: deploy
         run: ./ci/deploy/dockerhub-tag-version.bash
diff --git a/.github/workflows/ci-testing-deploy.yml b/.github/workflows/ci-testing-deploy.yml
@@ -49,7 +49,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -85,7 +85,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -134,7 +134,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -189,7 +189,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -238,7 +238,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -287,7 +287,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - uses: actions/setup-node@v1
         with:
           node-version: ${{ matrix.node }}
@@ -317,7 +317,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -353,7 +353,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -402,7 +402,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -451,7 +451,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -500,7 +500,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -549,7 +549,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -599,7 +599,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -649,7 +649,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -699,7 +699,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -749,7 +749,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -795,7 +795,7 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: show system environs
         if: github.event_name == 'push'
         run: ./ci/helpers/show_system_versions.bash
@@ -807,7 +807,7 @@ jobs:
         run: ./ci/build/test-images.bash build_images
       - name: set owner variable
         if: github.event_name == 'push'
-        run: echo ::set-env name=OWNER::${GITHUB_REPOSITORY%/*}
+        run: echo "OWNER=${GITHUB_REPOSITORY%/*}" >> $GITHUB_ENV
       - name: push images
         # only pushes have access to the docker credentials
         if: github.event_name == 'push'
@@ -828,14 +828,14 @@ jobs:
         if: github.event_name == 'pull_request'
         run: |
           export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
-          echo ::set-env name=DOCKER_REGISTRY::${TMP_DOCKER_REGISTRY,,}
+          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
       - uses: actions/checkout@v2
       - name: setup docker
         run: |
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -893,14 +893,14 @@ jobs:
         if: github.event_name == 'pull_request'
         run: |
           export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
-          echo ::set-env name=DOCKER_REGISTRY::${TMP_DOCKER_REGISTRY,,}
+          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
       - uses: actions/checkout@v2
       - name: setup docker
         run: |
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -958,14 +958,14 @@ jobs:
         if: github.event_name == 'pull_request'
         run: |
           export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
-          echo ::set-env name=DOCKER_REGISTRY::${TMP_DOCKER_REGISTRY,,}
+          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
       - uses: actions/checkout@v2
       - name: setup docker
         run: |
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -1023,14 +1023,14 @@ jobs:
         if: github.event_name == 'pull_request'
         run: |
           export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
-          echo ::set-env name=DOCKER_REGISTRY::${TMP_DOCKER_REGISTRY,,}
+          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
       - uses: actions/checkout@v2
       - name: setup docker
         run: |
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -1072,14 +1072,14 @@ jobs:
         if: github.event_name == 'pull_request'
         run: |
           export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
-          echo ::set-env name=DOCKER_REGISTRY::${TMP_DOCKER_REGISTRY,,}
+          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
       - uses: actions/checkout@v2
       - name: setup docker
         run: |
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: setup python environment
         uses: actions/setup-python@v2
         with:
@@ -1152,7 +1152,7 @@ jobs:
         if: github.event_name == 'pull_request'
         run: |
           export TMP_DOCKER_REGISTRY=${GITHUB_REPOSITORY%/*}
-          echo ::set-env name=DOCKER_REGISTRY::${TMP_DOCKER_REGISTRY,,}
+          echo "DOCKER_REGISTRY=${TMP_DOCKER_REGISTRY,,}" >> $GITHUB_ENV
       - uses: actions/checkout@v2
       - name: setup python environment
         uses: actions/setup-python@v2
@@ -1307,9 +1307,9 @@ jobs:
           sudo ./ci/github/helpers/setup_docker_compose.bash
           ./ci/github/helpers/setup_docker_experimental.bash
           ./ci/github/helpers/setup_docker_buildx.bash
-          echo ::set-env name=DOCKER_BUILDX::1
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
       - name: set owner variable
-        run: echo ::set-env name=OWNER::${GITHUB_REPOSITORY%/*}
+        run: echo "OWNER=${GITHUB_REPOSITORY%/*}" >> $GITHUB_ENV
       - name: deploy master
         if: github.ref == 'refs/heads/master'
         env:

diff --git a/packages/pytest-simcore/src/pytest_simcore/postgres_service.py b/packages/pytest-simcore/src/pytest_simcore/postgres_service.py
@@ -43,7 +43,7 @@ def create_template_db(postgres_dsn: Dict, postgres_engine: sa.engine.Engine) ->
     queries = [
         # disconnect existing users
         f"""
-        SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity 
+        SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity
         WHERE pg_stat_activity.datname = '{postgres_dsn["database"]}' AND pid <> pg_backend_pid();
         """,
         # drop template database
@@ -116,7 +116,7 @@ def database_from_template_before_each_function(
     queries = [
         # terminate existing connections to the database
         f"""
-        SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity 
+        SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity
         WHERE pg_stat_activity.datname = '{postgres_dsn["database"]}';
         """,
         # drop database
@@ -209,7 +209,7 @@ def postgres_session(postgres_db: sa.engine.Engine) -> sa.orm.session.Session:
 
 @tenacity.retry(
     wait=tenacity.wait_fixed(5),
-    stop=tenacity.stop_after_attempt(20),
+    stop=tenacity.stop_after_attempt(60),
     before_sleep=tenacity.before_sleep_log(log, logging.INFO),
     reraise=True,
 )

diff --git a/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py b/packages/pytest-simcore/src/pytest_simcore/rabbit_service.py
@@ -2,18 +2,20 @@
 # pylint:disable=unused-argument
 # pylint:disable=redefined-outer-name
 
+import logging
 import os
 import socket
 from typing import Any, Dict, Optional, Tuple
 
 import aio_pika
 import pytest
 import tenacity
-from servicelib.rabbitmq_utils import RabbitMQRetryPolicyUponInitialization
 from simcore_sdk.config.rabbit import Config
 
 from .helpers.utils_docker import get_service_published_port
 
+log = logging.getLogger(__name__)
+
 
 @pytest.fixture(scope="module")
 def rabbit_config(docker_stack: Dict, devel_environ: Dict) -> Config:
@@ -127,7 +129,12 @@ async def rabbit_queue(
 # HELPERS --
 
 
-@tenacity.retry(**RabbitMQRetryPolicyUponInitialization().kwargs)
+@tenacity.retry(
+    wait=tenacity.wait_fixed(5),
+    stop=tenacity.stop_after_attempt(60),
+    before_sleep=tenacity.before_sleep_log(log, logging.INFO),
+    reraise=True,
+)
 async def wait_till_rabbit_responsive(url: str) -> None:
     connection = await aio_pika.connect(url)
     await connection.close()
diff --git a/services/catalog/docker-compose-extra.yml b/services/catalog/docker-compose-extra.yml
@@ -45,6 +45,7 @@ services:
       - REGISTRY_SSL=${REGISTRY_SSL}
       - DIRECTOR_REGISTRY_CACHING=${DIRECTOR_REGISTRY_CACHING}
       - DIRECTOR_REGISTRY_CACHING_TTL=${DIRECTOR_REGISTRY_CACHING_TTL}
+      - DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS=${DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS}
       - DIRECTOR_SELF_SIGNED_SSL_SECRET_ID=${DIRECTOR_SELF_SIGNED_SSL_SECRET_ID}
       - DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME=${DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME}
       - DIRECTOR_SELF_SIGNED_SSL_FILENAME=${DIRECTOR_SELF_SIGNED_SSL_FILENAME}

diff --git a/services/director/src/simcore_service_director/config.py b/services/director/src/simcore_service_director/config.py
@@ -11,7 +11,8 @@
 LOGLEVEL_STR = os.environ.get("LOGLEVEL", "WARNING").upper()
 log_level = getattr(logging, LOGLEVEL_STR)
 logging.basicConfig(
-    level=log_level, format="%(levelname)s:%(name)s-%(lineno)d: %(message)s",
+    level=log_level,
+    format="%(levelname)s:%(name)s-%(lineno)d: %(message)s",
 )
 logging.root.setLevel(log_level)
 
@@ -38,6 +39,10 @@
     os.environ.get("DIRECTOR_REGISTRY_CACHING_TTL", 15 * 60)
 )
 
+DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS: str = os.environ.get(
+    "DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS", ""
+)
+
 # for passing self-signed certificate to spawned services
 DIRECTOR_SELF_SIGNED_SSL_SECRET_ID: str = os.environ.get(
     "DIRECTOR_SELF_SIGNED_SSL_SECRET_ID", ""