♻️⬆️ Migrating director service to fastapi ⚠️ 🚨

.env-devel

@@ -69,11 +69,20 @@ DASK_TLS_CA_FILE=/home/scu/.dask/dask-crt.pem
 DASK_TLS_CERT=/home/scu/.dask/dask-crt.pem
 DASK_TLS_KEY=/home/scu/.dask/dask-key.pem
 
+DIRECTOR_DEFAULT_MAX_MEMORY=2147483648
+DIRECTOR_DEFAULT_MAX_NANO_CPUS=1000000000
 DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}'
 DIRECTOR_HOST=director
-DIRECTOR_PORT=8080
+DIRECTOR_LOGLEVEL=INFO
+DIRECTOR_MONITORING_ENABLED=True
+DIRECTOR_PORT=8000
+DIRECTOR_PUBLISHED_HOST_NAME=
 DIRECTOR_REGISTRY_CACHING_TTL=900
 DIRECTOR_REGISTRY_CACHING=True
+DIRECTOR_SELF_SIGNED_SSL_FILENAME=
+DIRECTOR_SELF_SIGNED_SSL_SECRET_ID=
+DIRECTOR_SELF_SIGNED_SSL_SECRET_NAME=
+DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS=
 DIRECTOR_TRACING={}
 
 EFS_USER_ID=8006
@@ -185,6 +194,7 @@ REDIS_SECURE=false
 REDIS_USER=null
 
 REGISTRY_AUTH=True
+REGISTRY_PATH=""
 REGISTRY_PW=adminadminadmin
 REGISTRY_SSL=True
 REGISTRY_URL=registry.osparc-master.speag.com

.github/workflows/ci-testing-deploy.yml

@@ -189,6 +189,8 @@ jobs:
               - 'packages/**'
               - 'services/director/**'
               - 'services/docker-compose*'
+              - 'scripts/mypy/*'
+              - 'mypy.ini'
             director-v2:
               - 'packages/**'
               - 'services/director-v2/**'
@@ -912,11 +914,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        # KEEP 3.6 Development of this service is frozen
-        # KEEP ubuntu 20.04, else no python 3.6
-        python: [3.6]
-        os: [ubuntu-20.04]
-        docker_buildx: [v0.10.4]
+        python: ["3.11"]
+        os: [ubuntu-22.04]
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
@@ -929,12 +928,27 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python }}
+      - name: install uv
+        uses: astral-sh/setup-uv@v3
+        with:
+          version: "0.4.x"
+          enable-cache: false
+          cache-dependency-glob: "**/director/requirements/ci.txt"
       - name: show system version
         run: ./ci/helpers/show_system_versions.bash
       - name: install
         run: ./ci/github/unit-testing/director.bash install
+      - name: typecheck
+        run: ./ci/github/unit-testing/director.bash typecheck
       - name: test
+        if: ${{ !cancelled() }}
         run: ./ci/github/unit-testing/director.bash test
+      - name: upload failed tests logs
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ github.job }}_docker_logs
+          path: ./services/director/test_failures
       - uses: codecov/[email protected]
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

ci/github/unit-testing/director.bash

@@ -6,44 +6,29 @@ set -o pipefail # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {
-  # Replaces 'bash ci/helpers/ensure_python_pip.bash'
-
-  echo "INFO:" "$(python --version)" "@" "$(command -v python)"
-
-  # installs pip if not in place
-  python -m ensurepip
-
-  echo "INFO:" "$(pip --version)" "@" "$(command -v pip)"
-  # NOTE: pip<22.0 for python 3.6
-  pip3 install --upgrade \
-    pip~=21.0 \
-    wheel \
-    setuptools
-  python3 -m venv .venv
+  make devenv
   # shellcheck source=/dev/null
   source .venv/bin/activate
   pushd services/director
-  pip3 install -r requirements/ci.txt
+  make install-ci
   popd
+  uv pip list
 }
 
 test() {
   # shellcheck source=/dev/null
   source .venv/bin/activate
   pushd services/director
-  pytest \
-    --color=yes \
-    --cov-append \
-    --cov-config=.coveragerc \
-    --cov-report=term-missing \
-    --cov-report=xml \
-    --cov=simcore_service_director \
-    --durations=10 \
-    --keep-docker-up \
-    --log-date-format="%Y-%m-%d %H:%M:%S" \
-    --log-format="%(asctime)s %(levelname)s %(message)s" \
-    --verbose \
-    tests/
+  make test-ci-unit
+  popd
+}
+
+typecheck() {
+  # shellcheck source=/dev/null
+  source .venv/bin/activate
+  uv pip install mypy
+  pushd services/director
+  make mypy
   popd
 }
 

packages/models-library/src/models_library/api_schemas_director/services.py

@@ -0,0 +1,5 @@
+from ..services_metadata_published import ServiceMetaDataPublished
+
+
+class ServiceDataGet(ServiceMetaDataPublished):
+    ...

packages/notifications-library/requirements/_base.txt

@@ -16,6 +16,10 @@ attrs==24.2.0
     #   referencing
 click==8.1.7
     # via typer
+deprecated==1.2.14
+    # via
+    #   opentelemetry-api
+    #   opentelemetry-semantic-conventions
 dnspython==2.6.1
     # via email-validator
 email-validator==2.2.0
@@ -26,6 +30,8 @@ idna==3.10
     # via
     #   email-validator
     #   yarl
+importlib-metadata==8.5.0
+    # via opentelemetry-api
 jinja2==3.1.4
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -54,13 +60,28 @@ mdurl==0.1.2
     # via markdown-it-py
 multidict==6.1.0
     # via yarl
+opentelemetry-api==1.28.0
+    # via
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-asyncpg
+    #   opentelemetry-semantic-conventions
+opentelemetry-instrumentation==0.49b0
+    # via opentelemetry-instrumentation-asyncpg
+opentelemetry-instrumentation-asyncpg==0.49b0
+    # via -r requirements/../../../packages/postgres-database/requirements/_base.in
+opentelemetry-semantic-conventions==0.49b0
+    # via
+    #   opentelemetry-instrumentation
+    #   opentelemetry-instrumentation-asyncpg
 orjson==3.10.7
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/models-library/requirements/_base.in
+packaging==24.1
+    # via opentelemetry-instrumentation
 psycopg2-binary==2.9.9
     # via sqlalchemy
 pydantic==1.10.18
@@ -109,5 +130,11 @@ typing-extensions==4.12.2
     #   alembic
     #   pydantic
     #   typer
+wrapt==1.16.0
+    # via
+    #   deprecated
+    #   opentelemetry-instrumentation
 yarl==1.12.1
     # via -r requirements/../../../packages/postgres-database/requirements/_base.in
+zipp==3.20.2
+    # via importlib-metadata

packages/notifications-library/requirements/_test.in

@@ -8,6 +8,7 @@
 #
 --constraint _base.txt
 
+aiodocker
 coverage
 docker
 faker

packages/notifications-library/requirements/_test.txt

@@ -1,3 +1,17 @@
+aiodocker==0.23.0
+    # via -r requirements/_test.in
+aiohappyeyeballs==2.4.3
+    # via aiohttp
+aiohttp==3.10.10
+    # via
+    #   -c requirements/../../../requirements/constraints.txt
+    #   aiodocker
+aiosignal==1.3.1
+    # via aiohttp
+attrs==24.2.0
+    # via
+    #   -c requirements/_base.txt
+    #   aiohttp
 certifi==2024.8.30
     # via
     #   -c requirements/../../../requirements/constraints.txt
@@ -12,6 +26,10 @@ docker==7.1.0
     # via -r requirements/_test.in
 faker==29.0.0
     # via -r requirements/_test.in
+frozenlist==1.5.0
+    # via
+    #   aiohttp
+    #   aiosignal
 greenlet==3.1.1
     # via
     #   -c requirements/_base.txt
@@ -22,14 +40,21 @@ idna==3.10
     # via
     #   -c requirements/_base.txt
     #   requests
+    #   yarl
 iniconfig==2.0.0
     # via pytest
+multidict==6.1.0
+    # via
+    #   -c requirements/_base.txt
+    #   aiohttp
+    #   yarl
 mypy==1.12.0
     # via sqlalchemy
 mypy-extensions==1.0.0
     # via mypy
 packaging==24.1
     # via
+    #   -c requirements/_base.txt
     #   pytest
     #   pytest-sugar
 pluggy==1.5.0
@@ -100,3 +125,7 @@ urllib3==2.2.3
     #   -c requirements/../../../requirements/constraints.txt
     #   docker
     #   requests
+yarl==1.12.1
+    # via
+    #   -c requirements/_base.txt
+    #   aiohttp

packages/notifications-library/requirements/_tools.txt

@@ -40,6 +40,7 @@ nodeenv==1.9.1
     # via pre-commit
 packaging==24.1
     # via
+    #   -c requirements/_base.txt
     #   -c requirements/_test.txt
     #   black
     #   build

packages/pytest-simcore/src/pytest_simcore/docker.py

@@ -1,3 +1,9 @@
+# pylint:disable=protected-access
+# pylint:disable=redefined-outer-name
+# pylint:disable=too-many-arguments
+# pylint:disable=unused-argument
+# pylint:disable=unused-variable
+
 import asyncio
 import contextlib
 from collections.abc import AsyncIterator, Callable
@@ -7,6 +13,12 @@
 import pytest
 
 
+@pytest.fixture
+async def async_docker_client() -> AsyncIterator[aiodocker.Docker]:
+    async with aiodocker.Docker() as docker_client:
+        yield docker_client
+
+
 @contextlib.asynccontextmanager
 async def _pause_container(
     async_docker_client: aiodocker.Docker, container_name: str

packages/pytest-simcore/src/pytest_simcore/docker_swarm.py

@@ -10,13 +10,15 @@
 from collections.abc import Iterator
 from contextlib import suppress
 from pathlib import Path
-from typing import Any
+from typing import Any, AsyncIterator, Awaitable, Callable
 
+import aiodocker
 import docker
 import pytest
 import yaml
 from docker.errors import APIError
-from tenacity import Retrying, TryAgain, retry
+from faker import Faker
+from tenacity import AsyncRetrying, Retrying, TryAgain, retry
 from tenacity.before_sleep import before_sleep_log
 from tenacity.retry import retry_if_exception_type
 from tenacity.stop import stop_after_delay
@@ -390,3 +392,41 @@ async def _check_all_services_are_running():
                         raise _ResourceStillNotRemovedError(msg)
 
     _fetch_and_print_services(docker_client, "[AFTER REMOVED]")
+
+
+@pytest.fixture
+async def docker_network(
+    docker_swarm: None,
+    async_docker_client: aiodocker.Docker,
+    faker: Faker,
+) -> AsyncIterator[Callable[..., Awaitable[dict[str, Any]]]]:
+    networks = []
+
+    async def _network_creator(**network_config_kwargs) -> dict[str, Any]:
+        network = await async_docker_client.networks.create(
+            config={"Name": faker.uuid4(), "Driver": "overlay"} | network_config_kwargs
+        )
+        assert network
+        print(f"--> created network {network=}")
+        networks.append(network)
+        return await network.show()
+
+    yield _network_creator
+
+    # wait until all networks are really gone
+    async def _wait_for_network_deletion(network: aiodocker.docker.DockerNetwork):
+        network_name = (await network.show())["Name"]
+        await network.delete()
+        async for attempt in AsyncRetrying(
+            reraise=True, wait=wait_fixed(1), stop=stop_after_delay(60)
+        ):
+            with attempt:
+                print(f"<-- waiting for network '{network_name}' deletion...")
+                list_of_network_names = [
+                    n["Name"] for n in await async_docker_client.networks.list()
+                ]
+                assert network_name not in list_of_network_names
+            print(f"<-- network '{network_name}' deleted")
+
+    print(f"<-- removing all networks {networks=}")
+    await asyncio.gather(*[_wait_for_network_deletion(network) for network in networks])

packages/pytest-simcore/src/pytest_simcore/simcore_service_library_fixtures.py

@@ -1,4 +1,4 @@
-from typing import AsyncIterable
+from collections.abc import AsyncIterable
 
 import pytest
 from servicelib.async_utils import cancel_sequential_workers

packages/service-library/src/servicelib/async_utils.py

@@ -39,6 +39,7 @@ class QueueElement:
     output: Any | None = None
 
 
+# NOTE: If you get issues with event loop already closed error use ensure_run_in_sequence_context_is_empty fixture in your tests
 _sequential_jobs_contexts: dict[str, Context] = {}
 
 

services/autoscaling/tests/unit/conftest.py

@@ -86,6 +86,7 @@
     "pytest_simcore.aws_iam_service",
     "pytest_simcore.aws_ssm_service",
     "pytest_simcore.dask_scheduler",
+    "pytest_simcore.docker",
     "pytest_simcore.docker_compose",
     "pytest_simcore.docker_swarm",
     "pytest_simcore.environment_configs",
@@ -426,12 +427,6 @@ async def autoscaling_docker() -> AsyncIterator[AutoscalingDocker]:
         yield cast(AutoscalingDocker, docker_client)
 
 
-@pytest.fixture
-async def async_docker_client() -> AsyncIterator[aiodocker.Docker]:
-    async with aiodocker.Docker() as docker_client:
-        yield docker_client
-
-
 @pytest.fixture
 async def host_node(
     docker_swarm: None,