From 0c233862ca51e0d63fe3eee5167cc119f652aabe Mon Sep 17 00:00:00 2001 From: Pedro Crespo-Valero <32402063+pcrespov@users.noreply.github.com> Date: Tue, 16 Jul 2024 17:42:08 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=A8=20web-api=20`services`=20api=20res?= =?UTF-8?q?ponse=20=20includes=20manifest=20info=20(part=205)=20(#6061)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/pytest_simcore/docker_compose.py | 28 ++-- .../src/pytest_simcore/helpers/constants.py | 2 +- .../api/dependencies/services.py | 4 +- .../api/rpc/_services.py | 12 +- .../db/repositories/services.py | 9 +- .../services/director.py | 7 - .../services/manifest.py | 38 ++++- .../services/{catalog.py => services_api.py} | 55 +++++-- .../tests/unit/test_services_director.py | 13 +- .../tests/unit/test_services_manifest.py | 16 +- .../catalog/tests/unit/with_dbs/conftest.py | 39 ++++- .../with_dbs/test_api_rest_services__get.py | 1 + .../tests/unit/with_dbs/test_api_rpc.py | 74 ++++++--- .../unit/with_dbs/test_services_catalog.py | 79 ---------- .../with_dbs/test_services_services_api.py | 145 ++++++++++++++++++ 15 files changed, 356 insertions(+), 166 deletions(-) rename services/catalog/src/simcore_service_catalog/services/{catalog.py => services_api.py} (80%) delete mode 100644 services/catalog/tests/unit/with_dbs/test_services_catalog.py create mode 100644 services/catalog/tests/unit/with_dbs/test_services_services_api.py diff --git a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py index a242e819b19..aa1e2ebecee 100644 --- a/packages/pytest-simcore/src/pytest_simcore/docker_compose.py +++ b/packages/pytest-simcore/src/pytest_simcore/docker_compose.py @@ -167,20 +167,13 @@ def simcore_docker_compose( docker_compose_path.exists() for docker_compose_path in docker_compose_paths ) - compose_specs = run_docker_compose_config( + return run_docker_compose_config( project_dir=osparc_simcore_root_dir / "services", scripts_dir=osparc_simcore_scripts_dir, docker_compose_paths=docker_compose_paths, env_file_path=env_file_for_testing, destination_path=temp_folder / "simcore_docker_compose.yml", ) - # NOTE: do not add indent. Copy&Paste log into editor instead - print( - HEADER_STR.format("simcore docker-compose"), - json.dumps(compose_specs), - HEADER_STR.format("-"), - ) - return compose_specs @pytest.fixture(scope="module") @@ -203,20 +196,13 @@ def ops_docker_compose( ) assert docker_compose_path.exists() - compose_specs = run_docker_compose_config( + return run_docker_compose_config( project_dir=osparc_simcore_root_dir / "services", scripts_dir=osparc_simcore_scripts_dir, docker_compose_paths=docker_compose_path, env_file_path=env_file_for_testing, destination_path=temp_folder / "ops_docker_compose.yml", ) - # NOTE: do not add indent. Copy&Paste log into editor instead - print( - HEADER_STR.format("ops docker-compose"), - json.dumps(compose_specs), - HEADER_STR.format("-"), - ) - return compose_specs @pytest.fixture(scope="module") @@ -245,6 +231,11 @@ def core_docker_compose_file( core_services_selection, simcore_docker_compose, docker_compose_path ) + print( + HEADER_STR.format(f"{docker_compose_path}"), + json.dumps(docker_compose_path.read_text()), + HEADER_STR.format("-"), + ) return docker_compose_path @@ -281,6 +272,11 @@ def ops_docker_compose_file( ops_services_selection, ops_docker_compose, docker_compose_path ) + print( + HEADER_STR.format(f"{docker_compose_path}"), + json.dumps(docker_compose_path.read_text()), + HEADER_STR.format("-"), + ) return docker_compose_path diff --git a/packages/pytest-simcore/src/pytest_simcore/helpers/constants.py b/packages/pytest-simcore/src/pytest_simcore/helpers/constants.py index cd6edbf427f..5d517b9a071 100644 --- a/packages/pytest-simcore/src/pytest_simcore/helpers/constants.py +++ b/packages/pytest-simcore/src/pytest_simcore/helpers/constants.py @@ -7,4 +7,4 @@ # string templates -HEADER_STR: str = "{:-^50}\n" +HEADER_STR: str = "{:-^100}\n" diff --git a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py index 7855bc97ad5..7d55dab5f59 100644 --- a/services/catalog/src/simcore_service_catalog/api/dependencies/services.py +++ b/services/catalog/src/simcore_service_catalog/api/dependencies/services.py @@ -93,8 +93,8 @@ async def get_service_from_manifest( """ try: return await manifest.get_service( - service_key=service_key, - service_version=service_version, + key=service_key, + version=service_version, director_client=director_client, ) diff --git a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py index 61623d05ee6..20df47ad97a 100644 --- a/services/catalog/src/simcore_service_catalog/api/rpc/_services.py +++ b/services/catalog/src/simcore_service_catalog/api/rpc/_services.py @@ -19,7 +19,8 @@ ) from ...db.repositories.services import ServicesRepository -from ...services import catalog +from ...services import services_api +from ..dependencies.director import get_director_api _logger = logging.getLogger(__name__) @@ -38,8 +39,9 @@ async def list_services_paginated( ) -> PageRpcServicesGetV2: assert app.state.engine # nosec - total_count, items = await catalog.list_services_paginated( + total_count, items = await services_api.list_services_paginated( repo=ServicesRepository(app.state.engine), + director_api=get_director_api(app), product_name=product_name, user_id=user_id, limit=limit, @@ -69,8 +71,9 @@ async def get_service( ) -> ServiceGetV2: assert app.state.engine # nosec - service = await catalog.get_service( + service = await services_api.get_service( repo=ServicesRepository(app.state.engine), + director_api=get_director_api(app), product_name=product_name, user_id=user_id, service_key=service_key, @@ -98,8 +101,9 @@ async def update_service( assert app.state.engine # nosec - service = await catalog.update_service( + service = await services_api.update_service( repo=ServicesRepository(app.state.engine), + director_api=get_director_api(app), product_name=product_name, user_id=user_id, service_key=service_key, diff --git a/services/catalog/src/simcore_service_catalog/db/repositories/services.py b/services/catalog/src/simcore_service_catalog/db/repositories/services.py index b9ad6c745df..9b787e3741b 100644 --- a/services/catalog/src/simcore_service_catalog/db/repositories/services.py +++ b/services/catalog/src/simcore_service_catalog/db/repositories/services.py @@ -432,12 +432,9 @@ async def list_services_access_rights( ) async with self.db_engine.connect() as conn: async for row in await conn.stream(query): - service_to_access_rights[ - ( - row[services_access_rights.c.key], - row[services_access_rights.c.version], - ) - ].append(ServiceAccessRightsAtDB.from_orm(row)) + service_to_access_rights[(row.key, row.version)].append( + ServiceAccessRightsAtDB.from_orm(row) + ) return service_to_access_rights async def upsert_service_access_rights( diff --git a/services/catalog/src/simcore_service_catalog/services/director.py b/services/catalog/src/simcore_service_catalog/services/director.py index 1f40b8ea3ec..bde5ce8a5f3 100644 --- a/services/catalog/src/simcore_service_catalog/services/director.py +++ b/services/catalog/src/simcore_service_catalog/services/director.py @@ -11,7 +11,6 @@ from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion from models_library.utils.json_serialization import json_dumps -from pydantic import parse_obj_as from servicelib.logging_utils import log_context from starlette import status from tenacity._asyncio import AsyncRetrying @@ -140,12 +139,6 @@ async def is_responsive(self) -> bool: except (httpx.HTTPStatusError, httpx.RequestError, httpx.TimeoutException): return False - async def list_all_services(self) -> list[ServiceMetaDataPublished]: - # WARNING: this function probably raise ValidationError since director does NOT offer guarantees. - # SEE list_registered_services - data = await self.get("/services") - return parse_obj_as(list[ServiceMetaDataPublished], data) - async def get_service( self, service_key: ServiceKey, service_version: ServiceVersion ) -> ServiceMetaDataPublished: diff --git a/services/catalog/src/simcore_service_catalog/services/manifest.py b/services/catalog/src/simcore_service_catalog/services/manifest.py index 33ab9bfe657..59149111780 100644 --- a/services/catalog/src/simcore_service_catalog/services/manifest.py +++ b/services/catalog/src/simcore_service_catalog/services/manifest.py @@ -27,11 +27,14 @@ import logging from typing import Any, TypeAlias, cast +from aiocache import cached from models_library.function_services_catalog.api import iter_service_docker_data from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion from pydantic import ValidationError +from servicelib.utils import limited_gather +from .._constants import DIRECTOR_CACHING_TTL from .director import DirectorApi from .function_services import get_function_service, is_function_service @@ -80,18 +83,43 @@ async def get_services_map( return services +@cached( + ttl=DIRECTOR_CACHING_TTL, + namespace=__name__, + key_builder=lambda f, *ag, **kw: f"{f.__name__}/{kw['key']}/{kw['version']}", +) async def get_service( - service_key: ServiceKey, - service_version: ServiceVersion, director_client: DirectorApi, + *, + key: ServiceKey, + version: ServiceVersion, ) -> ServiceMetaDataPublished: """ Retrieves service metadata from the docker registry via the director and accounting + + raises if does not exist or if validation fails """ - if is_function_service(service_key): - service = get_function_service(key=service_key, version=service_version) + if is_function_service(key): + service = get_function_service(key=key, version=version) else: service = await director_client.get_service( - service_key=service_key, service_version=service_version + service_key=key, service_version=version ) return service + + +async def get_batch_services( + selection: list[tuple[ServiceKey, ServiceVersion]], + director_client: DirectorApi, +) -> list[ServiceMetaDataPublished | BaseException]: + + batch: list[ServiceMetaDataPublished | BaseException] = await limited_gather( + *( + get_service(key=k, version=v, director_client=director_client) + for k, v in selection + ), + reraise=False, + log=_logger, + tasks_group_prefix="manifest.get_batch_services", + ) + return batch diff --git a/services/catalog/src/simcore_service_catalog/services/catalog.py b/services/catalog/src/simcore_service_catalog/services/services_api.py similarity index 80% rename from services/catalog/src/simcore_service_catalog/services/catalog.py rename to services/catalog/src/simcore_service_catalog/services/services_api.py index 5a8b044d47d..28fd0ebe347 100644 --- a/services/catalog/src/simcore_service_catalog/services/catalog.py +++ b/services/catalog/src/simcore_service_catalog/services/services_api.py @@ -7,8 +7,8 @@ ) from models_library.products import ProductName from models_library.rest_pagination import PageLimitInt -from models_library.services_authoring import Author from models_library.services_enums import ServiceType +from models_library.services_metadata_published import ServiceMetaDataPublished from models_library.services_types import ServiceKey, ServiceVersion from models_library.users import UserID from pydantic import NonNegativeInt @@ -21,6 +21,8 @@ ServiceMetaDataAtDB, ServiceWithHistoryFromDB, ) +from simcore_service_catalog.services import manifest +from simcore_service_catalog.services.director import DirectorApi from ..db.repositories.services import ServicesRepository from .function_services import is_function_service @@ -39,29 +41,33 @@ def _deduce_service_type_from(key: str) -> ServiceType: def _db_to_api_model( service_db: ServiceWithHistoryFromDB, access_rights_db: list[ServiceAccessRightsAtDB], + service_manifest: ServiceMetaDataPublished, ) -> ServiceGetV2: + assert ( + _deduce_service_type_from(service_db.key) == service_manifest.service_type + ) # nosec return ServiceGetV2( key=service_db.key, version=service_db.version, name=service_db.name, thumbnail=service_db.thumbnail or None, description=service_db.description, - version_display=f"V{service_db.version}", # rg.version_display, - type=_deduce_service_type_from(service_db.key), # rg.service_type, - contact=Author.Config.schema_extra["examples"][0]["email"], # rg.contact, - authors=Author.Config.schema_extra["examples"], + version_display=service_manifest.version_display, + type=service_manifest.service_type, + contact=service_manifest.contact, + authors=service_manifest.authors, owner=service_db.owner_email or None, - inputs={}, # rg.inputs, - outputs={}, # rg.outputs, - boot_options=None, # rg.boot_options, - min_visible_inputs=None, # rg.min_visible_inputs, + inputs=service_manifest.inputs or {}, + outputs=service_manifest.outputs or {}, + boot_options=service_manifest.boot_options, + min_visible_inputs=service_manifest.min_visible_inputs, access_rights={ a.gid: ServiceGroupAccessRightsV2.construct( execute=a.execute_access, write=a.write_access, ) for a in access_rights_db - }, # db.access_rights, + }, classifiers=service_db.classifiers, quality=service_db.quality, history=[h.to_api_model() for h in service_db.history], @@ -70,6 +76,7 @@ def _db_to_api_model( async def list_services_paginated( repo: ServicesRepository, + director_api: DirectorApi, product_name: ProductName, user_id: UserID, limit: PageLimitInt | None, @@ -94,11 +101,23 @@ async def list_services_paginated( product_name=product_name, ) + # get manifest of those with access rights + got = await manifest.get_batch_services( + [(s.key, s.version) for s in services if access_rights.get((s.key, s.version))], + director_api, + ) + service_manifest = { + (s.key, s.version): s for s in got if isinstance(s, ServiceMetaDataPublished) + } + # NOTE: aggregates published (i.e. not editable) is still missing in this version items = [ - _db_to_api_model(s, ar) + _db_to_api_model(s, ar, sm) for s in services - if (ar := access_rights.get((s.key, s.version))) + if ( + (ar := access_rights.get((s.key, s.version))) + and (sm := service_manifest.get((s.key, s.version))) + ) ] return total_count, items @@ -106,7 +125,7 @@ async def list_services_paginated( async def get_service( repo: ServicesRepository, - # image_registry, + director_api: DirectorApi, product_name: ProductName, user_id: UserID, service_key: ServiceKey, @@ -142,11 +161,18 @@ async def get_service( product_name=product_name, ) - return _db_to_api_model(service, access_rights) + service_manifest = await manifest.get_service( + key=service_key, + version=service_version, + director_client=director_api, + ) + + return _db_to_api_model(service, access_rights, service_manifest) async def update_service( repo: ServicesRepository, + director_api: DirectorApi, *, product_name: ProductName, user_id: UserID, @@ -229,6 +255,7 @@ async def update_service( return await get_service( repo=repo, + director_api=director_api, product_name=product_name, user_id=user_id, service_key=service_key, diff --git a/services/catalog/tests/unit/test_services_director.py b/services/catalog/tests/unit/test_services_director.py index b809e7a8315..7458ad6f7ad 100644 --- a/services/catalog/tests/unit/test_services_director.py +++ b/services/catalog/tests/unit/test_services_director.py @@ -11,6 +11,7 @@ import pytest from fastapi import FastAPI +from models_library.services_metadata_published import ServiceMetaDataPublished from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict from respx.router import MockRouter @@ -35,6 +36,7 @@ def app_environment( async def test_director_client_high_level_api( background_tasks_setup_disabled: None, rabbitmq_and_rpc_setup_disabled: None, + expected_director_list_services: list[dict[str, Any]], mocked_director_service_api: MockRouter, app: FastAPI, ): @@ -47,20 +49,13 @@ async def test_director_client_high_level_api( # PING assert await director_api.is_responsive() - # LIST - all_services = await director_api.list_all_services() - assert mocked_director_service_api["list_services"].called - - services_image_digest = {service.image_digest for service in all_services} - assert None not in services_image_digest - assert len(services_image_digest) == len(all_services) - # GET - expected_service = all_services[0] + expected_service = ServiceMetaDataPublished(**expected_director_list_services[0]) assert ( await director_api.get_service(expected_service.key, expected_service.version) == expected_service ) + # TODO: error handling! async def test_director_client_low_level_api( diff --git a/services/catalog/tests/unit/test_services_manifest.py b/services/catalog/tests/unit/test_services_manifest.py index 4a6fcbdd025..a43d82d5220 100644 --- a/services/catalog/tests/unit/test_services_manifest.py +++ b/services/catalog/tests/unit/test_services_manifest.py @@ -7,6 +7,7 @@ import pytest +import toolz from fastapi import FastAPI from models_library.function_services_catalog.api import is_function_service from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict @@ -59,9 +60,22 @@ async def test_services_manifest_api( # GET for expected_service in all_services_map.values(): service = await manifest.get_service( - expected_service.key, expected_service.version, director_api + key=expected_service.key, + version=expected_service.version, + director_client=director_api, ) assert service == expected_service if not is_function_service(service.key): assert mocked_director_service_api["get_service"].called + + # BATCH + for expected_services in toolz.partition(2, all_services_map.values()): + selection = [(s.key, s.version) for s in expected_services] + got_services = await manifest.get_batch_services(selection, director_api) + + assert [(s.key, s.version) for s in got_services] == selection + + # NOTE: simplier to visualize + for got, expected in zip(got_services, expected_services, strict=True): + assert got == expected diff --git a/services/catalog/tests/unit/with_dbs/conftest.py b/services/catalog/tests/unit/with_dbs/conftest.py index f720e89bcdc..0d4aea3d4e2 100644 --- a/services/catalog/tests/unit/with_dbs/conftest.py +++ b/services/catalog/tests/unit/with_dbs/conftest.py @@ -14,10 +14,11 @@ import pytest import sqlalchemy as sa from faker import Faker +from fastapi.encoders import jsonable_encoder from models_library.products import ProductName from models_library.services import ServiceMetaDataPublished from models_library.users import UserID -from pydantic import parse_obj_as +from pydantic import Extra, parse_obj_as from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.postgres_tools import ( PostgresTestConfig, @@ -447,3 +448,39 @@ def _fake_factory( return tuple(fakes) return _fake_factory + + +@pytest.fixture +def create_director_list_services_from() -> Callable[ + [list[dict[str, Any]], list], list[dict[str, Any]] +]: + """Convenience function to merge outputs of + - `create_fake_service_data` callable with those of + - `expected_director_list_services` fixture + + to produce a new expected_director_list_services + """ + + class _Loader(ServiceMetaDataPublished): + class Config: + extra = Extra.ignore + allow_population_by_field_name = True + + def _( + expected_director_list_services: list[dict[str, Any]], + fake_services_data: list, + ): + return [ + jsonable_encoder( + _Loader.parse_obj( + { + **next(itertools.cycle(expected_director_list_services)), + **data[0], # service, **access_rights = data + } + ), + exclude_unset=True, + ) + for data in fake_services_data + ] + + return _ diff --git a/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py b/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py index aa6e569fff8..a3c85d3f31b 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rest_services__get.py @@ -65,6 +65,7 @@ async def expected_service( def test_get_service_with_details( + service_caching_disabled: None, background_tasks_setup_disabled: None, rabbitmq_and_rpc_setup_disabled: None, mocked_director_service_api: respx.MockRouter, diff --git a/services/catalog/tests/unit/with_dbs/test_api_rpc.py b/services/catalog/tests/unit/with_dbs/test_api_rpc.py index 09e2678b872..817b0efc573 100644 --- a/services/catalog/tests/unit/with_dbs/test_api_rpc.py +++ b/services/catalog/tests/unit/with_dbs/test_api_rpc.py @@ -5,6 +5,7 @@ from collections.abc import Callable +from typing import Any import pytest from fastapi import FastAPI @@ -14,6 +15,7 @@ from pydantic import ValidationError from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict from pytest_simcore.helpers.typing_env import EnvVarsDict +from respx.router import MockRouter from servicelib.rabbitmq import RabbitMQRPCClient from servicelib.rabbitmq.rpc_interfaces.catalog.errors import CatalogItemNotFoundError from servicelib.rabbitmq.rpc_interfaces.catalog.services import ( @@ -43,35 +45,65 @@ def app_environment( @pytest.fixture -async def fake_services_inserted_in_db( +def num_services() -> int: + return 5 + + +@pytest.fixture +def num_versions_per_service() -> int: + return 20 + + +@pytest.fixture +def fake_data_for_services( target_product: ProductName, create_fake_service_data: Callable, + num_services: int, + num_versions_per_service: int, +) -> list: + return [ + create_fake_service_data( + f"simcore/services/comp/test-api-rpc-service-{n}", + f"{v}.0.0", + team_access=None, + everyone_access=None, + product=target_product, + ) + for n in range(num_services) + for v in range(num_versions_per_service) + ] + + +@pytest.fixture +def expected_director_list_services( + expected_director_list_services: list[dict[str, Any]], + fake_data_for_services: list, + create_director_list_services_from: Callable, +) -> list[dict[str, Any]]: + # OVERRIDES: Changes the values returned by the mocked_director_service_api + + return create_director_list_services_from( + expected_director_list_services, fake_data_for_services + ) + + +@pytest.fixture +async def background_sync_task_mocked( + background_tasks_setup_disabled: None, services_db_tables_injector: Callable, + fake_data_for_services: list, ) -> None: - num_services = 5 - num_versions_per_service = 20 - await services_db_tables_injector( - [ - create_fake_service_data( - f"simcore/services/dynamic/some-service-{n}", - f"{v}.0.0", - team_access=None, - everyone_access=None, - product=target_product, - ) - for n in range(num_services) - for v in range(num_versions_per_service) - ] - ) + # inject db services (typically done by the sync background task) + await services_db_tables_injector(fake_data_for_services) async def test_rpc_catalog_client( - director_setup_disabled: None, - fake_services_inserted_in_db: None, - app: FastAPI, + background_sync_task_mocked: None, + mocked_director_service_api: MockRouter, rpc_client: RabbitMQRPCClient, product_name: ProductName, user_id: UserID, + app: FastAPI, ): assert app @@ -136,8 +168,8 @@ async def test_rpc_catalog_client( async def test_rpc_service_not_found_error( - director_setup_disabled: None, - fake_services_inserted_in_db: None, + background_sync_task_mocked: None, + mocked_director_service_api: MockRouter, app: FastAPI, rpc_client: RabbitMQRPCClient, product_name: ProductName, diff --git a/services/catalog/tests/unit/with_dbs/test_services_catalog.py b/services/catalog/tests/unit/with_dbs/test_services_catalog.py deleted file mode 100644 index f828b300d56..00000000000 --- a/services/catalog/tests/unit/with_dbs/test_services_catalog.py +++ /dev/null @@ -1,79 +0,0 @@ -# pylint: disable=redefined-outer-name -# pylint: disable=unused-argument -# pylint: disable=unused-variable - -from collections.abc import Callable - -import pytest -from models_library.products import ProductName -from models_library.users import UserID -from simcore_service_catalog.db.repositories.services import ServicesRepository -from simcore_service_catalog.services import catalog -from sqlalchemy.ext.asyncio import AsyncEngine - -pytest_simcore_core_services_selection = [ - "postgres", -] -pytest_simcore_ops_services_selection = [ - "adminer", -] - - -@pytest.fixture -def services_repo(sqlalchemy_async_engine: AsyncEngine): - return ServicesRepository(sqlalchemy_async_engine) - - -async def test_list_services_paginated( - target_product: ProductName, - create_fake_service_data: Callable, - services_db_tables_injector: Callable, - services_repo: ServicesRepository, - user_id: UserID, -): - # inject services - num_services = 5 - num_versions_per_service = 20 - await services_db_tables_injector( - [ - create_fake_service_data( - f"simcore/services/dynamic/some-service-{n}", - f"{v}.0.0", - team_access=None, - everyone_access=None, - product=target_product, - ) - for n in range(num_services) - for v in range(num_versions_per_service) - ] - ) - - limit = 2 - assert limit < num_services - offset = 1 - - total_count, page_items = await catalog.list_services_paginated( - services_repo, - product_name=target_product, - user_id=user_id, - limit=limit, - offset=offset, - ) - - assert total_count == num_services - assert len(page_items) <= limit - - for item in page_items: - assert item.access_rights - assert item.owner is not None - assert item.history[0].version == item.version - - got = await catalog.get_service( - services_repo, - product_name=target_product, - user_id=user_id, - service_key=item.key, - service_version=item.version, - ) - - assert got == item diff --git a/services/catalog/tests/unit/with_dbs/test_services_services_api.py b/services/catalog/tests/unit/with_dbs/test_services_services_api.py new file mode 100644 index 00000000000..63f0fd06e56 --- /dev/null +++ b/services/catalog/tests/unit/with_dbs/test_services_services_api.py @@ -0,0 +1,145 @@ +# pylint: disable=redefined-outer-name +# pylint: disable=unused-argument +# pylint: disable=unused-variable + +from collections.abc import Callable +from typing import Any + +import pytest +from fastapi import FastAPI +from models_library.products import ProductName +from models_library.users import UserID +from respx.router import MockRouter +from simcore_service_catalog.api.dependencies.director import get_director_api +from simcore_service_catalog.db.repositories.services import ServicesRepository +from simcore_service_catalog.services import manifest, services_api +from simcore_service_catalog.services.director import DirectorApi +from sqlalchemy.ext.asyncio import AsyncEngine + +pytest_simcore_core_services_selection = [ + "postgres", +] +pytest_simcore_ops_services_selection = [ + "adminer", +] + + +@pytest.fixture +def services_repo(sqlalchemy_async_engine: AsyncEngine): + return ServicesRepository(sqlalchemy_async_engine) + + +@pytest.fixture +def num_services() -> int: + return 5 + + +@pytest.fixture +def num_versions_per_service() -> int: + return 20 + + +@pytest.fixture +def fake_services_data( + target_product: ProductName, + create_fake_service_data: Callable, + num_services: int, + num_versions_per_service: int, +) -> list: + return [ + create_fake_service_data( + f"simcore/services/comp/some-service-{n}", + f"{v}.0.0", + team_access=None, + everyone_access=None, + product=target_product, + ) + for n in range(num_services) + for v in range(num_versions_per_service) + ] + + +@pytest.fixture +def expected_director_list_services( + expected_director_list_services: list[dict[str, Any]], + fake_services_data: list, + create_director_list_services_from: Callable, +) -> list[dict[str, Any]]: + # OVERRIDES: Changes the values returned by the mocked_director_service_api + + return create_director_list_services_from( + expected_director_list_services, fake_services_data + ) + + +@pytest.fixture +async def background_sync_task_mocked( + background_tasks_setup_disabled: None, + services_db_tables_injector: Callable, + fake_services_data: list, +) -> None: + # inject db services (typically done by the sync background task) + await services_db_tables_injector(fake_services_data) + + +@pytest.fixture +async def director_client(app: FastAPI) -> DirectorApi: + director_api = get_director_api(app) + + # ensures manifest API cache is reset + assert hasattr(manifest.get_service, "cache") + assert manifest.get_service.cache.clear() + + return director_api + + +async def test_list_services_paginated( + background_sync_task_mocked: None, + rabbitmq_and_rpc_setup_disabled: None, + mocked_director_service_api: MockRouter, + target_product: ProductName, + services_repo: ServicesRepository, + user_id: UserID, + director_client: DirectorApi, + num_services: int, +): + + offset = 1 + limit = 2 + assert limit < num_services + + assert not mocked_director_service_api["get_service"].called + + total_count, page_items = await services_api.list_services_paginated( + services_repo, + director_client, + product_name=target_product, + user_id=user_id, + limit=limit, + offset=offset, + ) + + assert total_count == num_services + assert page_items + assert len(page_items) <= limit + assert mocked_director_service_api["get_service"].called + assert mocked_director_service_api["get_service"].call_count == limit + + for item in page_items: + assert item.access_rights + assert item.owner is not None + assert item.history[0].version == item.version + + got = await services_api.get_service( + services_repo, + director_client, + product_name=target_product, + user_id=user_id, + service_key=item.key, + service_version=item.version, + ) + + assert got == item + + # since it is cached, it should only call it `limit` times + assert mocked_director_service_api["get_service"].call_count == limit