diff --git a/integrations/gitlab/CHANGELOG.md b/integrations/gitlab/CHANGELOG.md index 3883ef0fce..a53a603593 100644 --- a/integrations/gitlab/CHANGELOG.md +++ b/integrations/gitlab/CHANGELOG.md @@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +0.1.107 (2024-08-19) +==================== + +### Bug Fixes + +- Fixed merge requests and issue resync methods to use an async method of listing root groups to avoid blocking the event loop + + 0.1.106 (2024-08-19) ==================== diff --git a/integrations/gitlab/gitlab_integration/gitlab_service.py b/integrations/gitlab/gitlab_integration/gitlab_service.py index 326cdb0842..8b67f7580c 100644 --- a/integrations/gitlab/gitlab_integration/gitlab_service.py +++ b/integrations/gitlab/gitlab_integration/gitlab_service.py @@ -354,6 +354,25 @@ async def get_all_groups(self) -> typing.AsyncIterator[List[Group]]: ) yield groups + async def get_all_root_groups(self) -> typing.AsyncIterator[List[Group]]: + logger.info("fetching all root groups for the token") + + def is_root_group(group: Group) -> bool: + return group.parent_id is None + + async for groups_batch in AsyncFetcher.fetch_batch( + fetch_func=self.gitlab_client.groups.list, + validation_func=is_root_group, + pagination="offset", + order_by="id", + sort="asc", + ): + groups: List[Group] = typing.cast(List[Group], groups_batch) + logger.info( + f"Queried {len(groups)} root groups {[group.path for group in groups]}" + ) + yield groups + async def get_all_projects(self) -> typing.AsyncIterator[List[Project]]: logger.info("fetching all projects for the token") port_app_config: GitlabPortAppConfig = typing.cast( diff --git a/integrations/gitlab/gitlab_integration/ocean.py b/integrations/gitlab/gitlab_integration/ocean.py index d7f1bd24eb..10c4166720 100644 --- a/integrations/gitlab/gitlab_integration/ocean.py +++ b/integrations/gitlab/gitlab_integration/ocean.py @@ -192,21 +192,29 @@ async def resync_merge_requests(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: updated_after = datetime.now() - timedelta(days=14) for service in get_cached_all_services(): - for group in service.get_root_groups(): - async for merge_request_batch in service.get_opened_merge_requests(group): - yield [merge_request.asdict() for merge_request in merge_request_batch] - async for merge_request_batch in service.get_closed_merge_requests( - group, updated_after - ): - yield [merge_request.asdict() for merge_request in merge_request_batch] + async for groups_batch in service.get_all_root_groups(): + for group in groups_batch: + async for merge_request_batch in service.get_opened_merge_requests( + group + ): + yield [ + merge_request.asdict() for merge_request in merge_request_batch + ] + async for merge_request_batch in service.get_closed_merge_requests( + group, updated_after + ): + yield [ + merge_request.asdict() for merge_request in merge_request_batch + ] @ocean.on_resync(ObjectKind.ISSUE) async def resync_issues(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: for service in get_cached_all_services(): - for group in service.get_root_groups(): - async for issues_batch in service.get_all_issues(group): - yield [issue.asdict() for issue in issues_batch] + async for groups_batch in service.get_all_root_groups(): + for group in groups_batch: + async for issues_batch in service.get_all_issues(group): + yield [issue.asdict() for issue in issues_batch] @ocean.on_resync(ObjectKind.JOB) diff --git a/integrations/gitlab/poetry.lock b/integrations/gitlab/poetry.lock index a1844f3bfa..977970416c 100644 --- a/integrations/gitlab/poetry.lock +++ b/integrations/gitlab/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. [[package]] name = "aiofiles" @@ -909,13 +909,13 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "port-ocean" -version = "0.9.13" +version = "0.9.14" description = "Port Ocean is a CLI tool for managing your Port projects." optional = false python-versions = "<4.0,>=3.11" files = [ - {file = "port_ocean-0.9.13-py3-none-any.whl", hash = "sha256:602bf1d1d5d8cdb29e9292a124cc9feb6b500dd80a4f2e6e1907001682c79219"}, - {file = "port_ocean-0.9.13.tar.gz", hash = "sha256:850637e3e8b9f4a500b43a80531cf936580a6bd496b7deba490c4f336c26f94e"}, + {file = "port_ocean-0.9.14-py3-none-any.whl", hash = "sha256:8ce0946e9b23c0ca7e5e6821efdaddadde2b2ccca39dd9e376132c38f27b9ec6"}, + {file = "port_ocean-0.9.14.tar.gz", hash = "sha256:f733ce2fcbc69a843d980e3d4182774dcfb1c7800f91372e52943f0d45808678"}, ] [package.dependencies] @@ -1896,4 +1896,4 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "4583f3d9e5029b50565628069e49d5a05e12a8f86bc614613644bb9df889c80e" +content-hash = "fa19c4a66cf429f3e619f3a7d6ed395bfb76633ea02f75953a913e457bf7ac89" diff --git a/integrations/gitlab/pyproject.toml b/integrations/gitlab/pyproject.toml index 794c10f2ef..5847a3dee8 100644 --- a/integrations/gitlab/pyproject.toml +++ b/integrations/gitlab/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gitlab" -version = "0.1.106" +version = "0.1.107" description = "Gitlab integration for Port using Port-Ocean Framework" authors = ["Yair Siman-Tov "] @@ -10,7 +10,7 @@ aiofiles = "^0.6.0" python-gitlab = "^3.14.0" pathlib = "^1.0.1" jsonschema = "^4.17.3" -port_ocean = {version = "^0.9.13", extras = ["cli"]} +port_ocean = {version = "^0.9.14", extras = ["cli"]} [tool.poetry.group.dev.dependencies] pytest = "^7.2"