[Gitlab] Fixed merge requests and issue resync methods to use an asyn…

…c method of listing root groups (#917)

integrations/gitlab/CHANGELOG.md

@@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 <!-- towncrier release notes start -->
 
+0.1.107 (2024-08-19)
+====================
+
+### Bug Fixes 
+
+- Fixed merge requests and issue resync methods to use an async method of listing root groups to avoid blocking the event loop
+
+
 0.1.106 (2024-08-19)
 ====================
 

integrations/gitlab/gitlab_integration/gitlab_service.py

@@ -354,6 +354,25 @@ async def get_all_groups(self) -> typing.AsyncIterator[List[Group]]:
             )
             yield groups
 
+    async def get_all_root_groups(self) -> typing.AsyncIterator[List[Group]]:
+        logger.info("fetching all root groups for the token")
+
+        def is_root_group(group: Group) -> bool:
+            return group.parent_id is None
+
+        async for groups_batch in AsyncFetcher.fetch_batch(
+            fetch_func=self.gitlab_client.groups.list,
+            validation_func=is_root_group,
+            pagination="offset",
+            order_by="id",
+            sort="asc",
+        ):
+            groups: List[Group] = typing.cast(List[Group], groups_batch)
+            logger.info(
+                f"Queried {len(groups)} root groups {[group.path for group in groups]}"
+            )
+            yield groups
+
     async def get_all_projects(self) -> typing.AsyncIterator[List[Project]]:
         logger.info("fetching all projects for the token")
         port_app_config: GitlabPortAppConfig = typing.cast(

integrations/gitlab/gitlab_integration/ocean.py

@@ -192,21 +192,29 @@ async def resync_merge_requests(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
     updated_after = datetime.now() - timedelta(days=14)
 
     for service in get_cached_all_services():
-        for group in service.get_root_groups():
-            async for merge_request_batch in service.get_opened_merge_requests(group):
-                yield [merge_request.asdict() for merge_request in merge_request_batch]
-            async for merge_request_batch in service.get_closed_merge_requests(
-                group, updated_after
-            ):
-                yield [merge_request.asdict() for merge_request in merge_request_batch]
+        async for groups_batch in service.get_all_root_groups():
+            for group in groups_batch:
+                async for merge_request_batch in service.get_opened_merge_requests(
+                    group
+                ):
+                    yield [
+                        merge_request.asdict() for merge_request in merge_request_batch
+                    ]
+                async for merge_request_batch in service.get_closed_merge_requests(
+                    group, updated_after
+                ):
+                    yield [
+                        merge_request.asdict() for merge_request in merge_request_batch
+                    ]
 
 
 @ocean.on_resync(ObjectKind.ISSUE)
 async def resync_issues(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
     for service in get_cached_all_services():
-        for group in service.get_root_groups():
-            async for issues_batch in service.get_all_issues(group):
-                yield [issue.asdict() for issue in issues_batch]
+        async for groups_batch in service.get_all_root_groups():
+            for group in groups_batch:
+                async for issues_batch in service.get_all_issues(group):
+                    yield [issue.asdict() for issue in issues_batch]
 
 
 @ocean.on_resync(ObjectKind.JOB)

integrations/gitlab/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gitlab"
-version = "0.1.106"
+version = "0.1.107"
 description = "Gitlab integration for Port using Port-Ocean Framework"
 authors = ["Yair Siman-Tov <[email protected]>"]
 
@@ -10,7 +10,7 @@ aiofiles = "^0.6.0"
 python-gitlab = "^3.14.0"
 pathlib = "^1.0.1"
 jsonschema = "^4.17.3"
-port_ocean = {version = "^0.9.13", extras = ["cli"]}
+port_ocean = {version = "^0.9.14", extras = ["cli"]}
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2"