[Integrations][Gitlab] Add support for project folders and monorepo (#…

…194)

integrations/gitlab/CHANGELOG.md

@@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 <!-- towncrier release notes start -->
 
+0.1.23 (2023-10-30)
+===================
+
+### Features
+
+- Added support for project folders (PORT-5060)
+
+
 0.1.22 (2023-10-30)
 ===================
 

integrations/gitlab/gitlab_integration/core/paging.py

@@ -1,25 +1,76 @@
 import asyncio
 import os
-from typing import List, Union, Callable, AsyncIterator, TypeVar
+from typing import List, Union, Callable, AsyncIterator, TypeVar, Any, Dict
 
+from gitlab import GitlabList, Gitlab
 from gitlab.base import RESTObject, RESTObjectList
+from gitlab.v4.objects import Project, ProjectPipelineJob, ProjectPipeline, Issue
 from loguru import logger
 
 T = TypeVar("T", bound=RESTObject)
 
 
 class AsyncFetcher:
-    def __init__(self, gitlab_client):
+    def __init__(self, gitlab_client: Gitlab):
         self.gitlab_client = gitlab_client
 
     @staticmethod
     async def fetch(
-        fetch_func: Callable[..., Union[RESTObjectList, List[RESTObject]]],
+        fetch_func: Callable[
+            ...,
+            Union[
+                RESTObjectList,
+                List[RESTObject],
+                List[ProjectPipelineJob],
+                List[ProjectPipeline],
+                List[Issue],
+                List[Dict[str, Any]],
+                GitlabList,
+                List[Project],
+                List[Union[RESTObject, Dict[str, Any]]],
+            ],
+        ],
+        validation_func: Callable[
+            [
+                Union[
+                    RESTObject,
+                    ProjectPipelineJob,
+                    Issue,
+                    ProjectPipeline,
+                    Dict[str, Any],
+                    Project,
+                ]
+            ],
+            bool,
+        ],
         batch_size: int = int(os.environ.get("GITLAB_BATCH_SIZE", 100)),
-        validation_func: Callable[[T], bool] | None = None,
         **kwargs,
-    ) -> AsyncIterator[List[T]]:
-        def fetch_batch(page_idx: int):
+    ) -> AsyncIterator[
+        Union[
+            List[Union[RESTObject, Dict[str, Any]]],
+            RESTObjectList,
+            List[ProjectPipelineJob],
+            List[ProjectPipeline],
+            List[Issue],
+            List[Project],
+            List[RESTObject],
+            List[Dict[str, Any]],
+            GitlabList,
+        ]
+    ]:
+        def fetch_batch(
+            page_idx: int,
+        ) -> Union[
+            List[Union[RESTObject, Dict[str, Any]]],
+            RESTObjectList,
+            List[ProjectPipelineJob],
+            List[ProjectPipeline],
+            List[Issue],
+            List[Project],
+            List[RESTObject],
+            List[Dict[str, Any]],
+            GitlabList,
+        ]:
             logger.info(f"Fetching page {page_idx}. Batch size: {batch_size}")
             return fetch_func(
                 page=page_idx, per_page=batch_size, get_all=False, **kwargs

integrations/gitlab/gitlab_integration/git_integration.py

@@ -5,7 +5,7 @@
     SEARCH_PROPERTY_PREFIX,
 )
 from loguru import logger
-from pydantic import Field
+from pydantic import Field, BaseModel
 from gitlab.v4.objects import Project
 from gitlab_integration.gitlab_service import PROJECTS_CACHE_KEY
 from gitlab_integration.utils import get_cached_all_services
@@ -14,18 +14,26 @@
 from port_ocean.core.handlers import JQEntityProcessor
 from port_ocean.core.handlers.port_app_config.models import (
     PortAppConfig,
+    Selector,
+    ResourceConfig,
 )
 
 
 class FileEntityProcessor(JQEntityProcessor):
     prefix = FILE_PROPERTY_PREFIX
 
     def _search(self, data: Dict[str, Any], pattern: str) -> Any:
-        project_id, ref = _validate_project_scope(data)
+        project_id, ref, base_path = _validate_project_scope(data)
         project = _get_project_from_cache(project_id)
-        logger.info(f"Searching for file {pattern} in Project {project_id}, ref {ref}")
 
         file_path = pattern.replace(self.prefix, "")
+        if base_path:
+            file_path = f"{base_path}/{file_path}"
+        if not project:
+            return None
+        logger.info(
+            f"Searching for file {file_path} in Project {project_id}: {project.path_with_namespace}, ref {ref}"
+        )
         return (
             project.files.get(file_path=file_path, ref=ref).decode().decode("utf-8")
             if project
@@ -44,17 +52,32 @@ def _search(self, data: Dict[str, Any], pattern: str) -> Any:
         :param pattern: e.g. search://scope=blobs&&query=filename:port.yml
         :return: True if the search pattern matches, False otherwise
         """
-        project_id, _ = _validate_project_scope(data)
+        project_id, _, base_path = _validate_project_scope(data)
         scope, query = self._parse_search_pattern(pattern)
 
         # assuming that the code is being called after event initialization and as part of the GitLab service
         # initialization
         project = _get_project_from_cache(project_id)
-        logger.info(f"Searching for {query} in Project {project_id}, scope {scope}")
-
+        if not project:
+            return None
+        base_path_message = f" in base path {base_path}" if base_path else ""
+        logger.info(
+            f"Searching {query} {base_path_message} in Project {project_id}: {project.path_with_namespace}, "
+            f"scope {scope}"
+        )
         match = None
         if project:
-            match = bool(project.search(scope=scope, search=query))
+            if scope == "blobs":
+                # if the query does not contain a path filter, we add the base path to the query
+                # this is done to avoid searching the entire project for the file, if the base path is known
+                # having the base path applies to the case where we export a folder as a monorepo
+                if base_path and "path:" not in query:
+                    query = f"{query} path:{base_path}"
+                results = project.search(scope=scope, search=query)
+                match = bool(results)
+            else:
+                results = project.search(scope=scope, search=query)
+                match = bool(results)
         return match
 
     def _parse_search_pattern(self, pattern: str) -> Tuple[str, str]:
@@ -87,6 +110,20 @@ def _search(self, data: Dict[str, Any], pattern: str) -> Any:
         return entity_processor(self.context)._search(data, pattern)
 
 
+class FoldersSelector(BaseModel):
+    path: str
+    repos: List[str] = Field(default_factory=list)
+    branch: str | None = None
+
+
+class GitlabSelector(Selector):
+    folders: List[FoldersSelector] = Field(default_factory=list)
+
+
+class GitlabResourceConfig(ResourceConfig):
+    selector: GitlabSelector
+
+
 class GitlabPortAppConfig(PortAppConfig):
     spec_path: str | List[str] = Field(alias="specPath", default="**/port.yml")
     branch: str = "main"
@@ -96,6 +133,7 @@ class GitlabPortAppConfig(PortAppConfig):
     project_visibility_filter: str | None = Field(
         alias="projectVisibilityFilter", default=None
     )
+    resources: list[GitlabResourceConfig] = Field(default_factory=list)  # type: ignore
 
 
 def _get_project_from_cache(project_id: int) -> Project | None:
@@ -129,7 +167,17 @@ def _get_project_from_cache(project_id: int) -> Project | None:
     return None
 
 
-def _validate_project_scope(data: Dict[str, Any]) -> Tuple[int, str]:
-    if (project_id := data.get("id")) and (ref := data.get("default_branch")):
-        return project_id, ref
+def _validate_project_scope(data: Dict[str, Any]) -> Tuple[int, str, str]:
+    # repo.id can be set when exporting folders as monorepo from a repo
+    project_id = data.get("repo", {}).get("id") or data.get("id")
+    # __branch is enriched when exporting folders as monorepo from a repo, or when exporting a single repo
+    ref = (
+        data.get("__branch")
+        or data.get("default_branch")
+        or data.get("repo", {}).get("default_branch")
+    )
+    # folder.path is enriched when exporting folders as monorepo from a repo
+    path = data.get("folder", {}).get("path", "")
+    if project_id and ref:
+        return project_id, ref, path
     raise ValueError("Project id and ref are required")