diff --git a/flux_local/git_repo.py b/flux_local/git_repo.py index 43430bdc..e9bbce9f 100644 --- a/flux_local/git_repo.py +++ b/flux_local/git_repo.py @@ -329,14 +329,42 @@ async def get_fluxtomizations( ] +# default_path=root_path_selector.relative_path +# sources=path-selector.sources or () +def adjust_ks_path( + doc: Kustomization, default_path: Path, sources: list[Source] +) -> Path | None: + """Make adjustments to the Kustomizations path.""" + # Source path is relative to the search path. Update to have the + # full prefix relative to the root. + if not doc.path: + _LOGGER.debug("Assigning implicit path %s", default_path) + return default_path + + if doc.source_kind == OCI_REPO_KIND: + for source in sources: + if source.name == doc.source_name: + _LOGGER.debug( + "Updated Source for OCIRepository %s: %s", doc.name, doc.path + ) + return source.root / doc.path + + _LOGGER.info( + "Unknown cluster source for OCIRepository %s: %s", doc.name, doc.path + ) + return None + + return Path(doc.path) + + async def kustomization_traversal( root_path_selector: PathSelector, path_selector: PathSelector, build: bool ) -> list[Kustomization]: """Search for kustomizations in the specified path.""" kustomizations: list[Kustomization] = [] - visited: set[Path] = set() # Relative paths within the cluster - nodes: set[str] = set() + visited_paths: set[Path] = set() # Relative paths within the cluster + visited_ks: set[str] = set() path_queue: queue.Queue[Path] = queue.Queue() path_queue.put(path_selector.relative_path) @@ -346,65 +374,42 @@ async def kustomization_traversal( try: docs = await get_fluxtomizations(root_path_selector.root, path, build=build) except FluxException as err: - detail = ERROR_DETAIL_BAD_KS if visited else ERROR_DETAIL_BAD_PATH + detail = ERROR_DETAIL_BAD_KS if visited_paths else ERROR_DETAIL_BAD_PATH raise FluxException( f"Error building Fluxtomization in '{root_path_selector.root}' " f"path '{path}': {err} - {detail}" ) - visited |= set({path}) + visited_paths |= set({path}) + + orig_len = len(docs) + docs = [doc for doc in docs if doc.namespaced_name not in visited_ks] + visited_ks |= set({doc.namespaced_name for doc in docs}) + new_len = len(docs) + _LOGGER.debug("Found %s Kustomizations (%s unique)", orig_len, new_len) - _LOGGER.debug("Found %s Kustomizations", len(docs)) result_docs = [] for doc in docs: - if doc.namespaced_name in nodes: - _LOGGER.debug( - "Ignoring duplicate Kustomization %s", doc.namespaced_name - ) - continue - nodes.add(doc.namespaced_name) - # Source path is relative to the search path. Update to have the - # full prefix relative to the root. - if not doc.path: - _LOGGER.debug( - "Assigning implicit path %s", root_path_selector.relative_path - ) - doc.path = str(root_path_selector.relative_path) - - found_path: Path | None = None _LOGGER.debug( - "Kustomization '%s' has sourceRef.kind '%s' of '%s'", + "Kustomization '%s' sourceRef.kind '%s' of '%s'", doc.name, doc.source_kind, doc.source_name, ) - if not doc.source_kind or doc.source_kind == GIT_REPO_KIND: - found_path = Path(doc.path) - elif doc.source_kind == OCI_REPO_KIND: - for source in path_selector.sources or (): - if source.name == doc.source_name: - found_path = source.root / doc.path - doc.path = str(found_path) - _LOGGER.debug( - "Updated Source for OCIRepository %s: %s", - doc.name, - doc.path, - ) - break - - elif not doc.source_kind or doc.source_kind == GIT_REPO_KIND: - found_path = Path(doc.path) - - if not found_path: - _LOGGER.debug("Skipping kustomization %s; not known source", doc.name) + if not ( + doc_path := adjust_ks_path( + doc, root_path_selector.relative_path, path_selector.sources or [] + ) + ): continue - - if found_path not in visited: - path_queue.put(found_path) + doc.path = str(doc_path) + if doc_path not in visited_paths: + path_queue.put(doc_path) else: - _LOGGER.debug("Already visited %s", found_path) + _LOGGER.debug("Already visited %s", doc_path) result_docs.append(doc) kustomizations.extend(result_docs) + kustomizations.sort(key=lambda x: (x.namespace, x.name)) return kustomizations @@ -452,15 +457,21 @@ async def get_clusters( ] build = False + tasks = [] for cluster in clusters: _LOGGER.debug("Building cluster %s %s", cluster.name, cluster.path) - results = await kustomization_traversal( - path_selector, - PathSelector(path=Path(cluster.path), sources=path_selector.sources), - build=build, + tasks.append( + kustomization_traversal( + path_selector, + PathSelector(path=Path(cluster.path), sources=path_selector.sources), + build=build, + ) ) - results.sort(key=lambda x: (x.namespace, x.name)) - cluster.kustomizations = list(filter(kustomization_selector.predicate, results)) + finished = await asyncio.gather(*tasks) + for cluster, results in zip(clusters, finished): + cluster.kustomizations = [ + ks for ks in results if kustomization_selector.predicate(ks) + ] clusters.sort(key=lambda x: (x.path, x.namespace, x.name)) return clusters diff --git a/tests/test_git_repo.py b/tests/test_git_repo.py index 2e3f9d4f..01a19a9b 100644 --- a/tests/test_git_repo.py +++ b/tests/test_git_repo.py @@ -123,7 +123,7 @@ async def write(w: Path, x: Path, y: Any, cmd: Kustomize | None) -> None: kustomization = cluster.kustomizations[0] assert kustomization.name == "apps" assert kustomization.namespace == "flux-system" - assert kustomization.path == "./tests/testdata/cluster/apps/prod" + assert kustomization.path == "tests/testdata/cluster/apps/prod" visits.sort() assert visits == [ diff --git a/tests/tool/testdata/get_cluster_yaml.yaml b/tests/tool/testdata/get_cluster_yaml.yaml index b488cdda..817edea6 100644 --- a/tests/tool/testdata/get_cluster_yaml.yaml +++ b/tests/tool/testdata/get_cluster_yaml.yaml @@ -14,7 +14,7 @@ stdout: | kustomizations: - name: apps namespace: flux-system - path: ./tests/testdata/cluster/apps/prod + path: tests/testdata/cluster/apps/prod helm_repos: [] helm_releases: - name: podinfo @@ -26,13 +26,13 @@ stdout: | cluster_policies: [] - name: flux-system namespace: flux-system - path: ./tests/testdata/cluster/clusters/prod + path: tests/testdata/cluster/clusters/prod helm_repos: [] helm_releases: [] cluster_policies: [] - name: infra-configs namespace: flux-system - path: ./tests/testdata/cluster/infrastructure/configs + path: tests/testdata/cluster/infrastructure/configs helm_repos: - name: bitnami namespace: flux-system @@ -52,7 +52,7 @@ stdout: | namespace: null - name: infra-controllers namespace: flux-system - path: ./tests/testdata/cluster/infrastructure/controllers + path: tests/testdata/cluster/infrastructure/controllers helm_repos: [] helm_releases: - name: weave-gitops diff --git a/tests/tool/testdata/get_ks.yaml b/tests/tool/testdata/get_ks.yaml index 35b28da1..46c85838 100644 --- a/tests/tool/testdata/get_ks.yaml +++ b/tests/tool/testdata/get_ks.yaml @@ -4,8 +4,8 @@ args: - --path - tests/testdata/cluster stdout: | - NAME PATH - apps ./tests/testdata/cluster/apps/prod - flux-system ./tests/testdata/cluster/clusters/prod - infra-configs ./tests/testdata/cluster/infrastructure/configs - infra-controllers ./tests/testdata/cluster/infrastructure/controllers + NAME PATH + apps tests/testdata/cluster/apps/prod + flux-system tests/testdata/cluster/clusters/prod + infra-configs tests/testdata/cluster/infrastructure/configs + infra-controllers tests/testdata/cluster/infrastructure/controllers diff --git a/tests/tool/testdata/get_ks2.yaml b/tests/tool/testdata/get_ks2.yaml index 73114334..c178a996 100644 --- a/tests/tool/testdata/get_ks2.yaml +++ b/tests/tool/testdata/get_ks2.yaml @@ -4,9 +4,9 @@ args: - --path - tests/testdata/cluster2 stdout: | - NAME PATH - cluster ./tests/testdata/cluster2/flux - cluster-apps ./tests/testdata/cluster2/apps - cluster-apps-ingress-nginx ./tests/testdata/cluster2/apps/networking/ingress-nginx/app - cluster-apps-ingress-nginx-certificates ./tests/testdata/cluster2/apps/networking/ingress-nginx/certificates - cluster-apps-kubernetes-dashboard ./tests/testdata/cluster2/apps/monitoring/kubernetes-dashboard/app + NAME PATH + cluster tests/testdata/cluster2/flux + cluster-apps tests/testdata/cluster2/apps + cluster-apps-ingress-nginx tests/testdata/cluster2/apps/networking/ingress-nginx/app + cluster-apps-ingress-nginx-certificates tests/testdata/cluster2/apps/networking/ingress-nginx/certificates + cluster-apps-kubernetes-dashboard tests/testdata/cluster2/apps/monitoring/kubernetes-dashboard/app diff --git a/tests/tool/testdata/get_ks4.yaml b/tests/tool/testdata/get_ks4.yaml index 21aeeda2..b529beec 100644 --- a/tests/tool/testdata/get_ks4.yaml +++ b/tests/tool/testdata/get_ks4.yaml @@ -4,7 +4,7 @@ args: - --path - tests/testdata/cluster4 stdout: | - NAME PATH - cluster ./tests/testdata/cluster4/flux - cluster-apps ./tests/testdata/cluster4/apps - cluster-apps-kubernetes-dashboard ./tests/testdata/cluster4/apps/monitoring/kubernetes-dashboard + NAME PATH + cluster tests/testdata/cluster4/flux + cluster-apps tests/testdata/cluster4/apps + cluster-apps-kubernetes-dashboard tests/testdata/cluster4/apps/monitoring/kubernetes-dashboard diff --git a/tests/tool/testdata/get_ks5.yaml b/tests/tool/testdata/get_ks5.yaml index 60cf634a..78b376d2 100644 --- a/tests/tool/testdata/get_ks5.yaml +++ b/tests/tool/testdata/get_ks5.yaml @@ -4,5 +4,5 @@ args: - --path - tests/testdata/cluster5 stdout: | - NAME PATH - flux-system ./tests/testdata/cluster5/clusters/prod + NAME PATH + flux-system tests/testdata/cluster5/clusters/prod diff --git a/tests/tool/testdata/get_ks5_all.yaml b/tests/tool/testdata/get_ks5_all.yaml index e4206267..b56dd9cc 100644 --- a/tests/tool/testdata/get_ks5_all.yaml +++ b/tests/tool/testdata/get_ks5_all.yaml @@ -5,6 +5,6 @@ args: - --path - tests/testdata/cluster5 stdout: | - NAMESPACE NAME PATH - controllers infra-controllers tests/testdata/cluster5 - flux-system flux-system ./tests/testdata/cluster5/clusters/prod + NAMESPACE NAME PATH + controllers infra-controllers tests/testdata/cluster5 + flux-system flux-system tests/testdata/cluster5/clusters/prod diff --git a/tests/tool/testdata/get_ks6.yaml b/tests/tool/testdata/get_ks6.yaml index 1e07ca40..2d5db8e1 100644 --- a/tests/tool/testdata/get_ks6.yaml +++ b/tests/tool/testdata/get_ks6.yaml @@ -4,6 +4,6 @@ args: - --path - tests/testdata/cluster6 stdout: | - NAME PATH - apps ./tests/testdata/cluster6/apps/ - flux-system ./tests/testdata/cluster6/cluster + NAME PATH + apps tests/testdata/cluster6/apps + flux-system tests/testdata/cluster6/cluster diff --git a/tests/tool/testdata/get_ks_path.yaml b/tests/tool/testdata/get_ks_path.yaml index fdefdc3c..6240dec9 100644 --- a/tests/tool/testdata/get_ks_path.yaml +++ b/tests/tool/testdata/get_ks_path.yaml @@ -4,8 +4,8 @@ args: - --path - ./tests/testdata/cluster/clusters/prod stdout: | - NAME PATH - apps ./tests/testdata/cluster/apps/prod - flux-system ./tests/testdata/cluster/clusters/prod - infra-configs ./tests/testdata/cluster/infrastructure/configs - infra-controllers ./tests/testdata/cluster/infrastructure/controllers + NAME PATH + apps tests/testdata/cluster/apps/prod + flux-system tests/testdata/cluster/clusters/prod + infra-configs tests/testdata/cluster/infrastructure/configs + infra-controllers tests/testdata/cluster/infrastructure/controllers diff --git a/tests/tool/testdata/get_ks_wide.yaml b/tests/tool/testdata/get_ks_wide.yaml index 92bf0838..2c938c2c 100644 --- a/tests/tool/testdata/get_ks_wide.yaml +++ b/tests/tool/testdata/get_ks_wide.yaml @@ -6,8 +6,8 @@ args: - --path - tests/testdata/cluster stdout: | - NAME PATH HELMREPOS RELEASES - apps ./tests/testdata/cluster/apps/prod 0 1 - flux-system ./tests/testdata/cluster/clusters/prod 0 0 - infra-configs ./tests/testdata/cluster/infrastructure/configs 3 0 - infra-controllers ./tests/testdata/cluster/infrastructure/controllers 0 2 + NAME PATH HELMREPOS RELEASES + apps tests/testdata/cluster/apps/prod 0 1 + flux-system tests/testdata/cluster/clusters/prod 0 0 + infra-configs tests/testdata/cluster/infrastructure/configs 3 0 + infra-controllers tests/testdata/cluster/infrastructure/controllers 0 2