Skip to content

Commit

Permalink
Cleanup paths on output to remove unnecessary ./ prefixes (#269)
Browse files Browse the repository at this point in the history
  • Loading branch information
@allenporter
allenporter authored Jul 7, 2023
1 parent b0ae075 commit ac4d5e7
Show file tree
Hide file tree
Showing 11 changed files with 99 additions and 88 deletions.
111 changes: 61 additions & 50 deletions flux_local/git_repo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,14 +329,42 @@ async def get_fluxtomizations(
]


# default_path=root_path_selector.relative_path
# sources=path-selector.sources or ()
def adjust_ks_path(
doc: Kustomization, default_path: Path, sources: list[Source]
) -> Path | None:
"""Make adjustments to the Kustomizations path."""
# Source path is relative to the search path. Update to have the
# full prefix relative to the root.
if not doc.path:
_LOGGER.debug("Assigning implicit path %s", default_path)
return default_path

if doc.source_kind == OCI_REPO_KIND:
for source in sources:
if source.name == doc.source_name:
_LOGGER.debug(
"Updated Source for OCIRepository %s: %s", doc.name, doc.path
)
return source.root / doc.path

_LOGGER.info(
"Unknown cluster source for OCIRepository %s: %s", doc.name, doc.path
)
return None

return Path(doc.path)


async def kustomization_traversal(
root_path_selector: PathSelector, path_selector: PathSelector, build: bool
) -> list[Kustomization]:
"""Search for kustomizations in the specified path."""

kustomizations: list[Kustomization] = []
visited: set[Path] = set() # Relative paths within the cluster
nodes: set[str] = set()
visited_paths: set[Path] = set() # Relative paths within the cluster
visited_ks: set[str] = set()

path_queue: queue.Queue[Path] = queue.Queue()
path_queue.put(path_selector.relative_path)
Expand All @@ -346,65 +374,42 @@ async def kustomization_traversal(
try:
docs = await get_fluxtomizations(root_path_selector.root, path, build=build)
except FluxException as err:
detail = ERROR_DETAIL_BAD_KS if visited else ERROR_DETAIL_BAD_PATH
detail = ERROR_DETAIL_BAD_KS if visited_paths else ERROR_DETAIL_BAD_PATH
raise FluxException(
f"Error building Fluxtomization in '{root_path_selector.root}' "
f"path '{path}': {err} - {detail}"
)

visited |= set({path})
visited_paths |= set({path})

orig_len = len(docs)
docs = [doc for doc in docs if doc.namespaced_name not in visited_ks]
visited_ks |= set({doc.namespaced_name for doc in docs})
new_len = len(docs)
_LOGGER.debug("Found %s Kustomizations (%s unique)", orig_len, new_len)

_LOGGER.debug("Found %s Kustomizations", len(docs))
result_docs = []
for doc in docs:
if doc.namespaced_name in nodes:
_LOGGER.debug(
"Ignoring duplicate Kustomization %s", doc.namespaced_name
)
continue
nodes.add(doc.namespaced_name)
# Source path is relative to the search path. Update to have the
# full prefix relative to the root.
if not doc.path:
_LOGGER.debug(
"Assigning implicit path %s", root_path_selector.relative_path
)
doc.path = str(root_path_selector.relative_path)

found_path: Path | None = None
_LOGGER.debug(
"Kustomization '%s' has sourceRef.kind '%s' of '%s'",
"Kustomization '%s' sourceRef.kind '%s' of '%s'",
doc.name,
doc.source_kind,
doc.source_name,
)
if not doc.source_kind or doc.source_kind == GIT_REPO_KIND:
found_path = Path(doc.path)
elif doc.source_kind == OCI_REPO_KIND:
for source in path_selector.sources or ():
if source.name == doc.source_name:
found_path = source.root / doc.path
doc.path = str(found_path)
_LOGGER.debug(
"Updated Source for OCIRepository %s: %s",
doc.name,
doc.path,
)
break

elif not doc.source_kind or doc.source_kind == GIT_REPO_KIND:
found_path = Path(doc.path)

if not found_path:
_LOGGER.debug("Skipping kustomization %s; not known source", doc.name)
if not (
doc_path := adjust_ks_path(
doc, root_path_selector.relative_path, path_selector.sources or []
)
):
continue

if found_path not in visited:
path_queue.put(found_path)
doc.path = str(doc_path)
if doc_path not in visited_paths:
path_queue.put(doc_path)
else:
_LOGGER.debug("Already visited %s", found_path)
_LOGGER.debug("Already visited %s", doc_path)
result_docs.append(doc)
kustomizations.extend(result_docs)
kustomizations.sort(key=lambda x: (x.namespace, x.name))
return kustomizations


Expand Down Expand Up @@ -452,15 +457,21 @@ async def get_clusters(
]
build = False

tasks = []
for cluster in clusters:
_LOGGER.debug("Building cluster %s %s", cluster.name, cluster.path)
results = await kustomization_traversal(
path_selector,
PathSelector(path=Path(cluster.path), sources=path_selector.sources),
build=build,
tasks.append(
kustomization_traversal(
path_selector,
PathSelector(path=Path(cluster.path), sources=path_selector.sources),
build=build,
)
)
results.sort(key=lambda x: (x.namespace, x.name))
cluster.kustomizations = list(filter(kustomization_selector.predicate, results))
finished = await asyncio.gather(*tasks)
for cluster, results in zip(clusters, finished):
cluster.kustomizations = [
ks for ks in results if kustomization_selector.predicate(ks)
]
clusters.sort(key=lambda x: (x.path, x.namespace, x.name))
return clusters

Expand Down
2 changes: 1 addition & 1 deletion tests/test_git_repo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ async def write(w: Path, x: Path, y: Any, cmd: Kustomize | None) -> None:
kustomization = cluster.kustomizations[0]
assert kustomization.name == "apps"
assert kustomization.namespace == "flux-system"
assert kustomization.path == "./tests/testdata/cluster/apps/prod"
assert kustomization.path == "tests/testdata/cluster/apps/prod"

visits.sort()
assert visits == [
Expand Down
8 changes: 4 additions & 4 deletions tests/tool/testdata/get_cluster_yaml.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ stdout: |
kustomizations:
- name: apps
namespace: flux-system
path: ./tests/testdata/cluster/apps/prod
path: tests/testdata/cluster/apps/prod
helm_repos: []
helm_releases:
- name: podinfo
Expand All @@ -26,13 +26,13 @@ stdout: |
cluster_policies: []
- name: flux-system
namespace: flux-system
path: ./tests/testdata/cluster/clusters/prod
path: tests/testdata/cluster/clusters/prod
helm_repos: []
helm_releases: []
cluster_policies: []
- name: infra-configs
namespace: flux-system
path: ./tests/testdata/cluster/infrastructure/configs
path: tests/testdata/cluster/infrastructure/configs
helm_repos:
- name: bitnami
namespace: flux-system
Expand All @@ -52,7 +52,7 @@ stdout: |
namespace: null
- name: infra-controllers
namespace: flux-system
path: ./tests/testdata/cluster/infrastructure/controllers
path: tests/testdata/cluster/infrastructure/controllers
helm_repos: []
helm_releases:
- name: weave-gitops
Expand Down
10 changes: 5 additions & 5 deletions tests/tool/testdata/get_ks.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ args:
- --path
- tests/testdata/cluster
stdout: |
NAME PATH
apps ./tests/testdata/cluster/apps/prod
flux-system ./tests/testdata/cluster/clusters/prod
infra-configs ./tests/testdata/cluster/infrastructure/configs
infra-controllers ./tests/testdata/cluster/infrastructure/controllers
NAME PATH
apps tests/testdata/cluster/apps/prod
flux-system tests/testdata/cluster/clusters/prod
infra-configs tests/testdata/cluster/infrastructure/configs
infra-controllers tests/testdata/cluster/infrastructure/controllers
12 changes: 6 additions & 6 deletions tests/tool/testdata/get_ks2.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ args:
- --path
- tests/testdata/cluster2
stdout: |
NAME PATH
cluster ./tests/testdata/cluster2/flux
cluster-apps ./tests/testdata/cluster2/apps
cluster-apps-ingress-nginx ./tests/testdata/cluster2/apps/networking/ingress-nginx/app
cluster-apps-ingress-nginx-certificates ./tests/testdata/cluster2/apps/networking/ingress-nginx/certificates
cluster-apps-kubernetes-dashboard ./tests/testdata/cluster2/apps/monitoring/kubernetes-dashboard/app
NAME PATH
cluster tests/testdata/cluster2/flux
cluster-apps tests/testdata/cluster2/apps
cluster-apps-ingress-nginx tests/testdata/cluster2/apps/networking/ingress-nginx/app
cluster-apps-ingress-nginx-certificates tests/testdata/cluster2/apps/networking/ingress-nginx/certificates
cluster-apps-kubernetes-dashboard tests/testdata/cluster2/apps/monitoring/kubernetes-dashboard/app
8 changes: 4 additions & 4 deletions tests/tool/testdata/get_ks4.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ args:
- --path
- tests/testdata/cluster4
stdout: |
NAME PATH
cluster ./tests/testdata/cluster4/flux
cluster-apps ./tests/testdata/cluster4/apps
cluster-apps-kubernetes-dashboard ./tests/testdata/cluster4/apps/monitoring/kubernetes-dashboard
NAME PATH
cluster tests/testdata/cluster4/flux
cluster-apps tests/testdata/cluster4/apps
cluster-apps-kubernetes-dashboard tests/testdata/cluster4/apps/monitoring/kubernetes-dashboard
4 changes: 2 additions & 2 deletions tests/tool/testdata/get_ks5.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,5 +4,5 @@ args:
- --path
- tests/testdata/cluster5
stdout: |
NAME PATH
flux-system ./tests/testdata/cluster5/clusters/prod
NAME PATH
flux-system tests/testdata/cluster5/clusters/prod
6 changes: 3 additions & 3 deletions tests/tool/testdata/get_ks5_all.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,6 @@ args:
- --path
- tests/testdata/cluster5
stdout: |
NAMESPACE NAME PATH
controllers infra-controllers tests/testdata/cluster5
flux-system flux-system ./tests/testdata/cluster5/clusters/prod
NAMESPACE NAME PATH
controllers infra-controllers tests/testdata/cluster5
flux-system flux-system tests/testdata/cluster5/clusters/prod
6 changes: 3 additions & 3 deletions tests/tool/testdata/get_ks6.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@ args:
- --path
- tests/testdata/cluster6
stdout: |
NAME PATH
apps ./tests/testdata/cluster6/apps/
flux-system ./tests/testdata/cluster6/cluster
NAME PATH
apps tests/testdata/cluster6/apps
flux-system tests/testdata/cluster6/cluster
10 changes: 5 additions & 5 deletions tests/tool/testdata/get_ks_path.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ args:
- --path
- ./tests/testdata/cluster/clusters/prod
stdout: |
NAME PATH
apps ./tests/testdata/cluster/apps/prod
flux-system ./tests/testdata/cluster/clusters/prod
infra-configs ./tests/testdata/cluster/infrastructure/configs
infra-controllers ./tests/testdata/cluster/infrastructure/controllers
NAME PATH
apps tests/testdata/cluster/apps/prod
flux-system tests/testdata/cluster/clusters/prod
infra-configs tests/testdata/cluster/infrastructure/configs
infra-controllers tests/testdata/cluster/infrastructure/controllers
10 changes: 5 additions & 5 deletions tests/tool/testdata/get_ks_wide.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ args:
- --path
- tests/testdata/cluster
stdout: |
NAME PATH HELMREPOS RELEASES
apps ./tests/testdata/cluster/apps/prod 0 1
flux-system ./tests/testdata/cluster/clusters/prod 0 0
infra-configs ./tests/testdata/cluster/infrastructure/configs 3 0
infra-controllers ./tests/testdata/cluster/infrastructure/controllers 0 2
NAME PATH HELMREPOS RELEASES
apps tests/testdata/cluster/apps/prod 0 1
flux-system tests/testdata/cluster/clusters/prod 0 0
infra-configs tests/testdata/cluster/infrastructure/configs 3 0
infra-controllers tests/testdata/cluster/infrastructure/controllers 0 2

0 comments on commit ac4d5e7

Please sign in to comment.