Fix build in postbuild substitution (#734)

Fixes #707

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ repos:
         exclude: '^tests/.*/__snapshots__/.*.ambr$'
       - id: end-of-file-fixer
       - id: check-yaml
-        exclude: '^tests/testdata/cluster8/apps/.*\.yaml$'
+        exclude: '^tests/testdata/cluster8/apps/.*\.yaml|tests/testdata/cluster/apps/prod/.*\.yaml$'
       - id: check-added-large-files
   - repo: https://github.com/psf/black
     rev: 24.4.2

flux_local/git_repo.py

@@ -403,27 +403,54 @@ def remove(self, kustomization: Kustomization) -> None:
                 del self._cache[key]
 
 
+@dataclass
+class VisitResult:
+    """Result of visiting a kustomization."""
+
+    kustomizations: list[Kustomization]
+    config_maps: list[ConfigMap]
+    secrets: list[Secret]
+
+    def __post_init__(self) -> None:
+        """Validate the object"""
+        unique = {ks.namespaced_name for ks in self.kustomizations}
+        if len(unique) != len(self.kustomizations):
+            ks_names = [ks.namespaced_name for ks in self.kustomizations]
+            dupes = list(filter(lambda x: ks_names.count(x) > 1, ks_names))
+            raise FluxException(
+                f"Detected multiple Fluxtomizations with the same name: {dupes}. "
+                "This indicates either (1) an incorrect Kustomization which needs to be fixed "
+                "or (2) a multi-cluster setup which requires flux-local to run with a more strict --path."
+            )
+
+
 async def visit_kustomization(
     selector: PathSelector,
     builder: CachableBuilder,
     path: Path,
     visit_ks: Kustomization | None,
-) -> list[Kustomization]:
+) -> VisitResult:
     """Visit a path and return a list of Kustomizations."""
 
     _LOGGER.debug("Visiting path (%s) %s", selector.path, path)
     label = visit_ks.namespaced_name if visit_ks else str(path)
+
+    kinds = [CLUSTER_KUSTOMIZE_KIND, CONFIG_MAP_KIND, SECRET_KIND]
+
     with trace_context(f"Kustomization '{label}'"):
         cmd: kustomize.Kustomize
         if visit_ks is None:
-            cmd = kustomize.grep(f"kind={CLUSTER_KUSTOMIZE_KIND}", selector.root / path)
+            cmd = kustomize.filter_resources(kinds, selector.root / path)
         else:
             cmd = await builder.build(visit_ks, selector.root / path)
-            cmd = cmd.grep(f"kind={CLUSTER_KUSTOMIZE_KIND}")
-        cmd = cmd.grep(GREP_SOURCE_REF_KIND)
+            cmd = cmd.filter_resources(kinds)
+        cmd = await cmd.stash()
+        ks_cmd = cmd.grep(GREP_SOURCE_REF_KIND)
+        cfg_cmd = cmd.filter_resources([CONFIG_MAP_KIND, SECRET_KIND])
 
         try:
-            docs = await cmd.objects()
+            ks_docs = await ks_cmd.objects()
+            cfg_docs = await cfg_cmd.objects()
         except KustomizePathException as err:
             raise FluxException(err) from err
         except FluxException as err:
@@ -436,25 +463,26 @@ async def visit_kustomization(
                 f"Error building Fluxtomization '{visit_ks.namespaced_name}' "
                 f"path '{path}': {ERROR_DETAIL_BAD_KS} {err}"
             ) from err
-    kustomizations = list(
-        filter(
-            is_allowed_source(selector.sources or []),
-            [
-                Kustomization.parse_doc(doc)
-                for doc in filter(FLUXTOMIZE_DOMAIN_FILTER, docs)
-            ],
-        )
+
+    return VisitResult(
+        kustomizations=list(
+            filter(
+                is_allowed_source(selector.sources or []),
+                [
+                    Kustomization.parse_doc(doc)
+                    for doc in filter(FLUXTOMIZE_DOMAIN_FILTER, ks_docs)
+                ],
+            )
+        ),
+        config_maps=[
+            ConfigMap.parse_doc(doc)
+            for doc in cfg_docs
+            if doc.get("kind") == CONFIG_MAP_KIND
+        ],
+        secrets=[
+            Secret.parse_doc(doc) for doc in cfg_docs if doc.get("kind") == SECRET_KIND
+        ],
     )
-    unique = {ks.namespaced_name for ks in kustomizations}
-    if len(unique) != len(kustomizations):
-        ks_names = [ks.namespaced_name for ks in kustomizations]
-        dupes = list(filter(lambda x: ks_names.count(x) > 1, ks_names))
-        raise FluxException(
-            f"Detected multiple Fluxtomizations with the same name: {dupes}. "
-            "This indicates either (1) an incorrect Kustomization which needs to be fixed "
-            "or (2) a multi-cluster setup which requires flux-local to run with a more strict --path."
-        )
-    return kustomizations
 
 
 async def kustomization_traversal(
@@ -468,6 +496,7 @@ async def kustomization_traversal(
 
     path_queue: deque[tuple[Path, Kustomization | None]] = deque()
     path_queue.append((selector.relative_path, None))
+    cluster_config = values.cluster_config([], [])
     while path_queue:
         # Fully empty the queue, running all tasks in parallel
         tasks = []
@@ -484,7 +513,10 @@ async def kustomization_traversal(
         # Find new kustomizations
         kustomizations = []
         for result in await asyncio.gather(*tasks):
-            for ks in result:
+            cluster_config = values.merge_cluster_config(
+                cluster_config, result.secrets, result.config_maps
+            )
+            for ks in result.kustomizations:
                 if ks.namespaced_name in visited_ks:
                     continue
                 kustomizations.append(ks)
@@ -502,6 +534,12 @@ async def kustomization_traversal(
             if not (ks_path := adjust_ks_path(ks, selector)):
                 continue
             ks.path = str(ks_path)
+            if ks.postbuild_substitute_from:
+                values.expand_postbuild_substitute_reference(
+                    ks,
+                    cluster_config,
+                )
+
             path_queue.append((ks_path, ks))
             response_kustomizations.append(ks)
 
@@ -581,8 +619,7 @@ async def build_kustomization(
         if not kinds:
             return
 
-        regexp = f"kind=^({'|'.join(kinds)})$"
-        docs = await cmd.grep(regexp).objects(
+        docs = await cmd.filter_resources(kinds).objects(
             target_namespace=kustomization.target_namespace
         )
 

flux_local/kustomize.py

@@ -141,6 +141,13 @@ def skip_resources(self, kinds: list[str]) -> "Kustomize":
         skip_re = "|".join(kinds)
         return self.grep(f"kind=^({skip_re})$", invert=True)
 
+    def filter_resources(self, kinds: list[str]) -> "Kustomize":
+        """Skip resources kinds of the specified types."""
+        if not kinds:
+            return self
+        skip_re = "|".join(kinds)
+        return self.grep(f"kind=^({skip_re})$", invert=False)
+
     async def validate_policies(self, policies: list[manifest.ClusterPolicy]) -> None:
         """Apply kyverno policies to objects built so far."""
         if not policies:
@@ -283,6 +290,12 @@ def grep(expr: str, path: Path, invert: bool = False) -> Kustomize:
     return Kustomize([Command(args, cwd=cwd, exc=KustomizeException)])
 
 
+def filter_resources(kinds: list[str], path: Path) -> Kustomize:
+    """Filter resources in the specified path based of a specific kind."""
+    regexp = f"kind=^({'|'.join(kinds)})$"
+    return grep(regexp, path)
+
+
 def update_namespace(doc: dict[str, Any], namespace: str) -> dict[str, Any]:
     """Update the namespace of the specified document.
 

flux_local/values.py

@@ -59,6 +59,16 @@ def cluster_config(
     )
 
 
+def merge_cluster_config(
+    config: ClusterConfig, secrets: list[Secret], config_maps: list[ConfigMap]
+) -> ClusterConfig:
+    """Create a ClusterConfig from a list of secrets and configmaps."""
+    return ClusterConfig(
+        lambda: list(config.secrets) + secrets,
+        lambda: list(config.config_maps) + config_maps,
+    )
+
+
 def ks_cluster_config(kustomizations: list[Kustomization]) -> ClusterConfig:
     """Create a ClusterConfig from a list of Kustomizations."""
 
@@ -262,7 +272,9 @@ def expand_postbuild_substitute_reference(
             continue
 
         if found_data is None:
-            if not ref.optional and not ref.kind == SECRET_KIND:  # Secrets are commonly filtered
+            if (
+                not ref.optional and not ref.kind == SECRET_KIND
+            ):  # Secrets are commonly filtered
                 _LOGGER.warning(
                     "Unable to find SubstituteReference for %s: %s",
                     ks.namespaced_name,
@@ -271,5 +283,6 @@ def expand_postbuild_substitute_reference(
             continue
 
         values.update(found_data)
+    _LOGGER.debug("update_postbuild_substitutions=%s", values)
     ks.update_postbuild_substitutions(values)
     return ks