add some comments and docs

examples/README.md

@@ -0,0 +1,12 @@
+# 🔥Config Examples
+
+## Workflows
+
+The workflows directory contains a set of GitHub actions that you can use to have 🔥Config automatically compute the
+mermaid DAG and diff of changes to your Kubernetes objects, and then leave a comment on the PR with the DAG and diff.
+You _should_ just be able to copy these into your `.github/workflows` directory.  You'll need to set up a personal
+access token (PAT) with read access to your actions and read and write access to pull requests.  This PAT then needs to
+be injected into your actions as a GitHub secret.
+
+- [Managing your Personal Access Tokens](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens)
+- [Using secrets in GitHub Actions](https://docs.github.com/en/actions/security-guides/using-secrets-in-github-actions)

examples/workflows/k8s_plan.yml

@@ -0,0 +1,49 @@
+name: Compute k8s plan
+
+on:
+  pull_request:
+    paths:
+      - 'k8s/**'
+
+jobs:
+  plan:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out master
+        uses: actions/checkout@v4
+        with:
+          ref: master
+          submodules: recursive
+
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+
+      - name: Compile k8s charts
+        run: make k8s
+
+      - name: Check out PR
+        uses: actions/checkout@v4
+        with:
+          clean: false
+
+      - name: Compute dag/diff
+        run: make k8s
+
+      - name: Save artifacts
+        run: |
+          mkdir -p ./artifacts
+          echo ${{ github.event.number }} > ./artifacts/PR
+          mv .build/dag.mermaid ./artifacts/dag.mermaid
+          mv .build/k8s.df ./artifacts/k8s.df
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: k8s-plan-artifacts
+          path: artifacts/

examples/workflows/pr_comment_finished.yml

@@ -0,0 +1,58 @@
+name: Comment on the PR
+
+on:
+  workflow_run:
+    workflows: ["Compute k8s plan"]
+    types:
+      - completed
+
+jobs:
+  pr-comment:
+    runs-on: ubuntu-latest
+    if: >
+      github.event.workflow_run.event == 'pull_request' &&
+      github.event.workflow_run.conclusion == 'success'
+
+    steps:
+      - name: Download artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: k8s-plan-artifacts
+          github-token: ${{ secrets.PR_COMMENT_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          path: k8s-plan-artifacts
+
+      - name: Get PR number
+        uses: mathiasvr/[email protected]
+        id: pr
+        with:
+          run: cat k8s-plan-artifacts/PR
+
+      - name: Find previous comment ID
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          token: ${{ secrets.PR_COMMENT_TOKEN }}
+          issue-number: ${{ steps.pr.outputs.stdout }}
+          body-includes: "<!-- 🔥config summary -->"
+
+      - name: Render Comment Template
+        run: |
+          echo "<!-- 🔥config summary -->" > fireconfig-comment.md
+          echo "## Kubernetes Object DAG" >> fireconfig-comment.md
+          cat k8s-plan-artifacts/dag.mermaid >> fireconfig-comment.md
+          echo '<img src="https://raw.githubusercontent.com/acrlabs/fireconfig/master/assets/new.png" width=10/> New object' >> fireconfig-comment.md
+          echo '<img src="https://raw.githubusercontent.com/acrlabs/fireconfig/master/assets/removed.png" width=10/> Deleted object' >> fireconfig-comment.md
+          echo '<img src="https://raw.githubusercontent.com/acrlabs/fireconfig/master/assets/changed.png" width=10/> Updated object' >> fireconfig-comment.md
+          echo '<img src="https://raw.githubusercontent.com/acrlabs/fireconfig/master/assets/pod_recreate.png" width=10/> Updated object (causes pod recreation)' >> fireconfig-comment.md
+          echo "## Detailed Diff" >> fireconfig-comment.md
+          cat k8s-plan-artifacts/k8s.df >> fireconfig-comment.md
+
+      - name: Comment on PR
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          token: ${{ secrets.PR_COMMENT_TOKEN }}
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ steps.pr.outputs.stdout }}
+          body-path: fireconfig-comment.md
+          edit-mode: replace

examples/workflows/pr_comment_starting.yml

@@ -0,0 +1,44 @@
+name: Update the PR Comment
+
+on:
+  #######################################################################################
+  # WARNING: DO NOT CHANGE THIS ACTION TO CHECK OUT OR EXECUTE ANY CODE!!!!!            #
+  #                                                                                     #
+  # This can allow an attacker to gain write access to code in the repository or read   #
+  # any repository secrets!  This should _only_ be used to update or add a PR comment.  #
+  #                                                                                     #
+  # See https://securitylab.github.com/research/github-actions-preventing-pwn-requests/ #
+  # for more details.                                                                   #
+  #######################################################################################
+  pull_request_target:
+    paths:
+      - 'k8s/**'
+
+jobs:
+  pr-comment:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Find previous comment ID
+        uses: peter-evans/find-comment@v3
+        id: fc
+        with:
+          token: ${{ secrets.PR_COMMENT_TOKEN }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body-includes: "<!-- 🔥config summary -->"
+
+      - name: Render Comment Template
+        run: |
+          echo
+
+      - name: Comment on PR
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          token: ${{ secrets.PR_COMMENT_TOKEN }}
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          body: |
+            <!-- 🔥config summary -->
+            ## Updating Kubernetes DAG...
+            Please wait until the job has finished.
+          edit-mode: replace

fireconfig/__init__.py

@@ -32,6 +32,10 @@
 
 
 class AppPackage(metaclass=ABCMeta):
+    """
+    Users should implement the AppPackage class to pass into fireconfig
+    """
+
     @property
     @abstractmethod
     def id(self):
@@ -48,9 +52,27 @@ def compile(
     cdk8s_outdir: T.Optional[str] = None,
     dry_run: bool = False,
 ) -> T.Tuple[str, str]:
+    """
+    `compile` takes a list of "packages" and generates Kubernetes manifests from them.  It
+    also generates a Markdown-ified "diff" and a mermaid graph representing the Kubernetes
+    manifest structure and changes.
+
+    :param pkgs: the list of packages to compile
+    :param dag_filename: the location of a previous DAG, for use in generating diffs
+    :param cdk8s_outdir: where to save the generated Kubernetes manifests
+    :param dry_run: actually generate the manifests, or not
+
+    :returns: the mermaid DAG and markdown-ified diff as a tuple of strings
+    """
+
     app = App(outdir=cdk8s_outdir)
+
+    # Anything that is a "global" dependency (e.g., namespaces) that should be generated before
+    # everything else, or that should only be generated once, belongs in the global chart
     gl = Chart(app, GLOBAL_CHART_NAME, disable_resource_name_hashes=True)
 
+    # For each cdk8s chart, we generate a sub-DAG (stored in `subgraphs`) and then we connect
+    # all the subgraphs together via the `subgraph_dag`
     subgraph_dag = defaultdict(list)
     subgraphs = {}
     subgraphs[GLOBAL_CHART_NAME] = ChartSubgraph(GLOBAL_CHART_NAME)
@@ -66,6 +88,16 @@ def compile(
             subgraphs[pkg.id] = ChartSubgraph(pkg.id)
             subgraph_dag[gl.node.id].append(pkg.id)
 
+    # cdk8s doesn't compute the full dependency graph until you call `synth`, and there's no
+    # public access to it at that point, which is annoying.  Until that point, the dependency
+    # graph only includes the dependencies that you've explicitly added.  The format is
+    #
+    # [root (empty node)] ---> leaf nodes of created objects ---> tree in reverse
+    #   |
+    #   -----> [list of chart objects]
+    #
+    # The consequence being that we need to start at the root node, walk forwards, look at all the things
+    # that have "chart" fields, and then from there walk in reverse.  It's somewhat annoying.
     for obj in DependencyGraph(app.node).root.outbound:
         walk_dep_graph(obj, subgraphs)
     diff, kinds = compute_diff(app)

fireconfig/deployment.py

@@ -152,6 +152,7 @@ def _build(self, meta: k8s.ObjectMeta, chart: Chart) -> k8s.KubeDeployment:
 
         return depl
 
+    # TODO maybe move these into separate files at some point?
     def _build_service_account(self, chart: Chart) -> k8s.KubeServiceAccount:
         return k8s.KubeServiceAccount(chart, f"{self._tag}sa")
 

fireconfig/plan.py

@@ -1,3 +1,13 @@
+"""
+The bulk of the logic for computing the plan (a la terraform plan) lives in this file.
+The rough outline of steps taken is
+
+1. For each chart, walk the dependency graph to construct a DAG (`walk_dep_graph`)
+2. Compute a diff between the newly-generated manifests and the old ones (`compute_diff`)
+3. Turn that diff into a list of per-resource changes (`get_resource_changes`)
+4. Find out what's been deleted since the last run and add that into the graph (`find_deleted_nodes`)
+"""
+
 import re
 import typing as T
 from collections import defaultdict
@@ -47,6 +57,20 @@ def changes(self) -> T.List[ChangeTuple]:
         return self._changes
 
     def update_state(self, change_type: str, path: str, kind: T.Optional[str]):
+        """
+        Given a particular resource, update the state (added, removed, changed, etc) for
+        that resource.  We use the list of "change types" from DeepDiff defined here:
+
+        https://github.com/seperman/deepdiff/blob/89c5cc227c48b63be4a0e1ad4af59d3c1b0272d7/deepdiff/serialization.py#L388
+
+        Since these are Kubernetes objects, we expect the root object to be a dictionary, if it's
+        not, something has gone horribly wrong.  If the root object was added or removed, we mark the
+        entire object as added or removed; otherwise if some sub-dictionary was added or removed,
+        the root object was just "changed".
+
+        We use the `kind` field to determine whether pod recreation needs to happen.  This entire
+        function is currently very hacky and incomplete, it would be good to make this more robust sometime.
+        """
         if self._state in {ResourceState.Added, ResourceState.Removed}:
             return
 
@@ -77,6 +101,11 @@ def add_change(self, path: str, r1: T.Union[T.Mapping, notpresent], r2: T.Union[
 
 
 def compute_diff(app: App) -> T.Tuple[T.Mapping[str, T.Any], T.Mapping[str, str]]:
+    """
+    To compute a diff, we look at the old YAML files that were written out "last time", and
+    compare them to the generated YAML by cdk8s "this time".
+    """
+
     kinds = {}
     old_defs = {}
     for filename in glob(f"{app.outdir}/*{app.output_file_extension}"):
@@ -112,6 +141,8 @@ def walk_dep_graph(v: DependencyVertex, subgraphs: T.Mapping[str, ChartSubgraph]
         if not hasattr(v.value, "chart"):
             return
 
+        # Note that cdk8s does things backwards, so instead of adding the edge from v->dep,
+        # we add an edge from dep->v
         subgraphs[chart].add_edge(dep, v)
         walk_dep_graph(dep, subgraphs)
 
@@ -133,6 +164,14 @@ def find_deleted_nodes(
     resource_changes: T.Mapping[str, ResourceChanges],
     old_dag_filename: T.Optional[str],
 ):
+    """
+    To determine the location and connections of deleted nodes in the DAG,
+    we just look at the old DAG file and copy out the edge lines that contain the
+    removed objects.  The DAG file is split into subgraphs, so we parse it line-by-line
+    and only copy entries that are a) inside a subgraph block, and b) weren't marked as
+    deleted "last time".  We use special comment markers in the DAG file to tell which
+    things were deleted "last time".
+    """
     if not old_dag_filename:
         return