feat: default-required feature flag

.github/workflows/test-upgrade.yml

@@ -69,7 +69,7 @@ jobs:
 
       - name: Initialize the cluster with the release version
         # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed
-        #       in a previous step. This test run will the current release to create a K3s cluster.
+        #       in a previous step. This test run will use the current release to create a K3s cluster.
         run: |
           sudo env "PATH=$PATH" CI=true zarf init --components k3s,git-server,logging --nodeport 31337 --confirm
 
@@ -79,7 +79,7 @@ jobs:
 
       - name: Create and deploy the upgrade test packages
         # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed
-        #       in a previous step. This test run will the current release to create a K3s cluster.
+        #       in a previous step. This test run will use the current release to create a K3s cluster.
         run: |
           zarf package create src/test/upgrade --set PODINFO_VERSION=6.3.3 --confirm
           sudo env "PATH=$PATH" CI=true zarf package deploy zarf-package-test-upgrade-package-amd64-6.3.3.tar.zst --confirm
@@ -105,12 +105,10 @@ jobs:
 
       - name: Run the upgrade tests
         # NOTE: "PATH=$PATH" preserves the default user $PATH. This is needed to maintain the version of zarf installed
-        #       in a previous step. This test run will the current release to create a K3s cluster.
+        #       in a previous step. This test run will use the current release to create a K3s cluster.
         run: |
           sudo env "PATH=$PATH" CI=true zarf tools kubectl describe nodes
 
-          zarf package create src/test/upgrade --set PODINFO_VERSION=6.3.4 --confirm
-
           sudo env "PATH=$PATH" CI=true make test-upgrade ARCH=amd64
 
       - name: Save logs

Makefile

@@ -29,7 +29,7 @@ else
 	endif
 endif
 
-CLI_VERSION ?= $(if $(shell git describe --tags),$(shell git describe --tags),"UnknownVersion")
+CLI_VERSION ?= $(if $(shell git describe --tags),$(shell git describe --tags),"unset-development-only")
 BUILD_ARGS := -s -w -X github.com/defenseunicorns/zarf/src/config.CLIVersion=$(CLI_VERSION)
 K8S_MODULES_VER=$(subst ., ,$(subst v,,$(shell go list -f '{{.Version}}' -m k8s.io/client-go)))
 K8S_MODULES_MAJOR_VER=$(shell echo $$(($(firstword $(K8S_MODULES_VER)) + 1)))
@@ -203,7 +203,7 @@ test-upgrade: ## Run the Zarf CLI E2E tests for an external registry and cluster
 	@test -s $(ZARF_BIN) || $(MAKE) build-cli
 	[ -n "$(shell zarf version)" ] || (echo "Zarf must be installed prior to the upgrade test" && exit 1)
 	[ -n "$(shell zarf package list 2>&1 | grep test-upgrade-package)" ] || (echo "Zarf must be initialized and have the 6.3.3 upgrade-test package installed prior to the upgrade test" && exit 1)
-	@test -s "zarf-package-test-upgrade-package-amd64-6.3.4.tar.zst" || zarf package create src/test/upgrade/ --set PODINFO_VERSION=6.3.4 --confirm
+	@test -s ./zarf-package-test-upgrade-package-$(ARCH)-6.3.4.tar.zst || zarf package create src/test/upgrade --set PODINFO_VERSION=6.3.4 --confirm
 	cd src/test/upgrade && go test -failfast -v -timeout 30m
 
 .PHONY: test-unit

adr/0025-required-optional.md

@@ -0,0 +1,127 @@
+# 25. Components can be required by default + introduction of feature flags
+
+Date: 2024-04-29
+
+## Status
+
+Accepted
+
+## Context
+
+> Feature request: <https://github.com/defenseunicorns/zarf/issues/2059>
+
+Currently, all Zarf components default to being optional due to the `required` key being _optional_ in the YAML. This leads to package authors needing to ensure that they annotate this key for each component, and since nothing in the current validations prompts them about this they may be confused about the "all things are optional" default state.
+
+When Zarf was first created, we didn't really know how it would evolve and this key was introduced in those very early days. At this point it would be better to require all components by default--especially with the introduction of composability and the OCI skeleton work, there is plenty of flexibility in the API to compose bespoke packages assembled from other packages.
+
+A few ways to handle this:
+
+1. Simply force the `required` key to be a non-optional, so that package authors would be forced to specify it for each component, thereby removing any ambiguity--but also force one more key for every single component ever created 🫠
+
+2. Deprecate `required` and introduce an optional `optional` key, which would default to _false_.
+
+3. Do something more significant like combine various condition-based things such as `only`, `optional` (instead of `required`), or `default`.
+
+4. Introduce `feature` flags to allow for certain schema behavior to be configurable to the user.
+
+## Decision
+
+Option 4. Introduce `.metadata.features` flags to `zarf.yaml` to allow for certain schema behavior to be configurable to the user.
+
+The `features` key will be added to the `metadata` section of the package schema. This key will be an array of strings, where each string is the name of a beta feature that can be enabled. To enable a feature, the user will need to add the name of the feature to this array and edit the package schema accordingly.
+
+> Such feature migrations can also be accomplished using `zarf dev migrate`, see the [Consequences](#consequences) section for more information.
+
+e.g.
+
+```diff
+kind: ZarfInitConfig
+metadata:
+  name: init
+  description: Used to establish a new Zarf cluster
++ features:
++   - default-required
+
+components:
+  - name: k3s
++   required: false
+    import:
+      path: packages/distros/k3s
+
+  # This package moves the injector & registries binaries
+  - name: zarf-injector
+-   required: true
+    import:
+      path: packages/zarf-registry
+```
+
+## Consequences
+
+The introduction of feature flags will allow Zarf to introduce new features and schema behavior without breaking existing packages and schemas, but also introduces more complexity. This will require more documentation and user education to ensure that users understand how to use these flags.
+
+Beta feature flags will become the default behavior of Zarf upon the next major release, and will _not_ be configurable by the user at that time. This will allow for a more consistent experience across all Zarf packages.
+
+There will be a flag added to the `zarf dev migrate` command `--enable-feature <feature-name>` to allow users to enable features on a per-package basis. This will allow users to test new features in a controlled environment before they are enabled by default.
+
+> Tab autocompletion for the `--enable-feature` flag is enabled for the `zarf dev migrate` command.
+
+e.g. (some output omitted for brevity)
+
+```bash
+$ zarf dev migrate --enable-feature default-required > migrated-zarf.yaml
+
+ NOTE  Using config file ...
+
+ NOTE  Saving log file to ...
+
+
+     Migration        | Type         | Affected
+     default-required | feature      | .
+
+```
+
+```diff
+$ git diff --no-index zarf.yaml migrated-zarf.yaml
+
+kind: ZarfInitConfig
+ metadata:
+   name: init
+   description: Used to establish a new Zarf cluster
++  features:
++    - default-required
+ components:
+   - name: k3s
++    required: false
+     import:
+       path: packages/distros/k3s
+
+   - name: zarf-injector
+-    required: true
+     import:
+       path: packages/zarf-registry
+
+   - name: zarf-seed-registry
+-    required: true
+     import:
+       path: packages/zarf-registry
+
+   - name: zarf-registry
+-    required: true
+     import:
+       path: packages/zarf-registry
+
+   - name: zarf-agent
+-    required: true
+     import:
+       path: packages/zarf-agent
+
+   - name: logging
++    required: false
+     import:
+       path: packages/logging-pgl
+
+   - name: git-server
++    required: false
+     import:
+       path: packages/gitea
+```

packages/distros/k3s/zarf.yaml

@@ -1,7 +1,10 @@
 kind: ZarfInitConfig
 metadata:
   name: distro-k3s
-  description: Used to establish a new Zarf cluster
+  description: |
+    Used to establish a new Zarf cluster
+
+    This package is NOT meant to be used as a standalone package. It is meant to be used as a dependency of an 'init' package.
 
 components:
   # AMD-64 version of the K3s stack

packages/gitea/zarf.yaml

@@ -1,6 +1,8 @@
 kind: ZarfPackageConfig
 metadata:
-  name: init-package-git-server
+  name: git-server
+  description: |
+    This package is NOT meant to be used as a standalone package. It is meant to be used as a dependency of an 'init' package.
 
 variables:
   - name: GIT_SERVER_EXISTING_PVC

packages/logging-pgl/zarf.yaml

@@ -1,6 +1,10 @@
 kind: ZarfPackageConfig
 metadata:
-  name: init-package-logging
+  name: logging
+  description: |
+    Deploys the Promtail Grafana & Loki (PGL) stack.
+
+    This package is NOT meant to be used as a standalone package. It is meant to be used as a dependency of an 'init' package.
 
 components:
   - name: logging

packages/zarf-agent/zarf.yaml

@@ -1,7 +1,10 @@
 kind: ZarfPackageConfig
 metadata:
-  name: init-package-zarf-agent
-  description: Install the zarf agent mutating webhook on a new cluster
+  name: zarf-agent
+  description: |
+    Install the zarf agent mutating webhook on a new cluster
+
+    This package is NOT meant to be used as a standalone package. It is meant to be used as a dependency of an 'init' package.
 
 constants:
   - name: AGENT_IMAGE
@@ -16,7 +19,6 @@ components:
       images and git repository references in Kubernetes manifests. This prevents
       the need to manually update URLs from their original sources to the Zarf-managed
       docker registry and git server.
-    required: true
     images:
       - "###ZARF_PKG_TMPL_AGENT_IMAGE_DOMAIN######ZARF_PKG_TMPL_AGENT_IMAGE###:###ZARF_PKG_TMPL_AGENT_IMAGE_TAG###"
     manifests:

packages/zarf-registry/zarf.yaml

@@ -1,6 +1,11 @@
 kind: ZarfPackageConfig
 metadata:
-  name: init-package-zarf-registry
+  name: zarf-registry
+  description: |
+    Initializes the Zarf Registry by bootstrapping a Kubernetes cluster with a running pod and hosting the registry image.
+    The Zarf Registry is then updated to use the self-hosted registry image.
+
+    This package is NOT meant to be used as a standalone package. It is meant to be used as a dependency of an 'init' package.
 
 variables:
   - name: REGISTRY_EXISTING_PVC
@@ -105,7 +110,6 @@ components:
     description: |
       Bootstraps a Kubernetes cluster by cloning a running pod in the cluster and hosting the registry image.
       Removed and destroyed after the Zarf Registry is self-hosting the registry image.
-    required: true
     only:
       cluster:
         architecture: amd64
@@ -120,7 +124,6 @@ components:
     description: |
       Bootstraps a Kubernetes cluster by cloning a running pod in the cluster and hosting the registry image.
       Removed and destroyed after the Zarf Registry is self-hosting the registry image.
-    required: true
     only:
       cluster:
         architecture: arm64

site/src/content/docs/commands/zarf_dev.md

@@ -37,6 +37,7 @@ Commands useful for developing packages
 * [zarf dev generate](/commands/zarf_dev_generate/)	 - [alpha] Creates a zarf.yaml automatically from a given remote (git) Helm chart
 * [zarf dev generate-config](/commands/zarf_dev_generate-config/)	 - Generates a config file for Zarf
 * [zarf dev lint](/commands/zarf_dev_lint/)	 - Lints the given package for valid schema and recommended practices
+* [zarf dev migrate](/commands/zarf_dev_migrate/)	 - [alpha] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format
 * [zarf dev patch-git](/commands/zarf_dev_patch-git/)	 - Converts all .git URLs to the specified Zarf HOST and with the Zarf URL pattern in a given FILE.  NOTE:
 This should only be used for manifests that are not mutated by the Zarf Agent Mutating Webhook.
 * [zarf dev sha256sum](/commands/zarf_dev_sha256sum/)	 - Generates a SHA256SUM for the given file

site/src/content/docs/commands/zarf_dev_migrate.md

@@ -0,0 +1,41 @@
+---
+title: zarf dev migrate
+description: Zarf CLI command reference for <code>zarf dev migrate</code>.
+tableOfContents: false
+---
+
+<!-- Page generated by Zarf; DO NOT EDIT -->
+
+## zarf dev migrate
+
+[alpha] Migrates the zarf.yaml in a given directory to the latest version of the zarf.yaml format
+
+```
+zarf dev migrate [flags]
+```
+
+### Options
+
+```
+      --enable-feature stringArray   feature migrations to run and enable (available: default-required)
+  -h, --help                         help for migrate
+      --run stringArray              migrations of deprecated features to run (default: all, available: scripts-to-actions, pluralize-set-variable)
+```
+
+### Options inherited from parent commands
+
+```
+  -a, --architecture string   Architecture for OCI images and Zarf packages
+      --insecure              Allow access to insecure registries and disable other recommended security enforcements such as package checksum and signature validation. This flag should only be used if you have a specific reason and accept the reduced security posture.
+  -l, --log-level string      Log level when running Zarf. Valid options are: warn, info, debug, trace (default "info")
+      --no-color              Disable colors in output
+      --no-log-file           Disable log file creation
+      --no-progress           Disable fancy UI progress bars, spinners, logos, etc
+      --tmpdir string         Specify the temporary directory to use for intermediate files
+      --zarf-cache string     Specify the location of the Zarf cache directory (default "~/.zarf-cache")
+```
+
+### SEE ALSO
+
+* [zarf dev](/commands/zarf_dev/)	 - Commands useful for developing packages
+