Merge branch 'main' into 2665-zarf-connect-options

.github/actions/cleanup-files/action.yaml

@@ -7,10 +7,16 @@ runs:
     - run: |
         lsblk -f
 
+        echo "removing some github actions pre-installed tools to save space"
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /opt/ghc
+        sudo rm -rf /opt/hostedtoolcache/CodeQL
+        sudo docker system prune --all --force
+
+        echo "removing zarf sboms, packages, cache"
         sudo rm -rf zarf-sbom /tmp/zarf-*
         sudo env "PATH=$PATH" CI=true make delete-packages
         sudo build/zarf tools clear-cache
-        sudo docker system prune --all --force
 
         lsblk -f
       shell: bash

.github/actions/debug-cluster/action.yaml

@@ -0,0 +1,22 @@
+name: debug-cluster
+description: "Setup Go binary and caching"
+
+runs:
+  using: composite
+  steps:
+    - run: |
+        echo "***** Getting pods *****"
+        kubectl get pods -A
+
+        echo "***** Getting pods yaml *****"
+        kubectl get pods -A -o yaml
+
+        echo "***** Describing pods *****"
+        kubectl describe pods -A
+
+        echo "***** Getting nodes *****"
+        kubectl get nodes -A
+
+        echo "***** describing nodes *****"
+        kubectl describe nodes -A
+      shell: bash

.github/workflows/nightly-eks.yml

@@ -57,21 +57,9 @@ jobs:
       - name: Run tests
         run: make test-e2e-with-cluster ARCH=amd64
 
-      - name: get pods
+      - name: show cluster logs
+        uses: ./.github/actions/debug-cluster
         if: always()
-        run: kubectl get pods -n kiwix -o yaml
-
-      - name: describe pod
-        if: always()
-        run: kubectl describe pods -n kiwix
-
-      - name: get nodes
-        if: always()
-        run: kubectl get nodes -o yaml
-
-      - name: describe nodes
-        if: always()
-        run: kubectl describe nodes
 
       - name: Teardown the cluster
         if: always()

.github/workflows/release.yml

@@ -72,7 +72,7 @@ jobs:
 
       # Upload the contents of the build directory for later stages to use
       - name: Upload build artifacts
-        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
+        uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
         with:
           name: build-artifacts
           path: build/
@@ -186,7 +186,7 @@ jobs:
           HOMEBREW_TAP_GITHUB_TOKEN: ${{ steps.brew-tap-token.outputs.token }}
 
       - name: Save CVE report
-        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
+        uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
         with:
           name: cve-report
           path: build/zarf-known-cves.csv

.github/workflows/scorecard.yaml

@@ -36,7 +36,7 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
+        uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
         with:
           name: SARIF file
           path: results.sarif

.github/workflows/test-bigbang.yml

@@ -63,7 +63,7 @@ jobs:
 
       # Upload the contents of the build directory for later stages to use
       - name: Upload build artifacts
-        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
+        uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
         with:
           name: build-artifacts
           path: build/

.github/workflows/test-e2e.yml

@@ -47,7 +47,7 @@ jobs:
 
       # Upload the contents of the build directory for later stages to use
       - name: Upload build artifacts
-        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
+        uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
         with:
           name: build-artifacts
           path: build/
@@ -119,6 +119,10 @@ jobs:
         run: |
           make test-e2e-with-cluster ARCH=amd64
 
+      - name: get cluster info
+        uses: ./.github/actions/debug-cluster
+        if: always()
+
       - name: Save logs
         if: always()
         uses: ./.github/actions/save-logs
@@ -197,6 +201,10 @@ jobs:
         run: |
           make test-e2e-with-cluster ARCH=amd64
 
+      - name: get cluster info
+        uses: ./.github/actions/debug-cluster
+        if: always()
+
       - name: Save logs
         if: always()
         uses: ./.github/actions/save-logs
@@ -235,6 +243,10 @@ jobs:
         run: |
           make test-e2e-with-cluster ARCH=amd64
 
+      - name: get cluster info
+        uses: ./.github/actions/debug-cluster
+        if: always()
+
       - name: Save logs
         if: always()
         uses: ./.github/actions/save-logs

.github/workflows/test-external.yml

@@ -48,9 +48,16 @@ jobs:
       - name: Setup k3d
         uses: ./.github/actions/k3d
 
+      - name: Cleanup files
+        uses: ./.github/actions/cleanup-files
+
       - name: Run external service test
         run: make test-external
 
+      - name: get cluster info
+        uses: ./.github/actions/debug-cluster
+        if: always()
+
       - name: Save logs
         if: always()
         uses: ./.github/actions/save-logs

.github/workflows/test-upgrade.yml

@@ -46,7 +46,7 @@ jobs:
 
       # Upload the contents of the build directory for later stages to use
       - name: Upload build artifacts
-        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
+        uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5
         with:
           name: build-artifacts
           path: build/

examples/podinfo-flux/git/podinfo-kustomization.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: kustomize.toolkit.fluxcd.io/v1beta2
+apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
   name: podinfo-git

examples/podinfo-flux/git/podinfo-source.yaml

@@ -1,5 +1,5 @@
 ---
-apiVersion: source.toolkit.fluxcd.io/v1beta2
+apiVersion: source.toolkit.fluxcd.io/v1
 kind: GitRepository
 metadata:
   name: podinfo

src/cmd/internal.go

@@ -278,7 +278,7 @@ var createPackageRegistryToken = &cobra.Command{
 		}
 
 		// If we are setup to use an internal artifact server, create the artifact registry token
-		if state.ArtifactServer.InternalServer {
+		if state.ArtifactServer.IsInternal() {
 			tunnel, err := c.NewTunnel(cluster.ZarfNamespaceName, cluster.SvcResource, cluster.ZarfGitServerName, "", 0, cluster.ZarfGitServerPort)
 			if err != nil {
 				return err

src/cmd/tools/zarf.go

@@ -148,7 +148,7 @@ var updateCredsCmd = &cobra.Command{
 			}
 
 			// Update artifact token (if internal)
-			if slices.Contains(args, message.ArtifactKey) && newState.ArtifactServer.PushToken == "" && newState.ArtifactServer.InternalServer {
+			if slices.Contains(args, message.ArtifactKey) && newState.ArtifactServer.PushToken == "" && newState.ArtifactServer.IsInternal() {
 				tunnel, err := c.NewTunnel(cluster.ZarfNamespaceName, cluster.SvcResource, cluster.ZarfGitServerName, "", 0, cluster.ZarfGitServerPort)
 				if err != nil {
 					return err
@@ -186,14 +186,14 @@ var updateCredsCmd = &cobra.Command{
 			// Update Zarf 'init' component Helm releases if present
 			h := helm.NewClusterOnly(&types.PackagerConfig{}, template.GetZarfVariableConfig(), newState, c)
 
-			if slices.Contains(args, message.RegistryKey) && newState.RegistryInfo.InternalRegistry {
+			if slices.Contains(args, message.RegistryKey) && newState.RegistryInfo.IsInternal() {
 				err = h.UpdateZarfRegistryValues(ctx)
 				if err != nil {
 					// Warn if we couldn't actually update the registry (it might not be installed and we should try to continue)
 					message.Warnf(lang.CmdToolsUpdateCredsUnableUpdateRegistry, err.Error())
 				}
 			}
-			if slices.Contains(args, message.GitKey) && newState.GitServer.InternalServer {
+			if slices.Contains(args, message.GitKey) && newState.GitServer.IsInternal() {
 				tunnel, err := c.NewTunnel(cluster.ZarfNamespaceName, cluster.SvcResource, cluster.ZarfGitServerName, "", 0, cluster.ZarfGitServerPort)
 				if err != nil {
 					return err

src/internal/agent/hooks/flux-helmrepo.go

@@ -80,7 +80,7 @@ func mutateHelmRepo(ctx context.Context, r *v1.AdmissionRequest, cluster *cluste
 
 	message.Debugf("original HelmRepo URL of (%s) got mutated to (%s)", src.Spec.URL, patchedURL)
 
-	patches := populateHelmRepoPatchOperations(patchedURL, zarfState.RegistryInfo.InternalRegistry)
+	patches := populateHelmRepoPatchOperations(patchedURL, zarfState.RegistryInfo.IsInternal())
 
 	patches = append(patches, getLabelPatch(src.Labels))
 

src/internal/agent/hooks/flux-ocirepo.go

@@ -99,7 +99,7 @@ func mutateOCIRepo(ctx context.Context, r *v1.AdmissionRequest, cluster *cluster
 
 	message.Debugf("original OCIRepo URL of (%s) got mutated to (%s)", src.Spec.URL, patchedURL)
 
-	patches := populateOCIRepoPatchOperations(patchedURL, zarfState.RegistryInfo.InternalRegistry, patchedRef)
+	patches := populateOCIRepoPatchOperations(patchedURL, zarfState.RegistryInfo.IsInternal(), patchedRef)
 
 	patches = append(patches, getLabelPatch(src.Labels))
 	return &operations.Result{

src/internal/packager/template/template.go

@@ -107,7 +107,7 @@ func GetZarfTemplates(componentName string, state *types.ZarfState) (templateMap
 // generateHtpasswd returns an htpasswd string for the current state's RegistryInfo.
 func generateHtpasswd(regInfo *types.RegistryInfo) (string, error) {
 	// Only calculate this for internal registries to allow longer external passwords
-	if regInfo.InternalRegistry {
+	if regInfo.IsInternal() {
 		pushUser, err := utils.GetHtpasswdString(regInfo.PushUsername, regInfo.PushPassword)
 		if err != nil {
 			return "", fmt.Errorf("error generating htpasswd string: %w", err)

src/pkg/cluster/secrets.go

@@ -64,7 +64,7 @@ func (c *Cluster) GenerateRegistryPullCreds(ctx context.Context, namespace, name
 	// Convert to JSON
 	dockerConfigData, err := json.Marshal(dockerConfigJSON)
 	if err != nil {
-		return nil, fmt.Errorf("Unable to marshal the .dockerconfigjson secret data for the image pull secret: %w", err)
+		return nil, fmt.Errorf("unable to marshal the .dockerconfigjson secret data for the image pull secret: %w", err)
 	}
 
 	secretDockerConfig := &corev1.Secret{

src/pkg/cluster/state.go

@@ -35,15 +35,16 @@ const (
 
 // InitZarfState initializes the Zarf state with the given temporary directory and init configs.
 func (c *Cluster) InitZarfState(ctx context.Context, initOptions types.ZarfInitOptions) error {
-	var distro string
-
 	spinner := message.NewProgressSpinner("Gathering cluster state information")
 	defer spinner.Stop()
 
 	// Attempt to load an existing state prior to init.
 	// NOTE: We are ignoring the error here because we don't really expect a state to exist yet.
 	spinner.Updatef("Checking cluster for existing Zarf deployment")
-	state, _ := c.LoadZarfState(ctx)
+	state, err := c.LoadZarfState(ctx)
+	if err != nil && !kerrors.IsNotFound(err) {
+		return fmt.Errorf("failed to check for existing state: %w", err)
+	}
 
 	// If state is nil, this is a new cluster.
 	if state == nil {
@@ -52,7 +53,7 @@ func (c *Cluster) InitZarfState(ctx context.Context, initOptions types.ZarfInitO
 
 		if initOptions.ApplianceMode {
 			// If the K3s component is being deployed, skip distro detection.
-			distro = DistroIsK3s
+			state.Distro = DistroIsK3s
 			state.ZarfAppliance = true
 		} else {
 			// Otherwise, trying to detect the K8s distro type.
@@ -67,16 +68,13 @@ func (c *Cluster) InitZarfState(ctx context.Context, initOptions types.ZarfInitO
 			if err != nil {
 				return err
 			}
-			distro = detectDistro(nodeList.Items[0], namespaceList.Items)
+			state.Distro = detectDistro(nodeList.Items[0], namespaceList.Items)
 		}
 
-		if distro != DistroIsUnknown {
-			spinner.Updatef("Detected K8s distro %s", distro)
+		if state.Distro != DistroIsUnknown {
+			spinner.Updatef("Detected K8s distro %s", state.Distro)
 		}
 
-		// Defaults
-		state.Distro = distro
-
 		// Setup zarf agent PKI
 		agentTLS, err := pki.GeneratePKI(config.ZarfAgentHost)
 		if err != nil {
@@ -100,8 +98,7 @@ func (c *Cluster) InitZarfState(ctx context.Context, initOptions types.ZarfInitO
 			namespaceCopy := namespace
 			_, err := c.Clientset.CoreV1().Namespaces().Update(ctx, &namespaceCopy, metav1.UpdateOptions{})
 			if err != nil {
-				// This is not a hard failure, but we should log it.
-				message.WarnErrf(err, "Unable to mark the namespace %s as ignored by Zarf Agent", namespace.Name)
+				return fmt.Errorf("unable to mark the namespace %s as ignored by Zarf Agent: %w", namespace.Name, err)
 			}
 		}
 
@@ -306,21 +303,14 @@ func MergeZarfState(oldState *types.ZarfState, initOptions types.ZarfInitOptions
 	if slices.Contains(services, message.RegistryKey) {
 		// TODO: Replace use of reflections with explicit setting
 		newState.RegistryInfo = helpers.MergeNonZero(newState.RegistryInfo, initOptions.RegistryInfo)
-		// Set the state of the internal registry if it has changed
-		// TODO: Internal registry should be a function of the address and not a property.
-		if newState.RegistryInfo.Address == fmt.Sprintf("%s:%d", helpers.IPV4Localhost, newState.RegistryInfo.NodePort) {
-			newState.RegistryInfo.InternalRegistry = true
-		} else {
-			newState.RegistryInfo.InternalRegistry = false
-		}
 
 		// Set the new passwords if they should be autogenerated
-		if newState.RegistryInfo.PushPassword == oldState.RegistryInfo.PushPassword && oldState.RegistryInfo.InternalRegistry {
+		if newState.RegistryInfo.PushPassword == oldState.RegistryInfo.PushPassword && oldState.RegistryInfo.IsInternal() {
 			if newState.RegistryInfo.PushPassword, err = helpers.RandomString(types.ZarfGeneratedPasswordLen); err != nil {
 				return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err)
 			}
 		}
-		if newState.RegistryInfo.PullPassword == oldState.RegistryInfo.PullPassword && oldState.RegistryInfo.InternalRegistry {
+		if newState.RegistryInfo.PullPassword == oldState.RegistryInfo.PullPassword && oldState.RegistryInfo.IsInternal() {
 			if newState.RegistryInfo.PullPassword, err = helpers.RandomString(types.ZarfGeneratedPasswordLen); err != nil {
 				return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err)
 			}
@@ -330,21 +320,13 @@ func MergeZarfState(oldState *types.ZarfState, initOptions types.ZarfInitOptions
 		// TODO: Replace use of reflections with explicit setting
 		newState.GitServer = helpers.MergeNonZero(newState.GitServer, initOptions.GitServer)
 
-		// Set the state of the internal git server if it has changed
-		// TODO: Internal server should be a function of the address and not a property.
-		if newState.GitServer.Address == types.ZarfInClusterGitServiceURL {
-			newState.GitServer.InternalServer = true
-		} else {
-			newState.GitServer.InternalServer = false
-		}
-
 		// Set the new passwords if they should be autogenerated
-		if newState.GitServer.PushPassword == oldState.GitServer.PushPassword && oldState.GitServer.InternalServer {
+		if newState.GitServer.PushPassword == oldState.GitServer.PushPassword && oldState.GitServer.IsInternal() {
 			if newState.GitServer.PushPassword, err = helpers.RandomString(types.ZarfGeneratedPasswordLen); err != nil {
 				return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err)
 			}
 		}
-		if newState.GitServer.PullPassword == oldState.GitServer.PullPassword && oldState.GitServer.InternalServer {
+		if newState.GitServer.PullPassword == oldState.GitServer.PullPassword && oldState.GitServer.IsInternal() {
 			if newState.GitServer.PullPassword, err = helpers.RandomString(types.ZarfGeneratedPasswordLen); err != nil {
 				return nil, fmt.Errorf("%s: %w", lang.ErrUnableToGenerateRandomSecret, err)
 			}
@@ -354,16 +336,8 @@ func MergeZarfState(oldState *types.ZarfState, initOptions types.ZarfInitOptions
 		// TODO: Replace use of reflections with explicit setting
 		newState.ArtifactServer = helpers.MergeNonZero(newState.ArtifactServer, initOptions.ArtifactServer)
 
-		// Set the state of the internal artifact server if it has changed
-		// TODO: Internal server should be a function of the address and not a property.
-		if newState.ArtifactServer.Address == types.ZarfInClusterArtifactServiceURL {
-			newState.ArtifactServer.InternalServer = true
-		} else {
-			newState.ArtifactServer.InternalServer = false
-		}
-
 		// Set an empty token if it should be autogenerated
-		if newState.ArtifactServer.PushToken == oldState.ArtifactServer.PushToken && oldState.ArtifactServer.InternalServer {
+		if newState.ArtifactServer.PushToken == oldState.ArtifactServer.PushToken && oldState.ArtifactServer.IsInternal() {
 			newState.ArtifactServer.PushToken = ""
 		}
 	}