From 05479771f25b1501acc06cc96d4024b5a0ea320c Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Wed, 10 Jul 2024 10:45:32 -0400 Subject: [PATCH 1/8] fix: revert fix: remove ignore label when adopting resource (#2711) --- src/pkg/cluster/namespace.go | 1 - src/pkg/cluster/namespace_test.go | 25 ------------------------- 2 files changed, 26 deletions(-) delete mode 100644 src/pkg/cluster/namespace_test.go diff --git a/src/pkg/cluster/namespace.go b/src/pkg/cluster/namespace.go index 997247bfa9..c65c968d0e 100644 --- a/src/pkg/cluster/namespace.go +++ b/src/pkg/cluster/namespace.go @@ -67,6 +67,5 @@ func AdoptZarfManagedLabels(labels map[string]string) map[string]string { labels = make(map[string]string) } labels[ZarfManagedByLabel] = "zarf" - delete(labels, AgentLabel) return labels } diff --git a/src/pkg/cluster/namespace_test.go b/src/pkg/cluster/namespace_test.go deleted file mode 100644 index 1fcc9fe8e9..0000000000 --- a/src/pkg/cluster/namespace_test.go +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -package cluster - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestAdoptZarfManagedLabels(t *testing.T) { - t.Parallel() - - labels := map[string]string{ - "foo": "bar", - AgentLabel: "ignore", - } - adoptedLabels := AdoptZarfManagedLabels(labels) - expectedLabels := map[string]string{ - "foo": "bar", - ZarfManagedByLabel: "zarf", - } - require.Equal(t, expectedLabels, adoptedLabels) -} From ecdac0bdcd4cb474cd7fa24a0cc6c5ed597a13f6 Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Wed, 10 Jul 2024 11:21:14 -0400 Subject: [PATCH 2/8] ci: run e2e tests (#2710) --- src/test/e2e/main_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/e2e/main_test.go b/src/test/e2e/main_test.go index 6c9c283a55..9670335641 100644 --- a/src/test/e2e/main_test.go +++ b/src/test/e2e/main_test.go @@ -38,6 +38,7 @@ func TestMain(m *testing.M) { e2e.ZarfBinPath = filepath.Join("build", test.GetCLIName()) e2e.ApplianceMode = os.Getenv(applianceModeEnvVar) == "true" e2e.ApplianceModeKeep = os.Getenv(applianceModeKeepEnvVar) == "true" + e2e.RunClusterTests = true message.SetLogLevel(message.TraceLevel) From d05c409c1cbfe8bf83bf698acfa835555e3b9d05 Mon Sep 17 00:00:00 2001 From: Philip Laine Date: Wed, 10 Jul 2024 19:51:19 +0200 Subject: [PATCH 3/8] refactor: test and refactor split file (#2708) Co-authored-by: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> --- src/pkg/layout/package.go | 2 +- src/pkg/layout/split.go | 109 +++++++++++++++++++++++++++ src/pkg/layout/split_test.go | 96 +++++++++++++++++++++++ src/pkg/utils/io.go | 142 ----------------------------------- 4 files changed, 206 insertions(+), 143 deletions(-) create mode 100644 src/pkg/layout/split.go create mode 100644 src/pkg/layout/split_test.go diff --git a/src/pkg/layout/package.go b/src/pkg/layout/package.go index e68f77b551..1b99379e81 100644 --- a/src/pkg/layout/package.go +++ b/src/pkg/layout/package.go @@ -243,7 +243,7 @@ func (pp *PackagePaths) ArchivePackage(destinationTarball string, maxPackageSize return fmt.Errorf("unable to split the package archive into multiple files: must be less than 1,000 files") } message.Notef("Package is larger than %dMB, splitting into multiple files", maxPackageSizeMB) - err := utils.SplitFile(destinationTarball, chunkSize) + err := splitFile(destinationTarball, chunkSize) if err != nil { return fmt.Errorf("unable to split the package archive into multiple files: %w", err) } diff --git a/src/pkg/layout/split.go b/src/pkg/layout/split.go new file mode 100644 index 0000000000..aecf295dad --- /dev/null +++ b/src/pkg/layout/split.go @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package layout contains functions for interacting with Zarf's package layout on disk. +package layout + +import ( + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "os" + + "github.com/defenseunicorns/pkg/helpers/v2" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/types" +) + +// splitFile will split the file into chunks and remove the original file. +func splitFile(srcPath string, chunkSize int) error { + srcFile, err := os.Open(srcPath) + if err != nil { + return err + } + defer srcFile.Close() + fi, err := srcFile.Stat() + if err != nil { + return err + } + + title := fmt.Sprintf("[0/%d] MB bytes written", fi.Size()/1000/1000) + progressBar := message.NewProgressBar(fi.Size(), title) + defer progressBar.Close() + + hash := sha256.New() + fileCount := 0 + for { + path := fmt.Sprintf("%s.part%03d", srcPath, fileCount+1) + dstFile, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, helpers.ReadAllWriteUser) + if err != nil { + return err + } + defer dstFile.Close() + + written, copyErr := io.CopyN(dstFile, srcFile, int64(chunkSize)) + if copyErr != nil && !errors.Is(copyErr, io.EOF) { + return err + } + progressBar.Add(int(written)) + title := fmt.Sprintf("[%d/%d] MB bytes written", progressBar.GetCurrent()/1000/1000, fi.Size()/1000/1000) + progressBar.Updatef(title) + + _, err = dstFile.Seek(0, io.SeekStart) + if err != nil { + return err + } + _, err = io.Copy(hash, dstFile) + if err != nil { + return err + } + err = dstFile.Close() + if err != nil { + return err + } + + // EOF error could be returned on 0 bytes written. + if written == 0 { + err = os.Remove(path) + if err != nil { + return err + } + break + } + + fileCount++ + if errors.Is(copyErr, io.EOF) { + break + } + } + + // Remove original file + err = srcFile.Close() + if err != nil { + return err + } + err = os.Remove(srcPath) + if err != nil { + return err + } + + // Write header file + data := types.ZarfSplitPackageData{ + Count: fileCount, + Bytes: fi.Size(), + Sha256Sum: fmt.Sprintf("%x", hash.Sum(nil)), + } + b, err := json.Marshal(data) + if err != nil { + return fmt.Errorf("unable to marshal the split package data: %w", err) + } + path := fmt.Sprintf("%s.part000", srcPath) + if err := os.WriteFile(path, b, helpers.ReadAllWriteUser); err != nil { + return fmt.Errorf("unable to write the file %s: %w", path, err) + } + progressBar.Successf("Package split across %d files", fileCount+1) + + return nil +} diff --git a/src/pkg/layout/split_test.go b/src/pkg/layout/split_test.go new file mode 100644 index 0000000000..b7f48fc9f6 --- /dev/null +++ b/src/pkg/layout/split_test.go @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package layout + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/defenseunicorns/zarf/src/types" + "github.com/stretchr/testify/require" +) + +func TestSplitFile(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + fileSize int + chunkSize int + expectedFileSize int64 + expectedLastFileSize int64 + expectedFileCount int + expectedSha256Sum string + }{ + { + name: "split evenly", + fileSize: 2048, + chunkSize: 16, + expectedFileSize: 16, + expectedLastFileSize: 16, + expectedFileCount: 128, + expectedSha256Sum: "93ecad679eff0df493aaf5d7d615211b0f1d7a919016efb15c98f0b8efb1ba43", + }, + { + name: "split with remainder", + fileSize: 2048, + chunkSize: 10, + expectedFileSize: 10, + expectedLastFileSize: 8, + expectedFileCount: 205, + expectedSha256Sum: "fe8460f4d53d3578aa37191acf55b3db7bbcb706056f4b6b02a0c70f24b0d95a", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + name := "random" + p := filepath.Join(dir, name) + f, err := os.Create(p) + require.NoError(t, err) + b := make([]byte, tt.fileSize) + for i := range tt.fileSize { + b[i] = byte(tt.chunkSize) + } + require.NoError(t, err) + _, err = f.Write(b) + require.NoError(t, err) + f.Close() + + err = splitFile(p, tt.chunkSize) + require.NoError(t, err) + + _, err = os.Stat(p) + require.ErrorIs(t, err, os.ErrNotExist) + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.Len(t, entries, tt.expectedFileCount+1) + for i, entry := range entries[1:] { + require.Equal(t, fmt.Sprintf("%s.part%03d", name, i+1), entry.Name()) + + fi, err := entry.Info() + require.NoError(t, err) + if i == len(entries)-2 { + require.Equal(t, tt.expectedLastFileSize, fi.Size()) + } else { + require.Equal(t, tt.expectedFileSize, fi.Size()) + } + } + + b, err = os.ReadFile(filepath.Join(dir, fmt.Sprintf("%s.part000", name))) + require.NoError(t, err) + var data types.ZarfSplitPackageData + err = json.Unmarshal(b, &data) + require.NoError(t, err) + require.Equal(t, tt.expectedFileCount, data.Count) + require.Equal(t, int64(tt.fileSize), data.Bytes) + require.Equal(t, tt.expectedSha256Sum, data.Sha256Sum) + }) + } +} diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index 8b9592def3..2fd4cfc72b 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -5,17 +5,13 @@ package utils import ( - "crypto/sha256" - "encoding/json" "fmt" - "io" "os" "path/filepath" "github.com/defenseunicorns/pkg/helpers/v2" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/types" ) const ( @@ -73,141 +69,3 @@ func GetFinalExecutableCommand() (string, error) { return zarfCommand, err } - -// SplitFile will take a srcFile path and split it into files based on chunkSizeBytes -// the first file will be a metadata file containing: -// - sha256sum of the original file -// - number of bytes in the original file -// - number of files the srcFile was split into -// SplitFile will delete the original file -// -// Returns: -// - fileNames: list of file paths srcFile was split across -// - sha256sum: sha256sum of the srcFile before splitting -// - err: any errors encountered -func SplitFile(srcPath string, chunkSizeBytes int) (err error) { - var fileNames []string - var sha256sum string - hash := sha256.New() - - // Set buffer size to some multiple of 4096 KiB for modern file system cluster sizes - bufferSize := 16 * 1024 * 1024 // 16 MiB - // if chunkSizeBytes is less than bufferSize, use chunkSizeBytes as bufferSize for simplicity - if chunkSizeBytes < bufferSize { - bufferSize = chunkSizeBytes - } - buf := make([]byte, bufferSize) - - // get file size - fi, err := os.Stat(srcPath) - if err != nil { - return err - } - fileSize := fi.Size() - - // start progress bar - title := fmt.Sprintf("[0/%d] MB bytes written", fileSize/1000/1000) - progressBar := message.NewProgressBar(fileSize, title) - defer progressBar.Close() - - // open srcFile - srcFile, err := os.Open(srcPath) - if err != nil { - return err - } - defer srcFile.Close() - - // create file path starting from part 001 - path := fmt.Sprintf("%s.part001", srcPath) - chunkFile, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, helpers.ReadAllWriteUser) - if err != nil { - return err - } - fileNames = append(fileNames, path) - defer chunkFile.Close() - - // setup counter for tracking how many bytes are left to write to file - chunkBytesRemaining := chunkSizeBytes - // Loop over the tarball hashing as we go and breaking it into chunks based on the chunkSizeBytes - for { - bytesRead, err := srcFile.Read(buf) - - if err != nil { - if err == io.EOF { - // At end of file, break out of loop - break - } - return err - } - - // Pass data to hash - hash.Write(buf[0:bytesRead]) - - // handle if we should split the data between two chunks - if chunkBytesRemaining < bytesRead { - // write the remaining chunk size to file - _, err := chunkFile.Write(buf[0:chunkBytesRemaining]) - if err != nil { - return err - } - err = chunkFile.Close() - if err != nil { - return err - } - - // create new file - path = fmt.Sprintf("%s.part%03d", srcPath, len(fileNames)+1) - chunkFile, err = os.OpenFile(path, os.O_CREATE|os.O_WRONLY, helpers.ReadAllWriteUser) - if err != nil { - return err - } - fileNames = append(fileNames, path) - defer chunkFile.Close() - - // write to new file where we left off - _, err = chunkFile.Write(buf[chunkBytesRemaining:bytesRead]) - if err != nil { - return err - } - - // set chunkBytesRemaining considering how many bytes are already written to new file - chunkBytesRemaining = chunkSizeBytes - (bufferSize - chunkBytesRemaining) - } else { - _, err := chunkFile.Write(buf[0:bytesRead]) - if err != nil { - return err - } - chunkBytesRemaining = chunkBytesRemaining - bytesRead - } - - // update progress bar - progressBar.Add(bufferSize) - title := fmt.Sprintf("[%d/%d] MB bytes written", progressBar.GetCurrent()/1000/1000, fileSize/1000/1000) - progressBar.Updatef(title) - } - srcFile.Close() - _ = os.RemoveAll(srcPath) - - // calculate sha256 sum - sha256sum = fmt.Sprintf("%x", hash.Sum(nil)) - - // Marshal the data into a json file. - jsonData, err := json.Marshal(types.ZarfSplitPackageData{ - Count: len(fileNames), - Bytes: fileSize, - Sha256Sum: sha256sum, - }) - if err != nil { - return fmt.Errorf("unable to marshal the split package data: %w", err) - } - - // write header file - path = fmt.Sprintf("%s.part000", srcPath) - if err := os.WriteFile(path, jsonData, helpers.ReadAllWriteUser); err != nil { - return fmt.Errorf("unable to write the file %s: %w", path, err) - } - fileNames = append(fileNames, path) - progressBar.Successf("Package split across %d files", len(fileNames)) - - return nil -} From 2ef121628231b4cdd60e90db5de89685e25cdfaf Mon Sep 17 00:00:00 2001 From: Philip Laine Date: Wed, 10 Jul 2024 20:28:10 +0200 Subject: [PATCH 4/8] refactor: remove unused message functions and verbose logging (#2712) Co-authored-by: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> --- src/cmd/dev.go | 7 +++- .../agent/hooks/argocd-application.go | 1 - src/internal/agent/hooks/argocd-repository.go | 1 - src/internal/agent/hooks/flux-gitrepo.go | 1 - src/internal/agent/hooks/flux-helmrepo.go | 1 - src/internal/agent/hooks/flux-ocirepo.go | 1 - src/internal/agent/hooks/pods.go | 4 --- src/internal/agent/http/admission/handler.go | 3 -- src/internal/agent/start.go | 2 -- src/pkg/cluster/data.go | 7 +++- src/pkg/cluster/secrets.go | 2 -- src/pkg/cluster/state.go | 6 +++- src/pkg/message/connect.go | 2 -- src/pkg/message/message.go | 36 ------------------- src/pkg/utils/cosign.go | 2 -- src/pkg/utils/io.go | 2 -- src/pkg/utils/yaml.go | 2 -- src/test/e2e/05_tarball_test.go | 11 ------ 18 files changed, 17 insertions(+), 74 deletions(-) diff --git a/src/cmd/dev.go b/src/cmd/dev.go index f6fad80cbe..e304ca2edf 100644 --- a/src/cmd/dev.go +++ b/src/cmd/dev.go @@ -23,6 +23,8 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" + "github.com/pterm/pterm" + "github.com/sergi/go-diff/diffmatchpatch" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -110,7 +112,10 @@ var devTransformGitLinksCmd = &cobra.Command{ processedText := transform.MutateGitURLsInText(message.Warnf, pkgConfig.InitOpts.GitServer.Address, text, pkgConfig.InitOpts.GitServer.PushUsername) // Print the differences - message.PrintDiff(text, processedText) + dmp := diffmatchpatch.New() + diffs := dmp.DiffMain(text, processedText, true) + diffs = dmp.DiffCleanupSemantic(diffs) + pterm.Println(dmp.DiffPrettyText(diffs)) // Ask the user before this destructive action confirm := false diff --git a/src/internal/agent/hooks/argocd-application.go b/src/internal/agent/hooks/argocd-application.go index a09b6d003f..b9197bae51 100644 --- a/src/internal/agent/hooks/argocd-application.go +++ b/src/internal/agent/hooks/argocd-application.go @@ -48,7 +48,6 @@ type ApplicationSource struct { // NewApplicationMutationHook creates a new instance of the ArgoCD Application mutation hook. func NewApplicationMutationHook(ctx context.Context, cluster *cluster.Cluster) operations.Hook { - message.Debug("hooks.NewApplicationMutationHook()") return operations.Hook{ Create: func(r *v1.AdmissionRequest) (*operations.Result, error) { return mutateApplication(ctx, r, cluster) diff --git a/src/internal/agent/hooks/argocd-repository.go b/src/internal/agent/hooks/argocd-repository.go index 40506a395f..23e05df999 100644 --- a/src/internal/agent/hooks/argocd-repository.go +++ b/src/internal/agent/hooks/argocd-repository.go @@ -36,7 +36,6 @@ type RepoCreds struct { // NewRepositorySecretMutationHook creates a new instance of the ArgoCD repository secret mutation hook. func NewRepositorySecretMutationHook(ctx context.Context, cluster *cluster.Cluster) operations.Hook { - message.Debug("hooks.NewRepositoryMutationHook()") return operations.Hook{ Create: func(r *v1.AdmissionRequest) (*operations.Result, error) { return mutateRepositorySecret(ctx, r, cluster) diff --git a/src/internal/agent/hooks/flux-gitrepo.go b/src/internal/agent/hooks/flux-gitrepo.go index 2638da0b5b..8256b40aae 100644 --- a/src/internal/agent/hooks/flux-gitrepo.go +++ b/src/internal/agent/hooks/flux-gitrepo.go @@ -26,7 +26,6 @@ const AgentErrTransformGitURL = "unable to transform the git url" // NewGitRepositoryMutationHook creates a new instance of the git repo mutation hook. func NewGitRepositoryMutationHook(ctx context.Context, cluster *cluster.Cluster) operations.Hook { - message.Debug("hooks.NewGitRepositoryMutationHook()") return operations.Hook{ Create: func(r *v1.AdmissionRequest) (*operations.Result, error) { return mutateGitRepo(ctx, r, cluster) diff --git a/src/internal/agent/hooks/flux-helmrepo.go b/src/internal/agent/hooks/flux-helmrepo.go index 750eeb0406..6fb0e7e712 100644 --- a/src/internal/agent/hooks/flux-helmrepo.go +++ b/src/internal/agent/hooks/flux-helmrepo.go @@ -24,7 +24,6 @@ import ( // NewHelmRepositoryMutationHook creates a new instance of the helm repo mutation hook. func NewHelmRepositoryMutationHook(ctx context.Context, cluster *cluster.Cluster) operations.Hook { - message.Debug("hooks.NewHelmRepositoryMutationHook()") return operations.Hook{ Create: func(r *v1.AdmissionRequest) (*operations.Result, error) { return mutateHelmRepo(ctx, r, cluster) diff --git a/src/internal/agent/hooks/flux-ocirepo.go b/src/internal/agent/hooks/flux-ocirepo.go index c026b2e7cc..00d7ded917 100644 --- a/src/internal/agent/hooks/flux-ocirepo.go +++ b/src/internal/agent/hooks/flux-ocirepo.go @@ -23,7 +23,6 @@ import ( // NewOCIRepositoryMutationHook creates a new instance of the oci repo mutation hook. func NewOCIRepositoryMutationHook(ctx context.Context, cluster *cluster.Cluster) operations.Hook { - message.Debug("hooks.NewOCIRepositoryMutationHook()") return operations.Hook{ Create: func(r *v1.AdmissionRequest) (*operations.Result, error) { return mutateOCIRepo(ctx, r, cluster) diff --git a/src/internal/agent/hooks/pods.go b/src/internal/agent/hooks/pods.go index 299b841f31..a152aa4a8a 100644 --- a/src/internal/agent/hooks/pods.go +++ b/src/internal/agent/hooks/pods.go @@ -22,7 +22,6 @@ import ( // NewPodMutationHook creates a new instance of pods mutation hook. func NewPodMutationHook(ctx context.Context, cluster *cluster.Cluster) operations.Hook { - message.Debug("hooks.NewMutationHook()") return operations.Hook{ Create: func(r *v1.AdmissionRequest) (*operations.Result, error) { return mutatePod(ctx, r, cluster) @@ -34,7 +33,6 @@ func NewPodMutationHook(ctx context.Context, cluster *cluster.Cluster) operation } func parsePod(object []byte) (*corev1.Pod, error) { - message.Debugf("pods.parsePod(%s)", string(object)) var pod corev1.Pod if err := json.Unmarshal(object, &pod); err != nil { return nil, err @@ -43,8 +41,6 @@ func parsePod(object []byte) (*corev1.Pod, error) { } func mutatePod(ctx context.Context, r *v1.AdmissionRequest, cluster *cluster.Cluster) (*operations.Result, error) { - message.Debugf("hooks.mutatePod()(*v1.AdmissionRequest) - %#v , %s/%s: %#v", r.Kind, r.Namespace, r.Name, r.Operation) - pod, err := parsePod(r.Object.Raw) if err != nil { return nil, fmt.Errorf(lang.AgentErrParsePod, err) diff --git a/src/internal/agent/http/admission/handler.go b/src/internal/agent/http/admission/handler.go index 68a17cbee5..f1d3cd8ead 100644 --- a/src/internal/agent/http/admission/handler.go +++ b/src/internal/agent/http/admission/handler.go @@ -35,10 +35,7 @@ func NewHandler() *Handler { // Serve returns an http.HandlerFunc for an admission webhook. func (h *Handler) Serve(hook operations.Hook) http.HandlerFunc { - message.Debugf("http.Serve(%#v)", hook) return func(w http.ResponseWriter, r *http.Request) { - message.Debugf("http.Serve()(writer, %#v)", r.URL) - w.Header().Set("Content-Type", "application/json") if r.Method != http.MethodPost { http.Error(w, lang.AgentErrInvalidMethod, http.StatusMethodNotAllowed) diff --git a/src/internal/agent/start.go b/src/internal/agent/start.go index f110ce2b52..9acd2afcc4 100644 --- a/src/internal/agent/start.go +++ b/src/internal/agent/start.go @@ -29,7 +29,6 @@ const ( // StartWebhook launches the Zarf agent mutating webhook in the cluster. func StartWebhook(ctx context.Context) error { - message.Debug("agent.StartWebhook()") srv, err := agentHttp.NewAdmissionServer(ctx, httpPort) if err != nil { return err @@ -39,7 +38,6 @@ func StartWebhook(ctx context.Context) error { // StartHTTPProxy launches the zarf agent proxy in the cluster. func StartHTTPProxy(ctx context.Context) error { - message.Debug("agent.StartHttpProxy()") return startServer(ctx, agentHttp.NewProxyServer(httpPort)) } diff --git a/src/pkg/cluster/data.go b/src/pkg/cluster/data.go index e47e80cbfe..21a060742e 100644 --- a/src/pkg/cluster/data.go +++ b/src/pkg/cluster/data.go @@ -6,6 +6,7 @@ package cluster import ( "context" + "encoding/json" "fmt" "os" "path/filepath" @@ -46,8 +47,12 @@ func (c *Cluster) HandleDataInjection(ctx context.Context, wg *sync.WaitGroup, d // Pod filter to ensure we only use the current deployment's pods podFilterByInitContainer := func(pod corev1.Pod) bool { + b, err := json.Marshal(pod) + if err != nil { + return false + } // Look everywhere in the pod for a matching data injection marker - return strings.Contains(message.JSONValue(pod), config.GetDataInjectionMarker()) + return strings.Contains(string(b), config.GetDataInjectionMarker()) } // Get the OS shell to execute commands in diff --git a/src/pkg/cluster/secrets.go b/src/pkg/cluster/secrets.go index 09be8beba5..d083f6bbf4 100644 --- a/src/pkg/cluster/secrets.go +++ b/src/pkg/cluster/secrets.go @@ -89,8 +89,6 @@ func (c *Cluster) GenerateRegistryPullCreds(ctx context.Context, namespace, name // GenerateGitPullCreds generates a secret containing the git credentials. func (c *Cluster) GenerateGitPullCreds(namespace, name string, gitServerInfo types.GitServerInfo) *corev1.Secret { - message.Debugf("k8s.GenerateGitPullCreds(%s, %s, gitServerInfo)", namespace, name) - gitServerSecret := &corev1.Secret{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), diff --git a/src/pkg/cluster/state.go b/src/pkg/cluster/state.go index ad4ff27736..65fba1431a 100644 --- a/src/pkg/cluster/state.go +++ b/src/pkg/cluster/state.go @@ -250,7 +250,11 @@ func (c *Cluster) debugPrintZarfState(state *types.ZarfState) { // this is a shallow copy, nested pointers WILL NOT be copied oldState := *state sanitized := c.sanitizeZarfState(&oldState) - message.Debugf("ZarfState - %s", message.JSONValue(sanitized)) + b, err := json.MarshalIndent(sanitized, "", " ") + if err != nil { + return + } + message.Debugf("ZarfState - %s", string(b)) } // SaveZarfState takes a given state and persists it to the Zarf/zarf-state secret. diff --git a/src/pkg/message/connect.go b/src/pkg/message/connect.go index 1c7e43cfe1..75f87fdd64 100644 --- a/src/pkg/message/connect.go +++ b/src/pkg/message/connect.go @@ -12,8 +12,6 @@ import ( // PrintConnectStringTable prints a table of connect strings. func PrintConnectStringTable(connectStrings types.ConnectStrings) { - Debugf("message.PrintConnectStringTable(%#v)", connectStrings) - if len(connectStrings) > 0 { connectData := [][]string{} // Loop over each connectStrings and convert to a string matrix diff --git a/src/pkg/message/message.go b/src/pkg/message/message.go index 591165559e..0ead8b0e01 100644 --- a/src/pkg/message/message.go +++ b/src/pkg/message/message.go @@ -5,10 +5,8 @@ package message import ( - "encoding/json" "fmt" "io" - "net/http" "os" "strings" "time" @@ -17,7 +15,6 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/fatih/color" "github.com/pterm/pterm" - "github.com/sergi/go-diff/diffmatchpatch" ) // LogLevel is the level of logging to display. @@ -97,11 +94,6 @@ func SetLogLevel(lvl LogLevel) { } } -// GetLogLevel returns the current log level. -func GetLogLevel() LogLevel { - return logLevel -} - // DisableColor disables color in output func DisableColor() { pterm.DisableColor() @@ -129,14 +121,6 @@ func Debugf(format string, a ...any) { debugPrinter(2, message) } -// ErrorWebf prints an error message and returns a web response. -func ErrorWebf(err any, w http.ResponseWriter, format string, a ...any) { - debugPrinter(2, err) - message := fmt.Sprintf(format, a...) - Warn(message) - http.Error(w, message, http.StatusInternalServerError) -} - // Warn prints a warning message. func Warn(message string) { Warnf("%s", message) @@ -242,15 +226,6 @@ func HorizontalRule() { pterm.Println(RuleLine) } -// JSONValue prints any value as JSON. -func JSONValue(value any) string { - bytes, err := json.MarshalIndent(value, "", " ") - if err != nil { - debugPrinter(2, fmt.Sprintf("ERROR marshalling json: %s", err.Error())) - } - return string(bytes) -} - // Paragraph formats text into a paragraph matching the TermWidth func Paragraph(format string, a ...any) string { return Paragraphn(TermWidth, format, a...) @@ -269,17 +244,6 @@ func Paragraphn(n int, format string, a ...any) string { return strings.Join(formattedLines, "\n") } -// PrintDiff prints the differences between a and b with a as original and b as new -func PrintDiff(textA, textB string) { - dmp := diffmatchpatch.New() - - diffs := dmp.DiffMain(textA, textB, true) - - diffs = dmp.DiffCleanupSemantic(diffs) - - pterm.Println(dmp.DiffPrettyText(diffs)) -} - // Table prints a padded table containing the specified header and data func Table(header []string, data [][]string) { pterm.Println() diff --git a/src/pkg/utils/cosign.go b/src/pkg/utils/cosign.go index d4b8fe695d..fc66d92ba9 100644 --- a/src/pkg/utils/cosign.go +++ b/src/pkg/utils/cosign.go @@ -50,8 +50,6 @@ func Sget(ctx context.Context, image, key string, out io.Writer) error { // Remove the custom protocol header from the url image = strings.TrimPrefix(image, helpers.SGETURLPrefix) - message.Debugf("utils.Sget: image=%s, key=%s", image, key) - spinner := message.NewProgressSpinner("Loading signed file %s", image) defer spinner.Stop() diff --git a/src/pkg/utils/io.go b/src/pkg/utils/io.go index 2fd4cfc72b..16c1abf377 100755 --- a/src/pkg/utils/io.go +++ b/src/pkg/utils/io.go @@ -38,8 +38,6 @@ func MakeTempDir(basePath string) (string, error) { // GetFinalExecutablePath returns the absolute path to the current executable, following any symlinks along the way. func GetFinalExecutablePath() (string, error) { - message.Debug("utils.GetExecutablePath()") - binaryPath, err := os.Executable() if err != nil { return "", err diff --git a/src/pkg/utils/yaml.go b/src/pkg/utils/yaml.go index f2c73d5f77..bc49d46502 100644 --- a/src/pkg/utils/yaml.go +++ b/src/pkg/utils/yaml.go @@ -124,8 +124,6 @@ func AddRootHint(hints map[string]string, rootKey string, hintText string) map[s // ReadYaml reads a yaml file and unmarshals it into a given config. func ReadYaml(path string, destConfig any) error { - message.Debugf("Reading YAML at %s", path) - file, err := os.ReadFile(path) if err != nil { return err diff --git a/src/test/e2e/05_tarball_test.go b/src/test/e2e/05_tarball_test.go index 7293b378bc..e4ebfde06d 100644 --- a/src/test/e2e/05_tarball_test.go +++ b/src/test/e2e/05_tarball_test.go @@ -13,7 +13,6 @@ import ( "github.com/defenseunicorns/pkg/helpers/v2" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/types" "github.com/stretchr/testify/require" @@ -98,10 +97,6 @@ func TestReproducibleTarballs(t *testing.T) { err = utils.ReadYaml(filepath.Join(unpack1, layout.ZarfYAML), &pkg1) require.NoError(t, err) - b, err := os.ReadFile(filepath.Join(unpack1, layout.Checksums)) - require.NoError(t, err) - checksums1 := string(b) - e2e.CleanFiles(unpack1, tb) stdOut, stdErr, err = e2e.Zarf("package", "create", createPath, "--confirm", "--output", tmp) @@ -114,11 +109,5 @@ func TestReproducibleTarballs(t *testing.T) { err = utils.ReadYaml(filepath.Join(unpack2, layout.ZarfYAML), &pkg2) require.NoError(t, err) - b, err = os.ReadFile(filepath.Join(unpack2, layout.Checksums)) - require.NoError(t, err) - checksums2 := string(b) - - message.PrintDiff(checksums1, checksums2) - require.Equal(t, pkg1.Metadata.AggregateChecksum, pkg2.Metadata.AggregateChecksum) } From f2c77ef21623999202c43592927a8096b0814606 Mon Sep 17 00:00:00 2001 From: Philip Laine Date: Wed, 10 Jul 2024 20:39:58 +0200 Subject: [PATCH 5/8] refactor: connect command list printing (#2703) Co-authored-by: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> --- src/cmd/connect.go | 135 +++++++++++----------- src/config/config.go | 4 - src/internal/packager/helm/post-render.go | 6 +- src/pkg/cluster/tunnel.go | 32 ++--- src/pkg/cluster/tunnel_test.go | 38 ++++++ 5 files changed, 123 insertions(+), 92 deletions(-) diff --git a/src/cmd/connect.go b/src/cmd/connect.go index 7111abf813..89eb341b45 100644 --- a/src/cmd/connect.go +++ b/src/cmd/connect.go @@ -5,14 +5,14 @@ package cmd import ( - "context" "fmt" + "github.com/spf13/cobra" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/cluster" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/utils/exec" - "github.com/spf13/cobra" ) var ( @@ -22,83 +22,80 @@ var ( connectLocalPort int connectRemotePort int cliOnly bool +) - connectCmd = &cobra.Command{ - Use: "connect { REGISTRY | GIT | connect-name }", - Aliases: []string{"c"}, - Short: lang.CmdConnectShort, - Long: lang.CmdConnectLong, - RunE: func(cmd *cobra.Command, args []string) error { - var target string - if len(args) > 0 { - target = args[0] - } - spinner := message.NewProgressSpinner(lang.CmdConnectPreparingTunnel, target) - defer spinner.Stop() - c, err := cluster.NewCluster() - if err != nil { - return err - } +var connectCmd = &cobra.Command{ + Use: "connect { REGISTRY | GIT | connect-name }", + Aliases: []string{"c"}, + Short: lang.CmdConnectShort, + Long: lang.CmdConnectLong, + RunE: func(cmd *cobra.Command, args []string) error { + target := "" + if len(args) > 0 { + target = args[0] + } - ctx := cmd.Context() + spinner := message.NewProgressSpinner(lang.CmdConnectPreparingTunnel, target) + defer spinner.Stop() - var tunnel *cluster.Tunnel - if connectResourceName != "" { - zt := cluster.NewTunnelInfo(connectNamespace, connectResourceType, connectResourceName, "", connectLocalPort, connectRemotePort) - tunnel, err = c.ConnectTunnelInfo(ctx, zt) - } else { - tunnel, err = c.Connect(ctx, target) - } - if err != nil { - return fmt.Errorf("unable to connect to the service: %w", err) - } + c, err := cluster.NewCluster() + if err != nil { + return err + } - defer tunnel.Close() - url := tunnel.FullURL() + ctx := cmd.Context() - // Dump the tunnel URL to the console for other tools to use. - fmt.Print(url) + var tunnel *cluster.Tunnel + if connectResourceName == "" { + tunnel, err = c.Connect(ctx, target) + } else { + zt := cluster.NewTunnelInfo(connectNamespace, connectResourceType, connectResourceName, "", connectLocalPort, connectRemotePort) + tunnel, err = c.ConnectTunnelInfo(ctx, zt) + } + if err != nil { + return fmt.Errorf("unable to connect to the service: %w", err) + } + defer tunnel.Close() - if cliOnly { - spinner.Updatef(lang.CmdConnectEstablishedCLI, url) - } else { - spinner.Updatef(lang.CmdConnectEstablishedWeb, url) + // Dump the tunnel URL to the console for other tools to use. + fmt.Print(tunnel.FullURL()) - if err := exec.LaunchURL(url); err != nil { - message.Debug(err) - } + if cliOnly { + spinner.Updatef(lang.CmdConnectEstablishedCLI, tunnel.FullURL()) + } else { + spinner.Updatef(lang.CmdConnectEstablishedWeb, tunnel.FullURL()) + if err := exec.LaunchURL(tunnel.FullURL()); err != nil { + message.Debug(err) } + } - // Wait for the interrupt signal or an error. - select { - case <-ctx.Done(): - spinner.Successf(lang.CmdConnectTunnelClosed, url) - case err = <-tunnel.ErrChan(): - return fmt.Errorf("lost connection to the service: %w", err) - } - return nil - }, - } - - connectListCmd = &cobra.Command{ - Use: "list", - Aliases: []string{"l"}, - Short: lang.CmdConnectListShort, - RunE: func(cmd *cobra.Command, _ []string) error { - timeoutCtx, cancel := context.WithTimeout(cmd.Context(), cluster.DefaultTimeout) - defer cancel() - c, err := cluster.NewClusterWithWait(timeoutCtx) - if err != nil { - return err - } - err = c.PrintConnectTable(cmd.Context()) - if err != nil { - return err - } + select { + case <-ctx.Done(): + spinner.Successf(lang.CmdConnectTunnelClosed, tunnel.FullURL()) return nil - }, - } -) + case err = <-tunnel.ErrChan(): + return fmt.Errorf("lost connection to the service: %w", err) + } + }, +} + +var connectListCmd = &cobra.Command{ + Use: "list", + Aliases: []string{"l"}, + Short: lang.CmdConnectListShort, + RunE: func(cmd *cobra.Command, _ []string) error { + c, err := cluster.NewCluster() + if err != nil { + return err + } + connections, err := c.ListConnections(cmd.Context()) + if err != nil { + return err + } + message.PrintConnectStringTable(connections) + return nil + }, +} func init() { rootCmd.AddCommand(connectCmd) diff --git a/src/config/config.go b/src/config/config.go index d3b0c07a21..f9dfc580ab 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -21,10 +21,6 @@ const ( ZarfAgentHost = "agent-hook.zarf.svc" - ZarfConnectLabelName = "zarf.dev/connect-name" - ZarfConnectAnnotationDescription = "zarf.dev/connect-description" - ZarfConnectAnnotationURL = "zarf.dev/connect-url" - ZarfCleanupScriptsPath = "/opt/zarf" ZarfPackagePrefix = "zarf-package-" diff --git a/src/internal/packager/helm/post-render.go b/src/internal/packager/helm/post-render.go index 43bc102a6b..b03aa6a93a 100644 --- a/src/internal/packager/helm/post-render.go +++ b/src/internal/packager/helm/post-render.go @@ -254,14 +254,14 @@ func (r *renderer) editHelmResources(ctx context.Context, resources []releaseuti if annotations == nil { annotations = map[string]string{} } - if key, keyExists := labels[config.ZarfConnectLabelName]; keyExists { + if key, keyExists := labels[cluster.ZarfConnectLabelName]; keyExists { // If there is a zarf-connect label message.Debugf("Match helm service %s for zarf connection %s", rawData.GetName(), key) // Add the connectString for processing later in the deployment r.connectStrings[key] = types.ConnectString{ - Description: annotations[config.ZarfConnectAnnotationDescription], - URL: annotations[config.ZarfConnectAnnotationURL], + Description: annotations[cluster.ZarfConnectAnnotationDescription], + URL: annotations[cluster.ZarfConnectAnnotationURL], } } } diff --git a/src/pkg/cluster/tunnel.go b/src/pkg/cluster/tunnel.go index 10a287986d..3f8433ed9f 100644 --- a/src/pkg/cluster/tunnel.go +++ b/src/pkg/cluster/tunnel.go @@ -23,13 +23,16 @@ import ( "k8s.io/client-go/transport/spdy" "github.com/defenseunicorns/pkg/helpers/v2" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" ) // Zarf specific connect strings const ( + ZarfConnectLabelName = "zarf.dev/connect-name" + ZarfConnectAnnotationDescription = "zarf.dev/connect-description" + ZarfConnectAnnotationURL = "zarf.dev/connect-url" + ZarfRegistry = "REGISTRY" ZarfGit = "GIT" ZarfInjector = "INJECTOR" @@ -64,33 +67,30 @@ func NewTunnelInfo(namespace, resourceType, resourceName, urlSuffix string, loca } } -// PrintConnectTable will print a table of all Zarf connect matches found in the cluster. -func (c *Cluster) PrintConnectTable(ctx context.Context) error { +// ListConnections will return a list of all Zarf connect matches found in the cluster. +func (c *Cluster) ListConnections(ctx context.Context) (types.ConnectStrings, error) { selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ MatchExpressions: []metav1.LabelSelectorRequirement{{ Operator: metav1.LabelSelectorOpExists, - Key: config.ZarfConnectLabelName, + Key: ZarfConnectLabelName, }}, }) if err != nil { - return err + return nil, err } serviceList, err := c.Clientset.CoreV1().Services("").List(ctx, metav1.ListOptions{LabelSelector: selector.String()}) if err != nil { - return err + return nil, err } - - connections := make(types.ConnectStrings) + connections := types.ConnectStrings{} for _, svc := range serviceList.Items { - name := svc.Labels[config.ZarfConnectLabelName] - // Add the connectString for processing later in the deployment. + name := svc.Labels[ZarfConnectLabelName] connections[name] = types.ConnectString{ - Description: svc.Annotations[config.ZarfConnectAnnotationDescription], - URL: svc.Annotations[config.ZarfConnectAnnotationURL], + Description: svc.Annotations[ZarfConnectAnnotationDescription], + URL: svc.Annotations[ZarfConnectAnnotationURL], } } - message.PrintConnectStringTable(connections) - return nil + return connections, nil } // Connect will establish a tunnel to the specified target. @@ -189,7 +189,7 @@ func (c *Cluster) checkForZarfConnectLabel(ctx context.Context, name string) (Tu selector, err := metav1.LabelSelectorAsSelector(&metav1.LabelSelector{ MatchLabels: map[string]string{ - config.ZarfConnectLabelName: name, + ZarfConnectLabelName: name, }, }) if err != nil { @@ -222,7 +222,7 @@ func (c *Cluster) checkForZarfConnectLabel(ctx context.Context, name string) (Tu } // Add the url suffix too. - zt.urlSuffix = svc.Annotations[config.ZarfConnectAnnotationURL] + zt.urlSuffix = svc.Annotations[ZarfConnectAnnotationURL] message.Debugf("tunnel connection match: %s/%s on port %d", svc.Namespace, svc.Name, zt.remotePort) } else { diff --git a/src/pkg/cluster/tunnel_test.go b/src/pkg/cluster/tunnel_test.go index f9dfc7b5c8..fd1866da19 100644 --- a/src/pkg/cluster/tunnel_test.go +++ b/src/pkg/cluster/tunnel_test.go @@ -4,13 +4,51 @@ package cluster import ( + "context" "testing" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + + "github.com/defenseunicorns/zarf/src/types" ) +func TestListConnections(t *testing.T) { + t.Parallel() + + c := &Cluster{ + Clientset: fake.NewSimpleClientset(), + } + svc := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "connect", + Labels: map[string]string{ + ZarfConnectLabelName: "connect name", + }, + Annotations: map[string]string{ + ZarfConnectAnnotationDescription: "description", + ZarfConnectAnnotationURL: "url", + }, + }, + Spec: corev1.ServiceSpec{}, + } + _, err := c.Clientset.CoreV1().Services(svc.ObjectMeta.Namespace).Create(context.Background(), &svc, metav1.CreateOptions{}) + require.NoError(t, err) + + connections, err := c.ListConnections(context.Background()) + require.NoError(t, err) + expectedConnections := types.ConnectStrings{ + "connect name": types.ConnectString{ + Description: "description", + URL: "url", + }, + } + require.Equal(t, expectedConnections, connections) +} + func TestServiceInfoFromNodePortURL(t *testing.T) { t.Parallel() From c89e2dd65bd00353cd28e591f2fd45607fc558d0 Mon Sep 17 00:00:00 2001 From: Xander Grzywinski Date: Thu, 11 Jul 2024 05:11:26 -0700 Subject: [PATCH 6/8] docs: add contributing doc to root and add tsc (#2706) Signed-off-by: Xander Grzywinski --- .github/CONTRIBUTING.md => CONTRIBUTING.md | 28 ++++++++++++++++++- .../docs/contribute/contributor-guide.mdx | 2 +- 2 files changed, 28 insertions(+), 2 deletions(-) rename .github/CONTRIBUTING.md => CONTRIBUTING.md (88%) diff --git a/.github/CONTRIBUTING.md b/CONTRIBUTING.md similarity index 88% rename from .github/CONTRIBUTING.md rename to CONTRIBUTING.md index 4dda9e1668..700f161b7a 100644 --- a/.github/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ First off, thanks so much for wanting to help out! :tada: -This document describes the steps and requirements for contributing a bug fix or feature in a Pull Request to Zarf! If you have any questions about the process or the pull request you are working on feel free to reach out in the [Zarf Dev Kubernetes Slack Channel](https://kubernetes.slack.com/archives/C03BP9Z3CMA). +This document describes the steps and requirements for contributing a bug fix or feature in a Pull Request to Zarf! If you have any questions about the process or the pull request you are working on feel free to reach out in the [Zarf Dev Kubernetes Slack Channel](https://kubernetes.slack.com/archives/C03BP9Z3CMA). The doc also details a bit about the governance structure of the project. ## Developer Experience @@ -83,3 +83,29 @@ adr new -l "15:Amends:Amended by" Use store-bought butter for all waffle making # Get full help docs. There are all sorts of other helpful commands that help manage the decision log. adr help ``` + +## Governance + +### Technical Steering Committee +The Technical Steering Committee (the “TSC”) will be responsible for all technical oversight of the project. The TSC may elect a TSC Chair, who will preside over meetings of the TSC and will serve until their resignation or replacement by the TSC. Current members of the TSC include: + +#### Austin Abro +Affiliation: Defense Unicorns +GitHub: @AustinAbro321 + +#### Danny Gershman +Affiliation: Radius Method +GitHub: @dgershman + +#### Jeff McCoy (TSC Chair) +Affiliation: Defense Unicorns +GitHub: @jeff-mccoy + +#### Sarah Christoff +Affiliation: Defense Unicorns +GitHub: @schristoff-du + +#### Wayne Starr +Affiliation: Defense Unicorns +GitHub: @Racer159 + diff --git a/site/src/content/docs/contribute/contributor-guide.mdx b/site/src/content/docs/contribute/contributor-guide.mdx index 02277d13d4..70b1174c64 100644 --- a/site/src/content/docs/contribute/contributor-guide.mdx +++ b/site/src/content/docs/contribute/contributor-guide.mdx @@ -5,7 +5,7 @@ tableOfContents: false --- import StripH1 from "@components/StripH1.astro"; -import Contributing from "../../../../../.github/CONTRIBUTING.md"; +import Contributing from "../../../../../CONTRIBUTING.md"; From eca0c530a04c3ed155c03761eb45bd16ba717b79 Mon Sep 17 00:00:00 2001 From: Jason Washburn <35488541+jasonwashburn@users.noreply.github.com> Date: Thu, 11 Jul 2024 09:36:52 -0500 Subject: [PATCH 7/8] fix: remove unpinned image warning in lint for cosign signatures (#2681) --- src/pkg/packager/lint/lint.go | 12 +++++++++++- src/pkg/packager/lint/lint_test.go | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/lint/lint.go b/src/pkg/packager/lint/lint.go index ddcdea76db..27a823deff 100644 --- a/src/pkg/packager/lint/lint.go +++ b/src/pkg/packager/lint/lint.go @@ -64,7 +64,6 @@ func lintComponents(ctx context.Context, pkg types.ZarfPackage, createOpts types } chain, err := composer.NewImportChain(ctx, component, i, pkg.Metadata.Name, arch, createOpts.Flavor) - if err != nil { return nil, err } @@ -144,9 +143,20 @@ func isPinnedImage(image string) (bool, error) { } return false, err } + if isCosignSignature(transformedImage.Tag) || isCosignAttestation(transformedImage.Tag) { + return true, nil + } return (transformedImage.Digest != ""), err } +func isCosignSignature(image string) bool { + return strings.HasSuffix(image, ".sig") +} + +func isCosignAttestation(image string) bool { + return strings.HasSuffix(image, ".att") +} + func isPinnedRepo(repo string) bool { return (strings.Contains(repo, "@")) } diff --git a/src/pkg/packager/lint/lint_test.go b/src/pkg/packager/lint/lint_test.go index 78d4c4102d..d20257b23a 100644 --- a/src/pkg/packager/lint/lint_test.go +++ b/src/pkg/packager/lint/lint_test.go @@ -217,10 +217,14 @@ func TestValidateComponent(t *testing.T) { t.Parallel() unpinnedImage := "registry.com:9001/whatever/image:1.0.0" badImage := "badimage:badimage@@sha256:3fbc632167424a6d997e74f5" + cosignSignature := "ghcr.io/stefanprodan/podinfo:sha256-57a654ace69ec02ba8973093b6a786faa15640575fbf0dbb603db55aca2ccec8.sig" + cosignAttestation := "ghcr.io/stefanprodan/podinfo:sha256-57a654ace69ec02ba8973093b6a786faa15640575fbf0dbb603db55aca2ccec8.att" component := types.ZarfComponent{Images: []string{ unpinnedImage, "busybox:latest@sha256:3fbc632167424a6d997e74f52b878d7cc478225cffac6bc977eedfe51c7f4e79", badImage, + cosignSignature, + cosignAttestation, }} findings := checkForUnpinnedImages(component, 0) expected := []types.PackageFinding{ @@ -333,6 +337,16 @@ func TestValidateComponent(t *testing.T) { expected: true, err: nil, }, + { + input: "ghcr.io/stefanprodan/podinfo:sha256-57a654ace69ec02ba8973093b6a786faa15640575fbf0dbb603db55aca2ccec8.sig", + expected: true, + err: nil, + }, + { + input: "ghcr.io/stefanprodan/podinfo:sha256-57a654ace69ec02ba8973093b6a786faa15640575fbf0dbb603db55aca2ccec8.att", + expected: true, + err: nil, + }, } for _, tc := range tests { t.Run(tc.input, func(t *testing.T) { From a567687b84fc69ced50e4e674f79295114024aed Mon Sep 17 00:00:00 2001 From: Austin Abro <37223396+AustinAbro321@users.noreply.github.com> Date: Fri, 12 Jul 2024 12:35:50 -0400 Subject: [PATCH 8/8] test: simplifying e2e test checks (#2721) --- .pre-commit-config.yaml | 2 +- src/test/common.go | 9 --------- src/test/e2e/20_zarf_init_test.go | 1 - src/test/e2e/21_connect_creds_test.go | 2 -- src/test/e2e/22_git_and_gitops_test.go | 3 --- src/test/e2e/23_data_injection_test.go | 1 - src/test/e2e/24_variables_test.go | 1 - src/test/e2e/25_helm_test.go | 1 - src/test/e2e/26_simple_packages_test.go | 3 --- src/test/e2e/27_deploy_regression_test.go | 2 -- src/test/e2e/28_wait_test.go | 1 - src/test/e2e/29_config_file_test.go | 1 - src/test/e2e/30_component_action_cluster_test.go | 2 -- src/test/e2e/31_checksum_and_signature_test.go | 1 - src/test/e2e/32_component_webhooks_test.go | 1 - src/test/e2e/34_custom_init_package_test.go | 2 +- src/test/e2e/35_custom_retries_test.go | 1 - src/test/e2e/50_oci_publish_deploy_test.go | 1 - src/test/e2e/51_oci_compose_test.go | 1 - src/test/e2e/99_appliance_remove_test.go | 2 -- src/test/e2e/99_yolo_test.go | 3 --- src/test/e2e/main_test.go | 1 - src/test/nightly/ecr_publish_test.go | 2 -- 23 files changed, 2 insertions(+), 42 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b77cc86c1c..ad9ea6d447 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,7 @@ repos: args: - "--allow-missing-credentials" - id: detect-private-key - exclude: "src/test/e2e/30_config_file_test.go" + exclude: "src/test/e2e/29_config_file_test.go" - id: end-of-file-fixer exclude: site/src/content/docs/commands/.* - id: fix-byte-order-marker diff --git a/src/test/common.go b/src/test/common.go index 4eb9844045..70492f014c 100644 --- a/src/test/common.go +++ b/src/test/common.go @@ -27,7 +27,6 @@ type ZarfE2ETest struct { Arch string ApplianceMode bool ApplianceModeKeep bool - RunClusterTests bool } var logRegex = regexp.MustCompile(`Saving log file to (?P.*?\.log)`) @@ -53,14 +52,6 @@ func GetCLIName() string { return binaryName } -// SetupWithCluster performs actions for each test that requires a K8s cluster. -func (e2e *ZarfE2ETest) SetupWithCluster(t *testing.T) { - if !e2e.RunClusterTests { - t.Skip("") - } - _ = exec.CmdWithPrint("sh", "-c", fmt.Sprintf("%s tools kubectl describe nodes | grep -A 99 Non-terminated", e2e.ZarfBinPath)) -} - // Zarf executes a Zarf command. func (e2e *ZarfE2ETest) Zarf(args ...string) (string, string, error) { if !slices.Contains(args, "--tmpdir") && !slices.Contains(args, "tools") { diff --git a/src/test/e2e/20_zarf_init_test.go b/src/test/e2e/20_zarf_init_test.go index 92fe26bc98..f54fc54f85 100644 --- a/src/test/e2e/20_zarf_init_test.go +++ b/src/test/e2e/20_zarf_init_test.go @@ -18,7 +18,6 @@ import ( func TestZarfInit(t *testing.T) { t.Log("E2E: Zarf init") - e2e.SetupWithCluster(t) initComponents := "git-server" if e2e.ApplianceMode { diff --git a/src/test/e2e/21_connect_creds_test.go b/src/test/e2e/21_connect_creds_test.go index 5c68277ff7..3a929ac596 100644 --- a/src/test/e2e/21_connect_creds_test.go +++ b/src/test/e2e/21_connect_creds_test.go @@ -23,7 +23,6 @@ type RegistryResponse struct { func TestConnectAndCreds(t *testing.T) { t.Log("E2E: Connect") - e2e.SetupWithCluster(t) prevAgentSecretData, _, err := e2e.Kubectl("get", "secret", "agent-hook-tls", "-n", "zarf", "-o", "jsonpath={.data}") require.NoError(t, err) @@ -44,7 +43,6 @@ func TestConnectAndCreds(t *testing.T) { func TestMetrics(t *testing.T) { t.Log("E2E: Emits metrics") - e2e.SetupWithCluster(t) c, err := cluster.NewCluster() require.NoError(t, err) diff --git a/src/test/e2e/22_git_and_gitops_test.go b/src/test/e2e/22_git_and_gitops_test.go index 8b53bacfaf..956ded5ed9 100644 --- a/src/test/e2e/22_git_and_gitops_test.go +++ b/src/test/e2e/22_git_and_gitops_test.go @@ -21,7 +21,6 @@ import ( func TestGit(t *testing.T) { t.Log("E2E: Git") - e2e.SetupWithCluster(t) buildPath := filepath.Join("src", "test", "packages", "22-git-data") stdOut, stdErr, err := e2e.Zarf("package", "create", buildPath, "-o=build", "--confirm") @@ -48,14 +47,12 @@ func TestGit(t *testing.T) { func TestGitOpsFlux(t *testing.T) { t.Log("E2E: GitOps / Flux") - e2e.SetupWithCluster(t) waitFluxPodInfoDeployment(t) } func TestGitOpsArgoCD(t *testing.T) { t.Log("E2E: ArgoCD / Flux") - e2e.SetupWithCluster(t) waitArgoDeployment(t) } diff --git a/src/test/e2e/23_data_injection_test.go b/src/test/e2e/23_data_injection_test.go index efbce9bc13..9e4bbef7c3 100644 --- a/src/test/e2e/23_data_injection_test.go +++ b/src/test/e2e/23_data_injection_test.go @@ -19,7 +19,6 @@ import ( func TestDataInjection(t *testing.T) { t.Log("E2E: Data injection") - e2e.SetupWithCluster(t) ctx := context.Background() diff --git a/src/test/e2e/24_variables_test.go b/src/test/e2e/24_variables_test.go index c0f546ad83..765c8902bf 100644 --- a/src/test/e2e/24_variables_test.go +++ b/src/test/e2e/24_variables_test.go @@ -15,7 +15,6 @@ import ( func TestVariables(t *testing.T) { t.Log("E2E: Package variables") - e2e.SetupWithCluster(t) evilSrc := filepath.Join("src", "test", "packages", "24-evil-variables") evilPath := fmt.Sprintf("zarf-package-evil-variables-%s.tar.zst", e2e.Arch) diff --git a/src/test/e2e/25_helm_test.go b/src/test/e2e/25_helm_test.go index 7c643288c0..280ba3d7ec 100644 --- a/src/test/e2e/25_helm_test.go +++ b/src/test/e2e/25_helm_test.go @@ -17,7 +17,6 @@ var helmChartsPkg string func TestHelm(t *testing.T) { t.Log("E2E: Helm chart") - e2e.SetupWithCluster(t) helmChartsPkg = filepath.Join("build", fmt.Sprintf("zarf-package-helm-charts-%s-0.0.1.tar.zst", e2e.Arch)) diff --git a/src/test/e2e/26_simple_packages_test.go b/src/test/e2e/26_simple_packages_test.go index 347aad89fd..97a67423e3 100644 --- a/src/test/e2e/26_simple_packages_test.go +++ b/src/test/e2e/26_simple_packages_test.go @@ -17,7 +17,6 @@ import ( func TestDosGames(t *testing.T) { t.Log("E2E: Dos games") - e2e.SetupWithCluster(t) path := filepath.Join("build", fmt.Sprintf("zarf-package-dos-games-%s-1.0.0.tar.zst", e2e.Arch)) @@ -53,7 +52,6 @@ func TestDosGames(t *testing.T) { func TestManifests(t *testing.T) { t.Log("E2E: Local, Remote, and Kustomize Manifests") - e2e.SetupWithCluster(t) path := filepath.Join("build", fmt.Sprintf("zarf-package-manifests-%s-0.0.1.tar.zst", e2e.Arch)) @@ -68,7 +66,6 @@ func TestManifests(t *testing.T) { func TestAgentIgnore(t *testing.T) { t.Log("E2E: Test Manifests that are Agent Ignored") - e2e.SetupWithCluster(t) testCreate := filepath.Join("src", "test", "packages", "26-agent-ignore") testDeploy := filepath.Join("build", fmt.Sprintf("zarf-package-agent-ignore-namespace-%s.tar.zst", e2e.Arch)) diff --git a/src/test/e2e/27_deploy_regression_test.go b/src/test/e2e/27_deploy_regression_test.go index 0f70365a63..fec8d55433 100644 --- a/src/test/e2e/27_deploy_regression_test.go +++ b/src/test/e2e/27_deploy_regression_test.go @@ -15,7 +15,6 @@ import ( func TestGHCRDeploy(t *testing.T) { t.Log("E2E: GHCR OCI deploy") - e2e.SetupWithCluster(t) var sha string // shas for package published 2023-08-08T22:13:51Z @@ -36,7 +35,6 @@ func TestGHCRDeploy(t *testing.T) { func TestCosignDeploy(t *testing.T) { t.Log("E2E: Cosign deploy") - e2e.SetupWithCluster(t) // Test with command from https://docs.zarf.dev/getting-started/install/ command := fmt.Sprintf("%s package deploy sget://defenseunicorns/zarf-hello-world:$(uname -m) --confirm", e2e.ZarfBinPath) diff --git a/src/test/e2e/28_wait_test.go b/src/test/e2e/28_wait_test.go index 16d2ddff4f..0de68f243e 100644 --- a/src/test/e2e/28_wait_test.go +++ b/src/test/e2e/28_wait_test.go @@ -27,7 +27,6 @@ func zarfCommandWStruct(e2e test.ZarfE2ETest, path string) (result zarfCommandRe func TestNoWait(t *testing.T) { t.Log("E2E: Helm Wait") - e2e.SetupWithCluster(t) stdOut, stdErr, err := e2e.Zarf("package", "create", "src/test/packages/28-helm-no-wait", "-o=build", "--confirm") require.NoError(t, err, stdOut, stdErr) diff --git a/src/test/e2e/29_config_file_test.go b/src/test/e2e/29_config_file_test.go index 5095efa91f..40c66b4204 100644 --- a/src/test/e2e/29_config_file_test.go +++ b/src/test/e2e/29_config_file_test.go @@ -15,7 +15,6 @@ import ( func TestConfigFile(t *testing.T) { t.Log("E2E: Config file") - e2e.SetupWithCluster(t) var ( path = fmt.Sprintf("zarf-package-config-file-%s.tar.zst", e2e.Arch) diff --git a/src/test/e2e/30_component_action_cluster_test.go b/src/test/e2e/30_component_action_cluster_test.go index 47efd3a6a4..5d6658152f 100644 --- a/src/test/e2e/30_component_action_cluster_test.go +++ b/src/test/e2e/30_component_action_cluster_test.go @@ -14,7 +14,6 @@ import ( func TestComponentActionRemove(t *testing.T) { t.Log("E2E: Component action remove") - e2e.SetupWithCluster(t) packagePath := filepath.Join("build", fmt.Sprintf("zarf-package-component-actions-%s.tar.zst", e2e.Arch)) @@ -31,7 +30,6 @@ func TestComponentActionRemove(t *testing.T) { func TestComponentActionEdgeCases(t *testing.T) { t.Log("E2E: Component action edge cases") - e2e.SetupWithCluster(t) sourcePath := filepath.Join("src", "test", "packages", "31-component-actions-edgecases") packagePath := fmt.Sprintf("zarf-package-component-actions-edgecases-%s.tar.zst", e2e.Arch) diff --git a/src/test/e2e/31_checksum_and_signature_test.go b/src/test/e2e/31_checksum_and_signature_test.go index 957deaeaa8..2b3856779f 100644 --- a/src/test/e2e/31_checksum_and_signature_test.go +++ b/src/test/e2e/31_checksum_and_signature_test.go @@ -13,7 +13,6 @@ import ( func TestChecksumAndSignature(t *testing.T) { t.Log("E2E: Checksum and Signature") - e2e.SetupWithCluster(t) testPackageDirPath := "examples/dos-games" pkgName := fmt.Sprintf("zarf-package-dos-games-%s-1.0.0.tar.zst", e2e.Arch) diff --git a/src/test/e2e/32_component_webhooks_test.go b/src/test/e2e/32_component_webhooks_test.go index cbb8321b08..d409e62521 100644 --- a/src/test/e2e/32_component_webhooks_test.go +++ b/src/test/e2e/32_component_webhooks_test.go @@ -13,7 +13,6 @@ import ( func TestComponentWebhooks(t *testing.T) { t.Log("E2E: Component Webhooks") - e2e.SetupWithCluster(t) // Deploy example Pepr webhook. webhookPath := fmt.Sprintf("build/zarf-package-component-webhooks-%s-0.0.1.tar.zst", e2e.Arch) diff --git a/src/test/e2e/34_custom_init_package_test.go b/src/test/e2e/34_custom_init_package_test.go index aae69e30e9..6229255aca 100644 --- a/src/test/e2e/34_custom_init_package_test.go +++ b/src/test/e2e/34_custom_init_package_test.go @@ -14,7 +14,7 @@ import ( func TestCustomInit(t *testing.T) { t.Log("E2E: Custom Init Package") - e2e.SetupWithCluster(t) + buildPath := filepath.Join("src", "test", "packages", "35-custom-init-package") pkgName := fmt.Sprintf("zarf-init-%s-%s.tar.zst", e2e.Arch, e2e.GetZarfVersion(t)) privateKeyFlag := "--signing-key=src/test/packages/zarf-test.prv-key" diff --git a/src/test/e2e/35_custom_retries_test.go b/src/test/e2e/35_custom_retries_test.go index cbe5428a25..92d7e2fffe 100644 --- a/src/test/e2e/35_custom_retries_test.go +++ b/src/test/e2e/35_custom_retries_test.go @@ -15,7 +15,6 @@ import ( func TestRetries(t *testing.T) { t.Log("E2E: Custom Retries") - e2e.SetupWithCluster(t) tmpDir := t.TempDir() diff --git a/src/test/e2e/50_oci_publish_deploy_test.go b/src/test/e2e/50_oci_publish_deploy_test.go index 96f0d6c7f1..f3d5d89428 100644 --- a/src/test/e2e/50_oci_publish_deploy_test.go +++ b/src/test/e2e/50_oci_publish_deploy_test.go @@ -164,7 +164,6 @@ func (suite *PublishDeploySuiteTestSuite) Test_3_Copy() { } func TestPublishDeploySuite(t *testing.T) { - e2e.SetupWithCluster(t) suite.Run(t, new(PublishDeploySuiteTestSuite)) } diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index c59d7477c1..124dbb953a 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -284,7 +284,6 @@ func (suite *SkeletonSuite) verifyComponentPaths(unpackedPath string, components } func TestSkeletonSuite(t *testing.T) { - e2e.SetupWithCluster(t) suite.Run(t, new(SkeletonSuite)) } diff --git a/src/test/e2e/99_appliance_remove_test.go b/src/test/e2e/99_appliance_remove_test.go index 1e34af99e0..321f353228 100644 --- a/src/test/e2e/99_appliance_remove_test.go +++ b/src/test/e2e/99_appliance_remove_test.go @@ -19,8 +19,6 @@ func TestApplianceRemove(t *testing.T) { return } - e2e.SetupWithCluster(t) - initPackageVersion := e2e.GetZarfVersion(t) path := fmt.Sprintf("build/zarf-init-%s-%s.tar.zst", e2e.Arch, initPackageVersion) diff --git a/src/test/e2e/99_yolo_test.go b/src/test/e2e/99_yolo_test.go index bce9ab1450..01671b8600 100644 --- a/src/test/e2e/99_yolo_test.go +++ b/src/test/e2e/99_yolo_test.go @@ -22,8 +22,6 @@ func TestYOLOMode(t *testing.T) { return } - e2e.SetupWithCluster(t) - // Destroy the cluster to test Zarf cleaning up after itself stdOut, stdErr, err := e2e.Zarf("destroy", "--confirm", "--remove-components") require.NoError(t, err, stdOut, stdErr) @@ -54,7 +52,6 @@ func TestDevDeploy(t *testing.T) { if e2e.ApplianceMode { return } - e2e.SetupWithCluster(t) stdOut, stdErr, err := e2e.Zarf("dev", "deploy", "examples/dos-games") require.NoError(t, err, stdOut, stdErr) diff --git a/src/test/e2e/main_test.go b/src/test/e2e/main_test.go index 9670335641..6c9c283a55 100644 --- a/src/test/e2e/main_test.go +++ b/src/test/e2e/main_test.go @@ -38,7 +38,6 @@ func TestMain(m *testing.M) { e2e.ZarfBinPath = filepath.Join("build", test.GetCLIName()) e2e.ApplianceMode = os.Getenv(applianceModeEnvVar) == "true" e2e.ApplianceModeKeep = os.Getenv(applianceModeKeepEnvVar) == "true" - e2e.RunClusterTests = true message.SetLogLevel(message.TraceLevel) diff --git a/src/test/nightly/ecr_publish_test.go b/src/test/nightly/ecr_publish_test.go index a1d07e73e1..8d696660c8 100644 --- a/src/test/nightly/ecr_publish_test.go +++ b/src/test/nightly/ecr_publish_test.go @@ -39,8 +39,6 @@ func TestECRPublishing(t *testing.T) { // Set up the e2e configs e2e.Arch = config.GetArch() e2e.ZarfBinPath = path.Join("build", test.GetCLIName()) - e2e.ApplianceMode = true - e2e.RunClusterTests = false // Set up variables for common names/locations testPackageName := "helm-charts"