diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f7f4b765c8..88e9a3e650 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,7 +72,7 @@ jobs: # Upload the contents of the build directory for later stages to use - name: Upload build artifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: build-artifacts path: build/ @@ -186,7 +186,7 @@ jobs: HOMEBREW_TAP_GITHUB_TOKEN: ${{ steps.brew-tap-token.outputs.token }} - name: Save CVE report - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: cve-report path: build/zarf-known-cves.csv diff --git a/.github/workflows/scan-codeql.yml b/.github/workflows/scan-codeql.yml index e5ab9bd6d8..486fdaa8f8 100644 --- a/.github/workflows/scan-codeql.yml +++ b/.github/workflows/scan-codeql.yml @@ -53,7 +53,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3.25.15 + uses: github/codeql-action/init@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0 with: languages: ${{ matrix.language }} config-file: ./.github/codeql.yaml @@ -62,6 +62,6 @@ jobs: run: make build-cli-linux-amd - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3.25.15 + uses: github/codeql-action/analyze@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/scorecard.yaml b/.github/workflows/scorecard.yaml index e4c9399538..c1a7d1915c 100644 --- a/.github/workflows/scorecard.yaml +++ b/.github/workflows/scorecard.yaml @@ -36,7 +36,7 @@ jobs: # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF # format to the repository Actions tab. - name: "Upload artifact" - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: SARIF file path: results.sarif @@ -44,6 +44,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard. - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@afb54ba388a7dca6ecae48f608c4ff05ff4cc77a # v3.25.15 + uses: github/codeql-action/upload-sarif@eb055d739abdc2e8de2e5f4ba1a8b246daa779aa # v3.26.0 with: sarif_file: results.sarif diff --git a/.github/workflows/test-bigbang.yml b/.github/workflows/test-bigbang.yml index 48d2f795cf..6b78a21c4c 100644 --- a/.github/workflows/test-bigbang.yml +++ b/.github/workflows/test-bigbang.yml @@ -63,7 +63,7 @@ jobs: # Upload the contents of the build directory for later stages to use - name: Upload build artifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: build-artifacts path: build/ diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 51faff752d..bdcc0d0c6b 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -47,7 +47,7 @@ jobs: # Upload the contents of the build directory for later stages to use - name: Upload build artifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: build-artifacts path: build/ diff --git a/.github/workflows/test-upgrade.yml b/.github/workflows/test-upgrade.yml index a3230eb6a7..72aa9dfa90 100644 --- a/.github/workflows/test-upgrade.yml +++ b/.github/workflows/test-upgrade.yml @@ -46,7 +46,7 @@ jobs: # Upload the contents of the build directory for later stages to use - name: Upload build artifacts - uses: actions/upload-artifact@89ef406dd8d7e03cfd12d9e0a4a378f454709029 # v4.3.5 + uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6 with: name: build-artifacts path: build/ diff --git a/src/cmd/internal.go b/src/cmd/internal.go index d989d6ce0a..4a393d01e6 100644 --- a/src/cmd/internal.go +++ b/src/cmd/internal.go @@ -43,7 +43,11 @@ var agentCmd = &cobra.Command{ Short: lang.CmdInternalAgentShort, Long: lang.CmdInternalAgentLong, RunE: func(cmd *cobra.Command, _ []string) error { - return agent.StartWebhook(cmd.Context()) + cluster, err := cluster.NewCluster() + if err != nil { + return err + } + return agent.StartWebhook(cmd.Context(), cluster) }, } @@ -52,7 +56,11 @@ var httpProxyCmd = &cobra.Command{ Short: lang.CmdInternalProxyShort, Long: lang.CmdInternalProxyLong, RunE: func(cmd *cobra.Command, _ []string) error { - return agent.StartHTTPProxy(cmd.Context()) + cluster, err := cluster.NewCluster() + if err != nil { + return err + } + return agent.StartHTTPProxy(cmd.Context(), cluster) }, } diff --git a/src/config/lang/english.go b/src/config/lang/english.go index e4057e00bd..8e821b03f2 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -611,7 +611,6 @@ const ( AgentErrMarshallJSONPatch = "unable to marshall the json patch" AgentErrMarshalResponse = "unable to marshal the response" AgentErrNilReq = "malformed admission review: request is nil" - AgentErrUnableTransform = "unable to transform the provided request; see zarf http proxy logs for more details" ) // Package create diff --git a/src/internal/agent/http/proxy.go b/src/internal/agent/http/proxy.go index 33709dfff7..760ba709ec 100644 --- a/src/internal/agent/http/proxy.go +++ b/src/internal/agent/http/proxy.go @@ -5,7 +5,6 @@ package http import ( - "context" "crypto/tls" "fmt" "io" @@ -14,51 +13,43 @@ import ( "net/url" "strings" - "github.com/zarf-dev/zarf/src/config/lang" "github.com/zarf-dev/zarf/src/pkg/cluster" "github.com/zarf-dev/zarf/src/pkg/message" "github.com/zarf-dev/zarf/src/pkg/transform" + "github.com/zarf-dev/zarf/src/types" ) // ProxyHandler constructs a new httputil.ReverseProxy and returns an http handler. -func ProxyHandler() http.HandlerFunc { +func ProxyHandler(cluster *cluster.Cluster) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - err := proxyRequestTransform(r) + state, err := cluster.LoadZarfState(r.Context()) if err != nil { message.Debugf("%#v", err) w.WriteHeader(http.StatusInternalServerError) //nolint: errcheck // ignore - w.Write([]byte(lang.AgentErrUnableTransform)) + w.Write([]byte("unable to load Zarf state, see the Zarf HTTP proxy logs for more details")) + return + } + err = proxyRequestTransform(r, state) + if err != nil { + message.Debugf("%#v", err) + w.WriteHeader(http.StatusInternalServerError) + //nolint: errcheck // ignore + w.Write([]byte("unable to transform the provided request, see the Zarf HTTP proxy logs for more details")) return } - proxy := &httputil.ReverseProxy{Director: func(_ *http.Request) {}, ModifyResponse: proxyResponseTransform} proxy.ServeHTTP(w, r) } } -func proxyRequestTransform(r *http.Request) error { - message.Debugf("Before Req %#v", r) - message.Debugf("Before Req URL %#v", r.URL) - +func proxyRequestTransform(r *http.Request, state *types.ZarfState) error { // We add this so that we can use it to rewrite urls in the response if needed r.Header.Add("X-Forwarded-Host", r.Host) // We remove this so that go will encode and decode on our behalf (see https://pkg.go.dev/net/http#Transport DisableCompression) r.Header.Del("Accept-Encoding") - c, err := cluster.NewCluster() - if err != nil { - return err - } - ctx := context.Background() - state, err := c.LoadZarfState(ctx) - if err != nil { - return err - } - - var targetURL *url.URL - // Setup authentication for each type of service based on User Agent switch { case isGitUserAgent(r.UserAgent()): @@ -70,6 +61,8 @@ func proxyRequestTransform(r *http.Request) error { } // Transform the URL; if we see the NoTransform prefix, strip it; otherwise, transform the URL based on User Agent + var err error + var targetURL *url.URL if strings.HasPrefix(r.URL.Path, transform.NoTransform) { switch { case isGitUserAgent(r.UserAgent()): @@ -89,7 +82,6 @@ func proxyRequestTransform(r *http.Request) error { targetURL, err = transform.GenTransformURL(state.ArtifactServer.Address, getTLSScheme(r.TLS)+r.Host+r.URL.String()) } } - if err != nil { return err } @@ -98,19 +90,12 @@ func proxyRequestTransform(r *http.Request) error { r.URL = targetURL r.RequestURI = getRequestURI(targetURL.Path, targetURL.RawQuery, targetURL.Fragment) - message.Debugf("After Req %#v", r) - message.Debugf("After Req URL%#v", r.URL) - return nil } func proxyResponseTransform(resp *http.Response) error { - message.Debugf("Before Resp %#v", resp) - // Handle redirection codes (3xx) by adding a marker to let Zarf know this has been redirected if resp.StatusCode/100 == 3 { - message.Debugf("Before Resp Location %#v", resp.Header.Get("Location")) - locationURL, err := url.Parse(resp.Header.Get("Location")) if err != nil { return err @@ -119,72 +104,46 @@ func proxyResponseTransform(resp *http.Response) error { locationURL.Host = resp.Request.Header.Get("X-Forwarded-Host") resp.Header.Set("Location", locationURL.String()) - - message.Debugf("After Resp Location %#v", resp.Header.Get("Location")) } - contentType := resp.Header.Get("Content-Type") - // Handle text content returns that may contain links + contentType := resp.Header.Get("Content-Type") if strings.HasPrefix(contentType, "text") || strings.HasPrefix(contentType, "application/json") || strings.HasPrefix(contentType, "application/xml") { - err := replaceBodyLinks(resp) - + forwardedPrefix := fmt.Sprintf("%s%s%s", getTLSScheme(resp.Request.TLS), resp.Request.Header.Get("X-Forwarded-Host"), transform.NoTransform) + targetPrefix := fmt.Sprintf("%s%s", getTLSScheme(resp.TLS), resp.Request.Host) + b, err := io.ReadAll(resp.Body) if err != nil { - message.Debugf("%#v", err) + return err } - } - - message.Debugf("After Resp %#v", resp) - - return nil -} - -func replaceBodyLinks(resp *http.Response) error { - message.Debugf("Resp Request: %#v", resp.Request) - - // Create the forwarded (online) and target (offline) URL prefixes to replace - forwardedPrefix := fmt.Sprintf("%s%s%s", getTLSScheme(resp.Request.TLS), resp.Request.Header.Get("X-Forwarded-Host"), transform.NoTransform) - targetPrefix := fmt.Sprintf("%s%s", getTLSScheme(resp.TLS), resp.Request.Host) + err = resp.Body.Close() + if err != nil { + return err + } + bodyString := strings.ReplaceAll(string(b), targetPrefix, forwardedPrefix) - b, err := io.ReadAll(resp.Body) - if err != nil { - return err - } - err = resp.Body.Close() - if err != nil { - return err + resp.Body = io.NopCloser(strings.NewReader(bodyString)) + resp.ContentLength = int64(len(bodyString)) + resp.Header.Set("Content-Length", fmt.Sprint(int64(len(bodyString)))) } - bodyString := strings.ReplaceAll(string(b), targetPrefix, forwardedPrefix) - - // Setup the new reader, and correct the content length - resp.Body = io.NopCloser(strings.NewReader(bodyString)) - resp.ContentLength = int64(len(bodyString)) - resp.Header.Set("Content-Length", fmt.Sprint(int64(len(bodyString)))) - return nil } func getTLSScheme(tls *tls.ConnectionState) string { scheme := "https://" - if tls == nil { scheme = "http://" } - return scheme } func getRequestURI(path, query, fragment string) string { uri := path - if query != "" { uri += "?" + query } - if fragment != "" { uri += "#" + fragment } - return uri } diff --git a/src/internal/agent/http/proxy_test.go b/src/internal/agent/http/proxy_test.go new file mode 100644 index 0000000000..c5883d25b4 --- /dev/null +++ b/src/internal/agent/http/proxy_test.go @@ -0,0 +1,192 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package http + +import ( + "crypto/tls" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" + "github.com/zarf-dev/zarf/src/types" +) + +func TestProxyRequestTransform(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + target string + state *types.ZarfState + expectedPath string + }{ + { + name: "basic request", + target: "http://example.com/zarf-3xx-no-transform/test", + state: &types.ZarfState{ + ArtifactServer: types.ArtifactServerInfo{ + PushUsername: "push-user", + PushToken: "push-token", + }, + }, + expectedPath: "/test", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + req := httptest.NewRequest(http.MethodGet, tt.target, nil) + req.Header.Set("Accept-Encoding", "foo") + err := proxyRequestTransform(req, tt.state) + require.NoError(t, err) + + require.Empty(t, req.Header.Get("Accept-Encoding")) + + username, password, ok := req.BasicAuth() + require.True(t, ok) + require.Equal(t, tt.state.ArtifactServer.PushUsername, username) + require.Equal(t, tt.state.ArtifactServer.PushToken, password) + + require.Equal(t, tt.expectedPath, req.URL.Path) + }) + } +} + +func TestGetTLSScheme(t *testing.T) { + t.Parallel() + + scheme := getTLSScheme(nil) + require.Equal(t, "http://", scheme) + scheme = getTLSScheme(&tls.ConnectionState{}) + require.Equal(t, "https://", scheme) +} + +func TestGetRequestURI(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + path string + query string + fragment string + expected string + }{ + { + name: "basic", + path: "/foo", + query: "", + fragment: "", + expected: "/foo", + }, + { + name: "query", + path: "/foo", + query: "key=value", + fragment: "", + expected: "/foo?key=value", + }, + { + name: "fragment", + path: "/foo", + query: "", + fragment: "bar", + expected: "/foo#bar", + }, + { + name: "query and fragment", + path: "/foo", + query: "key=value", + fragment: "bar", + expected: "/foo?key=value#bar", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + uri := getRequestURI(tt.path, tt.query, tt.fragment) + require.Equal(t, tt.expected, uri) + }) + } +} + +func TestUserAgent(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + userAgent string + expectedGit bool + expectedPip bool + expectedNpm bool + }{ + { + name: "unknown user agent", + userAgent: "Firefox", + expectedGit: false, + expectedPip: false, + expectedNpm: false, + }, + { + name: "git user agent", + userAgent: "git/2.0.0", + expectedGit: true, + expectedPip: false, + expectedNpm: false, + }, + { + name: "pip user agent", + userAgent: "pip/1.2.3", + expectedGit: false, + expectedPip: true, + expectedNpm: false, + }, + { + name: "twine user agent", + userAgent: "twine/1.8.1", + expectedGit: false, + expectedPip: true, + expectedNpm: false, + }, + { + name: "npm user agent", + userAgent: "npm/1.0.0", + expectedGit: false, + expectedPip: false, + expectedNpm: true, + }, + { + name: "pnpm user agent", + userAgent: "pnpm/1.0.0", + expectedGit: false, + expectedPip: false, + expectedNpm: true, + }, + { + name: "yarn user agent", + userAgent: "yarn/1.0.0", + expectedGit: false, + expectedPip: false, + expectedNpm: true, + }, + { + name: "bun user agent", + userAgent: "bun/1.0.0", + expectedGit: false, + expectedPip: false, + expectedNpm: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + require.Equal(t, tt.expectedGit, isGitUserAgent(tt.userAgent)) + require.Equal(t, tt.expectedPip, isPipUserAgent(tt.userAgent)) + require.Equal(t, tt.expectedNpm, isNpmUserAgent(tt.userAgent)) + }) + } +} diff --git a/src/internal/agent/http/server.go b/src/internal/agent/http/server.go deleted file mode 100644 index 6a79aaa449..0000000000 --- a/src/internal/agent/http/server.go +++ /dev/null @@ -1,74 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package http provides a http server for the webhook and proxy. -package http - -import ( - "context" - "fmt" - "net/http" - "time" - - "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/zarf-dev/zarf/src/internal/agent/hooks" - "github.com/zarf-dev/zarf/src/internal/agent/http/admission" - "github.com/zarf-dev/zarf/src/pkg/cluster" -) - -// NewAdmissionServer creates a http.Server for the mutating webhook admission handler. -func NewAdmissionServer(ctx context.Context, port string) (*http.Server, error) { - c, err := cluster.NewCluster() - if err != nil { - return nil, err - } - - // Routers - admissionHandler := admission.NewHandler() - podsMutation := hooks.NewPodMutationHook(ctx, c) - fluxGitRepositoryMutation := hooks.NewGitRepositoryMutationHook(ctx, c) - argocdApplicationMutation := hooks.NewApplicationMutationHook(ctx, c) - argocdRepositoryMutation := hooks.NewRepositorySecretMutationHook(ctx, c) - fluxHelmRepositoryMutation := hooks.NewHelmRepositoryMutationHook(ctx, c) - fluxOCIRepositoryMutation := hooks.NewOCIRepositoryMutationHook(ctx, c) - - // Routers - mux := http.NewServeMux() - mux.Handle("/healthz", healthz()) - mux.Handle("/mutate/pod", admissionHandler.Serve(podsMutation)) - mux.Handle("/mutate/flux-gitrepository", admissionHandler.Serve(fluxGitRepositoryMutation)) - mux.Handle("/mutate/flux-helmrepository", admissionHandler.Serve(fluxHelmRepositoryMutation)) - mux.Handle("/mutate/flux-ocirepository", admissionHandler.Serve(fluxOCIRepositoryMutation)) - mux.Handle("/mutate/argocd-application", admissionHandler.Serve(argocdApplicationMutation)) - mux.Handle("/mutate/argocd-repository", admissionHandler.Serve(argocdRepositoryMutation)) - mux.Handle("/metrics", promhttp.Handler()) - - srv := &http.Server{ - Addr: fmt.Sprintf(":%s", port), - Handler: mux, - ReadHeaderTimeout: 5 * time.Second, // Set ReadHeaderTimeout to avoid Slowloris attacks - } - return srv, nil -} - -// NewProxyServer creates and returns an http proxy server. -func NewProxyServer(port string) *http.Server { - mux := http.NewServeMux() - mux.Handle("/healthz", healthz()) - mux.Handle("/", ProxyHandler()) - mux.Handle("/metrics", promhttp.Handler()) - - return &http.Server{ - Addr: fmt.Sprintf(":%s", port), - Handler: mux, - ReadHeaderTimeout: 5 * time.Second, // Set ReadHeaderTimeout to avoid Slowloris attacks - } -} - -func healthz() http.HandlerFunc { - return func(w http.ResponseWriter, _ *http.Request) { - w.WriteHeader(http.StatusOK) - //nolint: errcheck // ignore - w.Write([]byte("ok")) - } -} diff --git a/src/internal/agent/start.go b/src/internal/agent/start.go index 80a04c642f..e620c3648a 100644 --- a/src/internal/agent/start.go +++ b/src/internal/agent/start.go @@ -7,13 +7,18 @@ package agent import ( "context" "errors" + "fmt" "net/http" "time" + "github.com/prometheus/client_golang/prometheus/promhttp" "golang.org/x/sync/errgroup" "github.com/zarf-dev/zarf/src/config/lang" + "github.com/zarf-dev/zarf/src/internal/agent/hooks" agentHttp "github.com/zarf-dev/zarf/src/internal/agent/http" + "github.com/zarf-dev/zarf/src/internal/agent/http/admission" + "github.com/zarf-dev/zarf/src/pkg/cluster" "github.com/zarf-dev/zarf/src/pkg/message" ) @@ -28,20 +33,48 @@ const ( ) // StartWebhook launches the Zarf agent mutating webhook in the cluster. -func StartWebhook(ctx context.Context) error { - srv, err := agentHttp.NewAdmissionServer(ctx, httpPort) - if err != nil { - return err - } - return startServer(ctx, srv) +func StartWebhook(ctx context.Context, cluster *cluster.Cluster) error { + // Routers + admissionHandler := admission.NewHandler() + podsMutation := hooks.NewPodMutationHook(ctx, cluster) + fluxGitRepositoryMutation := hooks.NewGitRepositoryMutationHook(ctx, cluster) + argocdApplicationMutation := hooks.NewApplicationMutationHook(ctx, cluster) + argocdRepositoryMutation := hooks.NewRepositorySecretMutationHook(ctx, cluster) + fluxHelmRepositoryMutation := hooks.NewHelmRepositoryMutationHook(ctx, cluster) + fluxOCIRepositoryMutation := hooks.NewOCIRepositoryMutationHook(ctx, cluster) + + // Routers + mux := http.NewServeMux() + mux.Handle("/mutate/pod", admissionHandler.Serve(podsMutation)) + mux.Handle("/mutate/flux-gitrepository", admissionHandler.Serve(fluxGitRepositoryMutation)) + mux.Handle("/mutate/flux-helmrepository", admissionHandler.Serve(fluxHelmRepositoryMutation)) + mux.Handle("/mutate/flux-ocirepository", admissionHandler.Serve(fluxOCIRepositoryMutation)) + mux.Handle("/mutate/argocd-application", admissionHandler.Serve(argocdApplicationMutation)) + mux.Handle("/mutate/argocd-repository", admissionHandler.Serve(argocdRepositoryMutation)) + + return startServer(ctx, httpPort, mux) } // StartHTTPProxy launches the zarf agent proxy in the cluster. -func StartHTTPProxy(ctx context.Context) error { - return startServer(ctx, agentHttp.NewProxyServer(httpPort)) +func StartHTTPProxy(ctx context.Context, cluster *cluster.Cluster) error { + mux := http.NewServeMux() + mux.Handle("/", agentHttp.ProxyHandler(cluster)) + return startServer(ctx, httpPort, mux) } -func startServer(ctx context.Context, srv *http.Server) error { +func startServer(ctx context.Context, port string, mux *http.ServeMux) error { + mux.Handle("/metrics", promhttp.Handler()) + mux.Handle("/healthz", http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + //nolint: errcheck // ignore + w.Write([]byte("ok")) + })) + srv := &http.Server{ + Addr: fmt.Sprintf(":%s", port), + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, // Set ReadHeaderTimeout to avoid Slowloris attacks + } + g, gCtx := errgroup.WithContext(ctx) g.Go(func() error { err := srv.ListenAndServeTLS(tlsCert, tlsKey) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 7a313c5db0..67907feb9d 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -40,6 +40,26 @@ import ( "golang.org/x/sync/errgroup" ) +func checkForIndex(refInfo transform.Image, desc *remote.Descriptor) error { + if refInfo.Digest != "" && desc != nil && types.MediaType(desc.MediaType).IsIndex() { + var idx v1.IndexManifest + if err := json.Unmarshal(desc.Manifest, &idx); err != nil { + return fmt.Errorf("unable to unmarshal index.json: %w", err) + } + lines := []string{"The following images are available in the index:"} + name := refInfo.Name + if refInfo.Tag != "" { + name += ":" + refInfo.Tag + } + for _, desc := range idx.Manifests { + lines = append(lines, fmt.Sprintf("image - %s@%s with platform %s", name, desc.Digest.String(), desc.Platform.String())) + } + imageOptions := strings.Join(lines, "\n") + return fmt.Errorf("%s resolved to an OCI image index which is not supported by Zarf, select a specific platform to use: %s", refInfo.Reference, imageOptions) + } + return nil +} + // Pull pulls all of the images from the given config. func Pull(ctx context.Context, cfg PullConfig) (map[transform.Image]v1.Image, error) { var longer string @@ -146,23 +166,8 @@ func Pull(ctx context.Context, cfg PullConfig) (map[transform.Image]v1.Image, er } } - if refInfo.Digest != "" && desc != nil && types.MediaType(desc.MediaType).IsIndex() { - message.Warn("Zarf does not currently support direct consumption of OCI image indexes or Docker manifest lists") - - var idx v1.IndexManifest - if err := json.Unmarshal(desc.Manifest, &idx); err != nil { - return fmt.Errorf("unable to unmarshal index manifest: %w", err) - } - lines := []string{"The following images are available in the index:"} - name := refInfo.Name - if refInfo.Tag != "" { - name += ":" + refInfo.Tag - } - for _, desc := range idx.Manifests { - lines = append(lines, fmt.Sprintf("\n(%s) %s@%s", desc.Platform, name, desc.Digest)) - } - message.Warn(strings.Join(lines, "\n")) - return fmt.Errorf("%s resolved to an index, please select a specific platform to use", refInfo.Reference) + if err := checkForIndex(refInfo, desc); err != nil { + return err } cacheImg, err := utils.OnlyHasImageLayers(img) diff --git a/src/internal/packager/images/pull_test.go b/src/internal/packager/images/pull_test.go index eef8bbc9e8..ad38bb3aac 100644 --- a/src/internal/packager/images/pull_test.go +++ b/src/internal/packager/images/pull_test.go @@ -6,14 +6,83 @@ package images import ( "context" + "encoding/json" + "fmt" "os" "path/filepath" "testing" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/stretchr/testify/require" "github.com/zarf-dev/zarf/src/pkg/transform" ) +func TestCheckForIndex(t *testing.T) { + t.Parallel() + testCases := []struct { + name string + ref string + file string + expectedErr string + }{ + { + name: "index sha", + ref: "ghcr.io/zarf-dev/zarf/agent:v0.32.6@sha256:05a82656df5466ce17c3e364c16792ae21ce68438bfe06eeab309d0520c16b48", + file: "agent-index.json", + expectedErr: "%s resolved to an OCI image index which is not supported by Zarf, select a specific platform to use", + }, + { + name: "docker manifest list", + ref: "defenseunicorns/zarf-game@sha256:0b694ca1c33afae97b7471488e07968599f1d2470c629f76af67145ca64428af", + file: "game-index.json", + expectedErr: "%s resolved to an OCI image index which is not supported by Zarf, select a specific platform to use", + }, + { + name: "image manifest", + ref: "ghcr.io/zarf-dev/zarf/agent:v0.32.6", + file: "agent-manifest.json", + expectedErr: "", + }, + { + name: "image manifest sha'd", + ref: "ghcr.io/zarf-dev/zarf/agent:v0.32.6@sha256:b3fabdc7d4ecd0f396016ef78da19002c39e3ace352ea0ae4baa2ce9d5958376", + file: "agent-manifest.json", + expectedErr: "", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + refInfo, err := transform.ParseImageRef(tc.ref) + require.NoError(t, err) + file := filepath.Join("testdata", tc.file) + manifest, err := os.ReadFile(file) + require.NoError(t, err) + var idx v1.IndexManifest + err = json.Unmarshal(manifest, &idx) + require.NoError(t, err) + desc := &remote.Descriptor{ + Descriptor: v1.Descriptor{ + MediaType: idx.MediaType, + }, + Manifest: manifest, + } + err = checkForIndex(refInfo, desc) + if tc.expectedErr != "" { + require.ErrorContains(t, err, fmt.Sprintf(tc.expectedErr, refInfo.Reference)) + // Ensure the error message contains the digest of the manifests the user can use + for _, manifest := range idx.Manifests { + require.ErrorContains(t, err, manifest.Digest.String()) + } + return + } + require.NoError(t, err) + }) + } +} + func TestPull(t *testing.T) { t.Run("pulling a cosign image is successful and doesn't add anything to the cache", func(t *testing.T) { ref, err := transform.ParseImageRef("ghcr.io/stefanprodan/podinfo:sha256-57a654ace69ec02ba8973093b6a786faa15640575fbf0dbb603db55aca2ccec8.sig") diff --git a/src/internal/packager/images/testdata/agent-index.json b/src/internal/packager/images/testdata/agent-index.json new file mode 100644 index 0000000000..2288b7f242 --- /dev/null +++ b/src/internal/packager/images/testdata/agent-index.json @@ -0,0 +1,50 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:b3fabdc7d4ecd0f396016ef78da19002c39e3ace352ea0ae4baa2ce9d5958376", + "size": 673, + "platform": { + "architecture": "arm64", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:454bf871b5d826b6a31ab14c983583ae9d9e30c2036606b500368c5b552d8fdf", + "size": 673, + "platform": { + "architecture": "amd64", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:449cd2cd763614f07219f38f8514b56c7831d583f7c6f767491d3ad31572189c", + "size": 566, + "annotations": { + "vnd.docker.reference.digest": "sha256:b3fabdc7d4ecd0f396016ef78da19002c39e3ace352ea0ae4baa2ce9d5958376", + "vnd.docker.reference.type": "attestation-manifest" + }, + "platform": { + "architecture": "unknown", + "os": "unknown" + } + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:2b51d89bf901b4e65dd85d8c6f135278fc7f86bb48608f0681a83291deadd645", + "size": 566, + "annotations": { + "vnd.docker.reference.digest": "sha256:454bf871b5d826b6a31ab14c983583ae9d9e30c2036606b500368c5b552d8fdf", + "vnd.docker.reference.type": "attestation-manifest" + }, + "platform": { + "architecture": "unknown", + "os": "unknown" + } + } + ] +} diff --git a/src/internal/packager/images/testdata/agent-manifest.json b/src/internal/packager/images/testdata/agent-manifest.json new file mode 100644 index 0000000000..5ef1467f7e --- /dev/null +++ b/src/internal/packager/images/testdata/agent-manifest.json @@ -0,0 +1,21 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:cd4790c66d0f72eb0e0c12d75c1025478294af876ff0042d1c3501db57b0244b", + "size": 1284 + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:32f8946351207153325c9d5eb56c61bbd96f9cd8deaa6dcecc4a3d4867609bec", + "size": 327582 + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:72dbbe2df2af4ee1b7a0c7ee66782999e8f6f846ffed250aec3738791d941ba7", + "size": 38937433 + } + ] +} diff --git a/src/internal/packager/images/testdata/game-index.json b/src/internal/packager/images/testdata/game-index.json new file mode 100644 index 0000000000..b8eadb836d --- /dev/null +++ b/src/internal/packager/images/testdata/game-index.json @@ -0,0 +1,34 @@ +{ + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "schemaVersion": 2, + "manifests": [ + { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:f78e442f0f3eb3e9459b5ae6b1a8fda62f8dfe818112e7d130a4e8ae72b3cbff", + "size": 739, + "platform": { + "architecture": "arm", + "os": "linux", + "variant": "v7" + } + }, + { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:e4d27fe4b7bf6d5cb7ef02ed0d33ec0846796c09d6ed4bd94c8b946119a01b09", + "size": 739, + "platform": { + "architecture": "arm64", + "os": "linux" + } + }, + { + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "digest": "sha256:e81b1467b812019f8e8e81450728b084471806dc7e959b7beb9f39933c337e7d", + "size": 739, + "platform": { + "architecture": "amd64", + "os": "linux" + } + } + ] +} diff --git a/src/pkg/packager/creator/creator_test.go b/src/pkg/packager/creator/creator_test.go index 9312fe0a2c..95803cc0f4 100644 --- a/src/pkg/packager/creator/creator_test.go +++ b/src/pkg/packager/creator/creator_test.go @@ -6,7 +6,6 @@ package creator import ( "context" - "io/fs" "os" "path/filepath" "testing" @@ -14,22 +13,10 @@ import ( "github.com/stretchr/testify/require" "github.com/zarf-dev/zarf/src/pkg/layout" "github.com/zarf-dev/zarf/src/pkg/lint" + "github.com/zarf-dev/zarf/src/test/testutil" "github.com/zarf-dev/zarf/src/types" ) -type mockSchemaLoader struct { - b []byte -} - -func (m *mockSchemaLoader) ReadFile(_ string) ([]byte, error) { - return m.b, nil -} - -// Satisfy fs.ReadFileFS interface -func (m *mockSchemaLoader) Open(_ string) (fs.File, error) { - return nil, nil -} - func TestLoadPackageDefinition(t *testing.T) { // TODO once creator is refactored to not expect to be in the same directory as the zarf.yaml file // this test can be re-parallelized @@ -64,9 +51,7 @@ func TestLoadPackageDefinition(t *testing.T) { creator: NewSkeletonCreator(types.ZarfCreateOptions{}, types.ZarfPublishOptions{}), }, } - b, err := os.ReadFile("../../../../zarf.schema.json") - require.NoError(t, err) - lint.ZarfSchema = &mockSchemaLoader{b: b} + lint.ZarfSchema = testutil.LoadSchema(t, "../../../../zarf.schema.json") for _, tt := range tests { tt := tt diff --git a/src/pkg/packager/prepare.go b/src/pkg/packager/prepare.go index d007e862bd..214258634a 100644 --- a/src/pkg/packager/prepare.go +++ b/src/pkg/packager/prepare.go @@ -13,10 +13,15 @@ import ( "sort" "strings" - "github.com/goccy/go-yaml" - "github.com/defenseunicorns/pkg/helpers/v2" + "github.com/goccy/go-yaml" "github.com/google/go-containerregistry/pkg/crane" + v1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "github.com/zarf-dev/zarf/src/api/v1alpha1" "github.com/zarf-dev/zarf/src/config/lang" "github.com/zarf-dev/zarf/src/internal/packager/helm" @@ -27,12 +32,6 @@ import ( "github.com/zarf-dev/zarf/src/pkg/packager/creator" "github.com/zarf-dev/zarf/src/pkg/utils" "github.com/zarf-dev/zarf/src/types" - v1 "k8s.io/api/apps/v1" - batchv1 "k8s.io/api/batch/v1" - corev1 "k8s.io/api/core/v1" - - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" ) // imageMap is a map of image/boolean pairs. diff --git a/src/pkg/packager/prepare_test.go b/src/pkg/packager/prepare_test.go new file mode 100644 index 0000000000..1f704fdfdc --- /dev/null +++ b/src/pkg/packager/prepare_test.go @@ -0,0 +1,42 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package packager + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/zarf-dev/zarf/src/pkg/lint" + "github.com/zarf-dev/zarf/src/test/testutil" + "github.com/zarf-dev/zarf/src/types" +) + +func TestFindImages(t *testing.T) { + t.Parallel() + + ctx := testutil.TestContext(t) + + lint.ZarfSchema = testutil.LoadSchema(t, "../../../zarf.schema.json") + + cfg := &types.PackagerConfig{ + CreateOpts: types.ZarfCreateOptions{ + BaseDir: "../../../examples/dos-games/", + }, + } + p, err := New(cfg) + require.NoError(t, err) + images, err := p.FindImages(ctx) + require.NoError(t, err) + expectedImages := map[string][]string{ + "baseline": { + "defenseunicorns/zarf-game:multi-tile-dark", + "index.docker.io/defenseunicorns/zarf-game:sha256-0b694ca1c33afae97b7471488e07968599f1d2470c629f76af67145ca64428af.sig", + }, + } + require.Equal(t, len(expectedImages), len(images)) + for k, v := range expectedImages { + require.ElementsMatch(t, v, images[k]) + } +} diff --git a/src/test/e2e/14_create_sha_index_test.go b/src/test/e2e/14_create_sha_index_test.go deleted file mode 100644 index 40b198738b..0000000000 --- a/src/test/e2e/14_create_sha_index_test.go +++ /dev/null @@ -1,40 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package test provides e2e tests for Zarf. -package test - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestCreateIndexShaErrors(t *testing.T) { - t.Log("E2E: CreateIndexShaErrors") - - testCases := []struct { - name string - packagePath string - expectedImageInStderr string - }{ - { - name: "Image Index", - packagePath: "src/test/packages/14-index-sha/image-index", - expectedImageInStderr: "ghcr.io/zarf-dev/zarf/agent:v0.32.6@sha256:b3fabdc7d4ecd0f396016ef78da19002c39e3ace352ea0ae4baa2ce9d5958376", - }, - { - name: "Manifest List", - packagePath: "src/test/packages/14-index-sha/manifest-list", - expectedImageInStderr: "docker.io/defenseunicorns/zarf-game@sha256:f78e442f0f3eb3e9459b5ae6b1a8fda62f8dfe818112e7d130a4e8ae72b3cbff", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - _, stderr, err := e2e.Zarf(t, "package", "create", tc.packagePath, "--confirm") - require.Error(t, err) - require.Contains(t, stderr, tc.expectedImageInStderr) - }) - } -} diff --git a/src/test/packages/14-index-sha/image-index/zarf.yaml b/src/test/packages/14-index-sha/image-index/zarf.yaml deleted file mode 100644 index 76d3e6ab4f..0000000000 --- a/src/test/packages/14-index-sha/image-index/zarf.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: ZarfPackageConfig -metadata: - name: image-index - -components: - - name: baseline - required: true - images: - - ghcr.io/zarf-dev/zarf/agent:v0.32.6@sha256:05a82656df5466ce17c3e364c16792ae21ce68438bfe06eeab309d0520c16b48 diff --git a/src/test/packages/14-index-sha/manifest-list/zarf.yaml b/src/test/packages/14-index-sha/manifest-list/zarf.yaml deleted file mode 100644 index 9d7ad76b20..0000000000 --- a/src/test/packages/14-index-sha/manifest-list/zarf.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: ZarfPackageConfig -metadata: - name: manifest-list - -components: - - name: baseline - required: true - images: - - defenseunicorns/zarf-game@sha256:0b694ca1c33afae97b7471488e07968599f1d2470c629f76af67145ca64428af diff --git a/src/test/testutil/schema.go b/src/test/testutil/schema.go new file mode 100644 index 0000000000..967e6e64ba --- /dev/null +++ b/src/test/testutil/schema.go @@ -0,0 +1,33 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package testutil + +import ( + "io/fs" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +type schemaFS struct { + b []byte +} + +func (m *schemaFS) ReadFile(_ string) ([]byte, error) { + return m.b, nil +} + +func (m *schemaFS) Open(_ string) (fs.File, error) { + return nil, nil +} + +// LoadSchema returns the schema file as a FS. +func LoadSchema(t *testing.T, path string) fs.ReadFileFS { + t.Helper() + + b, err := os.ReadFile(path) + require.NoError(t, err) + return &schemaFS{b: b} +} diff --git a/src/test/testutil/testutil.go b/src/test/testutil/testutil.go index 862bca937d..672f1ae194 100644 --- a/src/test/testutil/testutil.go +++ b/src/test/testutil/testutil.go @@ -12,6 +12,8 @@ import ( // TestContext takes a testing.T and returns a context that is // attached to the test by t.Cleanup() func TestContext(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) return ctx