From 0ad979947a618bcb8613a27d17985535614b36e8 Mon Sep 17 00:00:00 2001 From: Philip Laine Date: Fri, 6 Sep 2024 19:48:47 +0200 Subject: [PATCH] refactor: mirror-resources Signed-off-by: Philip Laine --- .../commands/zarf_package_mirror-resources.md | 2 +- src/cmd/package.go | 45 ++-- src/config/lang/english.go | 2 +- src/internal/dns/dns.go | 45 ++++ src/internal/dns/dns_test.go | 84 +++++++ src/internal/packager2/load.go | 203 ++++++++++++++++ src/internal/packager2/load_test.go | 76 ++++++ src/internal/packager2/mirror.go | 225 ++++++++++++++++++ src/internal/packager2/packager.go | 5 + src/internal/packager2/pull.go | 32 ++- src/test/external/ext_in_cluster_test.go | 18 +- 11 files changed, 703 insertions(+), 34 deletions(-) create mode 100644 src/internal/dns/dns.go create mode 100644 src/internal/dns/dns_test.go create mode 100644 src/internal/packager2/load.go create mode 100644 src/internal/packager2/load_test.go create mode 100644 src/internal/packager2/mirror.go create mode 100644 src/internal/packager2/packager.go diff --git a/site/src/content/docs/commands/zarf_package_mirror-resources.md b/site/src/content/docs/commands/zarf_package_mirror-resources.md index 5070a968f2..048f86d851 100644 --- a/site/src/content/docs/commands/zarf_package_mirror-resources.md +++ b/site/src/content/docs/commands/zarf_package_mirror-resources.md @@ -25,7 +25,7 @@ zarf package mirror-resources [ PACKAGE_SOURCE ] [flags] # Mirror resources to internal Zarf resources $ zarf package mirror-resources \ - --registry-url 127.0.0.1:31999 \ + --registry-url http://zarf-docker-registry.zarf.svc.cluster.local:5000 \ --registry-push-username zarf-push \ --registry-push-password \ --git-url http://zarf-gitea-http.zarf.svc.cluster.local:3000 \ diff --git a/src/cmd/package.go b/src/cmd/package.go index dfdd400bfc..1f56ac46ef 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -11,26 +11,27 @@ import ( "os" "path/filepath" "regexp" + "runtime" "strings" + "github.com/AlecAivazis/survey/v2" + "github.com/defenseunicorns/pkg/helpers/v2" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "oras.land/oras-go/v2/registry" + "github.com/zarf-dev/zarf/src/cmd/common" + "github.com/zarf-dev/zarf/src/config" "github.com/zarf-dev/zarf/src/config/lang" + "github.com/zarf-dev/zarf/src/internal/dns" "github.com/zarf-dev/zarf/src/internal/packager2" + "github.com/zarf-dev/zarf/src/pkg/cluster" "github.com/zarf-dev/zarf/src/pkg/lint" "github.com/zarf-dev/zarf/src/pkg/message" + "github.com/zarf-dev/zarf/src/pkg/packager" "github.com/zarf-dev/zarf/src/pkg/packager/filters" "github.com/zarf-dev/zarf/src/pkg/packager/sources" "github.com/zarf-dev/zarf/src/types" - - "oras.land/oras-go/v2/registry" - - "github.com/AlecAivazis/survey/v2" - "github.com/defenseunicorns/pkg/helpers/v2" - "github.com/spf13/cobra" - "github.com/spf13/viper" - "github.com/zarf-dev/zarf/src/config" - "github.com/zarf-dev/zarf/src/pkg/cluster" - "github.com/zarf-dev/zarf/src/pkg/packager" ) var packageCmd = &cobra.Command{ @@ -128,18 +129,30 @@ var packageMirrorCmd = &cobra.Command{ } }, RunE: func(cmd *cobra.Command, args []string) error { - packageSource, err := choosePackage(args) + var c *cluster.Cluster + if dns.IsServiceURL(pkgConfig.InitOpts.RegistryInfo.Address) || dns.IsServiceURL(pkgConfig.InitOpts.GitServer.Address) { + var err error + c, err = cluster.NewCluster() + if err != nil { + return err + } + } + src, err := choosePackage(args) if err != nil { return err } - pkgConfig.PkgOpts.PackageSource = packageSource - pkgClient, err := packager.New(&pkgConfig) + filter := filters.Combine( + filters.ByLocalOS(runtime.GOOS), + filters.BySelectState(pkgConfig.PkgOpts.OptionalComponents), + ) + pkgPaths, err := packager2.LoadPackage(cmd.Context(), src, pkgConfig.PkgOpts.Shasum, pkgConfig.PkgOpts.PublicKeyPath, pkgConfig.PkgOpts.SkipSignatureValidation, filter) if err != nil { return err } - defer pkgClient.ClearTempPaths() - if err := pkgClient.Mirror(cmd.Context()); err != nil { - return fmt.Errorf("failed to mirror package: %w", err) + defer os.Remove(pkgPaths.Base) + err = packager2.Mirror(cmd.Context(), c, *pkgPaths, filter, pkgConfig.InitOpts.RegistryInfo, pkgConfig.InitOpts.GitServer, pkgConfig.MirrorOpts.NoImgChecksum, pkgConfig.PkgOpts.Retries) + if err != nil { + return err } return nil }, diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 26f67b60c5..2e2a7e2177 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -233,7 +233,7 @@ $ zarf init --artifact-push-password={PASSWORD} --artifact-push-username={USERNA CmdPackageMirrorExample = ` # Mirror resources to internal Zarf resources $ zarf package mirror-resources \ - --registry-url 127.0.0.1:31999 \ + --registry-url http://zarf-docker-registry.zarf.svc.cluster.local:5000 \ --registry-push-username zarf-push \ --registry-push-password \ --git-url http://zarf-gitea-http.zarf.svc.cluster.local:3000 \ diff --git a/src/internal/dns/dns.go b/src/internal/dns/dns.go new file mode 100644 index 0000000000..fcc627a6f5 --- /dev/null +++ b/src/internal/dns/dns.go @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package dns contains DNS related functionality. +package dns + +import ( + "errors" + "fmt" + "net/url" + "regexp" + "strconv" +) + +var ( + // localClusterServiceRegex is used to match the local cluster service format: + localClusterServiceRegex = regexp.MustCompile(`^(?P[^\.]+)\.(?P[^\.]+)\.svc\.cluster\.local$`) +) + +// IsServiceURL returns true if the give url complies with the service url format. +func IsServiceURL(serviceURL string) bool { + _, _, _, err := ParseServiceURL(serviceURL) + return err == nil +} + +// ParseServiceURL takes a serviceURL and parses it to find the service info for connecting to the cluster. The string is expected to follow the following format: +// Example serviceURL: http://{SERVICE_NAME}.{NAMESPACE}.svc.cluster.local:{PORT}. +func ParseServiceURL(serviceURL string) (string, string, int, error) { + parsedURL, err := url.Parse(serviceURL) + if err != nil { + return "", "", 0, err + } + if parsedURL.Port() == "" { + return "", "", 0, errors.New("service url does not have a port") + } + remotePort, err := strconv.Atoi(parsedURL.Port()) + if err != nil { + return "", "", 0, err + } + matches := localClusterServiceRegex.FindStringSubmatch(parsedURL.Hostname()) + if len(matches) != 3 { + return "", "", 0, fmt.Errorf("invalid service url %s", serviceURL) + } + return matches[2], matches[1], remotePort, nil +} diff --git a/src/internal/dns/dns_test.go b/src/internal/dns/dns_test.go new file mode 100644 index 0000000000..e9b62f6777 --- /dev/null +++ b/src/internal/dns/dns_test.go @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package dns + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsServiceURL(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + serviceURL string + expected bool + }{ + { + name: "is service url", + serviceURL: "http://registry.zarf.svc.cluster.local:1", + expected: true, + }, + { + name: "is not service url", + serviceURL: "https://zarf.dev", + expected: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + result := IsServiceURL(tt.serviceURL) + require.Equal(t, tt.expected, result) + }) + } +} + +func TestParseServiceURL(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + serviceURL string + expectedErr string + expectedNamespace string + expectedName string + expectedPort int + }{ + { + name: "correct service url", + serviceURL: "http://foo.bar.svc.cluster.local:5000", + expectedNamespace: "bar", + expectedName: "foo", + expectedPort: 5000, + }, + { + name: "invalid service url without port", + serviceURL: "http://google.com", + expectedErr: "service url does not have a port", + }, + { + name: "invalid service url with port", + serviceURL: "http://google.com:3000", + expectedErr: "invalid service url http://google.com:3000", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + namespace, name, port, err := ParseServiceURL(tt.serviceURL) + if tt.expectedErr != "" { + require.EqualError(t, err, tt.expectedErr) + return + } + require.Equal(t, tt.expectedNamespace, namespace) + require.Equal(t, tt.expectedName, name) + require.Equal(t, tt.expectedPort, port) + }) + } +} diff --git a/src/internal/packager2/load.go b/src/internal/packager2/load.go new file mode 100644 index 0000000000..30ba7157a1 --- /dev/null +++ b/src/internal/packager2/load.go @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package packager2 + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/defenseunicorns/pkg/helpers/v2" + "github.com/mholt/archiver/v3" + + "github.com/zarf-dev/zarf/src/config" + "github.com/zarf-dev/zarf/src/pkg/layout" + "github.com/zarf-dev/zarf/src/pkg/packager/filters" + "github.com/zarf-dev/zarf/src/pkg/packager/sources" + "github.com/zarf-dev/zarf/src/pkg/utils" + "github.com/zarf-dev/zarf/src/types" +) + +// LoadPackage optionally fetches and loads the package from the given source. +func LoadPackage(ctx context.Context, src, shasum, publicKeyPath string, skipSignatureValidation bool, filter filters.ComponentFilterStrategy) (*layout.PackagePaths, error) { + srcType, err := identifySource(src) + if err != nil { + return nil, err + } + + tarDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return nil, err + } + defer os.RemoveAll(tarDir) + tarPath := filepath.Join(tarDir, "packager.tar") + isPartial := false + switch srcType { + case "oci": + isPartial, err = pullOCI(ctx, src, tarPath, shasum, filter) + if err != nil { + return nil, err + } + case "http", "https": + err = pullHTTP(ctx, src, tarPath, shasum) + if err != nil { + return nil, err + } + case "split": + err = assembleSplitTar(src, tarPath) + if err != nil { + return nil, err + } + case "tarball": + tarPath = src + default: + return nil, fmt.Errorf("unknown source type: %s", src) + } + if srcType != "oci" && shasum != "" { + err := helpers.SHAsMatch(tarPath, shasum) + if err != nil { + return nil, err + } + } + + // Extract the package + packageDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return nil, err + } + pathsExtracted := []string{} + err = archiver.Walk(tarPath, func(f archiver.File) error { + if f.IsDir() { + return nil + } + header, ok := f.Header.(*tar.Header) + if !ok { + return fmt.Errorf("expected header to be *tar.Header but was %T", f.Header) + } + // If path has nested directories we want to create them. + dir := filepath.Dir(header.Name) + if dir != "." { + err := os.MkdirAll(filepath.Join(packageDir, dir), helpers.ReadExecuteAllWriteUser) + if err != nil { + return err + } + } + dst, err := os.Create(filepath.Join(packageDir, header.Name)) + if err != nil { + return err + } + defer dst.Close() + _, err = io.Copy(dst, f) + if err != nil { + return err + } + pathsExtracted = append(pathsExtracted, header.Name) + return nil + }) + if err != nil { + return nil, err + } + + // Load the package paths + pkgPaths := layout.New(packageDir) + pkgPaths.SetFromPaths(pathsExtracted) + pkg, _, err := pkgPaths.ReadZarfYAML() + if err != nil { + return nil, err + } + // TODO: Filter is not persistently applied. + pkg.Components, err = filter.Apply(pkg) + if err != nil { + return nil, err + } + if err := pkgPaths.MigrateLegacy(); err != nil { + return nil, err + } + if !pkgPaths.IsLegacyLayout() { + if err := sources.ValidatePackageIntegrity(pkgPaths, pkg.Metadata.AggregateChecksum, isPartial); err != nil { + return nil, err + } + if skipSignatureValidation { + if err := sources.ValidatePackageSignature(ctx, pkgPaths, publicKeyPath); err != nil { + return nil, err + } + } + } + for _, component := range pkg.Components { + if err := pkgPaths.Components.Unarchive(component); err != nil { + if errors.Is(err, layout.ErrNotLoaded) { + _, err := pkgPaths.Components.Create(component) + if err != nil { + return nil, err + } + } else { + return nil, err + } + } + } + if pkgPaths.SBOMs.Path != "" { + if err := pkgPaths.SBOMs.Unarchive(); err != nil { + return nil, err + } + } + return pkgPaths, nil +} + +func identifySource(src string) (string, error) { + parsed, err := url.Parse(src) + if err == nil && parsed.Scheme != "" && parsed.Host != "" { + return parsed.Scheme, nil + } + if strings.HasSuffix(src, ".tar.zst") || strings.HasSuffix(src, ".tar") { + return "tarball", nil + } + if strings.Contains(src, ".part000") { + return "split", nil + } + return "", fmt.Errorf("unknown source %s", src) +} + +func assembleSplitTar(src, tarPath string) error { + pattern := strings.Replace(src, ".part000", ".part*", 1) + splitFiles, err := filepath.Glob(pattern) + if err != nil { + return fmt.Errorf("unable to find split tarball files: %w", err) + } + // Ensure the files are in order so they are appended in the correct order + sort.Strings(splitFiles) + + tarFile, err := os.Create(tarPath) + if err != nil { + return err + } + defer tarFile.Close() + + var pkgData types.ZarfSplitPackageData + for _, splitFile := range splitFiles { + f, err := os.Open(splitFile) + if err != nil { + return fmt.Errorf("unable to open file %s: %w", splitFile, err) + } + defer f.Close() + _, err = io.Copy(tarFile, f) + if err != nil { + return fmt.Errorf("unable to copy file %s: %w", splitFile, err) + } + err = f.Close() + if err != nil { + return fmt.Errorf("unable to close file %s: %w", splitFile, err) + } + } + if err := helpers.SHAsMatch(tarPath, pkgData.Sha256Sum); err != nil { + return fmt.Errorf("package integrity check failed: %w", err) + } + return nil +} diff --git a/src/internal/packager2/load_test.go b/src/internal/packager2/load_test.go new file mode 100644 index 0000000000..93ad6b23cd --- /dev/null +++ b/src/internal/packager2/load_test.go @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package packager2 + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIdentifySource(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + src string + expectedSrcType string + }{ + { + name: "oci", + src: "oci://ghcr.io/defenseunicorns/packages/init:1.0.0", + expectedSrcType: "oci", + }, + { + name: "sget with sub path", + src: "sget://github.com/defenseunicorns/zarf-hello-world:x86", + expectedSrcType: "sget", + }, + { + name: "sget without host", + src: "sget://defenseunicorns/zarf-hello-world:x86_64", + expectedSrcType: "sget", + }, + { + name: "https", + src: "https://github.com/zarf-dev/zarf/releases/download/v1.0.0/zarf-init-amd64-v1.0.0.tar.zst", + expectedSrcType: "https", + }, + { + name: "http", + src: "http://github.com/zarf-dev/zarf/releases/download/v1.0.0/zarf-init-amd64-v1.0.0.tar.zst", + expectedSrcType: "http", + }, + { + name: "local tar init zst", + src: "zarf-init-amd64-v1.0.0.tar.zst", + expectedSrcType: "tarball", + }, + { + name: "local tar", + src: "zarf-package-manifests-amd64-v1.0.0.tar", + expectedSrcType: "tarball", + }, + { + name: "local tar manifest zst", + src: "zarf-package-manifests-amd64-v1.0.0.tar.zst", + expectedSrcType: "tarball", + }, + { + name: "local tar split", + src: "testdata/.part000", + expectedSrcType: "split", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + srcType, err := identifySource(tt.src) + require.NoError(t, err) + require.Equal(t, tt.expectedSrcType, srcType) + }) + } +} diff --git a/src/internal/packager2/mirror.go b/src/internal/packager2/mirror.go new file mode 100644 index 0000000000..56d261636c --- /dev/null +++ b/src/internal/packager2/mirror.go @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +package packager2 + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/avast/retry-go/v4" + "github.com/defenseunicorns/pkg/helpers/v2" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/crane" + "github.com/google/go-containerregistry/pkg/logs" + v1 "github.com/google/go-containerregistry/pkg/v1" + + "github.com/zarf-dev/zarf/src/config" + "github.com/zarf-dev/zarf/src/internal/dns" + "github.com/zarf-dev/zarf/src/internal/git" + "github.com/zarf-dev/zarf/src/internal/gitea" + "github.com/zarf-dev/zarf/src/pkg/cluster" + "github.com/zarf-dev/zarf/src/pkg/layout" + "github.com/zarf-dev/zarf/src/pkg/message" + "github.com/zarf-dev/zarf/src/pkg/packager/filters" + "github.com/zarf-dev/zarf/src/pkg/transform" + "github.com/zarf-dev/zarf/src/pkg/utils" + "github.com/zarf-dev/zarf/src/types" +) + +// Mirror mirrors the package contents to the give registry and git server. +func Mirror(ctx context.Context, c *cluster.Cluster, pkgPaths layout.PackagePaths, filter filters.ComponentFilterStrategy, regInfo types.RegistryInfo, gitInfo types.GitServerInfo, noImgChecksum bool, retries int) error { + err := pushImagesToRegistry(ctx, c, pkgPaths, filter, regInfo, noImgChecksum, retries) + if err != nil { + return err + } + err = pushReposToRepository(ctx, c, pkgPaths, filter, gitInfo, retries) + if err != nil { + return err + } + return nil +} + +func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, pkgPaths layout.PackagePaths, filter filters.ComponentFilterStrategy, regInfo types.RegistryInfo, noImgChecksum bool, retries int) error { + logs.Warn.SetOutput(&message.DebugWriter{}) + logs.Progress.SetOutput(&message.DebugWriter{}) + + pkg, _, err := pkgPaths.ReadZarfYAML() + if err != nil { + return err + } + components, err := filter.Apply(pkg) + if err != nil { + return err + } + pkg.Components = components + + images := map[transform.Image]v1.Image{} + for _, component := range pkg.Components { + for _, img := range component.Images { + ref, err := transform.ParseImageRef(img) + if err != nil { + return fmt.Errorf("failed to create ref for image %s: %w", img, err) + } + if _, ok := images[ref]; ok { + continue + } + ociImage, err := utils.LoadOCIImage(pkgPaths.Images.Base, ref) + if err != nil { + return err + } + images[ref] = ociImage + } + } + if len(images) == 0 { + return nil + } + + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig.InsecureSkipVerify = config.CommonOptions.InsecureSkipTLSVerify + // TODO (@WSTARR) This is set to match the TLSHandshakeTimeout to potentially mitigate effects of https://github.com/zarf-dev/zarf/issues/1444 + transport.ResponseHeaderTimeout = 10 * time.Second + transportWithProgressBar := helpers.NewTransport(transport, nil) + + pushOptions := []crane.Option{ + crane.WithPlatform(&v1.Platform{OS: "linux", Architecture: pkg.Build.Architecture}), + crane.WithTransport(transportWithProgressBar), + crane.WithAuth(authn.FromConfig(authn.AuthConfig{ + Username: regInfo.PushUsername, + Password: regInfo.PushPassword, + })), + crane.WithUserAgent("zarf"), + crane.WithNoClobber(true), + crane.WithJobs(1), + } + if config.CommonOptions.InsecureSkipTLSVerify { + pushOptions = append(pushOptions, crane.Insecure) + } + + for refInfo, img := range images { + err = retry.Do(func() error { + pushImage := func(registryUrl string) error { + names := []string{} + if !noImgChecksum { + offlineNameCRC, err := transform.ImageTransformHost(registryUrl, refInfo.Reference) + if err != nil { + return retry.Unrecoverable(err) + } + names = append(names, offlineNameCRC) + } + offlineName, err := transform.ImageTransformHostWithoutChecksum(registryUrl, refInfo.Reference) + if err != nil { + return retry.Unrecoverable(err) + } + names = append(names, offlineName) + for _, name := range names { + err = crane.Push(img, name, pushOptions...) + if err != nil { + return err + } + } + return nil + } + + if c == nil { + return pushImage(regInfo.Address) + } + + namespace, name, port, err := dns.ParseServiceURL(regInfo.Address) + if err != nil { + return err + } + tunnel, err := c.NewTunnel(namespace, cluster.SvcResource, name, "", 0, port) + if err != nil { + return err + } + _, err = tunnel.Connect(ctx) + if err != nil { + return err + } + defer tunnel.Close() + err = tunnel.Wrap(func() error { + return pushImage(tunnel.Endpoint()) + }) + if err != nil { + return err + } + return nil + }, retry.Context(ctx), retry.Attempts(uint(retries)), retry.Delay(500*time.Millisecond)) + if err != nil { + return err + } + } + return nil +} + +func pushReposToRepository(ctx context.Context, c *cluster.Cluster, pkgPaths layout.PackagePaths, filter filters.ComponentFilterStrategy, gitInfo types.GitServerInfo, retries int) error { + pkg, _, err := pkgPaths.ReadZarfYAML() + if err != nil { + return err + } + components, err := filter.Apply(pkg) + if err != nil { + return err + } + pkg.Components = components + + for _, component := range pkg.Components { + for _, repoURL := range component.Repos { + repository, err := git.Open(pkgPaths.Components.Dirs[component.Name].Repos, repoURL) + if err != nil { + return err + } + err = retry.Do(func() error { + if c == nil { + err = repository.Push(ctx, gitInfo.Address, gitInfo.PushUsername, gitInfo.PushPassword) + if err != nil { + return err + } + return nil + } + + namespace, name, port, err := dns.ParseServiceURL(gitInfo.Address) + if err != nil { + return retry.Unrecoverable(err) + } + tunnel, err := c.NewTunnel(namespace, cluster.SvcResource, name, "", 0, port) + if err != nil { + return err + } + _, err = tunnel.Connect(ctx) + if err != nil { + return err + } + defer tunnel.Close() + giteaClient, err := gitea.NewClient(tunnel.HTTPEndpoint(), gitInfo.PushUsername, gitInfo.PushPassword) + if err != nil { + return err + } + return tunnel.Wrap(func() error { + err = repository.Push(ctx, tunnel.HTTPEndpoint(), gitInfo.PushUsername, gitInfo.PushPassword) + if err != nil { + return err + } + // Add the read-only user to this repo + // TODO: This should not be done here. Or the function name should be changed. + repoName, err := transform.GitURLtoRepoName(repoURL) + if err != nil { + return retry.Unrecoverable(err) + } + err = giteaClient.AddReadOnlyUserToRepository(ctx, repoName, gitInfo.PullUsername) + if err != nil { + return fmt.Errorf("unable to add the read only user to the repo %s: %w", repoName, err) + } + return nil + }) + }, retry.Context(ctx), retry.Attempts(uint(retries)), retry.Delay(500*time.Millisecond)) + if err != nil { + return fmt.Errorf("unable to push repo %s to the Git Server: %w", repoURL, err) + } + } + } + return nil +} diff --git a/src/internal/packager2/packager.go b/src/internal/packager2/packager.go new file mode 100644 index 0000000000..b0e8dc79a0 --- /dev/null +++ b/src/internal/packager2/packager.go @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package packager2 is the new implementation for packager. +package packager2 diff --git a/src/internal/packager2/pull.go b/src/internal/packager2/pull.go index bc2930ce16..538facc5b9 100644 --- a/src/internal/packager2/pull.go +++ b/src/internal/packager2/pull.go @@ -50,7 +50,7 @@ func Pull(ctx context.Context, src, dir, shasum string, filter filters.Component switch u.Scheme { case "oci": - err := pullOCI(ctx, src, tmpPath, shasum, filter) + _, err := pullOCI(ctx, src, tmpPath, shasum, filter) if err != nil { return err } @@ -89,10 +89,10 @@ func Pull(ctx context.Context, src, dir, shasum string, filter filters.Component return nil } -func pullOCI(ctx context.Context, src, tarPath, shasum string, filter filters.ComponentFilterStrategy) error { +func pullOCI(ctx context.Context, src, tarPath, shasum string, filter filters.ComponentFilterStrategy) (bool, error) { tmpDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { - return err + return false, err } defer os.Remove(tmpDir) if shasum != "" { @@ -101,40 +101,48 @@ func pullOCI(ctx context.Context, src, tarPath, shasum string, filter filters.Co arch := config.GetArch() remote, err := zoci.NewRemote(src, oci.PlatformForArch(arch)) if err != nil { - return err + return false, err } desc, err := remote.ResolveRoot(ctx) if err != nil { - return fmt.Errorf("could not fetch images index: %w", err) + return false, fmt.Errorf("could not fetch images index: %w", err) } layersToPull := []ocispec.Descriptor{} + isPartial := false if supportsFiltering(desc.Platform) { + root, err := remote.FetchRoot(ctx) + if err != nil { + return false, err + } + if len(root.Layers) != len(layersToPull) { + isPartial = true + } pkg, err := remote.FetchZarfYAML(ctx) if err != nil { - return err + return false, err } pkg.Components, err = filter.Apply(pkg) if err != nil { - return err + return false, err } layersToPull, err = remote.LayersFromRequestedComponents(ctx, pkg.Components) if err != nil { - return err + return false, err } } _, err = remote.PullPackage(ctx, tmpDir, config.CommonOptions.OCIConcurrency, layersToPull...) if err != nil { - return err + return false, err } allTheLayers, err := filepath.Glob(filepath.Join(tmpDir, "*")) if err != nil { - return err + return false, err } err = archiver.Archive(allTheLayers, tarPath) if err != nil { - return err + return false, err } - return nil + return isPartial, nil } func pullHTTP(ctx context.Context, src, tarPath, shasum string) error { diff --git a/src/test/external/ext_in_cluster_test.go b/src/test/external/ext_in_cluster_test.go index ffe5a08b73..ad1e338935 100644 --- a/src/test/external/ext_in_cluster_test.go +++ b/src/test/external/ext_in_cluster_test.go @@ -24,13 +24,23 @@ import ( "sigs.k8s.io/cli-utils/pkg/object" ) -var inClusterCredentialArgs = []string{ +var inClusterMirrorCredentialArgs = []string{ "--git-push-username=git-user", "--git-push-password=superSecurePassword", "--git-url=http://gitea-http.git-server.svc.cluster.local:3000", "--registry-push-username=push-user", "--registry-push-password=superSecurePassword", - "--registry-url=127.0.0.1:31999"} + "--registry-url=http://external-registry-docker-registry.external-registry.svc.cluster.local:5000", +} + +var inClusterInitCredentialArgs = []string{ + "--git-push-username=git-user", + "--git-push-password=superSecurePassword", + "--git-url=http://gitea-http.git-server.svc.cluster.local:3000", + "--registry-push-username=push-user", + "--registry-push-password=superSecurePassword", + "--registry-url=127.0.0.1:31999", +} type ExtInClusterTestSuite struct { suite.Suite @@ -97,7 +107,7 @@ func (suite *ExtInClusterTestSuite) TearDownSuite() { func (suite *ExtInClusterTestSuite) Test_0_Mirror() { // Use Zarf to mirror a package to the services (do this as test 0 so that the registry is unpolluted) mirrorArgs := []string{"package", "mirror-resources", "../../../build/zarf-package-argocd-amd64.tar.zst", "--confirm"} - mirrorArgs = append(mirrorArgs, inClusterCredentialArgs...) + mirrorArgs = append(mirrorArgs, inClusterMirrorCredentialArgs...) err := exec.CmdWithPrint(zarfBinPath, mirrorArgs...) suite.NoError(err, "unable to mirror the package with zarf") @@ -143,7 +153,7 @@ func (suite *ExtInClusterTestSuite) Test_0_Mirror() { func (suite *ExtInClusterTestSuite) Test_1_Deploy() { // Use Zarf to initialize the cluster initArgs := []string{"init", "--confirm"} - initArgs = append(initArgs, inClusterCredentialArgs...) + initArgs = append(initArgs, inClusterInitCredentialArgs...) err := exec.CmdWithPrint(zarfBinPath, initArgs...) suite.NoError(err, "unable to initialize the k8s server with zarf") temp := suite.T().TempDir()