From 96f9ac83683db29f0d88830db5f2d84dd69da933 Mon Sep 17 00:00:00 2001 From: Philip Laine Date: Mon, 23 Sep 2024 21:42:06 +0200 Subject: [PATCH] refactor: layout Signed-off-by: Philip Laine --- src/cmd/package.go | 7 +- src/internal/packager2/layout/layout.go | 300 ++++++++++++++++++++++++ src/internal/packager2/load.go | 102 +------- src/internal/packager2/load_test.go | 10 +- src/internal/packager2/mirror.go | 47 ++-- 5 files changed, 341 insertions(+), 125 deletions(-) create mode 100644 src/internal/packager2/layout/layout.go diff --git a/src/cmd/package.go b/src/cmd/package.go index d5c1141196..7b178a7e4b 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -153,14 +153,15 @@ var packageMirrorCmd = &cobra.Command{ SkipSignatureValidation: pkgConfig.PkgOpts.SkipSignatureValidation, Filter: filter, } - pkgPaths, err := packager2.LoadPackage(cmd.Context(), loadOpt) + pkgLayout, err := packager2.LoadPackage(cmd.Context(), loadOpt) if err != nil { return err } - defer os.RemoveAll(pkgPaths.Base) + //nolint: errcheck // ignore + defer pkgLayout.Cleanup() mirrorOpt := packager2.MirrorOptions{ Cluster: c, - PackagePaths: *pkgPaths, + PkgLayouit: pkgLayout, Filter: filter, RegistryInfo: pkgConfig.InitOpts.RegistryInfo, GitInfo: pkgConfig.InitOpts.GitServer, diff --git a/src/internal/packager2/layout/layout.go b/src/internal/packager2/layout/layout.go new file mode 100644 index 0000000000..451e79c130 --- /dev/null +++ b/src/internal/packager2/layout/layout.go @@ -0,0 +1,300 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package layout contains functions for inteacting the Zarf packages. +package layout + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/defenseunicorns/pkg/helpers/v2" + goyaml "github.com/goccy/go-yaml" + registryv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/mholt/archiver/v3" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sigstore/cosign/v2/cmd/cosign/cli/options" + "github.com/sigstore/cosign/v2/cmd/cosign/cli/verify" + + "github.com/zarf-dev/zarf/src/api/v1alpha1" + "github.com/zarf-dev/zarf/src/config" + "github.com/zarf-dev/zarf/src/pkg/packager/deprecated" + "github.com/zarf-dev/zarf/src/pkg/transform" + "github.com/zarf-dev/zarf/src/pkg/utils" +) + +const ( + zarfYAML = "zarf.yaml" + signature = "zarf.yaml.sig" + checksums = "checksums.txt" + imagesDir = "images" + componentsDir = "components" +) + +// PackageLayout manages the layout for a package. +type PackageLayout struct { + dirPath string + Pkg v1alpha1.ZarfPackage +} + +// PackageLayoutOptions are the options used when loading a package. +type PackageLayoutOptions struct { + PublicKeyPath string + SkipSignatureValidation bool + IsPartial bool +} + +// LoadFromTar unpacks the give compressed package and loads it. +func LoadFromTar(ctx context.Context, tarPath string, opt PackageLayoutOptions) (*PackageLayout, error) { + dirPath, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return nil, err + } + err = archiver.Walk(tarPath, func(f archiver.File) error { + if f.IsDir() { + return nil + } + header, ok := f.Header.(*tar.Header) + if !ok { + return fmt.Errorf("expected header to be *tar.Header but was %T", f.Header) + } + // If path has nested directories we want to create them. + dir := filepath.Dir(header.Name) + if dir != "." { + err := os.MkdirAll(filepath.Join(dirPath, dir), helpers.ReadExecuteAllWriteUser) + if err != nil { + return err + } + } + dst, err := os.Create(filepath.Join(dirPath, header.Name)) + if err != nil { + return err + } + defer dst.Close() + _, err = io.Copy(dst, f) + if err != nil { + return err + } + return nil + }) + if err != nil { + return nil, err + } + p, err := LoadFromDir(ctx, dirPath, opt) + if err != nil { + return nil, err + } + return p, nil +} + +// LoadFromDir loads and validates a package from the given directory path. +func LoadFromDir(ctx context.Context, dirPath string, opt PackageLayoutOptions) (*PackageLayout, error) { + b, err := os.ReadFile(filepath.Join(dirPath, zarfYAML)) + if err != nil { + return nil, err + } + var pkg v1alpha1.ZarfPackage + err = goyaml.Unmarshal(b, &pkg) + if err != nil { + return nil, err + } + if len(pkg.Build.Migrations) > 0 { + for idx, component := range pkg.Components { + pkg.Components[idx], _ = deprecated.MigrateComponent(pkg.Build, component) + } + } + + pkgLayout := &PackageLayout{ + dirPath: dirPath, + Pkg: pkg, + } + err = validatePackageIntegrity(pkgLayout, opt.IsPartial) + if err != nil { + return nil, err + } + err = validatePackageSignature(ctx, pkgLayout, opt.PublicKeyPath, opt.SkipSignatureValidation) + if err != nil { + return nil, err + } + return pkgLayout, nil +} + +// Cleanup removes any temporary directories created. +func (p *PackageLayout) Cleanup() error { + err := os.RemoveAll(p.dirPath) + if err != nil { + return err + } + return nil +} + +// GetDirectory returns a path to the directory in the given component. +// TODO (phillebaba): Make dirType an enum. +func (p *PackageLayout) GetDirectory(component string, dirType string) (string, error) { + sourcePath := filepath.Join(p.dirPath, componentsDir, fmt.Sprintf("%s.tar", component)) + _, err := os.Stat(sourcePath) + if err != nil { + return "", err + } + dirPath, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return "", err + } + err = archiver.Extract(sourcePath, filepath.Join(component, dirType), dirPath) + if err != nil { + return "", err + } + _, err = os.Stat(filepath.Join(dirPath, dirType)) + if err != nil { + return "", err + } + return filepath.Join(dirPath, dirType), nil +} + +// GetImage returns the image with the given reference in the package layout. +func (p *PackageLayout) GetImage(ref transform.Image) (registryv1.Image, error) { + // Use the manifest within the index.json to load the specific image we want + layoutPath := layout.Path(filepath.Join(p.dirPath, imagesDir)) + imgIdx, err := layoutPath.ImageIndex() + if err != nil { + return nil, err + } + idxManifest, err := imgIdx.IndexManifest() + if err != nil { + return nil, err + } + // Search through all the manifests within this package until we find the annotation that matches our ref + for _, manifest := range idxManifest.Manifests { + if manifest.Annotations[ocispec.AnnotationBaseImageName] == ref.Reference || + // A backwards compatibility shim for older Zarf versions that would leave docker.io off of image annotations + (manifest.Annotations[ocispec.AnnotationBaseImageName] == ref.Path+ref.TagOrDigest && ref.Host == "docker.io") { + // This is the image we are looking for, load it and then return + return layoutPath.Image(manifest.Digest) + } + } + return nil, fmt.Errorf("unable to find the image %s", ref.Reference) +} + +func validatePackageIntegrity(pkgLayout *PackageLayout, isPartial bool) error { + _, err := os.Stat(filepath.Join(pkgLayout.dirPath, zarfYAML)) + if err != nil { + return err + } + _, err = os.Stat(filepath.Join(pkgLayout.dirPath, checksums)) + if err != nil { + return err + } + err = helpers.SHAsMatch(filepath.Join(pkgLayout.dirPath, checksums), pkgLayout.Pkg.Metadata.AggregateChecksum) + if err != nil { + return err + } + + packageFiles := map[string]interface{}{} + err = filepath.Walk(pkgLayout.dirPath, func(path string, info fs.FileInfo, err error) error { + if info.IsDir() { + return nil + } + packageFiles[path] = nil + return err + }) + if err != nil { + return err + } + // Remove files which are not in the checksums. + delete(packageFiles, filepath.Join(pkgLayout.dirPath, zarfYAML)) + delete(packageFiles, filepath.Join(pkgLayout.dirPath, checksums)) + delete(packageFiles, filepath.Join(pkgLayout.dirPath, signature)) + + b, err := os.ReadFile(filepath.Join(pkgLayout.dirPath, checksums)) + if err != nil { + return err + } + lines := strings.Split(string(b), "\n") + for _, line := range lines { + // If the line is empty (i.e. there is no checksum) simply skip it, this can result from a package with no images/components. + if line == "" { + continue + } + + split := strings.Split(line, " ") + if len(split) != 2 { + return fmt.Errorf("invalid checksum line: %s", line) + } + sha := split[0] + rel := split[1] + if sha == "" || rel == "" { + return fmt.Errorf("invalid checksum line: %s", line) + } + + path := filepath.Join(pkgLayout.dirPath, rel) + _, ok := packageFiles[path] + if !ok && isPartial { + delete(packageFiles, path) + continue + } + if !ok { + return fmt.Errorf("file %s from checksum missing in layout", rel) + } + err = helpers.SHAsMatch(path, sha) + if err != nil { + return err + } + delete(packageFiles, path) + } + + if len(packageFiles) > 0 { + // TODO (phillebaba): Replace with maps.Keys after upgrading to Go 1.23. + filePaths := []string{} + for k := range packageFiles { + filePaths = append(filePaths, k) + } + return fmt.Errorf("package contains additional files not present in the checksum %s", strings.Join(filePaths, ", ")) + } + + return nil +} + +func validatePackageSignature(ctx context.Context, pkgLayout *PackageLayout, publicKeyPath string, skipSignatureValidation bool) error { + if skipSignatureValidation { + return nil + } + + signaturePath := filepath.Join(pkgLayout.dirPath, signature) + zarfYamlPath := filepath.Join(pkgLayout.dirPath, zarfYAML) + + sigExist := true + _, err := os.Stat(signaturePath) + if err != nil { + sigExist = false + } + if !sigExist && publicKeyPath == "" { + // Nobody was expecting a signature, so we can just return + return nil + } else if sigExist && publicKeyPath == "" { + return errors.New("package is signed but no key was provided") + } else if !sigExist && publicKeyPath != "" { + return errors.New("a key was provided but the package is not signed") + } + + keyOptions := options.KeyOpts{KeyRef: publicKeyPath} + cmd := &verify.VerifyBlobCmd{ + KeyOpts: keyOptions, + SigRef: signaturePath, + IgnoreSCT: true, + Offline: true, + IgnoreTlog: true, + } + err = cmd.Exec(ctx, zarfYamlPath) + if err != nil { + return fmt.Errorf("package signature did not match the provided key: %w", err) + } + return nil +} diff --git a/src/internal/packager2/load.go b/src/internal/packager2/load.go index 49dc7dea5b..f8e6030515 100644 --- a/src/internal/packager2/load.go +++ b/src/internal/packager2/load.go @@ -4,10 +4,8 @@ package packager2 import ( - "archive/tar" "context" "encoding/json" - "errors" "fmt" "io" "net/url" @@ -17,14 +15,12 @@ import ( "strings" "github.com/defenseunicorns/pkg/helpers/v2" - "github.com/mholt/archiver/v3" "github.com/zarf-dev/zarf/src/api/v1alpha1" "github.com/zarf-dev/zarf/src/config" + "github.com/zarf-dev/zarf/src/internal/packager2/layout" "github.com/zarf-dev/zarf/src/pkg/cluster" - "github.com/zarf-dev/zarf/src/pkg/layout" "github.com/zarf-dev/zarf/src/pkg/packager/filters" - "github.com/zarf-dev/zarf/src/pkg/packager/sources" "github.com/zarf-dev/zarf/src/pkg/utils" "github.com/zarf-dev/zarf/src/types" ) @@ -39,7 +35,7 @@ type LoadOptions struct { } // LoadPackage optionally fetches and loads the package from the given source. -func LoadPackage(ctx context.Context, opt LoadOptions) (*layout.PackagePaths, error) { +func LoadPackage(ctx context.Context, opt LoadOptions) (*layout.PackageLayout, error) { srcType, err := identifySource(opt.Source) if err != nil { return nil, err @@ -81,87 +77,16 @@ func LoadPackage(ctx context.Context, opt LoadOptions) (*layout.PackagePaths, er } } - // Extract the package - packageDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) - if err != nil { - return nil, err + layoutOpt := layout.PackageLayoutOptions{ + PublicKeyPath: opt.PublicKeyPath, + SkipSignatureValidation: opt.SkipSignatureValidation, + IsPartial: isPartial, } - pathsExtracted := []string{} - err = archiver.Walk(tarPath, func(f archiver.File) error { - if f.IsDir() { - return nil - } - header, ok := f.Header.(*tar.Header) - if !ok { - return fmt.Errorf("expected header to be *tar.Header but was %T", f.Header) - } - // If path has nested directories we want to create them. - dir := filepath.Dir(header.Name) - if dir != "." { - err := os.MkdirAll(filepath.Join(packageDir, dir), helpers.ReadExecuteAllWriteUser) - if err != nil { - return err - } - } - dst, err := os.Create(filepath.Join(packageDir, header.Name)) - if err != nil { - return err - } - defer dst.Close() - _, err = io.Copy(dst, f) - if err != nil { - return err - } - pathsExtracted = append(pathsExtracted, header.Name) - return nil - }) + pkgLayout, err := layout.LoadFromTar(ctx, tarPath, layoutOpt) if err != nil { return nil, err } - - // Load the package paths - pkgPaths := layout.New(packageDir) - pkgPaths.SetFromPaths(pathsExtracted) - pkg, _, err := pkgPaths.ReadZarfYAML() - if err != nil { - return nil, err - } - // TODO: Filter is not persistently applied. - pkg.Components, err = opt.Filter.Apply(pkg) - if err != nil { - return nil, err - } - if err := pkgPaths.MigrateLegacy(); err != nil { - return nil, err - } - if !pkgPaths.IsLegacyLayout() { - if err := sources.ValidatePackageIntegrity(pkgPaths, pkg.Metadata.AggregateChecksum, isPartial); err != nil { - return nil, err - } - if opt.SkipSignatureValidation { - if err := sources.ValidatePackageSignature(ctx, pkgPaths, opt.PublicKeyPath); err != nil { - return nil, err - } - } - } - for _, component := range pkg.Components { - if err := pkgPaths.Components.Unarchive(component); err != nil { - if errors.Is(err, layout.ErrNotLoaded) { - _, err := pkgPaths.Components.Create(component) - if err != nil { - return nil, err - } - } else { - return nil, err - } - } - } - if pkgPaths.SBOMs.Path != "" { - if err := pkgPaths.SBOMs.Unarchive(); err != nil { - return nil, err - } - } - return pkgPaths, nil + return pkgLayout, nil } // identifySource returns the source type for the given source. @@ -245,14 +170,11 @@ func packageFromSourceOrCluster(ctx context.Context, cluster *cluster.Cluster, s SkipSignatureValidation: skipSignatureValidation, Filter: filters.Empty(), } - pkgPaths, err := LoadPackage(ctx, loadOpt) - if err != nil { - return v1alpha1.ZarfPackage{}, err - } - defer os.RemoveAll(pkgPaths.Base) - pkg, _, err := pkgPaths.ReadZarfYAML() + p, err := LoadPackage(ctx, loadOpt) if err != nil { return v1alpha1.ZarfPackage{}, err } - return pkg, nil + //nolint: errcheck // ignore + defer p.Cleanup() + return p.Pkg, nil } diff --git a/src/internal/packager2/load_test.go b/src/internal/packager2/load_test.go index 208f68ba65..5783218ef9 100644 --- a/src/internal/packager2/load_test.go +++ b/src/internal/packager2/load_test.go @@ -48,14 +48,12 @@ func TestLoadPackage(t *testing.T) { SkipSignatureValidation: false, Filter: filters.Empty(), } - pkgPaths, err := LoadPackage(ctx, opt) + pkgLayout, err := LoadPackage(ctx, opt) require.NoError(t, err) - pkg, _, err := pkgPaths.ReadZarfYAML() - require.NoError(t, err) - require.Equal(t, "test", pkg.Metadata.Name) - require.Equal(t, "0.0.1", pkg.Metadata.Version) - require.Len(t, pkg.Components, 1) + require.Equal(t, "test", pkgLayout.Pkg.Metadata.Name) + require.Equal(t, "0.0.1", pkgLayout.Pkg.Metadata.Version) + require.Len(t, pkgLayout.Pkg.Components, 1) } opt := LoadOptions{ diff --git a/src/internal/packager2/mirror.go b/src/internal/packager2/mirror.go index 7649b62757..435639cea2 100644 --- a/src/internal/packager2/mirror.go +++ b/src/internal/packager2/mirror.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "net/http" + "os" "time" "github.com/avast/retry-go/v4" @@ -21,19 +22,18 @@ import ( "github.com/zarf-dev/zarf/src/internal/dns" "github.com/zarf-dev/zarf/src/internal/git" "github.com/zarf-dev/zarf/src/internal/gitea" + "github.com/zarf-dev/zarf/src/internal/packager2/layout" "github.com/zarf-dev/zarf/src/pkg/cluster" - "github.com/zarf-dev/zarf/src/pkg/layout" "github.com/zarf-dev/zarf/src/pkg/message" "github.com/zarf-dev/zarf/src/pkg/packager/filters" "github.com/zarf-dev/zarf/src/pkg/transform" - "github.com/zarf-dev/zarf/src/pkg/utils" "github.com/zarf-dev/zarf/src/types" ) // MirrorOptions are the options for Mirror. type MirrorOptions struct { Cluster *cluster.Cluster - PackagePaths layout.PackagePaths + PkgLayouit *layout.PackageLayout Filter filters.ComponentFilterStrategy RegistryInfo types.RegistryInfo GitInfo types.GitServerInfo @@ -43,33 +43,29 @@ type MirrorOptions struct { // Mirror mirrors the package contents to the given registry and git server. func Mirror(ctx context.Context, opt MirrorOptions) error { - err := pushImagesToRegistry(ctx, opt.Cluster, opt.PackagePaths, opt.Filter, opt.RegistryInfo, opt.NoImageChecksum, opt.Retries) + err := pushImagesToRegistry(ctx, opt.Cluster, opt.PkgLayouit, opt.Filter, opt.RegistryInfo, opt.NoImageChecksum, opt.Retries) if err != nil { return err } - err = pushReposToRepository(ctx, opt.Cluster, opt.PackagePaths, opt.Filter, opt.GitInfo, opt.Retries) + err = pushReposToRepository(ctx, opt.Cluster, opt.PkgLayouit, opt.Filter, opt.GitInfo, opt.Retries) if err != nil { return err } return nil } -func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, pkgPaths layout.PackagePaths, filter filters.ComponentFilterStrategy, regInfo types.RegistryInfo, noImgChecksum bool, retries int) error { +func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, pkgLayout *layout.PackageLayout, filter filters.ComponentFilterStrategy, regInfo types.RegistryInfo, noImgChecksum bool, retries int) error { logs.Warn.SetOutput(&message.DebugWriter{}) logs.Progress.SetOutput(&message.DebugWriter{}) - pkg, _, err := pkgPaths.ReadZarfYAML() + components, err := filter.Apply(pkgLayout.Pkg) if err != nil { return err } - components, err := filter.Apply(pkg) - if err != nil { - return err - } - pkg.Components = components + // pkg.Components = components images := map[transform.Image]v1.Image{} - for _, component := range pkg.Components { + for _, component := range components { for _, img := range component.Images { ref, err := transform.ParseImageRef(img) if err != nil { @@ -78,11 +74,11 @@ func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, pkgPaths layo if _, ok := images[ref]; ok { continue } - ociImage, err := utils.LoadOCIImage(pkgPaths.Images.Base, ref) + img, err := pkgLayout.GetImage(ref) if err != nil { return err } - images[ref] = ociImage + images[ref] = img } } if len(images) == 0 { @@ -96,7 +92,7 @@ func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, pkgPaths layo transportWithProgressBar := helpers.NewTransport(transport, nil) pushOptions := []crane.Option{ - crane.WithPlatform(&v1.Platform{OS: "linux", Architecture: pkg.Build.Architecture}), + crane.WithPlatform(&v1.Platform{OS: "linux", Architecture: pkgLayout.Pkg.Build.Architecture}), crane.WithTransport(transportWithProgressBar), crane.WithAuth(authn.FromConfig(authn.AuthConfig{ Username: regInfo.PushUsername, @@ -171,20 +167,19 @@ func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, pkgPaths layo return nil } -func pushReposToRepository(ctx context.Context, c *cluster.Cluster, pkgPaths layout.PackagePaths, filter filters.ComponentFilterStrategy, gitInfo types.GitServerInfo, retries int) error { - pkg, _, err := pkgPaths.ReadZarfYAML() - if err != nil { - return err - } - components, err := filter.Apply(pkg) +func pushReposToRepository(ctx context.Context, c *cluster.Cluster, pkgLayout *layout.PackageLayout, filter filters.ComponentFilterStrategy, gitInfo types.GitServerInfo, retries int) error { + components, err := filter.Apply(pkgLayout.Pkg) if err != nil { return err } - pkg.Components = components - - for _, component := range pkg.Components { + for _, component := range components { for _, repoURL := range component.Repos { - repository, err := git.Open(pkgPaths.Components.Dirs[component.Name].Repos, repoURL) + reposPath, err := pkgLayout.GetDirectory(component.Name, "repos") + if err != nil { + return err + } + defer os.RemoveAll(reposPath) + repository, err := git.Open(reposPath, repoURL) if err != nil { return err }