diff --git a/src/cmd/package.go b/src/cmd/package.go index d168ed80dd..6a92f9a88c 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -153,14 +153,14 @@ var packageMirrorCmd = &cobra.Command{ SkipSignatureValidation: pkgConfig.PkgOpts.SkipSignatureValidation, Filter: filter, } - pkgPaths, err := packager2.LoadPackage(cmd.Context(), loadOpt) + p, err := packager2.LoadPackage(cmd.Context(), loadOpt) if err != nil { return err } - defer os.RemoveAll(pkgPaths.Base) + defer p.Cleanup() mirrorOpt := packager2.MirrorOptions{ Cluster: c, - PackagePaths: *pkgPaths, + Package: p, Filter: filter, RegistryInfo: pkgConfig.InitOpts.RegistryInfo, GitInfo: pkgConfig.InitOpts.GitServer, diff --git a/src/internal/packager2/load.go b/src/internal/packager2/load.go index b20eea6195..6aa0a8cc1c 100644 --- a/src/internal/packager2/load.go +++ b/src/internal/packager2/load.go @@ -4,10 +4,8 @@ package packager2 import ( - "archive/tar" "context" "encoding/json" - "errors" "fmt" "io" "net/url" @@ -17,12 +15,9 @@ import ( "strings" "github.com/defenseunicorns/pkg/helpers/v2" - "github.com/mholt/archiver/v3" "github.com/zarf-dev/zarf/src/config" - "github.com/zarf-dev/zarf/src/pkg/layout" "github.com/zarf-dev/zarf/src/pkg/packager/filters" - "github.com/zarf-dev/zarf/src/pkg/packager/sources" "github.com/zarf-dev/zarf/src/pkg/utils" "github.com/zarf-dev/zarf/src/types" ) @@ -37,7 +32,7 @@ type LoadOptions struct { } // LoadPackage optionally fetches and loads the package from the given source. -func LoadPackage(ctx context.Context, opt LoadOptions) (*layout.PackagePaths, error) { +func LoadPackage(ctx context.Context, opt LoadOptions) (*Package, error) { srcType, err := identifySource(opt.Source) if err != nil { return nil, err @@ -79,87 +74,12 @@ func LoadPackage(ctx context.Context, opt LoadOptions) (*layout.PackagePaths, er } } - // Extract the package - packageDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + fmt.Println(isPartial) + p, err := LoadFromTar(ctx, tarPath, opt.PublicKeyPath) if err != nil { return nil, err } - pathsExtracted := []string{} - err = archiver.Walk(tarPath, func(f archiver.File) error { - if f.IsDir() { - return nil - } - header, ok := f.Header.(*tar.Header) - if !ok { - return fmt.Errorf("expected header to be *tar.Header but was %T", f.Header) - } - // If path has nested directories we want to create them. - dir := filepath.Dir(header.Name) - if dir != "." { - err := os.MkdirAll(filepath.Join(packageDir, dir), helpers.ReadExecuteAllWriteUser) - if err != nil { - return err - } - } - dst, err := os.Create(filepath.Join(packageDir, header.Name)) - if err != nil { - return err - } - defer dst.Close() - _, err = io.Copy(dst, f) - if err != nil { - return err - } - pathsExtracted = append(pathsExtracted, header.Name) - return nil - }) - if err != nil { - return nil, err - } - - // Load the package paths - pkgPaths := layout.New(packageDir) - pkgPaths.SetFromPaths(pathsExtracted) - pkg, _, err := pkgPaths.ReadZarfYAML() - if err != nil { - return nil, err - } - // TODO: Filter is not persistently applied. - pkg.Components, err = opt.Filter.Apply(pkg) - if err != nil { - return nil, err - } - if err := pkgPaths.MigrateLegacy(); err != nil { - return nil, err - } - if !pkgPaths.IsLegacyLayout() { - if err := sources.ValidatePackageIntegrity(pkgPaths, pkg.Metadata.AggregateChecksum, isPartial); err != nil { - return nil, err - } - if opt.SkipSignatureValidation { - if err := sources.ValidatePackageSignature(ctx, pkgPaths, opt.PublicKeyPath); err != nil { - return nil, err - } - } - } - for _, component := range pkg.Components { - if err := pkgPaths.Components.Unarchive(component); err != nil { - if errors.Is(err, layout.ErrNotLoaded) { - _, err := pkgPaths.Components.Create(component) - if err != nil { - return nil, err - } - } else { - return nil, err - } - } - } - if pkgPaths.SBOMs.Path != "" { - if err := pkgPaths.SBOMs.Unarchive(); err != nil { - return nil, err - } - } - return pkgPaths, nil + return p, nil } func identifySource(src string) (string, error) { diff --git a/src/internal/packager2/load_test.go b/src/internal/packager2/load_test.go index b9b6cf37c2..35cca50cbf 100644 --- a/src/internal/packager2/load_test.go +++ b/src/internal/packager2/load_test.go @@ -46,14 +46,12 @@ func TestLoadPackage(t *testing.T) { SkipSignatureValidation: false, Filter: filters.Empty(), } - pkgPaths, err := LoadPackage(ctx, opt) + p, err := LoadPackage(ctx, opt) require.NoError(t, err) - pkg, _, err := pkgPaths.ReadZarfYAML() - require.NoError(t, err) - require.Equal(t, "test", pkg.Metadata.Name) - require.Equal(t, "0.0.1", pkg.Metadata.Version) - require.Len(t, pkg.Components, 1) + require.Equal(t, "test", p.pkg.Metadata.Name) + require.Equal(t, "0.0.1", p.pkg.Metadata.Version) + require.Len(t, p.pkg.Components, 1) } opt := LoadOptions{ diff --git a/src/internal/packager2/mirror.go b/src/internal/packager2/mirror.go index 7649b62757..557949b168 100644 --- a/src/internal/packager2/mirror.go +++ b/src/internal/packager2/mirror.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "net/http" + "os" "time" "github.com/avast/retry-go/v4" @@ -22,18 +23,16 @@ import ( "github.com/zarf-dev/zarf/src/internal/git" "github.com/zarf-dev/zarf/src/internal/gitea" "github.com/zarf-dev/zarf/src/pkg/cluster" - "github.com/zarf-dev/zarf/src/pkg/layout" "github.com/zarf-dev/zarf/src/pkg/message" "github.com/zarf-dev/zarf/src/pkg/packager/filters" "github.com/zarf-dev/zarf/src/pkg/transform" - "github.com/zarf-dev/zarf/src/pkg/utils" "github.com/zarf-dev/zarf/src/types" ) // MirrorOptions are the options for Mirror. type MirrorOptions struct { Cluster *cluster.Cluster - PackagePaths layout.PackagePaths + Package *Package Filter filters.ComponentFilterStrategy RegistryInfo types.RegistryInfo GitInfo types.GitServerInfo @@ -43,33 +42,29 @@ type MirrorOptions struct { // Mirror mirrors the package contents to the given registry and git server. func Mirror(ctx context.Context, opt MirrorOptions) error { - err := pushImagesToRegistry(ctx, opt.Cluster, opt.PackagePaths, opt.Filter, opt.RegistryInfo, opt.NoImageChecksum, opt.Retries) + err := pushImagesToRegistry(ctx, opt.Cluster, opt.Package, opt.Filter, opt.RegistryInfo, opt.NoImageChecksum, opt.Retries) if err != nil { return err } - err = pushReposToRepository(ctx, opt.Cluster, opt.PackagePaths, opt.Filter, opt.GitInfo, opt.Retries) + err = pushReposToRepository(ctx, opt.Cluster, opt.Package, opt.Filter, opt.GitInfo, opt.Retries) if err != nil { return err } return nil } -func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, pkgPaths layout.PackagePaths, filter filters.ComponentFilterStrategy, regInfo types.RegistryInfo, noImgChecksum bool, retries int) error { +func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, p *Package, filter filters.ComponentFilterStrategy, regInfo types.RegistryInfo, noImgChecksum bool, retries int) error { logs.Warn.SetOutput(&message.DebugWriter{}) logs.Progress.SetOutput(&message.DebugWriter{}) - pkg, _, err := pkgPaths.ReadZarfYAML() + components, err := filter.Apply(p.pkg) if err != nil { return err } - components, err := filter.Apply(pkg) - if err != nil { - return err - } - pkg.Components = components + // pkg.Components = components images := map[transform.Image]v1.Image{} - for _, component := range pkg.Components { + for _, component := range components { for _, img := range component.Images { ref, err := transform.ParseImageRef(img) if err != nil { @@ -78,11 +73,11 @@ func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, pkgPaths layo if _, ok := images[ref]; ok { continue } - ociImage, err := utils.LoadOCIImage(pkgPaths.Images.Base, ref) + img, err := p.GetImage(ref) if err != nil { return err } - images[ref] = ociImage + images[ref] = img } } if len(images) == 0 { @@ -96,7 +91,7 @@ func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, pkgPaths layo transportWithProgressBar := helpers.NewTransport(transport, nil) pushOptions := []crane.Option{ - crane.WithPlatform(&v1.Platform{OS: "linux", Architecture: pkg.Build.Architecture}), + crane.WithPlatform(&v1.Platform{OS: "linux", Architecture: p.pkg.Build.Architecture}), crane.WithTransport(transportWithProgressBar), crane.WithAuth(authn.FromConfig(authn.AuthConfig{ Username: regInfo.PushUsername, @@ -171,20 +166,19 @@ func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, pkgPaths layo return nil } -func pushReposToRepository(ctx context.Context, c *cluster.Cluster, pkgPaths layout.PackagePaths, filter filters.ComponentFilterStrategy, gitInfo types.GitServerInfo, retries int) error { - pkg, _, err := pkgPaths.ReadZarfYAML() - if err != nil { - return err - } - components, err := filter.Apply(pkg) +func pushReposToRepository(ctx context.Context, c *cluster.Cluster, p *Package, filter filters.ComponentFilterStrategy, gitInfo types.GitServerInfo, retries int) error { + components, err := filter.Apply(p.pkg) if err != nil { return err } - pkg.Components = components - - for _, component := range pkg.Components { + for _, component := range components { for _, repoURL := range component.Repos { - repository, err := git.Open(pkgPaths.Components.Dirs[component.Name].Repos, repoURL) + reposPath, err := p.GetDirectory(component.Name, "repos") + if err != nil { + return err + } + defer os.RemoveAll(reposPath) + repository, err := git.Open(reposPath, repoURL) if err != nil { return err } diff --git a/src/internal/packager2/package.go b/src/internal/packager2/package.go new file mode 100644 index 0000000000..5d60bff3e0 --- /dev/null +++ b/src/internal/packager2/package.go @@ -0,0 +1,269 @@ +package packager2 + +import ( + "archive/tar" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/defenseunicorns/pkg/helpers/v2" + goyaml "github.com/goccy/go-yaml" + registryv1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/layout" + "github.com/mholt/archiver/v3" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/sigstore/cosign/v2/cmd/cosign/cli/options" + "github.com/sigstore/cosign/v2/cmd/cosign/cli/verify" + + "github.com/zarf-dev/zarf/src/api/v1alpha1" + "github.com/zarf-dev/zarf/src/config" + "github.com/zarf-dev/zarf/src/pkg/transform" + "github.com/zarf-dev/zarf/src/pkg/utils" +) + +const ( + ZarfYAML = "zarf.yaml" + Signature = "zarf.yaml.sig" + Checksums = "checksums.txt" + ImagesDir = "images" + ComponentsDir = "components" + SBOMDir = "zarf-sbom" + SBOMTar = "sboms.tar" + IndexJSON = "index.json" + OCILayout = "oci-layout" +) + +type Package struct { + dirPath string + pkg v1alpha1.ZarfPackage +} + +// LoadFromTar unpacks the give compressed package and loads it. +func LoadFromTar(ctx context.Context, tarPath, publicKeyPath string) (*Package, error) { + dirPath, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return nil, err + } + err = archiver.Walk(tarPath, func(f archiver.File) error { + if f.IsDir() { + return nil + } + header, ok := f.Header.(*tar.Header) + if !ok { + return fmt.Errorf("expected header to be *tar.Header but was %T", f.Header) + } + // If path has nested directories we want to create them. + dir := filepath.Dir(header.Name) + if dir != "." { + err := os.MkdirAll(filepath.Join(dirPath, dir), helpers.ReadExecuteAllWriteUser) + if err != nil { + return err + } + } + dst, err := os.Create(filepath.Join(dirPath, header.Name)) + if err != nil { + return err + } + defer dst.Close() + _, err = io.Copy(dst, f) + if err != nil { + return err + } + return nil + }) + if err != nil { + return nil, err + } + p, err := LoadFromDir(ctx, dirPath, publicKeyPath) + if err != nil { + return nil, err + } + return p, nil +} + +// LoadFromDir loads and validates a package from the given directory path. +func LoadFromDir(ctx context.Context, dirPath, publicKeyPath string) (*Package, error) { + b, err := os.ReadFile(filepath.Join(dirPath, ZarfYAML)) + if err != nil { + return nil, err + } + var pkg v1alpha1.ZarfPackage + err = goyaml.Unmarshal(b, &pkg) + if err != nil { + return nil, err + } + p := &Package{ + dirPath: dirPath, + pkg: pkg, + } + err = validatePackageIntegrity(p) + if err != nil { + return nil, err + } + err = validatePackageSignature(ctx, p, publicKeyPath) + if err != nil { + return nil, err + } + return p, nil +} + +func (p *Package) Cleanup() error { + err := os.RemoveAll(p.dirPath) + if err != nil { + return err + } + return nil +} + +// TODO (phillebaba): Make dirType an enum. +func (p *Package) GetDirectory(component string, dirType string) (string, error) { + dirPath, err := utils.MakeTempDir(config.CommonOptions.TempDirectory) + if err != nil { + return "", err + } + sourcePath := filepath.Join(p.dirPath, ComponentsDir, fmt.Sprintf("%s.tar", component)) + err = archiver.Extract(sourcePath, dirType, dirPath) + if err != nil { + return "", err + } + return dirPath, nil +} + +func (p *Package) GetImage(ref transform.Image) (registryv1.Image, error) { + // Use the manifest within the index.json to load the specific image we want + layoutPath := layout.Path(filepath.Join(p.dirPath, ImagesDir)) + imgIdx, err := layoutPath.ImageIndex() + if err != nil { + return nil, err + } + idxManifest, err := imgIdx.IndexManifest() + if err != nil { + return nil, err + } + + // Search through all the manifests within this package until we find the annotation that matches our ref + for _, manifest := range idxManifest.Manifests { + if manifest.Annotations[ocispec.AnnotationBaseImageName] == ref.Reference || + // A backwards compatibility shim for older Zarf versions that would leave docker.io off of image annotations + (manifest.Annotations[ocispec.AnnotationBaseImageName] == ref.Path+ref.TagOrDigest && ref.Host == "docker.io") { + // This is the image we are looking for, load it and then return + return layoutPath.Image(manifest.Digest) + } + } + return nil, fmt.Errorf("unable to find the image %s", ref.Reference) +} + +func validatePackageIntegrity(p *Package) error { + _, err := os.Stat(filepath.Join(p.dirPath, ZarfYAML)) + if err != nil { + return err + } + _, err = os.Stat(filepath.Join(p.dirPath, Checksums)) + if err != nil { + return err + } + err = helpers.SHAsMatch(filepath.Join(p.dirPath, Checksums), p.pkg.Metadata.AggregateChecksum) + if err != nil { + return err + } + + packageFiles := map[string]interface{}{} + err = filepath.Walk(p.dirPath, func(path string, info fs.FileInfo, err error) error { + if info.IsDir() { + return nil + } + packageFiles[path] = nil + return err + }) + if err != nil { + return err + } + // Remove files which are not in the checksums. + delete(packageFiles, filepath.Join(p.dirPath, ZarfYAML)) + delete(packageFiles, filepath.Join(p.dirPath, Checksums)) + delete(packageFiles, filepath.Join(p.dirPath, Signature)) + + b, err := os.ReadFile(filepath.Join(p.dirPath, Checksums)) + if err != nil { + return err + } + lines := strings.Split(string(b), "\n") + for _, line := range lines { + // If the line is empty (i.e. there is no checksum) simply skip it, this can result from a package with no images/components. + if line == "" { + continue + } + + split := strings.Split(line, " ") + if len(split) != 2 { + return fmt.Errorf("invalid checksum line: %s", line) + } + sha := split[0] + rel := split[1] + if sha == "" || rel == "" { + return fmt.Errorf("invalid checksum line: %s", line) + } + + // TODO (phillebaba): support partial + path := filepath.Join(p.dirPath, rel) + _, err := os.Stat(path) + if err != nil { + return err + } + err = helpers.SHAsMatch(path, sha) + if err != nil { + return err + } + + delete(packageFiles, path) + } + + if len(packageFiles) > 0 { + // TODO (phillebaba): Replace with maps.Keys after upgrading to Go 1.23. + filePaths := []string{} + for k := range packageFiles { + filePaths = append(filePaths, k) + } + return fmt.Errorf("package contains additional files not present in the checksum %s", strings.Join(filePaths, ", ")) + } + + return nil +} + +func validatePackageSignature(ctx context.Context, p *Package, publicKeyPath string) error { + signaturePath := filepath.Join(p.dirPath, Signature) + zarfYamlPath := filepath.Join(p.dirPath, ZarfYAML) + + sigExist := true + _, err := os.Stat(signaturePath) + if err != nil { + sigExist = false + } + if !sigExist && publicKeyPath == "" { + // Nobody was expecting a signature, so we can just return + return nil + } else if sigExist && publicKeyPath == "" { + return errors.New("package is signed but no key was provided") + } else if !sigExist && publicKeyPath != "" { + return errors.New("a key was provided but the package is not signed") + } + + keyOptions := options.KeyOpts{KeyRef: publicKeyPath} + cmd := &verify.VerifyBlobCmd{ + KeyOpts: keyOptions, + SigRef: signaturePath, + IgnoreSCT: true, + Offline: true, + IgnoreTlog: true, + } + err = cmd.Exec(ctx, zarfYamlPath) + if err != nil { + return fmt.Errorf("package signature did not match the provided key: %w", err) + } + return nil +}