Features/differential packages (#1650)

Add a differential flag when building zarf packages so that packages can
be created that are iterations of a prior package without needing to
fully download any packages resources that have stayed the same since
the last package build.

Some discussion for this capability can be found here: #1438 

#1426 

TODO: 
- [x] Write a test
- [x] find a better way to get the ref,tag,branch,etc from gitURLs when
considering differential package contents

---------

Co-authored-by: Wayne Starr <[email protected]>

docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md

@@ -17,6 +17,7 @@ zarf package create [DIRECTORY] [flags]
 
 ```
       --confirm                   Confirm package creation without prompting
+      --differential string       Build a package that only contains the differential changes from local resources and differing remote resources from the specified previously built package
   -h, --help                      help for create
   -k, --key string                Path to private key file for signing packages
       --key-pass string           Password to the private key file used for signing packages

docs/3-create-a-zarf-package/5-zarf-schema.md

@@ -406,6 +406,24 @@ Must be one of:
 </blockquote>
 </details>
 
+<details>
+<summary>
+<strong> <a name="build_differential"></a>differential *</strong>
+</summary>
+&nbsp;
+<blockquote>
+
+![Required](https://img.shields.io/badge/Required-red)
+
+**Description:** Whether this package was created with differential components
+
+|          |           |
+| -------- | --------- |
+| **Type** | `boolean` |
+
+</blockquote>
+</details>
+
 </blockquote>
 </details>
 

src/cmd/package.go

@@ -306,6 +306,7 @@ func bindCreateFlags() {
 	v.SetDefault(V_PKG_CREATE_MAX_PACKAGE_SIZE, 0)
 	v.SetDefault(V_PKG_CREATE_SIGNING_KEY, "")
 
+	createFlags.StringVar(&pkgConfig.CreateOpts.DifferentialData.DifferentialPackagePath, "differential", v.GetString(V_PKG_CREATE_DIFFERENTIAL), lang.CmdPackageCreateFlagDifferential)
 	createFlags.StringToStringVar(&pkgConfig.CreateOpts.SetVariables, "set", v.GetStringMapString(V_PKG_CREATE_SET), lang.CmdPackageCreateFlagSet)
 	createFlags.StringVarP(&pkgConfig.CreateOpts.OutputDirectory, "output-directory", "o", v.GetString(V_PKG_CREATE_OUTPUT_DIR), lang.CmdPackageCreateFlagOutputDirectory)
 	createFlags.BoolVarP(&pkgConfig.CreateOpts.ViewSBOM, "sbom", "s", v.GetBool(V_PKG_CREATE_SBOM), lang.CmdPackageCreateFlagSbom)

src/cmd/viper.go

@@ -58,6 +58,7 @@ const (
 	V_PKG_CREATE_MAX_PACKAGE_SIZE     = "package.create.max_package_size"
 	V_PKG_CREATE_SIGNING_KEY          = "package.create.signing_key"
 	V_PKG_CREATE_SIGNING_KEY_PASSWORD = "package.create.signing_key_password"
+	V_PKG_CREATE_DIFFERENTIAL         = "package.create.differential"
 
 	// Package deploy config keys
 	V_PKG_DEPLOY_SET        = "package.deploy.set"

src/config/lang/english.go

@@ -216,6 +216,7 @@ zarf init --git-push-password={PASSWORD} --git-push-username={USERNAME} --git-ur
 	CmdPackageCreateFlagMaxPackageSize     = "Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts. Use 0 to disable splitting."
 	CmdPackageCreateFlagSigningKey         = "Path to private key file for signing packages"
 	CmdPackageCreateFlagSigningKeyPassword = "Password to the private key file used for signing packages"
+	CmdPackageCreateFlagDifferential       = "Build a package that only contains the differential changes from local resources and differing remote resources from the specified previously built package"
 
 	CmdPackageDeployFlagConfirm                = "Confirms package deployment without prompting. ONLY use with packages you trust. Skips prompts to review SBOM, configure variables, select optional components and review potential breaking changes."
 	CmdPackageDeployFlagAdoptExistingResources = "Adopts any pre-existing K8s resources into the Helm charts managed by Zarf. ONLY use when you have existing deployments you want Zarf to takeover."
@@ -352,6 +353,11 @@ const (
 	AgentErrUnableTransform        = "unable to transform the provided request; see zarf http proxy logs for more details"
 )
 
+// src/internal/packager/create
+const (
+	PkgCreateErrDifferentialSameVersion = "unable to create a differential package with the same version as the package you are using as a reference; the package version must be incremented"
+)
+
 // src/internal/packager/validate.
 const (
 	PkgValidateTemplateDeprecation        = "Package template '%s' is using the deprecated syntax ###ZARF_PKG_VAR_%s###.  This will be removed in a future Zarf version.  Please update to ###ZARF_PKG_TMPL_%s###."

src/internal/packager/git/checkout.go

@@ -18,7 +18,7 @@ func (g *Git) CheckoutTag(tag string) error {
 	message.Debugf("git.CheckoutTag(%s)", tag)
 
 	options := &git.CheckoutOptions{
-		Branch: g.parseRef(tag),
+		Branch: ParseRef(tag),
 	}
 	return g.checkout(options)
 }

src/internal/packager/git/common.go

@@ -42,8 +42,8 @@ func NewWithSpinner(server types.GitServerInfo, spinner *message.Spinner) *Git {
 	}
 }
 
-// parseRef parses the provided ref into a ReferenceName if it's not a hash.
-func (g *Git) parseRef(r string) plumbing.ReferenceName {
+// ParseRef parses the provided ref into a ReferenceName if it's not a hash.
+func ParseRef(r string) plumbing.ReferenceName {
 	// If not a full ref, assume it's a tag at this point.
 	if !plumbing.IsHash(r) && !strings.HasPrefix(r, "refs/") {
 		r = fmt.Sprintf("refs/tags/%s", r)

src/internal/packager/git/pull.go

@@ -48,7 +48,7 @@ func (g *Git) Pull(gitURL, targetFolder string) error {
 
 	// Parse the ref from the git URL.
 	if refPlain != emptyRef {
-		ref = g.parseRef(refPlain)
+		ref = ParseRef(refPlain)
 	}
 
 	// Construct a path unique to this git repo

src/pkg/packager/common.go

@@ -108,11 +108,14 @@ func (p *Packager) GetPackageName() string {
 		suffix = "tar"
 	}
 
-	if p.cfg.Pkg.Metadata.Version == "" {
-		return fmt.Sprintf("%s%s-%s.%s", config.ZarfPackagePrefix, packageName, p.arch, suffix)
+	packageFileName := fmt.Sprintf("%s%s-%s", config.ZarfPackagePrefix, packageName, p.arch)
+	if p.cfg.Pkg.Build.Differential {
+		packageFileName = fmt.Sprintf("%s-%s-differential-%s", packageFileName, p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion, p.cfg.Pkg.Metadata.Version)
+	} else if p.cfg.Pkg.Metadata.Version != "" {
+		packageFileName = fmt.Sprintf("%s-%s", packageFileName, p.cfg.Pkg.Metadata.Version)
 	}
 
-	return fmt.Sprintf("%s%s-%s-%s.%s", config.ZarfPackagePrefix, packageName, p.arch, p.cfg.Pkg.Metadata.Version, suffix)
+	return fmt.Sprintf("%s.%s", packageFileName, suffix)
 }
 
 // ClearTempPaths removes the temp directory and any files within it.

src/pkg/packager/create.go

@@ -7,6 +7,7 @@ package packager
 import (
 	"crypto"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path"
@@ -25,15 +26,26 @@ import (
 	"github.com/defenseunicorns/zarf/src/internal/packager/sbom"
 	"github.com/defenseunicorns/zarf/src/internal/packager/validate"
 	"github.com/defenseunicorns/zarf/src/pkg/message"
+	"github.com/defenseunicorns/zarf/src/pkg/transform"
 	"github.com/defenseunicorns/zarf/src/pkg/utils"
 	"github.com/defenseunicorns/zarf/src/types"
+	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/mholt/archiver/v3"
 )
 
 // Create generates a Zarf package tarball for a given PackageConfig and optional base directory.
 func (p *Packager) Create(baseDir string) error {
 	var originalDir string
 
+	if err := p.readYaml(filepath.Join(baseDir, config.ZarfYAML), false); err != nil {
+		return fmt.Errorf("unable to read the zarf.yaml file: %s", err.Error())
+	}
+
+	// Load the images and repos from the 'reference' package
+	if err := p.loadDifferentialData(); err != nil {
+		return err
+	}
+
 	// Change the working directory if this run has an alternate base dir.
 	if baseDir != "" {
 		originalDir, _ = os.Getwd()
@@ -43,10 +55,6 @@ func (p *Packager) Create(baseDir string) error {
 		message.Note(fmt.Sprintf("Using build directory %s", baseDir))
 	}
 
-	if err := p.readYaml(config.ZarfYAML, false); err != nil {
-		return fmt.Errorf("unable to read the zarf.yaml file: %w", err)
-	}
-
 	if p.cfg.Pkg.Kind == "ZarfInitConfig" {
 		p.cfg.Pkg.Metadata.Version = config.CLIVersion
 		p.cfg.IsInitConfig = true
@@ -61,6 +69,23 @@ func (p *Packager) Create(baseDir string) error {
 		return fmt.Errorf("unable to fill values in template: %s", err.Error())
 	}
 
+	// Remove unnecessary repos and images if we are building a differential package
+	if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath != "" {
+		// Verify the package version of the package we're using as a 'reference' for the differential build is different than the package we're building
+		// If the package versions are the same return an error
+		if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == p.cfg.Pkg.Metadata.Version {
+			return errors.New(lang.PkgCreateErrDifferentialSameVersion)
+		}
+		if p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion == "" || p.cfg.Pkg.Metadata.Version == "" {
+			fmt.Errorf("unable to build differential package when either the differential package version or the referenced package version is not set")
+		}
+
+		// Handle any potential differential images/repos before going forward
+		if err := p.removeCopiesFromDifferentialPackage(); err != nil {
+			return err
+		}
+	}
+
 	// Create component paths and process extensions for each component.
 	for i, c := range p.cfg.Pkg.Components {
 		componentPath, err := p.createOrGetComponentPaths(c)
@@ -498,3 +523,115 @@ func generatePackageChecksums(basePath string) (string, error) {
 	// Calculate the checksum of the checksum file
 	return utils.GetSHA256OfFile(checksumsFilePath)
 }
+
+// loadDifferentialData extracts the zarf config of a designated 'reference' package that we are building a differential over and creates a list of all images and repos that are in the reference package
+func (p *Packager) loadDifferentialData() error {
+	if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" {
+		return nil
+	}
+
+	tmpDir, _ := utils.MakeTempDir("")
+	defer os.RemoveAll(tmpDir)
+
+	// Load the package spec of the package we're using as a 'reference' for the differential build
+	if utils.IsOCIURL(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath) {
+		if err := p.pullPackageLayers(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, tmpDir, []string{config.ZarfYAML}); err != nil {
+			return fmt.Errorf("unable to pull the differential zarf package spec: %s", err.Error())
+		}
+	} else {
+		if err := archiver.Extract(p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath, config.ZarfYAML, tmpDir); err != nil {
+			return fmt.Errorf("unable to extract the differential zarf package spec: %s", err.Error())
+		}
+	}
+
+	var differentialZarfConfig types.ZarfPackage
+	if err := utils.ReadYaml(filepath.Join(tmpDir, config.ZarfYAML), &differentialZarfConfig); err != nil {
+		return fmt.Errorf("unable to load the differential zarf package spec: %s", err.Error())
+	}
+
+	// Generate a map of all the images and repos that are included in the provided package
+	allIncludedImagesMap := map[string]bool{}
+	allIncludedReposMap := map[string]bool{}
+	for _, component := range differentialZarfConfig.Components {
+		for _, image := range component.Images {
+			allIncludedImagesMap[image] = true
+		}
+		for _, repo := range component.Repos {
+			allIncludedReposMap[repo] = true
+		}
+	}
+
+	p.cfg.CreateOpts.DifferentialData.DifferentialImages = allIncludedImagesMap
+	p.cfg.CreateOpts.DifferentialData.DifferentialRepos = allIncludedReposMap
+	p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version
+
+	return nil
+}
+
+// removeCopiesFromDifferentialPackage will remove any images and repos that are already included in the reference package from the new package
+func (p *Packager) removeCopiesFromDifferentialPackage() error {
+	// If a differential build was not requested, continue on as normal
+	if p.cfg.CreateOpts.DifferentialData.DifferentialPackagePath == "" {
+		return nil
+	}
+
+	// Loop through all of the components to determine if any of them are using already included images or repos
+	componentMap := make(map[int]types.ZarfComponent)
+	for idx, component := range p.cfg.Pkg.Components {
+		newImageList := []string{}
+		newRepoList := []string{}
+
+		// Generate a list of all unique images for this component
+		for _, img := range component.Images {
+			// If a image doesn't have a tag (or is a commonly reused tag), we will include this image in the differential package
+			imgRef, err := transform.ParseImageRef(img)
+			if err != nil {
+				return fmt.Errorf("unable to parse image ref %s: %s", img, err.Error())
+			}
+
+			// Only include new images or images that have a commonly overwritten tag
+			imgTag := imgRef.TagOrDigest
+			useImgAnyways := imgTag == ":latest" || imgTag == ":stable" || imgTag == ":nightly"
+			if useImgAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialImages[img] {
+				newImageList = append(newImageList, img)
+			} else {
+				message.Debugf("Image %s is already included in the differential package", img)
+			}
+		}
+
+		// Generate a list of all unique repos for this component
+		for _, repoURL := range component.Repos {
+			// Split the remote url and the zarf reference
+			_, refPlain, err := transform.GitTransformURLSplitRef(repoURL)
+			if err != nil {
+				return err
+			}
+
+			var ref plumbing.ReferenceName
+			// Parse the ref from the git URL.
+			if refPlain != "" {
+				ref = git.ParseRef(refPlain)
+			}
+
+			// Only include new repos or repos that were not referenced by a specific commit sha or tag
+			useRepoAnyways := ref == "" || (!ref.IsTag() && !plumbing.IsHash(refPlain))
+			if useRepoAnyways || !p.cfg.CreateOpts.DifferentialData.DifferentialRepos[repoURL] {
+				newRepoList = append(newRepoList, repoURL)
+			} else {
+				message.Debugf("Repo %s is already included in the differential package", repoURL)
+			}
+		}
+
+		// Update the component with the unique lists of repos and images
+		component.Images = newImageList
+		component.Repos = newRepoList
+		componentMap[idx] = component
+	}
+
+	// Update the package with the new component list
+	for idx, component := range componentMap {
+		p.cfg.Pkg.Components[idx] = component
+	}
+
+	return nil
+}

src/pkg/packager/inspect.go

@@ -7,17 +7,13 @@ package packager
 import (
 	"fmt"
 	"os"
-	"path/filepath"
-	"strings"
 
+	"github.com/defenseunicorns/zarf/src/config"
 	"github.com/defenseunicorns/zarf/src/internal/packager/sbom"
 	"github.com/defenseunicorns/zarf/src/pkg/message"
 	"github.com/defenseunicorns/zarf/src/pkg/utils"
-	"github.com/defenseunicorns/zarf/src/types"
 	"github.com/mholt/archiver/v3"
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pterm/pterm"
-	"oras.land/oras-go/v2/registry"
 )
 
 // Inspect list the contents of a package.
@@ -28,7 +24,17 @@ func (p *Packager) Inspect(includeSBOM bool, outputSBOM string, inspectPublicKey
 		// Download all the layers we need
 		pullSBOM := includeSBOM || outputSBOM != ""
 		pullZarfSig := inspectPublicKey != ""
-		if err := pullLayersForInspect(p.cfg.DeployOpts.PackagePath, p.tmp, pullSBOM, pullZarfSig); err != nil {
+
+		layersToPull := []string{config.ZarfYAML}
+		if pullSBOM {
+			layersToPull = append(layersToPull, "sboms.tar")
+		}
+		if pullZarfSig {
+			layersToPull = append(layersToPull, "zarf.yaml.sig")
+		}
+
+		message.Debugf("Pulling layers %v from %s", layersToPull, p.cfg.DeployOpts.PackagePath)
+		if err := p.pullPackageLayers(p.cfg.DeployOpts.PackagePath, p.tmp.Base, layersToPull); err != nil {
 			return fmt.Errorf("unable to pull layers for inspect: %w", err)
 		}
 		err := utils.ReadYaml(p.tmp.ZarfYaml, &p.cfg.Pkg)
@@ -95,56 +101,3 @@ func (p *Packager) Inspect(includeSBOM bool, outputSBOM string, inspectPublicKey
 
 	return nil
 }
-
-func pullLayersForInspect(packagePath string, tmpPath types.TempPaths, includeSBOM bool, includeSig bool) error {
-	spinner := message.NewProgressSpinner("Loading Zarf Package %s", packagePath)
-	ref, err := registry.ParseReference(strings.TrimPrefix(packagePath, "oci://"))
-	if err != nil {
-		return err
-	}
-
-	dst, err := utils.NewOrasRemote(ref)
-	if err != nil {
-		return err
-	}
-
-	// get the manifest
-	spinner.Updatef("Fetching the manifest for %s", packagePath)
-	layers, err := getLayers(dst)
-	if err != nil {
-		return err
-	}
-	spinner.Updatef("Loading Zarf Package %s", packagePath)
-	zarfYamlDesc := utils.Find(layers, func(d ocispec.Descriptor) bool {
-		return d.Annotations["org.opencontainers.image.title"] == "zarf.yaml"
-	})
-	err = pullLayer(dst, zarfYamlDesc, tmpPath.ZarfYaml)
-	if err != nil {
-		return err
-	}
-
-	if includeSBOM {
-		sbmomsTarDesc := utils.Find(layers, func(d ocispec.Descriptor) bool {
-			return d.Annotations["org.opencontainers.image.title"] == "sboms.tar"
-		})
-		err = pullLayer(dst, sbmomsTarDesc, tmpPath.SbomTar)
-		if err != nil {
-			return err
-		}
-		if err := archiver.Unarchive(tmpPath.SbomTar, filepath.Join(tmpPath.Base, "sboms")); err != nil {
-			return err
-		}
-	}
-
-	if includeSig {
-		sigTarDesc := utils.Find(layers, func(d ocispec.Descriptor) bool {
-			return d.Annotations["org.opencontainers.image.title"] == "zarf.yaml.sig"
-		})
-		err = pullLayer(dst, sigTarDesc, tmpPath.ZarfSig)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}