Record info about container images and Docker Compose apps in bundle …

…manifests (#364)

* Disaggregate HTTP file downloads from OCI image downloads in bundle manifest file

* Rearrange downloads list's nested hierarchy in bundle manifest file

* Record Compose app names in bundle export manifest

* Summarize info about each Compose app in the bundle manifest

* Print info about Compose apps in `show-bun`

* Aggregate the lists of exports in `show-bun`

* Aggregate the lists of downloads in `stage show-bun`

* Add Compose app container images to list of downloaded images in manifest

* Include local files in packages into manifest's list of bind mounts

* Make `stage show-bun` show info about required pallets

CHANGELOG.md

@@ -7,9 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+### Added
+
+- (spec) Now the bundle manifest file's `exports` section lists information about the Docker Compose apps created by the bundle's package deployments.
+- (spec) Now the bundle manifest file's `downloads` section lists OCI images to be cached for Docker Compose apps; this is enabled by a breaking change in the layout of that section, described below.
+- (cli) Now `stage show-bun` prints information about required pallets in the "Includes" section.
+
 ### Changed
 
 - (Breaking change; cli) The verbs `rm` and `remove` have been deleted from all commands (e.g. `forklift pallet rm`), because the two-character verb `rm` doesn't line up nicely with the three-character verb `add`; `del` or `delete` should be used instead (e.g. `forklift pallet del`).
+- (Breaking change; spec) Now the bundle manifest file's `downloads` section lists different download types (e.g. HTTP file download vs. OCI image download) separately, instead of merging them all into one list.
+- (Breaking change; spec) Now the bundle manifest file's `exports` section lists different exports types (e.g. exported file vs. Docker compose app) separately, instead of merging them all into one list.
+
+### Removed
+
+- (cli) Now empty lists are omitted from bundle manifest files.
 
 ### Fixed
 

cmd/forklift/main.go

@@ -113,9 +113,9 @@ const (
 	bundleMinVersion = "v0.7.0"
 	// newBundleVersion is the Forklift version reported in new staged pallet bundles made by Forklift.
 	// Older versions of the Forklift tool cannot use such bundles.
-	newBundleVersion = "v0.8.0-alpha.2"
+	newBundleVersion = "v0.8.0-alpha.6"
 	// newStageStoreVersion is the Forklift version reported in a stage store initialized by Forklift.
-	// Older versions of the Forklift tool cannot use the state store.
+	// Older versions of the Forklift tool cannot use the stage store.
 	newStageStoreVersion = "v0.7.0"
 	// fallbackVersion is the version reported which the Forklift tool reports itself as if its actual
 	// version is unknown.

internal/app/forklift/bundles-models.go

@@ -56,14 +56,14 @@ type BundleManifest struct {
 	// files, while values are lists showing the chain of provenance of the respective files (with
 	// the deepest ancestor at the end of each list).
 	Imports map[string][]string `yaml:"imports,omitempty"`
-	// Downloads lists the URLs of files and OCI images downloaded for export by the bundle's
-	// deployments. Keys are names of the bundle's deployments which export downloaded files.
-	Downloads map[string][]string `yaml:"downloads,omitempty"`
 	// Deploys describes deployments provided by the bundle. Keys are names of deployments.
 	Deploys map[string]DeplDef `yaml:"deploys,omitempty"`
-	// Exports lists the target paths of file exports provided by the bundle's deployments. Keys are
-	// names of the bundle's deployments which provide file exports.
-	Exports map[string][]string `yaml:"exports,omitempty"`
+	// Downloads lists the downloadable paths of resources downloaded for creation and/or use of the
+	// bundle. Keys are the names of the bundle's deployments which include downloads.
+	Downloads map[string]BundleDeplDownloads `yaml:"downloads,omitempty"`
+	// Exports lists the exposed paths of resources created by the bundle's deployments. Keys are
+	// names of the bundle's deployments which provide resources.
+	Exports map[string]BundleDeplExports `yaml:"exports,omitempty"`
 }
 
 // BundlePallet describes a bundle's bundled pallet.
@@ -112,6 +112,7 @@ type BundleRepoInclusion struct {
 	Override BundleInclusionOverride `yaml:"override,omitempty"`
 }
 
+// BundleInclusionOverride describes a pallet used to override a required pallet.
 type BundleInclusionOverride struct {
 	// Path is the path of the override. This should be a filesystem path.
 	Path string `yaml:"path"`
@@ -121,3 +122,43 @@ type BundleInclusionOverride struct {
 	// Git commit, if the it's version-controlled with Git.
 	Clean bool `yaml:"clean"`
 }
+
+// BundleDeplDownloads lists the downloadable paths of resources which are downloaded for a
+// deployment, whether during creation of the bundle or during staging of the bundle.
+type BundleDeplDownloads struct {
+	// HTTPFile lists HTTP(S) URLs of files downloaded for export by the deployment.
+	HTTPFile []string `yaml:"http,omitempty"`
+	// OCIImage lists URLs of OCI images downloaded either for export by the deployment or for use in
+	// the deployment's Docker Compose app.
+	OCIImage []string `yaml:"oci-image,omitempty"`
+}
+
+// BundleDeplExports lists the exposed paths of resources which are provided by a deployment.
+type BundleDeplExports struct {
+	// File lists the filesystem target paths of files exported by the deployment.
+	File []string `yaml:"file,omitempty"`
+	// ComposeApp lists the name of the Docker Compose app exported by the deployment.
+	ComposeApp BundleDeplComposeApp `yaml:"compose-app,omitempty"`
+}
+
+// BundleDeplComposeApp lists information about a Docker Compose app provided by a deployment.
+type BundleDeplComposeApp struct {
+	// Name is the name of the Docker Compose app.
+	Name string `yaml:"name,omitempty"`
+	// Services lists the names of the services of the Docker Compose app.
+	Services []string `yaml:"services,omitempty"`
+	// Images lists the names of the container images used by services of the Docker Compose app.
+	Images []string `yaml:"images,omitempty"`
+	// CreatedBindMounts lists the names of the bind mounts created by the Docker Compose app.
+	CreatedBindMounts []string `yaml:"created-bind-mounts,omitempty"`
+	// RequiredBindMounts lists the names of the bind mounts required by the Docker Compose app.
+	RequiredBindMounts []string `yaml:"required-bind-mounts,omitempty"`
+	// CreatedVolumes lists the names of the volumes created by the Docker Compose app.
+	CreatedVolumes []string `yaml:"created-volumes,omitempty"`
+	// RequiredVolumes lists the names of the volumes required by the Docker Compose app.
+	RequiredVolumes []string `yaml:"required-volumes,omitempty"`
+	// CreatedNetworks lists the names of the networks created by the Docker Compose app.
+	CreatedNetworks []string `yaml:"created-networks,omitempty"`
+	// RequiredNetworks lists the names of the networks required by the Docker Compose app.
+	RequiredNetworks []string `yaml:"required-networks,omitempty"`
+}

internal/app/forklift/bundles.go

@@ -3,6 +3,7 @@ package forklift
 import (
 	"archive/tar"
 	"bytes"
+	"cmp"
 	"compress/gzip"
 	"fmt"
 	"io"
@@ -14,12 +15,14 @@ import (
 	"strings"
 
 	"github.com/bmatcuk/doublestar/v4"
+	dct "github.com/compose-spec/compose-go/v2/types"
 	"github.com/h2non/filetype"
 	ftt "github.com/h2non/filetype/types"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v3"
 
 	"github.com/PlanktoScope/forklift/pkg/core"
+	"github.com/PlanktoScope/forklift/pkg/structures"
 )
 
 // FSBundle
@@ -209,14 +212,19 @@ func (b *FSBundle) getBundledMergedPalletPath() string {
 
 func (b *FSBundle) AddResolvedDepl(depl *ResolvedDepl) (err error) {
 	b.Manifest.Deploys[depl.Name] = depl.Depl.Def
-	if b.Manifest.Downloads[depl.Name], err = depl.GetDownloadURLs(); err != nil {
+	downloads := BundleDeplDownloads{}
+	if downloads.HTTPFile, err = depl.GetHTTPFileDownloadURLs(); err != nil {
 		return errors.Wrapf(
-			err, "couldn't determine HTTP file downloads for export by deployment %s", depl.Depl.Name,
+			err, "couldn't determine HTTP file downloads for deployment %s", depl.Depl.Name,
 		)
 	}
-	if b.Manifest.Exports[depl.Name], err = depl.GetFileExportTargets(); err != nil {
-		return errors.Wrapf(err, "couldn't determine file exports of deployment %s", depl.Depl.Name)
+	if downloads.OCIImage, err = depl.GetOCIImageDownloadNames(); err != nil {
+		return errors.Wrapf(
+			err, "couldn't determine OCI image downloads for deployment %s", depl.Depl.Name,
+		)
 	}
+	b.Manifest.Downloads[depl.Name] = downloads
+
 	if err = CopyFS(depl.Pkg.FS, filepath.FromSlash(
 		path.Join(b.getPackagesPath(), depl.Def.Package),
 	)); err != nil {
@@ -225,9 +233,146 @@ func (b *FSBundle) AddResolvedDepl(depl *ResolvedDepl) (err error) {
 			depl.Pkg.Path(), depl.Depl.Name, depl.Pkg.FS.Path(),
 		)
 	}
+
+	exports := BundleDeplExports{}
+	if exports.File, err = depl.GetFileExportTargets(); err != nil {
+		return errors.Wrapf(err, "couldn't determine file exports of deployment %s", depl.Depl.Name)
+	}
+	definesComposeApp, err := depl.DefinesComposeApp()
+	if err != nil {
+		return errors.Wrapf(
+			err, "couldn't check deployment %s for a Compose app", depl.Depl.Name,
+		)
+	}
+	if definesComposeApp {
+		exports.ComposeApp, err = makeComposeAppSummary(depl, b.FS)
+		if err != nil {
+			return errors.Wrap(err, "couldn't make summary of Compose app definition")
+		}
+	}
+	b.Manifest.Exports[depl.Name] = exports
+
+	allOCIImages := make(structures.Set[string])
+	allOCIImages.Add(downloads.OCIImage...)
+	allOCIImages.Add(exports.ComposeApp.Images...)
+	downloads.OCIImage = slices.Sorted(allOCIImages.All())
+	b.Manifest.Downloads[depl.Name] = downloads
+
+	if downloads.Empty() {
+		delete(b.Manifest.Downloads, depl.Name)
+	}
+	if exports.Empty() {
+		delete(b.Manifest.Exports, depl.Name)
+	}
 	return nil
 }
 
+func makeComposeAppSummary(
+	depl *ResolvedDepl, bundleFS core.PathedFS,
+) (BundleDeplComposeApp, error) {
+	bundlePkg, err := core.LoadFSPkg(bundleFS, path.Join(packagesDirName, depl.Def.Package))
+	if err != nil {
+		return BundleDeplComposeApp{}, errors.Wrapf(
+			err, "couldn't load bundled package %s", depl.Pkg.Path(),
+		)
+	}
+	depl = &ResolvedDepl{
+		Depl:   depl.Depl,
+		PkgReq: depl.PkgReq,
+		Pkg:    bundlePkg,
+	}
+
+	appDef, err := depl.LoadComposeAppDefinition(true)
+	if err != nil {
+		return BundleDeplComposeApp{}, errors.Wrap(err, "couldn't load Compose app definition")
+	}
+
+	services := make(structures.Set[string])
+	images := make(structures.Set[string])
+	for _, service := range appDef.Services {
+		services.Add(service.Name)
+		images.Add(service.Image)
+	}
+
+	createdBindMounts, requiredBindMounts := makeComposeAppBindMountSummaries(appDef, bundleFS.Path())
+	createdVolumes, requiredVolumes := makeComposeAppVolumeSummaries(appDef)
+	createdNetworks, requiredNetworks := makeComposeAppNetworkSummaries(appDef)
+
+	app := BundleDeplComposeApp{
+		Name:               appDef.Name,
+		Services:           slices.Sorted(services.All()),
+		Images:             slices.Sorted(images.All()),
+		CreatedBindMounts:  slices.Sorted(createdBindMounts.All()),
+		RequiredBindMounts: slices.Sorted(requiredBindMounts.All()),
+		CreatedVolumes:     slices.Sorted(createdVolumes.All()),
+		RequiredVolumes:    slices.Sorted(requiredVolumes.All()),
+		CreatedNetworks:    slices.Sorted(createdNetworks.All()),
+		RequiredNetworks:   slices.Sorted(requiredNetworks.All()),
+	}
+	return app, nil
+}
+
+func makeComposeAppBindMountSummaries(
+	appDef *dct.Project, bundleRoot string,
+) (created structures.Set[string], required structures.Set[string]) {
+	created = make(structures.Set[string])
+	required = make(structures.Set[string])
+	for _, service := range appDef.Services {
+		for _, volume := range service.Volumes {
+			if volume.Type != "bind" {
+				continue
+			}
+			// If the path on the host is declared as a relative path, then it's supposed to be a path
+			// managed by Forklift, and its location will depend on where the bundle is. So we record it
+			// relative to the path of the bundle.
+			volume.Source = strings.TrimPrefix(volume.Source, bundleRoot+"/")
+			if volume.Bind != nil && !volume.Bind.CreateHostPath {
+				required.Add(volume.Source)
+				continue
+			}
+			created.Add(volume.Source)
+		}
+	}
+
+	return created.Difference(required), required
+}
+
+func makeComposeAppVolumeSummaries(
+	appDef *dct.Project,
+) (created structures.Set[string], required structures.Set[string]) {
+	created = make(structures.Set[string])
+	required = make(structures.Set[string])
+	for volumeName, volume := range appDef.Volumes {
+		if volume.External {
+			required.Add(cmp.Or(volume.Name, volumeName))
+			continue
+		}
+		created.Add(cmp.Or(volume.Name, volumeName))
+	}
+	return created, required
+}
+
+func makeComposeAppNetworkSummaries(
+	appDef *dct.Project,
+) (created structures.Set[string], required structures.Set[string]) {
+	created = make(structures.Set[string])
+	required = make(structures.Set[string])
+	for networkName, network := range appDef.Networks {
+		if network.External {
+			if networkName == "default" && network.Name == "none" {
+				// If the network is Docker's pre-made "none" network (which uses the null network driver),
+				// we ignore it for brevity since the intention is to suppress creating a network for the
+				// container.
+				continue
+			}
+			required.Add(cmp.Or(network.Name, networkName))
+			continue
+		}
+		created.Add(cmp.Or(network.Name, networkName))
+	}
+	return created, required
+}
+
 func (b *FSBundle) LoadDepl(name string) (Depl, error) {
 	depl, ok := b.Manifest.Deploys[name]
 	if !ok {
@@ -637,3 +782,27 @@ func (i *BundleInclusions) HasOverrides() bool {
 	}
 	return false
 }
+
+// BundleDownloads
+
+func (d BundleDeplDownloads) Empty() bool {
+	if len(d.HTTPFile) > 0 {
+		return false
+	}
+	if len(d.OCIImage) > 0 {
+		return false
+	}
+	return true
+}
+
+// BundleExports
+
+func (d BundleDeplExports) Empty() bool {
+	if len(d.File) > 0 {
+		return false
+	}
+	if d.ComposeApp.Name != "" {
+		return false
+	}
+	return true
+}

internal/app/forklift/cli/deployments-printing.go

@@ -53,15 +53,15 @@ func FprintResolvedDepl(
 	}
 	indent++
 
-	definesApp, err := resolved.DefinesApp()
+	definesApp, err := resolved.DefinesComposeApp()
 	if err != nil {
 		return errors.Wrap(err, "couldn't determine whether package deployment defines a Compose app")
 	}
 	if !definesApp {
 		return nil
 	}
 
-	appDef, err := loadAppDefinition(resolved)
+	appDef, err := resolved.LoadComposeAppDefinition(false)
 	if err != nil {
 		return errors.Wrap(err, "couldn't load Compose app definition")
 	}

internal/app/forklift/cli/images.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"maps"
 	"os"
+	"slices"
 
 	"github.com/pkg/errors"
 	"golang.org/x/sync/errgroup"
@@ -107,10 +109,9 @@ func ListRequiredImages(
 		return nil, err
 	}
 
-	orderedImages := make([]string, 0, len(resolved))
 	images := make(structures.Set[string])
 	for _, depl := range resolved {
-		definesApp, err := depl.DefinesApp()
+		definesApp, err := depl.DefinesComposeApp()
 		if err != nil {
 			return nil, errors.Wrapf(
 				err, "couldn't determine whether package deployment %s defines a Compose app", depl.Name,
@@ -120,18 +121,15 @@ func ListRequiredImages(
 			continue
 		}
 
-		appDef, err := loadAppDefinition(depl)
+		appDef, err := depl.LoadComposeAppDefinition(false)
 		if err != nil {
 			return nil, errors.Wrap(err, "couldn't load Compose app definition")
 		}
 		for _, service := range appDef.Services {
-			if !images.Has(service.Image) {
-				images.Add(service.Image)
-				orderedImages = append(orderedImages, service.Image)
-			}
+			images.Add(service.Image)
 		}
 	}
-	return orderedImages, nil
+	return slices.Sorted(maps.Keys(images)), nil
 }
 
 func downloadImagesParallel(indent int, images []string, platform string, dc *docker.Client) error {