Skip to content

Commit

Permalink
refactor: layout
Browse files Browse the repository at this point in the history 
Signed-off-by: Philip Laine <[email protected]>
  • Loading branch information
@phillebaba
phillebaba committed Sep 23, 2024
1 parent 7ac8297 commit ac04223
Show file tree
Hide file tree
Showing 5 changed files with 300 additions and 119 deletions.
6 changes: 3 additions & 3 deletions src/cmd/package.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,14 +153,14 @@ var packageMirrorCmd = &cobra.Command{
SkipSignatureValidation: pkgConfig.PkgOpts.SkipSignatureValidation,
Filter: filter,
}
pkgPaths, err := packager2.LoadPackage(cmd.Context(), loadOpt)
p, err := packager2.LoadPackage(cmd.Context(), loadOpt)
if err != nil {
return err
}
defer os.RemoveAll(pkgPaths.Base)
defer p.Cleanup()
mirrorOpt := packager2.MirrorOptions{
Cluster: c,
PackagePaths: *pkgPaths,
Package: p,
Filter: filter,
RegistryInfo: pkgConfig.InitOpts.RegistryInfo,
GitInfo: pkgConfig.InitOpts.GitServer,
Expand Down
88 changes: 4 additions & 84 deletions src/internal/packager2/load.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,8 @@
package packager2

import (
"archive/tar"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/url"
Expand All @@ -17,12 +15,9 @@ import (
"strings"

"github.com/defenseunicorns/pkg/helpers/v2"
"github.com/mholt/archiver/v3"

"github.com/zarf-dev/zarf/src/config"
"github.com/zarf-dev/zarf/src/pkg/layout"
"github.com/zarf-dev/zarf/src/pkg/packager/filters"
"github.com/zarf-dev/zarf/src/pkg/packager/sources"
"github.com/zarf-dev/zarf/src/pkg/utils"
"github.com/zarf-dev/zarf/src/types"
)
Expand All @@ -37,7 +32,7 @@ type LoadOptions struct {
}

// LoadPackage optionally fetches and loads the package from the given source.
func LoadPackage(ctx context.Context, opt LoadOptions) (*layout.PackagePaths, error) {
func LoadPackage(ctx context.Context, opt LoadOptions) (*Package, error) {
srcType, err := identifySource(opt.Source)
if err != nil {
return nil, err
Expand Down Expand Up @@ -79,87 +74,12 @@ func LoadPackage(ctx context.Context, opt LoadOptions) (*layout.PackagePaths, er
}
}

// Extract the package
packageDir, err := utils.MakeTempDir(config.CommonOptions.TempDirectory)
fmt.Println(isPartial)
p, err := LoadFromTar(ctx, tarPath, opt.PublicKeyPath)
if err != nil {
return nil, err
}
pathsExtracted := []string{}
err = archiver.Walk(tarPath, func(f archiver.File) error {
if f.IsDir() {
return nil
}
header, ok := f.Header.(*tar.Header)
if !ok {
return fmt.Errorf("expected header to be *tar.Header but was %T", f.Header)
}
// If path has nested directories we want to create them.
dir := filepath.Dir(header.Name)
if dir != "." {
err := os.MkdirAll(filepath.Join(packageDir, dir), helpers.ReadExecuteAllWriteUser)
if err != nil {
return err
}
}
dst, err := os.Create(filepath.Join(packageDir, header.Name))
if err != nil {
return err
}
defer dst.Close()
_, err = io.Copy(dst, f)
if err != nil {
return err
}
pathsExtracted = append(pathsExtracted, header.Name)
return nil
})
if err != nil {
return nil, err
}

// Load the package paths
pkgPaths := layout.New(packageDir)
pkgPaths.SetFromPaths(pathsExtracted)
pkg, _, err := pkgPaths.ReadZarfYAML()
if err != nil {
return nil, err
}
// TODO: Filter is not persistently applied.
pkg.Components, err = opt.Filter.Apply(pkg)
if err != nil {
return nil, err
}
if err := pkgPaths.MigrateLegacy(); err != nil {
return nil, err
}
if !pkgPaths.IsLegacyLayout() {
if err := sources.ValidatePackageIntegrity(pkgPaths, pkg.Metadata.AggregateChecksum, isPartial); err != nil {
return nil, err
}
if opt.SkipSignatureValidation {
if err := sources.ValidatePackageSignature(ctx, pkgPaths, opt.PublicKeyPath); err != nil {
return nil, err
}
}
}
for _, component := range pkg.Components {
if err := pkgPaths.Components.Unarchive(component); err != nil {
if errors.Is(err, layout.ErrNotLoaded) {
_, err := pkgPaths.Components.Create(component)
if err != nil {
return nil, err
}
} else {
return nil, err
}
}
}
if pkgPaths.SBOMs.Path != "" {
if err := pkgPaths.SBOMs.Unarchive(); err != nil {
return nil, err
}
}
return pkgPaths, nil
return p, nil
}

func identifySource(src string) (string, error) {
Expand Down
10 changes: 4 additions & 6 deletions src/internal/packager2/load_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,12 @@ func TestLoadPackage(t *testing.T) {
SkipSignatureValidation: false,
Filter: filters.Empty(),
}
pkgPaths, err := LoadPackage(ctx, opt)
p, err := LoadPackage(ctx, opt)
require.NoError(t, err)

pkg, _, err := pkgPaths.ReadZarfYAML()
require.NoError(t, err)
require.Equal(t, "test", pkg.Metadata.Name)
require.Equal(t, "0.0.1", pkg.Metadata.Version)
require.Len(t, pkg.Components, 1)
require.Equal(t, "test", p.pkg.Metadata.Name)
require.Equal(t, "0.0.1", p.pkg.Metadata.Version)
require.Len(t, p.pkg.Components, 1)
}

opt := LoadOptions{
Expand Down
46 changes: 20 additions & 26 deletions src/internal/packager2/mirror.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"net/http"
"os"
"time"

"github.com/avast/retry-go/v4"
Expand All @@ -22,18 +23,16 @@ import (
"github.com/zarf-dev/zarf/src/internal/git"
"github.com/zarf-dev/zarf/src/internal/gitea"
"github.com/zarf-dev/zarf/src/pkg/cluster"
"github.com/zarf-dev/zarf/src/pkg/layout"
"github.com/zarf-dev/zarf/src/pkg/message"
"github.com/zarf-dev/zarf/src/pkg/packager/filters"
"github.com/zarf-dev/zarf/src/pkg/transform"
"github.com/zarf-dev/zarf/src/pkg/utils"
"github.com/zarf-dev/zarf/src/types"
)

// MirrorOptions are the options for Mirror.
type MirrorOptions struct {
Cluster *cluster.Cluster
PackagePaths layout.PackagePaths
Package *Package
Filter filters.ComponentFilterStrategy
RegistryInfo types.RegistryInfo
GitInfo types.GitServerInfo
Expand All @@ -43,33 +42,29 @@ type MirrorOptions struct {

// Mirror mirrors the package contents to the given registry and git server.
func Mirror(ctx context.Context, opt MirrorOptions) error {
err := pushImagesToRegistry(ctx, opt.Cluster, opt.PackagePaths, opt.Filter, opt.RegistryInfo, opt.NoImageChecksum, opt.Retries)
err := pushImagesToRegistry(ctx, opt.Cluster, opt.Package, opt.Filter, opt.RegistryInfo, opt.NoImageChecksum, opt.Retries)
if err != nil {
return err
}
err = pushReposToRepository(ctx, opt.Cluster, opt.PackagePaths, opt.Filter, opt.GitInfo, opt.Retries)
err = pushReposToRepository(ctx, opt.Cluster, opt.Package, opt.Filter, opt.GitInfo, opt.Retries)
if err != nil {
return err
}
return nil
}

func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, pkgPaths layout.PackagePaths, filter filters.ComponentFilterStrategy, regInfo types.RegistryInfo, noImgChecksum bool, retries int) error {
func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, p *Package, filter filters.ComponentFilterStrategy, regInfo types.RegistryInfo, noImgChecksum bool, retries int) error {
logs.Warn.SetOutput(&message.DebugWriter{})
logs.Progress.SetOutput(&message.DebugWriter{})

pkg, _, err := pkgPaths.ReadZarfYAML()
components, err := filter.Apply(p.pkg)
if err != nil {
return err
}
components, err := filter.Apply(pkg)
if err != nil {
return err
}
pkg.Components = components
// pkg.Components = components

images := map[transform.Image]v1.Image{}
for _, component := range pkg.Components {
for _, component := range components {
for _, img := range component.Images {
ref, err := transform.ParseImageRef(img)
if err != nil {
Expand All @@ -78,11 +73,11 @@ func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, pkgPaths layo
if _, ok := images[ref]; ok {
continue
}
ociImage, err := utils.LoadOCIImage(pkgPaths.Images.Base, ref)
img, err := p.GetImage(ref)
if err != nil {
return err
}
images[ref] = ociImage
images[ref] = img
}
}
if len(images) == 0 {
Expand All @@ -96,7 +91,7 @@ func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, pkgPaths layo
transportWithProgressBar := helpers.NewTransport(transport, nil)

pushOptions := []crane.Option{
crane.WithPlatform(&v1.Platform{OS: "linux", Architecture: pkg.Build.Architecture}),
crane.WithPlatform(&v1.Platform{OS: "linux", Architecture: p.pkg.Build.Architecture}),
crane.WithTransport(transportWithProgressBar),
crane.WithAuth(authn.FromConfig(authn.AuthConfig{
Username: regInfo.PushUsername,
Expand Down Expand Up @@ -171,20 +166,19 @@ func pushImagesToRegistry(ctx context.Context, c *cluster.Cluster, pkgPaths layo
return nil
}

func pushReposToRepository(ctx context.Context, c *cluster.Cluster, pkgPaths layout.PackagePaths, filter filters.ComponentFilterStrategy, gitInfo types.GitServerInfo, retries int) error {
pkg, _, err := pkgPaths.ReadZarfYAML()
if err != nil {
return err
}
components, err := filter.Apply(pkg)
func pushReposToRepository(ctx context.Context, c *cluster.Cluster, p *Package, filter filters.ComponentFilterStrategy, gitInfo types.GitServerInfo, retries int) error {
components, err := filter.Apply(p.pkg)
if err != nil {
return err
}
pkg.Components = components

for _, component := range pkg.Components {
for _, component := range components {
for _, repoURL := range component.Repos {
repository, err := git.Open(pkgPaths.Components.Dirs[component.Name].Repos, repoURL)
reposPath, err := p.GetDirectory(component.Name, "repos")
if err != nil {
return err
}
defer os.RemoveAll(reposPath)
repository, err := git.Open(reposPath, repoURL)
if err != nil {
return err
}
Expand Down
Loading

0 comments on commit ac04223

Please sign in to comment.