Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Populate content/index.json manifest list #784

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Populate content/index.json manifest list #784

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cmd/soci/commands/create.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,10 +86,11 @@ var CreateCommand = cli.Command{
return err
}

ctx, blobStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
ctx, blobStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
if err != nil {
return err
}
defer done(ctx)

ps, err := internal.GetPlatforms(ctx, cliContext, srcImg, cs)
if err != nil {
Expand Down
5 changes: 4 additions & 1 deletion cmd/soci/commands/index/info.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,10 +53,13 @@ var infoCommand = cli.Command{
}
ctx, cancel := context.WithTimeout(context.Background(), cliContext.GlobalDuration("timeout"))
defer cancel()
ctx, store, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))

ctx, store, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
if err != nil {
return err
}
defer done(ctx)

reader, err := store.Fetch(ctx, v1.Descriptor{Digest: digest})
if err != nil {
return err
Expand Down
3 changes: 2 additions & 1 deletion cmd/soci/commands/index/rm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,11 @@ var rmCommand = cli.Command{
}

ctx := context.Background()
ctx, contentStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
ctx, contentStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
if err != nil {
return fmt.Errorf("cannot create local content store: %w", err)
}
defer done(ctx)

db, err := soci.NewDB(soci.ArtifactsDbPath())
if err != nil {
Expand Down
3 changes: 2 additions & 1 deletion cmd/soci/commands/push.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,11 @@ if they are available in the snapshotter's local content store.
}, nil
}

ctx, src, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
ctx, src, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
if err != nil {
return fmt.Errorf("cannot create local content store: %w", err)
}
defer done(ctx)

dst.Client = authClient
dst.PlainHTTP = cliContext.Bool("plain-http")
Expand Down
3 changes: 2 additions & 1 deletion cmd/soci/commands/rebuild_db.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,11 @@ var RebuildDBCommand = cli.Command{
if err != nil {
return err
}
ctx, blobStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
ctx, blobStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
if err != nil {
return err
}
defer done(ctx)

contentStorePath, err := store.GetContentStorePath(store.ContentStoreType(cliContext.GlobalString("content-store")))
if err != nil {
Expand Down
3 changes: 2 additions & 1 deletion cmd/soci/commands/ztoc/get-file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,11 @@ var getFileCommand = cli.Command{
}

func getZtoc(ctx context.Context, cliContext *cli.Context, d digest.Digest) (*ztoc.Ztoc, error) {
ctx, blobStore, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
ctx, blobStore, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
if err != nil {
return nil, err
}
defer done(ctx)

reader, err := blobStore.Fetch(ctx, v1.Descriptor{Digest: d})
if err != nil {
Expand Down
5 changes: 4 additions & 1 deletion cmd/soci/commands/ztoc/info.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,13 @@ var infoCommand = cli.Command{
}
ctx, cancel := context.WithTimeout(context.Background(), cliContext.GlobalDuration("timeout"))
defer cancel()
ctx, store, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))

ctx, store, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cliContext.GlobalString("content-store"))), store.WithNamespace(cliContext.GlobalString("namespace")))
if err != nil {
return err
}
defer done(ctx)

reader, err := store.Fetch(ctx, v1.Descriptor{Digest: digest})
if err != nil {
return err
Expand Down
4 changes: 3 additions & 1 deletion fs/fs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,10 +141,12 @@ func NewFilesystem(ctx context.Context, root string, cfg config.FSConfig, opts .
return docker.ConfigureDefaultRegistries(docker.WithPlainHTTP(docker.MatchLocalhost))(refspec.Hostname())
})
}
ctx, store, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cfg.ContentStoreConfig.Type)), store.WithNamespace(cfg.ContentStoreConfig.Namespace))
ctx, store, done, err := store.NewContentStore(ctx, store.WithType(store.ContentStoreType(cfg.ContentStoreConfig.Type)), store.WithNamespace(cfg.ContentStoreConfig.Namespace))
if err != nil {
return nil, fmt.Errorf("cannot create content store: %w", err)
}
// FIXME: This runs far too soon. Writing index.json for the soci content store should happen after each change to the store and/or when the snapshotter shuts down.
defer done(ctx)
Comment on lines +148 to +149
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All the other cases are single CLI commands which perform some operation then clean up after themselves. Here we are creating the filesystem for the snapshotter which will be used for an indefinite period. This done should happen when the filesystem is closed, probably when the snapshotter is exiting. An equivalent operation (which wouldn't really be "done") may also be needed after each new content creation. Either or both of these options would require putting the function into the filesystem struct (or somewhere else persistent) to be called later.

I am seeking input on the desirable path forward here.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This operation could be moved into Push but would then happen far more often than is reasonable for sequential operations.


var bgFetcher *bf.BackgroundFetcher

Expand Down
35 changes: 27 additions & 8 deletions soci/store/store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,20 +136,20 @@ type CleanupFunc func(context.Context) error

func nopCleanup(context.Context) error { return nil }

func NewContentStore(ctx context.Context, opts ...Option) (context.Context, Store, error) {
func NewContentStore(ctx context.Context, opts ...Option) (context.Context, Store, CleanupFunc, error) {
storeConfig := NewStoreConfig(opts...)

contentStoreType, err := CanonicalizeContentStoreType(ContentStoreType(storeConfig.Type))
if err != nil {
return ctx, nil, err
return ctx, nil, nopCleanup, err
}
switch contentStoreType {
case ContainerdContentStoreType:
return NewContainerdStore(ctx, storeConfig)
case SociContentStoreType:
return NewSociStore(ctx)
}
return ctx, nil, errors.New("unexpectedly reached end of NewContentStore")
return ctx, nil, nopCleanup, errors.New("unexpectedly reached end of NewContentStore")
}

// SociStore wraps oci.Store and adds or stubs additional functionality of the Store interface.
Expand All @@ -161,9 +161,18 @@ type SociStore struct {
var _ Store = (*SociStore)(nil)

// NewSociStore creates a sociStore.
func NewSociStore(ctx context.Context) (context.Context, *SociStore, error) {
func NewSociStore(ctx context.Context) (context.Context, *SociStore, CleanupFunc, error) {
store, err := oci.New(DefaultSociContentStorePath)
return ctx, &SociStore{store}, err
return ctx, &SociStore{store}, func(_ context.Context) error { return store.SaveIndex() }, err
}

// Push adds a new content item to the sociStore then tags it for indexing
func (s *SociStore) Push(ctx context.Context, expected ocispec.Descriptor, reader io.Reader) error {
if err := s.Store.Push(ctx, expected, reader); err != nil {
return err
}

return s.Tag(ctx, expected, expected.Digest.String())
}

// Label is a no-op for sociStore until sociStore and ArtifactsDb are better integrated.
Expand All @@ -181,6 +190,11 @@ func (s *SociStore) BatchOpen(ctx context.Context) (context.Context, CleanupFunc
return ctx, nopCleanup, nil
}

// Cleanup on a sociStore will update the content/index.json file
func (s *SociStore) Cleanup(ctx context.Context) error {
return s.SaveIndex()
}

type ContainerdStore struct {
config.ContentStoreConfig
client *containerd.Client
Expand All @@ -189,10 +203,10 @@ type ContainerdStore struct {
// assert that ContainerdStore implements Store
var _ Store = (*ContainerdStore)(nil)

func NewContainerdStore(ctx context.Context, storeConfig config.ContentStoreConfig) (context.Context, *ContainerdStore, error) {
func NewContainerdStore(ctx context.Context, storeConfig config.ContentStoreConfig) (context.Context, *ContainerdStore, CleanupFunc, error) {
client, err := containerd.New(config.DefaultImageServiceAddress)
if err != nil {
return ctx, nil, fmt.Errorf("could not connect to containerd socket for content store access: %w", err)
return ctx, nil, nopCleanup, fmt.Errorf("could not connect to containerd socket for content store access: %w", err)
}

ctx = namespaces.WithNamespace(ctx, storeConfig.Namespace)
Expand All @@ -203,7 +217,7 @@ func NewContainerdStore(ctx context.Context, storeConfig config.ContentStoreConf

containerdStore.ContentStoreConfig = storeConfig

return ctx, &containerdStore, nil
return ctx, &containerdStore, nopCleanup, nil
}

// Exists returns true iff the described content exists.
Expand Down Expand Up @@ -333,3 +347,8 @@ func (s *ContainerdStore) BatchOpen(ctx context.Context) (context.Context, Clean
}
return ctx, leaseDone, nil
}

// Cleanup is a no-op for ContainerdStore
func (s *ContainerdStore) Cleanup(ctx context.Context) error {
return nil
}