From 2717533260de0c6d37702bc6c312aed42ae8cbec Mon Sep 17 00:00:00 2001 From: Gergely Nagy Date: Thu, 24 Oct 2024 14:13:07 +0200 Subject: [PATCH 1/5] Shift cache files into memory cache --- pkg/apiserver/apiserver.go | 10 +- pkg/cache/memory/cache.go | 214 ++++++++++++++++++++++ pkg/cache/{ => memory}/cache_test.go | 0 pkg/cache/{ => memory}/draft.go | 0 pkg/cache/{ => memory}/package.go | 0 pkg/cache/{ => memory}/packagerevision.go | 0 pkg/cache/{ => memory}/repository.go | 0 pkg/cache/{ => memory}/util.go | 0 pkg/engine/engine.go | 2 +- pkg/engine/options.go | 4 +- 10 files changed, 222 insertions(+), 8 deletions(-) create mode 100644 pkg/cache/memory/cache.go rename pkg/cache/{ => memory}/cache_test.go (100%) rename pkg/cache/{ => memory}/draft.go (100%) rename pkg/cache/{ => memory}/package.go (100%) rename pkg/cache/{ => memory}/packagerevision.go (100%) rename pkg/cache/{ => memory}/repository.go (100%) rename pkg/cache/{ => memory}/util.go (100%) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 27732306..ba06b0c4 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -26,7 +26,7 @@ import ( configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" internalapi "github.com/nephio-project/porch/internal/api/porchinternal/v1alpha1" "github.com/nephio-project/porch/internal/kpt/fnruntime" - "github.com/nephio-project/porch/pkg/cache" + memorycache "github.com/nephio-project/porch/pkg/cache/memory" "github.com/nephio-project/porch/pkg/engine" "github.com/nephio-project/porch/pkg/meta" "github.com/nephio-project/porch/pkg/registry/porch" @@ -92,7 +92,7 @@ type Config struct { type PorchServer struct { GenericAPIServer *genericapiserver.GenericAPIServer coreClient client.WithWatch - cache *cache.Cache + cache *memorycache.Cache } type completedConfig struct { @@ -225,7 +225,7 @@ func (c completedConfig) New() (*PorchServer, error) { watcherMgr := engine.NewWatcherManager() - cache := cache.NewCache(c.ExtraConfig.CacheDirectory, c.ExtraConfig.RepoSyncFrequency, c.ExtraConfig.UseGitCaBundle, cache.CacheOptions{ + memoryCache := memorycache.NewCache(c.ExtraConfig.CacheDirectory, c.ExtraConfig.RepoSyncFrequency, c.ExtraConfig.UseGitCaBundle, memorycache.CacheOptions{ CredentialResolver: credentialResolver, UserInfoProvider: userInfoProvider, MetadataStore: metadataStore, @@ -246,7 +246,7 @@ func (c completedConfig) New() (*PorchServer, error) { } cad, err := engine.NewCaDEngine( - engine.WithCache(cache), + engine.WithCache(memoryCache), // The order of registering the function runtimes matters here. When // evaluating a function, the runtimes will be tried in the same // order as they are registered. @@ -271,7 +271,7 @@ func (c completedConfig) New() (*PorchServer, error) { s := &PorchServer{ GenericAPIServer: genericServer, coreClient: coreClient, - cache: cache, + cache: memoryCache, } // Install the groups. diff --git a/pkg/cache/memory/cache.go b/pkg/cache/memory/cache.go new file mode 100644 index 00000000..9e194cb7 --- /dev/null +++ b/pkg/cache/memory/cache.go @@ -0,0 +1,214 @@ +// Copyright 2022 The kpt and Nephio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cache + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "sync" + "time" + + kptoci "github.com/GoogleContainerTools/kpt/pkg/oci" + configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" + "github.com/nephio-project/porch/pkg/git" + "github.com/nephio-project/porch/pkg/meta" + "github.com/nephio-project/porch/pkg/oci" + "github.com/nephio-project/porch/pkg/repository" + "go.opentelemetry.io/otel/trace" + "k8s.io/apimachinery/pkg/watch" +) + +// Cache allows us to keep state for repositories, rather than querying them every time. +// +// Cache Structure: +// /git/ +// * Caches bare git repositories in directories named based on the repository address. +// /oci/ +// * Caches oci images with further hierarchy underneath +// * We Cache image layers in /oci/layers/ (this might be obsolete with the flattened Cache) +// * We Cache flattened tar files in /oci/ (so we don't need to pull to read resources) +// * We poll the repositories (every minute) and Cache the discovered images in memory. +type Cache struct { + mutex sync.Mutex + repositories map[string]*cachedRepository + cacheDir string + credentialResolver repository.CredentialResolver + userInfoProvider repository.UserInfoProvider + metadataStore meta.MetadataStore + repoSyncFrequency time.Duration + objectNotifier objectNotifier + useGitCaBundle bool +} + +type objectNotifier interface { + NotifyPackageRevisionChange(eventType watch.EventType, obj repository.PackageRevision, objMeta meta.PackageRevisionMeta) int +} + +type CacheOptions struct { + CredentialResolver repository.CredentialResolver + UserInfoProvider repository.UserInfoProvider + MetadataStore meta.MetadataStore + ObjectNotifier objectNotifier +} + +func NewCache(cacheDir string, repoSyncFrequency time.Duration, useGitCaBundle bool, opts CacheOptions) *Cache { + return &Cache{ + repositories: make(map[string]*cachedRepository), + cacheDir: cacheDir, + credentialResolver: opts.CredentialResolver, + userInfoProvider: opts.UserInfoProvider, + metadataStore: opts.MetadataStore, + objectNotifier: opts.ObjectNotifier, + repoSyncFrequency: repoSyncFrequency, + useGitCaBundle: useGitCaBundle, + } +} + +func getCacheKey(repositorySpec *configapi.Repository) (string, error) { + switch repositoryType := repositorySpec.Spec.Type; repositoryType { + case configapi.RepositoryTypeOCI: + ociSpec := repositorySpec.Spec.Oci + if ociSpec == nil { + return "", fmt.Errorf("oci not configured") + } + return "oci://" + ociSpec.Registry, nil + + case configapi.RepositoryTypeGit: + gitSpec := repositorySpec.Spec.Git + if gitSpec == nil { + return "", errors.New("git property is required") + } + if gitSpec.Repo == "" { + return "", errors.New("git.repo property is required") + } + return fmt.Sprintf("git://%s/%s@%s/%s", gitSpec.Repo, gitSpec.Directory, repositorySpec.Namespace, repositorySpec.Name), nil + + default: + return "", fmt.Errorf("repository type %q not supported", repositoryType) + } +} + +func (c *Cache) OpenRepository(ctx context.Context, repositorySpec *configapi.Repository) (*cachedRepository, error) { + ctx, span := tracer.Start(ctx, "Cache::OpenRepository", trace.WithAttributes()) + defer span.End() + + key, err := getCacheKey(repositorySpec) + if err != nil { + return nil, err + } + c.mutex.Lock() + defer c.mutex.Unlock() + cachedRepo := c.repositories[key] + + switch repositoryType := repositorySpec.Spec.Type; repositoryType { + case configapi.RepositoryTypeOCI: + ociSpec := repositorySpec.Spec.Oci + if cachedRepo == nil { + cacheDir := filepath.Join(c.cacheDir, "oci") + storage, err := kptoci.NewStorage(cacheDir) + if err != nil { + return nil, err + } + + r, err := oci.OpenRepository(repositorySpec.Name, repositorySpec.Namespace, repositorySpec.Spec.Content, ociSpec, repositorySpec.Spec.Deployment, storage) + if err != nil { + return nil, err + } + cachedRepo = newRepository(key, repositorySpec, r, c.objectNotifier, c.metadataStore, c.repoSyncFrequency) + c.repositories[key] = cachedRepo + } + return cachedRepo, nil + + case configapi.RepositoryTypeGit: + gitSpec := repositorySpec.Spec.Git + if !isPackageContent(repositorySpec.Spec.Content) { + return nil, fmt.Errorf("git repository supports Package content only; got %q", string(repositorySpec.Spec.Content)) + } + if cachedRepo == nil { + var mbs git.MainBranchStrategy + if gitSpec.CreateBranch { + mbs = git.CreateIfMissing + } else { + mbs = git.ErrorIfMissing + } + + r, err := git.OpenRepository(ctx, repositorySpec.Name, repositorySpec.Namespace, gitSpec, repositorySpec.Spec.Deployment, filepath.Join(c.cacheDir, "git"), git.GitRepositoryOptions{ + CredentialResolver: c.credentialResolver, + UserInfoProvider: c.userInfoProvider, + MainBranchStrategy: mbs, + UseGitCaBundle: c.useGitCaBundle, + }) + if err != nil { + return nil, err + } + + cachedRepo = newRepository(key, repositorySpec, r, c.objectNotifier, c.metadataStore, c.repoSyncFrequency) + c.repositories[key] = cachedRepo + } else { + // If there is an error from the background refresh goroutine, return it. + if err := cachedRepo.getRefreshError(); err != nil { + return nil, err + } + } + return cachedRepo, nil + + default: + return nil, fmt.Errorf("type %q not supported", repositoryType) + } +} + +func isPackageContent(content configapi.RepositoryContent) bool { + return content == configapi.RepositoryContentPackage +} + +func (c *Cache) CloseRepository(repositorySpec *configapi.Repository, allRepos []configapi.Repository) error { + key, err := getCacheKey(repositorySpec) + if err != nil { + return err + } + + // check if repositorySpec shares the underlying cached repo with another repository + for _, r := range allRepos { + if r.Name == repositorySpec.Name && r.Namespace == repositorySpec.Namespace { + continue + } + otherKey, err := getCacheKey(&r) + if err != nil { + return err + } + if otherKey == key { + // do not close cached repo if it is shared + return nil + } + } + + var repository *cachedRepository + { + c.mutex.Lock() + if r, ok := c.repositories[key]; ok { + delete(c.repositories, key) + repository = r + } + c.mutex.Unlock() + } + + if repository != nil { + return repository.Close() + } else { + return nil + } +} diff --git a/pkg/cache/cache_test.go b/pkg/cache/memory/cache_test.go similarity index 100% rename from pkg/cache/cache_test.go rename to pkg/cache/memory/cache_test.go diff --git a/pkg/cache/draft.go b/pkg/cache/memory/draft.go similarity index 100% rename from pkg/cache/draft.go rename to pkg/cache/memory/draft.go diff --git a/pkg/cache/package.go b/pkg/cache/memory/package.go similarity index 100% rename from pkg/cache/package.go rename to pkg/cache/memory/package.go diff --git a/pkg/cache/packagerevision.go b/pkg/cache/memory/packagerevision.go similarity index 100% rename from pkg/cache/packagerevision.go rename to pkg/cache/memory/packagerevision.go diff --git a/pkg/cache/repository.go b/pkg/cache/memory/repository.go similarity index 100% rename from pkg/cache/repository.go rename to pkg/cache/memory/repository.go diff --git a/pkg/cache/util.go b/pkg/cache/memory/util.go similarity index 100% rename from pkg/cache/util.go rename to pkg/cache/memory/util.go diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index b1898949..9c0eeebb 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -28,7 +28,7 @@ import ( configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" "github.com/nephio-project/porch/internal/kpt/builtins" "github.com/nephio-project/porch/internal/kpt/fnruntime" - "github.com/nephio-project/porch/pkg/cache" + cache "github.com/nephio-project/porch/pkg/cache/memory" "github.com/nephio-project/porch/pkg/kpt" kptfile "github.com/nephio-project/porch/pkg/kpt/api/kptfile/v1" "github.com/nephio-project/porch/pkg/kpt/fn" diff --git a/pkg/engine/options.go b/pkg/engine/options.go index 3c05f0eb..61163b37 100644 --- a/pkg/engine/options.go +++ b/pkg/engine/options.go @@ -18,7 +18,7 @@ import ( "fmt" "github.com/nephio-project/porch/internal/kpt/fnruntime" - "github.com/nephio-project/porch/pkg/cache" + memorycache "github.com/nephio-project/porch/pkg/cache/memory" "github.com/nephio-project/porch/pkg/kpt" "github.com/nephio-project/porch/pkg/kpt/fn" "github.com/nephio-project/porch/pkg/meta" @@ -37,7 +37,7 @@ func (f EngineOptionFunc) apply(engine *cadEngine) error { return f(engine) } -func WithCache(cache *cache.Cache) EngineOption { +func WithCache(cache *memorycache.Cache) EngineOption { return EngineOptionFunc(func(engine *cadEngine) error { engine.cache = cache return nil From fe9829032b69095ff407ab8170641917a3aebf0f Mon Sep 17 00:00:00 2001 From: Gergely Nagy Date: Thu, 24 Oct 2024 14:13:32 +0200 Subject: [PATCH 2/5] Shift background.go to memory cache as well --- pkg/registry/porch/background.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/registry/porch/background.go b/pkg/registry/porch/background.go index a901ee2e..0b39a648 100644 --- a/pkg/registry/porch/background.go +++ b/pkg/registry/porch/background.go @@ -20,7 +20,7 @@ import ( "time" configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" - "github.com/nephio-project/porch/pkg/cache" + memorycache "github.com/nephio-project/porch/pkg/cache/memory" "k8s.io/apimachinery/pkg/api/meta" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func RunBackground(ctx context.Context, coreClient client.WithWatch, cache *cache.Cache) { +func RunBackground(ctx context.Context, coreClient client.WithWatch, cache *memorycache.Cache) { b := background{ coreClient: coreClient, cache: cache, @@ -39,7 +39,7 @@ func RunBackground(ctx context.Context, coreClient client.WithWatch, cache *cach // background manages background tasks type background struct { coreClient client.WithWatch - cache *cache.Cache + cache *memorycache.Cache } const ( From 07ea983edc32e3a0a816f38285ddf287fdc607ad Mon Sep 17 00:00:00 2001 From: Gergely Nagy Date: Mon, 28 Oct 2024 19:09:48 +0100 Subject: [PATCH 3/5] Provide packageRevision cache as an interface --- pkg/cache/cache.go | 211 ++-------------------------- pkg/cache/memory/cache.go | 7 +- pkg/cache/memory/cache_test.go | 6 +- pkg/cache/memory/draft.go | 4 +- pkg/cache/memory/package.go | 8 +- pkg/cache/memory/packagerevision.go | 4 +- pkg/cache/memory/repository.go | 4 +- pkg/cache/memory/util.go | 2 +- 8 files changed, 40 insertions(+), 206 deletions(-) diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 9e194cb7..11b61595 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -1,214 +1,33 @@ -// Copyright 2022 The kpt and Nephio Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - package cache import ( "context" - "errors" - "fmt" - "path/filepath" - "sync" - "time" - kptoci "github.com/GoogleContainerTools/kpt/pkg/oci" configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" - "github.com/nephio-project/porch/pkg/git" - "github.com/nephio-project/porch/pkg/meta" - "github.com/nephio-project/porch/pkg/oci" "github.com/nephio-project/porch/pkg/repository" - "go.opentelemetry.io/otel/trace" - "k8s.io/apimachinery/pkg/watch" ) -// Cache allows us to keep state for repositories, rather than querying them every time. -// -// Cache Structure: -// /git/ -// * Caches bare git repositories in directories named based on the repository address. -// /oci/ -// * Caches oci images with further hierarchy underneath -// * We Cache image layers in /oci/layers/ (this might be obsolete with the flattened Cache) -// * We Cache flattened tar files in /oci/ (so we don't need to pull to read resources) -// * We poll the repositories (every minute) and Cache the discovered images in memory. -type Cache struct { - mutex sync.Mutex - repositories map[string]*cachedRepository - cacheDir string - credentialResolver repository.CredentialResolver - userInfoProvider repository.UserInfoProvider - metadataStore meta.MetadataStore - repoSyncFrequency time.Duration - objectNotifier objectNotifier - useGitCaBundle bool -} - -type objectNotifier interface { - NotifyPackageRevisionChange(eventType watch.EventType, obj repository.PackageRevision, objMeta meta.PackageRevisionMeta) int +type Cache interface { + OpenRepository(ctx context.Context, repositorySpec *configapi.Repository) (CachedRepository, error) + CloseRepository(repositorySpec *configapi.Repository, allRepos []configapi.Repository) error } -type CacheOptions struct { - CredentialResolver repository.CredentialResolver - UserInfoProvider repository.UserInfoProvider - MetadataStore meta.MetadataStore - ObjectNotifier objectNotifier +type CachedRepository interface { + repository.Repository + // TODO: Remove this once https://github.com/nephio-project/porch/pull/119 is merged + repository.FunctionRepository + RefreshCache(ctx context.Context) error } -func NewCache(cacheDir string, repoSyncFrequency time.Duration, useGitCaBundle bool, opts CacheOptions) *Cache { - return &Cache{ - repositories: make(map[string]*cachedRepository), - cacheDir: cacheDir, - credentialResolver: opts.CredentialResolver, - userInfoProvider: opts.UserInfoProvider, - metadataStore: opts.MetadataStore, - objectNotifier: opts.ObjectNotifier, - repoSyncFrequency: repoSyncFrequency, - useGitCaBundle: useGitCaBundle, - } +type CachedPackageRevision interface { + repository.PackageRevision } -func getCacheKey(repositorySpec *configapi.Repository) (string, error) { - switch repositoryType := repositorySpec.Spec.Type; repositoryType { - case configapi.RepositoryTypeOCI: - ociSpec := repositorySpec.Spec.Oci - if ociSpec == nil { - return "", fmt.Errorf("oci not configured") - } - return "oci://" + ociSpec.Registry, nil - - case configapi.RepositoryTypeGit: - gitSpec := repositorySpec.Spec.Git - if gitSpec == nil { - return "", errors.New("git property is required") - } - if gitSpec.Repo == "" { - return "", errors.New("git.repo property is required") - } - return fmt.Sprintf("git://%s/%s@%s/%s", gitSpec.Repo, gitSpec.Directory, repositorySpec.Namespace, repositorySpec.Name), nil - - default: - return "", fmt.Errorf("repository type %q not supported", repositoryType) - } +type CachedPackageDraft interface { + repository.PackageDraft } -func (c *Cache) OpenRepository(ctx context.Context, repositorySpec *configapi.Repository) (*cachedRepository, error) { - ctx, span := tracer.Start(ctx, "Cache::OpenRepository", trace.WithAttributes()) - defer span.End() - - key, err := getCacheKey(repositorySpec) - if err != nil { - return nil, err - } - c.mutex.Lock() - defer c.mutex.Unlock() - cachedRepo := c.repositories[key] - - switch repositoryType := repositorySpec.Spec.Type; repositoryType { - case configapi.RepositoryTypeOCI: - ociSpec := repositorySpec.Spec.Oci - if cachedRepo == nil { - cacheDir := filepath.Join(c.cacheDir, "oci") - storage, err := kptoci.NewStorage(cacheDir) - if err != nil { - return nil, err - } - - r, err := oci.OpenRepository(repositorySpec.Name, repositorySpec.Namespace, repositorySpec.Spec.Content, ociSpec, repositorySpec.Spec.Deployment, storage) - if err != nil { - return nil, err - } - cachedRepo = newRepository(key, repositorySpec, r, c.objectNotifier, c.metadataStore, c.repoSyncFrequency) - c.repositories[key] = cachedRepo - } - return cachedRepo, nil - - case configapi.RepositoryTypeGit: - gitSpec := repositorySpec.Spec.Git - if !isPackageContent(repositorySpec.Spec.Content) { - return nil, fmt.Errorf("git repository supports Package content only; got %q", string(repositorySpec.Spec.Content)) - } - if cachedRepo == nil { - var mbs git.MainBranchStrategy - if gitSpec.CreateBranch { - mbs = git.CreateIfMissing - } else { - mbs = git.ErrorIfMissing - } - - r, err := git.OpenRepository(ctx, repositorySpec.Name, repositorySpec.Namespace, gitSpec, repositorySpec.Spec.Deployment, filepath.Join(c.cacheDir, "git"), git.GitRepositoryOptions{ - CredentialResolver: c.credentialResolver, - UserInfoProvider: c.userInfoProvider, - MainBranchStrategy: mbs, - UseGitCaBundle: c.useGitCaBundle, - }) - if err != nil { - return nil, err - } - - cachedRepo = newRepository(key, repositorySpec, r, c.objectNotifier, c.metadataStore, c.repoSyncFrequency) - c.repositories[key] = cachedRepo - } else { - // If there is an error from the background refresh goroutine, return it. - if err := cachedRepo.getRefreshError(); err != nil { - return nil, err - } - } - return cachedRepo, nil - - default: - return nil, fmt.Errorf("type %q not supported", repositoryType) - } -} - -func isPackageContent(content configapi.RepositoryContent) bool { - return content == configapi.RepositoryContentPackage -} - -func (c *Cache) CloseRepository(repositorySpec *configapi.Repository, allRepos []configapi.Repository) error { - key, err := getCacheKey(repositorySpec) - if err != nil { - return err - } - - // check if repositorySpec shares the underlying cached repo with another repository - for _, r := range allRepos { - if r.Name == repositorySpec.Name && r.Namespace == repositorySpec.Namespace { - continue - } - otherKey, err := getCacheKey(&r) - if err != nil { - return err - } - if otherKey == key { - // do not close cached repo if it is shared - return nil - } - } - - var repository *cachedRepository - { - c.mutex.Lock() - if r, ok := c.repositories[key]; ok { - delete(c.repositories, key) - repository = r - } - c.mutex.Unlock() - } - - if repository != nil { - return repository.Close() - } else { - return nil - } +// Remove? +type CachedPackage interface { + repository.Package } diff --git a/pkg/cache/memory/cache.go b/pkg/cache/memory/cache.go index 9e194cb7..e1c8caeb 100644 --- a/pkg/cache/memory/cache.go +++ b/pkg/cache/memory/cache.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cache +package memory import ( "context" @@ -24,6 +24,7 @@ import ( kptoci "github.com/GoogleContainerTools/kpt/pkg/oci" configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" + "github.com/nephio-project/porch/pkg/cache" "github.com/nephio-project/porch/pkg/git" "github.com/nephio-project/porch/pkg/meta" "github.com/nephio-project/porch/pkg/oci" @@ -54,6 +55,8 @@ type Cache struct { useGitCaBundle bool } +var _ cache.Cache = &Cache{} + type objectNotifier interface { NotifyPackageRevisionChange(eventType watch.EventType, obj repository.PackageRevision, objMeta meta.PackageRevisionMeta) int } @@ -102,7 +105,7 @@ func getCacheKey(repositorySpec *configapi.Repository) (string, error) { } } -func (c *Cache) OpenRepository(ctx context.Context, repositorySpec *configapi.Repository) (*cachedRepository, error) { +func (c *Cache) OpenRepository(ctx context.Context, repositorySpec *configapi.Repository) (cache.CachedRepository, error) { ctx, span := tracer.Start(ctx, "Cache::OpenRepository", trace.WithAttributes()) defer span.End() diff --git a/pkg/cache/memory/cache_test.go b/pkg/cache/memory/cache_test.go index d663d502..3b35b4e1 100644 --- a/pkg/cache/memory/cache_test.go +++ b/pkg/cache/memory/cache_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cache +package memory import ( "context" @@ -25,6 +25,8 @@ import ( "github.com/google/go-cmp/cmp" api "github.com/nephio-project/porch/api/porch/v1alpha1" "github.com/nephio-project/porch/api/porchconfig/v1alpha1" + "github.com/nephio-project/porch/pkg/cache" + fakecache "github.com/nephio-project/porch/pkg/cache/fake" "github.com/nephio-project/porch/pkg/git" "github.com/nephio-project/porch/pkg/meta" @@ -126,7 +128,7 @@ func TestPublishedLatest(t *testing.T) { } } -func openRepositoryFromArchive(t *testing.T, ctx context.Context, testPath, name string) *cachedRepository { +func openRepositoryFromArchive(t *testing.T, ctx context.Context, testPath, name string) cache.CachedRepository { t.Helper() tempdir := t.TempDir() diff --git a/pkg/cache/memory/draft.go b/pkg/cache/memory/draft.go index cb8e5bca..aa36435d 100644 --- a/pkg/cache/memory/draft.go +++ b/pkg/cache/memory/draft.go @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cache +package memory import ( "context" + "github.com/nephio-project/porch/pkg/cache" "github.com/nephio-project/porch/pkg/repository" ) @@ -26,6 +27,7 @@ type cachedDraft struct { } var _ repository.PackageDraft = &cachedDraft{} +var _ cache.CachedPackageDraft = &cachedDraft{} func (cd *cachedDraft) Close(ctx context.Context) (repository.PackageRevision, error) { if closed, err := cd.PackageDraft.Close(ctx); err != nil { diff --git a/pkg/cache/memory/package.go b/pkg/cache/memory/package.go index 027e00a8..0699583b 100644 --- a/pkg/cache/memory/package.go +++ b/pkg/cache/memory/package.go @@ -12,9 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cache +package memory -import "github.com/nephio-project/porch/pkg/repository" +import ( + "github.com/nephio-project/porch/pkg/cache" + "github.com/nephio-project/porch/pkg/repository" +) // We take advantage of the cache having a global view of all the packages // in a repository and compute the latest package revision in the cache @@ -23,6 +26,7 @@ import "github.com/nephio-project/porch/pkg/repository" // between Git and OCI. var _ repository.Package = &cachedPackage{} +var _ cache.CachedPackage = &cachedPackage{} type cachedPackage struct { repository.Package diff --git a/pkg/cache/memory/packagerevision.go b/pkg/cache/memory/packagerevision.go index 17fce423..7f0b468d 100644 --- a/pkg/cache/memory/packagerevision.go +++ b/pkg/cache/memory/packagerevision.go @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cache +package memory import ( "context" "github.com/nephio-project/porch/api/porch/v1alpha1" + "github.com/nephio-project/porch/pkg/cache" "github.com/nephio-project/porch/pkg/repository" ) @@ -28,6 +29,7 @@ import ( // between Git and OCI. var _ repository.PackageRevision = &cachedPackageRevision{} +var _ cache.CachedPackageRevision = &cachedPackageRevision{} type cachedPackageRevision struct { repository.PackageRevision diff --git a/pkg/cache/memory/repository.go b/pkg/cache/memory/repository.go index 80649934..4e02573e 100644 --- a/pkg/cache/memory/repository.go +++ b/pkg/cache/memory/repository.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cache +package memory import ( "context" @@ -22,6 +22,7 @@ import ( "github.com/nephio-project/porch/api/porch/v1alpha1" configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" + "github.com/nephio-project/porch/pkg/cache" "github.com/nephio-project/porch/pkg/git" "github.com/nephio-project/porch/pkg/meta" "github.com/nephio-project/porch/pkg/repository" @@ -42,6 +43,7 @@ var tracer = otel.Tracer("cache") // between Git and OCI. var _ repository.Repository = &cachedRepository{} +var _ cache.CachedRepository = &cachedRepository{} var _ repository.FunctionRepository = &cachedRepository{} type cachedRepository struct { diff --git a/pkg/cache/memory/util.go b/pkg/cache/memory/util.go index a8d35f13..f703e59c 100644 --- a/pkg/cache/memory/util.go +++ b/pkg/cache/memory/util.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cache +package memory import ( "sort" From 0d17de6820c0517de7f1da9dc16799a17188f486 Mon Sep 17 00:00:00 2001 From: Gergely Nagy Date: Mon, 28 Oct 2024 19:38:18 +0100 Subject: [PATCH 4/5] Fix unit test reference --- pkg/cache/memory/cache_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cache/memory/cache_test.go b/pkg/cache/memory/cache_test.go index 3b35b4e1..2133fe8a 100644 --- a/pkg/cache/memory/cache_test.go +++ b/pkg/cache/memory/cache_test.go @@ -38,7 +38,7 @@ import ( func TestLatestPackages(t *testing.T) { ctx := context.Background() - testPath := filepath.Join("..", "git", "testdata") + testPath := filepath.Join("..", "..", "git", "testdata") cachedRepo := openRepositoryFromArchive(t, ctx, testPath, "nested") @@ -84,7 +84,7 @@ func TestLatestPackages(t *testing.T) { func TestPublishedLatest(t *testing.T) { ctx := context.Background() - testPath := filepath.Join("..", "git", "testdata") + testPath := filepath.Join("..", "..", "git", "testdata") cachedRepo := openRepositoryFromArchive(t, ctx, testPath, "nested") revisions, err := cachedRepo.ListPackageRevisions(ctx, repository.ListPackageRevisionFilter{ From 4e1752f30dc4da267fe88960dfce74392b1e0024 Mon Sep 17 00:00:00 2001 From: Gergely Nagy Date: Tue, 29 Oct 2024 11:38:42 +0100 Subject: [PATCH 5/5] Moved engine and background to use the interface instead of the memory implementation --- pkg/apiserver/apiserver.go | 3 ++- pkg/engine/engine.go | 4 ++-- pkg/engine/options.go | 4 ++-- pkg/registry/porch/background.go | 6 +++--- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index ba06b0c4..af6631bc 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -26,6 +26,7 @@ import ( configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" internalapi "github.com/nephio-project/porch/internal/api/porchinternal/v1alpha1" "github.com/nephio-project/porch/internal/kpt/fnruntime" + "github.com/nephio-project/porch/pkg/cache" memorycache "github.com/nephio-project/porch/pkg/cache/memory" "github.com/nephio-project/porch/pkg/engine" "github.com/nephio-project/porch/pkg/meta" @@ -92,7 +93,7 @@ type Config struct { type PorchServer struct { GenericAPIServer *genericapiserver.GenericAPIServer coreClient client.WithWatch - cache *memorycache.Cache + cache cache.Cache } type completedConfig struct { diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 9c0eeebb..ef42d6c7 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -28,7 +28,7 @@ import ( configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" "github.com/nephio-project/porch/internal/kpt/builtins" "github.com/nephio-project/porch/internal/kpt/fnruntime" - cache "github.com/nephio-project/porch/pkg/cache/memory" + cache "github.com/nephio-project/porch/pkg/cache" "github.com/nephio-project/porch/pkg/kpt" kptfile "github.com/nephio-project/porch/pkg/kpt/api/kptfile/v1" "github.com/nephio-project/porch/pkg/kpt/fn" @@ -160,7 +160,7 @@ func NewCaDEngine(opts ...EngineOption) (CaDEngine, error) { } type cadEngine struct { - cache *cache.Cache + cache cache.Cache // runnerOptionsResolver returns the RunnerOptions for function execution in the specified namespace. runnerOptionsResolver func(namespace string) fnruntime.RunnerOptions diff --git a/pkg/engine/options.go b/pkg/engine/options.go index 61163b37..18f33e93 100644 --- a/pkg/engine/options.go +++ b/pkg/engine/options.go @@ -18,7 +18,7 @@ import ( "fmt" "github.com/nephio-project/porch/internal/kpt/fnruntime" - memorycache "github.com/nephio-project/porch/pkg/cache/memory" + "github.com/nephio-project/porch/pkg/cache" "github.com/nephio-project/porch/pkg/kpt" "github.com/nephio-project/porch/pkg/kpt/fn" "github.com/nephio-project/porch/pkg/meta" @@ -37,7 +37,7 @@ func (f EngineOptionFunc) apply(engine *cadEngine) error { return f(engine) } -func WithCache(cache *memorycache.Cache) EngineOption { +func WithCache(cache cache.Cache) EngineOption { return EngineOptionFunc(func(engine *cadEngine) error { engine.cache = cache return nil diff --git a/pkg/registry/porch/background.go b/pkg/registry/porch/background.go index 0b39a648..688e9280 100644 --- a/pkg/registry/porch/background.go +++ b/pkg/registry/porch/background.go @@ -20,7 +20,7 @@ import ( "time" configapi "github.com/nephio-project/porch/api/porchconfig/v1alpha1" - memorycache "github.com/nephio-project/porch/pkg/cache/memory" + "github.com/nephio-project/porch/pkg/cache" "k8s.io/apimachinery/pkg/api/meta" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/watch" @@ -28,7 +28,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -func RunBackground(ctx context.Context, coreClient client.WithWatch, cache *memorycache.Cache) { +func RunBackground(ctx context.Context, coreClient client.WithWatch, cache cache.Cache) { b := background{ coreClient: coreClient, cache: cache, @@ -39,7 +39,7 @@ func RunBackground(ctx context.Context, coreClient client.WithWatch, cache *memo // background manages background tasks type background struct { coreClient client.WithWatch - cache *memorycache.Cache + cache cache.Cache } const (