From 0726ecbb590d57c7644cfb5179e26f1d2699a39a Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Fri, 29 Nov 2024 16:21:05 +0100 Subject: [PATCH] multi: roll back change in request page size This fixes an issue with a new tapd (v0.5.0-rc1) attempting to sync with an old (v0.4.1) universe server. Because we now serve all the root nodes from a cache, we allow a much larger page size when requesting the root nodes. But old universe servers will not yet allow that higher request limit, resulting in the following error: [/universerpc.Universe/AssetLeafKeys]: invalid request limit So we need to use the old limit when _requesting_ pages. And once most of the universe servers have been updated (e.g. v0.5.1), we can then also use the larger limit on requests. --- cmd/tapcli/universe.go | 4 ++-- itest/assertions.go | 4 ++-- sample-tapd.conf | 4 ++-- tapdb/multiverse.go | 8 ++++---- tapdb/multiverse_cache.go | 2 +- tapdb/universe.go | 2 +- universe/interface.go | 7 +++++++ universe/syncer.go | 2 +- 8 files changed, 20 insertions(+), 13 deletions(-) diff --git a/cmd/tapcli/universe.go b/cmd/tapcli/universe.go index 579afb2c2..cb44d45c6 100644 --- a/cmd/tapcli/universe.go +++ b/cmd/tapcli/universe.go @@ -321,7 +321,7 @@ func universeKeys(ctx *cli.Context) error { ctxc, &unirpc.AssetLeafKeysRequest{ Id: universeID, Offset: int32(offset), - Limit: universe.MaxPageSize, + Limit: universe.RequestPageSize, }, ) @@ -336,7 +336,7 @@ func universeKeys(ctx *cli.Context) error { assetKeys.AssetKeys = append( assetKeys.AssetKeys, tempKeys.AssetKeys..., ) - offset += universe.MaxPageSize + offset += universe.RequestPageSize } printRespJSON(assetKeys) diff --git a/itest/assertions.go b/itest/assertions.go index 2c0abdd40..5638bbad8 100644 --- a/itest/assertions.go +++ b/itest/assertions.go @@ -1598,10 +1598,10 @@ func AssertUniverseRootsEqual(a, b *unirpc.AssetRootResponse) bool { func AssertUniverseStateEqual(t *testing.T, a, b unirpc.UniverseClient) bool { ctxb := context.Background() - rootsA, err := assetRoots(ctxb, a, universe.MaxPageSize/100) + rootsA, err := assetRoots(ctxb, a, universe.RequestPageSize/100) require.NoError(t, err) - rootsB, err := assetRoots(ctxb, b, universe.MaxPageSize/100) + rootsB, err := assetRoots(ctxb, b, universe.RequestPageSize/100) require.NoError(t, err) return AssertUniverseRootsEqual(rootsA, rootsB) diff --git a/sample-tapd.conf b/sample-tapd.conf index a052cb796..25b63ec8e 100644 --- a/sample-tapd.conf +++ b/sample-tapd.conf @@ -328,8 +328,8 @@ ; universe.multiverse-caches.syncer-cache-pre-alloc-size=100000 ; The size of the root node page cache for all requests that aren't served by -; the syncer cache. (default: 327680) -; universe.multiverse-caches.root-node-page-cache-size=327680 +; the syncer cache. (default: 10240) +; universe.multiverse-caches.root-node-page-cache-size=10240 [address] diff --git a/tapdb/multiverse.go b/tapdb/multiverse.go index e6c65ff66..5570305b0 100644 --- a/tapdb/multiverse.go +++ b/tapdb/multiverse.go @@ -446,7 +446,7 @@ func (b *MultiverseStore) RootNodes(ctx context.Context, NumOffset: q.Offset, NumLimit: func() int32 { if q.Limit == 0 { - return universe.MaxPageSize + return universe.RequestPageSize } return q.Limit @@ -573,7 +573,7 @@ func (b *MultiverseStore) fillSyncerCache(ctx context.Context) error { params := sqlc.UniverseRootsParams{ SortDirection: sqlInt16(universe.SortAscending), NumOffset: 0, - NumLimit: universe.MaxPageSize, + NumLimit: universe.RequestPageSize, } allRoots := make( @@ -586,9 +586,9 @@ func (b *MultiverseStore) fillSyncerCache(ctx context.Context) error { } allRoots = append(allRoots, newRoots...) - params.NumOffset += universe.MaxPageSize + params.NumOffset += universe.RequestPageSize - if len(newRoots) < universe.MaxPageSize { + if len(newRoots) < universe.RequestPageSize { break } } diff --git a/tapdb/multiverse_cache.go b/tapdb/multiverse_cache.go index 60bdd9550..de523f8b1 100644 --- a/tapdb/multiverse_cache.go +++ b/tapdb/multiverse_cache.go @@ -63,7 +63,7 @@ func DefaultMultiverseCacheConfig() MultiverseCacheConfig { LeavesPerUniverse: 50, SyncerCacheEnabled: false, SyncerCachePreAllocSize: 100_000, - RootNodePageCacheSize: 20 * universe.MaxPageSize, + RootNodePageCacheSize: 20 * universe.RequestPageSize, } } diff --git a/tapdb/universe.go b/tapdb/universe.go index ee263125a..11e174db7 100644 --- a/tapdb/universe.go +++ b/tapdb/universe.go @@ -723,7 +723,7 @@ func mintingKeys(ctx context.Context, dbTx BaseUniverseStore, NumOffset: q.Offset, NumLimit: func() int32 { if q.Limit == 0 { - return universe.MaxPageSize + return universe.RequestPageSize } return q.Limit diff --git a/universe/interface.go b/universe/interface.go index 8192078ec..436563ca3 100644 --- a/universe/interface.go +++ b/universe/interface.go @@ -39,6 +39,13 @@ const ( // MaxPageSize is the maximum page size that can be used when querying // for asset roots and leaves. MaxPageSize = 16384 + + // RequestPageSize is the default page size that should be used when + // querying for asset roots and leaves. + // + // TODO(guggero): Bump this to the value of MaxPageSize once the + // universe servers have been updated to v0.5.0-rc1 or later. + RequestPageSize = 512 ) // IdentifierKey is the compact representation of a universe identifier that can diff --git a/universe/syncer.go b/universe/syncer.go index 104b24890..decc14c84 100644 --- a/universe/syncer.go +++ b/universe/syncer.go @@ -16,7 +16,7 @@ import ( ) const ( - defaultPageSize = int32(MaxPageSize) + defaultPageSize = int32(RequestPageSize) ) var (