diff --git a/api/layout/example_test.go b/api/layout/example_test.go index 87a90d49..db824a6e 100644 --- a/api/layout/example_test.go +++ b/api/layout/example_test.go @@ -28,19 +28,19 @@ func ExampleNodeCoordsToTileAddress() { } func ExampleTilePath() { - tilePath := layout.TilePath(0, 1234067, 315921160) + tilePath := layout.TilePath(0, 1234067, 8) fmt.Printf("tile path: %s", tilePath) // Output: tile path: tile/0/x001/x234/067.p/8 } func ExampleEntriesPath() { - entriesPath := layout.EntriesPath(1234067, 315921160) + entriesPath := layout.EntriesPath(1234067, 8) fmt.Printf("entries path: %s", entriesPath) // Output: entries path: tile/entries/x001/x234/067.p/8 } -func ExampleParseTileLevelIndexWidth() { - level, index, width, _ := layout.ParseTileLevelIndexWidth("0", "x001/x234/067.p/8") +func ExampleParseTileLevelIndexPartial() { + level, index, width, _ := layout.ParseTileLevelIndexPartial("0", "x001/x234/067.p/8") fmt.Printf("level: %d, index: %d, width: %d", level, index, width) // Output: level: 0, index: 1234067, width: 8 } diff --git a/api/layout/paths.go b/api/layout/paths.go index e8b4e5dc..07d2aa2b 100644 --- a/api/layout/paths.go +++ b/api/layout/paths.go @@ -37,13 +37,13 @@ const ( // The logSize is required so that a partial qualifier can be appended to tiles that // would contain fewer than 256 entries. func EntriesPathForLogIndex(seq, logSize uint64) string { - return EntriesPath(seq/256, logSize) + return EntriesPath(seq/EntryBundleWidth, PartialTileSize(0, seq, logSize)) } -// NWithSuffix returns a tiles-spec "N" path, with a partial suffix if applicable. -func NWithSuffix(l, n, logSize uint64) string { +// NWithSuffix returns a tiles-spec "N" path, with a partial suffix if p > 0. +func NWithSuffix(l, n uint64, p uint8) string { suffix := "" - if p := partialTileSize(l, n, logSize); p > 0 { + if p > 0 { suffix = fmt.Sprintf(".p/%d", p) } return fmt.Sprintf("%s%s", fmtN(n), suffix) @@ -51,13 +51,14 @@ func NWithSuffix(l, n, logSize uint64) string { // EntriesPath returns the local path for the nth entry bundle. p denotes the partial // tile size, or 0 if the tile is complete. -func EntriesPath(n, logSize uint64) string { - return fmt.Sprintf("tile/entries/%s", NWithSuffix(0, n, logSize)) +func EntriesPath(n uint64, p uint8) string { + return fmt.Sprintf("tile/entries/%s", NWithSuffix(0, n, p)) } // TilePath builds the path to the subtree tile with the given level and index in tile space. -func TilePath(tileLevel, tileIndex, logSize uint64) string { - return fmt.Sprintf("tile/%d/%s", tileLevel, NWithSuffix(tileLevel, tileIndex, logSize)) +// If p > 0 the path represents a partial tile. +func TilePath(tileLevel, tileIndex uint64, p uint8) string { + return fmt.Sprintf("tile/%d/%s", tileLevel, NWithSuffix(tileLevel, tileIndex, p)) } // fmtN returns the "N" part of a Tiles-spec path. @@ -83,13 +84,13 @@ func fmtN(N uint64) string { // Examples: // "/tile/0/x001/x234/067" means level 0 and index 1234067 of a full tile. // "/tile/0/x001/x234/067.p/8" means level 0, index 1234067 and width 8 of a partial tile. -func ParseTileLevelIndexWidth(level, index string) (uint64, uint64, uint64, error) { +func ParseTileLevelIndexPartial(level, index string) (uint64, uint64, uint8, error) { l, err := ParseTileLevel(level) if err != nil { return 0, 0, 0, err } - i, w, err := ParseTileIndexWidth(index) + i, w, err := ParseTileIndexPartial(index) if err != nil { return 0, 0, 0, err } @@ -107,17 +108,18 @@ func ParseTileLevel(level string) (uint64, error) { return l, err } -// ParseTileIndexWidth takes index in string, validates and returns the index and width in uint64. -func ParseTileIndexWidth(index string) (uint64, uint64, error) { - w := uint64(256) +// ParseTileIndexPartial takes index in string, validates and returns the index and width in uint64. +func ParseTileIndexPartial(index string) (uint64, uint8, error) { + w := uint8(0) indexPaths := strings.Split(index, "/") if strings.Contains(index, ".p") { var err error - w, err = strconv.ParseUint(indexPaths[len(indexPaths)-1], 10, 64) - if err != nil || w < 1 || w > 255 { - return 0, 0, fmt.Errorf("failed to parse tile index") + w64, err := strconv.ParseUint(indexPaths[len(indexPaths)-1], 10, 64) + if err != nil || w64 < 1 || w64 >= TileWidth { + return 0, 0, fmt.Errorf("failed to parse tile width") } + w = uint8(w64) indexPaths[len(indexPaths)-2] = strings.TrimSuffix(indexPaths[len(indexPaths)-2], ".p") indexPaths = indexPaths[:len(indexPaths)-1] } diff --git a/api/layout/paths_test.go b/api/layout/paths_test.go index 2ccb0b3f..093eab69 100644 --- a/api/layout/paths_test.go +++ b/api/layout/paths_test.go @@ -16,7 +16,6 @@ package layout import ( "fmt" - "math" "testing" ) @@ -65,39 +64,30 @@ func TestEntriesPathForLogIndex(t *testing.T) { func TestEntriesPath(t *testing.T) { for _, test := range []struct { N uint64 - logSize uint64 + p uint8 wantPath string + wantErr bool }{ { N: 0, - logSize: 289, wantPath: "tile/entries/000", }, { N: 0, - logSize: 8, + p: 8, wantPath: "tile/entries/000.p/8", }, { N: 255, - logSize: 256 * 256, wantPath: "tile/entries/255", }, { N: 255, - logSize: 255*256 - 3, + p: 253, wantPath: "tile/entries/255.p/253", - }, { - N: 256, - logSize: 257 * 256, - wantPath: "tile/entries/256", - }, { - N: 123456789000, - logSize: math.MaxUint64, - wantPath: "tile/entries/x123/x456/x789/000", }, } { desc := fmt.Sprintf("N %d", test.N) t.Run(desc, func(t *testing.T) { - gotPath := EntriesPath(test.N, test.logSize) + gotPath := EntriesPath(test.N, test.p) if gotPath != test.wantPath { t.Errorf("got file %q want %q", gotPath, test.wantPath) } @@ -109,59 +99,37 @@ func TestTilePath(t *testing.T) { for _, test := range []struct { level uint64 index uint64 - logSize uint64 + p uint8 wantPath string }{ { level: 0, index: 0, - logSize: 256, - wantPath: "tile/0/000", - }, { - level: 0, - index: 0, - logSize: 0, wantPath: "tile/0/000", }, { level: 0, index: 0, - logSize: 255, + p: 255, wantPath: "tile/0/000.p/255", }, { level: 1, index: 0, - logSize: math.MaxUint64, wantPath: "tile/1/000", - }, { - level: 1, - index: 0, - logSize: 256, - wantPath: "tile/1/000.p/1", - }, { - level: 1, - index: 0, - logSize: 1024, - wantPath: "tile/1/000.p/4", }, { level: 15, index: 455667, - logSize: math.MaxUint64, + p: 0, wantPath: "tile/15/x455/667", - }, { - level: 3, - index: 1234567, - logSize: math.MaxUint64, - wantPath: "tile/3/x001/x234/567", }, { level: 15, index: 123456789, - logSize: math.MaxUint64, - wantPath: "tile/15/x123/x456/789", + p: 41, + wantPath: "tile/15/x123/x456/789.p/41", }, } { desc := fmt.Sprintf("level %x index %x", test.level, test.index) t.Run(desc, func(t *testing.T) { - gotPath := TilePath(test.level, test.index, test.logSize) + gotPath := TilePath(test.level, test.index, test.p) if gotPath != test.wantPath { t.Errorf("Got path %q want %q", gotPath, test.wantPath) } @@ -173,59 +141,32 @@ func TestNWithSuffix(t *testing.T) { for _, test := range []struct { level uint64 index uint64 - logSize uint64 + p uint8 wantPath string }{ { level: 0, index: 0, - logSize: 256, - wantPath: "000", - }, { - level: 0, - index: 0, - logSize: 0, wantPath: "000", }, { level: 0, index: 0, - logSize: 255, + p: 255, wantPath: "000.p/255", - }, { - level: 1, - index: 0, - logSize: math.MaxUint64, - wantPath: "000", - }, { - level: 1, - index: 0, - logSize: 256, - wantPath: "000.p/1", - }, { - level: 1, - index: 0, - logSize: 1024, - wantPath: "000.p/4", }, { level: 15, index: 455667, - logSize: math.MaxUint64, wantPath: "x455/667", - }, { - level: 3, - index: 1234567, - logSize: math.MaxUint64, - wantPath: "x001/x234/567", }, { level: 15, index: 123456789, - logSize: math.MaxUint64, - wantPath: "x123/x456/789", + p: 65, + wantPath: "x123/x456/789.p/65", }, } { desc := fmt.Sprintf("level %x index %x", test.level, test.index) t.Run(desc, func(t *testing.T) { - gotPath := NWithSuffix(test.level, test.index, test.logSize) + gotPath := NWithSuffix(test.level, test.index, test.p) if gotPath != test.wantPath { t.Errorf("Got path %q want %q", gotPath, test.wantPath) } @@ -233,13 +174,13 @@ func TestNWithSuffix(t *testing.T) { } } -func TestParseTileLevelIndexWidth(t *testing.T) { +func TestParseTileLevelIndexPartial(t *testing.T) { for _, test := range []struct { pathLevel string pathIndex string wantLevel uint64 wantIndex uint64 - wantWidth uint64 + wantP uint8 wantErr bool }{ { @@ -247,28 +188,28 @@ func TestParseTileLevelIndexWidth(t *testing.T) { pathIndex: "x001/x234/067", wantLevel: 0, wantIndex: 1234067, - wantWidth: 256, + wantP: 0, }, { pathLevel: "0", pathIndex: "x001/x234/067.p/89", wantLevel: 0, wantIndex: 1234067, - wantWidth: 89, + wantP: 89, }, { pathLevel: "63", pathIndex: "x999/x999/x999/x999/x999/999.p/255", wantLevel: 63, wantIndex: 999999999999999999, - wantWidth: 255, + wantP: 255, }, { pathLevel: "0", pathIndex: "001", wantLevel: 0, wantIndex: 1, - wantWidth: 256, + wantP: 0, }, { pathLevel: "0", @@ -348,15 +289,15 @@ func TestParseTileLevelIndexWidth(t *testing.T) { } { desc := fmt.Sprintf("pathLevel: %q, pathIndex: %q", test.pathLevel, test.pathIndex) t.Run(desc, func(t *testing.T) { - gotLevel, gotIndex, gotWidth, err := ParseTileLevelIndexWidth(test.pathLevel, test.pathIndex) + gotLevel, gotIndex, gotWidth, err := ParseTileLevelIndexPartial(test.pathLevel, test.pathIndex) if gotLevel != test.wantLevel { t.Errorf("got level %d want %d", gotLevel, test.wantLevel) } if gotIndex != test.wantIndex { t.Errorf("got index %d want %d", gotIndex, test.wantIndex) } - if gotWidth != test.wantWidth { - t.Errorf("got width %d want %d", gotWidth, test.wantWidth) + if gotWidth != test.wantP { + t.Errorf("got width %d want %d", gotWidth, test.wantP) } gotErr := err != nil if gotErr != test.wantErr { diff --git a/api/layout/tile.go b/api/layout/tile.go index c6b31a97..ded1c6da 100644 --- a/api/layout/tile.go +++ b/api/layout/tile.go @@ -14,24 +14,35 @@ package layout -// partialTileSize returns the expected number of leaves in a tile at the given location within +const ( + // TileHeight is the maximum number of levels Merkle tree levels a tile represents. + // This is fixed at 8 by tlog-tile spec. + TileHeight = 8 + // TileWidth is the maximum number of hashes which can be present in the bottom row of a tile. + TileWidth = 1 << TileHeight + // EntryBundleWidth is the maximum number of entries which can be present in an EntryBundle. + // This is defined to be the same as the width of the node tiles by tlog-tile spec. + EntryBundleWidth = TileWidth +) + +// PartialTileSize returns the expected number of leaves in a tile at the given location within // a tree of the specified logSize, or 0 if the tile is expected to be fully populated. -func partialTileSize(level, index, logSize uint64) uint64 { - sizeAtLevel := logSize >> (level * 8) - fullTiles := sizeAtLevel / 256 +func PartialTileSize(level, index, logSize uint64) uint8 { + sizeAtLevel := logSize >> (level * TileHeight) + fullTiles := sizeAtLevel / TileWidth if index < fullTiles { return 0 } - return sizeAtLevel % 256 + return uint8(sizeAtLevel % TileWidth) } // NodeCoordsToTileAddress returns the (TileLevel, TileIndex) in tile-space, and the // (NodeLevel, NodeIndex) address within that tile of the specified tree node co-ordinates. func NodeCoordsToTileAddress(treeLevel, treeIndex uint64) (uint64, uint64, uint, uint64) { - tileRowWidth := uint64(1 << (8 - treeLevel%8)) - tileLevel := treeLevel / 8 + tileRowWidth := uint64(1 << (TileHeight - treeLevel%TileHeight)) + tileLevel := treeLevel / TileHeight tileIndex := treeIndex / tileRowWidth - nodeLevel := uint(treeLevel % 8) + nodeLevel := uint(treeLevel % TileHeight) nodeIndex := uint64(treeIndex % tileRowWidth) return tileLevel, tileIndex, nodeLevel, nodeIndex diff --git a/api/state.go b/api/state.go index 3c67af40..d86e67cc 100644 --- a/api/state.go +++ b/api/state.go @@ -22,6 +22,8 @@ import ( "crypto/sha256" "encoding/binary" "fmt" + + "github.com/transparency-dev/trillian-tessera/api/layout" ) // HashTile represents a tile within the Merkle hash tree. @@ -71,7 +73,7 @@ type EntryBundle struct { // UnmarshalText implements encoding/TextUnmarshaler and reads EntryBundles // which are encoded using the tlog-tiles spec. func (t *EntryBundle) UnmarshalText(raw []byte) error { - nodes := make([][]byte, 0, 256) + nodes := make([][]byte, 0, layout.EntryBundleWidth) for index := 0; index < len(raw); { dataIndex := index + 2 if dataIndex > len(raw) { diff --git a/client/client.go b/client/client.go index 564b280e..b3347a4f 100644 --- a/client/client.go +++ b/client/client.go @@ -53,7 +53,7 @@ type CheckpointFetcherFunc func(ctx context.Context) ([]byte, error) // Note that the implementation of this MUST return (either directly or wrapped) // an os.ErrIsNotExist when the file referenced by path does not exist, e.g. a HTTP // based implementation MUST return this error when it receives a 404 StatusCode. -type TileFetcherFunc func(ctx context.Context, level, index, logSize uint64) ([]byte, error) +type TileFetcherFunc func(ctx context.Context, level, index uint64, p uint8) ([]byte, error) // EntryBundleFetcherFunc is the signature of a function which can fetch the raw data // for a given entry bundle. @@ -61,7 +61,7 @@ type TileFetcherFunc func(ctx context.Context, level, index, logSize uint64) ([] // Note that the implementation of this MUST return (either directly or wrapped) // an os.ErrIsNotExist when the file referenced by path does not exist, e.g. a HTTP // based implementation MUST return this error when it receives a 404 StatusCode. -type EntryBundleFetcherFunc func(ctx context.Context, bundleIndex, logSize uint64) ([]byte, error) +type EntryBundleFetcherFunc func(ctx context.Context, bundleIndex uint64, p uint8) ([]byte, error) // ConsensusCheckpointFunc is a function which returns the largest checkpoint known which is // signed by logSigV and satisfies some consensus algorithm. @@ -255,7 +255,7 @@ func (n *nodeCache) GetNode(ctx context.Context, id compact.NodeID) ([]byte, err tKey := tileKey{tileLevel, tileIndex} t, ok := n.tiles[tKey] if !ok { - tileRaw, err := n.getTile(ctx, tileLevel, tileIndex, n.logSize) + tileRaw, err := n.getTile(ctx, tileLevel, tileIndex, layout.PartialTileSize(tileLevel, tileIndex, n.logSize)) if err != nil { return nil, fmt.Errorf("failed to fetch tile: %w", err) } @@ -286,7 +286,7 @@ func (n *nodeCache) GetNode(ctx context.Context, id compact.NodeID) ([]byte, err // GetEntryBundle fetches the entry bundle at the given _tile index_. func GetEntryBundle(ctx context.Context, f EntryBundleFetcherFunc, i, logSize uint64) (api.EntryBundle, error) { bundle := api.EntryBundle{} - sRaw, err := f(ctx, i, logSize) + sRaw, err := f(ctx, i, uint8(logSize%layout.EntryBundleWidth)) if err != nil { if errors.Is(err, os.ErrNotExist) { return bundle, fmt.Errorf("leaf bundle at index %d not found: %v", i, err) diff --git a/client/client_test.go b/client/client_test.go index 9807af02..eb67b2a8 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -76,8 +76,8 @@ func testLogFetcher(_ context.Context, p string) ([]byte, error) { return os.ReadFile(path) } -func testLogTileFetcher(ctx context.Context, l, i, s uint64) ([]byte, error) { - return testLogFetcher(ctx, layout.TilePath(l, i, s)) +func testLogTileFetcher(ctx context.Context, l, i uint64, p uint8) ([]byte, error) { + return testLogFetcher(ctx, layout.TilePath(l, i, p)) } // fetchCheckpointShim allows fetcher requests for checkpoints to be intercepted. @@ -309,7 +309,7 @@ func TestCheckConsistency(t *testing.T) { func TestNodeCacheHandlesInvalidRequest(t *testing.T) { ctx := context.Background() wantBytes := []byte("0123456789ABCDEF0123456789ABCDEF") - f := func(_ context.Context, _, _, _ uint64) ([]byte, error) { + f := func(_ context.Context, _, _ uint64, _ uint8) ([]byte, error) { h := &api.HashTile{ Nodes: [][]byte{wantBytes}, } diff --git a/client/fetcher.go b/client/fetcher.go index bfa0e9eb..681e0b82 100644 --- a/client/fetcher.go +++ b/client/fetcher.go @@ -98,12 +98,12 @@ func (h HTTPFetcher) ReadCheckpoint(ctx context.Context) ([]byte, error) { return h.fetch(ctx, layout.CheckpointPath) } -func (h HTTPFetcher) ReadTile(ctx context.Context, l, i, sz uint64) ([]byte, error) { - return h.fetch(ctx, layout.TilePath(l, i, sz)) +func (h HTTPFetcher) ReadTile(ctx context.Context, l, i uint64, p uint8) ([]byte, error) { + return h.fetch(ctx, layout.TilePath(l, i, p)) } -func (h HTTPFetcher) ReadEntryBundle(ctx context.Context, i, sz uint64) ([]byte, error) { - return h.fetch(ctx, layout.EntriesPath(i, sz)) +func (h HTTPFetcher) ReadEntryBundle(ctx context.Context, i uint64, p uint8) ([]byte, error) { + return h.fetch(ctx, layout.EntriesPath(i, p)) } // FileFetcher knows how to fetch log artifacts from a filesystem rooted at Root. @@ -115,10 +115,10 @@ func (f FileFetcher) ReadCheckpoint(_ context.Context) ([]byte, error) { return os.ReadFile(path.Join(f.Root, layout.CheckpointPath)) } -func (f FileFetcher) ReadTile(_ context.Context, l, i, sz uint64) ([]byte, error) { - return os.ReadFile(path.Join(f.Root, layout.TilePath(l, i, sz))) +func (f FileFetcher) ReadTile(_ context.Context, l, i uint64, p uint8) ([]byte, error) { + return os.ReadFile(path.Join(f.Root, layout.TilePath(l, i, p))) } -func (f FileFetcher) ReadEntryBundle(_ context.Context, i, sz uint64) ([]byte, error) { - return os.ReadFile(path.Join(f.Root, layout.EntriesPath(i, sz))) +func (f FileFetcher) ReadEntryBundle(_ context.Context, i uint64, p uint8) ([]byte, error) { + return os.ReadFile(path.Join(f.Root, layout.EntriesPath(i, p))) } diff --git a/cmd/conformance/mysql/main.go b/cmd/conformance/mysql/main.go index d8f7a4c6..00e24b29 100644 --- a/cmd/conformance/mysql/main.go +++ b/cmd/conformance/mysql/main.go @@ -159,7 +159,7 @@ func configureTilesReadAPI(mux *http.ServeMux, storage *mysql.Storage) { }) mux.HandleFunc("GET /tile/{level}/{index...}", func(w http.ResponseWriter, r *http.Request) { - level, index, width, err := layout.ParseTileLevelIndexWidth(r.PathValue("level"), r.PathValue("index")) + level, index, p, err := layout.ParseTileLevelIndexPartial(r.PathValue("level"), r.PathValue("index")) if err != nil { w.WriteHeader(http.StatusBadRequest) if _, werr := w.Write([]byte(fmt.Sprintf("Malformed URL: %s", err.Error()))); werr != nil { @@ -167,8 +167,7 @@ func configureTilesReadAPI(mux *http.ServeMux, storage *mysql.Storage) { } return } - inferredMinTreeSize := (index*256 + width) << (level * 8) - tile, err := storage.ReadTile(r.Context(), level, index, inferredMinTreeSize) + tile, err := storage.ReadTile(r.Context(), level, index, p) if err != nil { if errors.Is(err, fs.ErrNotExist) { w.WriteHeader(http.StatusNotFound) @@ -188,7 +187,7 @@ func configureTilesReadAPI(mux *http.ServeMux, storage *mysql.Storage) { }) mux.HandleFunc("GET /tile/entries/{index...}", func(w http.ResponseWriter, r *http.Request) { - index, width, err := layout.ParseTileIndexWidth(r.PathValue("index")) + index, p, err := layout.ParseTileIndexPartial(r.PathValue("index")) if err != nil { w.WriteHeader(http.StatusBadRequest) if _, werr := w.Write([]byte(fmt.Sprintf("Malformed URL: %s", err.Error()))); werr != nil { @@ -197,7 +196,7 @@ func configureTilesReadAPI(mux *http.ServeMux, storage *mysql.Storage) { return } - entryBundle, err := storage.ReadEntryBundle(r.Context(), index, width) + entryBundle, err := storage.ReadEntryBundle(r.Context(), index, p) if err != nil { klog.Errorf("/tile/entries/{index...}: %v", err) w.WriteHeader(http.StatusInternalServerError) diff --git a/ct_only.go b/ct_only.go index 34ede327..e39a21a6 100644 --- a/ct_only.go +++ b/ct_only.go @@ -69,6 +69,6 @@ func WithCTLayout() func(*options.StorageOptions) { } } -func ctEntriesPath(n, logSize uint64) string { - return fmt.Sprintf("tile/data/%s", layout.NWithSuffix(0, n, logSize)) +func ctEntriesPath(n uint64, p uint8) string { + return fmt.Sprintf("tile/data/%s", layout.NWithSuffix(0, n, p)) } diff --git a/ct_only_test.go b/ct_only_test.go index 7048a466..d5817da3 100644 --- a/ct_only_test.go +++ b/ct_only_test.go @@ -16,46 +16,41 @@ package tessera import ( "fmt" - "math" "testing" ) func TestCTEntriesPath(t *testing.T) { for _, test := range []struct { N uint64 - logSize uint64 + p uint8 wantPath string }{ { N: 0, - logSize: 289, wantPath: "tile/data/000", }, { N: 0, - logSize: 8, + p: 8, wantPath: "tile/data/000.p/8", }, { N: 255, - logSize: 256 * 256, wantPath: "tile/data/255", }, { N: 255, - logSize: 255*256 - 3, + p: 253, wantPath: "tile/data/255.p/253", }, { N: 256, - logSize: 257 * 256, wantPath: "tile/data/256", }, { N: 123456789000, - logSize: math.MaxUint64, wantPath: "tile/data/x123/x456/x789/000", }, } { desc := fmt.Sprintf("N %d", test.N) t.Run(desc, func(t *testing.T) { - gotPath := ctEntriesPath(test.N, test.logSize) + gotPath := ctEntriesPath(test.N, test.p) if gotPath != test.wantPath { t.Errorf("got file %q want %q", gotPath, test.wantPath) } diff --git a/integration/storage_uniformity_test.go b/integration/storage_uniformity_test.go index 123240aa..aeb12af4 100644 --- a/integration/storage_uniformity_test.go +++ b/integration/storage_uniformity_test.go @@ -28,8 +28,8 @@ import ( type StorageContract interface { Add(ctx context.Context, entry *tessera.Entry) tessera.IndexFuture ReadCheckpoint(ctx context.Context) ([]byte, error) - ReadTile(ctx context.Context, level, index, treeSize uint64) ([]byte, error) - ReadEntryBundle(ctx context.Context, index, treeSize uint64) ([]byte, error) + ReadTile(ctx context.Context, level, index uint64, p uint8) ([]byte, error) + ReadEntryBundle(ctx context.Context, index uint64, p uint8) ([]byte, error) } var ( diff --git a/internal/hammer/client.go b/internal/hammer/client.go index 58429100..63a04028 100644 --- a/internal/hammer/client.go +++ b/internal/hammer/client.go @@ -35,8 +35,8 @@ var ErrRetry = errors.New("retry") type fetcher interface { ReadCheckpoint(ctx context.Context) ([]byte, error) - ReadTile(ctx context.Context, l, i, sz uint64) ([]byte, error) - ReadEntryBundle(ctx context.Context, i, sz uint64) ([]byte, error) + ReadTile(ctx context.Context, l, i uint64, p uint8) ([]byte, error) + ReadEntryBundle(ctx context.Context, i uint64, p uint8) ([]byte, error) } // newLogClientsFromFlags returns a fetcher and a writer that will read @@ -110,14 +110,14 @@ func (rr *roundRobinFetcher) ReadCheckpoint(ctx context.Context) ([]byte, error) return f.ReadCheckpoint(ctx) } -func (rr *roundRobinFetcher) ReadTile(ctx context.Context, l, i, sz uint64) ([]byte, error) { +func (rr *roundRobinFetcher) ReadTile(ctx context.Context, l, i uint64, p uint8) ([]byte, error) { f := rr.next() - return f.ReadTile(ctx, l, i, sz) + return f.ReadTile(ctx, l, i, p) } -func (rr *roundRobinFetcher) ReadEntryBundle(ctx context.Context, i, sz uint64) ([]byte, error) { +func (rr *roundRobinFetcher) ReadEntryBundle(ctx context.Context, i uint64, p uint8) ([]byte, error) { f := rr.next() - return f.ReadEntryBundle(ctx, i, sz) + return f.ReadEntryBundle(ctx, i, p) } func (rr *roundRobinFetcher) next() fetcher { diff --git a/internal/options/options.go b/internal/options/options.go index 2d60c543..5a87c5ca 100644 --- a/internal/options/options.go +++ b/internal/options/options.go @@ -27,7 +27,7 @@ type NewCPFunc func(size uint64, hash []byte) ([]byte, error) type ParseCPFunc func(raw []byte) (*f_log.Checkpoint, error) // EntriesPathFunc is the signature of a function which knows how to format entry bundle paths. -type EntriesPathFunc func(n, logSize uint64) string +type EntriesPathFunc func(n uint64, p uint8) string // StorageOptions holds optional settings for all storage implementations. type StorageOptions struct { diff --git a/storage/aws/aws.go b/storage/aws/aws.go index c4766190..6950db1b 100644 --- a/storage/aws/aws.go +++ b/storage/aws/aws.go @@ -59,9 +59,8 @@ import ( ) const ( - entryBundleSize = 256 - logContType = "application/octet-stream" - ckptContType = "text/plain; charset=utf-8" + logContType = "application/octet-stream" + ckptContType = "text/plain; charset=utf-8" DefaultPushbackMaxOutstanding = 4096 DefaultIntegrationSizeLimit = 5 * 4096 @@ -225,12 +224,12 @@ func (s *Storage) ReadCheckpoint(ctx context.Context) ([]byte, error) { return s.get(ctx, layout.CheckpointPath) } -func (s *Storage) ReadTile(ctx context.Context, l, i, sz uint64) ([]byte, error) { - return s.get(ctx, layout.TilePath(l, i, sz)) +func (s *Storage) ReadTile(ctx context.Context, l, i uint64, p uint8) ([]byte, error) { + return s.get(ctx, layout.TilePath(l, i, p)) } -func (s *Storage) ReadEntryBundle(ctx context.Context, i, sz uint64) ([]byte, error) { - return s.get(ctx, s.entriesPath(i, sz)) +func (s *Storage) ReadEntryBundle(ctx context.Context, i uint64, p uint8) ([]byte, error) { + return s.get(ctx, s.entriesPath(i, p)) } // get returns the requested object. @@ -303,7 +302,7 @@ func (s *Storage) setTile(ctx context.Context, level, index, logSize uint64, til if err != nil { return err } - tPath := layout.TilePath(level, index, logSize) + tPath := layout.TilePath(level, index, layout.PartialTileSize(level, index, logSize)) klog.V(2).Infof("StoreTile: %s (%d entries)", tPath, len(tile.Nodes)) return s.objStore.setObjectIfNoneMatch(ctx, tPath, data, logContType) @@ -319,7 +318,7 @@ func (s *Storage) getTiles(ctx context.Context, tileIDs []storage.TileID, logSiz i := i id := id errG.Go(func() error { - objName := layout.TilePath(id.Level, id.Index, logSize) + objName := layout.TilePath(id.Level, id.Index, layout.PartialTileSize(id.Level, id.Index, logSize)) data, err := s.objStore.getObject(ctx, objName) if err != nil { // Do not use errors.Is. Keep errors.As to compare by type and not by value. @@ -349,8 +348,8 @@ func (s *Storage) getTiles(ctx context.Context, tileIDs []storage.TileID, logSiz // getEntryBundle returns the serialised entry bundle at the location implied by the given index and treeSize. // // Returns a wrapped os.ErrNotExist if the bundle does not exist. -func (s *Storage) getEntryBundle(ctx context.Context, bundleIndex uint64, logSize uint64) ([]byte, error) { - objName := s.entriesPath(bundleIndex, logSize) +func (s *Storage) getEntryBundle(ctx context.Context, bundleIndex uint64, p uint8) ([]byte, error) { + objName := s.entriesPath(bundleIndex, p) data, err := s.objStore.getObject(ctx, objName) if err != nil { // Do not use errors.Is. Keep errors.As to compare by type and not by value. @@ -367,8 +366,8 @@ func (s *Storage) getEntryBundle(ctx context.Context, bundleIndex uint64, logSiz } // setEntryBundle idempotently stores the serialised entry bundle at the location implied by the bundleIndex and treeSize. -func (s *Storage) setEntryBundle(ctx context.Context, bundleIndex uint64, logSize uint64, bundleRaw []byte) error { - objName := s.entriesPath(bundleIndex, logSize) +func (s *Storage) setEntryBundle(ctx context.Context, bundleIndex uint64, p uint8, bundleRaw []byte) error { + objName := s.entriesPath(bundleIndex, p) // Note that setObject does an idempotent interpretation of IfNoneMatch - it only // returns an error if the named object exists _and_ contains different data to what's // passed in here. @@ -433,11 +432,11 @@ func (s *Storage) updateEntryBundles(ctx context.Context, fromSeq uint64, entrie } numAdded := uint64(0) - bundleIndex, entriesInBundle := fromSeq/entryBundleSize, fromSeq%entryBundleSize + bundleIndex, entriesInBundle := fromSeq/layout.EntryBundleWidth, fromSeq%layout.EntryBundleWidth bundleWriter := &bytes.Buffer{} if entriesInBundle > 0 { // If the latest bundle is partial, we need to read the data it contains in for our newer, larger, bundle. - part, err := s.getEntryBundle(ctx, uint64(bundleIndex), uint64(entriesInBundle)) + part, err := s.getEntryBundle(ctx, uint64(bundleIndex), uint8(entriesInBundle)) if err != nil { return err } @@ -451,9 +450,9 @@ func (s *Storage) updateEntryBundles(ctx context.Context, fromSeq uint64, entrie // goSetEntryBundle is a function which uses seqErr to spin off a go-routine to write out an entry bundle. // It's used in the for loop below. - goSetEntryBundle := func(ctx context.Context, bundleIndex uint64, fromSeq uint64, bundleRaw []byte) { + goSetEntryBundle := func(ctx context.Context, bundleIndex uint64, p uint8, bundleRaw []byte) { seqErr.Go(func() error { - if err := s.setEntryBundle(ctx, bundleIndex, fromSeq, bundleRaw); err != nil { + if err := s.setEntryBundle(ctx, bundleIndex, p, bundleRaw); err != nil { return err } return nil @@ -468,10 +467,10 @@ func (s *Storage) updateEntryBundles(ctx context.Context, fromSeq uint64, entrie entriesInBundle++ fromSeq++ numAdded++ - if entriesInBundle == entryBundleSize { + if entriesInBundle == layout.EntryBundleWidth { // This bundle is full, so we need to write it out... klog.V(1).Infof("In-memory bundle idx %d is full, attempting write to S3", bundleIndex) - goSetEntryBundle(ctx, bundleIndex, fromSeq, bundleWriter.Bytes()) + goSetEntryBundle(ctx, bundleIndex, 0, bundleWriter.Bytes()) // ... and prepare the next entry bundle for any remaining entries in the batch bundleIndex++ entriesInBundle = 0 @@ -484,7 +483,7 @@ func (s *Storage) updateEntryBundles(ctx context.Context, fromSeq uint64, entrie // this needs writing out too. if entriesInBundle > 0 { klog.V(1).Infof("Attempting to write in-memory partial bundle idx %d.%d to S3", bundleIndex, entriesInBundle) - goSetEntryBundle(ctx, bundleIndex, fromSeq, bundleWriter.Bytes()) + goSetEntryBundle(ctx, bundleIndex, uint8(entriesInBundle), bundleWriter.Bytes()) } return seqErr.Wait() } diff --git a/storage/aws/aws_test.go b/storage/aws/aws_test.go index 5ce734be..7c866657 100644 --- a/storage/aws/aws_test.go +++ b/storage/aws/aws_test.go @@ -294,7 +294,7 @@ func TestTileRoundtrip(t *testing.T) { t.Fatalf("setTile: %v", err) } - expPath := layout.TilePath(test.level, test.index, test.logSize) + expPath := layout.TilePath(test.level, test.index, layout.PartialTileSize(test.level, test.index, test.logSize)) _, ok := m.mem[expPath] if !ok { t.Fatalf("want tile at %v but found none", expPath) @@ -334,29 +334,29 @@ func TestBundleRoundtrip(t *testing.T) { for _, test := range []struct { name string index uint64 - logSize uint64 + p uint8 bundleSize uint64 }{ { name: "ok", index: 3 * 256, - logSize: 3*256 + 20, + p: 20, bundleSize: 20, }, } { t.Run(test.name, func(t *testing.T) { wantBundle := makeBundle(t, test.bundleSize) - if err := s.setEntryBundle(ctx, test.index, test.logSize, wantBundle); err != nil { + if err := s.setEntryBundle(ctx, test.index, test.p, wantBundle); err != nil { t.Fatalf("setEntryBundle: %v", err) } - expPath := layout.EntriesPath(test.index, test.logSize) + expPath := layout.EntriesPath(test.index, test.p) _, ok := m.mem[expPath] if !ok { t.Fatalf("want bundle at %v but found none", expPath) } - got, err := s.getEntryBundle(ctx, test.index, test.logSize) + got, err := s.getEntryBundle(ctx, test.index, test.p) if err != nil { t.Fatalf("getEntryBundle: %v", err) } diff --git a/storage/gcp/gcp.go b/storage/gcp/gcp.go index aa4a374a..6947f2fa 100644 --- a/storage/gcp/gcp.go +++ b/storage/gcp/gcp.go @@ -59,9 +59,8 @@ import ( ) const ( - entryBundleSize = 256 - logContType = "application/octet-stream" - ckptContType = "text/plain; charset=utf-8" + logContType = "application/octet-stream" + ckptContType = "text/plain; charset=utf-8" DefaultPushbackMaxOutstanding = 4096 DefaultIntegrationSizeLimit = 5 * 4096 @@ -203,12 +202,12 @@ func (s *Storage) ReadCheckpoint(ctx context.Context) ([]byte, error) { return s.get(ctx, layout.CheckpointPath) } -func (s *Storage) ReadTile(ctx context.Context, l, i, sz uint64) ([]byte, error) { - return s.get(ctx, layout.TilePath(l, i, sz)) +func (s *Storage) ReadTile(ctx context.Context, l, i uint64, p uint8) ([]byte, error) { + return s.get(ctx, layout.TilePath(l, i, p)) } -func (s *Storage) ReadEntryBundle(ctx context.Context, i, sz uint64) ([]byte, error) { - return s.get(ctx, s.entriesPath(i, sz)) +func (s *Storage) ReadEntryBundle(ctx context.Context, i uint64, p uint8) ([]byte, error) { + return s.get(ctx, s.entriesPath(i, p)) } // get returns the requested object. @@ -277,7 +276,7 @@ func (s *Storage) setTile(ctx context.Context, level, index, logSize uint64, til if err != nil { return err } - tPath := layout.TilePath(level, index, logSize) + tPath := layout.TilePath(level, index, layout.PartialTileSize(level, index, logSize)) klog.V(2).Infof("StoreTile: %s (%d entries)", tPath, len(tile.Nodes)) return s.objStore.setObject(ctx, tPath, data, &gcs.Conditions{DoesNotExist: true}, logContType) @@ -293,7 +292,7 @@ func (s *Storage) getTiles(ctx context.Context, tileIDs []storage.TileID, logSiz i := i id := id errG.Go(func() error { - objName := layout.TilePath(id.Level, id.Index, logSize) + objName := layout.TilePath(id.Level, id.Index, layout.PartialTileSize(id.Level, id.Index, logSize)) data, _, err := s.objStore.getObject(ctx, objName) if err != nil { if errors.Is(err, gcs.ErrObjectNotExist) { @@ -318,11 +317,12 @@ func (s *Storage) getTiles(ctx context.Context, tileIDs []storage.TileID, logSiz } -// getEntryBundle returns the serialised entry bundle at the location implied by the given index and treeSize. +// getEntryBundle returns the serialised entry bundle at the location described by the given index and partial size. +// A partial size of zero implies a full tile. // // Returns a wrapped os.ErrNotExist if the bundle does not exist. -func (s *Storage) getEntryBundle(ctx context.Context, bundleIndex uint64, logSize uint64) ([]byte, error) { - objName := s.entriesPath(bundleIndex, logSize) +func (s *Storage) getEntryBundle(ctx context.Context, bundleIndex uint64, p uint8) ([]byte, error) { + objName := s.entriesPath(bundleIndex, p) data, _, err := s.objStore.getObject(ctx, objName) if err != nil { if errors.Is(err, gcs.ErrObjectNotExist) { @@ -337,8 +337,8 @@ func (s *Storage) getEntryBundle(ctx context.Context, bundleIndex uint64, logSiz } // setEntryBundle idempotently stores the serialised entry bundle at the location implied by the bundleIndex and treeSize. -func (s *Storage) setEntryBundle(ctx context.Context, bundleIndex uint64, logSize uint64, bundleRaw []byte) error { - objName := s.entriesPath(bundleIndex, logSize) +func (s *Storage) setEntryBundle(ctx context.Context, bundleIndex uint64, p uint8, bundleRaw []byte) error { + objName := s.entriesPath(bundleIndex, p) // Note that setObject does an idempotent interpretation of DoesNotExist - it only // returns an error if the named object exists _and_ contains different data to what's // passed in here. @@ -400,11 +400,11 @@ func (s *Storage) updateEntryBundles(ctx context.Context, fromSeq uint64, entrie } numAdded := uint64(0) - bundleIndex, entriesInBundle := fromSeq/entryBundleSize, fromSeq%entryBundleSize + bundleIndex, entriesInBundle := fromSeq/layout.EntryBundleWidth, fromSeq%layout.EntryBundleWidth bundleWriter := &bytes.Buffer{} if entriesInBundle > 0 { // If the latest bundle is partial, we need to read the data it contains in for our newer, larger, bundle. - part, err := s.getEntryBundle(ctx, uint64(bundleIndex), uint64(entriesInBundle)) + part, err := s.getEntryBundle(ctx, uint64(bundleIndex), uint8(entriesInBundle)) if err != nil { return err } @@ -418,9 +418,9 @@ func (s *Storage) updateEntryBundles(ctx context.Context, fromSeq uint64, entrie // goSetEntryBundle is a function which uses seqErr to spin off a go-routine to write out an entry bundle. // It's used in the for loop below. - goSetEntryBundle := func(ctx context.Context, bundleIndex uint64, fromSeq uint64, bundleRaw []byte) { + goSetEntryBundle := func(ctx context.Context, bundleIndex uint64, p uint8, bundleRaw []byte) { seqErr.Go(func() error { - if err := s.setEntryBundle(ctx, bundleIndex, fromSeq, bundleRaw); err != nil { + if err := s.setEntryBundle(ctx, bundleIndex, p, bundleRaw); err != nil { return err } return nil @@ -435,10 +435,10 @@ func (s *Storage) updateEntryBundles(ctx context.Context, fromSeq uint64, entrie entriesInBundle++ fromSeq++ numAdded++ - if entriesInBundle == entryBundleSize { + if entriesInBundle == layout.EntryBundleWidth { // This bundle is full, so we need to write it out... klog.V(1).Infof("In-memory bundle idx %d is full, attempting write to GCS", bundleIndex) - goSetEntryBundle(ctx, bundleIndex, fromSeq, bundleWriter.Bytes()) + goSetEntryBundle(ctx, bundleIndex, 0, bundleWriter.Bytes()) // ... and prepare the next entry bundle for any remaining entries in the batch bundleIndex++ entriesInBundle = 0 @@ -451,7 +451,7 @@ func (s *Storage) updateEntryBundles(ctx context.Context, fromSeq uint64, entrie // this needs writing out too. if entriesInBundle > 0 { klog.V(1).Infof("Attempting to write in-memory partial bundle idx %d.%d to GCS", bundleIndex, entriesInBundle) - goSetEntryBundle(ctx, bundleIndex, fromSeq, bundleWriter.Bytes()) + goSetEntryBundle(ctx, bundleIndex, uint8(entriesInBundle), bundleWriter.Bytes()) } return seqErr.Wait() } diff --git a/storage/gcp/gcp_test.go b/storage/gcp/gcp_test.go index 2210301b..66feddc8 100644 --- a/storage/gcp/gcp_test.go +++ b/storage/gcp/gcp_test.go @@ -232,7 +232,7 @@ func TestTileRoundtrip(t *testing.T) { t.Fatalf("setTile: %v", err) } - expPath := layout.TilePath(test.level, test.index, test.logSize) + expPath := layout.TilePath(test.level, test.index, layout.PartialTileSize(test.level, test.index, test.logSize)) _, ok := m.mem[expPath] if !ok { t.Fatalf("want tile at %v but found none", expPath) @@ -284,17 +284,17 @@ func TestBundleRoundtrip(t *testing.T) { } { t.Run(test.name, func(t *testing.T) { wantBundle := makeBundle(t, test.bundleSize) - if err := s.setEntryBundle(ctx, test.index, test.logSize, wantBundle); err != nil { + if err := s.setEntryBundle(ctx, test.index, uint8(test.bundleSize), wantBundle); err != nil { t.Fatalf("setEntryBundle: %v", err) } - expPath := layout.EntriesPath(test.index, test.logSize) + expPath := layout.EntriesPath(test.index, uint8(test.logSize%layout.EntryBundleWidth)) _, ok := m.mem[expPath] if !ok { t.Fatalf("want bundle at %v but found none", expPath) } - got, err := s.getEntryBundle(ctx, test.index, test.logSize) + got, err := s.getEntryBundle(ctx, test.index, uint8(test.logSize%layout.EntryBundleWidth)) if err != nil { t.Fatalf("getEntryBundle: %v", err) } diff --git a/storage/internal/integrate.go b/storage/internal/integrate.go index d04ac27b..275016c8 100644 --- a/storage/internal/integrate.go +++ b/storage/internal/integrate.go @@ -177,7 +177,7 @@ func newTileReadCache(getTiles func(ctx context.Context, tileIDs []TileID, treeS // Get returns a previously set tile and true, or, if no such tile is in the cache, attempt to fetch it. func (r *tileReadCache) Get(ctx context.Context, tileID TileID, treeSize uint64) (*populatedTile, error) { - k := layout.TilePath(uint64(tileID.Level), tileID.Index, treeSize) + k := layout.TilePath(uint64(tileID.Level), tileID.Index, layout.PartialTileSize(tileID.Level, tileID.Index, treeSize)) e, ok := r.entries[k] if !ok { klog.V(1).Infof("Readcache miss: %q", k) @@ -207,7 +207,7 @@ func (r *tileReadCache) Prewarm(ctx context.Context, tileIDs []TileID, treeSize if err != nil { return fmt.Errorf("failed to create fulltile: %v", err) } - k := layout.TilePath(uint64(tileIDs[i].Level), tileIDs[i].Index, treeSize) + k := layout.TilePath(uint64(tileIDs[i].Level), tileIDs[i].Index, layout.PartialTileSize(tileIDs[i].Level, tileIDs[i].Index, treeSize)) r.entries[k] = e } return nil diff --git a/storage/internal/integrate_test.go b/storage/internal/integrate_test.go index d29ae01b..07a6c737 100644 --- a/storage/internal/integrate_test.go +++ b/storage/internal/integrate_test.go @@ -221,7 +221,7 @@ func (m *memTileStore[T]) getTile(_ context.Context, id TileID, treeSize uint64) m.RLock() defer m.RUnlock() - k := layout.TilePath(id.Level, id.Index, treeSize) + k := layout.TilePath(id.Level, id.Index, layout.PartialTileSize(id.Level, id.Index, treeSize)) d := m.mem[k] return d, nil } @@ -232,7 +232,7 @@ func (m *memTileStore[T]) getTiles(_ context.Context, ids []TileID, treeSize uin r := make([]*T, len(ids)) for i, id := range ids { - k := layout.TilePath(id.Level, id.Index, treeSize) + k := layout.TilePath(id.Level, id.Index, layout.PartialTileSize(id.Level, id.Index, treeSize)) klog.V(1).Infof("mem.getTile(%q, %d)", k, treeSize) d, ok := m.mem[k] if !ok { @@ -247,7 +247,7 @@ func (m *memTileStore[T]) setTile(_ context.Context, id TileID, treeSize uint64, m.Lock() defer m.Unlock() - k := layout.TilePath(id.Level, id.Index, treeSize) + k := layout.TilePath(id.Level, id.Index, layout.PartialTileSize(id.Level, id.Index, treeSize)) klog.V(1).Infof("mem.setTile(%q, %d)", k, treeSize) _, ok := m.mem[k] if ok { diff --git a/storage/mysql/mysql.go b/storage/mysql/mysql.go index 8ac67862..7c17af13 100644 --- a/storage/mysql/mysql.go +++ b/storage/mysql/mysql.go @@ -243,7 +243,7 @@ func (s *Storage) writeTreeState(ctx context.Context, tx *sql.Tx, size uint64, r // Note that if a partial tile is requested, but a larger tile is available, this // will return the largest tile available. This could be trimmed to return only the // number of entries specifically requested if this behaviour becomes problematic. -func (s *Storage) ReadTile(ctx context.Context, level, index, minTreeSize uint64) ([]byte, error) { +func (s *Storage) ReadTile(ctx context.Context, level, index uint64, p uint8) ([]byte, error) { row := s.db.QueryRowContext(ctx, selectSubtreeByLevelAndIndexSQL, level, index) if err := row.Err(); err != nil { return nil, err @@ -258,10 +258,12 @@ func (s *Storage) ReadTile(ctx context.Context, level, index, minTreeSize uint64 return nil, fmt.Errorf("scan tile: %v", err) } - requestedWidth := partialTileSize(level, index, minTreeSize) numEntries := uint64(len(tile) / sha256.Size) - - if requestedWidth > numEntries { + requestedEntries := uint64(p) + if requestedEntries == 0 { + requestedEntries = 256 + } + if requestedEntries > numEntries { // If the user has requested a size larger than we have, they can't have it return nil, os.ErrNotExist } @@ -269,17 +271,6 @@ func (s *Storage) ReadTile(ctx context.Context, level, index, minTreeSize uint64 return tile, nil } -// partialTileSize returns the expected number of leaves in a tile at the given location within -// a tree of the specified logSize, or 0 if the tile is expected to be fully populated. -func partialTileSize(level, index, logSize uint64) uint64 { - sizeAtLevel := logSize >> (level * 8) - fullTiles := sizeAtLevel / 256 - if index < fullTiles { - return 256 - } - return sizeAtLevel % 256 -} - // writeTile replaces the tile nodes at the given level and index. func (s *Storage) writeTile(ctx context.Context, tx *sql.Tx, level, index uint64, nodes []byte) error { if _, err := tx.ExecContext(ctx, replaceSubtreeSQL, level, index, nodes); err != nil { @@ -299,7 +290,7 @@ func (s *Storage) writeTile(ctx context.Context, tx *sql.Tx, level, index uint64 // 3. Partial tile request with full/larger partial tile output: Return trimmed partial tile with correct tile width. // 4. Partial tile request with partial tile (same width) output: Return partial tile. // 5. Partial tile request with smaller partial tile output: Return error. -func (s *Storage) ReadEntryBundle(ctx context.Context, index, treeSize uint64) ([]byte, error) { +func (s *Storage) ReadEntryBundle(ctx context.Context, index uint64, p uint8) ([]byte, error) { row := s.db.QueryRowContext(ctx, selectTiledLeavesSQL, index) if err := row.Err(); err != nil { return nil, err diff --git a/storage/mysql/mysql_test.go b/storage/mysql/mysql_test.go index a9da561c..ada8f85d 100644 --- a/storage/mysql/mysql_test.go +++ b/storage/mysql/mysql_test.go @@ -191,54 +191,55 @@ func TestGetTile(t *testing.T) { } for _, test := range []struct { - name string - level, index, treeSize uint64 - wantEntries int - wantNotFound bool + name string + level, index uint64 + p uint8 + wantEntries int + wantNotFound bool }{ { name: "requested partial tile for a complete tile", - level: 0, index: 0, treeSize: 10, + level: 0, index: 0, p: 10, wantEntries: 256, wantNotFound: false, }, { name: "too small but that's ok", - level: 0, index: 1, treeSize: uint64(treeSize) - 1, + level: 0, index: 1, p: layout.PartialTileSize(0, 1, uint64(treeSize-1)), wantEntries: 2, wantNotFound: false, }, { name: "just right", - level: 0, index: 1, treeSize: uint64(treeSize), + level: 0, index: 1, p: layout.PartialTileSize(0, 1, uint64(treeSize)), wantEntries: 2, wantNotFound: false, }, { name: "too big", - level: 0, index: 1, treeSize: uint64(treeSize + 1), + level: 0, index: 1, p: layout.PartialTileSize(0, 1, uint64(treeSize+1)), wantNotFound: true, }, { name: "level 1 too small", - level: 1, index: 0, treeSize: uint64(treeSize - 1), + level: 1, index: 0, p: layout.PartialTileSize(1, 0, uint64(treeSize-1)), wantEntries: 1, wantNotFound: false, }, { name: "level 1 just right", - level: 1, index: 0, treeSize: uint64(treeSize), + level: 1, index: 0, p: layout.PartialTileSize(1, 0, uint64(treeSize)), wantEntries: 1, wantNotFound: false, }, { name: "level 1 too big", - level: 1, index: 0, treeSize: 550, + level: 1, index: 0, p: layout.PartialTileSize(1, 0, 550), wantNotFound: true, }, } { t.Run(test.name, func(t *testing.T) { - tile, err := s.ReadTile(ctx, test.level, test.index, test.treeSize) + tile, err := s.ReadTile(ctx, test.level, test.index, test.p) if err != nil { if notFound, wantNotFound := errors.Is(err, fs.ErrNotExist), test.wantNotFound; notFound != wantNotFound { t.Errorf("wantNotFound %v but notFound %v", wantNotFound, notFound) @@ -261,20 +262,21 @@ func TestReadMissingTile(t *testing.T) { s := newTestMySQLStorage(t, ctx) for _, test := range []struct { - name string - level, index, width uint64 + name string + level, index uint64 + p uint8 }{ { name: "0/0/0", - level: 0, index: 0, width: 0, + level: 0, index: 0, p: 0, }, { name: "123/456/789", - level: 123, index: 456, width: 789, + level: 123, index: 456, p: 789 % layout.TileWidth, }, } { t.Run(test.name, func(t *testing.T) { - tile, err := s.ReadTile(ctx, test.level, test.index, test.width) + tile, err := s.ReadTile(ctx, test.level, test.index, test.p) if err != nil { if errors.Is(err, fs.ErrNotExist) { // this is success for this test @@ -307,7 +309,7 @@ func TestReadMissingEntryBundle(t *testing.T) { }, } { t.Run(test.name, func(t *testing.T) { - entryBundle, err := s.ReadEntryBundle(ctx, test.index, test.index) + entryBundle, err := s.ReadEntryBundle(ctx, test.index, uint8(test.index%layout.TileWidth)) if err != nil { if errors.Is(err, fs.ErrNotExist) { // this is success for this test @@ -387,7 +389,7 @@ func TestTileRoundTrip(t *testing.T) { } tileLevel, tileIndex, _, nodeIndex := layout.NodeCoordsToTileAddress(0, entryIndex) - tileRaw, err := s.ReadTile(ctx, tileLevel, tileIndex, nodeIndex+1) + tileRaw, err := s.ReadTile(ctx, tileLevel, tileIndex, uint8(nodeIndex+1)) if err != nil { t.Errorf("ReadTile got err: %v", err) } @@ -436,7 +438,7 @@ func TestEntryBundleRoundTrip(t *testing.T) { if err != nil { t.Errorf("Add got err: %v", err) } - entryBundleRaw, err := s.ReadEntryBundle(ctx, entryIndex/256, entryIndex) + entryBundleRaw, err := s.ReadEntryBundle(ctx, entryIndex/256, uint8(entryIndex%layout.TileWidth)) if err != nil { t.Errorf("ReadEntryBundle got err: %v", err) } diff --git a/storage/posix/files.go b/storage/posix/files.go index 5e78658f..4fca1a4c 100644 --- a/storage/posix/files.go +++ b/storage/posix/files.go @@ -148,12 +148,12 @@ func (s *Storage) ReadCheckpoint(_ context.Context) ([]byte, error) { } // ReadEntryBundle retrieves the Nth entries bundle for a log of the given size. -func (s *Storage) ReadEntryBundle(_ context.Context, index, logSize uint64) ([]byte, error) { - return os.ReadFile(filepath.Join(s.path, s.entriesPath(index, logSize))) +func (s *Storage) ReadEntryBundle(_ context.Context, index uint64, p uint8) ([]byte, error) { + return os.ReadFile(filepath.Join(s.path, s.entriesPath(index, p))) } -func (s *Storage) ReadTile(_ context.Context, level, index, logSize uint64) ([]byte, error) { - return os.ReadFile(filepath.Join(s.path, layout.TilePath(level, index, logSize))) +func (s *Storage) ReadTile(_ context.Context, level, index uint64, p uint8) ([]byte, error) { + return os.ReadFile(filepath.Join(s.path, layout.TilePath(level, index, p))) } // sequenceBatch writes the entries from the provided batch into the entry bundle files of the log. @@ -194,7 +194,7 @@ func (s *Storage) sequenceBatch(ctx context.Context, entries []*tessera.Entry) e bundleIndex, entriesInBundle := seq/uint64(256), seq%uint64(256) if entriesInBundle > 0 { // If the latest bundle is partial, we need to read the data it contains in for our newer, larger, bundle. - part, err := s.ReadEntryBundle(ctx, bundleIndex, s.curSize) + part, err := s.ReadEntryBundle(ctx, bundleIndex, uint8(s.curSize%layout.EntryBundleWidth)) if err != nil { return err } @@ -203,7 +203,7 @@ func (s *Storage) sequenceBatch(ctx context.Context, entries []*tessera.Entry) e } } writeBundle := func(bundleIndex uint64) error { - bf := filepath.Join(s.path, s.entriesPath(bundleIndex, newSize)) + bf := filepath.Join(s.path, s.entriesPath(bundleIndex, uint8(newSize%layout.EntryBundleWidth))) if err := os.MkdirAll(filepath.Dir(bf), dirPerm); err != nil { return fmt.Errorf("failed to make entries directory structure: %w", err) } @@ -287,7 +287,7 @@ func (s *Storage) doIntegrate(ctx context.Context, fromSeq uint64, entries []sto func (s *Storage) readTiles(ctx context.Context, tileIDs []storage.TileID, treeSize uint64) ([]*api.HashTile, error) { r := make([]*api.HashTile, 0, len(tileIDs)) for _, id := range tileIDs { - t, err := s.readTile(ctx, id.Level, id.Index, treeSize) + t, err := s.readTile(ctx, id.Level, id.Index, layout.PartialTileSize(id.Level, id.Index, treeSize)) if err != nil { return nil, err } @@ -299,8 +299,8 @@ func (s *Storage) readTiles(ctx context.Context, tileIDs []storage.TileID, treeS // readTile returns the parsed tile at the given tile-level and tile-index. // If no complete tile exists at that location, it will attempt to find a // partial tile for the given tree size at that location. -func (s *Storage) readTile(ctx context.Context, level, index, logSize uint64) (*api.HashTile, error) { - t, err := s.ReadTile(ctx, level, index, logSize) +func (s *Storage) readTile(ctx context.Context, level, index uint64, p uint8) (*api.HashTile, error) { + t, err := s.ReadTile(ctx, level, index, p) if err != nil { if errors.Is(err, os.ErrNotExist) { // We'll signal to higher levels that it wasn't found by retuning a nil for this tile. @@ -331,7 +331,7 @@ func (s *Storage) storeTile(_ context.Context, level, index, logSize uint64, til return fmt.Errorf("failed to marshal tile: %w", err) } - tPath := filepath.Join(s.path, layout.TilePath(level, index, logSize)) + tPath := filepath.Join(s.path, layout.TilePath(level, index, layout.PartialTileSize(level, index, logSize))) tDir := filepath.Dir(tPath) if err := os.MkdirAll(tDir, dirPerm); err != nil { return fmt.Errorf("failed to create directory %q: %w", tDir, err)