From 60243a5f7f9b454d190471e264e336b992697959 Mon Sep 17 00:00:00 2001 From: Al Cutter Date: Thu, 9 Jan 2025 16:32:47 +0000 Subject: [PATCH] Integrate operates in terms of hashes (#437) --- storage/aws/aws.go | 6 +++++- storage/gcp/gcp.go | 6 +++++- storage/internal/integrate.go | 12 ++++++------ storage/internal/integrate_test.go | 14 ++++---------- storage/mysql/mysql.go | 6 +++++- storage/posix/files.go | 17 +++++++---------- 6 files changed, 32 insertions(+), 29 deletions(-) diff --git a/storage/aws/aws.go b/storage/aws/aws.go index b01a40a3..49b2d7a6 100644 --- a/storage/aws/aws.go +++ b/storage/aws/aws.go @@ -428,7 +428,11 @@ func (s *Storage) integrate(ctx context.Context, fromSeq uint64, entries []stora }) errG.Go(func() error { - newSize, root, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, entries) + lh := make([][]byte, len(entries)) + for i, e := range entries { + lh[i] = e.LeafHash + } + newSize, root, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, lh) if err != nil { return fmt.Errorf("Integrate: %v", err) } diff --git a/storage/gcp/gcp.go b/storage/gcp/gcp.go index 1f5ad5df..7bf1287a 100644 --- a/storage/gcp/gcp.go +++ b/storage/gcp/gcp.go @@ -382,7 +382,11 @@ func (s *Storage) integrate(ctx context.Context, fromSeq uint64, entries []stora return n, nil } - newSize, root, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, entries) + lh := make([][]byte, len(entries)) + for i, e := range entries { + lh[i] = e.LeafHash + } + newSize, root, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, lh) if err != nil { return fmt.Errorf("Integrate: %v", err) } diff --git a/storage/internal/integrate.go b/storage/internal/integrate.go index 0f83de68..45b573ef 100644 --- a/storage/internal/integrate.go +++ b/storage/internal/integrate.go @@ -36,9 +36,9 @@ type SequencedEntry struct { LeafHash []byte } -func Integrate(ctx context.Context, getTiles func(ctx context.Context, tileIDs []TileID, treeSize uint64) ([]*api.HashTile, error), fromSize uint64, entries []SequencedEntry) (newSize uint64, rootHash []byte, tiles map[TileID]*api.HashTile, err error) { +func Integrate(ctx context.Context, getTiles func(ctx context.Context, tileIDs []TileID, treeSize uint64) ([]*api.HashTile, error), fromSize uint64, leafHashes [][]byte) (newSize uint64, rootHash []byte, tiles map[TileID]*api.HashTile, err error) { tb := newTreeBuilder(getTiles) - return tb.integrate(ctx, fromSize, entries) + return tb.integrate(ctx, fromSize, leafHashes) } // getPopulatedTileFunc is the signature of a function which can return a fully populated tile for the given tile coords. @@ -98,7 +98,7 @@ func (t *treeBuilder) newRange(ctx context.Context, treeSize uint64) (*compact.R return t.rf.NewRange(0, treeSize, hashes) } -func (t *treeBuilder) integrate(ctx context.Context, fromSize uint64, entries []SequencedEntry) (newSize uint64, rootHash []byte, tiles map[TileID]*api.HashTile, err error) { +func (t *treeBuilder) integrate(ctx context.Context, fromSize uint64, leafHashes [][]byte) (newSize uint64, rootHash []byte, tiles map[TileID]*api.HashTile, err error) { baseRange, err := t.newRange(ctx, fromSize) if err != nil { return 0, nil, nil, fmt.Errorf("failed to create range covering existing log: %w", err) @@ -109,7 +109,7 @@ func (t *treeBuilder) integrate(ctx context.Context, fromSize uint64, entries [] if err != nil { return 0, nil, nil, fmt.Errorf("invalid log state, unable to recalculate root: %w", err) } - if len(entries) == 0 { + if len(leafHashes) == 0 { klog.V(1).Infof("Nothing to do.") // C2SP.org/log-tiles says all Merkle operations are those from RFC6962, we need to override // the root of the empty tree to match (compact.Range will return an empty slice). @@ -125,9 +125,9 @@ func (t *treeBuilder) integrate(ctx context.Context, fromSize uint64, entries [] newRange := t.rf.NewEmptyRange(fromSize) tc := newTileWriteCache(fromSize, t.readCache.Get) visitor := tc.Visitor(ctx) - for _, e := range entries { + for _, e := range leafHashes { // Update range and set nodes - if err := newRange.Append(e.LeafHash, visitor); err != nil { + if err := newRange.Append(e, visitor); err != nil { return 0, nil, nil, fmt.Errorf("newRange.Append(): %v", err) } diff --git a/storage/internal/integrate_test.go b/storage/internal/integrate_test.go index 458b0e1b..54da7998 100644 --- a/storage/internal/integrate_test.go +++ b/storage/internal/integrate_test.go @@ -130,14 +130,11 @@ func TestIntegrate(t *testing.T) { seq := uint64(0) for chunk := 0; chunk < numChunks; chunk++ { oldSeq := seq - c := make([]SequencedEntry, chunkSize) + c := make([][]byte, chunkSize) for i := range c { leaf := []byte{byte(seq)} entry := tessera.NewEntry(leaf) - c[i] = SequencedEntry{ - BundleData: entry.MarshalBundleData(seq), - LeafHash: entry.LeafHash(), - } + c[i] = entry.LeafHash() if err := cr.Append(rfc6962.DefaultHasher.HashLeaf(leaf), nil); err != nil { t.Fatalf("compact Append: %v", err) } @@ -173,14 +170,11 @@ func BenchmarkIntegrate(b *testing.B) { seq := uint64(0) for chunk := 0; chunk < b.N; chunk++ { oldSeq := seq - c := make([]SequencedEntry, chunkSize) + c := make([][]byte, chunkSize) for i := range c { leaf := []byte{byte(seq)} entry := tessera.NewEntry(leaf) - c[i] = SequencedEntry{ - BundleData: entry.MarshalBundleData(seq), - LeafHash: entry.LeafHash(), - } + c[i] = entry.LeafHash() seq++ } _, _, gotTiles, err := Integrate(ctx, m.getTiles, oldSeq, c) diff --git a/storage/mysql/mysql.go b/storage/mysql/mysql.go index 0c79e1d6..6096d4a8 100644 --- a/storage/mysql/mysql.go +++ b/storage/mysql/mysql.go @@ -501,7 +501,11 @@ func (s *Storage) integrate(ctx context.Context, tx *sql.Tx, fromSeq uint64, ent } } - newSize, newRoot, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, sequencedEntries) + lh := make([][]byte, len(sequencedEntries)) + for i, e := range sequencedEntries { + lh[i] = e.LeafHash + } + newSize, newRoot, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, lh) if err != nil { return fmt.Errorf("tb.Integrate: %v", err) } diff --git a/storage/posix/files.go b/storage/posix/files.go index 63ec6490..50e030ac 100644 --- a/storage/posix/files.go +++ b/storage/posix/files.go @@ -31,7 +31,7 @@ import ( "github.com/transparency-dev/trillian-tessera/api" "github.com/transparency-dev/trillian-tessera/api/layout" "github.com/transparency-dev/trillian-tessera/internal/options" - "github.com/transparency-dev/trillian-tessera/storage/internal" + storage "github.com/transparency-dev/trillian-tessera/storage/internal" "k8s.io/klog/v2" ) @@ -219,17 +219,14 @@ func (s *Storage) sequenceBatch(ctx context.Context, entries []*tessera.Entry) e return nil } - seqEntries := make([]storage.SequencedEntry, 0, len(entries)) + leafHashes := make([][]byte, 0, len(entries)) // Add new entries to the bundle for i, e := range entries { bundleData := e.MarshalBundleData(seq + uint64(i)) if _, err := currTile.Write(bundleData); err != nil { return fmt.Errorf("failed to write entry %d to currTile: %v", i, err) } - seqEntries = append(seqEntries, storage.SequencedEntry{ - BundleData: bundleData, - LeafHash: e.LeafHash(), - }) + leafHashes = append(leafHashes, e.LeafHash()) entriesInBundle++ if entriesInBundle == layout.EntryBundleWidth { @@ -258,15 +255,15 @@ func (s *Storage) sequenceBatch(ctx context.Context, entries []*tessera.Entry) e } // For simplicity, in-line the integration of these new entries into the Merkle structure too. - if err := s.doIntegrate(ctx, seq, seqEntries); err != nil { + if err := s.doIntegrate(ctx, seq, leafHashes); err != nil { klog.Errorf("Integrate failed: %v", err) return err } return nil } -// doIntegrate handles integrating new entries into the log, and updating the tree state. -func (s *Storage) doIntegrate(ctx context.Context, fromSeq uint64, entries []storage.SequencedEntry) error { +// doIntegrate handles integrating new leaf hashes into the log, and updating the tree state. +func (s *Storage) doIntegrate(ctx context.Context, fromSeq uint64, leafHashes [][]byte) error { getTiles := func(ctx context.Context, tileIDs []storage.TileID, treeSize uint64) ([]*api.HashTile, error) { n, err := s.readTiles(ctx, tileIDs, treeSize) if err != nil { @@ -275,7 +272,7 @@ func (s *Storage) doIntegrate(ctx context.Context, fromSeq uint64, entries []sto return n, nil } - newSize, newRoot, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, entries) + newSize, newRoot, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, leafHashes) if err != nil { klog.Errorf("Integrate: %v", err) return fmt.Errorf("Integrate: %v", err)