Integrate operates in terms of hashes (#437)

storage/aws/aws.go

@@ -428,7 +428,11 @@ func (s *Storage) integrate(ctx context.Context, fromSeq uint64, entries []stora
 	})
 
 	errG.Go(func() error {
-		newSize, root, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, entries)
+		lh := make([][]byte, len(entries))
+		for i, e := range entries {
+			lh[i] = e.LeafHash
+		}
+		newSize, root, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, lh)
 		if err != nil {
 			return fmt.Errorf("Integrate: %v", err)
 		}

storage/gcp/gcp.go

@@ -382,7 +382,11 @@ func (s *Storage) integrate(ctx context.Context, fromSeq uint64, entries []stora
 			return n, nil
 		}
 
-		newSize, root, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, entries)
+		lh := make([][]byte, len(entries))
+		for i, e := range entries {
+			lh[i] = e.LeafHash
+		}
+		newSize, root, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, lh)
 		if err != nil {
 			return fmt.Errorf("Integrate: %v", err)
 		}

storage/internal/integrate.go

@@ -36,9 +36,9 @@ type SequencedEntry struct {
 	LeafHash []byte
 }
 
-func Integrate(ctx context.Context, getTiles func(ctx context.Context, tileIDs []TileID, treeSize uint64) ([]*api.HashTile, error), fromSize uint64, entries []SequencedEntry) (newSize uint64, rootHash []byte, tiles map[TileID]*api.HashTile, err error) {
+func Integrate(ctx context.Context, getTiles func(ctx context.Context, tileIDs []TileID, treeSize uint64) ([]*api.HashTile, error), fromSize uint64, leafHashes [][]byte) (newSize uint64, rootHash []byte, tiles map[TileID]*api.HashTile, err error) {
 	tb := newTreeBuilder(getTiles)
-	return tb.integrate(ctx, fromSize, entries)
+	return tb.integrate(ctx, fromSize, leafHashes)
 }
 
 // getPopulatedTileFunc is the signature of a function which can return a fully populated tile for the given tile coords.
@@ -98,7 +98,7 @@ func (t *treeBuilder) newRange(ctx context.Context, treeSize uint64) (*compact.R
 	return t.rf.NewRange(0, treeSize, hashes)
 }
 
-func (t *treeBuilder) integrate(ctx context.Context, fromSize uint64, entries []SequencedEntry) (newSize uint64, rootHash []byte, tiles map[TileID]*api.HashTile, err error) {
+func (t *treeBuilder) integrate(ctx context.Context, fromSize uint64, leafHashes [][]byte) (newSize uint64, rootHash []byte, tiles map[TileID]*api.HashTile, err error) {
 	baseRange, err := t.newRange(ctx, fromSize)
 	if err != nil {
 		return 0, nil, nil, fmt.Errorf("failed to create range covering existing log: %w", err)
@@ -109,7 +109,7 @@ func (t *treeBuilder) integrate(ctx context.Context, fromSize uint64, entries []
 	if err != nil {
 		return 0, nil, nil, fmt.Errorf("invalid log state, unable to recalculate root: %w", err)
 	}
-	if len(entries) == 0 {
+	if len(leafHashes) == 0 {
 		klog.V(1).Infof("Nothing to do.")
 		// C2SP.org/log-tiles says all Merkle operations are those from RFC6962, we need to override
 		// the root of the empty tree to match (compact.Range will return an empty slice).
@@ -125,9 +125,9 @@ func (t *treeBuilder) integrate(ctx context.Context, fromSize uint64, entries []
 	newRange := t.rf.NewEmptyRange(fromSize)
 	tc := newTileWriteCache(fromSize, t.readCache.Get)
 	visitor := tc.Visitor(ctx)
-	for _, e := range entries {
+	for _, e := range leafHashes {
 		// Update range and set nodes
-		if err := newRange.Append(e.LeafHash, visitor); err != nil {
+		if err := newRange.Append(e, visitor); err != nil {
 			return 0, nil, nil, fmt.Errorf("newRange.Append(): %v", err)
 		}
 

storage/internal/integrate_test.go

@@ -130,14 +130,11 @@ func TestIntegrate(t *testing.T) {
 	seq := uint64(0)
 	for chunk := 0; chunk < numChunks; chunk++ {
 		oldSeq := seq
-		c := make([]SequencedEntry, chunkSize)
+		c := make([][]byte, chunkSize)
 		for i := range c {
 			leaf := []byte{byte(seq)}
 			entry := tessera.NewEntry(leaf)
-			c[i] = SequencedEntry{
-				BundleData: entry.MarshalBundleData(seq),
-				LeafHash:   entry.LeafHash(),
-			}
+			c[i] = entry.LeafHash()
 			if err := cr.Append(rfc6962.DefaultHasher.HashLeaf(leaf), nil); err != nil {
 				t.Fatalf("compact Append: %v", err)
 			}
@@ -173,14 +170,11 @@ func BenchmarkIntegrate(b *testing.B) {
 	seq := uint64(0)
 	for chunk := 0; chunk < b.N; chunk++ {
 		oldSeq := seq
-		c := make([]SequencedEntry, chunkSize)
+		c := make([][]byte, chunkSize)
 		for i := range c {
 			leaf := []byte{byte(seq)}
 			entry := tessera.NewEntry(leaf)
-			c[i] = SequencedEntry{
-				BundleData: entry.MarshalBundleData(seq),
-				LeafHash:   entry.LeafHash(),
-			}
+			c[i] = entry.LeafHash()
 			seq++
 		}
 		_, _, gotTiles, err := Integrate(ctx, m.getTiles, oldSeq, c)

storage/mysql/mysql.go

@@ -501,7 +501,11 @@ func (s *Storage) integrate(ctx context.Context, tx *sql.Tx, fromSeq uint64, ent
 		}
 	}
 
-	newSize, newRoot, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, sequencedEntries)
+	lh := make([][]byte, len(sequencedEntries))
+	for i, e := range sequencedEntries {
+		lh[i] = e.LeafHash
+	}
+	newSize, newRoot, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, lh)
 	if err != nil {
 		return fmt.Errorf("tb.Integrate: %v", err)
 	}

storage/posix/files.go

@@ -31,7 +31,7 @@ import (
 	"github.com/transparency-dev/trillian-tessera/api"
 	"github.com/transparency-dev/trillian-tessera/api/layout"
 	"github.com/transparency-dev/trillian-tessera/internal/options"
-	"github.com/transparency-dev/trillian-tessera/storage/internal"
+	storage "github.com/transparency-dev/trillian-tessera/storage/internal"
 	"k8s.io/klog/v2"
 )
 
@@ -219,17 +219,14 @@ func (s *Storage) sequenceBatch(ctx context.Context, entries []*tessera.Entry) e
 		return nil
 	}
 
-	seqEntries := make([]storage.SequencedEntry, 0, len(entries))
+	leafHashes := make([][]byte, 0, len(entries))
 	// Add new entries to the bundle
 	for i, e := range entries {
 		bundleData := e.MarshalBundleData(seq + uint64(i))
 		if _, err := currTile.Write(bundleData); err != nil {
 			return fmt.Errorf("failed to write entry %d to currTile: %v", i, err)
 		}
-		seqEntries = append(seqEntries, storage.SequencedEntry{
-			BundleData: bundleData,
-			LeafHash:   e.LeafHash(),
-		})
+		leafHashes = append(leafHashes, e.LeafHash())
 
 		entriesInBundle++
 		if entriesInBundle == layout.EntryBundleWidth {
@@ -258,15 +255,15 @@ func (s *Storage) sequenceBatch(ctx context.Context, entries []*tessera.Entry) e
 	}
 
 	// For simplicity, in-line the integration of these new entries into the Merkle structure too.
-	if err := s.doIntegrate(ctx, seq, seqEntries); err != nil {
+	if err := s.doIntegrate(ctx, seq, leafHashes); err != nil {
 		klog.Errorf("Integrate failed: %v", err)
 		return err
 	}
 	return nil
 }
 
-// doIntegrate handles integrating new entries into the log, and updating the tree state.
-func (s *Storage) doIntegrate(ctx context.Context, fromSeq uint64, entries []storage.SequencedEntry) error {
+// doIntegrate handles integrating new leaf hashes into the log, and updating the tree state.
+func (s *Storage) doIntegrate(ctx context.Context, fromSeq uint64, leafHashes [][]byte) error {
 	getTiles := func(ctx context.Context, tileIDs []storage.TileID, treeSize uint64) ([]*api.HashTile, error) {
 		n, err := s.readTiles(ctx, tileIDs, treeSize)
 		if err != nil {
@@ -275,7 +272,7 @@ func (s *Storage) doIntegrate(ctx context.Context, fromSeq uint64, entries []sto
 		return n, nil
 	}
 
-	newSize, newRoot, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, entries)
+	newSize, newRoot, tiles, err := storage.Integrate(ctx, getTiles, fromSeq, leafHashes)
 	if err != nil {
 		klog.Errorf("Integrate: %v", err)
 		return fmt.Errorf("Integrate: %v", err)