Add support for Static CT API entry bundle paths

api/layout/paths.go

@@ -37,25 +37,24 @@ func EntriesPathForLogIndex(seq, logSize uint64) string {
 	return EntriesPath(seq/256, logSize)
 }
 
-// EntriesPath returns the local path for the nth entry bundle. p denotes the partial
-// tile size, or 0 if the tile is complete.
-func EntriesPath(n, logSize uint64) string {
+// NWithSuffix returns a tiles-spec "N" path, with a partial suffix if applicable.
+func NWithSuffix(l, n, logSize uint64) string {
 	suffix := ""
-	if p := partialTileSize(0, n, logSize); p > 0 {
+	if p := partialTileSize(l, n, logSize); p > 0 {
 		suffix = fmt.Sprintf(".p/%d", p)
 	}
-	return fmt.Sprintf("tile/entries%s%s", fmtN(n), suffix)
+	return fmt.Sprintf("%s%s", fmtN(n), suffix)
+}
+
+// EntriesPath returns the local path for the nth entry bundle. p denotes the partial
+// tile size, or 0 if the tile is complete.
+func EntriesPath(n, logSize uint64) string {
+	return fmt.Sprintf("tile/entries%s", NWithSuffix(0, n, logSize))
 }
 
 // TilePath builds the path to the subtree tile with the given level and index in tile space.
 func TilePath(tileLevel, tileIndex, logSize uint64) string {
-	suffix := ""
-	p := partialTileSize(tileLevel, tileIndex, logSize)
-	if p > 0 {
-		suffix = fmt.Sprintf(".p/%d", p)
-	}
-
-	return fmt.Sprintf("tile/%d%s%s", tileLevel, fmtN(tileIndex), suffix)
+	return fmt.Sprintf("tile/%d%s", tileLevel, NWithSuffix(tileLevel, tileIndex, logSize))
 }
 
 // fmtN returns the "N" part of a Tiles-spec path.

ct_only.go

@@ -16,7 +16,9 @@ package tessera
 
 import (
 	"context"
+	"fmt"
 
+	"github.com/transparency-dev/trillian-tessera/api/layout"
 	"github.com/transparency-dev/trillian-tessera/ctonly"
 )
 
@@ -56,3 +58,12 @@ func convertCTEntry(e *ctonly.Entry) *Entry {
 
 	return r
 }
+
+// WithCTLayout instructs the underlying storage to use a Static CT API compatible scheme for layout.
+func WithCTLayout() func(*StorageOptions) {
+	return func(opts *StorageOptions) {
+		opts.EntriesPath = func(n, logSize uint64) string {
+			return fmt.Sprintf("tile/data%s", layout.NWithSuffix(0, n, logSize))
+		}
+	}
+}

log.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	f_log "github.com/transparency-dev/formats/log"
+	"github.com/transparency-dev/trillian-tessera/api/layout"
 	"golang.org/x/mod/sumdb/note"
 )
 
@@ -41,6 +42,9 @@ type NewCPFunc func(size uint64, hash []byte) ([]byte, error)
 // ParseCPFunc is the signature of a function which knows how to verify and parse checkpoints.
 type ParseCPFunc func(raw []byte) (*f_log.Checkpoint, error)
 
+// EntriesPathFunc is the signature of a function which knows how to format entry bundle paths.
+type EntriesPathFunc func(n, logSize uint64) string
+
 // StorageOptions holds optional settings for all storage implementations.
 type StorageOptions struct {
 	NewCP   NewCPFunc
@@ -50,13 +54,16 @@ type StorageOptions struct {
 	BatchMaxSize uint
 
 	PushbackMaxOutstanding uint
+
+	EntriesPath EntriesPathFunc
 }
 
 // ResolveStorageOptions turns a variadic array of storage options into a StorageOptions instance.
 func ResolveStorageOptions(opts ...func(*StorageOptions)) *StorageOptions {
 	defaults := &StorageOptions{
 		BatchMaxSize: DefaultBatchMaxSize,
 		BatchMaxAge:  DefaultBatchMaxAge,
+		EntriesPath:  layout.EntriesPath,
 	}
 	for _, opt := range opts {
 		opt(defaults)

storage/gcp/gcp.go

@@ -70,7 +70,8 @@ type Storage struct {
 	projectID string
 	bucket    string
 
-	newCP tessera.NewCPFunc
+	newCP       tessera.NewCPFunc
+	entriesPath tessera.EntriesPathFunc
 
 	sequencer sequencer
 	objStore  objStore
@@ -129,12 +130,13 @@ func New(ctx context.Context, cfg Config, opts ...func(*tessera.StorageOptions))
 	}
 
 	r := &Storage{
-		gcsClient: c,
-		projectID: cfg.ProjectID,
-		bucket:    cfg.Bucket,
-		objStore:  gcsStorage,
-		sequencer: seq,
-		newCP:     opt.NewCP,
+		gcsClient:   c,
+		projectID:   cfg.ProjectID,
+		bucket:      cfg.Bucket,
+		objStore:    gcsStorage,
+		sequencer:   seq,
+		newCP:       opt.NewCP,
+		entriesPath: opt.EntriesPath,
 	}
 	r.queue = storage.NewQueue(ctx, opt.BatchMaxAge, opt.BatchMaxSize, r.sequencer.assignEntries)
 
@@ -229,7 +231,7 @@ func (s *Storage) getTiles(ctx context.Context, tileIDs []storage.TileID, logSiz
 //
 // Returns a wrapped os.ErrNotExist if the bundle does not exist.
 func (s *Storage) getEntryBundle(ctx context.Context, bundleIndex uint64, logSize uint64) ([]byte, error) {
-	objName := layout.EntriesPath(bundleIndex, logSize)
+	objName := s.entriesPath(bundleIndex, logSize)
 	data, _, err := s.objStore.getObject(ctx, objName)
 	if err != nil {
 		if errors.Is(err, gcs.ErrObjectNotExist) {
@@ -245,7 +247,7 @@ func (s *Storage) getEntryBundle(ctx context.Context, bundleIndex uint64, logSiz
 
 // setEntryBundle idempotently stores the serialised entry bundle at the location implied by the bundleIndex and treeSize.
 func (s *Storage) setEntryBundle(ctx context.Context, bundleIndex uint64, logSize uint64, bundleRaw []byte) error {
-	objName := layout.EntriesPath(bundleIndex, logSize)
+	objName := s.entriesPath(bundleIndex, logSize)
 	// Note that setObject does an idempotent interpretation of DoesNotExist - it only
 	// returns an error if the named object exists _and_ contains different data to what's
 	// passed in here.

storage/gcp/gcp_test.go

@@ -264,7 +264,8 @@ func TestBundleRoundtrip(t *testing.T) {
 	ctx := context.Background()
 	m := newMemObjStore()
 	s := &Storage{
-		objStore: m,
+		objStore:    m,
+		entriesPath: layout.EntriesPath,
 	}
 
 	for _, test := range []struct {

storage/posix/files.go

@@ -49,6 +49,8 @@ type Storage struct {
 
 	curSize uint64
 	newCP   tessera.NewCPFunc
+
+	entriesPath tessera.EntriesPathFunc
 }
 
 // NewTreeFunc is the signature of a function which receives information about newly integrated trees.
@@ -66,10 +68,11 @@ func New(ctx context.Context, path string, curTree func() (uint64, []byte, error
 	opt := tessera.ResolveStorageOptions(opts...)
 
 	r := &Storage{
-		path:    path,
-		curSize: curSize,
-		curTree: curTree,
-		newCP:   opt.NewCP,
+		path:        path,
+		curSize:     curSize,
+		curTree:     curTree,
+		newCP:       opt.NewCP,
+		entriesPath: opt.EntriesPath,
 	}
 	r.queue = storage.NewQueue(ctx, opt.BatchMaxAge, opt.BatchMaxSize, r.sequenceBatch)
 
@@ -124,7 +127,7 @@ func (s *Storage) Add(ctx context.Context, e *tessera.Entry) (uint64, error) {
 
 // GetEntryBundle retrieves the Nth entries bundle for a log of the given size.
 func (s *Storage) GetEntryBundle(ctx context.Context, index, logSize uint64) ([]byte, error) {
-	return os.ReadFile(filepath.Join(s.path, layout.EntriesPath(index, logSize)))
+	return os.ReadFile(filepath.Join(s.path, s.entriesPath(index, logSize)))
 }
 
 // sequenceBatch writes the entries from the provided batch into the entry bundle files of the log.
@@ -173,7 +176,7 @@ func (s *Storage) sequenceBatch(ctx context.Context, entries []*tessera.Entry) e
 		}
 	}
 	writeBundle := func(bundleIndex uint64) error {
-		bf := filepath.Join(s.path, layout.EntriesPath(bundleIndex, newSize))
+		bf := filepath.Join(s.path, s.entriesPath(bundleIndex, newSize))
 		if err := os.MkdirAll(filepath.Dir(bf), dirPerm); err != nil {
 			return fmt.Errorf("failed to make entries directory structure: %w", err)
 		}