Add TODO

personalities/sctfe/ct_server_gcp/main.go

@@ -290,6 +290,7 @@ func newGCPStorage(ctx context.Context, vCfg *sctfe.ValidatedLogConfig, signer n
 		return nil, fmt.Errorf("Failed to initialize GCP issuer storage: %v", err)
 	}
 
+	// TODO: repalce with a global dedup storage for GCP
 	dedupStorage, err := bbolt.NewStorage(*dedupPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize BBolt deduplication database: %v", err)

personalities/sctfe/modules/dedup/dedup.go

@@ -48,13 +48,13 @@ type LocalDedupStorage interface {
 
 type LocalBEDedup struct {
 	DedupStorage
-	LogSize func() (uint64, error) // returns the largest contiguous idx Add has successfully been called with
+	logSize func() (uint64, error) // returns the largest contiguous idx Add has successfully been called with
 	fetcher client.Fetcher
 }
 
 // NewLocalBestEffortDedup instantiates a local dedup storage and kicks off a synchronisation routine in the background.
 func NewLocalBestEffortDedup(ctx context.Context, lds LocalDedupStorage, t time.Duration, f client.Fetcher, parseBundle func([]byte, uint64) ([]LeafIdx, error)) *LocalBEDedup {
-	ret := &LocalBEDedup{DedupStorage: lds, LogSize: lds.LogSize, fetcher: f}
+	ret := &LocalBEDedup{DedupStorage: lds, logSize: lds.LogSize, fetcher: f}
 	go func() {
 		tck := time.NewTicker(t)
 		defer tck.Stop()
@@ -87,7 +87,7 @@ func (d *LocalBEDedup) sync(ctx context.Context, parseBundle func([]byte, uint64
 	if err != nil {
 		return fmt.Errorf("invalid checkpoint - can't extract size: %v", err)
 	}
-	oldSize, err := d.LogSize()
+	oldSize, err := d.logSize()
 	if err != nil {
 		return fmt.Errorf("OldSize(): %v", err)
 	}

personalities/sctfe/storage/bbolt/dedup.go

@@ -88,8 +88,7 @@ func NewStorage(path string) (*Storage, error) {
 
 // Add inserts entries in the dedup bucket and updates the size bucket if need be.
 //
-// If an entry is already stored under a given key, Add does not update it, even if the new value
-// is different.
+// If an entry is already stored under a given key, Add only updates it if the new value is smaller.
 // The context is here for consistency with interfaces, but isn't used by BBolt.
 func (s *Storage) Add(_ context.Context, lidxs []dedup.LeafIdx) error {
 	for _, lidx := range lidxs {
@@ -102,10 +101,9 @@ func (s *Storage) Add(_ context.Context, lidxs []dedup.LeafIdx) error {
 			}
 			size := btoi(sizeB)
 
-			if old := db.Get(lidx.LeafID); old != nil {
-				klog.V(3).Infof("Add(): bucket %q already contains an entry for %q, not updating", dedupBucket, hex.EncodeToString(lidx.LeafID))
-			}
-			if err := db.Put(lidx.LeafID, itob(lidx.Idx)); err != nil {
+			if old := db.Get(lidx.LeafID); old != nil && btoi(old) <= lidx.Idx {
+				klog.V(3).Infof("Add(): bucket %q already contains a smaller index %d < %d for entry %q, not updating", dedupBucket, btoi(old), lidx.Idx, hex.EncodeToString(lidx.LeafID))
+			} else if err := db.Put(lidx.LeafID, itob(lidx.Idx)); err != nil {
 				return err
 			}
 			// size is a length, kv.V an index, so if they're equal,