From 833607421bdaf51606daee996f96b6196f2917cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Duchesneau?= Date: Fri, 22 Mar 2024 15:48:23 -0400 Subject: [PATCH] when scanning store files, find old partial files with a traceID and delete them on sight (they are leftovers, possibly slowing down next runs) --- storage/store/config.go | 7 +++++++ storage/store/filename.go | 18 ++++++++++-------- storage/store/filename_test.go | 6 ++++++ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/storage/store/config.go b/storage/store/config.go index 5cb3a699e..6f9bb7d15 100644 --- a/storage/store/config.go +++ b/storage/store/config.go @@ -147,6 +147,13 @@ func (c *Config) ListSnapshotFiles(ctx context.Context, below uint64) (files []* return nil } + if fileInfo.WithTraceID { + if err := c.objStore.DeleteObject(ctx, filename); err != nil { // clean up all old files with traceID in them, they will only slow every next run + logger.Warn("cannot delete old partial file with trace_id", zap.String("filename", filename), zap.Error(err)) + } + return nil + } + if fileInfo.Range.StartBlock >= below { return dstore.StopIteration } diff --git a/storage/store/filename.go b/storage/store/filename.go index b4294f782..a4a03fd35 100644 --- a/storage/store/filename.go +++ b/storage/store/filename.go @@ -35,10 +35,11 @@ func (f FileInfos) String() string { } type FileInfo struct { - ModuleName string - Filename string - Range *block.Range - Partial bool + ModuleName string + Filename string + Range *block.Range + Partial bool + WithTraceID bool } func NewCompleteFileInfo(moduleName string, moduleInitialBlock uint64, exclusiveEndBlock uint64) *FileInfo { @@ -70,10 +71,11 @@ func parseFileName(moduleName, filename string) (*FileInfo, bool) { } return &FileInfo{ - ModuleName: moduleName, - Filename: filename, - Range: block.NewRange(uint64(mustAtoi(res[0][2])), uint64(mustAtoi(res[0][1]))), - Partial: res[0][4] == "partial", + ModuleName: moduleName, + Filename: filename, + Range: block.NewRange(uint64(mustAtoi(res[0][2])), uint64(mustAtoi(res[0][1]))), + Partial: res[0][4] == "partial", + WithTraceID: res[0][3] != "", }, true } diff --git a/storage/store/filename_test.go b/storage/store/filename_test.go index 896d3192c..87cf864ca 100644 --- a/storage/store/filename_test.go +++ b/storage/store/filename_test.go @@ -27,6 +27,12 @@ func Test_parseFileName(t *testing.T) { &FileInfo{ModuleName: "test", Filename: "0000000100-0000000000.kv", Range: block.NewRange(0, 100), Partial: false}, true, }, + { + "old-partial-with-trace-id", + fmt.Sprintf("%010d-%010d.deadbeefdeadbeefdeadbeefdeadbeef.partial", 100, 0), + &FileInfo{ModuleName: "test", Filename: "0000000100-0000000000.deadbeefdeadbeefdeadbeefdeadbeef.partial", Range: block.NewRange(0, 100), Partial: true, WithTraceID: true}, + true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) {