Skip to content

Commit

Permalink
improve log message when retrying tier2 request
Browse files Browse the repository at this point in the history
  • Loading branch information
sduchesneau authored and ArnaudBger committed Mar 29, 2024
1 parent fbe7d32 commit 53f4e32
Showing 1 changed file with 11 additions and 7 deletions.
18 changes: 11 additions & 7 deletions orchestrator/work/worker.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,12 +104,22 @@ func (w *RemoteWorker) Work(ctx context.Context, unit stage.Unit, workRange *blo
retryIdx := 0
startTime := time.Now()
maxRetries := 720 //TODO: make this configurable
var previousError error
err := derr.RetryContext(ctx, uint64(maxRetries), func(ctx context.Context) error {
w.logger.Info("launching remote worker",
zap.Int64("start_block_num", int64(request.StartBlockNum)),
zap.Uint64("stop_block_num", request.StopBlockNum),
zap.Uint32("stage", request.Stage),
zap.String("output_module", request.OutputModule),
zap.Int("attempt", retryIdx+1),
zap.NamedError("previous_error", previousError),
)

res = w.work(ctx, request, moduleNames, upstream)
err := res.Error
switch err.(type) {
case *RetryableErr:
logger.Debug("worker failed with retryable error", zap.Error(err))
previousError = err
retryIdx++
return err
default:
Expand Down Expand Up @@ -180,12 +190,6 @@ func (w *RemoteWorker) work(ctx context.Context, request *pbssinternal.ProcessRa
return &Result{Error: fmt.Errorf("unable to create grpc client: %w", err)}
}

w.logger.Info("launching remote worker",
zap.Int64("start_block_num", int64(request.StartBlockNum)),
zap.Uint64("stop_block_num", request.StopBlockNum),
zap.String("output_module", request.OutputModule),
)

stats := reqctx.ReqStats(ctx)
jobIdx := stats.RecordNewSubrequest(request.Stage, request.StartBlockNum, request.StopBlockNum)
defer stats.RecordEndSubrequest(jobIdx)
Expand Down

0 comments on commit 53f4e32

Please sign in to comment.