From 53f4e326c346d7881b287cde1c21e9efb8dc2a8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Duchesneau?= Date: Mon, 25 Mar 2024 09:01:36 -0400 Subject: [PATCH] improve log message when retrying tier2 request --- orchestrator/work/worker.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/orchestrator/work/worker.go b/orchestrator/work/worker.go index c11d7efbd..8898c964e 100644 --- a/orchestrator/work/worker.go +++ b/orchestrator/work/worker.go @@ -104,12 +104,22 @@ func (w *RemoteWorker) Work(ctx context.Context, unit stage.Unit, workRange *blo retryIdx := 0 startTime := time.Now() maxRetries := 720 //TODO: make this configurable + var previousError error err := derr.RetryContext(ctx, uint64(maxRetries), func(ctx context.Context) error { + w.logger.Info("launching remote worker", + zap.Int64("start_block_num", int64(request.StartBlockNum)), + zap.Uint64("stop_block_num", request.StopBlockNum), + zap.Uint32("stage", request.Stage), + zap.String("output_module", request.OutputModule), + zap.Int("attempt", retryIdx+1), + zap.NamedError("previous_error", previousError), + ) + res = w.work(ctx, request, moduleNames, upstream) err := res.Error switch err.(type) { case *RetryableErr: - logger.Debug("worker failed with retryable error", zap.Error(err)) + previousError = err retryIdx++ return err default: @@ -180,12 +190,6 @@ func (w *RemoteWorker) work(ctx context.Context, request *pbssinternal.ProcessRa return &Result{Error: fmt.Errorf("unable to create grpc client: %w", err)} } - w.logger.Info("launching remote worker", - zap.Int64("start_block_num", int64(request.StartBlockNum)), - zap.Uint64("stop_block_num", request.StopBlockNum), - zap.String("output_module", request.OutputModule), - ) - stats := reqctx.ReqStats(ctx) jobIdx := stats.RecordNewSubrequest(request.Stage, request.StartBlockNum, request.StopBlockNum) defer stats.RecordEndSubrequest(jobIdx)