Skip to content

Commit

Permalink
fix: ptah-sh/ptah-server#239 better tracking of failed tasks
Browse files Browse the repository at this point in the history
  • Loading branch information
bohdan-shulha committed Oct 20, 2024
1 parent a525cc9 commit 3b775d2
Showing 1 changed file with 11 additions and 5 deletions.
16 changes: 11 additions & 5 deletions internal/app/ptah-agent/service_monitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ func (e *taskExecutor) monitorDaemonServiceLaunch(ctx context.Context, service *
timeout := time.After(time.Duration(5) * time.Minute)

successfullChecks := 0
lastFailedTasks := 0
lastTasks := 0

for {
select {
Expand Down Expand Up @@ -89,6 +91,12 @@ func (e *taskExecutor) monitorDaemonServiceLaunch(ctx context.Context, service *
return nil
}

if len(tasks) == lastTasks {
continue
}

lastTasks = len(tasks)

failedTasks := 0
var lastErr string
for _, t := range tasks {
Expand All @@ -98,13 +106,11 @@ func (e *taskExecutor) monitorDaemonServiceLaunch(ctx context.Context, service *
}
}

if failedTasks > 5 {
return errors.Errorf("task failed: %s", lastErr)
}

if failedTasks > 0 {
if failedTasks > lastFailedTasks {
log.Debug("service has failed tasks", "failed_tasks", failedTasks, "last_error", lastErr)

lastFailedTasks = failedTasks

continue
}

Expand Down

0 comments on commit 3b775d2

Please sign in to comment.