feat: handle interrupt signal, add --timeout option

.github/workflows/test.yml

@@ -32,7 +32,7 @@ jobs:
         with:
           go-version-file: 'go.mod'
       - name: Run unit tests
-        run: go test -v ./...
+        run: go test -timeout 30s -v ./...
   examples:
     strategy:
       matrix:

Makefile

@@ -4,9 +4,14 @@ DATE    ?= $(shell date +%FT%T%z)
 VERSION ?= $(shell git describe --tags --always --dirty --match=v* 2> /dev/null || \
 			cat $(CURDIR)/.version 2> /dev/null || echo v0)
 
+ifeq ($(shell $(GO) env GOOS),windows)
+    EXT = ".exe"
+else
+    EXT = ""
+endif
 
 BIN_DIR              = $(CURDIR)/bin
-CLI_BIN              = mdtest
+CLI_BIN              = mdtest$(EXT)
 
 .PHONY: build
 build: $(BIN_DIR)

cmd/cmd_internal_test.go

@@ -32,12 +32,12 @@ func TestRoot_testdata(t *testing.T) {
 		},
 		{
 			testPath: "../testdata",
-			exitCode: 3,
+			exitCode: 4,
 		},
 	} {
 		test := test
 		t.Run(test.testPath, func(t *testing.T) {
-			rootCmd.SetArgs([]string{test.testPath})
+			rootCmd.SetArgs([]string{"--timeout", "1s", test.testPath})
 			exitCode := Execute()
 			assert.Equal(t, test.exitCode, exitCode)
 		})

cmd/root.go

@@ -2,14 +2,17 @@ package cmd
 
 import (
 	"errors"
+	"fmt"
 	"runtime"
+	"time"
 
 	"github.com/UpCloudLtd/mdtest/testrun"
 	"github.com/spf13/cobra"
 )
 
 var (
-	numberOfJobs int
+	numberOfJobs  int
+	timeoutString string
 
 	rootCmd = &cobra.Command{
 		Use:   "mdtest [flags] path ...",
@@ -21,18 +24,24 @@ var (
 
 func init() {
 	rootCmd.Flags().IntVarP(&numberOfJobs, "jobs", "j", runtime.NumCPU()*2, "number of jobs to use for executing tests in parallel")
+	rootCmd.Flags().StringVar(&timeoutString, "timeout", "", "timeout for the test run as a `duration` string, e.g., 1s, 1m, 1h")
 	rootCmd.RunE = func(cmd *cobra.Command, args []string) error {
 		cmd.SilenceUsage = true
 		cmd.SilenceErrors = true
 
+		timeout, err := time.ParseDuration(timeoutString)
+		if err != nil && timeoutString != "" {
+			return fmt.Errorf("failed to parse timeout: %w", err)
+		}
+
 		params := testrun.RunParameters{
 			NumberOfJobs: numberOfJobs,
 			OutputTarget: rootCmd.OutOrStdout(),
+			Timeout:      timeout,
 		}
 
 		res := testrun.Execute(args, params)
-		err := testrun.NewRunError(res)
-		if err != nil {
+		if err := testrun.NewRunError(res); err != nil {
 			return err
 		}
 

testcase/envstep.go

@@ -1,14 +1,15 @@
 package testcase
 
 import (
+	"context"
 	"strings"
 )
 
 type envStep struct {
 	envUpdates []string
 }
 
-func (s envStep) Execute(t *testStatus) StepResult {
+func (s envStep) Execute(_ context.Context, t *testStatus) StepResult {
 	t.Env = append(t.Env, s.envUpdates...)
 
 	return StepResult{

testcase/filenamestep.go

@@ -1,6 +1,7 @@
 package testcase
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -13,7 +14,7 @@ type filenameStep struct {
 	filename string
 }
 
-func (s filenameStep) Execute(t *testStatus) StepResult {
+func (s filenameStep) Execute(_ context.Context, t *testStatus) StepResult {
 	target := filepath.Join(getTestDirPath(t.Params), s.filename)
 	dir := filepath.Dir(target)
 	if _, err := os.Stat(dir); errors.Is(err, fs.ErrNotExist) {

testcase/shstep.go

@@ -1,10 +1,13 @@
 package testcase
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"os/exec"
 	"strconv"
+
+	"github.com/UpCloudLtd/mdtest/utils"
 )
 
 type shStep struct {
@@ -16,10 +19,15 @@ func unexpectedExitCode(expected, got int) error {
 	return fmt.Errorf("expected exit code %d, got %d", expected, got)
 }
 
-func (s shStep) Execute(t *testStatus) StepResult {
-	cmd := exec.Command("sh", "-xec", s.script) //nolint:gosec // Here we trust that the user knows what their tests do
+func (s shStep) Execute(ctx context.Context, t *testStatus) StepResult {
+	cmd := exec.CommandContext(ctx, "sh", "-xec", s.script) //nolint:gosec // Here we trust that the user knows what their tests do
+	cmd.Cancel = func() error {
+		return utils.Terminate(cmd)
+	}
 	cmd.Dir = getTestDirPath(t.Params)
 	cmd.Env = t.Env
+	utils.UseProcessGroup(cmd)
+
 	output, err := cmd.CombinedOutput()
 	if err != nil {
 		var exit *exec.ExitError
@@ -30,7 +38,17 @@ func (s shStep) Execute(t *testStatus) StepResult {
 				Error:   fmt.Errorf("unexpected error (%w)", err),
 				Output:  string(output),
 			}
-		} else if got := exit.ExitCode(); got != s.exitCode {
+		}
+		got := exit.ExitCode()
+
+		if ctxErr := ctx.Err(); got == -1 && ctxErr != nil {
+			return StepResult{
+				Success: false,
+				Error:   utils.GetContextError(ctxErr),
+				Output:  string(output),
+			}
+		}
+		if got != s.exitCode {
 			return StepResult{
 				Success: false,
 				Error:   unexpectedExitCode(s.exitCode, got),

testcase/testcase.go

@@ -2,12 +2,14 @@ package testcase
 
 import (
 	"bufio"
+	"context"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 
 	"github.com/UpCloudLtd/mdtest/globals"
+	"github.com/UpCloudLtd/mdtest/utils"
 	"github.com/UpCloudLtd/progress"
 	"github.com/UpCloudLtd/progress/messages"
 )
@@ -96,7 +98,12 @@ func removeTestDir(params TestParameters) error {
 }
 
 func getFailureDetails(test TestResult) string {
-	details := "Failures:"
+	details := ""
+	if test.FailureCount > 0 {
+		details += "Failures:"
+	} else if test.Error != nil {
+		details += "Canceled: " + test.Error.Error()
+	}
 	for i, res := range test.Results {
 		if err := res.Error; err != nil {
 			details += fmt.Sprintf("\n\nStep %d: %s", i+1, err.Error())
@@ -110,10 +117,15 @@ func getFailureDetails(test TestResult) string {
 
 	details += fmt.Sprintf("\n\n%d of %d test steps failed", test.FailureCount, test.StepsCount)
 
+	skippedCount := test.StepsCount - test.SuccessCount - test.FailureCount
+	if skippedCount > 0 {
+		details += fmt.Sprintf(" (%d skipped)", skippedCount)
+	}
+
 	return details
 }
 
-func Execute(path string, params TestParameters) TestResult {
+func Execute(ctx context.Context, path string, params TestParameters) TestResult {
 	testLog := params.TestLog
 
 	_ = testLog.Push(messages.Update{Key: path, Message: fmt.Sprintf("Parsing %s", path), Status: messages.MessageStatusStarted})
@@ -150,17 +162,20 @@ func Execute(path string, params TestParameters) TestResult {
 			ProgressMessage: fmt.Sprintf("(Step %d of %d)", i+1, len(steps)),
 		})
 
-		res := step.Execute(&status)
-		if res.Success {
-			test.SuccessCount++
+		if err := ctx.Err(); err == nil {
+			res := step.Execute(ctx, &status)
+			test.Results = append(test.Results, res)
+			if res.Success {
+				test.SuccessCount++
+			} else {
+				test.FailureCount++
+			}
 		} else {
-			test.FailureCount++
+			test.Error = utils.GetContextError(err)
 		}
-
-		test.Results = append(test.Results, res)
 	}
 
-	test.Success = test.FailureCount == 0
+	test.Success = test.SuccessCount == test.StepsCount
 	if test.Success {
 		_ = testLog.Push(messages.Update{Key: path, Status: messages.MessageStatusSuccess})
 		_ = removeTestDir(params)

testcase/teststep.go

@@ -2,6 +2,7 @@ package testcase
 
 import (
 	"bufio"
+	"context"
 	"fmt"
 	"strings"
 
@@ -15,7 +16,7 @@ type StepResult struct {
 }
 
 type Step interface {
-	Execute(*testStatus) StepResult
+	Execute(context.Context, *testStatus) StepResult
 }
 
 func parseCodeBlock(lang string, options map[string]string, content string) (Step, error) {

testdata/success_sleep.md

@@ -0,0 +1,13 @@
+# Sleep
+
+Sleep ten minutes.
+
+```sh
+sleep 600
+```
+
+And after that, sleep ten minutes again.
+
+```sh
+sleep 600
+```

testrun/execute.go

@@ -1,12 +1,14 @@
 package testrun
 
 import (
+	"context"
+
 	"github.com/UpCloudLtd/mdtest/id"
 	"github.com/UpCloudLtd/mdtest/testcase"
 	"github.com/UpCloudLtd/progress"
 )
 
-func executeTests(paths []string, params RunParameters, testLog *progress.Progress, run *RunResult) {
+func executeTests(ctx context.Context, paths []string, params RunParameters, testLog *progress.Progress, run *RunResult) {
 	if len(paths) == 0 {
 		return
 	}
@@ -32,7 +34,7 @@ func executeTests(paths []string, params RunParameters, testLog *progress.Progre
 				defer func() {
 					jobQueue <- jobID
 				}()
-				returnChan <- testcase.Execute(curTest, testcase.TestParameters{
+				returnChan <- testcase.Execute(ctx, curTest, testcase.TestParameters{
 					JobID:   jobID,
 					RunID:   run.ID,
 					TestID:  id.NewTestID(),

testrun/testrun.go

@@ -1,8 +1,11 @@
 package testrun
 
 import (
+	"context"
 	"fmt"
 	"io"
+	"os"
+	"os/signal"
 	"time"
 
 	"github.com/UpCloudLtd/mdtest/id"
@@ -15,6 +18,7 @@ import (
 type RunParameters struct {
 	NumberOfJobs int
 	OutputTarget io.Writer
+	Timeout      time.Duration
 }
 
 type RunResult struct {
@@ -47,12 +51,26 @@ func PrintSummary(target io.Writer, run RunResult) {
 }
 
 func Execute(rawPaths []string, params RunParameters) RunResult {
+	ctx, cancel := context.WithCancel(context.Background())
+	if params.Timeout > 0 {
+		ctx, cancel = context.WithTimeout(context.Background(), params.Timeout)
+		defer cancel()
+	}
+
 	started := time.Now()
 	paths, warnings := utils.ParseFilePaths(rawPaths, 1)
 
 	testLog := progress.NewProgress(nil)
 	testLog.Start()
 
+	// Handle possible interrupts during execution
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, os.Interrupt)
+	go func() {
+		<-signalChan
+		cancel()
+	}()
+
 	for _, warning := range warnings {
 		_ = testLog.Push(warning.Message())
 	}
@@ -62,7 +80,7 @@ func Execute(rawPaths []string, params RunParameters) RunResult {
 		Started: started,
 		Success: true,
 	}
-	executeTests(paths, params, testLog, &run)
+	executeTests(ctx, paths, params, testLog, &run)
 
 	testLog.Stop()
 	run.Success = run.FailureCount == 0

utils/context.go

@@ -0,0 +1,18 @@
+package utils
+
+import (
+	"context"
+	"errors"
+	"fmt"
+)
+
+func GetContextError(err error) error {
+	switch {
+	case errors.Is(err, context.DeadlineExceeded):
+		return fmt.Errorf("test run timeout exceeded")
+	case errors.Is(err, context.Canceled):
+		return fmt.Errorf("test run was canceled with interrupt signal")
+	default:
+		return fmt.Errorf("unexpected context error (%w)", err)
+	}
+}

utils/process.go

@@ -0,0 +1,23 @@
+//go:build !windows
+
+package utils
+
+import (
+	"fmt"
+	"os/exec"
+	"syscall"
+)
+
+func UseProcessGroup(cmd *exec.Cmd) {
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+}
+
+func Terminate(cmd *exec.Cmd) error {
+	err := syscall.Kill(-cmd.Process.Pid, syscall.SIGTERM)
+	if err != nil {
+		return fmt.Errorf("failed to terminate process group: %w", err)
+	}
+	return nil
+}