diff --git a/.gitignore b/.gitignore index 0d4be26..05856fe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ +junit*.xml /framework-tests /example-tests diff --git a/Makefile b/Makefile index 065dd6f..e0beaa3 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,11 @@ unit: go test ./... integration: build - ./framework-tests run-suite framework +ifeq ($(OPENSHIFT_CI), true) + ./framework-tests run-suite openshift-tests-extension/framework --junit-path $(ARTIFACT_DIR)/junit_$(shell date +%Y%m%d-%H%M%S).xml +else + ./framework-tests run-suite openshift-tests-extension/framework +endif lint: ./hack/go-lint.sh run ./... diff --git a/cmd/framework-tests/main.go b/cmd/framework-tests/main.go index 67bbdc3..b6fea2d 100644 --- a/cmd/framework-tests/main.go +++ b/cmd/framework-tests/main.go @@ -20,7 +20,7 @@ func main() { registry := e.NewRegistry() ext := e.NewExtension("openshift", "framework", "default") ext.AddSuite(e.Suite{ - Name: "framework", + Name: "openshift-tests-extension/framework", }) // If using Ginkgo, build test specs automatically diff --git a/pkg/cmd/cmdrun/runsuite.go b/pkg/cmd/cmdrun/runsuite.go index c79b3ff..1b80fea 100644 --- a/pkg/cmd/cmdrun/runsuite.go +++ b/pkg/cmd/cmdrun/runsuite.go @@ -17,10 +17,12 @@ func NewRunSuiteCommand(registry *extension.Registry) *cobra.Command { componentFlags *flags.ComponentFlags outputFlags *flags.OutputFlags concurrencyFlags *flags.ConcurrencyFlags + junitPath string }{ componentFlags: flags.NewComponentFlags(), outputFlags: flags.NewOutputFlags(), concurrencyFlags: flags.NewConcurrencyFlags(), + junitPath: "", } cmd := &cobra.Command{ @@ -35,29 +37,47 @@ func NewRunSuiteCommand(registry *extension.Registry) *cobra.Command { if len(args) != 1 { return fmt.Errorf("must specify one suite name") } - - w, err := extensiontests.NewResultWriter(os.Stdout, extensiontests.ResultFormat(opts.outputFlags.Output)) + suite, err := ext.GetSuite(args[0]) if err != nil { - return err + return errors.Wrapf(err, "couldn't find suite: %s", args[0]) } - defer w.Flush() - suite, err := ext.GetSuite(args[0]) + compositeWriter := extensiontests.NewCompositeResultWriter() + defer func() { + if err = compositeWriter.Flush(); err != nil { + fmt.Fprintf(os.Stderr, "failed to write results: %v\n", err) + } + }() + + // JUnit writer if needed + if opts.junitPath != "" { + junitWriter, err := extensiontests.NewJUnitResultWriter(opts.junitPath, suite.Name) + if err != nil { + return errors.Wrap(err, "couldn't create junit writer") + } + compositeWriter.AddWriter(junitWriter) + } + + // JSON writer + jsonWriter, err := extensiontests.NewJSONResultWriter(os.Stdout, + extensiontests.ResultFormat(opts.outputFlags.Output)) if err != nil { - return errors.Wrapf(err, "couldn't find suite: %s", args[0]) + return err } + compositeWriter.AddWriter(jsonWriter) specs, err := ext.GetSpecs().Filter(suite.Qualifiers) if err != nil { return errors.Wrap(err, "couldn't filter specs") } - return specs.Run(w, opts.concurrencyFlags.MaxConcurency) + return specs.Run(compositeWriter, opts.concurrencyFlags.MaxConcurency) }, } opts.componentFlags.BindFlags(cmd.Flags()) opts.outputFlags.BindFlags(cmd.Flags()) opts.concurrencyFlags.BindFlags(cmd.Flags()) + cmd.Flags().StringVarP(&opts.junitPath, "junit-path", "j", opts.junitPath, "write results to junit XML") return cmd } diff --git a/pkg/cmd/cmdrun/runtest.go b/pkg/cmd/cmdrun/runtest.go index ea4b62c..c44f124 100644 --- a/pkg/cmd/cmdrun/runtest.go +++ b/pkg/cmd/cmdrun/runtest.go @@ -63,7 +63,7 @@ func NewRunTestCommand(registry *extension.Registry) *cobra.Command { return err } - w, err := extensiontests.NewResultWriter(os.Stdout, extensiontests.ResultFormat(opts.outputFlags.Output)) + w, err := extensiontests.NewJSONResultWriter(os.Stdout, extensiontests.ResultFormat(opts.outputFlags.Output)) if err != nil { return err } diff --git a/pkg/extension/extensiontests/result.go b/pkg/extension/extensiontests/result.go index b0ae1c4..ae21b38 100644 --- a/pkg/extension/extensiontests/result.go +++ b/pkg/extension/extensiontests/result.go @@ -1,7 +1,60 @@ package extensiontests +import ( + "fmt" + "strings" + + "github.com/openshift-eng/openshift-tests-extension/pkg/junit" +) + func (results ExtensionTestResults) Walk(walkFn func(*ExtensionTestResult)) { for i := range results { walkFn(results[i]) } } + +func (result ExtensionTestResult) ToJUnit() *junit.TestCase { + tc := &junit.TestCase{ + Name: result.Name, + Duration: float64(result.Duration) / 1000.0, + } + switch result.Result { + case ResultFailed: + tc.FailureOutput = &junit.FailureOutput{ + Message: result.Error, + Output: result.Error, + } + case ResultSkipped: + tc.SkipMessage = &junit.SkipMessage{ + Message: strings.Join(result.Details, "\n"), + } + case ResultPassed: + tc.SystemOut = result.Output + } + + return tc +} + +func (results ExtensionTestResults) ToJUnit(suiteName string) junit.TestSuite { + suite := junit.TestSuite{ + Name: suiteName, + } + + results.Walk(func(result *ExtensionTestResult) { + suite.NumTests++ + switch result.Result { + case ResultFailed: + suite.NumFailed++ + case ResultSkipped: + suite.NumSkipped++ + case ResultPassed: + // do nothing + default: + panic(fmt.Sprintf("unknown result type: %s", result.Result)) + } + + suite.TestCases = append(suite.TestCases, result.ToJUnit()) + }) + + return suite +} diff --git a/pkg/extension/extensiontests/result_writer.go b/pkg/extension/extensiontests/result_writer.go index 104e266..f4feb30 100644 --- a/pkg/extension/extensiontests/result_writer.go +++ b/pkg/extension/extensiontests/result_writer.go @@ -2,10 +2,105 @@ package extensiontests import ( "encoding/json" + "encoding/xml" "fmt" "io" + "os" + "sync" + + "github.com/openshift-eng/openshift-tests-extension/pkg/junit" ) +// ResultWriter is an interface for recording ExtensionTestResults in a particular format. +// Implementations must be threadsafe. +type ResultWriter interface { + Write(*ExtensionTestResult) + Flush() error +} + +type CompositeResultWriter struct { + writers []ResultWriter +} + +func NewCompositeResultWriter(writers ...ResultWriter) *CompositeResultWriter { + return &CompositeResultWriter{ + writers: writers, + } +} + +func (w *CompositeResultWriter) AddWriter(writer ResultWriter) { + w.writers = append(w.writers, writer) +} + +func (w *CompositeResultWriter) Write(res *ExtensionTestResult) { + for _, writer := range w.writers { + writer.Write(res) + } +} + +func (w *CompositeResultWriter) Flush() error { + var errs []error + for _, writer := range w.writers { + if err := writer.Flush(); err != nil { + errs = append(errs, err) + } + } + + if len(errs) > 0 { + return fmt.Errorf("encountered errors from writers: %v", errs) + } + + return nil +} + +type JUnitResultWriter struct { + lock sync.Mutex + testSuite *junit.TestSuite + out *os.File + suiteName string + path string + results ExtensionTestResults +} + +func NewJUnitResultWriter(path, suiteName string) (ResultWriter, error) { + file, err := os.Create(path) + if err != nil { + return nil, err + } + + return &JUnitResultWriter{ + testSuite: &junit.TestSuite{ + Name: suiteName, + }, + out: file, + suiteName: suiteName, + path: path, + }, nil +} + +func (w *JUnitResultWriter) Write(res *ExtensionTestResult) { + w.lock.Lock() + defer w.lock.Unlock() + w.results = append(w.results, res) +} + +func (w *JUnitResultWriter) Flush() error { + w.lock.Lock() + defer w.lock.Unlock() + data, err := xml.Marshal(w.results.ToJUnit(w.suiteName)) + if err != nil { + panic(err) + } + if _, err := w.out.Write(data); err != nil { + return err + } + if err := w.out.Close(); err != nil { + return err + } + + return nil +} + type ResultFormat string var ( @@ -13,13 +108,14 @@ var ( JSONL ResultFormat = "jsonl" ) -type ResultWriter struct { +type JSONResultWriter struct { + lock sync.Mutex out io.Writer format ResultFormat results ExtensionTestResults } -func NewResultWriter(out io.Writer, format ResultFormat) (*ResultWriter, error) { +func NewJSONResultWriter(out io.Writer, format ResultFormat) (*JSONResultWriter, error) { switch format { case JSON, JSONL: // do nothing @@ -27,13 +123,15 @@ func NewResultWriter(out io.Writer, format ResultFormat) (*ResultWriter, error) return nil, fmt.Errorf("unsupported result format: %s", format) } - return &ResultWriter{ + return &JSONResultWriter{ out: out, format: format, }, nil } -func (w *ResultWriter) Write(result *ExtensionTestResult) { +func (w *JSONResultWriter) Write(result *ExtensionTestResult) { + w.lock.Lock() + defer w.lock.Unlock() switch w.format { case JSONL: // JSONL gets written to out as we get the items @@ -47,15 +145,20 @@ func (w *ResultWriter) Write(result *ExtensionTestResult) { } } -func (w *ResultWriter) Flush() { +func (w *JSONResultWriter) Flush() error { + w.lock.Lock() + defer w.lock.Unlock() switch w.format { case JSONL: // we already wrote it out case JSON: data, err := json.MarshalIndent(w.results, "", " ") if err != nil { - panic(err) + return err } - fmt.Fprintf(w.out, "%s\n", string(data)) + _, err = w.out.Write(data) + return err } + + return nil } diff --git a/pkg/extension/extensiontests/spec.go b/pkg/extension/extensiontests/spec.go index ee5a744..318988e 100644 --- a/pkg/extension/extensiontests/spec.go +++ b/pkg/extension/extensiontests/spec.go @@ -39,7 +39,7 @@ func (specs ExtensionTestSpecs) Names() []string { return names } -func (specs ExtensionTestSpecs) Run(w *ResultWriter, maxConcurrent int) error { +func (specs ExtensionTestSpecs) Run(w ResultWriter, maxConcurrent int) error { queue := make(chan *ExtensionTestSpec) failures := atomic.Int64{} diff --git a/pkg/junit/types.go b/pkg/junit/types.go new file mode 100644 index 0000000..e5423e2 --- /dev/null +++ b/pkg/junit/types.go @@ -0,0 +1,104 @@ +package junit + +import ( + "encoding/xml" +) + +// The below types are directly marshalled into XML. The types correspond to jUnit +// XML schema, but do not contain all valid fields. For instance, the class name +// field for test cases is omitted, as this concept does not directly apply to Go. +// For XML specifications see http://help.catchsoftware.com/display/ET/JUnit+Format +// or view the XSD included in this package as 'junit.xsd' + +// TestSuites represents a flat collection of jUnit test suites. +type TestSuites struct { + XMLName xml.Name `xml:"testsuites"` + + // Suites are the jUnit test suites held in this collection + Suites []*TestSuite `xml:"testsuite"` +} + +// TestSuite represents a single jUnit test suite, potentially holding child suites. +type TestSuite struct { + XMLName xml.Name `xml:"testsuite"` + + // Name is the name of the test suite + Name string `xml:"name,attr"` + + // NumTests records the number of tests in the TestSuite + NumTests uint `xml:"tests,attr"` + + // NumSkipped records the number of skipped tests in the suite + NumSkipped uint `xml:"skipped,attr"` + + // NumFailed records the number of failed tests in the suite + NumFailed uint `xml:"failures,attr"` + + // Duration is the time taken in seconds to run all tests in the suite + Duration float64 `xml:"time,attr"` + + // Properties holds other properties of the test suite as a mapping of name to value + Properties []*TestSuiteProperty `xml:"properties,omitempty"` + + // TestCases are the test cases contained in the test suite + TestCases []*TestCase `xml:"testcase"` + + // Children holds nested test suites + Children []*TestSuite `xml:"testsuite"` //nolint +} + +// TestSuiteProperty contains a mapping of a property name to a value +type TestSuiteProperty struct { + XMLName xml.Name `xml:"properties"` + + Name string `xml:"name,attr"` + Value string `xml:"value,attr"` +} + +// TestCase represents a jUnit test case +type TestCase struct { + XMLName xml.Name `xml:"testcase"` + + // Name is the name of the test case + Name string `xml:"name,attr"` + + // Classname is an attribute set by the package type and is required + Classname string `xml:"classname,attr,omitempty"` + + // Duration is the time taken in seconds to run the test + Duration float64 `xml:"time,attr"` + + // SkipMessage holds the reason why the test was skipped + SkipMessage *SkipMessage `xml:"skipped"` + + // FailureOutput holds the output from a failing test + FailureOutput *FailureOutput `xml:"failure"` + + // SystemOut is output written to stdout during the execution of this test case + SystemOut string `xml:"system-out,omitempty"` + + // SystemErr is output written to stderr during the execution of this test case + SystemErr string `xml:"system-err,omitempty"` +} + +// SkipMessage holds a message explaining why a test was skipped +type SkipMessage struct { + XMLName xml.Name `xml:"skipped"` + + // Message explains why the test was skipped + Message string `xml:"message,attr,omitempty"` +} + +// FailureOutput holds the output from a failing test +type FailureOutput struct { + XMLName xml.Name `xml:"failure"` + + // Message holds the failure message from the test + Message string `xml:"message,attr"` + + // Output holds verbose failure output from the test + Output string `xml:",chardata"` +} + +// TestResult is the result of a test case +type TestResult string