diff --git a/CHANGELOG-developer.asciidoc b/CHANGELOG-developer.asciidoc index fb22d9bd4435..3da4811bd2ef 100644 --- a/CHANGELOG-developer.asciidoc +++ b/CHANGELOG-developer.asciidoc @@ -39,6 +39,10 @@ The list below covers the major changes between 6.3.0 and master only. - Libbeat provides a global registry for beats developer that allow to register and retrieve plugin. {pull}7392[7392] - Added more options to control required and optional fields in schema.Apply(), error returned is a plain nil if no error happened {pull}7335[7335] - Packaging on MacOS now produces a .dmg file containing an installer (.pkg) and uninstaller for the Beat. {pull}7481[7481] +- Added mage targets `goTestUnit` and `goTestIntegration` for executing + 'go test'. This captures the log to a file, summarizes the result, produces a + coverage profile (.cov), and produces an HTML coverage report. See + `mage -h goTestUnit`. {pull}7766[7766] - New function `AddTagsWithKey` is added, so `common.MapStr` can be enriched with tags with an arbitrary key. {pull}7991[7991] - Libbeat provides a new function `cmd.GenRootCmdWithSettings` that should be preferred over deprecated functions `cmd.GenRootCmd`, `cmd.GenRootCmdWithRunFlags`, and `cmd.GenRootCmdWithIndexPrefixWithRunFlags`. {pull}7850[7850] diff --git a/NOTICE.txt b/NOTICE.txt index 5fa0a65f5889..2fcf6d11a74e 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1390,6 +1390,33 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +-------------------------------------------------------------------- +Dependency: github.com/jstemmer/go-junit-report +Revision: 385fac0ced9acaae6dc5b39144194008ded00697 +License type (autodetected): MIT +./vendor/github.com/jstemmer/go-junit-report/LICENSE: +-------------------------------------------------------------------- +Copyright (c) 2012 Joel Stemmer + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + -------------------------------------------------------------------- Dependency: github.com/klauspost/compress Revision: 14c9a76e3c95e47f8ccce949bba2c1101a8b85e6 diff --git a/auditbeat/magefile.go b/auditbeat/magefile.go index 083d35e72f78..e4d66ccdd9cf 100644 --- a/auditbeat/magefile.go +++ b/auditbeat/magefile.go @@ -20,6 +20,7 @@ package main import ( + "context" "fmt" "regexp" "time" @@ -96,6 +97,20 @@ func Fields() error { return mage.GenerateFieldsYAML("module") } +// GoTestUnit executes the Go unit tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestUnit(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestUnitArgs()) +} + +// GoTestIntegration executes the Go integration tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestIntegration(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) +} + // ----------------------------------------------------------------------------- // Customizations specific to Auditbeat. // - Config files are Go templates. diff --git a/dev-tools/jenkins_ci.ps1 b/dev-tools/jenkins_ci.ps1 index 3ac3053ee0fa..08614ea1e30c 100755 --- a/dev-tools/jenkins_ci.ps1 +++ b/dev-tools/jenkins_ci.ps1 @@ -21,12 +21,16 @@ $env:PATH = "$env:GOPATH\bin;C:\tools\mingw64\bin;$env:PATH" # each run starts from a clean slate. $env:MAGEFILE_CACHE = "$env:WORKSPACE\.magefile" +# Configure testing parameters. +$env:TEST_COVERAGE = "true" +$env:RACE_DETECTOR = "true" + +# Install mage from vendor. exec { go install github.com/elastic/beats/vendor/github.com/magefile/mage } echo "Fetching testing dependencies" # TODO (elastic/beats#5050): Use a vendored copy of this. exec { go get github.com/docker/libcompose } -exec { go get github.com/jstemmer/go-junit-report } if (Test-Path "$env:beat") { cd "$env:beat" @@ -49,8 +53,7 @@ echo "Building $env:beat" exec { mage build } "Build FAILURE" echo "Unit testing $env:beat" -go test -v $(go list ./... | select-string -Pattern "vendor" -NotMatch) 2>&1 | Out-File -encoding UTF8 build/TEST-go-unit.out -exec { Get-Content build/TEST-go-unit.out | go-junit-report.exe -set-exit-code | Out-File -encoding UTF8 build/TEST-go-unit.xml } "Unit test FAILURE, view testReport or TEST-go-unit.out jenkins artifact for detailed error info." +exec { mage goTestUnit } echo "System testing $env:beat" # Get a CSV list of package names. diff --git a/dev-tools/mage/gotest.go b/dev-tools/mage/gotest.go new file mode 100644 index 000000000000..88dd6e701234 --- /dev/null +++ b/dev-tools/mage/gotest.go @@ -0,0 +1,327 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package mage + +import ( + "bytes" + "context" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + "time" + + "github.com/jstemmer/go-junit-report/formatter" + "github.com/jstemmer/go-junit-report/parser" + "github.com/magefile/mage/mg" + "github.com/magefile/mage/sh" + "github.com/pkg/errors" +) + +// GoTestArgs are the arguments used for the "goTest*" targets and they define +// how "go test" is invoked. "go test" is always invoked with -v for verbose. +type GoTestArgs struct { + TestName string // Test name used in logging. + Race bool // Enable race detector. + Tags []string // Build tags to enable. + ExtraFlags []string // Extra flags to pass to 'go test'. + Packages []string // Packages to test. + Env map[string]string // Env vars to add to the current env. + OutputFile string // File to write verbose test output to. + JUnitReportFile string // File to write a JUnit XML test report to. + CoverageProfileFile string // Test coverage profile file (enables -cover). +} + +func makeGoTestArgs(name string) GoTestArgs { + fileName := fmt.Sprintf("build/TEST-go-%s", strings.Replace(strings.ToLower(name), " ", "_", -1)) + params := GoTestArgs{ + TestName: name, + Race: RaceDetector, + Packages: []string{"./..."}, + OutputFile: fileName + ".out", + JUnitReportFile: fileName + ".xml", + } + if TestCoverage { + params.CoverageProfileFile = fileName + ".cov" + } + return params +} + +// DefaultGoTestUnitArgs returns a default set of arguments for running +// all unit tests. We tag unit test files with '!integration'. +func DefaultGoTestUnitArgs() GoTestArgs { return makeGoTestArgs("Unit") } + +// DefaultGoTestIntegrationArgs returns a default set of arguments for running +// all integration tests. We tag integration test files with 'integration'. +func DefaultGoTestIntegrationArgs() GoTestArgs { + args := makeGoTestArgs("Integration") + args.Tags = append(args.Tags, "integration") + return args +} + +// GoTest invokes "go test" and reports the results to stdout. It returns an +// error if there was any failuring executing the tests or if there were any +// test failures. +func GoTest(ctx context.Context, params GoTestArgs) error { + fmt.Println(">> go test:", params.TestName, "Testing") + + // Build args list to Go. + args := []string{"test", "-v"} + if len(params.Tags) > 0 { + args = append(args, "-tags", strings.Join(params.Tags, " ")) + } + if params.CoverageProfileFile != "" { + params.CoverageProfileFile = createDir(filepath.Clean(params.CoverageProfileFile)) + args = append(args, + "-covermode=atomic", + "-coverprofile="+params.CoverageProfileFile, + ) + } + args = append(args, params.ExtraFlags...) + args = append(args, params.Packages...) + + goTest := makeCommand(ctx, params.Env, "go", args...) + + // Wire up the outputs. + bufferOutput := new(bytes.Buffer) + outputs := []io.Writer{bufferOutput} + if mg.Verbose() { + outputs = append(outputs, os.Stdout) + } + if params.OutputFile != "" { + fileOutput, err := os.Create(createDir(params.OutputFile)) + if err != nil { + return errors.Wrap(err, "failed to create go test output file") + } + defer fileOutput.Close() + outputs = append(outputs, fileOutput) + } + output := io.MultiWriter(outputs...) + goTest.Stdout = output + goTest.Stderr = output + + // Execute 'go test' and measure duration. + start := time.Now() + err := goTest.Run() + duration := time.Since(start) + var goTestErr *exec.ExitError + if err != nil { + // Command ran. + exitErr, ok := err.(*exec.ExitError) + if !ok { + return errors.Wrap(err, "failed to execute go") + } + + // Command ran but failed. Process the output. + goTestErr = exitErr + } + + // Parse the verbose test output. + report, err := parser.Parse(bytes.NewBuffer(bufferOutput.Bytes()), BeatName) + if err != nil { + return errors.Wrap(err, "failed to parse go test output") + } + if goTestErr != nil && len(report.Packages) == 0 { + // No packages were tested. Probably the code didn't compile. + fmt.Println(bytes.NewBuffer(bufferOutput.Bytes()).String()) + return errors.Wrap(goTestErr, "go test returned a non-zero value") + } + + // Generate a JUnit XML report. + if params.JUnitReportFile != "" { + junitReport, err := os.Create(createDir(params.JUnitReportFile)) + if err != nil { + return errors.Wrap(err, "failed to create junit report") + } + defer junitReport.Close() + + if err = formatter.JUnitReportXML(report, false, runtime.Version(), junitReport); err != nil { + return errors.Wrap(err, "failed to write junit report") + } + } + + // Generate a HTML code coverage report. + var htmlCoverReport string + if params.CoverageProfileFile != "" { + htmlCoverReport = strings.TrimSuffix(params.CoverageProfileFile, + filepath.Ext(params.CoverageProfileFile)) + ".html" + coverToHTML := sh.RunCmd("go", "tool", "cover", + "-html="+params.CoverageProfileFile, + "-o", htmlCoverReport) + if err = coverToHTML(); err != nil { + return errors.Wrap(err, "failed to write HTML code coverage report") + } + } + + // Summarize the results and log to stdout. + summary, err := NewGoTestSummary(duration, report, map[string]string{ + "Output File": params.OutputFile, + "JUnit Report": params.JUnitReportFile, + "Coverage Report": htmlCoverReport, + }) + if err != nil { + return err + } + if !mg.Verbose() && summary.Fail > 0 { + fmt.Println(summary.Failures()) + } + fmt.Println(summary.String()) + + // Return an error indicating that testing failed. + if summary.Fail > 0 || goTestErr != nil { + fmt.Println(">> go test:", params.TestName, "Test Failed") + if summary.Fail > 0 { + return errors.Errorf("go test failed: %d test failures", summary.Fail) + } + + return errors.Wrap(goTestErr, "go test returned a non-zero value") + } + + fmt.Println(">> go test:", params.TestName, "Test Passed") + return nil +} + +func makeCommand(ctx context.Context, env map[string]string, cmd string, args ...string) *exec.Cmd { + c := exec.CommandContext(ctx, "go", args...) + c.Env = os.Environ() + for k, v := range env { + c.Env = append(c.Env, k+"="+v) + } + c.Stdout = ioutil.Discard + if mg.Verbose() { + c.Stdout = os.Stdout + } + c.Stderr = os.Stderr + c.Stdin = os.Stdin + log.Println("exec:", cmd, strings.Join(args, " ")) + return c +} + +// GoTestSummary is a summary of test results. +type GoTestSummary struct { + *parser.Report // Report generated by parsing test output. + Pass int // Number of passing tests. + Fail int // Number of failed tests. + Skip int // Number of skipped tests. + Packages int // Number of packages tested. + Duration time.Duration // Total go test running duration. + Files map[string]string +} + +// NewGoTestSummary builds a new GoTestSummary. It returns an error if it cannot +// resolve the absolute paths to the given files. +func NewGoTestSummary(d time.Duration, r *parser.Report, outputFiles map[string]string) (*GoTestSummary, error) { + files := map[string]string{} + for name, file := range outputFiles { + if file == "" { + continue + } + absFile, err := filepath.Abs(file) + if err != nil { + return nil, errors.Wrapf(err, "failed resolving absolute path for %v", file) + } + files[name+":"] = absFile + } + + summary := &GoTestSummary{ + Report: r, + Duration: d, + Packages: len(r.Packages), + Files: files, + } + + for _, pkg := range r.Packages { + for _, t := range pkg.Tests { + switch t.Result { + case parser.PASS: + summary.Pass++ + case parser.FAIL: + summary.Fail++ + case parser.SKIP: + summary.Skip++ + default: + return nil, errors.Errorf("Unknown test result value: %v", t.Result) + } + } + } + + return summary, nil +} + +// Failures returns a string containing the list of failed test cases and their +// output. +func (s *GoTestSummary) Failures() string { + b := new(strings.Builder) + + if s.Fail > 0 { + fmt.Fprintln(b, "FAILURES:") + for _, pkg := range s.Report.Packages { + for _, t := range pkg.Tests { + if t.Result != parser.FAIL { + continue + } + fmt.Fprintln(b, "Package:", pkg.Name) + fmt.Fprintln(b, "Test: ", t.Name) + for _, line := range t.Output { + if strings.TrimSpace(line) != "" { + fmt.Fprintln(b, line) + } + } + fmt.Fprintln(b, "----") + } + } + } + + return strings.TrimRight(b.String(), "\n") +} + +// String returns a summary of the testing results (number of fail/pass/skip, +// test duration, number packages, output files). +func (s *GoTestSummary) String() string { + b := new(strings.Builder) + + fmt.Fprintln(b, "SUMMARY:") + fmt.Fprintln(b, " Fail: ", s.Fail) + fmt.Fprintln(b, " Skip: ", s.Skip) + fmt.Fprintln(b, " Pass: ", s.Pass) + fmt.Fprintln(b, " Packages:", len(s.Report.Packages)) + fmt.Fprintln(b, " Duration:", s.Duration) + + // Sort the list of files and compute the column width. + var names []string + var nameWidth int + for name := range s.Files { + if len(name) > nameWidth { + nameWidth = len(name) + } + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + fmt.Fprintf(b, " %-*s %s\n", nameWidth, name, s.Files[name]) + } + + return strings.TrimRight(b.String(), "\n") +} diff --git a/dev-tools/mage/settings.go b/dev-tools/mage/settings.go index c49d11565a6b..6b98f1076a21 100644 --- a/dev-tools/mage/settings.go +++ b/dev-tools/mage/settings.go @@ -47,11 +47,13 @@ const ( // Common settings with defaults derived from files, CWD, and environment. var ( - GOOS = build.Default.GOOS - GOARCH = build.Default.GOARCH - GOARM = EnvOr("GOARM", "") - Platform = MakePlatformAttributes(GOOS, GOARCH, GOARM) - BinaryExt = "" + GOOS = build.Default.GOOS + GOARCH = build.Default.GOARCH + GOARM = EnvOr("GOARM", "") + Platform = MakePlatformAttributes(GOOS, GOARCH, GOARM) + BinaryExt = "" + RaceDetector = false + TestCoverage = false BeatName = EnvOr("BEAT_NAME", filepath.Base(CWD())) BeatServiceName = EnvOr("BEAT_SERVICE_NAME", BeatName) @@ -82,9 +84,19 @@ func init() { } var err error + RaceDetector, err = strconv.ParseBool(EnvOr("RACE_DETECTOR", "false")) + if err != nil { + panic(errors.Wrap(err, "failed to parse RACE_DETECTOR env value")) + } + + TestCoverage, err = strconv.ParseBool(EnvOr("TEST_COVERAGE", "false")) + if err != nil { + panic(errors.Wrap(err, "failed to parse TEST_COVERAGE env value")) + } + Snapshot, err = strconv.ParseBool(EnvOr("SNAPSHOT", "false")) if err != nil { - panic(errors.Errorf("failed to parse SNAPSHOT value", err)) + panic(errors.Errorf("failed to parse SNAPSHOT env value", err)) } } diff --git a/dev-tools/packaging/templates/common/magefile.go.tmpl b/dev-tools/packaging/templates/common/magefile.go.tmpl index db7f7132110f..c86079fac87c 100644 --- a/dev-tools/packaging/templates/common/magefile.go.tmpl +++ b/dev-tools/packaging/templates/common/magefile.go.tmpl @@ -3,6 +3,7 @@ package main import ( + "context" "fmt" "time" @@ -73,5 +74,19 @@ func Update() error { // Fields generates a fields.yml for the Beat. func Fields() error { - return mage.GenerateFieldsYAML("protos") + return mage.GenerateFieldsYAML() +} + +// GoTestUnit executes the Go unit tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestUnit(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestUnitArgs()) +} + +// GoTestIntegration executes the Go integration tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestIntegration(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) } diff --git a/filebeat/magefile.go b/filebeat/magefile.go index f81483584b3a..f6de07b1d69a 100644 --- a/filebeat/magefile.go +++ b/filebeat/magefile.go @@ -20,6 +20,7 @@ package main import ( + "context" "fmt" "path/filepath" "time" @@ -96,6 +97,20 @@ func Fields() error { return mage.GenerateFieldsYAML("module") } +// GoTestUnit executes the Go unit tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestUnit(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestUnitArgs()) +} + +// GoTestIntegration executes the Go integration tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestIntegration(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) +} + // ----------------------------------------------------------------------------- // Customizations specific to Filebeat. // - Include modules directory in packages (minus _meta and test files). diff --git a/generator/beat/{beat}/magefile.go b/generator/beat/{beat}/magefile.go index 13c6184b5fa7..11ebe5d9101e 100644 --- a/generator/beat/{beat}/magefile.go +++ b/generator/beat/{beat}/magefile.go @@ -20,6 +20,7 @@ package main import ( + "context" "fmt" "time" @@ -94,3 +95,17 @@ func Update() error { func Fields() error { return mage.GenerateFieldsYAML() } + +// GoTestUnit executes the Go unit tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestUnit(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestUnitArgs()) +} + +// GoTestIntegration executes the Go integration tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestIntegration(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) +} diff --git a/generator/metricbeat/{beat}/magefile.go b/generator/metricbeat/{beat}/magefile.go index fa75eff3a806..a8a5931f823b 100644 --- a/generator/metricbeat/{beat}/magefile.go +++ b/generator/metricbeat/{beat}/magefile.go @@ -20,6 +20,7 @@ package main import ( + "context" "fmt" "time" @@ -94,3 +95,17 @@ func Update() error { func Fields() error { return mage.GenerateFieldsYAML("module") } + +// GoTestUnit executes the Go unit tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestUnit(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestUnitArgs()) +} + +// GoTestIntegration executes the Go integration tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestIntegration(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) +} diff --git a/heartbeat/magefile.go b/heartbeat/magefile.go index a23b8687e00a..507ffc8c75c2 100644 --- a/heartbeat/magefile.go +++ b/heartbeat/magefile.go @@ -20,6 +20,7 @@ package main import ( + "context" "fmt" "time" @@ -93,3 +94,17 @@ func Update() error { func Fields() error { return mage.GenerateFieldsYAML("monitors/active") } + +// GoTestUnit executes the Go unit tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestUnit(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestUnitArgs()) +} + +// GoTestIntegration executes the Go integration tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestIntegration(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) +} diff --git a/libbeat/magefile.go b/libbeat/magefile.go index 939eac4adb02..61290e0835ce 100644 --- a/libbeat/magefile.go +++ b/libbeat/magefile.go @@ -20,6 +20,8 @@ package main import ( + "context" + "github.com/elastic/beats/dev-tools/mage" ) @@ -37,3 +39,17 @@ func Clean() error { func Fields() error { return mage.GenerateFieldsYAML("processors") } + +// GoTestUnit executes the Go unit tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestUnit(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestUnitArgs()) +} + +// GoTestIntegration executes the Go integration tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestIntegration(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) +} diff --git a/metricbeat/magefile.go b/metricbeat/magefile.go index 056189f628f1..ff92e9bb7363 100644 --- a/metricbeat/magefile.go +++ b/metricbeat/magefile.go @@ -20,6 +20,7 @@ package main import ( + "context" "fmt" "regexp" "time" @@ -96,6 +97,20 @@ func Fields() error { return mage.GenerateFieldsYAML("module") } +// GoTestUnit executes the Go unit tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestUnit(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestUnitArgs()) +} + +// GoTestIntegration executes the Go integration tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestIntegration(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) +} + // ----------------------------------------------------------------------------- // Customizations specific to Metricbeat. // - Include modules.d directory in packages. diff --git a/packetbeat/magefile.go b/packetbeat/magefile.go index 03132ca70ae1..cb4a65330a95 100644 --- a/packetbeat/magefile.go +++ b/packetbeat/magefile.go @@ -20,6 +20,7 @@ package main import ( + "context" "fmt" "log" "regexp" @@ -123,6 +124,20 @@ func Fields() error { return mage.GenerateFieldsYAML("protos") } +// GoTestUnit executes the Go unit tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestUnit(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestUnitArgs()) +} + +// GoTestIntegration executes the Go integration tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestIntegration(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) +} + // ----------------------------------------------------------------------------- // Customizations specific to Packetbeat. // - Config file contains an OS specific device name (affects darwin, windows). diff --git a/vendor/github.com/jstemmer/go-junit-report/LICENSE b/vendor/github.com/jstemmer/go-junit-report/LICENSE new file mode 100644 index 000000000000..f346564cefd9 --- /dev/null +++ b/vendor/github.com/jstemmer/go-junit-report/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2012 Joel Stemmer + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/jstemmer/go-junit-report/README.md b/vendor/github.com/jstemmer/go-junit-report/README.md new file mode 100644 index 000000000000..4ea26582a345 --- /dev/null +++ b/vendor/github.com/jstemmer/go-junit-report/README.md @@ -0,0 +1,46 @@ +# go-junit-report + +Converts `go test` output to an xml report, suitable for applications that +expect junit xml reports (e.g. [Jenkins](http://jenkins-ci.org)). + +[![Build Status][travis-badge]][travis-link] +[![Report Card][report-badge]][report-link] + +## Installation + +Go version 1.1 or higher is required. Install or update using the `go get` +command: + +```bash +go get -u github.com/jstemmer/go-junit-report +``` + +## Contribution + +Create an Issue and discuss the fix or feature, then fork the package. +Clone to github.com/jstemmer/go-junit-report. This is necessary because go import uses this path. +Fix or implement feature. Test and then commit change. +Specify #Issue and describe change in the commit message. +Create Pull Request. It can be merged by owner or administrator then. + +## Run Tests +go test + +## Usage + +go-junit-report reads the `go test` verbose output from standard in and writes +junit compatible XML to standard out. + +```bash +go test -v 2>&1 | go-junit-report > report.xml +``` + +Note that it also can parse benchmark output with `-bench` flag: +```bash +go test -v -bench . -count 5 2>&1 | go-junit-report > report.xml + ``` + +[travis-badge]: https://travis-ci.org/jstemmer/go-junit-report.svg +[travis-link]: https://travis-ci.org/jstemmer/go-junit-report +[report-badge]: https://goreportcard.com/badge/github.com/jstemmer/go-junit-report +[report-link]: https://goreportcard.com/report/github.com/jstemmer/go-junit-report diff --git a/vendor/github.com/jstemmer/go-junit-report/formatter/formatter.go b/vendor/github.com/jstemmer/go-junit-report/formatter/formatter.go new file mode 100644 index 000000000000..bcc647451b08 --- /dev/null +++ b/vendor/github.com/jstemmer/go-junit-report/formatter/formatter.go @@ -0,0 +1,182 @@ +package formatter + +import ( + "bufio" + "encoding/xml" + "fmt" + "io" + "runtime" + "strings" + "time" + + "github.com/jstemmer/go-junit-report/parser" +) + +// JUnitTestSuites is a collection of JUnit test suites. +type JUnitTestSuites struct { + XMLName xml.Name `xml:"testsuites"` + Suites []JUnitTestSuite +} + +// JUnitTestSuite is a single JUnit test suite which may contain many +// testcases. +type JUnitTestSuite struct { + XMLName xml.Name `xml:"testsuite"` + Tests int `xml:"tests,attr"` + Failures int `xml:"failures,attr"` + Time string `xml:"time,attr"` + Name string `xml:"name,attr"` + Properties []JUnitProperty `xml:"properties>property,omitempty"` + TestCases []JUnitTestCase +} + +// JUnitTestCase is a single test case with its result. +type JUnitTestCase struct { + XMLName xml.Name `xml:"testcase"` + Classname string `xml:"classname,attr"` + Name string `xml:"name,attr"` + Time string `xml:"time,attr"` + SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"` + Failure *JUnitFailure `xml:"failure,omitempty"` +} + +// JUnitSkipMessage contains the reason why a testcase was skipped. +type JUnitSkipMessage struct { + Message string `xml:"message,attr"` +} + +// JUnitProperty represents a key/value pair used to define properties. +type JUnitProperty struct { + Name string `xml:"name,attr"` + Value string `xml:"value,attr"` +} + +// JUnitFailure contains data related to a failed test. +type JUnitFailure struct { + Message string `xml:"message,attr"` + Type string `xml:"type,attr"` + Contents string `xml:",chardata"` +} + +// JUnitReportXML writes a JUnit xml representation of the given report to w +// in the format described at http://windyroad.org/dl/Open%20Source/JUnit.xsd +func JUnitReportXML(report *parser.Report, noXMLHeader bool, goVersion string, w io.Writer) error { + suites := JUnitTestSuites{} + + // convert Report to JUnit test suites + for _, pkg := range report.Packages { + pkg.Benchmarks = mergeBenchmarks(pkg.Benchmarks) + ts := JUnitTestSuite{ + Tests: len(pkg.Tests) + len(pkg.Benchmarks), + Failures: 0, + Time: formatTime(pkg.Duration), + Name: pkg.Name, + Properties: []JUnitProperty{}, + TestCases: []JUnitTestCase{}, + } + + classname := pkg.Name + if idx := strings.LastIndex(classname, "/"); idx > -1 && idx < len(pkg.Name) { + classname = pkg.Name[idx+1:] + } + + // properties + if goVersion == "" { + // if goVersion was not specified as a flag, fall back to version reported by runtime + goVersion = runtime.Version() + } + ts.Properties = append(ts.Properties, JUnitProperty{"go.version", goVersion}) + if pkg.CoveragePct != "" { + ts.Properties = append(ts.Properties, JUnitProperty{"coverage.statements.pct", pkg.CoveragePct}) + } + + // individual test cases + for _, test := range pkg.Tests { + testCase := JUnitTestCase{ + Classname: classname, + Name: test.Name, + Time: formatTime(test.Duration), + Failure: nil, + } + + if test.Result == parser.FAIL { + ts.Failures++ + testCase.Failure = &JUnitFailure{ + Message: "Failed", + Type: "", + Contents: strings.Join(test.Output, "\n"), + } + } + + if test.Result == parser.SKIP { + testCase.SkipMessage = &JUnitSkipMessage{strings.Join(test.Output, "\n")} + } + + ts.TestCases = append(ts.TestCases, testCase) + } + + // individual benchmarks + for _, benchmark := range pkg.Benchmarks { + benchmarkCase := JUnitTestCase{ + Classname: classname, + Name: benchmark.Name, + Time: formatBenchmarkTime(benchmark.Duration), + } + + ts.TestCases = append(ts.TestCases, benchmarkCase) + } + + suites.Suites = append(suites.Suites, ts) + } + + // to xml + bytes, err := xml.MarshalIndent(suites, "", "\t") + if err != nil { + return err + } + + writer := bufio.NewWriter(w) + + if !noXMLHeader { + writer.WriteString(xml.Header) + } + + writer.Write(bytes) + writer.WriteByte('\n') + writer.Flush() + + return nil +} + +func mergeBenchmarks(benchmarks []*parser.Benchmark) []*parser.Benchmark { + var merged []*parser.Benchmark + benchmap := make(map[string][]*parser.Benchmark) + for _, bm := range benchmarks { + if _, ok := benchmap[bm.Name]; !ok { + merged = append(merged, &parser.Benchmark{Name: bm.Name}) + } + benchmap[bm.Name] = append(benchmap[bm.Name], bm) + } + + for _, bm := range merged { + for _, b := range benchmap[bm.Name] { + bm.Allocs += b.Allocs + bm.Bytes += b.Bytes + bm.Duration += b.Duration + } + n := len(benchmap[bm.Name]) + bm.Allocs /= n + bm.Bytes /= n + bm.Duration /= time.Duration(n) + } + + return merged +} + +func formatTime(d time.Duration) string { + return fmt.Sprintf("%.3f", d.Seconds()) +} + +func formatBenchmarkTime(d time.Duration) string { + return fmt.Sprintf("%.9f", d.Seconds()) +} diff --git a/vendor/github.com/jstemmer/go-junit-report/go-junit-report.go b/vendor/github.com/jstemmer/go-junit-report/go-junit-report.go new file mode 100644 index 000000000000..5c2e433a881e --- /dev/null +++ b/vendor/github.com/jstemmer/go-junit-report/go-junit-report.go @@ -0,0 +1,46 @@ +package main + +import ( + "flag" + "fmt" + "os" + + "github.com/jstemmer/go-junit-report/formatter" + "github.com/jstemmer/go-junit-report/parser" +) + +var ( + noXMLHeader bool + packageName string + goVersionFlag string + setExitCode bool +) + +func init() { + flag.BoolVar(&noXMLHeader, "no-xml-header", false, "do not print xml header") + flag.StringVar(&packageName, "package-name", "", "specify a package name (compiled test have no package name in output)") + flag.StringVar(&goVersionFlag, "go-version", "", "specify the value to use for the go.version property in the generated XML") + flag.BoolVar(&setExitCode, "set-exit-code", false, "set exit code to 1 if tests failed") +} + +func main() { + flag.Parse() + + // Read input + report, err := parser.Parse(os.Stdin, packageName) + if err != nil { + fmt.Printf("Error reading input: %s\n", err) + os.Exit(1) + } + + // Write xml + err = formatter.JUnitReportXML(report, noXMLHeader, goVersionFlag, os.Stdout) + if err != nil { + fmt.Printf("Error writing XML: %s\n", err) + os.Exit(1) + } + + if setExitCode && report.Failures() > 0 { + os.Exit(1) + } +} diff --git a/vendor/github.com/jstemmer/go-junit-report/parser/parser.go b/vendor/github.com/jstemmer/go-junit-report/parser/parser.go new file mode 100644 index 000000000000..b095b7cf81b0 --- /dev/null +++ b/vendor/github.com/jstemmer/go-junit-report/parser/parser.go @@ -0,0 +1,287 @@ +package parser + +import ( + "bufio" + "io" + "regexp" + "strconv" + "strings" + "time" +) + +// Result represents a test result. +type Result int + +// Test result constants +const ( + PASS Result = iota + FAIL + SKIP +) + +// Report is a collection of package tests. +type Report struct { + Packages []Package +} + +// Package contains the test results of a single package. +type Package struct { + Name string + Duration time.Duration + Tests []*Test + Benchmarks []*Benchmark + CoveragePct string + + // Time is deprecated, use Duration instead. + Time int // in milliseconds +} + +// Test contains the results of a single test. +type Test struct { + Name string + Duration time.Duration + Result Result + Output []string + + // Time is deprecated, use Duration instead. + Time int // in milliseconds +} + +// Benchmark contains the results of a single benchmark. +type Benchmark struct { + Name string + Duration time.Duration + // number of B/op + Bytes int + // number of allocs/op + Allocs int +} + +var ( + regexStatus = regexp.MustCompile(`--- (PASS|FAIL|SKIP): (.+) \((\d+\.\d+)(?: seconds|s)\)`) + regexCoverage = regexp.MustCompile(`^coverage:\s+(\d+\.\d+)%\s+of\s+statements(?:\sin\s.+)?$`) + regexResult = regexp.MustCompile(`^(ok|FAIL)\s+([^ ]+)\s+(?:(\d+\.\d+)s|\(cached\)|(\[\w+ failed]))(?:\s+coverage:\s+(\d+\.\d+)%\sof\sstatements(?:\sin\s.+)?)?$`) + // regexBenchmark captures 3-5 groups: benchmark name, number of times ran, ns/op (with or without decimal), B/op (optional), and allocs/op (optional). + regexBenchmark = regexp.MustCompile(`^(Benchmark[^ -]+)(?:-\d+\s+|\s+)(\d+)\s+(\d+|\d+\.\d+)\sns/op(?:\s+(\d+)\sB/op)?(?:\s+(\d+)\sallocs/op)?`) + regexOutput = regexp.MustCompile(`( )*\t(.*)`) + regexSummary = regexp.MustCompile(`^(PASS|FAIL|SKIP)$`) +) + +// Parse parses go test output from reader r and returns a report with the +// results. An optional pkgName can be given, which is used in case a package +// result line is missing. +func Parse(r io.Reader, pkgName string) (*Report, error) { + reader := bufio.NewReader(r) + + report := &Report{make([]Package, 0)} + + // keep track of tests we find + var tests []*Test + + // keep track of benchmarks we find + var benchmarks []*Benchmark + + // sum of tests' time, use this if current test has no result line (when it is compiled test) + var testsTime time.Duration + + // current test + var cur string + + // keep track if we've already seen a summary for the current test + var seenSummary bool + + // coverage percentage report for current package + var coveragePct string + + // stores mapping between package name and output of build failures + var packageCaptures = map[string][]string{} + + // the name of the package which it's build failure output is being captured + var capturedPackage string + + // capture any non-test output + var buffers = map[string][]string{} + + // parse lines + for { + l, _, err := reader.ReadLine() + if err != nil && err == io.EOF { + break + } else if err != nil { + return nil, err + } + + line := string(l) + + if strings.HasPrefix(line, "=== RUN ") { + // new test + cur = strings.TrimSpace(line[8:]) + tests = append(tests, &Test{ + Name: cur, + Result: FAIL, + Output: make([]string, 0), + }) + + // clear the current build package, so output lines won't be added to that build + capturedPackage = "" + seenSummary = false + } else if matches := regexBenchmark.FindStringSubmatch(line); len(matches) == 6 { + bytes, _ := strconv.Atoi(matches[4]) + allocs, _ := strconv.Atoi(matches[5]) + + benchmarks = append(benchmarks, &Benchmark{ + Name: matches[1], + Duration: parseNanoseconds(matches[3]), + Bytes: bytes, + Allocs: allocs, + }) + } else if strings.HasPrefix(line, "=== PAUSE ") { + continue + } else if strings.HasPrefix(line, "=== CONT ") { + cur = strings.TrimSpace(line[8:]) + continue + } else if matches := regexResult.FindStringSubmatch(line); len(matches) == 6 { + if matches[5] != "" { + coveragePct = matches[5] + } + if strings.HasSuffix(matches[4], "failed]") { + // the build of the package failed, inject a dummy test into the package + // which indicate about the failure and contain the failure description. + tests = append(tests, &Test{ + Name: matches[4], + Result: FAIL, + Output: packageCaptures[matches[2]], + }) + } else if matches[1] == "FAIL" && len(tests) == 0 && len(buffers[cur]) > 0 { + // This package didn't have any tests, but it failed with some + // output. Create a dummy test with the output. + tests = append(tests, &Test{ + Name: "Failure", + Result: FAIL, + Output: buffers[cur], + }) + buffers[cur] = buffers[cur][0:0] + } + + // all tests in this package are finished + report.Packages = append(report.Packages, Package{ + Name: matches[2], + Duration: parseSeconds(matches[3]), + Tests: tests, + Benchmarks: benchmarks, + CoveragePct: coveragePct, + + Time: int(parseSeconds(matches[3]) / time.Millisecond), // deprecated + }) + + buffers[cur] = buffers[cur][0:0] + tests = make([]*Test, 0) + benchmarks = make([]*Benchmark, 0) + coveragePct = "" + cur = "" + testsTime = 0 + } else if matches := regexStatus.FindStringSubmatch(line); len(matches) == 4 { + cur = matches[2] + test := findTest(tests, cur) + if test == nil { + continue + } + + // test status + if matches[1] == "PASS" { + test.Result = PASS + } else if matches[1] == "SKIP" { + test.Result = SKIP + } else { + test.Result = FAIL + } + test.Output = buffers[cur] + + test.Name = matches[2] + test.Duration = parseSeconds(matches[3]) + testsTime += test.Duration + + test.Time = int(test.Duration / time.Millisecond) // deprecated + } else if matches := regexCoverage.FindStringSubmatch(line); len(matches) == 2 { + coveragePct = matches[1] + } else if matches := regexOutput.FindStringSubmatch(line); capturedPackage == "" && len(matches) == 3 { + // Sub-tests start with one or more series of 4-space indents, followed by a hard tab, + // followed by the test output + // Top-level tests start with a hard tab. + test := findTest(tests, cur) + if test == nil { + continue + } + test.Output = append(test.Output, matches[2]) + } else if strings.HasPrefix(line, "# ") { + // indicates a capture of build output of a package. set the current build package. + capturedPackage = line[2:] + } else if capturedPackage != "" { + // current line is build failure capture for the current built package + packageCaptures[capturedPackage] = append(packageCaptures[capturedPackage], line) + } else if regexSummary.MatchString(line) { + // don't store any output after the summary + seenSummary = true + } else if !seenSummary { + // buffer anything else that we didn't recognize + buffers[cur] = append(buffers[cur], line) + } + } + + if len(tests) > 0 { + // no result line found + report.Packages = append(report.Packages, Package{ + Name: pkgName, + Duration: testsTime, + Time: int(testsTime / time.Millisecond), + Tests: tests, + Benchmarks: benchmarks, + CoveragePct: coveragePct, + }) + } + + return report, nil +} + +func parseSeconds(t string) time.Duration { + if t == "" { + return time.Duration(0) + } + // ignore error + d, _ := time.ParseDuration(t + "s") + return d +} + +func parseNanoseconds(t string) time.Duration { + // note: if input < 1 ns precision, result will be 0s. + if t == "" { + return time.Duration(0) + } + // ignore error + d, _ := time.ParseDuration(t + "ns") + return d +} + +func findTest(tests []*Test, name string) *Test { + for i := len(tests) - 1; i >= 0; i-- { + if tests[i].Name == name { + return tests[i] + } + } + return nil +} + +// Failures counts the number of failed tests in this report +func (r *Report) Failures() int { + count := 0 + + for _, p := range r.Packages { + for _, t := range p.Tests { + if t.Result == FAIL { + count++ + } + } + } + + return count +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 91f5cf641444..f8e7e912c83c 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -1142,6 +1142,24 @@ "revision": "69b34d4ec901851247ae7e77d33909caf9df99ed", "revisionTime": "2014-01-24T17:37:10Z" }, + { + "checksumSHA1": "tUaGicGy0FLinVWKYFM5ejGzMcY=", + "path": "github.com/jstemmer/go-junit-report", + "revision": "385fac0ced9acaae6dc5b39144194008ded00697", + "revisionTime": "2018-06-14T14:38:34Z" + }, + { + "checksumSHA1": "IH4jnWcj4d4h+hgsHsHOWg/F+rk=", + "path": "github.com/jstemmer/go-junit-report/formatter", + "revision": "385fac0ced9acaae6dc5b39144194008ded00697", + "revisionTime": "2018-06-14T14:38:34Z" + }, + { + "checksumSHA1": "Tx9cQqKFUHzu1l6H2XEl8G7ivlI=", + "path": "github.com/jstemmer/go-junit-report/parser", + "revision": "385fac0ced9acaae6dc5b39144194008ded00697", + "revisionTime": "2018-06-14T14:38:34Z" + }, { "checksumSHA1": "+CqJGh7NIDMnHgScq9sl9tPrnVM=", "path": "github.com/klauspost/compress/flate", diff --git a/winlogbeat/magefile.go b/winlogbeat/magefile.go index 1d8ad837d83a..ad93f4feda77 100644 --- a/winlogbeat/magefile.go +++ b/winlogbeat/magefile.go @@ -20,6 +20,7 @@ package main import ( + "context" "fmt" "time" @@ -93,3 +94,17 @@ func Update() error { func Fields() error { return mage.GenerateFieldsYAML() } + +// GoTestUnit executes the Go unit tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestUnit(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestUnitArgs()) +} + +// GoTestIntegration executes the Go integration tests. +// Use TEST_COVERAGE=true to enable code coverage profiling. +// Use RACE_DETECTOR=true to enable the race detector. +func GoTestIntegration(ctx context.Context) error { + return mage.GoTest(ctx, mage.DefaultGoTestIntegrationArgs()) +}