diff --git a/tools/src/cmd/cts/roll/roll.go b/tools/src/cmd/cts/roll/roll.go index 8e4e66f0a3f..16e1b1a3e23 100644 --- a/tools/src/cmd/cts/roll/roll.go +++ b/tools/src/cmd/cts/roll/roll.go @@ -121,6 +121,8 @@ func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, e flag.BoolVar(&c.flags.preserve, "preserve", false, "do not abandon existing rolls") flag.BoolVar(&c.flags.sendToGardener, "send-to-gardener", false, "send the CL to the WebGPU gardener for review") flag.BoolVar(&c.flags.verbose, "verbose", false, "emit additional logging") + // Deprecated. + // TODO(crbug.com/372730248): Remove this flag once the roller stops passing it in. flag.BoolVar(&c.flags.useSimplifiedCodepath, "use-simplified-codepath", false, "use the simplified codepath that only looks at unexpected failures") flag.StringVar(&c.flags.parentSwarmingRunID, "parent-swarming-run-id", "", "parent swarming run id. All triggered tasks will be children of this task and will be canceled if the parent is canceled.") flag.IntVar(&c.flags.maxAttempts, "max-attempts", 3, "number of update attempts before giving up") @@ -414,11 +416,7 @@ func (r *roller) roll(ctx context.Context) error { // Gather the build results log.Println("gathering results...") - if r.flags.useSimplifiedCodepath { - psResultsByExecutionMode, err = common.CacheUnsuppressedFailingResults(ctx, r.cfg, ps, r.flags.cacheDir, r.client, builds) - } else { - psResultsByExecutionMode, err = common.CacheResults(ctx, r.cfg, ps, r.flags.cacheDir, r.client, builds) - } + psResultsByExecutionMode, err = common.CacheUnsuppressedFailingResults(ctx, r.cfg, ps, r.flags.cacheDir, r.client, builds) if err != nil { return err } @@ -431,26 +429,14 @@ func (r *roller) roll(ctx context.Context) error { // Rebuild the expectations with the accumulated results log.Println("building new expectations...") for _, exInfo := range exInfos { - if r.flags.useSimplifiedCodepath { - // TODO(crbug.com/372730248): Modify exInfo.expectations in place once - // the old code path is removed. - exInfo.newExpectations = exInfo.expectations.Clone() - err := exInfo.newExpectations.AddExpectationsForFailingResults(psResultsByExecutionMode[exInfo.executionMode], testlist, r.flags.verbose) - if err != nil { - return err - } - exInfo.expectations = exInfo.newExpectations - } else { - // Merge the new results into the accumulated results - log.Printf("merging results for %s ...\n", exInfo.executionMode) - exInfo.results = result.Merge(exInfo.results, psResultsByExecutionMode[exInfo.executionMode]) - - exInfo.newExpectations = exInfo.expectations.Clone() - _, err := exInfo.newExpectations.Update(exInfo.results, testlist, r.flags.verbose) - if err != nil { - return err - } + // TODO(crbug.com/372730248): Modify exInfo.expectations in place once + // the old code path is removed. + exInfo.newExpectations = exInfo.expectations.Clone() + err := exInfo.newExpectations.AddExpectationsForFailingResults(psResultsByExecutionMode[exInfo.executionMode], testlist, r.flags.verbose) + if err != nil { + return err } + exInfo.expectations = exInfo.newExpectations } // Otherwise, push the updated expectations, and try again diff --git a/tools/src/cmd/cts/update/expectations/expectations.go b/tools/src/cmd/cts/update/expectations/expectations.go index 8d490947f23..b32a059b7e0 100644 --- a/tools/src/cmd/cts/update/expectations/expectations.go +++ b/tools/src/cmd/cts/update/expectations/expectations.go @@ -80,6 +80,8 @@ func (c *cmd) RegisterFlags(ctx context.Context, cfg common.Config) ([]string, e c.flags.results.RegisterFlags(cfg) c.flags.auth.Register(flag.CommandLine, auth.DefaultAuthOptions()) flag.BoolVar(&c.flags.verbose, "verbose", false, "emit additional logging") + // Deprecated. + // TODO(crbug.com/372730248): Remove this flag once the roller stops passing it in. flag.BoolVar(&c.flags.useSimplifiedCodepath, "use-simplified-codepath", false, "use the simplified codepath that only looks at unexpected failures") flag.Var(&c.flags.expectations, "expectations", "path to CTS expectations file(s) to update") return nil, nil @@ -111,12 +113,7 @@ func (c *cmd) Run(ctx context.Context, cfg common.Config) error { // Fetch the results log.Println("fetching results...") - var resultsByExecutionMode result.ResultsByExecutionMode - if c.flags.useSimplifiedCodepath { - resultsByExecutionMode, err = c.flags.results.GetUnsuppressedFailingResults(ctx, cfg, auth) - } else { - resultsByExecutionMode, err = c.flags.results.GetResults(ctx, cfg, auth) - } + resultsByExecutionMode, err := c.flags.results.GetUnsuppressedFailingResults(ctx, cfg, auth) if err != nil { return err } @@ -154,14 +151,9 @@ func (c *cmd) Run(ctx context.Context, cfg common.Config) error { name = "compat" } - var diag expectations.Diagnostics - if c.flags.useSimplifiedCodepath { - err = ex.AddExpectationsForFailingResults(resultsByExecutionMode[name], testlist, c.flags.verbose) - // TODO(crbug.com/372730248): Report actual diagnostics. - diag = expectations.Diagnostics{} - } else { - diag, err = ex.Update(resultsByExecutionMode[name], testlist, c.flags.verbose) - } + err = ex.AddExpectationsForFailingResults(resultsByExecutionMode[name], testlist, c.flags.verbose) + // TODO(crbug.com/372730248): Report actual diagnostics. + diag := expectations.Diagnostics{} if err != nil { return err } diff --git a/tools/src/cts/expectations/update.go b/tools/src/cts/expectations/update.go index fba9912128f..5a107d7fd62 100644 --- a/tools/src/cts/expectations/update.go +++ b/tools/src/cts/expectations/update.go @@ -28,18 +28,18 @@ package expectations import ( - "errors" "fmt" - "log" - "os" "strings" "time" "dawn.googlesource.com/dawn/tools/src/container" "dawn.googlesource.com/dawn/tools/src/cts/query" "dawn.googlesource.com/dawn/tools/src/cts/result" - "dawn.googlesource.com/dawn/tools/src/progressbar" - "github.com/mattn/go-isatty" +) + +const ( + // Chunk comment for the AddExpectationsForFailingResults path. + ROLLER_AUTOGENERATED_FAILURES = "# ##ROLLER_AUTOGENERATED_FAILURES##" ) // AddExpectationsForFailingResults adds new expectations for the provided @@ -246,777 +246,3 @@ func (c *Content) addExpectationsToMutableChunk(results *result.List) error { chunkToModify.Expectations.SortPrioritizeQuery() return nil } - -// Update performs an incremental update on the expectations using the provided -// results. -// -// Update will: -// - Remove any expectation lines that have a query where no results match. -// - Remove expectations lines that are in a chunk which is not annotated with -// 'KEEP', and all test results have the status 'Pass'. -// - Remove chunks that have had all expectation lines removed. -// - Appends new chunks for flaky and failing tests which are not covered by -// existing expectation lines. -// -// Update returns a list of diagnostics for things that should be addressed. -// -// Note: Validate() should be called before attempting to update the -// expectations. If Validate() returns errors, then Update() behaviour is -// undefined. -// TODO(crbug.com/371501714): Remove the Diagnostics return value since it is -// no longer used with the removal of commenting on CLs. -func (c *Content) Update(results result.List, testlist []query.Query, verbose bool) (Diagnostics, error) { - // Make a copy of the results. This code mutates the list. - results = append(result.List{}, results...) - - // Replace statuses that the CTS runner doesn't recognize with 'Failure' - simplifyStatuses(results) - - // Produce a list of tag sets. - // We reverse the declared order, as webgpu-cts/expectations.txt lists the - // most important first (OS, GPU, etc), and result.MinimalVariantTags will - // prioritize folding away the earlier tag-sets. - tagSets := make([]result.Tags, len(c.Tags.Sets)) - for i, s := range c.Tags.Sets { - tagSets[len(tagSets)-i-1] = s.Tags - } - - // Scan the full result list to obtain all the test variants - // (unique tag combinations). - variants := results.Variants() - - if verbose { - fmt.Println("result variants:") - for i, tags := range variants { - fmt.Printf(" (%.2d) %v\n", i, tags.List()) - } - } - - // Add 'consumed' results for tests that were skipped. - // This ensures that skipped results are not included in reduced trees. - results = c.appendConsumedResultsForSkippedTests(results, testlist, variants) - - var pb *progressbar.ProgressBar - if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stderr.Fd()) { - pb = progressbar.New(os.Stdout, nil) - defer pb.Stop() - } - - testQueryTree, _ := query.NewTree[struct{}]() - for _, query := range testlist { - testQueryTree.Add(query, struct{}{}) - } - - u := updater{ - in: *c, - out: Content{}, - resultQueryTree: buildResultQueryTree(results), - testQueryTree: testQueryTree, - variants: variants, - tagSets: tagSets, - pb: pb, - } - - if err := u.preserveRetryOnFailures(); err != nil { - return nil, err - } - - // Update those expectations! - if err := u.build(); err != nil { - return nil, fmt.Errorf("while updating expectations: %w", err) - } - - *c = u.out - return u.diags, nil -} - -// updater holds the state used for updating the expectations -type updater struct { - in Content // the original expectations Content - out Content // newly built expectations Content - resultQueryTree resultQueryTree // the results query tree - testQueryTree query.Tree[struct{}] - variants []container.Set[string] - diags []Diagnostic // diagnostics raised during update - tagSets []result.Tags // reverse-ordered tag-sets of 'in' - pb *progressbar.ProgressBar // Progress bar, may be nil -} - -// Returns 'results' with additional 'consumed' results for tests that have -// 'Skip' expectations. This fills in gaps for results, preventing tree -// reductions from marking skipped results as failure, which could result in -// expectation collisions. -func (c *Content) appendConsumedResultsForSkippedTests(results result.List, - testlist []query.Query, - variants []container.Set[string]) result.List { - tree := query.Tree[struct{}]{} - for _, q := range testlist { - tree.Add(q, struct{}{}) - } - // For each variant... - for _, variant := range variants { - resultsForVariant := container.NewSet[string]() - for _, result := range results.FilterByVariant(variant) { - resultsForVariant.Add(result.Query.String()) - } - - // For each expectation... - for _, c := range c.Chunks { - for _, ex := range c.Expectations { - // Does this expectation apply for variant? - if !variant.ContainsAll(ex.Tags) { - continue // Nope. - } - - // Does the expectation contain a Skip status? - if !container.NewSet(ex.Status...).Contains(string(result.Skip)) { - continue // Nope. - } - - // Gather all the tests that apply to the expectation - glob, _ := tree.Glob(query.Parse(ex.Query)) - for _, qd := range glob { - // If we don't have a result for the test, then append a - // synthetic 'consumed' result. - if query := qd.Query.String(); !resultsForVariant.Contains(query) { - resultsForVariant.Add(query) - results = append(results, result.Result{ - Query: qd.Query, - Tags: variant, - Status: consumed, - }) - } - } - } - } - } - return results -} - -// simplifyStatuses replaces all result statuses that are not one of -// 'Pass', 'RetryOnFailure', 'Slow', 'Skip' with 'Failure', and also replaces -// 'Skip' results with 'Pass'. -func simplifyStatuses(results result.List) { - for i, r := range results { - switch r.Status { - case result.Pass, result.RetryOnFailure, result.Slow: - // keep - case result.Skip: - // Typically represents a .unimplemented() test - results[i].Status = result.Pass - default: - results[i].Status = result.Failure - } - } -} - -const ( - // Status used to mark results that have been already handled by an - // expectation. - consumed result.Status = "<>" - // Chunk comment for new flakes - newFlakesComment = "# New flakes. Please triage - will be discarded/regenerated by the next roll:" - // Chunk comment for new failures - newFailuresComment = "# New failures. Please triage - will be discarded/regenerated by the next roll:" - // Chunk comment for expectations the roller is allowed to mutate - ROLLER_MUTABLE = "# ##ROLLER_MUTABLE##" - // Chunk comment for expectations the roller should discard and rewrite - ROLLER_DISCARD_AND_REWRITE = "# ##ROLLER_DISCARD_AND_REWRITE##" - // Chunk comment for the AddExpectationsForFailingResults path. - ROLLER_AUTOGENERATED_FAILURES = "# ##ROLLER_AUTOGENERATED_FAILURES##" -) - -// resultQueryTree holds tree of queries to all results (no filtering by tag or -// status). The resultQueryTree is used to glob all the results that match a -// particular query. -type resultQueryTree struct { - // All the results. - results result.List - // consumedAt is a list of line numbers for the i'th result in 'results' - // Initially all line numbers are 0. When a result is consumed the line - // number is set. - consumedAt []int - // Each tree node holds a list of indices to results. - tree query.Tree[[]int] -} - -// buildResultQueryTree builds the queryTree from the list of results. -func buildResultQueryTree(results result.List) resultQueryTree { - log.Println("building query tree...") - - // Build a map of query to result indices - queryToIndices := map[query.Query][]int{} - for i, r := range results { - l := queryToIndices[r.Query] - l = append(l, i) - queryToIndices[r.Query] = l - } - - // Construct the query tree to result indices - tree := query.Tree[[]int]{} - for query, indices := range queryToIndices { - if err := tree.Add(query, indices); err != nil { - // Unreachable: The only error we could get is duplicate data for - // the same query, which should be impossible. - panic(err) - } - } - - consumedAt := make([]int, len(results)) - return resultQueryTree{results, consumedAt, tree} -} - -// glob returns the list of results matching the given tags under (or with) the -// given query. -func (qt *resultQueryTree) glob(q query.Query) (result.List, error) { - glob, err := qt.tree.Glob(q) - if err != nil { - return nil, fmt.Errorf("while gathering results for query '%v': %w", q, err) - } - - out := result.List{} - for _, indices := range glob { - for _, idx := range indices.Data { - out = append(out, qt.results[idx]) - } - } - - return out, nil -} - -// globTags returns the list of results matching the given tags under (or with) -// the given query. -func (qt *resultQueryTree) globTags(q query.Query, t result.Tags) (result.List, error) { - glob, err := qt.tree.Glob(q) - if err != nil { - return nil, err - } - - out := result.List{} - for _, indices := range glob { - for _, idx := range indices.Data { - if r := qt.results[idx]; r.Tags.ContainsAll(t) { - out = append(out, r) - } - } - } - return out, nil -} - -// markAsConsumed marks all the results matching the given tags -// under (or with) the given query, as consumed. -// line is used to record the line at which the results were consumed. If the -// results were consumed as part of generating new expectations then line should -// be 0. -func (qt *resultQueryTree) markAsConsumed(q query.Query, t result.Tags, line int) { - if glob, err := qt.tree.Glob(q); err == nil { - for _, indices := range glob { - for _, idx := range indices.Data { - r := &qt.results[idx] - if r.Tags.ContainsAll(t) { - r.Status = consumed - qt.consumedAt[idx] = line - } - } - } - } -} - -// preserveRetryOnFailures changes any results matching expectations with a -// RetryOnFailure expectation to RetryOnFailure. -func (u *updater) preserveRetryOnFailures() error { - // For each expectation... - for _, c := range u.in.Chunks { - for _, ex := range c.Expectations { - // Does the expectation contain a RetryOnFailure status? - if !container.NewSet(ex.Status...).Contains(string(result.RetryOnFailure)) { - continue // Nope. - } - - q := query.Parse(ex.Query) - - glob, err := u.resultQueryTree.tree.Glob(q) - if err != nil { - if errors.As(err, &query.ErrNoDataForQuery{}) { - // No results for this RetryOnFailure expectation. - // Flaky tests might have been removed from the CTS. - // These expectations will be automatically removed by updater.expectation() - continue - } - return err - } - for _, indices := range glob { - for _, idx := range indices.Data { - if u.resultQueryTree.results[idx].Tags.ContainsAll(ex.Tags) { - u.resultQueryTree.results[idx].Status = result.RetryOnFailure - } - } - } - } - } - return nil -} - -type Progress struct { - totalExpectations int - currentExpectation int -} - -// build is the updater top-level function. -// build first appends to u.out all chunks from 'u.in' with expectations updated -// using the new results, and then appends any new expectations to u.out. -func (u *updater) build() error { - progress := Progress{} - - // Chunks are considered immutable by default, unless annotated as - // ROLLER_MUTABLE or ROLLER_DISCARD_AND_REWRITE. - mutableTokens := []string{ - ROLLER_MUTABLE, - ROLLER_DISCARD_AND_REWRITE, - } - - // Bin the chunks into those that contain any of the strings in - // mutableTokens in the comments and those that do not have these strings. - immutableChunks, mutableChunks := []Chunk{}, []Chunk{} - for _, chunk := range u.in.Chunks { - keep := true - - comments: - for _, line := range chunk.Comments { - for _, token := range mutableTokens { - if strings.Contains(line, token) { - keep = false - break comments - } - } - } - - if keep { - immutableChunks = append(immutableChunks, chunk) - } else { - mutableChunks = append(mutableChunks, chunk) - } - - progress.totalExpectations += len(chunk.Expectations) - } - - log.Println("updating expectation chunks...") - - // Update all the existing chunks in two passes - those that are immutable - // then those that are mutable. We do this because the former can't be - // altered and may declare expectations that may collide with later - // expectations. - for _, group := range []struct { - chunks []Chunk - isImmutable bool - }{ - {immutableChunks, true}, - {mutableChunks, false}, - } { - for _, in := range group.chunks { - out := u.chunk(in, group.isImmutable, &progress) - - // If all chunk had expectations, but now they've gone, remove the chunk - if len(in.Expectations) > 0 && len(out.Expectations) == 0 { - continue - } - - u.out.Chunks = append(u.out.Chunks, out) - } - } - - // Emit new expectations (flaky, failing) - if err := u.addNewExpectations(); err != nil { - return fmt.Errorf("failed to add new expectations: %w", err) - } - - return nil -} - -// chunk returns a new Chunk, based on 'in', with the expectations updated. -// isImmutable is true if the chunk is labelled with 'KEEP' and can't be changed. -func (u *updater) chunk(in Chunk, isImmutable bool, progress *Progress) Chunk { - if len(in.Expectations) == 0 { - return in // Just a comment / blank line - } - - // Skip over ROLLER_DISCARD_AND_REWRITE chunks (untriaged failures/flakes). - // We'll just rebuild them at the end. - for _, line := range in.Comments { - if strings.HasPrefix(line, ROLLER_DISCARD_AND_REWRITE) { - return Chunk{} - } - } - - // Build the new chunk's expectations - newExpectations := container.NewMap[string, Expectation]() - for _, exIn := range in.Expectations { - if u.pb != nil { - u.pb.Update(progressbar.Status{Total: progress.totalExpectations, Segments: []progressbar.Segment{ - {Count: 1 + progress.currentExpectation}, - }}) - progress.currentExpectation++ - } - - u.addExpectations(newExpectations, exIn, isImmutable) - } - - // Sort the expectations to keep things clean and tidy. - out := Chunk{Comments: in.Comments, Expectations: newExpectations.Values()} - out.Expectations.Sort() - return out -} - -// expectation returns a new list of Expectations, based on the Expectation 'in', -// using the new result data. -func (u *updater) addExpectations(out container.Map[string, Expectation], in Expectation, isImmutable bool) { - q := query.Parse(in.Query) - - // keyOf returns the map key for out - keyOf := func(e Expectation) string { return fmt.Sprint(e.Tags, e.Query, e.Status) } - - // noResults is a helper for returning when the expectation has no test results. - noResults := func() { - if glob, err := u.testQueryTree.Glob(q); err == nil && len(glob) > 0 { - // At least one test is found with the query in the test list - likely a variant that is not being run. - if len(in.Tags) > 0 { - u.diag(Note, in.Line, "no results found for query '%v' with tags %v", in.Query, in.Tags) - } else { - u.diag(Note, in.Line, "no results found for query '%v'", in.Query) - } - // Preserve. - out.Add(keyOf(in), in) - } else { - // Remove the no-results expectation (do not add to out) - u.diag(Warning, in.Line, "no tests exist with query '%v' - removing", in.Query) - } - } - - // Glob the results for the expectation's query + tag combination. - // Ensure that none of these are already consumed. - results, err := u.resultQueryTree.globTags(q, in.Tags) - // If we can't find any results for this query + tag combination, then bail. - switch { - case errors.As(err, &query.ErrNoDataForQuery{}): - noResults() - return - case err != nil: - u.diag(Error, in.Line, "%v", err) - return - case len(results) == 0: - noResults() - return - } - - // Before returning, mark all the results as consumed. - // Note: this has to happen *after* we've generated the new expectations, as - // marking the results as 'consumed' will impact the logic of - // expectationsForRoot() - defer u.resultQueryTree.markAsConsumed(q, in.Tags, in.Line) - - if isImmutable { // Expectation chunk was marked with 'KEEP' - // Add a diagnostic if all tests of the expectation were 'Pass' - if s := results.Statuses(); len(s) == 1 && s.One() == result.Pass { - u.diagAllPass(in.Line, results) - } - out.Add(keyOf(in), in) - return - } - - // Rebuild the expectations for this query. - expectations, somePass, someConsumed := u.expectationsForRoot(q, in.Line, in.Bug, in.Comment) - - // Add the new expectations to out - for _, expectation := range expectations { - out.Add(keyOf(expectation), expectation) - } - - // Add a diagnostic if the expectation is filtered away - if !out.Contains(keyOf(in)) && len(expectations) == 0 { - switch { - case somePass && someConsumed: - u.diag(Note, in.Line, "expectation is partly covered by previous expectations and the remaining tests all pass") - case someConsumed: - u.diag(Note, in.Line, "expectation is fully covered by previous expectations") - case somePass: - u.diagAllPass(in.Line, results) - } - } - -} - -// addNewExpectations (potentially) appends to 'u.out' chunks for new flaky and -// failing tests. -func (u *updater) addNewExpectations() error { - // For each variant: - // • Build a query tree using the results filtered to the variant, and then - // reduce the tree. - // • Take all the reduced-tree leaf nodes, and add these to 'roots'. - // Once we've collected all the roots, we'll use these to build the - // expectations across the reduced set of tags. - log.Println("determining new expectation roots...") - roots := query.Tree[bool]{} - for i, variant := range u.variants { - if u.pb != nil { - u.pb.Update(progressbar.Status{Total: len(u.variants), Segments: []progressbar.Segment{ - {Count: 1 + i}, - }}) - } - - // Build a tree from the results matching the given variant. - filtered := u.resultQueryTree.results.FilterByVariant(variant) - tree, err := filtered.StatusTree() - if err != nil { - return fmt.Errorf("while building tree for tags '%v': %w", variant, err) - } - // Reduce the tree. - tree.Reduce(treeReducer) - // Add all the reduced leaf nodes to 'roots'. - for _, qd := range tree.List() { - if qd.Data != result.Pass { - roots.Add(qd.Query, true) - } - } - } - - // Build all the expectations for each of the roots. - log.Println("building new expectations...") - rootsList := roots.List() - expectations := []Expectation{} - for i, root := range rootsList { - if u.pb != nil { - u.pb.Update(progressbar.Status{Total: len(rootsList), Segments: []progressbar.Segment{ - {Count: 1 + i}, - }}) - } - rootExpectations, _, _ := u.expectationsForRoot( - root.Query, // Root query - 0, // Line number - "crbug.com/dawn/0000", // Bug - "", // Comment - ) - expectations = append(expectations, rootExpectations...) - } - - // Bin the expectations by failure or flake. - flakes, failures := []Expectation{}, []Expectation{} - for _, r := range expectations { - if container.NewSet(r.Status...).Contains(string(result.RetryOnFailure)) { - flakes = append(flakes, r) - } else { - failures = append(failures, r) - } - } - - // Create chunks for any flakes and failures, in that order. - for _, group := range []struct { - results Expectations - comment string - }{ - {flakes, newFlakesComment}, - {failures, newFailuresComment}, - } { - if len(group.results) > 0 { - group.results.Sort() - u.out.Chunks = append(u.out.Chunks, Chunk{ - Comments: []string{ - "################################################################################", - group.comment, - ROLLER_DISCARD_AND_REWRITE, - "################################################################################", - }, - Expectations: group.results, - }) - } - } - - return nil -} - -// expectationsForRoot builds a list of expectations that cover the failing -// tests for the results under root. -// The returned list of expectations is optimized by reducing queries to the -// most common root, and reducing tags to the smallest required set. -func (u *updater) expectationsForRoot( - root query.Query, // The sub-tree query root - line int, // The originating line, when producing diagnostics - bug string, // The bug to apply to all returned expectations - comment string, // The comment to apply to all returned expectations -) ( - expectations []Expectation, // The output expectations - somePass bool, // Some of the results for the query had a Pass status - someConsumed bool, // The query was at least partly consumed by previous expectations -) { - results, err := u.resultQueryTree.glob(root) - if err != nil { - u.diag(Error, line, "%v", err) - return nil, false, false - } - - // Using the full list of unfiltered tests, generate the minimal set of - // variants (tags) that uniquely classify the results with differing status. - minimalVariants := u. - removeUnknownTags(results). - MinimalVariantTags(u.tagSets) - - // For each minimized variant... - reduced := result.List{} - for _, variant := range minimalVariants { - // Build a query tree from this variant... - tree := result.StatusTree{} - filtered := results.FilterByTags(variant) - for _, r := range filtered { - // Note: variants may overlap, but overlaped queries will have - // identical statuses, so we can just ignore the error for Add(). - tree.Add(r.Query, r.Status) - } - - // ... and reduce the tree by collapsing sub-trees that have common - // statuses. - tree.ReduceUnder(root, treeReducer) - - // Append the reduced tree nodes to the results list - for _, qs := range tree.List() { - reduced = append(reduced, result.Result{ - Query: qs.Query, - Tags: variant, - Status: qs.Data, - }) - } - } - - // Filter out any results that passed or have already been consumed - filtered := result.List{} - for _, r := range reduced { - switch r.Status { - case result.Pass: - somePass = true - case consumed: - someConsumed = true - default: - filtered = append(filtered, r) - } - } - - // Mark all the new expectation results as consumed. - for _, r := range filtered { - u.resultQueryTree.markAsConsumed(r.Query, r.Tags, 0) - } - - // Transform the results to expectations. - expectations = u.resultsToExpectations(filtered, bug, comment) - return expectations, somePass, someConsumed -} - -// resultsToExpectations returns a list of expectations from the given results. -// Each expectation will have the same query, tags and status as the input -// result, along with the specified bug and comment. -// -// If the result query target is a test without a wildcard, then the expectation -// will have a wildcard automatically appended. This is to satisfy a requirement -// of the expectation validator. -func (u *updater) resultsToExpectations(results result.List, bug, comment string) []Expectation { - results.Sort() - - out := make([]Expectation, 0, len(results)) - addedExpectations := container.NewSet[string]() - for _, r := range results { - q := r.Query.String() - if r.Query.Target() == query.Tests && !r.Query.IsWildcard() { - // The expectation validator wants a trailing ':' for test queries - q += query.TargetDelimiter - } - e := Expectation{ - Bug: bug, - Tags: u.in.Tags.RemoveLowerPriorityTags(r.Tags), - Query: q, - Status: []string{string(r.Status)}, - Comment: comment, - } - key := e.AsExpectationFileString() - // We keep track of all expectations we've added so far to avoid cases where - // two distinct results create the same expectation due to - // RemoveLowerPriorityTags removing the distinguishing tags. - if !addedExpectations.Contains(key) { - out = append(out, e) - addedExpectations.Add(key) - } - } - - return out -} - -// removeUnknownTags returns a copy of the provided results with all tags not -// found in the expectations list removed -func (u *updater) removeUnknownTags(results result.List) result.List { - return results.TransformTags(func(t result.Tags) result.Tags { - filtered := result.NewTags() - for tag := range t { - if _, ok := u.in.Tags.ByName[tag]; ok { - filtered.Add(tag) - } - } - return filtered - }) -} - -// treeReducer is a function that can be used by StatusTree.Reduce() to reduce -// tree nodes with the same status. -// treeReducer will collapse trees nodes if any of the following are true: -// - All child nodes have the same status -// - More than 50% of the child nodes have a non-pass status, and none of the -// children are consumed. -// - There are more than 10 child nodes with a non-pass status, and none of the -// children are consumed. -func treeReducer(statuses []result.Status) *result.Status { - counts := map[result.Status]int{} - for _, s := range statuses { - counts[s] = counts[s] + 1 - } - if len(counts) == 1 { - return &statuses[0] // All the same status - } - if counts[consumed] > 0 { - return nil // Partially consumed trees cannot be merged - } - highestNonPassCount := 0 - highestNonPassStatus := result.Failure - for s, n := range counts { - if s != result.Pass { - if percent := (100 * n) / len(statuses); percent > 50 { - // Over 50% of all the children are of non-pass status s. - return &s - } - if n > highestNonPassCount { - highestNonPassCount = n - highestNonPassStatus = s - } - } - } - - if highestNonPassCount > 10 { - // Over 10 child node failed. - return &highestNonPassStatus - } - - return nil -} - -// diag appends a new diagnostic to u.diags with the given severity, line and -// message. -func (u *updater) diag(severity Severity, line int, msg string, args ...interface{}) { - u.diags = append(u.diags, Diagnostic{ - Severity: severity, - Line: line, - Message: fmt.Sprintf(msg, args...), - }) -} - -// diagAllPass appends a new note diagnostic that all the tests now pass -func (u *updater) diagAllPass(line int, results result.List) { - if c := len(results); c > 1 { - u.diag(Note, line, "all %d tests now pass", len(results)) - } else { - u.diag(Note, line, "test now passes") - } -} diff --git a/tools/src/cts/expectations/update_test.go b/tools/src/cts/expectations/update_test.go index a4c12737d7f..ce172dca54e 100644 --- a/tools/src/cts/expectations/update_test.go +++ b/tools/src/cts/expectations/update_test.go @@ -28,933 +28,15 @@ package expectations import ( - "strings" "testing" - "dawn.googlesource.com/dawn/tools/src/container" "dawn.googlesource.com/dawn/tools/src/cts/query" "dawn.googlesource.com/dawn/tools/src/cts/result" - "github.com/google/go-cmp/cmp" "github.com/stretchr/testify/assert" ) var Q = query.Parse -func TestUpdate(t *testing.T) { - header := `# BEGIN TAG HEADER -# OS -# tags: [ os-a os-b os-c ] -# GPU -# tags: [ gpu-a gpu-b gpu-c ] -# END TAG HEADER -` - headerLines := strings.Count(header, "\n") - - type Test struct { - name string - expectations string - results result.List - updated string - diagnostics Diagnostics - err string - } - for _, test := range []Test{ - { ////////////////////////////////////////////////////////////////////// - name: "empty results", - expectations: ``, - results: result.List{}, - }, - { ////////////////////////////////////////////////////////////////////// - name: "no results found", - expectations: ` -# ##ROLLER_MUTABLE## -crbug.com/a/123 a:missing,test,result:* [ Failure ] -crbug.com/a/123 [ tag ] another:missing,test,result:* [ Failure ] -some:other,test:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("some:other,test:*"), - Tags: result.NewTags("os-a", "gpu-a"), - Status: result.Failure, - }, - result.Result{ - Query: Q("some:other,test:*"), - Tags: result.NewTags("os-b", "gpu-b"), - Status: result.Failure, - }, - }, - updated: ` -# ##ROLLER_MUTABLE## -some:other,test:* [ Failure ] -crbug.com/a/123 a:missing,test,result:* [ Failure ] -crbug.com/a/123 [ tag ] another:missing,test,result:* [ Failure ] -`, - diagnostics: Diagnostics{ - { - Severity: Note, - Line: headerLines + 3, - Message: "no results found for query 'a:missing,test,result:*'", - }, - { - Severity: Note, - Line: headerLines + 4, - Message: "no results found for query 'another:missing,test,result:*' with tags [tag]", - }, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "no results found immutable", - expectations: ` -crbug.com/a/123 a:missing,test,result:* [ Failure ] - -some:other,test:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("some:other,test:*"), - Tags: result.NewTags("os-a", "gpu-a"), - Status: result.Failure, - }, - result.Result{ - Query: Q("some:other,test:*"), - Tags: result.NewTags("os-b", "gpu-b"), - Status: result.Failure, - }, - }, - updated: ` -crbug.com/a/123 a:missing,test,result:* [ Failure ] - -some:other,test:* [ Failure ] -`, - diagnostics: Diagnostics{ - { - Severity: Note, - Line: headerLines + 2, - Message: "no results found for query 'a:missing,test,result:*'", - }, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "unknown test", - expectations: ` -# ##ROLLER_MUTABLE## -crbug.com/a/123 an:unknown,test:* [ Failure ] -crbug.com/a/123 [ tag ] another:unknown:test [ Failure ] - -some:other,test:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("some:other,test:*"), - Tags: result.NewTags("os-a", "gpu-a"), - Status: result.Failure, - }, - result.Result{ - Query: Q("some:other,test:*"), - Tags: result.NewTags("os-b", "gpu-b"), - Status: result.Failure, - }, - }, - updated: ` -some:other,test:* [ Failure ] -`, - diagnostics: Diagnostics{ - { - Severity: Warning, - Line: headerLines + 3, - Message: "no tests exist with query 'an:unknown,test:*' - removing", - }, - { - Severity: Warning, - Line: headerLines + 4, - Message: "no tests exist with query 'another:unknown:test' - removing", - }, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "unknown test found in immutable chunk", - expectations: ` -crbug.com/a/123 an:unknown,test:* [ Failure ] - -some:other,test:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("some:other,test:*"), - Tags: result.NewTags("os-a", "gpu-a"), - Status: result.Failure, - }, - result.Result{ - Query: Q("some:other,test:*"), - Tags: result.NewTags("os-b", "gpu-b"), - Status: result.Failure, - }, - }, - updated: ` -some:other,test:* [ Failure ] -`, - diagnostics: Diagnostics{ - { - Severity: Warning, - Line: headerLines + 2, - Message: "no tests exist with query 'an:unknown,test:*' - removing", - }, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "simple expectation with tags", - expectations: ` -# ##ROLLER_MUTABLE## -[ os-a ] a:b,c:* [ Failure ] -[ gpu-b ] a:b,c:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("a:b,c:d:e"), - Tags: result.NewTags("os-a", "os-c", "gpu-b"), - Status: result.Failure, - }, - }, - updated: ` -# ##ROLLER_MUTABLE## -a:b,c:* [ Failure ] -`, - diagnostics: Diagnostics{ - { - Severity: Note, - Line: headerLines + 4, - Message: "expectation is fully covered by previous expectations", - }, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "simple expectation with tags new flakes implicitly mutable", - expectations: ` -################################################################################ -# New flakes. Please triage - will be discarded/regenerated by the next roll: -# ##ROLLER_DISCARD_AND_REWRITE## -################################################################################ -[ os-a ] a:b,c:* [ RetryOnFailure ] -[ gpu-b ] a:b,c:* [ RetryOnFailure ] -`, - results: result.List{ - result.Result{ - Query: Q("a:b,c:d:e"), - Tags: result.NewTags("os-a", "os-c", "gpu-b"), - Status: result.RetryOnFailure, - }, - }, - updated: ` -################################################################################ -# New flakes. Please triage - will be discarded/regenerated by the next roll: -# ##ROLLER_DISCARD_AND_REWRITE## -################################################################################ -crbug.com/dawn/0000 a:* [ RetryOnFailure ] -`, - }, - { ////////////////////////////////////////////////////////////////////// - name: "simple expectation with tags new failures implicitly mutable", - expectations: ` -################################################################################ -# New failures. Please triage - will be discarded/regenerated by the next roll: -# ##ROLLER_DISCARD_AND_REWRITE## -################################################################################ -[ os-a ] a:b,c:* [ Failure ] -[ gpu-b ] a:b,c:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("a:b,c:d:e"), - Tags: result.NewTags("os-a", "os-c", "gpu-b"), - Status: result.Failure, - }, - }, - updated: ` -################################################################################ -# New failures. Please triage - will be discarded/regenerated by the next roll: -# ##ROLLER_DISCARD_AND_REWRITE## -################################################################################ -crbug.com/dawn/0000 a:* [ Failure ] -`, - }, - { ////////////////////////////////////////////////////////////////////// - name: "simple expectation with tags immutable", - expectations: ` -[ os-a ] a:b,c:* [ Failure ] -[ gpu-b ] a:b,c:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("a:b,c:d:e"), - Tags: result.NewTags("os-a", "os-c", "gpu-b"), - Status: result.Failure, - }, - }, - updated: ` -[ gpu-b ] a:b,c:* [ Failure ] -[ os-a ] a:b,c:* [ Failure ] -`, - }, - { ////////////////////////////////////////////////////////////////////// - name: "expectation test now passes", - expectations: ` -# ##ROLLER_MUTABLE## -crbug.com/a/123 [ gpu-a os-a ] a:b,c:* [ Failure ] -crbug.com/a/123 [ gpu-b os-b ] a:b,c:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("a:b,c:*"), - Tags: result.NewTags("os-a", "gpu-a"), - Status: result.Pass, - }, - result.Result{ - Query: Q("a:b,c:*"), - Tags: result.NewTags("os-b", "gpu-b"), - Status: result.Abort, - }, - }, - updated: ` -# ##ROLLER_MUTABLE## -crbug.com/a/123 [ os-b ] a:b,c:* [ Failure ] -`, - diagnostics: Diagnostics{ - { - Severity: Note, - Line: headerLines + 4, - Message: "expectation is fully covered by previous expectations", - }, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "expectation case now passes", - expectations: ` -# ##ROLLER_MUTABLE## -crbug.com/a/123 [ gpu-a os-a ] a:b,c:d:* [ Failure ] -crbug.com/a/123 [ gpu-b os-b ] a:b,c:d:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("a:b,c:d:*"), - Tags: result.NewTags("os-a", "gpu-a"), - Status: result.Pass, - }, - result.Result{ - Query: Q("a:b,c:d:*"), - Tags: result.NewTags("os-b", "gpu-b"), - Status: result.Abort, - }, - }, - updated: ` -# ##ROLLER_MUTABLE## -crbug.com/a/123 [ os-b ] a:b,c:d:* [ Failure ] -`, - diagnostics: Diagnostics{ - { - Severity: Note, - Line: headerLines + 4, - Message: "expectation is fully covered by previous expectations", - }, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "first expectation expands to cover later expectations - no diagnostics", - expectations: ` -crbug.com/a/123 [ gpu-a os-a ] a:b,c:d:* [ Failure ] -crbug.com/a/123 [ gpu-c os-a ] a:b,c:d:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("a:b,c:d:e"), - Tags: result.NewTags("gpu-a", "os-a"), - Status: result.Failure, - }, - result.Result{ - Query: Q("a:b,c:d:e"), - Tags: result.NewTags("gpu-a", "os-b"), - Status: result.Pass, - }, - result.Result{ - Query: Q("a:b,c:d:e"), - Tags: result.NewTags("gpu-b", "os-a"), - Status: result.Pass, - }, - result.Result{ - Query: Q("a:b,c:d:e"), - Tags: result.NewTags("gpu-b", "os-b"), - Status: result.Pass, - }, - result.Result{ - Query: Q("a:b,c:d:e"), - Tags: result.NewTags("gpu-c", "os-a"), - Status: result.Failure, - }, - result.Result{ - Query: Q("a:b,c:d:e"), - Tags: result.NewTags("gpu-c", "os-b"), - Status: result.Pass, - }, - }, - updated: ` -crbug.com/a/123 [ gpu-a os-a ] a:b,c:d:* [ Failure ] -crbug.com/a/123 [ gpu-c os-a ] a:b,c:d:* [ Failure ] -`, - }, - { ////////////////////////////////////////////////////////////////////// - name: "expectation case now passes immutable - single", - expectations: ` -crbug.com/a/123 [ gpu-a os-a ] a:b,c:d:* [ Failure ] -crbug.com/a/123 [ gpu-b os-b ] a:b,c:d:* [ Failure ] -`, - results: result.List{ - result.Result{ - Query: Q("a:b,c:d:e"), - Tags: result.NewTags("os-a", "gpu-a"), - Status: result.Pass, - }, - result.Result{ - Query: Q("a:b,c:d:e"), - Tags: result.NewTags("os-b", "gpu-b"), - Status: result.Abort, - }, - }, - updated: ` -crbug.com/a/123 [ gpu-a os-a ] a:b,c:d:* [ Failure ] -crbug.com/a/123 [ gpu-b os-b ] a:b,c:d:* [ Failure ] -`, - diagnostics: Diagnostics{ - { - Severity: Note, - Line: headerLines + 2, - Message: "test now passes", - }, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "expectation case now passes immutable - multiple", - expectations: ` -crbug.com/a/123 a:b,c:d:* [ Failure ] -`, - results: result.List{ - result.Result{Query: Q("a:b,c:d:a"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:d:b"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:d:c"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:d:d"), Status: result.Pass}, - }, - updated: ` -crbug.com/a/123 a:b,c:d:* [ Failure ] -`, - diagnostics: Diagnostics{ - { - Severity: Note, - Line: headerLines + 2, - Message: "all 4 tests now pass", - }, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "new test results", - expectations: `# A comment`, - results: result.List{ - result.Result{ - Query: Q("suite:dir_a,dir_b:test_a:*"), - Tags: result.NewTags("os-a", "gpu-a"), - Status: result.Abort, - }, - result.Result{ - Query: Q("suite:dir_a,dir_b:test_a:*"), - Tags: result.NewTags("os-a", "gpu-b"), - Status: result.Abort, - }, - result.Result{ - Query: Q("suite:dir_a,dir_b:test_c:case=4;*"), - Tags: result.NewTags("os-a", "gpu-a"), - Status: result.Crash, - }, - result.Result{ - Query: Q("suite:dir_a,dir_b:test_c:case=4;*"), - Tags: result.NewTags("os-b", "gpu-b"), - Status: result.Crash, - }, - result.Result{ - Query: Q("suite:dir_a,dir_b:test_c:case=5;*"), - Tags: result.NewTags("os-b", "gpu-b"), - Status: result.RetryOnFailure, - }, - result.Result{ - Query: Q("suite:dir_a,dir_b:test_b:case=5;*"), - Tags: result.NewTags("os-b", "gpu-b"), - Status: result.Pass, - }, - result.Result{ - Query: Q("suite:dir_a,dir_b:test_b:case=6;*"), - Tags: result.NewTags("os-a", "gpu-a"), - Status: result.Slow, - }, - result.Result{ - Query: Q("suite:dir_a,dir_b:test_b:case=6;*"), - Tags: result.NewTags("os-b", "gpu-a"), - Status: result.Pass, - }, - result.Result{ - Query: Q("suite:dir_a,dir_b:test_c:case=6;*"), - Tags: result.NewTags("os-a", "gpu-a"), - Status: result.RetryOnFailure, - }, - }, - updated: `# A comment - -################################################################################ -# New flakes. Please triage - will be discarded/regenerated by the next roll: -# ##ROLLER_DISCARD_AND_REWRITE## -################################################################################ -crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_c:case=6;* [ RetryOnFailure ] -crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=5;* [ RetryOnFailure ] - -################################################################################ -# New failures. Please triage - will be discarded/regenerated by the next roll: -# ##ROLLER_DISCARD_AND_REWRITE## -################################################################################ -crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_a:* [ Failure ] -crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_b:* [ Slow ] -crbug.com/dawn/0000 [ gpu-a os-a ] suite:dir_a,dir_b:test_c:case=4;* [ Failure ] -crbug.com/dawn/0000 [ gpu-b os-a ] suite:* [ Failure ] -crbug.com/dawn/0000 [ gpu-b os-b ] suite:dir_a,dir_b:test_c:case=4;* [ Failure ] -`, - }, - - { ////////////////////////////////////////////////////////////////////// - name: "root node overlap", - expectations: `# A comment`, - results: result.List{ - // For variant ['os-a'], we have a root node 'a:b,c:d:*'. - result.Result{ - Query: Q("a:b,c:d:x,*"), - Tags: result.NewTags("os-a"), - Status: result.Failure, - }, - result.Result{ - Query: Q("a:b,c:d:y,*"), - Tags: result.NewTags("os-a"), - Status: result.Failure, - }, - result.Result{ - Query: Q("a:b,c:e:*"), - Tags: result.NewTags("os-a"), - Status: result.Pass, - }, - // For variant ['os-b'], we have a root node 'a:b,c:d:x,*'. - result.Result{ - Query: Q("a:b,c:d:x,*"), - Tags: result.NewTags("os-b"), - Status: result.Failure, - }, - result.Result{ - Query: Q("a:b,c:d:y,*"), - Tags: result.NewTags("os-b"), - Status: result.Pass, - }, - result.Result{ - Query: Q("a:b,c:e:*"), - Tags: result.NewTags("os-b"), - Status: result.Pass, - }, - }, - updated: `# A comment - -################################################################################ -# New failures. Please triage - will be discarded/regenerated by the next roll: -# ##ROLLER_DISCARD_AND_REWRITE## -################################################################################ -crbug.com/dawn/0000 [ os-a ] a:b,c:d:* [ Failure ] -crbug.com/dawn/0000 [ os-b ] a:b,c:d:x,* [ Failure ] -`, - }, - { ////////////////////////////////////////////////////////////////////// - name: "filter unknown tags", - expectations: ``, - results: result.List{ - result.Result{ - Query: Q("a:b,c:*"), - Tags: result.NewTags("os-a", "gpu-x"), - Status: result.Failure, - }, - result.Result{ - Query: Q("a:b,c:*"), - Tags: result.NewTags("os-b", "gpu-x"), - Status: result.Crash, - }, - result.Result{ - Query: Q("a:b,c:*"), - Tags: result.NewTags("os-x", "gpu-b"), - Status: result.Failure, - }, - result.Result{ - Query: Q("a:b,c:*"), - Tags: result.NewTags("os-x", "gpu-a"), - Status: result.Crash, - }, - result.Result{ - Query: Q("a:b,c:*"), - Tags: result.NewTags("os-c", "gpu-c"), - Status: result.Pass, - }, - }, - updated: ` -################################################################################ -# New failures. Please triage - will be discarded/regenerated by the next roll: -# ##ROLLER_DISCARD_AND_REWRITE## -################################################################################ -crbug.com/dawn/0000 [ gpu-a ] a:* [ Failure ] -crbug.com/dawn/0000 [ gpu-b ] a:* [ Failure ] -crbug.com/dawn/0000 [ os-a ] a:* [ Failure ] -crbug.com/dawn/0000 [ os-b ] a:* [ Failure ] -`, - }, - { ////////////////////////////////////////////////////////////////////// - name: "prioritize tag sets", - expectations: ``, - results: result.List{ - result.Result{ - Query: Q("a:b,c:*"), - Tags: result.NewTags("os-a", "os-c", "gpu-b"), - Status: result.Failure, - }, - result.Result{ - Query: Q("a:b,c:*"), - Tags: result.NewTags("gpu-a", "os-b", "gpu-c"), - Status: result.Failure, - }, - result.Result{ - Query: Q("a:b,c:*"), - Tags: result.NewTags("os-c", "gpu-c"), - Status: result.Pass, - }, - }, - updated: ` -################################################################################ -# New failures. Please triage - will be discarded/regenerated by the next roll: -# ##ROLLER_DISCARD_AND_REWRITE## -################################################################################ -crbug.com/dawn/0000 [ gpu-b os-c ] a:* [ Failure ] -crbug.com/dawn/0000 [ gpu-c os-b ] a:* [ Failure ] -`, - }, - { ////////////////////////////////////////////////////////////////////// - name: "merge when 50% or more children fail", - expectations: ``, - results: result.List{ // 4 pass, 6 fail (50%) - result.Result{Query: Q("a:b,c:0:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:1:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:2:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:3:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:4:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:5:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:6:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:7:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:8:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:9:*"), Status: result.Pass}, - }, - updated: ` -################################################################################ -# New failures. Please triage - will be discarded/regenerated by the next roll: -# ##ROLLER_DISCARD_AND_REWRITE## -################################################################################ -crbug.com/dawn/0000 a:* [ Failure ] -`, - }, - { ////////////////////////////////////////////////////////////////////// - name: "don't merge when 50% or fewer children fail", - expectations: ``, - results: result.List{ // 5 pass, 5 fail (50%) - result.Result{Query: Q("a:b,c:0:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:1:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:2:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:3:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:4:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:5:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:6:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:7:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:8:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:9:*"), Status: result.Pass}, - }, - updated: ` -################################################################################ -# New failures. Please triage - will be discarded/regenerated by the next roll: -# ##ROLLER_DISCARD_AND_REWRITE## -################################################################################ -crbug.com/dawn/0000 a:b,c:0:* [ Failure ] -crbug.com/dawn/0000 a:b,c:2:* [ Failure ] -crbug.com/dawn/0000 a:b,c:5:* [ Failure ] -crbug.com/dawn/0000 a:b,c:6:* [ Failure ] -crbug.com/dawn/0000 a:b,c:8:* [ Failure ] -`, - }, - { ////////////////////////////////////////////////////////////////////// - name: "merge when more than 10 children fail", - expectations: ``, - results: result.List{ // 19 pass, 11 fail (37%) - result.Result{Query: Q("a:b,c:00:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:01:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:02:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:03:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:04:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:05:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:06:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:07:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:08:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:09:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:10:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:11:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:12:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:13:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:14:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:15:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:16:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:17:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:18:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:19:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:20:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:21:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:22:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:23:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:24:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:25:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:26:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:27:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:28:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:29:*"), Status: result.Failure}, - }, - updated: ` -################################################################################ -# New failures. Please triage - will be discarded/regenerated by the next roll: -# ##ROLLER_DISCARD_AND_REWRITE## -################################################################################ -crbug.com/dawn/0000 a:* [ Failure ] -`, - }, - { ////////////////////////////////////////////////////////////////////// - name: "don't merge when 10 or fewer children fail", - expectations: ``, - results: result.List{ // 20 pass, 10 fail (33%) - result.Result{Query: Q("a:b,c:00:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:01:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:02:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:03:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:04:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:05:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:06:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:07:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:08:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:09:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:10:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:11:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:12:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:13:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:14:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:15:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:16:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:17:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:18:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:19:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:20:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:21:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:22:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:23:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:24:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:25:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:26:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:27:*"), Status: result.Pass}, - result.Result{Query: Q("a:b,c:28:*"), Status: result.Failure}, - result.Result{Query: Q("a:b,c:29:*"), Status: result.Failure}, - }, - updated: ` -################################################################################ -# New failures. Please triage - will be discarded/regenerated by the next roll: -# ##ROLLER_DISCARD_AND_REWRITE## -################################################################################ -crbug.com/dawn/0000 a:b,c:00:* [ Failure ] -crbug.com/dawn/0000 a:b,c:05:* [ Failure ] -crbug.com/dawn/0000 a:b,c:08:* [ Failure ] -crbug.com/dawn/0000 a:b,c:13:* [ Failure ] -crbug.com/dawn/0000 a:b,c:15:* [ Failure ] -crbug.com/dawn/0000 a:b,c:20:* [ Failure ] -crbug.com/dawn/0000 a:b,c:23:* [ Failure ] -crbug.com/dawn/0000 a:b,c:26:* [ Failure ] -crbug.com/dawn/0000 a:b,c:28:* [ Failure ] -crbug.com/dawn/0000 a:b,c:29:* [ Failure ] -`, - }, - } { - ex, err := Parse("expectations.txt", header+test.expectations) - if err != nil { - t.Fatalf("'%v': expectations.Parse():\n%v", test.name, err) - } - - testList := container.NewMap[string, query.Query]() - for _, r := range test.results { - testList.Add(r.Query.String(), r.Query) - } - for _, s := range []string{ - "a:missing,test,result:a=1,b=2", - "another:missing,test,result:cat=meow,dog=woof", - } { - testList.Add(s, query.Parse(s)) - } - - errMsg := "" - diagnostics, err := ex.Update(test.results, testList.Values() /* verbose */, false) - if err != nil { - errMsg = err.Error() - } - if diff := cmp.Diff(errMsg, test.err); diff != "" { - t.Errorf("'%v': expectations.Update() error:\n%v", test.name, diff) - } - - if diff := cmp.Diff(diagnostics, test.diagnostics); diff != "" { - t.Errorf("'%v': diagnostics were not as expected:\n%v", test.name, diff) - } - - if diff := cmp.Diff( - strings.Split(ex.String(), "\n"), - strings.Split(header+test.updated, "\n")); diff != "" { - t.Errorf("'%v': updated was not as expected:\n%v", test.name, diff) - } - } -} - -func createGenericUpdater(t *testing.T) updater { - header := ` -# BEGIN TAG HEADER -# OS -# tags: [ linux win10 ] -# GPU -# tags: [ intel -# nvidia nvidia-0x2184 ] -# Driver -# tags: [ nvidia_ge_31.0.15.4601 nvidia_lt_31.0.15.4601 -# nvidia_ge_535.183.01 nvidia_lt_535.183.01 ] -# END TAG HEADER -` - inContent, err := Parse("expectations.txt", header) - if err != nil { - t.Fatalf("Failed to parse expectations: %v", err) - } - - u := updater{ - in: inContent, - } - return u -} - -// Tests basic result -> expectation conversion. -func TestResultsToExpectationsBasic(t *testing.T) { - results := result.List{ - { - Query: query.Parse("webgpu:shader,execution,memory_layout:read_layout:"), - Tags: result.NewTags("linux", "nvidia"), - Status: result.Failure, - }, - { - Query: query.Parse("webgpu:shader,execution,memory_model,barrier:"), - Tags: result.NewTags("win10", "intel"), - Status: result.Failure, - }, - } - - expectedOutput := []Expectation{ - { - Bug: "crbug.com/1234", - Tags: result.NewTags("linux", "nvidia"), - Query: "webgpu:shader,execution,memory_layout:read_layout:", - Status: []string{"Failure"}, - Comment: "comment", - }, - { - Bug: "crbug.com/1234", - Tags: result.NewTags("win10", "intel"), - Query: "webgpu:shader,execution,memory_model,barrier", - Status: []string{"Failure"}, - Comment: "comment", - }, - } - - u := createGenericUpdater(t) - output := u.resultsToExpectations(results, "crbug.com/1234", "comment") - assert.Equal(t, output, expectedOutput) -} - -// Tests behavior when two unique results end up creating the same expectation -// due to lower priority tags being removed. -func TestResultsToExpectationsOverlappingExpectations(t *testing.T) { - results := result.List{ - { - Query: query.Parse("webgpu:shader,execution,memory_layout:read_layout:"), - Tags: result.NewTags("nvidia", "nvidia-0x2184", "nvidia_ge_31.0.15.4601", "nvidia_lt_535.183.01"), - Status: result.Failure, - }, - { - Query: query.Parse("webgpu:shader,execution,memory_layout:read_layout:"), - Tags: result.NewTags("nvidia", "nvidia-0x2184", "nvidia_lt_31.0.15.4601", "nvidia_lt_535.183.01"), - Status: result.Failure, - }, - } - - expectedOutput := []Expectation{ - { - Bug: "crbug.com/1234", - Tags: result.NewTags("nvidia-0x2184", "nvidia_lt_535.183.01"), - Query: "webgpu:shader,execution,memory_layout:read_layout:", - Status: []string{"Failure"}, - Comment: "comment", - }, - } - - u := createGenericUpdater(t) - output := u.resultsToExpectations(results, "crbug.com/1234", "comment") - assert.Equal(t, output, expectedOutput) -} - -// Tests behavior related to automatic inclusion of a trailing :. -func TestResultsToExpectationsTrailingColon(t *testing.T) { - results := result.List{ - // Should automatically have a : added since it's a test query. - { - Query: query.Parse("webgpu:shader,execution,memory_layout:read_layout"), - Tags: result.NewTags("linux", "nvidia"), - Status: result.Failure, - }, - // Should not have a : added since it is a wildcard query. - { - Query: query.Parse("webgpu:shader,execution,*"), - Tags: result.NewTags("win10", "intel"), - Status: result.Failure, - }, - } - - expectedOutput := []Expectation{ - { - Bug: "crbug.com/1234", - Tags: result.NewTags("win10", "intel"), - Query: "webgpu:shader,execution,*", - Status: []string{"Failure"}, - Comment: "comment", - }, - { - Bug: "crbug.com/1234", - Tags: result.NewTags("linux", "nvidia"), - Query: "webgpu:shader,execution,memory_layout:read_layout:", - Status: []string{"Failure"}, - Comment: "comment", - }, - } - - u := createGenericUpdater(t) - output := u.resultsToExpectations(results, "crbug.com/1234", "comment") - assert.Equal(t, output, expectedOutput) -} - /******************************************************************************* * removeExpectationsForUnknownTests tests ******************************************************************************/ diff --git a/tools/src/cts/query/tree.go b/tools/src/cts/query/tree.go index 91ac323b281..4288b6f1e1b 100644 --- a/tools/src/cts/query/tree.go +++ b/tools/src/cts/query/tree.go @@ -112,105 +112,6 @@ func (n *TreeNode[Data]) traverse(f func(n *TreeNode[Data]) error) error { return nil } -// Merger is a function used to merge the children nodes of a tree. -// Merger is called with the Data of each child node. If the function returns a -// non-nil Data pointer, then this is used as the merged result. If the function -// returns nil, then the node will not be merged. -type Merger[Data any] func([]Data) *Data - -// merge collapses tree nodes based on child node data, using the function f. -// merge operates on the leaf nodes first, working its way towards the root of -// the tree. -// Returns the merged target data for this node, or nil if the node is not a -// leaf and its children has non-uniform data. -func (n *TreeNode[Data]) merge(f Merger[Data]) *Data { - // If the node is a leaf, then simply return the node's data. - if len(n.Children) == 0 { - return n.Data - } - - // Build a map of child target to merged child data. - // A nil for the value indicates that one or more children could not merge. - mergedChildren := map[Target][]Data{} - for key, child := range n.Children { - // Call merge() on the child. Even if we cannot merge this node, we want - // to do this for all children so they can merge their sub-graphs. - childData := child.merge(f) - - if childData == nil { - // If merge() returned nil, then the data could not be merged. - // Mark the entire target as unmergeable. - mergedChildren[key.Target] = nil - continue - } - - // Fetch the merge list for this child's target. - list, found := mergedChildren[key.Target] - if !found { - // First child with the given target? - mergedChildren[key.Target] = []Data{*childData} - continue - } - if list != nil { - mergedChildren[key.Target] = append(list, *childData) - } - } - - merge := func(in []Data) *Data { - switch len(in) { - case 0: - return nil // nothing to merge. - case 1: - return &in[0] // merge of a single item results in that item - default: - return f(in) - } - } - - // Might it possible to merge this node? - maybeMergeable := true - - // The merged data, per target - mergedTargets := map[Target]Data{} - - // Attempt to merge each of the target's data - for target, list := range mergedChildren { - if list != nil { // nil == unmergeable target - if data := merge(list); data != nil { - // Merge success! - mergedTargets[target] = *data - continue - } - } - maybeMergeable = false // Merge of this node is not possible - } - - // Remove all children that have been merged - for key := range n.Children { - if _, merged := mergedTargets[key.Target]; merged { - delete(n.Children, key) - } - } - - // Add wildcards for merged targets - for target, data := range mergedTargets { - data := data // Don't take address of iterator - n.getOrCreateChild(TreeNodeChildKey{"*", target}).Data = &data - } - - // If any of the targets are unmergeable, then we cannot merge the node itself. - if !maybeMergeable { - return nil - } - - // All targets were merged. Attempt to merge each of the targets. - data := make([]Data, 0, len(mergedTargets)) - for _, d := range mergedTargets { - data = append(data, d) - } - return merge(data) -} - // print writes a textual representation of this node and its children to w. // prefix is used as the line prefix for each node, which is appended with // whitespace for each child node. @@ -280,23 +181,6 @@ func (t *Tree[Data]) Add(q Query, d Data) error { return nil } -// Split adds a new data to the tree, clearing any ancestor node's data. -// Returns ErrDuplicateData if the tree already contains a data for the given node at query -func (t *Tree[Data]) Split(q Query, d Data) error { - node := &t.TreeNode - q.Walk(func(q Query, t Target, n string) error { - delete(node.Children, TreeNodeChildKey{Name: "*", Target: t}) - node.Data = nil - node = node.getOrCreateChild(TreeNodeChildKey{n, t}) - return nil - }) - if node.Data != nil { - return ErrDuplicateData{node.Query} - } - node.Data = &d - return nil -} - // GetOrCreate returns existing, or adds a new data to the tree. func (t *Tree[Data]) GetOrCreate(q Query, create func() Data) *Data { node := &t.TreeNode @@ -327,40 +211,6 @@ func (t *Tree[Data]) Get(q Query) *TreeNode[Data] { return node } -// Reduce reduces the tree using the Merger function f. -// If the Merger function returns a non-nil Data value, then this will be used -// to replace the non-leaf node with a new leaf node holding the returned Data. -// This process recurses up to the tree root. -func (t *Tree[Data]) Reduce(f Merger[Data]) { - for _, root := range t.TreeNode.Children { - root.merge(f) - } -} - -// ReduceUnder reduces the sub-tree under the given query using the Merger -// function f. -// If the Merger function returns a non-nil Data value, then this will be used -// to replace the non-leaf node with a new leaf node holding the returned Data. -// This process recurses up to the node pointed at by the query to. -func (t *Tree[Data]) ReduceUnder(to Query, f Merger[Data]) error { - node := &t.TreeNode - return to.Walk(func(q Query, t Target, n string) error { - if n == "*" { - node.merge(f) - return nil - } - child, ok := node.Children[TreeNodeChildKey{n, t}] - if !ok { - return ErrNoDataForQuery{q} - } - node = child - if q == to { - node.merge(f) - } - return nil - }) -} - // glob calls f for every node under the given query. func (t *Tree[Data]) glob(fq Query, f func(f *TreeNode[Data]) error) error { node := &t.TreeNode @@ -403,44 +253,6 @@ func (t *Tree[Data]) glob(fq Query, f func(f *TreeNode[Data]) error) error { }) } -// Replace replaces the sub-tree matching the query 'what' with the Data 'with' -func (t *Tree[Data]) Replace(what Query, with Data) error { - node := &t.TreeNode - return what.Walk(func(q Query, t Target, n string) error { - childKey := TreeNodeChildKey{n, t} - if q == what { - for key, child := range node.Children { - // Use Query.Contains() to handle matching of Cases - // (which are not split into tree nodes) - if q.Contains(child.Query) { - delete(node.Children, key) - } - } - node = node.getOrCreateChild(childKey) - node.Data = &with - } else { - child, ok := node.Children[childKey] - if !ok { - return ErrNoDataForQuery{q} - } - node = child - } - return nil - }) -} - -// List returns the tree nodes flattened as a list of QueryData -func (t *Tree[Data]) List() []QueryData[Data] { - out := []QueryData[Data]{} - t.traverse(func(n *TreeNode[Data]) error { - if n.Data != nil { - out = append(out, QueryData[Data]{n.Query, *n.Data}) - } - return nil - }) - return out -} - // Glob returns a list of QueryData's for every node that is under the given // query, which holds data. // Glob handles wildcards as well as non-wildcard queries: diff --git a/tools/src/cts/query/tree_test.go b/tools/src/cts/query/tree_test.go index c2e9ce7fc5f..9ed14f52649 100644 --- a/tools/src/cts/query/tree_test.go +++ b/tools/src/cts/query/tree_test.go @@ -4,9 +4,7 @@ import ( "fmt" "testing" - "dawn.googlesource.com/dawn/tools/src/container" "dawn.googlesource.com/dawn/tools/src/cts/query" - "dawn.googlesource.com/dawn/tools/src/fileutils" "github.com/google/go-cmp/cmp" ) @@ -350,635 +348,6 @@ func TestNewWithCollision(t *testing.T) { } } -func TestSplit(t *testing.T) { - type Tree = query.Tree[string] - type Node = query.TreeNode[string] - type QueryData = query.QueryData[string] - type Children = query.TreeNodeChildren[string] - - type Test struct { - in QueryData - pre Tree - post Tree - } - for _, test := range []Test{ - { ///////////////////////////////////////////////////////////////////// - in: QueryData{ - Query: Q(`suite:*`), - Data: pass, - }, - post: Tree{ - TreeNode: Node{ - Children: Children{ - query.TreeNodeChildKey{`suite`, query.Suite}: { - Query: Q(`suite`), - Children: Children{ - query.TreeNodeChildKey{`*`, query.Files}: { - Query: Q(`suite:*`), - Data: &pass, - }, - }, - }, - }, - }, - }, - }, - { ///////////////////////////////////////////////////////////////////// - in: QueryData{ - Query: Q(`suite:a,b:*`), - Data: pass, - }, - pre: Tree{ - TreeNode: Node{ - Children: Children{ - query.TreeNodeChildKey{`suite`, query.Suite}: { - Query: Q(`suite`), - Children: Children{ - query.TreeNodeChildKey{`a`, query.Files}: { - Query: Q(`suite:a`), - Children: Children{ - query.TreeNodeChildKey{`*`, query.Files}: { - Query: Q(`suite:a,*`), - Data: &pass, - }, - }, - }, - }, - }, - }, - }, - }, - post: Tree{ - TreeNode: Node{ - Children: Children{ - query.TreeNodeChildKey{`suite`, query.Suite}: { - Query: Q(`suite`), - Children: Children{ - query.TreeNodeChildKey{`a`, query.Files}: { - Query: Q(`suite:a`), - Children: Children{ - query.TreeNodeChildKey{`b`, query.Files}: { - Query: Q(`suite:a,b`), - Children: Children{ - query.TreeNodeChildKey{`*`, query.Tests}: { - Query: Q(`suite:a,b:*`), - Data: &pass, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - { ///////////////////////////////////////////////////////////////////// - in: QueryData{ - Query: Q(`suite:a:*`), - Data: pass, - }, - pre: Tree{ - TreeNode: Node{ - Children: Children{ - query.TreeNodeChildKey{`suite`, query.Suite}: { - Query: Q(`suite`), - Children: Children{ - query.TreeNodeChildKey{`a`, query.Files}: { - Query: Q(`suite:a`), - Children: Children{ - query.TreeNodeChildKey{`b`, query.Files}: { - Query: Q(`suite:a,b`), - Children: Children{ - query.TreeNodeChildKey{`*`, query.Files}: { - Query: Q(`suite:a,b,*`), - Data: &pass, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - post: Tree{ - TreeNode: Node{ - Children: Children{ - query.TreeNodeChildKey{`suite`, query.Suite}: { - Query: Q(`suite`), - Children: Children{ - query.TreeNodeChildKey{`a`, query.Files}: { - Query: Q(`suite:a`), - Children: Children{ - query.TreeNodeChildKey{`*`, query.Tests}: { - Query: Q(`suite:a:*`), - Data: &pass, - }, - query.TreeNodeChildKey{`b`, query.Files}: { - Query: Q(`suite:a,b`), - Children: Children{ - query.TreeNodeChildKey{`*`, query.Files}: { - Query: Q(`suite:a,b,*`), - Data: &pass, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - { ///////////////////////////////////////////////////////////////////// - in: QueryData{ - Query: Q(`suite:a,b:c:*`), - Data: pass, - }, - pre: Tree{ - TreeNode: Node{ - Children: Children{ - query.TreeNodeChildKey{`suite`, query.Suite}: { - Query: Q(`suite`), - Children: Children{ - query.TreeNodeChildKey{`a`, query.Files}: { - Query: Q(`suite:a`), - Children: Children{ - query.TreeNodeChildKey{`b`, query.Files}: { - Query: Q(`suite:a,b`), - Children: Children{ - query.TreeNodeChildKey{`*`, query.Tests}: { - Query: Q(`suite:a,b:*`), - Data: &pass, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - post: Tree{ - TreeNode: Node{ - Children: Children{ - query.TreeNodeChildKey{`suite`, query.Suite}: { - Query: Q(`suite`), - Children: Children{ - query.TreeNodeChildKey{`a`, query.Files}: { - Query: Q(`suite:a`), - Children: Children{ - query.TreeNodeChildKey{`b`, query.Files}: { - Query: Q(`suite:a,b`), - Children: Children{ - query.TreeNodeChildKey{`c`, query.Tests}: { - Query: Q(`suite:a,b:c`), - Children: Children{ - query.TreeNodeChildKey{`*`, query.Cases}: { - Query: Q(`suite:a,b:c:*`), - Data: &pass, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } { - tree := test.pre - if err := tree.Split(test.in.Query, test.in.Data); err != nil { - t.Errorf("NewTree(%v): %v", test.in, err) - continue - } - if diff := cmp.Diff(tree, test.post); diff != "" { - t.Errorf("Split(%v) tree was not as expected:\n%v", test.in, diff) - } - } -} - -func TestList(t *testing.T) { - type QueryData = query.QueryData[string] - - tree, err := NewTree(t, - QueryData{Query: Q(`suite:*`), Data: skip}, - QueryData{Query: Q(`suite:a,*`), Data: failure}, - QueryData{Query: Q(`suite:a,b,*`), Data: failure}, - QueryData{Query: Q(`suite:a,b:c:*`), Data: failure}, - QueryData{Query: Q(`suite:a,b:c:d;*`), Data: failure}, - QueryData{Query: Q(`suite:a,b:c:d="e";*`), Data: failure}, - QueryData{Query: Q(`suite:h,b:c:f="g";*`), Data: abort}, - QueryData{Query: Q(`suite:a,b:c:f="g";*`), Data: skip}, - ) - if err != nil { - t.Fatalf("NewTree() returned %v", err) - } - - got := tree.List() - expect := []QueryData{ - {Query: Q(`suite:*`), Data: skip}, - {Query: Q(`suite:a,*`), Data: failure}, - {Query: Q(`suite:a,b,*`), Data: failure}, - {Query: Q(`suite:a,b:c:*`), Data: failure}, - {Query: Q(`suite:a,b:c:d;*`), Data: failure}, - {Query: Q(`suite:a,b:c:d="e";*`), Data: failure}, - {Query: Q(`suite:a,b:c:f="g";*`), Data: skip}, - {Query: Q(`suite:h,b:c:f="g";*`), Data: abort}, - } - if diff := cmp.Diff(got, expect); diff != "" { - t.Errorf("List() was not as expected:\n%v", diff) - } -} - -// reducer is used by Reduce() and ReduceUnder() tests for reducing the tree. -// reducer returns a pointer to the common string if all strings in data are -// equal, otherwise returns nil -func reducer(data []string) *string { - if s := container.NewSet(data...); len(s) == 1 { - item := s.One() - return &item - } - return nil -} - -func TestReduce(t *testing.T) { - type QueryData = query.QueryData[string] - - type Test struct { - name string - in []QueryData - expect []QueryData - } - for _, test := range []Test{ - { ////////////////////////////////////////////////////////////////////// - name: "Different file results - A", - in: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: failure}, - {Query: Q(`suite:a,c,*`), Data: pass}, - }, - expect: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: failure}, - {Query: Q(`suite:a,c,*`), Data: pass}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "Different file results - B", - in: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: failure}, - {Query: Q(`suite:a,c,*`), Data: pass}, - {Query: Q(`suite:a,d,*`), Data: skip}, - }, - expect: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: failure}, - {Query: Q(`suite:a,c,*`), Data: pass}, - {Query: Q(`suite:a,d,*`), Data: skip}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "Different test results", - in: []QueryData{ - {Query: Q(`suite:a,b:*`), Data: failure}, - {Query: Q(`suite:a,c:*`), Data: pass}, - }, - expect: []QueryData{ - {Query: Q(`suite:a,b:*`), Data: failure}, - {Query: Q(`suite:a,c:*`), Data: pass}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "Same file results", - in: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: failure}, - {Query: Q(`suite:a,c,*`), Data: failure}, - }, - expect: []QueryData{ - {Query: Q(`suite:*`), Data: failure}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "Same test results", - in: []QueryData{ - {Query: Q(`suite:a,b:*`), Data: failure}, - {Query: Q(`suite:a,c:*`), Data: failure}, - }, - expect: []QueryData{ - {Query: Q(`suite:*`), Data: failure}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "File vs test", - in: []QueryData{ - {Query: Q(`suite:a:b,c*`), Data: failure}, - {Query: Q(`suite:a,b,c*`), Data: pass}, - }, - expect: []QueryData{ - {Query: Q(`suite:a,*`), Data: pass}, - {Query: Q(`suite:a:*`), Data: failure}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "Sibling cases, no reduce", - in: []QueryData{ - {Query: Q(`suite:a:b:c;d=e;f=g;*`), Data: failure}, - {Query: Q(`suite:a:b:c;d=e;f=h;*`), Data: pass}, - }, - expect: []QueryData{ - {Query: Q(`suite:a:b:c;d=e;f=g;*`), Data: failure}, - {Query: Q(`suite:a:b:c;d=e;f=h;*`), Data: pass}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "Sibling cases, reduce to test", - in: []QueryData{ - {Query: Q(`suite:a:b:c=1;d="x";*`), Data: failure}, - {Query: Q(`suite:a:b:c=1;d="y";*`), Data: failure}, - {Query: Q(`suite:a:z:*`), Data: pass}, - }, - expect: []QueryData{ - {Query: Q(`suite:a:b:*`), Data: failure}, - {Query: Q(`suite:a:z:*`), Data: pass}, - }, - }, - } { - tree, err := NewTree(t, test.in...) - if err != nil { - t.Errorf("Test '%v':\nNewTree() returned %v", test.name, err) - continue - } - tree.Reduce(reducer) - results := tree.List() - if diff := cmp.Diff(results, test.expect); diff != "" { - t.Errorf("Test '%v':\n%v", test.name, diff) - } - } -} - -func TestReduceUnder(t *testing.T) { - type QueryData = query.QueryData[string] - - type Test struct { - location string - to query.Query - in []QueryData - expect []QueryData - expectErr error - } - for _, test := range []Test{ - { ////////////////////////////////////////////////////////////////////// - location: fileutils.ThisLine(), - to: Q(`suite:a,b,*`), - in: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: failure}, - }, - expect: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: failure}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - location: fileutils.ThisLine(), - to: Q(`suite:a,*`), - in: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: failure}, - }, - expect: []QueryData{ - {Query: Q(`suite:a,*`), Data: failure}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - location: fileutils.ThisLine(), - to: Q(`suite:*`), - in: []QueryData{ - {Query: Q(`suite:a,b:*`), Data: failure}, - }, - expect: []QueryData{ - {Query: Q(`suite:*`), Data: failure}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - location: fileutils.ThisLine(), - to: Q(`suite:a,*`), - in: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: failure}, - {Query: Q(`suite:a,c,*`), Data: pass}, - }, - expect: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: failure}, - {Query: Q(`suite:a,c,*`), Data: pass}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - location: fileutils.ThisLine(), - to: Q(`suite:a,*`), - in: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: pass}, - {Query: Q(`suite:a,c,*`), Data: pass}, - }, - expect: []QueryData{ - {Query: Q(`suite:a,*`), Data: pass}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - location: fileutils.ThisLine(), - to: Q(`suite:a`), - in: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: pass}, - {Query: Q(`suite:a,c,*`), Data: pass}, - }, - expect: []QueryData{ - {Query: Q(`suite:a,*`), Data: pass}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - location: fileutils.ThisLine(), - to: Q(`suite:x`), - in: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: pass}, - {Query: Q(`suite:a,c,*`), Data: pass}, - }, - expect: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: pass}, - {Query: Q(`suite:a,c,*`), Data: pass}, - }, - expectErr: query.ErrNoDataForQuery{ - Query: Q(`suite:x`), - }, - }, - { ////////////////////////////////////////////////////////////////////// - location: fileutils.ThisLine(), - to: Q(`suite:a,b,c,*`), - in: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: pass}, - }, - expect: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: pass}, - }, - expectErr: query.ErrNoDataForQuery{ - Query: Q(`suite:a,b,c`), - }, - }, - } { - tree, err := NewTree(t, test.in...) - if err != nil { - t.Errorf("\n%v NewTree(): %v", test.location, err) - continue - } - err = tree.ReduceUnder(test.to, reducer) - if diff := cmp.Diff(err, test.expectErr); diff != "" { - t.Errorf("\n%v ReduceUnder(): %v", test.location, err) - } - results := tree.List() - if diff := cmp.Diff(results, test.expect); diff != "" { - t.Errorf("\n%v List(): %v", test.location, diff) - } - } -} - -func TestReplace(t *testing.T) { - type QueryData = query.QueryData[string] - - type Test struct { - name string - base []QueryData - replacement QueryData - expect []QueryData - expectErr error - } - for _, test := range []Test{ - { ////////////////////////////////////////////////////////////////////// - name: "Replace file. Direct", - base: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: failure}, - {Query: Q(`suite:a,c,*`), Data: pass}, - }, - replacement: QueryData{Q(`suite:a,b,*`), skip}, - expect: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: skip}, - {Query: Q(`suite:a,c,*`), Data: pass}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "Replace file. Indirect", - base: []QueryData{ - {Query: Q(`suite:a,b,c,*`), Data: failure}, - {Query: Q(`suite:a,b,d,*`), Data: pass}, - {Query: Q(`suite:a,c,*`), Data: pass}, - }, - replacement: QueryData{Q(`suite:a,b,*`), skip}, - expect: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: skip}, - {Query: Q(`suite:a,c,*`), Data: pass}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "File vs Test", - base: []QueryData{ - {Query: Q(`suite:a,b:c,*`), Data: crash}, - {Query: Q(`suite:a,b:d,*`), Data: abort}, - {Query: Q(`suite:a,b,c,*`), Data: failure}, - {Query: Q(`suite:a,b,d,*`), Data: pass}, - }, - replacement: QueryData{Q(`suite:a,b,*`), skip}, - expect: []QueryData{ - {Query: Q(`suite:a,b,*`), Data: skip}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "Cases. * with *", - base: []QueryData{ - {Query: Q(`suite:file:test:*`), Data: failure}, - }, - replacement: QueryData{Q(`suite:file:test:*`), pass}, - expect: []QueryData{ - {Query: Q(`suite:file:test:*`), Data: pass}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "Cases. Mixed with *", - base: []QueryData{ - {Query: Q(`suite:file:test:a=1,*`), Data: failure}, - {Query: Q(`suite:file:test:a=2,*`), Data: skip}, - {Query: Q(`suite:file:test:a=3,*`), Data: crash}, - }, - replacement: QueryData{Q(`suite:file:test:*`), pass}, - expect: []QueryData{ - {Query: Q(`suite:file:test:*`), Data: pass}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "Cases. Replace partial - (a=1)", - base: []QueryData{ - {Query: Q(`suite:file:test:a=1;b=x;*`), Data: failure}, - {Query: Q(`suite:file:test:a=1;b=y;*`), Data: failure}, - {Query: Q(`suite:file:test:a=2;b=y;*`), Data: failure}, - }, - replacement: QueryData{Q(`suite:file:test:a=1;*`), pass}, - expect: []QueryData{ - {Query: Q(`suite:file:test:a=1;*`), Data: pass}, - {Query: Q(`suite:file:test:a=2;b=y;*`), Data: failure}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "Cases. Replace partial - (b=y)", - base: []QueryData{ - {Query: Q(`suite:file:test:a=1;b=x;*`), Data: failure}, - {Query: Q(`suite:file:test:a=1;b=y;*`), Data: failure}, - {Query: Q(`suite:file:test:a=2;b=y;*`), Data: failure}, - }, - replacement: QueryData{Q(`suite:file:test:b=y;*`), pass}, - expect: []QueryData{ - {Query: Q(`suite:file:test:a=1;b=x;*`), Data: failure}, - {Query: Q(`suite:file:test:b=y;*`), Data: pass}, - }, - }, - { ////////////////////////////////////////////////////////////////////// - name: "Error. No data for query - short", - base: []QueryData{ - {Query: Q(`suite:file:test:a=1;b=x;*`), Data: failure}, - }, - replacement: QueryData{Q(`suite:missing:*`), pass}, - expect: []QueryData{ - {Query: Q(`suite:file:test:a=1;b=x;*`), Data: failure}, - }, - expectErr: query.ErrNoDataForQuery{Q(`suite:missing`)}, - }, - { ////////////////////////////////////////////////////////////////////// - name: "Error. No data for query - long", - base: []QueryData{ - {Query: Q(`suite:file:test:*`), Data: failure}, - }, - replacement: QueryData{Q(`suite:file:test,missing,*`), pass}, - expect: []QueryData{ - {Query: Q(`suite:file:test:*`), Data: failure}, - }, - expectErr: query.ErrNoDataForQuery{Q(`suite:file:test,missing`)}, - }, - } { - tree, err := NewTree(t, test.base...) - if err != nil { - t.Errorf("Test '%v':\nNewTree(): %v", test.name, err) - continue - } - err = tree.Replace(test.replacement.Query, test.replacement.Data) - if diff := cmp.Diff(err, test.expectErr); diff != "" { - t.Errorf("Test '%v':\nReplace() error: %v", test.name, err) - continue - } - if diff := cmp.Diff(tree.List(), test.expect); diff != "" { - t.Errorf("Test '%v':\n%v", test.name, diff) - } - } -} - func TestGlob(t *testing.T) { type QueryData = query.QueryData[string] diff --git a/tools/src/cts/result/mvt.go b/tools/src/cts/result/mvt.go deleted file mode 100644 index 32059a62be7..00000000000 --- a/tools/src/cts/result/mvt.go +++ /dev/null @@ -1,159 +0,0 @@ -// Copyright 2022 The Dawn & Tint Authors -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package result - -import ( - "sort" - - "dawn.googlesource.com/dawn/tools/src/cts/query" -) - -// MinimalVariantTags accepts a list of tag-sets (e.g GPU tags, OS tags, etc), -// and returns an optimized list of variants, folding together variants that -// have identical result query-to-status mappings, and removing redundant tags. -// -// MinimalVariantTags will attempt to remove variant tags starting with the -// first set of tags in tagSets, then second, and so on. If a tag-set cannot -// be removed, then the tags of the set are left alone, and the algorithm will -// progress to the next tag-set. -// -// MinimalVariantTags assumes that there are no duplicate results (same query, -// same tags) in l. -func (l List) MinimalVariantTags(tagSets []Tags) []Variant { - type VariantData struct { - // The variant tags - tags Variant - // The query -> status for all results in l that have this variant's - // tags. - queryToStatus map[query.Query]Status - } - - variants := []VariantData{} - - // Build the initial list of variants from l. - // Bin result [query -> status] to the variant. - { - variantIndices := map[string]int{} - for _, r := range l { - key := TagsToString(r.Tags) - if idx, found := variantIndices[key]; !found { - variantIndices[key] = len(variants) - variants = append(variants, VariantData{ - tags: Variant(r.Tags.Clone()), - queryToStatus: map[query.Query]Status{ - r.Query: r.Status, - }, - }) - } else { - variants[idx].queryToStatus[r.Query] = r.Status - } - } - } - - // canReduce checks that the variant would match the same results if the - // tags were reduced to 'tags'. Returns true if the variant's tags could - // be reduced, otherwise false. - canReduce := func(variant VariantData, tags Tags) bool { - for _, r := range l.FilterByTags(tags) { - existing, found := variant.queryToStatus[r.Query] - if !found { - // Removing the tag has expanded the set of queries. - return false - } - if existing != r.Status { - // Removing the tag has resulted in two queries with different - // results. - return false - } - } - return true - } - - // tryToRemoveTags will remove all the tags in 'tags' from all variants - // iff doing so does not affect the set of results filtered by each variant. - // If it was possible to remove the tags, then variants that now have the - // same tags may be folded together, reducing the total number of variants. - tryToRemoveTags := func(tags Tags) { - newVariants := make([]VariantData, 0, len(variants)) - - for _, v := range variants { - // Does the variant even contain these tags? - if !v.tags.ContainsAny(tags) { - // Nope. Skip the canReduce() call, and keep the variant. - newVariants = append(newVariants, v) - continue - } - - // Build the new set of tags with 'tags' removed. - newTags := v.tags.Clone() - newTags.RemoveAll(tags) - - // Check wether removal of these tags affected the outcome. - if !canReduce(v, newTags) { - // Removing these tags resulted in differences. - return // Abort - } - newVariants = append(newVariants, VariantData{newTags, v.queryToStatus}) - } - - // Remove variants that are now subsets of others. - // Start by sorting the variants by number of tags. - // This ensures that the variants with fewer tags (fewer constraints) - // come first. - sort.Slice(newVariants, func(i, j int) bool { - return len(newVariants[i].tags) < len(newVariants[j].tags) - }) - - // Now check each variant's tags against the previous variant tags. - // As we've sorted, we know that supersets (fewer-tags) come before - // subsets (more-tags). - variants = []VariantData{} - - nextVariant: - for i, v1 := range newVariants { // for variants 0..N - for _, v2 := range newVariants[:i] { // for variants 0..i - if v1.tags.ContainsAll(v2.tags) { - continue nextVariant // v1 is a subset of v2. Omit. - } - } - variants = append(variants, v1) - } - } - - // Attempt to remove the tag sets from the variants, one by one. - for _, tags := range tagSets { - tryToRemoveTags(tags) - } - - // Return the final set of unique variants - out := make([]Variant, len(variants)) - for i, v := range variants { - out[i] = v.tags - } - return out -} diff --git a/tools/src/cts/result/mvt_test.go b/tools/src/cts/result/mvt_test.go deleted file mode 100644 index d1df908f4af..00000000000 --- a/tools/src/cts/result/mvt_test.go +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2022 The Dawn & Tint Authors -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// 3. Neither the name of the copyright holder nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package result_test - -import ( - "fmt" - "testing" - - "dawn.googlesource.com/dawn/tools/src/cts/result" - "dawn.googlesource.com/dawn/tools/src/fileutils" - "github.com/google/go-cmp/cmp" -) - -func TestMinimalVariantTags(t *testing.T) { - type Test struct { - location string - results result.List - expect []result.Variant - } - for _, test := range []Test{ - { ////////////////////////////////////////////////////////////////////// - location: fileutils.ThisLine(), - results: result.List{}, - expect: []result.Variant{}, - }, { /////////////////////////////////////////////////////////////////// - // Single variant, that can be entirely optimized away - location: fileutils.ThisLine(), - results: result.List{ - {Query: Q("a:b,c:d,*"), Tags: T("a0", "b1", "c2"), Status: result.Pass}, - }, - expect: []result.Variant{T()}, - }, { /////////////////////////////////////////////////////////////////// - // Multiple variants on the same query. - // Can also be entirely optimized away. - location: fileutils.ThisLine(), - results: result.List{ - {Query: Q("a:b,c:d,*"), Tags: T("a0", "b1", "c2"), Status: result.Pass}, - {Query: Q("a:b,c:d,*"), Tags: T("a1", "b2", "c0"), Status: result.Pass}, - {Query: Q("a:b,c:d,*"), Tags: T("a2", "b1", "c0"), Status: result.Pass}, - }, - expect: []result.Variant{T()}, - }, { /////////////////////////////////////////////////////////////////// - // Two variants where the 1st and 2nd tag-sets are redundant. - location: fileutils.ThisLine(), - results: result.List{ - {Query: Q("a:b,c:d,*"), Tags: T("a0", "b0", "c0"), Status: result.Pass}, - {Query: Q("a:b,c:d,*"), Tags: T("a1", "b1", "c1"), Status: result.Failure}, - }, - expect: []result.Variant{T("c0"), T("c1")}, - }, { /////////////////////////////////////////////////////////////////// - // Two variants where the 1st and 3rd tag-sets are redundant. - location: fileutils.ThisLine(), - results: result.List{ - {Query: Q("a:b,c:d,*"), Tags: T("a0", "b0", "c0"), Status: result.Pass}, - {Query: Q("a:b,c:d,*"), Tags: T("a1", "b1", "c1"), Status: result.Failure}, - {Query: Q("a:b,c:d,*"), Tags: T("a0", "b0", "c1"), Status: result.Pass}, - {Query: Q("a:b,c:d,*"), Tags: T("a1", "b1", "c0"), Status: result.Failure}, - }, - expect: []result.Variant{T("b0"), T("b1")}, - }, { /////////////////////////////////////////////////////////////////// - // Two variants where the 2nd and 3rd tag-sets are redundant. - location: fileutils.ThisLine(), - results: result.List{ - {Query: Q("a:b,c:d,*"), Tags: T("a0", "b0", "c0"), Status: result.Pass}, - {Query: Q("a:b,c:d,*"), Tags: T("a1", "b1", "c1"), Status: result.Failure}, - {Query: Q("a:b,c:d,*"), Tags: T("a0", "b1", "c1"), Status: result.Pass}, - {Query: Q("a:b,c:d,*"), Tags: T("a1", "b0", "c0"), Status: result.Failure}, - }, - expect: []result.Variant{T("a0"), T("a1")}, - }, { /////////////////////////////////////////////////////////////////// - // Check that variants aren't optimized to expand the set of results - // they target, even if results are uniform - location: fileutils.ThisLine(), - results: result.List{ - {Query: Q("a:b,c:d0,*"), Tags: T("a0", "b0", "c0"), Status: result.Pass}, - {Query: Q("a:b,c:d1,*"), Tags: T("a1", "b1", "c1"), Status: result.Pass}, - }, - expect: []result.Variant{T("c0"), T("c1")}, - }, { /////////////////////////////////////////////////////////////////// - // Exercise the optimizations to skip checks on tag removals that - // aren't found in all variants - location: fileutils.ThisLine(), - results: result.List{ - {Query: Q("a:b,c:d0,*"), Tags: T("a0"), Status: result.Pass}, - {Query: Q("a:b,c:d1,*"), Tags: T("b0"), Status: result.Pass}, - {Query: Q("a:b,c:d2,*"), Tags: T("c0"), Status: result.Pass}, - }, - expect: []result.Variant{T("a0"), T("b0"), T("c0")}, - }, - } { - preReduce := fmt.Sprint(test.results) - got := test.results.MinimalVariantTags([]result.Tags{ - T("a0", "a1", "a2"), - T("b0", "b1", "b2"), - T("c0", "c1", "c2"), - }) - postReduce := fmt.Sprint(test.results) - if diff := cmp.Diff(got, test.expect); diff != "" { - t.Errorf("%v MinimalVariantTags() diff:\n%v", test.location, diff) - } - if diff := cmp.Diff(preReduce, postReduce); diff != "" { - t.Errorf("%v MinimalVariantTags() modified original list:\n%v", test.location, diff) - } - } -} diff --git a/tools/src/cts/result/result.go b/tools/src/cts/result/result.go index 9f24b9b9ece..a3a9c7d3aa9 100644 --- a/tools/src/cts/result/result.go +++ b/tools/src/cts/result/result.go @@ -186,19 +186,6 @@ type ExecutionMode string // Lists of test results by execution mode. type ResultsByExecutionMode map[ExecutionMode]List -// Variant is a collection of tags that uniquely identify a test -// configuration (e.g the combination of OS, GPU, validation-modes, etc). -type Variant = Tags - -// Variants returns the list of unique tags (variants) across all results. -func (l List) Variants() []Variant { - tags := container.NewMap[string, Variant]() - for _, r := range l { - tags.Add(TagsToString(r.Tags), r.Tags) - } - return tags.Values() -} - // TransformTags returns the list of results with the tags transformed using f. // TransformTags assumes that f will return the same output for the same input. func (l List) TransformTags(f func(Tags) Tags) List { @@ -328,14 +315,6 @@ func (l List) FilterByTags(tags Tags) List { }) } -// FilterByVariant returns the results that exactly match the given tags -func (l List) FilterByVariant(tags Tags) List { - str := TagsToString(tags) - return l.Filter(func(r Result) bool { - return len(r.Tags) == len(tags) && TagsToString(r.Tags) == str - }) -} - // FilterByQuery returns the results that match the given query func (l List) FilterByQuery(q query.Query) List { return l.Filter(func(r Result) bool { @@ -358,21 +337,6 @@ func (l List) Statuses() Statuses { return set } -// StatusTree is a query tree of statuses -type StatusTree = query.Tree[Status] - -// StatusTree returns a query.Tree from the List, with the Status as the tree -// node data. -func (l List) StatusTree() (StatusTree, error) { - tree := StatusTree{} - for _, r := range l { - if err := tree.Add(r.Query, r.Status); err != nil { - return StatusTree{}, err - } - } - return tree, nil -} - // Load loads the result list from the file with the given path func Load(path string) (ResultsByExecutionMode, error) { file, err := os.Open(path) diff --git a/tools/src/cts/result/result_test.go b/tools/src/cts/result/result_test.go index 28b7007c8f9..456348f985c 100644 --- a/tools/src/cts/result/result_test.go +++ b/tools/src/cts/result/result_test.go @@ -116,120 +116,6 @@ func TestParseError(t *testing.T) { } } -func TestVariants(t *testing.T) { - type Test struct { - results result.List - expect []result.Tags - } - for _, test := range []Test{ - { ////////////////////////////////////////////////////////////////////// - results: result.List{ - result.Result{ - Query: Q(`a`), - Status: result.Pass, - Tags: result.NewTags(), - }, - }, - expect: []result.Tags{ - result.NewTags(), - }, - }, - { ////////////////////////////////////////////////////////////////////// - results: result.List{ - result.Result{ - Query: Q(`a`), - Status: result.Pass, - Tags: result.NewTags("x"), - }, - }, - expect: []result.Tags{ - result.NewTags("x"), - }, - }, - { ////////////////////////////////////////////////////////////////////// - results: result.List{ - result.Result{ - Query: Q(`a`), - Status: result.Pass, - Tags: result.NewTags("x", "y"), - }, - }, - expect: []result.Tags{ - result.NewTags("x", "y"), - }, - }, - { ////////////////////////////////////////////////////////////////////// - results: result.List{ - result.Result{ - Query: Q(`a`), - Status: result.Pass, - Tags: result.NewTags("x", "y"), - }, - result.Result{ - Query: Q(`b`), - Status: result.Pass, - Tags: result.NewTags("x", "y"), - }, - }, - expect: []result.Tags{ - result.NewTags("x", "y"), - }, - }, - { ////////////////////////////////////////////////////////////////////// - results: result.List{ - result.Result{ - Query: Q(`a`), - Status: result.Pass, - Tags: result.NewTags("x", "y"), - }, - result.Result{ - Query: Q(`b`), - Status: result.Pass, - Tags: result.NewTags("y", "z"), - }, - }, - expect: []result.Tags{ - result.NewTags("x", "y"), - result.NewTags("y", "z"), - }, - }, - { ////////////////////////////////////////////////////////////////////// - results: result.List{ - result.Result{ - Query: Q(`a`), - Status: result.Pass, - Tags: result.NewTags("x", "y"), - }, - result.Result{ - Query: Q(`b`), - Status: result.Pass, - Tags: result.NewTags("y", "z"), - }, - result.Result{ - Query: Q(`c`), - Status: result.Pass, - Tags: result.NewTags("z", "x"), - }, - result.Result{ - Query: Q(`d`), - Status: result.Pass, - Tags: result.NewTags("y", "z"), - }, - }, - expect: []result.Tags{ - result.NewTags("x", "y"), - result.NewTags("x", "z"), - result.NewTags("y", "z"), - }, - }, - } { - got := test.results.Variants() - if diff := cmp.Diff(got, test.expect); diff != "" { - t.Errorf("Results:\n%v\nUniqueTags() was not as expected:\n%v", test.results, diff) - } - } -} - func TestTransformTags(t *testing.T) { type Test struct { results result.List @@ -832,96 +718,6 @@ func TestFilterByTags(t *testing.T) { } } -func TestFilterByVariant(t *testing.T) { - type Test struct { - results result.List - tags result.Tags - expect result.List - } - for _, test := range []Test{ - { ////////////////////////////////////////////////////////////////////// - results: result.List{ - result.Result{ - Query: Q(`a`), - Status: result.Pass, - Tags: result.NewTags("x"), - }, - result.Result{ - Query: Q(`b`), - Status: result.Failure, - Tags: result.NewTags("y"), - }, - result.Result{ - Query: Q(`c`), - Status: result.Pass, - Tags: result.NewTags("x", "y"), - }, - }, - tags: result.NewTags("x", "y"), - expect: result.List{ - result.Result{ - Query: Q(`c`), - Status: result.Pass, - Tags: result.NewTags("x", "y"), - }, - }, - }, - { ////////////////////////////////////////////////////////////////////// - results: result.List{ - result.Result{ - Query: Q(`a`), - Status: result.Pass, - Tags: result.NewTags("x"), - }, - result.Result{ - Query: Q(`b`), - Status: result.Failure, - Tags: result.NewTags("y"), - }, - result.Result{ - Query: Q(`c`), - Status: result.Pass, - Tags: result.NewTags("x", "y"), - }, - }, - tags: result.NewTags("x"), - expect: result.List{ - result.Result{ - Query: Q(`a`), - Status: result.Pass, - Tags: result.NewTags("x"), - }, - }, - }, - { ////////////////////////////////////////////////////////////////////// - results: result.List{ - result.Result{ - Query: Q(`a`), - Status: result.Pass, - Tags: result.NewTags("x"), - }, - result.Result{ - Query: Q(`b`), - Status: result.Failure, - Tags: result.NewTags("y"), - }, - result.Result{ - Query: Q(`c`), - Status: result.Pass, - Tags: result.NewTags("x", "y"), - }, - }, - tags: result.NewTags("q"), - expect: result.List{}, - }, - } { - got := test.results.FilterByVariant(test.tags) - if diff := cmp.Diff(got, test.expect); diff != "" { - t.Errorf("Results:\n%v\nFilterByVariant(%v) was not as expected:\n%v", test.results, test.tags, diff) - } - } -} - func TestStatuses(t *testing.T) { type Test struct { results result.List @@ -992,67 +788,6 @@ func TestStatuses(t *testing.T) { } } -func TestStatusTree(t *testing.T) { - type Node = query.TreeNode[result.Status] - type Children = query.TreeNodeChildren[result.Status] - type ChildKey = query.TreeNodeChildKey - - pass := result.Pass - - type Test struct { - results result.List - expectErr error - expect result.StatusTree - } - for _, test := range []Test{ - { ////////////////////////////////////////////////////////////////////// - results: result.List{}, - expect: result.StatusTree{}, - }, - { ////////////////////////////////////////////////////////////////////// - results: result.List{ - {Query: Q(`suite:a:*`), Status: result.Pass}, - }, - expect: result.StatusTree{ - TreeNode: Node{ - Children: Children{ - ChildKey{Name: `suite`, Target: query.Suite}: &Node{ - Query: Q(`suite`), - Children: Children{ - ChildKey{Name: `a`, Target: query.Files}: &Node{ - Query: Q(`suite:a`), - Children: Children{ - ChildKey{Name: `*`, Target: query.Tests}: &Node{ - Query: Q(`suite:a:*`), - Data: &pass, - }, - }, - }, - }, - }, - }, - }, - }, - }, - { ////////////////////////////////////////////////////////////////////// - results: result.List{ - {Query: Q(`suite:a:*`), Status: result.Pass}, - {Query: Q(`suite:a:*`), Status: result.Failure}, - }, - expectErr: query.ErrDuplicateData{Query: Q(`suite:a:*`)}, - }, - } { - got, err := test.results.StatusTree() - if diff := cmp.Diff(err, test.expectErr); diff != "" { - t.Errorf("Results:\n%v\nStatusTree() error was not as expected:\n%v", test.results, diff) - continue - } - if diff := cmp.Diff(got, test.expect); diff != "" { - t.Errorf("Results:\n%v\nStatusTree() was not as expected:\n%v", test.results, diff) - } - } -} - func TestReadWrite(t *testing.T) { in := result.ResultsByExecutionMode{ "bar": result.List{