diff --git a/docs/matrix.md b/docs/matrix.md index fcffb8e7440..9727bc525c5 100644 --- a/docs/matrix.md +++ b/docs/matrix.md @@ -303,7 +303,7 @@ See the end-to-end example in [`PipelineRun` with `Matrix` and `Results`][pr-wit #### Results in Matrix.Params -`Matrix.Params` supports string replacements from `Results` of type String, Array or Object. +`Matrix.Params` supports whole array replacements and string replacements from `Results` of type String, Array or Object ```yaml tasks: @@ -314,16 +314,9 @@ tasks: matrix: params: - name: values - value: - - $(tasks.task-1.results.foo) # string replacement from string result - - $(tasks.task-2.results.bar[0]) # string replacement from array result - - $(tasks.task-3.results.rad.key) # string replacement from object result + value: $(tasks.task-4.results.whole-array[*]) ``` -For further information, see the example in [`PipelineRun` with `Matrix` and `Results`][pr-with-matrix-and-results]. - -We plan to add support passing whole Array `Results` into the `Matrix` (#5925): - ```yaml tasks: ... @@ -333,9 +326,16 @@ tasks: matrix: params: - name: values - value: $(tasks.task-4.results.foo) # array + value: + - $(tasks.task-1.results.a-string-result) + - $(tasks.task-2.results.an-array-result[0]) + - $(tasks.task-3.results.an-object-result.key) ``` + +For further information, see the example in [`PipelineRun` with `Matrix` and `Results`][pr-with-matrix-and-results]. + + #### Results in Matrix.Include.Params `Matrix.Include.Params` supports string replacements from `Results` of type String, Array or Object. diff --git a/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix-and-results.yaml b/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix-and-results.yaml index 20408bb168f..6ebd50e0f8e 100644 --- a/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix-and-results.yaml +++ b/examples/v1beta1/pipelineruns/alpha/pipelinerun-with-matrix-and-results.yaml @@ -21,7 +21,7 @@ kind: PipelineRun metadata: generateName: matrixed-pr- spec: - serviceAccountName: 'default' + serviceAccountName: "default" pipelineSpec: tasks: - name: get-platforms @@ -56,10 +56,7 @@ spec: matrix: params: - name: platform - value: - - $(tasks.get-platforms.results.platforms[0]) - - $(tasks.get-platforms.results.platforms[1]) - - $(tasks.get-platforms.results.platforms[2]) + value: $(tasks.get-platforms.results.platforms[*]) - name: browser value: - $(tasks.get-browsers-and-url.results.browsers[0]) diff --git a/pkg/apis/pipeline/v1/matrix_types.go b/pkg/apis/pipeline/v1/matrix_types.go index 03fba3f29aa..f51a0ac4af4 100644 --- a/pkg/apis/pipeline/v1/matrix_types.go +++ b/pkg/apis/pipeline/v1/matrix_types.go @@ -348,24 +348,3 @@ func (m *Matrix) validateParameterInOneOfMatrixOrParams(params []Param) (errs *a } return errs } - -// validateNoWholeArrayResults() is used to ensure a matrix parameter does not contain result references -// to entire arrays. This is temporary until whole array replacements for matrix parameters are supported. -// See issue #6056 for more details -func (m *Matrix) validateNoWholeArrayResults() (errs *apis.FieldError) { - if m.HasParams() { - for i, param := range m.Params { - val := param.Value.StringVal - expressions, ok := GetVarSubstitutionExpressionsForParam(param) - if ok { - if LooksLikeContainsResultRefs(expressions) { - _, stringIdx := ParseResultName(val) - if stringIdx == "*" { - errs = errs.Also(apis.ErrGeneric("matrix parameters cannot contain whole array result references", "").ViaFieldIndex("matrix.params", i)) - } - } - } - } - } - return errs -} diff --git a/pkg/apis/pipeline/v1/pipeline_types_test.go b/pkg/apis/pipeline/v1/pipeline_types_test.go index d0249b463ff..712e0bdd324 100644 --- a/pkg/apis/pipeline/v1/pipeline_types_test.go +++ b/pkg/apis/pipeline/v1/pipeline_types_test.go @@ -711,19 +711,6 @@ func TestPipelineTask_ValidateMatrix(t *testing.T) { Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-result)"}}, }}}, }, - }, { - name: "parameters in matrix contain whole array results references", - pt: &PipelineTask{ - Name: "task", - Matrix: &Matrix{ - Params: Params{{ - Name: "a-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.foo-task.results.arr-results[*])"}, - }}}, - }, - wantErrs: &apis.FieldError{ - Message: "matrix parameters cannot contain whole array result references", - Paths: []string{"matrix.params[0]"}, - }, }, { name: "count of combinations of parameters in the matrix exceeds the maximum", pt: &PipelineTask{ diff --git a/pkg/apis/pipeline/v1/pipeline_validation.go b/pkg/apis/pipeline/v1/pipeline_validation.go index 32ff89656b1..ecace0e7b9a 100644 --- a/pkg/apis/pipeline/v1/pipeline_validation.go +++ b/pkg/apis/pipeline/v1/pipeline_validation.go @@ -155,7 +155,6 @@ func (pt *PipelineTask) validateMatrix(ctx context.Context) (errs *apis.FieldErr // when the enable-api-fields feature gate is anything but "alpha". errs = errs.Also(version.ValidateEnabledAPIFields(ctx, "matrix", config.AlphaAPIFields)) errs = errs.Also(pt.Matrix.validateCombinationsCount(ctx)) - errs = errs.Also(pt.Matrix.validateNoWholeArrayResults()) errs = errs.Also(pt.Matrix.validateUniqueParams()) } errs = errs.Also(pt.Matrix.validateParameterInOneOfMatrixOrParams(pt.Params)) diff --git a/pkg/apis/pipeline/v1/pipeline_validation_test.go b/pkg/apis/pipeline/v1/pipeline_validation_test.go index 03149b92244..8b7ec90705a 100644 --- a/pkg/apis/pipeline/v1/pipeline_validation_test.go +++ b/pkg/apis/pipeline/v1/pipeline_validation_test.go @@ -3095,6 +3095,16 @@ func Test_validateMatrix(t *testing.T) { Name: "b-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.bar-task.results.b-result)"}}, }}}, }}, + }, { + name: "parameters in matrix contain whole array results references", + tasks: PipelineTaskList{{ + Name: "a-task", + TaskRef: &TaskRef{Name: "a-task"}, + Matrix: &Matrix{ + Params: Params{{ + Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-task-results[*])"}}, + }}}, + }}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/apis/pipeline/v1beta1/matrix_types.go b/pkg/apis/pipeline/v1beta1/matrix_types.go index ac44ff6891f..19042e11e4b 100644 --- a/pkg/apis/pipeline/v1beta1/matrix_types.go +++ b/pkg/apis/pipeline/v1beta1/matrix_types.go @@ -17,7 +17,6 @@ import ( "context" "fmt" "sort" - "strings" "github.com/tektoncd/pipeline/pkg/apis/config" "golang.org/x/exp/maps" @@ -349,21 +348,3 @@ func (m *Matrix) validateParameterInOneOfMatrixOrParams(params Params) (errs *ap } return errs } - -// validateNoWholeArrayResults() is used to ensure a matrix parameter does not contain result references -// to entire arrays. This is temporary until whole array replacements for matrix paraemeters are supported. -// See issue #6056 for more details -func (m *Matrix) validateNoWholeArrayResults() (errs *apis.FieldError) { - if m.HasParams() { - for i, param := range m.Params { - val := param.Value.StringVal - expressions, ok := GetVarSubstitutionExpressionsForParam(param) - if ok { - if LooksLikeContainsResultRefs(expressions) && strings.Contains(val, "[*]") { - errs = errs.Also(apis.ErrGeneric("matrix parameters cannot contain whole array result references", "").ViaFieldIndex("matrix.params", i)) - } - } - } - } - return errs -} diff --git a/pkg/apis/pipeline/v1beta1/pipeline_types_test.go b/pkg/apis/pipeline/v1beta1/pipeline_types_test.go index f137ae152a6..42bf0a8e996 100644 --- a/pkg/apis/pipeline/v1beta1/pipeline_types_test.go +++ b/pkg/apis/pipeline/v1beta1/pipeline_types_test.go @@ -684,19 +684,6 @@ func TestPipelineTask_validateMatrix(t *testing.T) { Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-result)"}}, }}}, }, - }, { - name: "parameters in matrix contain whole array results references", - pt: &PipelineTask{ - Name: "task", - Matrix: &Matrix{ - Params: Params{{ - Name: "a-param", Value: ParamValue{Type: ParamTypeString, StringVal: "$(tasks.foo-task.results.arr-results[*])"}, - }}}, - }, - wantErrs: &apis.FieldError{ - Message: "matrix parameters cannot contain whole array result references", - Paths: []string{"matrix.params[0]"}, - }, }, { name: "count of combinations of parameters in the matrix exceeds the maximum", pt: &PipelineTask{ diff --git a/pkg/apis/pipeline/v1beta1/pipeline_validation.go b/pkg/apis/pipeline/v1beta1/pipeline_validation.go index 80bb7c6d175..3b60b2e8b9d 100644 --- a/pkg/apis/pipeline/v1beta1/pipeline_validation.go +++ b/pkg/apis/pipeline/v1beta1/pipeline_validation.go @@ -167,7 +167,6 @@ func (pt *PipelineTask) validateMatrix(ctx context.Context) (errs *apis.FieldErr // when the enable-api-fields feature gate is anything but "alpha". errs = errs.Also(version.ValidateEnabledAPIFields(ctx, "matrix", config.AlphaAPIFields)) errs = errs.Also(pt.Matrix.validateCombinationsCount(ctx)) - errs = errs.Also(pt.Matrix.validateNoWholeArrayResults()) errs = errs.Also(pt.Matrix.validateUniqueParams()) } errs = errs.Also(pt.Matrix.validateParameterInOneOfMatrixOrParams(pt.Params)) diff --git a/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go b/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go index 04b366fa8f0..12432921cc8 100644 --- a/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go +++ b/pkg/apis/pipeline/v1beta1/pipeline_validation_test.go @@ -3139,6 +3139,16 @@ func Test_validateMatrix(t *testing.T) { Name: "b-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.bar-task.results.b-result)"}}, }}}, }}, + }, { + name: "parameters in matrix contain whole array results references", + tasks: PipelineTaskList{{ + Name: "a-task", + TaskRef: &TaskRef{Name: "a-task"}, + Matrix: &Matrix{ + Params: Params{{ + Name: "a-param", Value: ParamValue{Type: ParamTypeArray, ArrayVal: []string{"$(tasks.foo-task.results.a-task-results[*])"}}, + }}}, + }}, }} for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/reconciler/pipelinerun/pipelinerun.go b/pkg/reconciler/pipelinerun/pipelinerun.go index 39f89587e43..3ee3348f496 100644 --- a/pkg/reconciler/pipelinerun/pipelinerun.go +++ b/pkg/reconciler/pipelinerun/pipelinerun.go @@ -774,6 +774,12 @@ func (c *Reconciler) runNextSchedulableTask(ctx context.Context, pr *v1beta1.Pip switch { case rpt.IsCustomTask() && rpt.PipelineTask.IsMatrixed(): + // RunObjectNames Names have not yet been populated for a matrix that has whole array references + // Since we have already applied Task Results, we can now get the names of the RunObjects + // after array result substitution before fanning out the Matrix and creating RunObjects + if len(rpt.RunObjectName) == 0 { + rpt.RunObjectNames = resources.GetNamesOfRuns(pr.Status.ChildReferences, rpt.PipelineTask.Name, pr.Name, rpt.PipelineTask.Matrix.CountCombinations()) + } rpt.RunObjects, err = c.createRunObjects(ctx, rpt, pr) if err != nil { recorder.Eventf(pr, corev1.EventTypeWarning, "RunsCreationFailed", "Failed to create Runs %q: %v", rpt.RunObjectNames, err) @@ -788,6 +794,12 @@ func (c *Reconciler) runNextSchedulableTask(ctx context.Context, pr *v1beta1.Pip return err } case rpt.PipelineTask.IsMatrixed(): + // TaskRun Names have not yet been populated for a matrix that has whole array references + // Since we have already applied Task Results, we can now get the names of the TaskRuns + // after array result substitution before fanning out the Matrix and creating TaskRuns + if len(rpt.TaskRunNames) == 0 { + rpt.TaskRunNames = resources.GetNamesOfTaskRuns(pr.Status.ChildReferences, rpt.PipelineTask.Name, pr.Name, rpt.PipelineTask.Matrix.CountCombinations()) + } rpt.TaskRuns, err = c.createTaskRuns(ctx, rpt, pr) if err != nil { recorder.Eventf(pr, corev1.EventTypeWarning, "TaskRunsCreationFailed", "Failed to create TaskRuns %q: %v", rpt.TaskRunNames, err) diff --git a/pkg/reconciler/pipelinerun/pipelinerun_test.go b/pkg/reconciler/pipelinerun/pipelinerun_test.go index 4ad9719a842..895218914df 100644 --- a/pkg/reconciler/pipelinerun/pipelinerun_test.go +++ b/pkg/reconciler/pipelinerun/pipelinerun_test.go @@ -9601,7 +9601,7 @@ spec: } } -func TestReconciler_PipelineTaskMatrixResultsWithArrayIndexing(t *testing.T) { +func TestReconciler_PipelineTaskMatrixResultsWithArrays(t *testing.T) { names.TestingSeed() task := parse.MustParseV1beta1Task(t, ` metadata: @@ -9906,6 +9906,152 @@ status: kind: TaskRun name: pr-echo-platforms-2 pipelineTaskName: echo-platforms +`), + }, { + name: "whole array result replacements in matrix.params", + pName: "p-dag-3", + p: parse.MustParseV1beta1Pipeline(t, fmt.Sprintf(` +metadata: + name: %s + namespace: foo +spec: + tasks: + - name: pt-with-result + params: + - name: platforms + type: array + taskRef: + name: taskwithresults + - name: echo-platforms + matrix: + params: + - name: platform + value: $(tasks.pt-with-result.results.platforms[*]) + taskRef: + name: mytask +`, "p-dag-3")), + tr: mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-pt-with-result", "foo", + "pr", "p-dag-3", "pt-with-result", false), + ` +spec: + serviceAccountName: test-sa + taskRef: + name: taskwithresults +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + taskResults: + - name: platforms + value: + - linux + - mac + - windows +`), + expectedTaskRuns: []*v1beta1.TaskRun{ + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-echo-platforms-0", "foo", + "pr", "p-dag-3", "echo-platforms", false), + ` +spec: + params: + - name: platform + value: linux + serviceAccountName: test-sa + taskRef: + name: mytask + kind: Task +labels: + tekton.dev/memberOf: tasks + tekton.dev/pipeline: p-dag-3 +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-echo-platforms-1", "foo", + "pr", "p-dag-3", "echo-platforms", false), + ` +spec: + params: + - name: platform + value: mac + serviceAccountName: test-sa + taskRef: + name: mytask + kind: Task +labels: + tekton.dev/memberOf: tasks + tekton.dev/pipeline: p-dag-3 +`), + mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-echo-platforms-2", "foo", + "pr", "p-dag-3", "echo-platforms", false), + ` +spec: + params: + - name: platform + value: windows + serviceAccountName: test-sa + taskRef: + name: mytask + kind: Task +labels: + tekton.dev/memberOf: tasks + tekton.dev/pipeline: p-dag-3 +`), + }, + expectedPipelineRun: parse.MustParseV1beta1PipelineRun(t, ` +metadata: + name: pr + namespace: foo + annotations: {} + labels: + tekton.dev/pipeline: p-dag-3 +spec: + serviceAccountName: test-sa + pipelineRef: + name: p-dag-3 +status: + pipelineSpec: + tasks: + - name: pt-with-result + params: + - name: platforms + type: array + taskRef: + name: taskwithresults + kind: Task + - name: echo-platforms + taskRef: + name: mytask + kind: Task + matrix: + params: + - name: platform + value: $(tasks.pt-with-result.results.platforms[*]) + conditions: + - type: Succeeded + status: "Unknown" + reason: "Running" + message: "Tasks Completed: 1 (Failed: 0, Cancelled 0), Incomplete: 1, Skipped: 0" + childReferences: + - apiVersion: tekton.dev/v1beta1 + kind: TaskRun + name: pr-pt-with-result + pipelineTaskName: pt-with-result + - apiVersion: tekton.dev/v1beta1 + kind: TaskRun + name: pr-echo-platforms-0 + pipelineTaskName: echo-platforms + - apiVersion: tekton.dev/v1beta1 + kind: TaskRun + name: pr-echo-platforms-1 + pipelineTaskName: echo-platforms + - apiVersion: tekton.dev/v1beta1 + kind: TaskRun + name: pr-echo-platforms-2 + pipelineTaskName: echo-platforms `), }} for _, tt := range tests { @@ -10913,6 +11059,239 @@ spec: } } +func TestReconciler_CustomPipelineTaskMatrixResultsWholeArray(t *testing.T) { + names.TestingSeed() + + taskwithresults := parse.MustParseV1beta1Task(t, ` +metadata: + name: taskwithresults + namespace: foo +spec: + results: + - name: platforms + type: array + steps: + - name: produce-a-list-of-platforms + image: bash:latest + script: | + #!/usr/bin/env bash + echo -n "[\"linux\",\"mac\",\"windows\"]" | tee $(results.platforms.path) +`) + expectedCustomRuns := []*v1beta1.CustomRun{ + mustParseCustomRunWithObjectMeta(t, + taskRunObjectMeta("pr-pt-matrix-custom-task-0", "foo", + "pr", "p-dag", "pt-matrix-custom-task", false), + ` +spec: + customRef: + apiVersion: example.dev/v0 + kind: Example + params: + - name: platform + value: linux + serviceAccountName: test-sa + taskRef: + name: mytask + kind: Task + labels: + tekton.dev/memberOf: tasks + tekton.dev/pipeline: p-dag +`), + mustParseCustomRunWithObjectMeta(t, + taskRunObjectMeta("pr-pt-matrix-custom-task-1", "foo", + "pr", "p-dag", "pt-matrix-custom-task", false), + ` +spec: + customRef: + apiVersion: example.dev/v0 + kind: Example + params: + - name: platform + value: mac + serviceAccountName: test-sa + taskRef: + name: mytask + kind: Task + labels: + tekton.dev/memberOf: tasks + tekton.dev/pipeline: p-dag +`), + mustParseCustomRunWithObjectMeta(t, + taskRunObjectMeta("pr-pt-matrix-custom-task-2", "foo", + "pr", "p-dag", "pt-matrix-custom-task", false), + ` +spec: + customRef: + apiVersion: example.dev/v0 + kind: Example + params: + - name: platform + value: windows + serviceAccountName: test-sa + taskRef: + name: mytask + kind: Task + labels: + tekton.dev/memberOf: tasks + tekton.dev/pipeline: p-dag +`), + } + + cms := []*corev1.ConfigMap{withEnabledAlphaAPIFields(newFeatureFlagsConfigMap())} + cms = append(cms, withMaxMatrixCombinationsCount(newDefaultsConfigMap(), 10)) + + tests := []struct { + name string + memberOf string + p *v1beta1.Pipeline + tr *v1beta1.TaskRun + expectedPipelineRun *v1beta1.PipelineRun + }{{ + name: "p-dag", + memberOf: "tasks", + p: parse.MustParseV1beta1Pipeline(t, fmt.Sprintf(` +metadata: + name: %s + namespace: foo +spec: + tasks: + - name: pt-with-result + params: + - name: platforms + type: array + taskRef: + name: taskwithresults + - name: pt-matrix-custom-task + matrix: + params: + - name: platform + value: $(tasks.pt-with-result.results.platforms[*]) + taskRef: + apiVersion: example.dev/v0 + kind: Example +`, "p-dag")), + + tr: mustParseTaskRunWithObjectMeta(t, + taskRunObjectMeta("pr-pt-with-result", "foo", + "pr", "p-dag", "pt-with-result", false), + ` +spec: + serviceAccountName: test-sa + taskRef: + name: taskwithresults +status: + conditions: + - type: Succeeded + status: "True" + reason: Succeeded + message: All Tasks have completed executing + taskResults: + - name: platforms + value: + - linux + - mac + - windows +`), + expectedPipelineRun: parse.MustParseV1beta1PipelineRun(t, ` +metadata: + name: pr + namespace: foo + annotations: {} + labels: + tekton.dev/pipeline: p-dag +spec: + serviceAccountName: test-sa + pipelineRef: + name: p-dag +status: + pipelineSpec: + tasks: + - name: pt-with-result + params: + - name: platforms + type: array + taskRef: + name: taskwithresults + kind: Task + - name: pt-matrix-custom-task + taskRef: + apiVersion: example.dev/v0 + kind: Example + matrix: + params: + - name: platform + value: $(tasks.pt-with-result.results.platforms[*]) + conditions: + - type: Succeeded + status: "Unknown" + reason: "Running" + message: "Tasks Completed: 1 (Failed: 0, Cancelled 0), Incomplete: 1, Skipped: 0" + childReferences: + - apiVersion: tekton.dev/v1beta1 + kind: TaskRun + name: pr-pt-with-result + pipelineTaskName: pt-with-result + - apiVersion: tekton.dev/v1beta1 + kind: CustomRun + name: pr-pt-matrix-custom-task-0 + pipelineTaskName: pt-matrix-custom-task + - apiVersion: tekton.dev/v1beta1 + kind: CustomRun + name: pr-pt-matrix-custom-task-1 + pipelineTaskName: pt-matrix-custom-task + - apiVersion: tekton.dev/v1beta1 + kind: CustomRun + name: pr-pt-matrix-custom-task-2 + pipelineTaskName: pt-matrix-custom-task +`), + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pr := parse.MustParseV1beta1PipelineRun(t, fmt.Sprintf(` +metadata: + name: pr + namespace: foo +spec: + serviceAccountName: test-sa + pipelineRef: + name: %s +`, tt.name)) + d := test.Data{ + PipelineRuns: []*v1beta1.PipelineRun{pr}, + Pipelines: []*v1beta1.Pipeline{tt.p}, + Tasks: []*v1beta1.Task{taskwithresults}, + ConfigMaps: cms, + } + if tt.tr != nil { + d.TaskRuns = []*v1beta1.TaskRun{tt.tr} + } + prt := newPipelineRunTest(t, d) + defer prt.Cancel() + + pipelineRun, clients := prt.reconcileRun("foo", "pr", []string{}, false) + customRuns, err := clients.Pipeline.TektonV1beta1().CustomRuns("foo").List(prt.TestAssets.Ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("Failure to list customRuns's %s", err) + } + + if len(customRuns.Items) != 3 { + t.Fatalf("Expected 3 customRuns got %d", len(customRuns.Items)) + } + + for i := range customRuns.Items { + expectedCustomRun := expectedCustomRuns[i] + if d := cmp.Diff(expectedCustomRun, &customRuns.Items[i], ignoreResourceVersion, ignoreTypeMeta); d != "" { + t.Errorf("expected to see CustomRun %v created. Diff %s", expectedCustomRun.Name, diff.PrintWantGot(d)) + } + } + + if d := cmp.Diff(tt.expectedPipelineRun, pipelineRun, ignoreResourceVersion, ignoreTypeMeta, ignoreLastTransitionTime, ignoreStartTime, ignoreFinallyStartTime, cmpopts.EquateEmpty()); d != "" { + t.Errorf("expected PipelineRun was not created. Diff %s", diff.PrintWantGot(d)) + } + }) + } +} + func TestReconcile_SetDefaults(t *testing.T) { names.TestingSeed() diff --git a/pkg/reconciler/pipelinerun/resources/apply.go b/pkg/reconciler/pipelinerun/resources/apply.go index edcec080901..b0b30b1c305 100644 --- a/pkg/reconciler/pipelinerun/resources/apply.go +++ b/pkg/reconciler/pipelinerun/resources/apply.go @@ -184,9 +184,8 @@ func ApplyTaskResults(targets PipelineRunState, resolvedResultRefs ResolvedResul pipelineTask := resolvedPipelineRunTask.PipelineTask.DeepCopy() pipelineTask.Params = replaceParamValues(pipelineTask.Params, stringReplacements, arrayReplacements, objectReplacements) if pipelineTask.IsMatrixed() { - // Only string replacements from string, array or object results are supported - // We plan to support array replacements from array results soon (#5925) - pipelineTask.Matrix.Params = replaceParamValues(pipelineTask.Matrix.Params, stringReplacements, nil, nil) + // String replacements from string, array or object results or array replacements from array results are supported + pipelineTask.Matrix.Params = replaceParamValues(pipelineTask.Matrix.Params, stringReplacements, arrayReplacements, nil) for i := range pipelineTask.Matrix.Include { // matrix include parameters can only be type string pipelineTask.Matrix.Include[i].Params = replaceParamValues(pipelineTask.Matrix.Include[i].Params, stringReplacements, nil, nil) diff --git a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go index bc70193cd56..751d5eab0f4 100644 --- a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go +++ b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "regexp" "github.com/tektoncd/pipeline/pkg/apis/config" "github.com/tektoncd/pipeline/pkg/apis/pipeline" @@ -581,7 +582,12 @@ func ResolvePipelineTask( rpt.CustomTask = rpt.PipelineTask.TaskRef.IsCustomTask() || rpt.PipelineTask.TaskSpec.IsCustomTask() switch { case rpt.IsCustomTask() && rpt.PipelineTask.IsMatrixed(): - rpt.RunObjectNames = getNamesOfRuns(pipelineRun.Status.ChildReferences, pipelineTask.Name, pipelineRun.Name, pipelineTask.Matrix.CountCombinations()) + // If the matrix contains array results references, skip getting the RunObjectNames until applying TaskResults + // to the matrixed pipelineTask since the # of combinations is non-deterministic at this time + if containsArrayResultReferences(rpt.PipelineTask.Matrix) { + break + } + rpt.RunObjectNames = GetNamesOfRuns(pipelineRun.Status.ChildReferences, pipelineTask.Name, pipelineRun.Name, pipelineTask.Matrix.CountCombinations()) for _, runName := range rpt.RunObjectNames { run, err := getRun(runName) if err != nil && !kerrors.IsNotFound(err) { @@ -600,6 +606,13 @@ func ResolvePipelineTask( if run != nil { rpt.RunObject = run } + case rpt.PipelineTask.IsMatrixed() && containsArrayResultReferences(rpt.PipelineTask.Matrix): + // If the matrix contains array results references, get the singular TaskRunName. All the TaskRunNames won't be + // retrieved until applying TaskResults to the matrixed pipelineTask since the # of combinations is non-deterministic + rpt.TaskRunName = GetTaskRunName(pipelineRun.Status.ChildReferences, pipelineTask.Name, pipelineRun.Name) + if err := rpt.resolvePipelineRunTaskWithTaskRun(ctx, rpt.TaskRunName, getTask, getTaskRun, pipelineTask); err != nil { + return nil, err + } case rpt.PipelineTask.IsMatrixed(): rpt.TaskRunNames = GetNamesOfTaskRuns(pipelineRun.Status.ChildReferences, pipelineTask.Name, pipelineRun.Name, pipelineTask.Matrix.CountCombinations()) for _, taskRunName := range rpt.TaskRunNames { @@ -757,9 +770,9 @@ func getRunName(childRefs []v1beta1.ChildStatusReference, ptName, prName string) return kmeta.ChildName(prName, fmt.Sprintf("-%s", ptName)) } -// getNamesOfRuns should return a unique names for `RunObjects` if they have not already been defined, +// GetNamesOfRuns should return a unique names for `RunObjects` if they have not already been defined, // and the existing ones otherwise. -func getNamesOfRuns(childRefs []v1beta1.ChildStatusReference, ptName, prName string, combinationCount int) []string { +func GetNamesOfRuns(childRefs []v1beta1.ChildStatusReference, ptName, prName string, combinationCount int) []string { if runNames := getRunNamesFromChildRefs(childRefs, ptName); runNames != nil { return runNames } @@ -811,6 +824,25 @@ func (t *ResolvedPipelineTask) hasResultReferences() bool { return false } +// containsArrayResultReferences returns true if the Matrix contains references to array +// results either through array indexing [0] or reference to the whole array[*] +func containsArrayResultReferences(m *v1beta1.Matrix) bool { + params := m.GetAllParams() + for _, param := range params { + expressions, ok := v1beta1.GetVarSubstitutionExpressionsForParam(param) + if ok && v1beta1.LooksLikeContainsResultRefs(expressions) { + for _, expression := range expressions { + var isNum = regexp.MustCompile(`^[0-9]+$`) + _, stringIdx := v1beta1.ParseResultName(expression) + if stringIdx == "*" || isNum.MatchString(stringIdx) { + return true + } + } + } + } + return false +} + func isCustomRunCancelledByPipelineRunTimeout(ro v1beta1.RunObject) bool { cr := ro.(*v1beta1.CustomRun) return cr.Spec.StatusMessage == v1beta1.CustomRunCancelledByPipelineTimeoutMsg diff --git a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go index 3571a48910b..1053e1f5aa1 100644 --- a/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go +++ b/pkg/reconciler/pipelinerun/resources/pipelinerunresolution_test.go @@ -2910,7 +2910,7 @@ func TestGetNamesOfRuns(t *testing.T) { if tc.prName != "" { testPrName = tc.prName } - namesOfRunsFromChildRefs := getNamesOfRuns(childRefs, tc.ptName, testPrName, 2) + namesOfRunsFromChildRefs := GetNamesOfRuns(childRefs, tc.ptName, testPrName, 2) sort.Strings(namesOfRunsFromChildRefs) if d := cmp.Diff(tc.wantRunNames, namesOfRunsFromChildRefs); d != "" { t.Errorf("getRunName: %s", diff.PrintWantGot(d)) @@ -3103,6 +3103,16 @@ func TestResolvePipelineRunTask_WithMatrix(t *testing.T) { Name: "browsers", Value: v1beta1.ParamValue{Type: v1beta1.ParamTypeArray, ArrayVal: []string{"chrome", "safari", "firefox"}}, }}, + }, + }, { + Name: "pipelinetask-with-whole-array-results", + TaskRef: &v1beta1.TaskRef{ + Name: "my-task-with-results", + }, + Matrix: &v1beta1.Matrix{ + Params: v1beta1.Params{{ + Name: "foo", Value: v1beta1.ParamValue{Type: v1beta1.ParamTypeString, StringVal: "$(tasks.foo-task.results.arr-results[*])"}, + }}, }}} rtr := &resources.ResolvedTask{ @@ -3140,6 +3150,15 @@ func TestResolvePipelineRunTask_WithMatrix(t *testing.T) { PipelineTask: &pts[1], ResolvedTask: rtr, }, + }, { + name: "task with matrix - whole array results", + pt: pts[2], + want: &ResolvedPipelineTask{ + TaskRunName: "pipelinerun-pipelinetask-with-whole-array-references", + TaskRuns: nil, + PipelineTask: &pts[2], + ResolvedTask: rtr, + }, }} { t.Run(tc.name, func(t *testing.T) { ctx := context.Background() @@ -3214,6 +3233,17 @@ func TestResolvePipelineRunTask_WithMatrixedCustomTask(t *testing.T) { Name: "browsers", Value: v1beta1.ParamValue{Type: v1beta1.ParamTypeArray, ArrayVal: []string{"chrome", "safari", "firefox"}}, }}}, + }, { + Name: "pipelinetask-custom-task-with-whole-array-results", + TaskRef: &v1beta1.TaskRef{ + APIVersion: "example.dev/v0", + Kind: "Example", + Name: "my-task", + }, + Matrix: &v1beta1.Matrix{ + Params: v1beta1.Params{{ + Name: "foo", Value: v1beta1.ParamValue{Type: v1beta1.ParamTypeString, StringVal: "$(tasks.foo-task.results.arr-results[*])"}}, + }}, }} getTask := func(ctx context.Context, name string) (*v1beta1.Task, *v1beta1.RefSource, error) { @@ -3257,6 +3287,15 @@ func TestResolvePipelineRunTask_WithMatrixedCustomTask(t *testing.T) { RunObjects: nil, PipelineTask: &pts[1], }, + }, { + name: "custom task with matrix - whole array results", + pt: pts[2], + want: &ResolvedPipelineTask{ + CustomTask: true, + RunObjectNames: nil, + RunObjects: nil, + PipelineTask: &pts[2], + }, }} { t.Run(tc.name, func(t *testing.T) { ctx := context.Background()