From c5de2c5333176d5f49d798bd3480a1a125d7f15b Mon Sep 17 00:00:00 2001 From: JeromeJu Date: Fri, 12 Jan 2024 15:39:35 +0000 Subject: [PATCH] POC conformance test suite --- test/poc_conformance_test.go | 219 +++++++++++++++++++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 test/poc_conformance_test.go diff --git a/test/poc_conformance_test.go b/test/poc_conformance_test.go new file mode 100644 index 00000000000..6b7df58c5bb --- /dev/null +++ b/test/poc_conformance_test.go @@ -0,0 +1,219 @@ +//go:build conformance +// +build conformance + +/* +This serves as a POC for conformance test suite design including functionality, +behavioural and fields population. +It mocks the "black-box" execution of TaskRuns and PipelineRuns utilizing the +Tekton clients to mock the controller of a conformant vendor service. + +Please use the following for triggering the test: +go test -v -tags=conformance -count=1 ./test -run ^TestConformance + +The next step will be to integrate this test as POC with v2 API. +*/ + +package test + +import ( + "context" + "fmt" + "testing" + + v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" + "github.com/tektoncd/pipeline/test/parse" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" + knativetest "knative.dev/pkg/test" + "knative.dev/pkg/test/helpers" + "sigs.k8s.io/yaml" +) + +const ( + TaskRunInputType = "TaskRun" + PipelineRunInputType = "PipelineRun" +) + +func TestConformanceShouldProvideTaskResult(t *testing.T) { + inputYAML := fmt.Sprintf(` +apiVersion: tekton.dev/v1 +kind: TaskRun +metadata: + name: %s +spec: + taskSpec: + params: + - name: multiplicand + description: the first operand + default: %s + - name: multipliper + description: the second operand + default: %s + results: + - name: product + description: the product of the first and second operand + steps: + - name: add + image: alpine + env: + - name: OP1 + value: $(params.multiplicand) + - name: OP2 + value: $(params.multipliper) + command: ["/bin/sh", "-c"] + args: + - echo -n $((${OP1}*${OP2})) | tee $(results.product.path); +`, helpers.ObjectNameForTest(t), "3", "5") + + outputYAML, err := ProcessAndSendToTekton(inputYAML, TaskRunInputType, t) + if err != nil { + t.Fatalf("Vendor service failed processing inputYAML: %s", err) + } + + // Parse and validate output YAML + resolvedTR := parse.MustParseV1TaskRun(t, outputYAML) + if len(resolvedTR.Status.Results) != 1 { + t.Errorf("Expect vendor service to provide 1 result but not") + } + + if resolvedTR.Status.Results[0].Value.StringVal != "15" { + t.Errorf("Not producing correct result :%s", resolvedTR.Status.Results[0].Value.StringVal) + } +} + +func TestConformanceShouldHonorTaskRunTimeout(t *testing.T) { + expectedFailedStatus := true + inputYAML := fmt.Sprintf(` +apiVersion: tekton.dev/v1 +kind: TaskRun +metadata: + name: %s +spec: + timeout: 15s + taskSpec: + steps: + - image: busybox + command: ['/bin/sh'] + args: ['-c', 'sleep 15001'] +`, helpers.ObjectNameForTest(t)) + + outputYAML, err := ProcessAndSendToTekton(inputYAML, TaskRunInputType, t, expectedFailedStatus) + if err != nil { + t.Fatalf("Vendor service failed processing inputYAML: %s", err) + } + + resolvedTR := parse.MustParseV1TaskRun(t, outputYAML) + if len(resolvedTR.Status.Conditions) != 1 { + t.Errorf("Expect vendor service to populate 1 Condition but no") + } + + if resolvedTR.Status.Conditions[0].Type != "Succeeded" { + t.Errorf("Expect vendor service to populate Condition `Succeeded` but got: %s", resolvedTR.Status.Conditions[0].Type) + } + + if resolvedTR.Status.Conditions[0].Status != "False" { + t.Errorf("Expect vendor service to populate Condition `False` but got: %s", resolvedTR.Status.Conditions[0].Status) + } + + if resolvedTR.Status.Conditions[0].Reason != "TaskRunTimeout" { + t.Errorf("Expect vendor service to populate Condition Reason `TaskRunTimeout` but got: %s", resolvedTR.Status.Conditions[0].Reason) + } +} + +func TestConformanceShouldPopulateConditions(t *testing.T) { + inputYAML := fmt.Sprintf(` +apiVersion: tekton.dev/v1 +kind: TaskRun +metadata: + name: %s +spec: + taskSpec: + steps: + - name: add + image: ubuntu + script: + echo Hello world! +`, helpers.ObjectNameForTest(t)) + + outputYAML, err := ProcessAndSendToTekton(inputYAML, TaskRunInputType, t) + if err != nil { + t.Fatalf("Vendor service failed processing inputYAML: %s", err) + } + + resolvedTR := parse.MustParseV1TaskRun(t, outputYAML) + if len(resolvedTR.Status.Conditions) != 1 { + t.Errorf("Expect vendor service to populate 1 Condition but no") + } + + if resolvedTR.Status.Conditions[0].Type != "Succeeded" { + t.Errorf("Expect vendor service to populate Condition `Succeeded` but got: %s", resolvedTR.Status.Conditions[0].Type) + } + + if resolvedTR.Status.Conditions[0].Status != "True" { + t.Errorf("Expect vendor service to populate Condition `True` but got: %s", resolvedTR.Status.Conditions[0].Status) + } +} + +// ProcessAndSendToTekton takes in vanilla Tekton PipelineRun and TaskRun, waits for the object to succeed and outputs the final PipelineRun and TaskRun with status. +// The parameters are inputYAML and its Primitive type {PipelineRun, TaskRun} +// And the return values will be the output YAML string and errors. +func ProcessAndSendToTekton(inputYAML, primitiveType string, customInputs ...interface{}) (string, error) { + // Handle customInputs + var t *testing.T + var expectRunToFail bool + for _, customInput := range customInputs { + if ci, ok := customInput.(*testing.T); ok { + t = ci + } + if ci, ok := customInput.(bool); ok { + expectRunToFail = ci + } + } + + return mockTektonPipelineController(t, inputYAML, primitiveType, expectRunToFail) +} + +// mockTektonPipelineController fakes the behaviour of a vendor service by utilizing the Tekton test infrastructure. +// For the POC, it uses the Tetkon clients to Create, Wait for and Get the expected TaskRun. +func mockTektonPipelineController(t *testing.T, inputYAML, primitiveType string, expectRunToFail bool) (string, error) { + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + defer cancel() + c, namespace := setup(ctx, t) + + knativetest.CleanupOnInterrupt(func() { tearDown(ctx, t, c, namespace) }, t.Logf) + defer tearDown(ctx, t, c, namespace) + + // Parse inputYAML, parse.MustParseTaskRun + var tr v1.TaskRun + if _, _, err := scheme.Codecs.UniversalDeserializer().Decode([]byte(inputYAML), nil, &tr); err != nil { + return "", fmt.Errorf("must parse YAML (%s): %v", inputYAML, err) + } + + // Create TaskRun via TaskRunClient + trResolved, err := c.V1TaskRunClient.Create(ctx, &tr, metav1.CreateOptions{}) + if err != nil { + return "", fmt.Errorf("Failed to create TaskRun `%v`: %w", trResolved, err) + } + + var caf ConditionAccessorFn + caf = Succeed(trResolved.Name) + if expectRunToFail { + caf = Failed(trResolved.Name) + } + if err := WaitForTaskRunState(ctx, c, trResolved.Name, caf, "WaitTaskRunDone", v1Version); err != nil { + return "", fmt.Errorf("Error waiting for TaskRun to finish: %s", err) + } + + // Retrieve the TaskRun via TaskRunClient + trGot, err := c.V1TaskRunClient.Get(ctx, trResolved.Name, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("Failed to get TaskRun `%s`: %s", trGot.Name, err) + } + + outputYAML, err := yaml.Marshal(trGot) + if err != nil { + return "", err + } + return string(outputYAML[:]), nil +}