POC conformance test suite

test/poc_conformance_test.go

@@ -0,0 +1,224 @@
+//go:build conformance
+// +build conformance
+
+/*
+This serves as a POC for conformance test suite design including functionality,
+behavioural and fields population.
+It mocks the "black-box" execution of TaskRuns and PipelineRuns utilizing the
+Tekton clients to mock the controller of a conformant vendor service.
+
+Please use the following for triggering the test:
+go test -v -tags=conformance -count=1 ./test -run ^TestConformance
+
+The next step will be to integrate this test as POC with v2 API.
+*/
+
+package test
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1"
+	"github.com/tektoncd/pipeline/test/parse"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes/scheme"
+	knativetest "knative.dev/pkg/test"
+	"knative.dev/pkg/test/helpers"
+	"sigs.k8s.io/yaml"
+)
+
+const (
+	TaskRunInputType     = "TaskRun"
+	PipelineRunInputType = "PipelineRun"
+)
+
+func TestConformanceShouldProvideTaskResult(t *testing.T) {
+	inputYAML := fmt.Sprintf(`
+apiVersion: tekton.dev/v1
+kind: TaskRun
+metadata:
+  name: %s
+spec:
+  taskSpec:
+    params:
+    - name: multiplicand
+      description: the first operand
+      default: %s
+    - name: multipliper
+      description: the second operand
+      default: %s
+    results:
+    - name: product
+      description: the product of the first and second operand
+    steps:
+    - name: add
+      image: alpine
+      env:
+      - name: OP1
+        value: $(params.multiplicand)
+      - name: OP2
+        value: $(params.multipliper)
+      command: ["/bin/sh", "-c"]
+      args:
+      - echo -n $((${OP1}*${OP2})) | tee $(results.product.path);
+`, helpers.ObjectNameForTest(t), "3", "5")
+
+	// Black box execution of Pipeline CRDs that should be implemented by Vendor service
+	outputYAML, err := ProcessAndSendToTekton(inputYAML, TaskRunInputType, t)
+	if err != nil {
+		t.Fatalf("Vendor service failed processing inputYAML: %s", err)
+	}
+
+	// Parse and validate output YAML
+	resolvedTR := parse.MustParseV1TaskRun(t, outputYAML)
+	if len(resolvedTR.Status.Results) != 1 {
+		t.Errorf("Expect vendor service to provide 1 result but not")
+	}
+
+	if resolvedTR.Status.Results[0].Value.StringVal != "15" {
+		t.Errorf("Not producing correct result :%s", resolvedTR.Status.Results[0].Value.StringVal)
+	}
+}
+
+func TestConformanceShouldHonorTaskRunTimeout(t *testing.T) {
+	expectedFailedStatus := true
+	inputYAML := fmt.Sprintf(`
+apiVersion: tekton.dev/v1
+kind: TaskRun
+metadata:
+  name: %s
+spec:
+  timeout: 15s
+  taskSpec:
+    steps:
+    - image: busybox
+      command: ['/bin/sh']
+      args: ['-c', 'sleep 15001']
+`, helpers.ObjectNameForTest(t))
+
+	// Black box execution of Pipeline CRDs that should be implemented by Vendor service
+	outputYAML, err := ProcessAndSendToTekton(inputYAML, TaskRunInputType, t, expectedFailedStatus)
+	if err != nil {
+		t.Fatalf("Vendor service failed processing inputYAML: %s", err)
+	}
+
+	// Parse and validate output YAML
+	resolvedTR := parse.MustParseV1TaskRun(t, outputYAML)
+	if len(resolvedTR.Status.Conditions) != 1 {
+		t.Errorf("Expect vendor service to populate 1 Condition but no")
+	}
+
+	if resolvedTR.Status.Conditions[0].Type != "Succeeded" {
+		t.Errorf("Expect vendor service to populate Condition `Succeeded` but got: %s", resolvedTR.Status.Conditions[0].Type)
+	}
+
+	if resolvedTR.Status.Conditions[0].Status != "False" {
+		t.Errorf("Expect vendor service to populate Condition `False` but got: %s", resolvedTR.Status.Conditions[0].Status)
+	}
+
+	if resolvedTR.Status.Conditions[0].Reason != "TaskRunTimeout" {
+		t.Errorf("Expect vendor service to populate Condition Reason `TaskRunTimeout` but got: %s", resolvedTR.Status.Conditions[0].Reason)
+	}
+}
+
+func TestConformanceShouldPopulateConditions(t *testing.T) {
+	inputYAML := fmt.Sprintf(`
+apiVersion: tekton.dev/v1
+kind: TaskRun
+metadata:
+  name: %s
+spec:
+  taskSpec:
+    steps:
+    - name: add
+      image: ubuntu
+      script:
+        echo Hello world!
+`, helpers.ObjectNameForTest(t))
+
+	// Black box execution of Pipeline CRDs that should be implemented by Vendor service
+	outputYAML, err := ProcessAndSendToTekton(inputYAML, TaskRunInputType, t)
+	if err != nil {
+		t.Fatalf("Vendor service failed processing inputYAML: %s", err)
+	}
+
+	// Parse and validate output YAML
+	resolvedTR := parse.MustParseV1TaskRun(t, outputYAML)
+	if len(resolvedTR.Status.Conditions) != 1 {
+		t.Errorf("Expect vendor service to populate 1 Condition but no")
+	}
+
+	if resolvedTR.Status.Conditions[0].Type != "Succeeded" {
+		t.Errorf("Expect vendor service to populate Condition `Succeeded` but got: %s", resolvedTR.Status.Conditions[0].Type)
+	}
+
+	if resolvedTR.Status.Conditions[0].Status != "True" {
+		t.Errorf("Expect vendor service to populate Condition `True` but got: %s", resolvedTR.Status.Conditions[0].Status)
+	}
+}
+
+// ProcessAndSendToTekton takes in vanilla Tekton PipelineRun and TaskRun, waits for the object to succeed and outputs the final PipelineRun and TaskRun with status.
+// The parameters are inputYAML and its Primitive type {PipelineRun, TaskRun}
+// And the return values will be the output YAML string and errors.
+func ProcessAndSendToTekton(inputYAML, primitiveType string, customInputs ...interface{}) (string, error) {
+	// Handle customInputs
+	var t *testing.T
+	var expectRunToFail bool
+	for _, customInput := range customInputs {
+		if ci, ok := customInput.(*testing.T); ok {
+			t = ci
+		}
+		if ci, ok := customInput.(bool); ok {
+			expectRunToFail = ci
+		}
+	}
+
+	return mockTektonPipelineController(t, inputYAML, primitiveType, expectRunToFail)
+}
+
+// mockTektonPipelineController fakes the behaviour of a vendor service by utilizing the Tekton test infrastructure.
+// For the POC, it uses the Tetkon clients to Create, Wait for and Get the expected TaskRun.
+func mockTektonPipelineController(t *testing.T, inputYAML, primitiveType string, expectRunToFail bool) (string, error) {
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	c, namespace := setup(ctx, t)
+
+	knativetest.CleanupOnInterrupt(func() { tearDown(ctx, t, c, namespace) }, t.Logf)
+	defer tearDown(ctx, t, c, namespace)
+
+	// Parse inputYAML, parse.MustParseTaskRun
+	var tr v1.TaskRun
+	if _, _, err := scheme.Codecs.UniversalDeserializer().Decode([]byte(inputYAML), nil, &tr); err != nil {
+		return "", fmt.Errorf("must parse YAML (%s): %v", inputYAML, err)
+	}
+
+	// Create TaskRun via TaskRunClient
+	trResolved, err := c.V1TaskRunClient.Create(ctx, &tr, metav1.CreateOptions{})
+	if err != nil {
+		return "", fmt.Errorf("Failed to create TaskRun `%v`: %w", trResolved, err)
+	}
+
+	var caf ConditionAccessorFn
+	caf = Succeed(trResolved.Name)
+	if expectRunToFail {
+		caf = Failed(trResolved.Name)
+	}
+	if err := WaitForTaskRunState(ctx, c, trResolved.Name, caf, "WaitTaskRunDone", v1Version); err != nil {
+		return "", fmt.Errorf("Error waiting for TaskRun to finish: %s", err)
+	}
+
+	// Retrieve the TaskRun via TaskRunClient
+	trGot, err := c.V1TaskRunClient.Get(ctx, trResolved.Name, metav1.GetOptions{})
+	if err != nil {
+		return "", fmt.Errorf("Failed to get TaskRun `%s`: %s", trGot.Name, err)
+	}
+
+	outputYAML, err := yaml.Marshal(trGot)
+	if err != nil {
+		return "", err
+	}
+	return string(outputYAML[:]), nil
+}