feat: workflows - expected-fail cases extended (#6110)

* expected-fail cases extended * enterprise installation workflow - branch fixed * playwright workflow - missing image fixed
kubeshop · Jan 15, 2025 · 08bac87 · 08bac87
1 parent 878649a
commit 08bac87
Show file tree

Hide file tree

Showing 4 changed files with 204 additions and 43 deletions.
diff --git a/test/playwright/executor-tests/crd-workflow/smoke.yaml b/test/playwright/executor-tests/crd-workflow/smoke.yaml
@@ -43,6 +43,7 @@ spec:
       paths:
       - test/playwright/executor-tests/playwright-project
   container:
+    image: mcr.microsoft.com/playwright:v1.32.3-focal
     resources:
       requests:
         cpu: 2

diff --git a/test/special-cases/test-workflows/edge-cases-expected-fails.yaml b/test/special-cases/test-workflows/edge-cases-expected-fails.yaml
@@ -4,13 +4,51 @@ metadata:
   name: expected-fail-cypress-incorrect-repo-path
   labels:
     core-tests: expected-fail
+description: default no-cone mode
 spec:
   content:
     git:
       uri: https://github.com/kubeshop/testkube
       revision: main
       paths:
-      - test/cypress/executor-tests/some-incorrect-path
+      - test/cypress/executor-tests/some-incorrect-path # no-cone won't fail here
+  container:
+    resources:
+      requests:
+        cpu: 2
+        memory: 2Gi
+    workingDir: /data/repo/test/cypress/executor-tests/some-incorrect-path
+  steps:
+  - name: Run tests
+    run:
+      image: cypress/included:13.6.4
+      args:
+      - --env
+      - NON_CYPRESS_ENV=NON_CYPRESS_ENV_value
+      - --config
+      - '{"screenshotsFolder":"/data/artifacts/screenshots","videosFolder":"/data/artifacts/videos"}'
+      env:
+      - name: CYPRESS_CUSTOM_ENV
+        value: CYPRESS_CUSTOM_ENV_value
+    artifacts:
+      workingDir: /data/artifacts
+      paths:
+      - '**/*'
+---
+apiVersion: testworkflows.testkube.io/v1
+kind: TestWorkflow
+metadata:
+  name: expected-fail-cypress-incorrect-repo-path-cone-mode
+  labels:
+    core-tests: expected-fail
+spec:
+  content:
+    git:
+      uri: https://github.com/kubeshop/testkube
+      revision: main
+      cone: true
+      paths:
+      - test/cypress/executor-tests/some-incorrect-path # cone mode will fail here
   container:
     resources:
       requests:
@@ -148,6 +186,77 @@ spec:
 ---
 apiVersion: testworkflows.testkube.io/v1
 kind: TestWorkflow
+metadata:
+  name: expected-fail-cypress-artifacts-condition-always-with-artifacts
+  labels:
+    core-tests: expected-fail
+spec:
+  content:
+    git:
+      uri: https://github.com/kubeshop/testkube
+      revision: main
+      paths:
+      - test/cypress/executor-tests/cypress-13
+  container:
+    resources:
+      limits:
+        cpu: 2
+        memory: 2Gi
+    workingDir: /data/repo/test/cypress/executor-tests/cypress-13
+  steps:
+  - name: Run tests
+    run:
+      image: cypress/included:13.6.4
+      args:
+      - --config
+      - '{"screenshotsFolder":"/data/artifacts/screenshots","videosFolder":"/data/artifacts/videos"}'
+    steps:
+    - name: Saving artifacts
+      condition: always
+      workingDir: /data/artifacts
+      artifacts:
+        paths:
+        - '**/*'
+---
+apiVersion: testworkflows.testkube.io/v1
+kind: TestWorkflow
+metadata:
+  name: expected-fail-k6-test-report-condition-always-no-artifacts
+  labels:
+    core-tests: expected-fail
+spec:
+  content:
+    git:
+      uri: https://github.com/kubeshop/testkube
+      revision: main
+      paths:
+      - test/k6/executor-tests/k6-smoke-test.js
+  container:
+    resources:
+      requests:
+        cpu: 128m
+        memory: 128Mi
+    workingDir: /data/repo/test/k6/executor-tests
+  steps:
+  - name: Run test
+    container:
+      image: grafana/k6:0.49.0
+    steps:
+    - shell: mkdir /data/artifacts
+    - run:
+        args:
+        - run
+        - some-incorrect-file-name.js
+      steps:
+      - name: Saving artifacts # should be run despite step failure
+        condition: always
+        workingDir: /data/artifacts
+        artifacts:
+          paths:
+          - '*'
+---
+apiVersion: testworkflows.testkube.io/v1
+kind: TestWorkflow
 metadata:
   name: expected-fail-timeout-activedeadlineseconds
   labels:
@@ -215,45 +324,91 @@ spec:
 apiVersion: testworkflows.testkube.io/v1
 kind: TestWorkflow
 metadata:
-  name: expected-fail-cypress-artifacts-condition-always-with-artifacts
+  name: expected-fail-private-registry-auth
+  labels:
+    core-tests: expected-fail
+spec:
+  container:
+    resources:
+      requests:
+        cpu: 32m
+        memory: 32Mi
+  steps:
+    - name: Run tests
+      shell: curl -f -LI https://testkube-test-page-lipsum.pages.dev/ && sleep 10
+      container:
+        image: tkoniecznykubeshop/example-private-repo:curl-8.11.1
+---
+apiVersion: testworkflows.testkube.io/v1
+kind: TestWorkflow
+metadata:
+  name: expected-fail-private-repo-auth
   labels:
     core-tests: expected-fail
 spec:
   content:
     git:
-      uri: https://github.com/kubeshop/testkube
+      uri: https://github.com/kubeshop/some-example-repo
       revision: main
       paths:
-      - test/cypress/executor-tests/cypress-13
+      - test/postman/executor-tests/postman-executor-smoke.postman_collection.json
   container:
     resources:
-      limits:
-        cpu: 2
-        memory: 2Gi
-    workingDir: /data/repo/test/cypress/executor-tests/cypress-13
+      requests:
+        cpu: 256m
+        memory: 128Mi
+    workingDir: /data/repo/test/postman/executor-tests
   steps:
-  - name: Run tests
+  - name: Run test
     run:
-      image: cypress/included:13.6.4
+      image: postman/newman:6-alpine
       args:
-      - --env
-      - NON_CYPRESS_ENV=NON_CYPRESS_ENV_value
-      - --config
-      - '{"screenshotsFolder":"/data/artifacts/screenshots","videosFolder":"/data/artifacts/videos"}'
-      env:
-      - name: CYPRESS_CUSTOM_ENV
-        value: CYPRESS_CUSTOM_ENV_value
-    steps:
-    - name: Saving artifacts
-      workingDir: /data/artifacts
-      artifacts:
-        paths:
-        - '**/*'
+      - run
+      - postman-executor-smoke.postman_collection.json
+      - "--env-var"
+      - "TESTKUBE_POSTMAN_PARAM=TESTKUBE_POSTMAN_PARAM_value"
 ---
 apiVersion: testworkflows.testkube.io/v1
 kind: TestWorkflow
 metadata:
-  name: expected-fail-k6-test-report-condition-always-no-artifacts
+  name: expected-fail-missing-git-secret
+  labels:
+    core-tests: expected-fail
+spec:
+  content:
+    git:
+      uri: https://github.com/kubeshop/some-example-repo
+      revision: main
+      usernameFrom:
+        secretKeyRef:
+          name: non-existant-secret
+          key: git-username
+      tokenFrom:
+        secretKeyRef:
+          name: non-existant-secret2
+          key: git-token
+      paths:
+      - test/postman/executor-tests/postman-executor-smoke.postman_collection.json
+  container:
+    resources:
+      requests:
+        cpu: 256m
+        memory: 128Mi
+    workingDir: /data/repo/test/postman/executor-tests
+  steps:
+  - name: Run test
+    run:
+      image: postman/newman:6-alpine
+      args:
+      - run
+      - postman-executor-smoke.postman_collection.json
+      - "--env-var"
+      - "TESTKUBE_POSTMAN_PARAM=TESTKUBE_POSTMAN_PARAM_value"
+---
+apiVersion: testworkflows.testkube.io/v1
+kind: TestWorkflow
+metadata:
+  name: expected-fail-missing-env-secret
   labels:
     core-tests: expected-fail
 spec:
@@ -262,27 +417,25 @@ spec:
       uri: https://github.com/kubeshop/testkube
       revision: main
       paths:
-      - test/k6/executor-tests/k6-smoke-test.js
+      - test/postman/executor-tests/postman-executor-smoke.postman_collection.json
   container:
     resources:
       requests:
-        cpu: 128m
+        cpu: 256m
         memory: 128Mi
-    workingDir: /data/repo/test/k6/executor-tests
+    workingDir: /data/repo/test/postman/executor-tests
   steps:
   - name: Run test
-    container:
-      image: grafana/k6:0.49.0
-    steps:
-    - shell: mkdir /data/artifacts
-    - run:
-        args:
-        - run
-        - some-incorrect-file-name.js
-      steps:
-      - name: Saving artifacts # should be run despite step failure
-        condition: always
-        workingDir: /data/artifacts
-        artifacts:
-          paths:
-          - '*'
+    run:
+      image: postman/newman:6-alpine
+      env:
+      - name: EXAMPLE_ENV
+        valueFrom:
+          secretKeyRef:
+            name: non-existant-secret
+            key: key
+      args:
+      - run
+      - postman-executor-smoke.postman_collection.json
+      - "--env-var"
+      - "TESTKUBE_POSTMAN_PARAM=TESTKUBE_POSTMAN_PARAM_value"
diff --git a/test/suites/special-cases/test-workflows/edge-cases-expected-fails.yaml b/test/suites/special-cases/test-workflows/edge-cases-expected-fails.yaml
@@ -9,8 +9,15 @@ spec:
   - execute:
       workflows:
       - name: expected-fail-cypress-incorrect-repo-path
+      - name: expected-fail-cypress-incorrect-repo-path-cone-mode
       - name: expected-fail-cypress-incorrect-image-name
       - name: expected-fail-cypress-incorrect-workingdir
       - name: expected-fail-cypress-resource-limits
       - name: expected-fail-cypress-artifacts-condition-always-with-artifacts
       - name: expected-fail-k6-test-report-condition-always-no-artifacts
+      - name: expected-fail-timeout-activedeadlineseconds
+      - name: expected-fail-timeout-step
+      - name: expected-fail-private-registry-auth
+      - name: expected-fail-private-repo-auth
+      - name: expected-fail-missing-git-secret
+      - name: expected-fail-missing-env-secret
diff --git a/test/testkube/enterprise-installation.yaml b/test/testkube/enterprise-installation.yaml
@@ -19,7 +19,7 @@ spec:
           key: json-sa-key
     git:
       uri: https://github.com/kubeshop/testkube
-      revision: enterprise-installation-workflow
+      revision: main
       paths:
       - test/testkube/enterprise-installation-expect-file.exp # expect file to finalize installation (user choices required): TKC-3026
   container: