refactor(makefile): consolidate the make targets used for bats tests (#…

…1746) New examples of running tests: 1. To run a specific bats file (with and without verbose output): make run-blackbox-tests BATS_TEST_FILE_PATH=test/blackbox/delete_images.bats make run-blackbox-tests BATS_TEST_FILE_PATH=test/blackbox/delete_images.bats BATS_VERBOSITY=2 2. To run the CI tests (with and without verbose output) make run-blackbox-ci make run-blackbox-ci BATS_VERBOSITY=2 BATS_TEST_FILE_PATH is used to pass on the test file to run using `run-blackbox-tests` BATS_VERBOSITY controls the verbosity of the bats framework output, if unspecified the output only contains test results and failure message in case of failures. If BATS_VERBOSITY is 1, then also show commands as they are executed. If BATS_VERBOSITY is 2, on top of the above it also shows output of passed tests. Other changes in this PR: - Update some of the tests to show logs after the run ends. - Run the linters before the tests, as it saves time on failures when running in GH Signed-off-by: Andrei Aaron <[email protected]>
project-zot · Sep 7, 2023 · 6115eed · 6115eed
1 parent f5b6396
commit 6115eed
Show file tree

Hide file tree

Showing 22 changed files with 204 additions and 216 deletions.
diff --git a/.github/workflows/ecosystem-tools.yaml b/.github/workflows/ecosystem-tools.yaml
@@ -69,42 +69,9 @@ jobs:
           sudo du -sh /var/lib/docker/
           du -sh /home/runner/work/
           set +x
-      - name: Run referrers tests
+      - name: Run CI tests
         run: |
-            make test-bats-referrers
-      - name: Run metadata tests
-        run: |
-            make test-bats-metadata
-      - name: Run push-pull tests
-        run: |
-            make test-push-pull
-      - name: Run push-pull-authn tests
-        run: |
-            make test-push-pull-authn
-      - name: Run metrics tests
-        run: |
-            make test-bats-metrics
-      - name: Run cve tests
-        run: |
-            make test-bats-cve
-      - name: Run sync test
-        run: |
-            make test-bats-sync
-      - name: Run scrub tests
-        run: |
-            make test-bats-scrub
-      - name: Run anonymous-push-pull tests
-        run: |
-            make test-anonymous-push-pull
-      - name: Run detect-manifest-collision tests
-        run: |
-            make test-detect-manifest-collision
-      - name: Run annotations tests
-        run: |
-            make test-annotations
-      - name: Run garbage collect tests
-        run: |
-            make test-garbage-collect
+          make run-blackbox-ci
       - name: Install localstack
         run: |
           pip install --upgrade pyopenssl
@@ -117,7 +84,7 @@ jobs:
           echo "Startup complete"
       - name: Run cloud-only tests
         run: |
-            make test-cloud-only
+          make run-blackbox-cloud-ci
         env:
           AWS_ACCESS_KEY_ID: fake
           AWS_SECRET_ACCESS_KEY: fake

diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml
@@ -44,15 +44,11 @@ jobs:
           echo "Waiting for LocalStack startup..."  # Wait 30 seconds for the LocalStack container
           localstack wait -t 30                     # to become ready before timing out
           echo "Startup complete"
-      - name: Run restore s3 blobs after cache is deleted
+      - name: Run blackbox nightly dedupe tests
         run: |
-            make test-restore-s3-blobs
-        env:
-          AWS_ACCESS_KEY_ID: fake
-          AWS_SECRET_ACCESS_KEY: fake
-      - name: Run dedupe filesystem blobs after switching dedupe to enable.
-        run: |
-            make test-push-pull-running-dedupe
+            # test restoring s3 blobs after cache is deleted
+            # test deduping filesystem blobs after switching dedupe to enable
+            make run-blackbox-dedupe-nightly
         env:
           AWS_ACCESS_KEY_ID: fake
           AWS_SECRET_ACCESS_KEY: fake
@@ -73,7 +69,8 @@ jobs:
           go mod download
       - name: Run sync harness
         run: |
-            make test-sync-harness
+            make run-blackbox-sync-nightly
+
   gc-referrers-stress-s3:
     name: GC(with referrers) on S3 with short interval
     runs-on: ubuntu-latest

diff --git a/Makefile b/Makefile
@@ -54,8 +54,17 @@ filter-valid = $(foreach ext, $(merged-extensions), $(if $(findstring $(ext),$(A
 add-extensions = $(subst $(1),$(2),$(sort $(filter-valid)))
 BUILD_LABELS = $(call add-extensions,$(space),$(comma))
 
+BATS_TEST_FILE_PATH ?= replace_me
+ifeq ($(BATS_VERBOSITY),2)
+	BATS_FLAGS = --trace --verbose-run --show-output-of-passing-tests --print-output-on-failure
+else ifeq ($(BATS_VERBOSITY),1)
+	BATS_FLAGS = --trace --verbose-run --print-output-on-failure
+else
+	BATS_FLAGS = --print-output-on-failure
+endif
+
 .PHONY: all
-all: modcheck swaggercheck binary binary-minimal binary-debug cli bench exporter-minimal verify-config test covhtml check check-gh-actions
+all: modcheck swaggercheck binary binary-minimal binary-debug cli bench exporter-minimal verify-config check check-gh-actions test covhtml
 
 .PHONY: modtidy
 modtidy:
@@ -162,6 +171,10 @@ run-bench: binary bench
 check-skopeo:
 	skopeo -v || (echo "You need skopeo to be installed in order to run tests"; exit 1)
 
+.PHONY: check-awslocal
+check-awslocal:
+	awslocal --version || (echo "You need awslocal to be installed in order to run tests"; exit 1)
+
 $(NOTATION):
 	mkdir -p $(TOOLSDIR)/bin
 	curl -Lo notation.tar.gz https://github.com/notaryproject/notation/releases/download/v$(NOTATION_VERSION)/notation_$(NOTATION_VERSION)_linux_amd64.tar.gz
@@ -349,127 +362,57 @@ $(BATS):
 	cd bats-core; ./install.sh $(TOOLSDIR); cd ..; \
 	rm -rf bats-core
 
-.PHONY: test-push-pull
-test-push-pull: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
-	$(BATS) --trace --print-output-on-failure test/blackbox/pushpull.bats
-
-.PHONY: test-push-pull-verbose
-test-push-pull-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
-	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull.bats
-
-.PHONY: test-garbage-collect
-test-garbage-collect: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/garbage_collect.bats
-
-.PHONY: test-garbage-collect-verbose
-test-garbage-collect-verbose: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS)
-	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/garbage_collect.bats
-
-.PHONY: test-push-pull-running-dedupe
-test-push-pull-running-dedupe: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
-	$(BATS) --trace --print-output-on-failure test/blackbox/pushpull_running_dedupe.bats
-
-.PHONY: test-push-pull-running-dedupe-verbose
-test-push-pull-running-dedupe-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
-	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull_running_dedupe.bats
-
-.PHONY: test-push-pull-authn
-test-push-pull-authn: check-linux binary check-skopeo $(BATS) $(REGCLIENT)
-	$(BATS) --trace --print-output-on-failure test/blackbox/pushpull_authn.bats
-
-.PHONY: test-sync-harness
-test-sync-harness: check-linux binary binary-minimal bench check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/sync_harness.bats
-
-.PHONY: test-sync-harness-verbose
-test-sync-harness-verbose: check-linux binary binary-minimal bench check-skopeo $(BATS)
-	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_harness.bats
-
-.PHONY: test-restore-s3-blobs
-test-restore-s3-blobs: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
-	$(BATS) --trace --print-output-on-failure test/blackbox/restore_s3_blobs.bats
-
-.PHONY: test-restore-s3-blobs-verbose
-test-restore-s3-blobs-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
-	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/restore_s3_blobs.bats
-
-.PHONY: test-bats-referrers
-test-bats-referrers: BUILD_LABELS=search
-test-bats-referrers: check-linux binary check-skopeo $(BATS) $(ORAS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/referrers.bats
-
-.PHONY: test-bats-metadata
-test-bats-metadata: BUILD_LABELS=search,userprefs
-test-bats-metadata: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/metadata.bats
-
-.PHONY: test-cloud-only
-test-cloud-only: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/cloud_only.bats
-
-.PHONY: test-cloud-only-verbose
-test-cloud-only-verbose: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/cloud_only.bats
-
-.PHONY: test-bats-sync
-test-bats-sync: BUILD_LABELS=sync
-test-bats-sync: check-linux binary binary-minimal bench check-skopeo $(BATS) $(NOTATION) $(COSIGN) $(HELM)
-	$(BATS) --trace --print-output-on-failure test/blackbox/sync.bats
-	$(BATS) --trace --print-output-on-failure test/blackbox/sync_docker.bats
-	$(BATS) --trace --print-output-on-failure test/blackbox/sync_replica_cluster.bats
-
-.PHONY: test-bats-delete-image
-test-bats-delete-image: check-linux binary binary-minimal bench check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/delete_images.bats
-
-.PHONY: test-bats-sync-verbose
-test-bats-sync-verbose: BUILD_LABELS=sync
-test-bats-sync-verbose: check-linux binary binary-minimal bench check-skopeo $(BATS) $(NOTATION) $(COSIGN) $(HELM)
-	$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync.bats
-	$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_docker.bats
-	$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_replica_cluster.bats
-
-.PHONY: test-bats-cve
-test-bats-cve: BUILD_LABELS=search
-test-bats-cve: check-linux binary cli check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/cve.bats
-
-.PHONY: test-bats-cve-verbose
-test-bats-cve-verbose: BUILD_LABELS=search
-test-bats-cve-verbose: check-linux binary cli check-skopeo $(BATS)
-	$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/cve.bats
-
-.PHONY: test-bats-scrub
-test-bats-scrub: BUILD_LABELS=scrub
-test-bats-scrub: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/scrub.bats
-
-.PHONY: test-bats-scrub-verbose
-test-bats-scrub-verbose: BUILD_LABELS=scrub
-test-bats-scrub-verbose: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/scrub.bats
-
-.PHONY: test-bats-metrics
-test-bats-metrics: BUILD_LABELS=metrics
-test-bats-metrics: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/metrics.bats
-
-.PHONY: test-bats-metrics-verbose
-test-bats-metrics-verbose: BUILD_LABELS=metrics
-test-bats-metrics-verbose: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/metrics.bats
-
-.PHONY: test-anonymous-push-pull
-test-anonymous-push-pull: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/anonymous_policy.bats
-
-.PHONY: test-annotations
-test-annotations: check-linux binary check-skopeo $(BATS) $(STACKER) $(NOTATION) $(COSIGN)
-	$(BATS) --trace --print-output-on-failure test/blackbox/annotations.bats
-
-.PHONY: test-detect-manifest-collision
-test-detect-manifest-collision: check-linux binary check-skopeo $(BATS) $(REGCLIENT)
-	$(BATS) --trace --print-output-on-failure test/blackbox/detect_manifest_collision.bats
+.PHONY: check-blackbox-prerequisites
+check-blackbox-prerequisites: check-linux check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL) $(NOTATION) $(COSIGN) $(STACKER)
+	which skopeo && skopeo --version; \
+	which stacker && stacker --version; \
+	which regctl && regctl version; \
+	which oras && oras version; \
+	which helm && helm version; \
+	which crictl && crictl version; \
+	which notation && notation version; \
+	which cosign && cosign version;
+
+.PHONY: run-blackbox-tests
+run-blackbox-tests: $(BATS_TEST_FILE_PATH) check-blackbox-prerequisites binary binary-minimal cli bench
+	echo running bats test "$(BATS_TEST_FILE_PATH)"; \
+	$(BATS) $(BATS_FLAGS) $(BATS_TEST_FILE_PATH)
+
+.PHONY: run-blackbox-ci
+run-blackbox-ci: check-blackbox-prerequisites binary binary-minimal cli
+	# ideally we would run a single bats command but too much disk space would be used at once
+	echo running CI bats tests; \
+	$(BATS) $(BATS_FLAGS) test/blackbox/pushpull.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/pushpull_authn.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/delete_images.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/referrers.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/metadata.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/anonymous_policy.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/annotations.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/detect_manifest_collision.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/cve.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/sync.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/sync_docker.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/sync_replica_cluster.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/scrub.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/garbage_collect.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/metrics.bats
+
+.PHONY: run-blackbox-cloud-ci
+run-blackbox-cloud-ci: check-blackbox-prerequisites check-awslocal binary $(BATS)
+	echo running cloud CI bats tests; \
+	$(BATS) $(BATS_FLAGS) test/blackbox/cloud_only.bats
+
+.PHONY: run-blackbox-dedupe-nightly
+run-blackbox-dedupe-nightly: check-blackbox-prerequisites check-awslocal binary binary-minimal
+	echo running nightly dedupe tests; \
+	$(BATS) $(BATS_FLAGS) test/blackbox/restore_s3_blobs.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/pushpull_running_dedupe.bats
+
+.PHONY: run-blackbox-sync-nightly
+run-blackbox-sync-nightly: check-blackbox-prerequisites binary binary-minimal bench
+	echo running nightly sync tests; \
+	$(BATS) $(BATS_FLAGS) test/blackbox/sync_harness.bats
 
 .PHONY: fuzz-all
 fuzz-all: fuzztime=${1}

diff --git a/test/blackbox/annotations.bats b/test/blackbox/annotations.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-annotations"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -79,8 +79,12 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
-    cat ${BATS_FILE_TMPDIR}/zot.log >&3
     zot_stop_all
     run rm -rf ${HOME}/.config/notation
 }

diff --git a/test/blackbox/anonymous_policy.bats b/test/blackbox/anonymous_policy.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-anonymous-push-pull"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -59,14 +59,20 @@ function setup_file() {
         }
     },
     "log": {
-        "level": "debug"
+        "level": "debug",
+        "output": "${BATS_FILE_TMPDIR}/zot.log"
     }
 }
 EOF
     zot_serve ${ZOT_PATH} ${zot_config_file}
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
     zot_stop_all
 }