From 6115eed4ec110090c619d714cd1949ee28174fdb Mon Sep 17 00:00:00 2001
From: Andrei Aaron <aaaron@luxoft.com>
Date: Thu, 7 Sep 2023 21:06:21 +0300
Subject: [PATCH] refactor(makefile): consolidate the make targets used for
 bats tests (#1746)

New examples of running tests:

1. To run a specific bats file (with and without verbose output):
make run-blackbox-tests BATS_TEST_FILE_PATH=test/blackbox/delete_images.bats
make run-blackbox-tests BATS_TEST_FILE_PATH=test/blackbox/delete_images.bats BATS_VERBOSITY=2

2. To run the CI tests (with and without verbose output)
make run-blackbox-ci
make run-blackbox-ci BATS_VERBOSITY=2

BATS_TEST_FILE_PATH is used to pass on the test file to run using `run-blackbox-tests`
BATS_VERBOSITY controls the verbosity of the bats framework output, if unspecified the output only
contains test results and failure message in case of failures.
If BATS_VERBOSITY is 1, then also show commands as they are executed.
If BATS_VERBOSITY is 2, on top of the above it also shows output of passed tests.

Other changes in this PR:
- Update some of the tests to show logs after the run ends.
- Run the linters before the tests, as it saves time on failures when running in GH

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
---
 .github/workflows/ecosystem-tools.yaml       |  39 +---
 .github/workflows/nightly.yaml               |  15 +-
 Makefile                                     | 177 +++++++------------
 test/blackbox/annotations.bats               |   8 +-
 test/blackbox/anonymous_policy.bats          |  10 +-
 test/blackbox/cloud_only.bats                |  42 +++--
 test/blackbox/cve.bats                       |  10 +-
 test/blackbox/delete_images.bats             |  17 +-
 test/blackbox/detect_manifest_collision.bats |  10 +-
 test/blackbox/garbage_collect.bats           |   7 +-
 test/blackbox/metadata.bats                  |  11 +-
 test/blackbox/metrics.bats                   |   7 +-
 test/blackbox/pushpull.bats                  |  10 +-
 test/blackbox/pushpull_authn.bats            |   8 +-
 test/blackbox/pushpull_running_dedupe.bats   |  10 +-
 test/blackbox/referrers.bats                 |   7 +-
 test/blackbox/restore_s3_blobs.bats          |   9 +
 test/blackbox/scrub.bats                     |   5 +-
 test/blackbox/sync.bats                      |   2 +-
 test/blackbox/sync_docker.bats               |   2 +-
 test/blackbox/sync_harness.bats              |   2 +-
 test/blackbox/sync_replica_cluster.bats      |   2 +-
 22 files changed, 199 insertions(+), 211 deletions(-)

diff --git a/.github/workflows/ecosystem-tools.yaml b/.github/workflows/ecosystem-tools.yaml
index dd169560..4d6bf34a 100644
--- a/.github/workflows/ecosystem-tools.yaml
+++ b/.github/workflows/ecosystem-tools.yaml
@@ -69,42 +69,9 @@ jobs:
           sudo du -sh /var/lib/docker/
           du -sh /home/runner/work/
           set +x
-      - name: Run referrers tests
+      - name: Run CI tests
         run: |
-            make test-bats-referrers
-      - name: Run metadata tests
-        run: |
-            make test-bats-metadata
-      - name: Run push-pull tests
-        run: |
-            make test-push-pull
-      - name: Run push-pull-authn tests
-        run: |
-            make test-push-pull-authn
-      - name: Run metrics tests
-        run: |
-            make test-bats-metrics
-      - name: Run cve tests
-        run: |
-            make test-bats-cve
-      - name: Run sync test
-        run: |
-            make test-bats-sync
-      - name: Run scrub tests
-        run: |
-            make test-bats-scrub
-      - name: Run anonymous-push-pull tests
-        run: |
-            make test-anonymous-push-pull
-      - name: Run detect-manifest-collision tests
-        run: |
-            make test-detect-manifest-collision
-      - name: Run annotations tests
-        run: |
-            make test-annotations
-      - name: Run garbage collect tests
-        run: |
-            make test-garbage-collect
+          make run-blackbox-ci
       - name: Install localstack
         run: |
           pip install --upgrade pyopenssl
@@ -117,7 +84,7 @@ jobs:
           echo "Startup complete"
       - name: Run cloud-only tests
         run: |
-            make test-cloud-only
+          make run-blackbox-cloud-ci
         env:
           AWS_ACCESS_KEY_ID: fake
           AWS_SECRET_ACCESS_KEY: fake
diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml
index 8c8a1896..23eacf84 100644
--- a/.github/workflows/nightly.yaml
+++ b/.github/workflows/nightly.yaml
@@ -44,15 +44,11 @@ jobs:
           echo "Waiting for LocalStack startup..."  # Wait 30 seconds for the LocalStack container
           localstack wait -t 30                     # to become ready before timing out
           echo "Startup complete"
-      - name: Run restore s3 blobs after cache is deleted
+      - name: Run blackbox nightly dedupe tests
         run: |
-            make test-restore-s3-blobs
-        env:
-          AWS_ACCESS_KEY_ID: fake
-          AWS_SECRET_ACCESS_KEY: fake
-      - name: Run dedupe filesystem blobs after switching dedupe to enable.
-        run: |
-            make test-push-pull-running-dedupe
+            # test restoring s3 blobs after cache is deleted
+            # test deduping filesystem blobs after switching dedupe to enable
+            make run-blackbox-dedupe-nightly
         env:
           AWS_ACCESS_KEY_ID: fake
           AWS_SECRET_ACCESS_KEY: fake
@@ -73,7 +69,8 @@ jobs:
           go mod download
       - name: Run sync harness
         run: |
-            make test-sync-harness
+            make run-blackbox-sync-nightly
+
   gc-referrers-stress-s3:
     name: GC(with referrers) on S3 with short interval
     runs-on: ubuntu-latest
diff --git a/Makefile b/Makefile
index 9e8d6c6b..0927198c 100644
--- a/Makefile
+++ b/Makefile
@@ -54,8 +54,17 @@ filter-valid = $(foreach ext, $(merged-extensions), $(if $(findstring $(ext),$(A
 add-extensions = $(subst $(1),$(2),$(sort $(filter-valid)))
 BUILD_LABELS = $(call add-extensions,$(space),$(comma))
 
+BATS_TEST_FILE_PATH ?= replace_me
+ifeq ($(BATS_VERBOSITY),2)
+	BATS_FLAGS = --trace --verbose-run --show-output-of-passing-tests --print-output-on-failure
+else ifeq ($(BATS_VERBOSITY),1)
+	BATS_FLAGS = --trace --verbose-run --print-output-on-failure
+else
+	BATS_FLAGS = --print-output-on-failure
+endif
+
 .PHONY: all
-all: modcheck swaggercheck binary binary-minimal binary-debug cli bench exporter-minimal verify-config test covhtml check check-gh-actions
+all: modcheck swaggercheck binary binary-minimal binary-debug cli bench exporter-minimal verify-config check check-gh-actions test covhtml
 
 .PHONY: modtidy
 modtidy:
@@ -162,6 +171,10 @@ run-bench: binary bench
 check-skopeo:
 	skopeo -v || (echo "You need skopeo to be installed in order to run tests"; exit 1)
 
+.PHONY: check-awslocal
+check-awslocal:
+	awslocal --version || (echo "You need awslocal to be installed in order to run tests"; exit 1)
+
 $(NOTATION):
 	mkdir -p $(TOOLSDIR)/bin
 	curl -Lo notation.tar.gz https://github.com/notaryproject/notation/releases/download/v$(NOTATION_VERSION)/notation_$(NOTATION_VERSION)_linux_amd64.tar.gz
@@ -349,127 +362,57 @@ $(BATS):
 	cd bats-core; ./install.sh $(TOOLSDIR); cd ..; \
 	rm -rf bats-core
 
-.PHONY: test-push-pull
-test-push-pull: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
-	$(BATS) --trace --print-output-on-failure test/blackbox/pushpull.bats
+.PHONY: check-blackbox-prerequisites
+check-blackbox-prerequisites: check-linux check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL) $(NOTATION) $(COSIGN) $(STACKER)
+	which skopeo && skopeo --version; \
+	which stacker && stacker --version; \
+	which regctl && regctl version; \
+	which oras && oras version; \
+	which helm && helm version; \
+	which crictl && crictl version; \
+	which notation && notation version; \
+	which cosign && cosign version;
 
-.PHONY: test-push-pull-verbose
-test-push-pull-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
-	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull.bats
+.PHONY: run-blackbox-tests
+run-blackbox-tests: $(BATS_TEST_FILE_PATH) check-blackbox-prerequisites binary binary-minimal cli bench
+	echo running bats test "$(BATS_TEST_FILE_PATH)"; \
+	$(BATS) $(BATS_FLAGS) $(BATS_TEST_FILE_PATH)
 
-.PHONY: test-garbage-collect
-test-garbage-collect: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/garbage_collect.bats
+.PHONY: run-blackbox-ci
+run-blackbox-ci: check-blackbox-prerequisites binary binary-minimal cli
+	# ideally we would run a single bats command but too much disk space would be used at once
+	echo running CI bats tests; \
+	$(BATS) $(BATS_FLAGS) test/blackbox/pushpull.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/pushpull_authn.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/delete_images.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/referrers.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/metadata.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/anonymous_policy.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/annotations.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/detect_manifest_collision.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/cve.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/sync.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/sync_docker.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/sync_replica_cluster.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/scrub.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/garbage_collect.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/metrics.bats
 
-.PHONY: test-garbage-collect-verbose
-test-garbage-collect-verbose: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS)
-	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/garbage_collect.bats
+.PHONY: run-blackbox-cloud-ci
+run-blackbox-cloud-ci: check-blackbox-prerequisites check-awslocal binary $(BATS)
+	echo running cloud CI bats tests; \
+	$(BATS) $(BATS_FLAGS) test/blackbox/cloud_only.bats
 
-.PHONY: test-push-pull-running-dedupe
-test-push-pull-running-dedupe: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
-	$(BATS) --trace --print-output-on-failure test/blackbox/pushpull_running_dedupe.bats
+.PHONY: run-blackbox-dedupe-nightly
+run-blackbox-dedupe-nightly: check-blackbox-prerequisites check-awslocal binary binary-minimal
+	echo running nightly dedupe tests; \
+	$(BATS) $(BATS_FLAGS) test/blackbox/restore_s3_blobs.bats && \
+	$(BATS) $(BATS_FLAGS) test/blackbox/pushpull_running_dedupe.bats
 
-.PHONY: test-push-pull-running-dedupe-verbose
-test-push-pull-running-dedupe-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
-	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull_running_dedupe.bats
-
-.PHONY: test-push-pull-authn
-test-push-pull-authn: check-linux binary check-skopeo $(BATS) $(REGCLIENT)
-	$(BATS) --trace --print-output-on-failure test/blackbox/pushpull_authn.bats
-
-.PHONY: test-sync-harness
-test-sync-harness: check-linux binary binary-minimal bench check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/sync_harness.bats
-
-.PHONY: test-sync-harness-verbose
-test-sync-harness-verbose: check-linux binary binary-minimal bench check-skopeo $(BATS)
-	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_harness.bats
-
-.PHONY: test-restore-s3-blobs
-test-restore-s3-blobs: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
-	$(BATS) --trace --print-output-on-failure test/blackbox/restore_s3_blobs.bats
-
-.PHONY: test-restore-s3-blobs-verbose
-test-restore-s3-blobs-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
-	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/restore_s3_blobs.bats
-
-.PHONY: test-bats-referrers
-test-bats-referrers: BUILD_LABELS=search
-test-bats-referrers: check-linux binary check-skopeo $(BATS) $(ORAS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/referrers.bats
-
-.PHONY: test-bats-metadata
-test-bats-metadata: BUILD_LABELS=search,userprefs
-test-bats-metadata: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/metadata.bats
-
-.PHONY: test-cloud-only
-test-cloud-only: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/cloud_only.bats
-
-.PHONY: test-cloud-only-verbose
-test-cloud-only-verbose: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/cloud_only.bats
-
-.PHONY: test-bats-sync
-test-bats-sync: BUILD_LABELS=sync
-test-bats-sync: check-linux binary binary-minimal bench check-skopeo $(BATS) $(NOTATION) $(COSIGN) $(HELM)
-	$(BATS) --trace --print-output-on-failure test/blackbox/sync.bats
-	$(BATS) --trace --print-output-on-failure test/blackbox/sync_docker.bats
-	$(BATS) --trace --print-output-on-failure test/blackbox/sync_replica_cluster.bats
-
-.PHONY: test-bats-delete-image
-test-bats-delete-image: check-linux binary binary-minimal bench check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/delete_images.bats
-
-.PHONY: test-bats-sync-verbose
-test-bats-sync-verbose: BUILD_LABELS=sync
-test-bats-sync-verbose: check-linux binary binary-minimal bench check-skopeo $(BATS) $(NOTATION) $(COSIGN) $(HELM)
-	$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync.bats
-	$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_docker.bats
-	$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_replica_cluster.bats
-
-.PHONY: test-bats-cve
-test-bats-cve: BUILD_LABELS=search
-test-bats-cve: check-linux binary cli check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/cve.bats
-
-.PHONY: test-bats-cve-verbose
-test-bats-cve-verbose: BUILD_LABELS=search
-test-bats-cve-verbose: check-linux binary cli check-skopeo $(BATS)
-	$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/cve.bats
-
-.PHONY: test-bats-scrub
-test-bats-scrub: BUILD_LABELS=scrub
-test-bats-scrub: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/scrub.bats
-
-.PHONY: test-bats-scrub-verbose
-test-bats-scrub-verbose: BUILD_LABELS=scrub
-test-bats-scrub-verbose: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/scrub.bats
-
-.PHONY: test-bats-metrics
-test-bats-metrics: BUILD_LABELS=metrics
-test-bats-metrics: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/metrics.bats
-
-.PHONY: test-bats-metrics-verbose
-test-bats-metrics-verbose: BUILD_LABELS=metrics
-test-bats-metrics-verbose: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/metrics.bats
-
-.PHONY: test-anonymous-push-pull
-test-anonymous-push-pull: check-linux binary check-skopeo $(BATS)
-	$(BATS) --trace --print-output-on-failure test/blackbox/anonymous_policy.bats
-
-.PHONY: test-annotations
-test-annotations: check-linux binary check-skopeo $(BATS) $(STACKER) $(NOTATION) $(COSIGN)
-	$(BATS) --trace --print-output-on-failure test/blackbox/annotations.bats
-
-.PHONY: test-detect-manifest-collision
-test-detect-manifest-collision: check-linux binary check-skopeo $(BATS) $(REGCLIENT)
-	$(BATS) --trace --print-output-on-failure test/blackbox/detect_manifest_collision.bats
+.PHONY: run-blackbox-sync-nightly
+run-blackbox-sync-nightly: check-blackbox-prerequisites binary binary-minimal bench
+	echo running nightly sync tests; \
+	$(BATS) $(BATS_FLAGS) test/blackbox/sync_harness.bats
 
 .PHONY: fuzz-all
 fuzz-all: fuzztime=${1}
diff --git a/test/blackbox/annotations.bats b/test/blackbox/annotations.bats
index e886f130..dc9f9e3a 100644
--- a/test/blackbox/annotations.bats
+++ b/test/blackbox/annotations.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-annotations"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -79,8 +79,12 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
-    cat ${BATS_FILE_TMPDIR}/zot.log >&3
     zot_stop_all
     run rm -rf ${HOME}/.config/notation
 }
diff --git a/test/blackbox/anonymous_policy.bats b/test/blackbox/anonymous_policy.bats
index ca6d22c1..577e6c86 100644
--- a/test/blackbox/anonymous_policy.bats
+++ b/test/blackbox/anonymous_policy.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-anonymous-push-pull"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -59,7 +59,8 @@ function setup_file() {
         }
     },
     "log": {
-        "level": "debug"
+        "level": "debug",
+        "output": "${BATS_FILE_TMPDIR}/zot.log"
     }
 }
 EOF
@@ -67,6 +68,11 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
     zot_stop_all
 }
diff --git a/test/blackbox/cloud_only.bats b/test/blackbox/cloud_only.bats
index be823fb7..0894b347 100644
--- a/test/blackbox/cloud_only.bats
+++ b/test/blackbox/cloud_only.bats
@@ -1,3 +1,7 @@
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-cloud-ci"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
 load helpers_cloud
 load helpers_wait
 
@@ -17,8 +21,8 @@ function setup() {
 
     cat > ${zot_config_file}<<EOF
 {
-	"distSpecVersion": "1.1.0-dev",
-	"storage": {
+    "distSpecVersion": "1.1.0-dev",
+    "storage": {
         "rootDirectory": "${zot_root_dir}",
         "dedupe": true,
         "remoteCache": true,
@@ -43,10 +47,10 @@ function setup() {
             "apiKeyTablename":"ApiKeyTable",
             "versionTablename": "Version"
         }
-	},
-	"http": {
-		"address": "127.0.0.1",
-		"port": "8080",
+    },
+    "http": {
+        "address": "127.0.0.1",
+        "port": "8080",
         "realm": "zot",
         "auth": {
             "openid": {
@@ -68,25 +72,25 @@ function setup() {
                 }
             }
         }
-	},
-	"log": {
-		"level": "debug"
-	},
-	"extensions": {
-		"metrics": {
+    },
+    "log": {
+        "level": "debug"
+    },
+    "extensions": {
+        "metrics": {
             "enable": true,
             "prometheus": {
                 "path": "/metrics"
             }
         },
-		"search": {
+        "search": {
             "enable": true
-		},
-		"scrub": {
-			"enable": true,
-			"interval": "24h"
-		}
-	}
+        },
+        "scrub": {
+            "enable": true,
+            "interval": "24h"
+        }
+    }
 }
 EOF
     awslocal s3 --region "us-east-2" mb s3://zot-storage
diff --git a/test/blackbox/cve.bats b/test/blackbox/cve.bats
index c5ded544..326179ac 100644
--- a/test/blackbox/cve.bats
+++ b/test/blackbox/cve.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-cve" or "make test-bats-cve-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -42,7 +42,8 @@ function setup_file() {
         "port": "8080"
     },
     "log": {
-        "level": "debug"
+        "level": "debug",
+        "output": "${BATS_FILE_TMPDIR}/zot.log"
     },
     "extensions": {
         "search": {
@@ -63,6 +64,11 @@ EOF
     zli_add_config ${registry_name} ${registry_url}
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
     zot_stop_all
 }
diff --git a/test/blackbox/delete_images.bats b/test/blackbox/delete_images.bats
index 8f0a96a4..e099faba 100644
--- a/test/blackbox/delete_images.bats
+++ b/test/blackbox/delete_images.bats
@@ -1,3 +1,7 @@
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
 load helpers_zot
 
 function verify_prerequisites {
@@ -14,7 +18,7 @@ function verify_prerequisites {
 
 function setup_file() {
     # Verify prerequisites are available
-    if ! (verify_prerequisites); then
+    if ! verify_prerequisites; then
         exit 1
     fi
 
@@ -22,10 +26,10 @@ function setup_file() {
     skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/test-images/alpine:3.17.3 oci:${TEST_DATA_DIR}/alpine:3.17.3
 
     # Setup zot server
-    ZOT_ROOT_DIR=${BATS_RUN_TMPDIR}/zot
+    ZOT_ROOT_DIR=${BATS_FILE_TMPDIR}/zot
     echo ${ZOT_ROOT_DIR}
-    local zot_log_file=${BATS_RUN_TMPDIR}/zot-log.json
-    local zot_config_file=${BATS_RUN_TMPDIR}/zot_config.json
+    local zot_log_file=${BATS_FILE_TMPDIR}/zot-log.json
+    local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json
     mkdir -p ${ZOT_ROOT_DIR}
     touch ${zot_log_file}
     cat >${zot_config_file} <<EOF
@@ -65,6 +69,11 @@ EOF
     echo ${MANIFEST_DIGEST}
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot-log.json
+}
+
 function teardown_file() {
     zot_stop_all
 }
diff --git a/test/blackbox/detect_manifest_collision.bats b/test/blackbox/detect_manifest_collision.bats
index 2a3d6314..26ef67d9 100644
--- a/test/blackbox/detect_manifest_collision.bats
+++ b/test/blackbox/detect_manifest_collision.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-detect-manifest-collision"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -64,7 +64,8 @@ function setup_file() {
         }
     },
     "log": {
-        "level": "debug"
+        "level": "debug",
+        "output": "${BATS_FILE_TMPDIR}/zot.log"
     }
 }
 EOF
@@ -72,6 +73,11 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
     zot_stop_all
 }
diff --git a/test/blackbox/garbage_collect.bats b/test/blackbox/garbage_collect.bats
index 73d5f2a4..61ca08d5 100644
--- a/test/blackbox/garbage_collect.bats
+++ b/test/blackbox/garbage_collect.bats
@@ -45,7 +45,7 @@ function setup_file() {
     },
     "log": {
         "level": "debug",
-        "output": "/tmp/gc.log"
+        "output": "${BATS_FILE_TMPDIR}/gc.log"
     }
 }
 EOF
@@ -53,6 +53,11 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/gc.log
+}
+
 function teardown_file() {
     zot_stop_all
 }
diff --git a/test/blackbox/metadata.bats b/test/blackbox/metadata.bats
index 2c451491..3f24a0c4 100644
--- a/test/blackbox/metadata.bats
+++ b/test/blackbox/metadata.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-metadata"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -77,7 +77,8 @@ function setup_file() {
         }
     },
     "log": {
-        "level": "debug"
+        "level": "debug",
+        "output": "${BATS_FILE_TMPDIR}/zot.log"
     }
 }
 EOF
@@ -86,11 +87,15 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
     zot_stop_all
 }
 
-
 @test "push image user policy" {
     run skopeo --insecure-policy copy --dest-creds test:test --dest-tls-verify=false \
         oci:${TEST_DATA_DIR}/golang:1.18 \
diff --git a/test/blackbox/metrics.bats b/test/blackbox/metrics.bats
index 7bc27cc7..df7825d0 100644
--- a/test/blackbox/metrics.bats
+++ b/test/blackbox/metrics.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-metrics" or "make test-bats-metrics-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -59,6 +59,11 @@ EOF
 
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
+}
+
 function teardown_file() {
     zot_stop_all
 }
diff --git a/test/blackbox/pushpull.bats b/test/blackbox/pushpull.bats
index 14671b95..73efadb3 100644
--- a/test/blackbox/pushpull.bats
+++ b/test/blackbox/pushpull.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-push-pull" or "test-push-pull-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -42,7 +42,8 @@ function setup_file() {
         "port": "8080"
     },
     "log": {
-        "level": "debug"
+        "level": "debug",
+        "output": "${BATS_FILE_TMPDIR}/zot.log"
     }
 }
 EOF
@@ -51,6 +52,11 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
     zot_stop_all
 }
diff --git a/test/blackbox/pushpull_authn.bats b/test/blackbox/pushpull_authn.bats
index ca08417e..93d36477 100644
--- a/test/blackbox/pushpull_authn.bats
+++ b/test/blackbox/pushpull_authn.bats
@@ -69,7 +69,8 @@ function setup_file() {
     }
   },
   "log":{
-    "level":"debug"
+    "level":"debug",
+    "output": "${BATS_FILE_TMPDIR}/zot.log"
   }
 }
 EOF
@@ -77,6 +78,11 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
     zot_stop_all
 }
diff --git a/test/blackbox/pushpull_running_dedupe.bats b/test/blackbox/pushpull_running_dedupe.bats
index 1c3ae0ad..b18163a3 100644
--- a/test/blackbox/pushpull_running_dedupe.bats
+++ b/test/blackbox/pushpull_running_dedupe.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-push-pull-running-dedupe" or "test-push-pull-running-dedupe-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-dedupe-nightly"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -45,7 +45,8 @@ function setup_file() {
         "port": "8080"
     },
     "log": {
-        "level": "debug"
+        "level": "debug",
+        "output": "${BATS_FILE_TMPDIR}/zot.log"
     }
 }
 EOF
@@ -54,6 +55,11 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot.log
+}
+
 function teardown_file() {
     zot_stop_all
 }
diff --git a/test/blackbox/referrers.bats b/test/blackbox/referrers.bats
index 6efdb382..005d3aff 100644
--- a/test/blackbox/referrers.bats
+++ b/test/blackbox/referrers.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-referrers"
+# Note: Intended to be run as "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -87,6 +87,11 @@ EOF
     curl -X GET http://127.0.0.1:8080/v2/golang/referrers/${MANIFEST_DIGEST}?artifactType=image.artifact/type
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
+}
+
 function teardown_file() {
     zot_stop_all
 }
diff --git a/test/blackbox/restore_s3_blobs.bats b/test/blackbox/restore_s3_blobs.bats
index 4b04f514..498aea50 100644
--- a/test/blackbox/restore_s3_blobs.bats
+++ b/test/blackbox/restore_s3_blobs.bats
@@ -1,3 +1,7 @@
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-dedupe-nightly"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
 load helpers_cloud
 load helpers_wait
 
@@ -81,6 +85,11 @@ EOF
     wait_zot_reachable 8080
 }
 
+function teardown() {
+    # conditionally printing on failure is possible from teardown but not from from teardown_file
+    cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
+}
+
 function teardown_file() {
     local zot_root_dir=${BATS_FILE_TMPDIR}/zot
     zot_stop
diff --git a/test/blackbox/scrub.bats b/test/blackbox/scrub.bats
index 079ca1cf..11bae823 100644
--- a/test/blackbox/scrub.bats
+++ b/test/blackbox/scrub.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-scrub" or "make test-bats-scrub-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
@@ -53,11 +53,10 @@ EOF
 }
 
 function teardown() {
+    cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
     zot_stop_all
 }
 
-
-
 @test "blobs/manifest integrity not affected" {
     add_test_files
     echo ${ZOT_CONFIG_FILE}
diff --git a/test/blackbox/sync.bats b/test/blackbox/sync.bats
index 85733618..8093b15c 100644
--- a/test/blackbox/sync.bats
+++ b/test/blackbox/sync.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-sync" or "make test-bats-sync-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
diff --git a/test/blackbox/sync_docker.bats b/test/blackbox/sync_docker.bats
index ebbd2b09..25b6e4a2 100644
--- a/test/blackbox/sync_docker.bats
+++ b/test/blackbox/sync_docker.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-sync" or "make test-bats-sync-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
diff --git a/test/blackbox/sync_harness.bats b/test/blackbox/sync_harness.bats
index 8856d2e9..d4f902a7 100644
--- a/test/blackbox/sync_harness.bats
+++ b/test/blackbox/sync_harness.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-sync-harness" or "make test-sync-harness-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-sync-nightly"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
 
diff --git a/test/blackbox/sync_replica_cluster.bats b/test/blackbox/sync_replica_cluster.bats
index 6a64f3d3..cc96596a 100644
--- a/test/blackbox/sync_replica_cluster.bats
+++ b/test/blackbox/sync_replica_cluster.bats
@@ -1,4 +1,4 @@
-# Note: Intended to be run as "make test-bats-sync" or "make test-bats-sync-verbose"
+# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
 #       Makefile target installs & checks all necessary tooling
 #       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()