fix: bats test refactoring (#1731)

Signed-off-by: Alexei Dodon <adodon@cisco.com>

.github/workflows/ci-cd.yml

@@ -13,14 +13,6 @@ name: build-test
 permissions: read-all
 
 jobs:
-  build-test:
-    name: Build and test ZOT
-    runs-on: ubuntu-latest
-    steps:
-      - name: Run build and test
-        timeout-minutes: 60
-        run: |
-            echo "job deprecated"
   build-test-arch:
     name: Build and test ZOT
     permissions:

.github/workflows/ecosystem-tools.yaml

@@ -92,6 +92,9 @@ jobs:
       - name: Run anonymous-push-pull tests
         run: |
             make test-anonymous-push-pull
+      - name: Run detect-manifest-collision tests
+        run: |
+            make test-detect-manifest-collision
       - name: Run annotations tests
         run: |
             make test-annotations

Makefile

@@ -350,109 +350,109 @@ $(BATS):
 	rm -rf bats-core
 
 .PHONY: test-push-pull
-test-push-pull: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
+test-push-pull: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
 	$(BATS) --trace --print-output-on-failure test/blackbox/pushpull.bats
 
 .PHONY: test-push-pull-verbose
-test-push-pull-verbose: binary check-skopeo $(BATS)
+test-push-pull-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
 	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull.bats
 
 .PHONY: test-push-pull-running-dedupe
-test-push-pull-running-dedupe: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
+test-push-pull-running-dedupe: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
 	$(BATS) --trace --print-output-on-failure test/blackbox/pushpull_running_dedupe.bats
 
 .PHONY: test-push-pull-running-dedupe-verbose
-test-push-pull-running-dedupe-verbose: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
+test-push-pull-running-dedupe-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
 	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull_running_dedupe.bats
 
 .PHONY: test-sync-harness
-test-sync-harness: binary binary-minimal bench check-skopeo $(BATS)
+test-sync-harness: check-linux binary binary-minimal bench check-skopeo $(BATS)
 	$(BATS) --trace --print-output-on-failure test/blackbox/sync_harness.bats
 
 .PHONY: test-sync-harness-verbose
-test-sync-harness-verbose: binary binary-minimal bench check-skopeo $(BATS)
+test-sync-harness-verbose: check-linux binary binary-minimal bench check-skopeo $(BATS)
 	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_harness.bats
 
 .PHONY: test-restore-s3-blobs
-test-restore-s3-blobs: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
+test-restore-s3-blobs: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
 	$(BATS) --trace --print-output-on-failure test/blackbox/restore_s3_blobs.bats
 
 .PHONY: test-restore-s3-blobs-verbose
-test-restore-s3-blobs-verbose: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
+test-restore-s3-blobs-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
 	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/restore_s3_blobs.bats
 
 .PHONY: test-bats-referrers
 test-bats-referrers: BUILD_LABELS=search
-test-bats-referrers: binary check-skopeo $(BATS) $(ORAS)
+test-bats-referrers: check-linux binary check-skopeo $(BATS) $(ORAS)
 	$(BATS) --trace --print-output-on-failure test/blackbox/referrers.bats
 
 .PHONY: test-bats-metadata
 test-bats-metadata: BUILD_LABELS=search,userprefs
-test-bats-metadata: binary check-skopeo $(BATS)
+test-bats-metadata: check-linux binary check-skopeo $(BATS)
 	$(BATS) --trace --print-output-on-failure test/blackbox/metadata.bats
 
 .PHONY: test-cloud-only
-test-cloud-only: binary check-skopeo $(BATS)
+test-cloud-only: check-linux binary check-skopeo $(BATS)
 	$(BATS) --trace --print-output-on-failure test/blackbox/cloud-only.bats
 
 .PHONY: test-cloud-only-verbose
-test-cloud-only-verbose: binary check-skopeo $(BATS)
+test-cloud-only-verbose: check-linux binary check-skopeo $(BATS)
 	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/cloud-only.bats
 
 .PHONY: test-bats-sync
 test-bats-sync: BUILD_LABELS=sync
-test-bats-sync: binary binary-minimal bench check-skopeo $(BATS) $(NOTATION) $(COSIGN)
+test-bats-sync: check-linux binary binary-minimal bench check-skopeo $(BATS) $(NOTATION) $(COSIGN) $(HELM)
 	$(BATS) --trace --print-output-on-failure test/blackbox/sync.bats
 	$(BATS) --trace --print-output-on-failure test/blackbox/sync_docker.bats
 	$(BATS) --trace --print-output-on-failure test/blackbox/sync_replica_cluster.bats
 
 .PHONY: test-bats-sync-verbose
 test-bats-sync-verbose: BUILD_LABELS=sync
-test-bats-sync-verbose: binary binary-minimal bench check-skopeo $(BATS) $(NOTATION) $(COSIGN)
+test-bats-sync-verbose: check-linux binary binary-minimal bench check-skopeo $(BATS) $(NOTATION) $(COSIGN) $(HELM)
 	$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync.bats
 	$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_docker.bats
 	$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_replica_cluster.bats
 
 .PHONY: test-bats-cve
 test-bats-cve: BUILD_LABELS=search
-test-bats-cve: binary cli check-skopeo $(BATS)
+test-bats-cve: check-linux binary cli check-skopeo $(BATS)
 	$(BATS) --trace --print-output-on-failure test/blackbox/cve.bats
 
 .PHONY: test-bats-cve-verbose
 test-bats-cve-verbose: BUILD_LABELS=search
-test-bats-cve-verbose: binary cli check-skopeo $(BATS)
+test-bats-cve-verbose: check-linux binary cli check-skopeo $(BATS)
 	$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/cve.bats
 
 .PHONY: test-bats-scrub
 test-bats-scrub: BUILD_LABELS=scrub
-test-bats-scrub: binary check-skopeo $(BATS)
+test-bats-scrub: check-linux binary check-skopeo $(BATS)
 	$(BATS) --trace --print-output-on-failure test/blackbox/scrub.bats
 
 .PHONY: test-bats-scrub-verbose
 test-bats-scrub-verbose: BUILD_LABELS=scrub
-test-bats-scrub-verbose: binary check-skopeo $(BATS)
+test-bats-scrub-verbose: check-linux binary check-skopeo $(BATS)
 	$(BATS) --trace -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/scrub.bats
 
 .PHONY: test-bats-metrics
 test-bats-metrics: BUILD_LABELS=metrics
-test-bats-metrics: binary check-skopeo $(BATS)
+test-bats-metrics: check-linux binary check-skopeo $(BATS)
 	$(BATS) --trace --print-output-on-failure test/blackbox/metrics.bats
 
 .PHONY: test-bats-metrics-verbose
 test-bats-metrics-verbose: BUILD_LABELS=metrics
-test-bats-metrics-verbose: binary check-skopeo $(BATS)
+test-bats-metrics-verbose: check-linux binary check-skopeo $(BATS)
 	$(BATS) --trace -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/metrics.bats
 
 .PHONY: test-anonymous-push-pull
-test-anonymous-push-pull: binary check-skopeo $(BATS)
+test-anonymous-push-pull: check-linux binary check-skopeo $(BATS)
 	$(BATS) --trace --print-output-on-failure test/blackbox/anonymous_policy.bats
 
 .PHONY: test-annotations
-test-annotations: binary check-skopeo $(BATS) $(STACKER) $(NOTATION) $(COSIGN)
+test-annotations: check-linux binary check-skopeo $(BATS) $(STACKER) $(NOTATION) $(COSIGN)
 	$(BATS) --trace --print-output-on-failure test/blackbox/annotations.bats
 
 .PHONY: test-detect-manifest-collision
-test-detect-manifest-collision: binary check-skopeo $(BATS)
+test-detect-manifest-collision: check-linux binary check-skopeo $(BATS) $(REGCLIENT)
 	$(BATS) --trace --print-output-on-failure test/blackbox/detect_manifest_collision.bats
 
 .PHONY: fuzz-all
@@ -512,3 +512,8 @@ ui:
 		fi;\
 	fi;\
 
+.PHONY: check-linux
+check-linux:
+ifneq ($(shell go env GOOS),linux)
+	$(error makefile target can be run only on linux)
+endif

test/blackbox/annotations.bats

@@ -1,8 +1,32 @@
-load helpers_pushpull
+# Note: Intended to be run as "make test-annotations"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
+load helpers_zot
+
+function verify_prerequisites {
+    if [ ! $(command -v curl) ]; then
+        echo "you need to install curl as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    if [ ! $(command -v jq) ]; then
+        echo "you need to install jq as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    if [ ! $(command -v podman) ]; then
+        echo "you need to install podman as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    return 0
+}
 
 function setup_file() {
+    export COSIGN_PASSWORD=""
     # Verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
         exit 1
     fi
     # Download test data to folder common for the entire suite, not just this file
@@ -10,12 +34,10 @@ function setup_file() {
     # Setup zot server
     local zot_root_dir=${BATS_FILE_TMPDIR}/zot
     local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json
-    local oci_data_dir=${BATS_FILE_TMPDIR}/oci
     mkdir -p ${zot_root_dir}
-    mkdir -p ${oci_data_dir}
     cat > ${zot_config_file}<<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_root_dir}"
     },
@@ -53,22 +75,14 @@ EOF
 FROM public.ecr.aws/t0x7q1g8/centos:7
 CMD ["/bin/sh", "-c", "echo 'It works!'"]
 EOF
-    setup_zot_file_level ${zot_config_file}
+    zot_serve ${ZOT_PATH} ${zot_config_file}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
+    wait_zot_reachable 8080
 }
 
 function teardown_file() {
     cat ${BATS_FILE_TMPDIR}/zot.log >&3
-    local zot_root_dir=${BATS_FILE_TMPDIR}/zot
+    zot_stop_all
-    local oci_data_dir=${BATS_FILE_TMPDIR}/oci
+    run rm -rf ${HOME}/.config/notation
-    local roots_data_dir=${BATS_FILE_TMPDIR}/roots
-    local stacker_data_dir=${BATS_FILE_TMPDIR}/.stacker
-    local stackeroci_data_dir=${BATS_FILE_TMPDIR}/stackeroci
-    teardown_zot_file_level
-    rm -rf ${zot_root_dir}
-    rm -rf ${oci_data_dir}
-    rm -rf ${stackeroci_data_dir}
-    rm -rf ${roots_data_dir}    
 }
 
 @test "build image with podman and specify annotations" {
@@ -77,9 +91,8 @@ function teardown_file() {
     run podman push 127.0.0.1:8080/annotations:latest --tls-verify=false --format=oci
     [ "$status" -eq 0 ]
     run curl -X POST -H "Content-Type: application/json" --data '{ "query": "{ ImageList(repo: \"annotations\") { Results { RepoName Tag Manifests {Digest ConfigDigest Size Layers { Size Digest }} Vendor Licenses }}}"}' http://localhost:8080/v2/_zot/ext/search
-   
     [ "$status" -eq 0 ]
-    # [ $(echo "${lines[-1]}" | jq '.data.ImageList') ]
+
     [ $(echo "${lines[-1]}" | jq '.data.ImageList.Results[0].RepoName') = '"annotations"' ]
     [ $(echo "${lines[-1]}" | jq '.data.ImageList.Results[0].Vendor') = '"CentOS"' ]
     [ $(echo "${lines[-1]}" | jq '.data.ImageList.Results[0].Licenses') = '"GPLv2"' ]
@@ -103,17 +116,16 @@ function teardown_file() {
     [ "$status" -eq 0 ]
     [ $(echo "${lines[-1]}" | jq '.data.ImageList.Results[0].RepoName') = '"annotations"' ]
     local digest=$(echo "${lines[-1]}" | jq -r '.data.ImageList.Results[0].Manifests[0].Digest')
-    
+
     run cosign initialize
     [ "$status" -eq 0 ]
-    run cosign generate-key-pair --output-key-prefix "cosign-sign-test"
+    run cosign generate-key-pair --output-key-prefix "${BATS_FILE_TMPDIR}/cosign-sign-test"
     [ "$status" -eq 0 ]
-    run cosign sign --key cosign-sign-test.key localhost:8080/annotations:latest --yes
+    run cosign sign --key ${BATS_FILE_TMPDIR}/cosign-sign-test.key localhost:8080/annotations:latest --yes
     [ "$status" -eq 0 ]
-    run cosign verify --key cosign-sign-test.pub localhost:8080/annotations:latest
+    run cosign verify --key ${BATS_FILE_TMPDIR}/cosign-sign-test.pub localhost:8080/annotations:latest
     [ "$status" -eq 0 ]
     local sigName=$(echo "${lines[-1]}" | jq '.[].critical.image."docker-manifest-digest"')
-    [ "$status" -eq 0 ]
     [[ "$sigName" == *"${digest}"* ]]
 }
 
@@ -136,7 +148,7 @@ function teardown_file() {
             "name": "notation-sign-test",
             "registryScopes": [ "*" ],
             "signatureVerification": {
-                "level" : "strict" 
+                "level" : "strict"
             },
             "trustStores": [ "ca:notation-sign-test" ],
             "trustedIdentities": [

test/blackbox/anonymous_policy.bats

@@ -1,8 +1,16 @@
-load helpers_pushpull
+# Note: Intended to be run as "make test-anonymous-push-pull"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
+load helpers_zot
+
+function verify_prerequisites {
+    return 0
+}
 
 function setup_file() {
     # Verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
         exit 1
     fi
 
@@ -18,7 +26,7 @@ function setup_file() {
     echo 'test:$2a$10$EIIoeCnvsIDAJeDL4T1sEOnL2fWOvsq7ACZbs3RT40BBBXg.Ih7V.' >> ${htpasswordFile}
     cat > ${zot_config_file}<<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_root_dir}"
     },
@@ -55,16 +63,12 @@ function setup_file() {
     }
 }
 EOF
-    setup_zot_file_level ${zot_config_file}
+    zot_serve ${ZOT_PATH} ${zot_config_file}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"   
+    wait_zot_reachable 8080
 }
 
 function teardown_file() {
-    local zot_root_dir=${BATS_FILE_TMPDIR}/zot
+    zot_stop_all
-    local oci_data_dir=${BATS_FILE_TMPDIR}/oci
-    teardown_zot_file_level
-    rm -rf ${zot_root_dir}
-    rm -rf ${oci_data_dir}
 }
 
 @test "push image user policy" {

test/blackbox/cloud-only.bats

@@ -1,15 +1,16 @@
 load helpers_cloud
+load helpers_wait
 
 function setup() {
     # Verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
         exit 1
     fi
 
     # Setup zot server
     local zot_root_dir=${BATS_FILE_TMPDIR}/zot
     local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json
-    
+
     echo ${zot_root_dir} >&3
 
     mkdir -p ${zot_root_dir}
@@ -91,7 +92,7 @@ EOF
     awslocal s3 --region "us-east-2" mb s3://zot-storage
     awslocal dynamodb --region "us-east-2" create-table --table-name "BlobTable" --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
     zot_serve_strace ${zot_config_file}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
+    wait_zot_reachable 8080
 }
 
 function teardown() {

test/blackbox/cve.bats

@@ -1,9 +1,27 @@
-load helpers_cve
+# Note: Intended to be run as "make test-bats-cve" or "make test-bats-cve-verbose"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
+load helpers_zot
+
+function verify_prerequisites {
+    if [ ! $(command -v curl) ]; then
+        echo "you need to install curl as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    if [ ! $(command -v jq) ]; then
+        echo "you need to install jq as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    return 0
+}
 
 function setup_file() {
-
+    export REGISTRY_NAME=main
     # Verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
         exit 1
     fi
 
@@ -15,7 +33,7 @@ function setup_file() {
     mkdir -p ${zot_root_dir}
     cat >${zot_config_file} <<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_root_dir}"
     },
@@ -36,8 +54,8 @@ function setup_file() {
     }
 }
 EOF
-    setup_zot_file_level ${zot_config_file}
+    zot_serve ${ZOT_PATH} ${zot_config_file}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
+    wait_zot_reachable 8080
 
     # setup zli to add zot registry to configs
     local registry_name=main
@@ -46,9 +64,7 @@ EOF
 }
 
 function teardown_file() {
-    local zot_root_dir=${BATS_FILE_TMPDIR}/zot
+    zot_stop_all
-    teardown_zot_file_level
-    rm -rf ${zot_root_dir}
 }
 
 @test "cve by image name and tag" {

test/blackbox/detect_manifest_collision.bats

@@ -1,8 +1,16 @@
-load helpers_pushpull
+# Note: Intended to be run as "make test-detect-manifest-collision"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
+load helpers_zot
+
+function verify_prerequisites {
+    return 0
+}
 
 function setup_file() {
     # Verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
         exit 1
     fi
 
@@ -18,7 +26,7 @@ function setup_file() {
     echo 'test:$2a$10$EIIoeCnvsIDAJeDL4T1sEOnL2fWOvsq7ACZbs3RT40BBBXg.Ih7V.' >> ${htpasswordFile}
     cat > ${zot_config_file}<<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_root_dir}"
     },
@@ -33,7 +41,7 @@ function setup_file() {
         "accessControl": {
             "repositories": {
                 "**": {
-                    "anonymousPolicy": [ 
+                    "anonymousPolicy": [
                         "read",
                         "create",
                         "delete",
@@ -60,16 +68,12 @@ function setup_file() {
     }
 }
 EOF
-    setup_zot_file_level ${zot_config_file}
+    zot_serve ${ZOT_PATH} ${zot_config_file}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"   
+    wait_zot_reachable 8080
 }
 
 function teardown_file() {
-    local zot_root_dir=${BATS_FILE_TMPDIR}/zot
+    zot_stop_all
-    local oci_data_dir=${BATS_FILE_TMPDIR}/oci
-    teardown_zot_file_level
-    rm -rf ${zot_root_dir}
-    rm -rf ${oci_data_dir}
 }
 
 @test "push 2 images with same manifest with user policy" {
@@ -90,7 +94,7 @@ function teardown_file() {
         docker://127.0.0.1:8080/golang:1.20
     [ "$status" -eq 1 ]
     # conflict status code
-    [[ "$output" == *"409"* ]]
+    [[ "$output" == *"manifest invalid"* ]]
 }
 
 @test "regctl delete image with anonymous policy should fail" {

test/blackbox/helpers_cloud.bash

@@ -12,12 +12,12 @@ function verify_prerequisites {
         return 1
     fi
 
-    if [ ! command -v skopeo &> /dev/null ]; then
+    if [ ! $(command -v skopeo) ]; then
         echo "you need to install skopeo as a prerequisite to running the tests" >&3
         return 1
     fi
 
-    if [ ! command -v awslocal ] &>/dev/null; then
+    if [ ! $(command -v awslocal) ] &>/dev/null; then
         echo "you need to install aws cli as a prerequisite to running the tests" >&3
         return 1
     fi
@@ -39,37 +39,9 @@ function zot_stop() {
     pkill zot
 }
 
-function wait_str() {
-    local filepath="$1"
-    local search_term="$2"
-    local wait_time="${3:-2m}"
-
-    (timeout $wait_time tail -F -n0 "$filepath" &) | grep -q "$search_term" && return 0
-
-    echo "timeout of $wait_time reached. unable to find '$search_term' in '$filepath'"
-    
-    return 1
-}
-
-function wait_for_string() {
-    local search_term="$1"
-    local filepath="$2"
-    local wait_time="${3:-2m}"
-
-    wait_file "$filepath" 60 || { echo "server log file missing: '$filepath'"; return 1; }
-
-    wait_str "$filepath" "$search_term" "$wait_time"
-}
-
-function wait_file() {
-    local file="$1"; shift
-    local wait_seconds="${1:-60}"; shift
-
-    until test $((wait_seconds--)) -eq 0 -o -f "$file" ; do sleep 1; done
-}
-
 function wait_zot_reachable() {
-    zot_url=${1}
+    local zot_port=${1}
+    local zot_url=http://127.0.0.1:${zot_port}/v2/_catalog
     curl --connect-timeout 3 \
         --max-time 10 \
         --retry 10 \

test/blackbox/helpers_cve.bash

@@ -1,84 +0,0 @@
-ROOT_DIR=$(git rev-parse --show-toplevel)
-TEST_DATA_DIR=${ROOT_DIR}/test/data/
-OS="${OS:-linux}"
-ARCH="${ARCH:-amd64}"
-ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}
-ZLI_PATH=${ROOT_DIR}/bin/zli-${OS}-${ARCH}
-REGISTRY_NAME=main
-
-function verify_prerequisites {
-    if [ ! -f ${BATS_RUN_TMPDIR}/.firstrun ]; then
-        env | grep proxy >&3
-        touch ${BATS_RUN_TMPDIR}/.firstrun
-    fi
-
-    if [ ! -f ${ZOT_PATH} ]; then
-        echo "you need to build ${ZOT_PATH} before running the tests" >&3
-        return 1
-    fi
-
-    if [ ! -f ${ZLI_PATH} ]; then
-        echo "you need to build ${ZLI} before running tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v curl ] &>/dev/null; then
-        echo "you need to install curl as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v jq ] &>/dev/null; then
-        echo "you need to install jq as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v skopeo ] &>/dev/null; then
-        echo "you need to install skopeo as a prerequisite to running the tests" >&3
-        return 1
-    fi
-    return 0
-}
-
-function zot_serve() {
-    local zot_path=${1}
-    local config_file=${2}
-    local pid_dir=${3}
-    ${zot_path} serve ${config_file} &
-    echo $! >>${pid_dir}/zot.pid
-}
-
-function zot_stop() {
-    local pid_dir=${1}
-    cat ${pid_dir}/zot.pid
-    kill $(cat ${pid_dir}/zot.pid)
-    rm ${pid_dir}/zot.pid
-}
-
-function setup_zot_file_level() {
-    local config_file=${1}
-    zot_serve ${ZOT_PATH} ${config_file} ${BATS_FILE_TMPDIR}
-}
-
-function teardown_zot_file_level() {
-    zot_stop ${BATS_FILE_TMPDIR}
-}
-
-function wait_zot_reachable() {
-    zot_url=${1}
-    curl --connect-timeout 3 \
-        --max-time 3 \
-        --retry 10 \
-        --retry-delay 0 \
-        --retry-max-time 60 \
-        --retry-connrefused \
-        ${zot_url}
-}
-
-function zli_add_config() {
-    local registry_name=${1}
-    local registry_url=${2}
-    if ! ${ZLI_PATH} config --list | grep -q main; then
-        ${ZLI_PATH} config add ${registry_name} ${registry_url}
-    fi
-
-}

test/blackbox/helpers_metrics.bash

@@ -1,63 +0,0 @@
-ROOT_DIR=$(git rev-parse --show-toplevel)
-TEST_DATA_DIR=${ROOT_DIR}/test/data/
-OS="${OS:-linux}"
-ARCH="${ARCH:-amd64}"
-ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}
-
-function verify_prerequisites() {
-    if [ ! -f ${BATS_RUN_TMPDIR}/.firstrun ]; then
-        env | grep proxy >&3
-        touch ${BATS_RUN_TMPDIR}/.firstrun
-    fi
-
-    if [ ! command -v curl ] &>/dev/null; then
-        echo "you need to install curl as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v jq ] &>/dev/null; then
-        echo "you need to install jq as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v skopeo ] &>/dev/null; then
-        echo "you need to install skopeo as a prerequisite to running the tests" >&3
-        return 1
-    fi
-    return 0
-}
-
-function zot_serve() {
-    local zot_path=${1}
-    local config_file=${2}
-    local pid_dir=${3}
-    ${zot_path} serve ${config_file} &
-    echo $! >>${pid_dir}/zot.pid
-}
-
-function zot_stop() {
-    local pid_dir=${1}
-    cat ${pid_dir}/zot.pid
-    kill $(cat ${pid_dir}/zot.pid)
-    rm ${pid_dir}/zot.pid
-}
-
-function setup_zot_file_level() {
-    local config_file=${1}
-    zot_serve ${ZOT_PATH} ${config_file} ${BATS_FILE_TMPDIR}
-}
-
-function teardown_zot_file_level() {
-    zot_stop ${BATS_FILE_TMPDIR}
-}
-
-function wait_zot_reachable() {
-    zot_url=${1}
-    curl --connect-timeout 3 \
-        --max-time 3 \
-        --retry 10 \
-        --retry-delay 0 \
-        --retry-max-time 60 \
-        --retry-connrefused \
-        ${zot_url}
-}

test/blackbox/helpers_pushpull.bash

@@ -1,84 +0,0 @@
-ROOT_DIR=$(git rev-parse --show-toplevel)
-TEST_DATA_DIR=${ROOT_DIR}/test/data/
-OS="${OS:-linux}"
-ARCH="${ARCH:-amd64}"
-ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}
-
-mkdir -p ${TEST_DATA_DIR}
-
-function verify_prerequisites {
-    if [ ! -f ${BATS_RUN_TMPDIR}/.firstrun ]; then
-        env | grep proxy >&3
-        touch ${BATS_RUN_TMPDIR}/.firstrun
-    fi
-
-    if [ ! -f ${ZOT_PATH} ]; then
-        echo "you need to build ${ZOT_PATH} before running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v curl &> /dev/null ]; then
-        echo "you need to install curl as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v jq ] &>/dev/null; then
-        echo "you need to install jq as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v skopeo ] &>/dev/null; then
-        echo "you need to install skopeo as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v oras ] &>/dev/null; then
-        echo "you need to install oras as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v helm ] &>/dev/null; then
-        echo "you need to install helm as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v regctl ] &>/dev/null; then
-        echo "you need to install regclient as a prerequisite to running the tests" >&3
-        return 1
-    fi
-    return 0
-}
-
-function zot_serve() {
-    local zot_path=${1}
-    local config_file=${2}
-    local pid_dir=${3}
-    ${zot_path} serve ${config_file} &
-    echo $! > ${pid_dir}/zot.pid
-}
-
-function zot_stop() {
-    local pid_dir=${1}
-    kill $(cat ${pid_dir}/zot.pid)
-    rm ${pid_dir}/zot.pid
-}
-
-function setup_zot_file_level() {
-    local config_file=${1}
-    zot_serve ${ZOT_PATH} ${config_file} ${BATS_FILE_TMPDIR}
-}
-
-function teardown_zot_file_level() {
-    zot_stop ${BATS_FILE_TMPDIR}
-}
-
-function wait_zot_reachable() {
-    zot_url=${1}
-    curl --connect-timeout 3 \
-        --max-time 3 \
-        --retry 10 \
-        --retry-delay 0 \
-        --retry-max-time 60 \
-        --retry-connrefused \
-        ${zot_url}
-}

test/blackbox/helpers_referrers.bash

@@ -1,80 +0,0 @@
-ROOT_DIR=$(git rev-parse --show-toplevel)
-TEST_DATA_DIR=${ROOT_DIR}/test/data/
-OS="${OS:-linux}"
-ARCH="${ARCH:-amd64}"
-ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}
-
-mkdir -p ${TEST_DATA_DIR}
-
-function verify_prerequisites {
-    if [ ! -f ${BATS_RUN_TMPDIR}/.firstrun ]; then
-        env | grep proxy >&3
-        touch ${BATS_RUN_TMPDIR}/.firstrun
-    fi
-
-    if [ ! -f ${ZOT_PATH} ]; then
-        echo "you need to build ${ZOT_PATH} before running the tests" >&3
-        return 1
-    fi
-
-    if [ ! -f ${ZOT_MINIMAL_PATH} ]; then
-        echo "you need to build ${ZOT_MINIMAL_PATH} before running tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v curl ] &>/dev/null; then
-        echo "you need to install curl as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v jq ] &>/dev/null; then
-        echo "you need to install jq as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v skopeo ] &>/dev/null; then
-        echo "you need to install skopeo as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v oras ] &>/dev/null; then
-        echo "you need to install oras as a prerequisite to running the tests" >&3
-        return 1
-    fi
-    return 0
-}
-
-function zot_serve() {
-    local zot_path=${1}
-    local config_file=${2}
-    local pid_dir=${3}
-    ${zot_path} serve ${config_file} &
-    echo $! >>${pid_dir}/zot.pid
-}
-
-function zot_stop() {
-    local pid_dir=${1}
-    cat ${pid_dir}/zot.pid
-    kill $(cat ${pid_dir}/zot.pid)
-    rm ${pid_dir}/zot.pid
-}
-
-function setup_zot_file_level() {
-    local config_file=${1}
-    zot_serve ${ZOT_PATH} ${config_file} ${BATS_FILE_TMPDIR}
-}
-
-function teardown_zot_file_level() {
-    zot_stop ${BATS_FILE_TMPDIR}
-}
-
-function wait_zot_reachable() {
-    zot_url=${1}
-    curl --connect-timeout 3 \
-        --max-time 3 \
-        --retry 10 \
-        --retry-delay 0 \
-        --retry-max-time 60 \
-        --retry-connrefused \
-        ${zot_url}
-}

test/blackbox/helpers_scrub.bash

@@ -1,79 +1,7 @@
-ROOT_DIR=$(git rev-parse --show-toplevel)
-TEST_DATA_DIR=${ROOT_DIR}/test/data/
-OS="${OS:-linux}"
-ARCH="${ARCH:-amd64}"
-ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}
-ZOT_ROOT_DIR=?
-ZOT_LOG_FILE=?
-ZOT_CONFIG_FILE=
-
-function verify_prerequisites() {
-    if [ ! -f ${BATS_RUN_TMPDIR}/.firstrun ]; then
-        env | grep proxy >&3
-        touch ${BATS_RUN_TMPDIR}/.firstrun
-    fi
-
-    if [ ! -f ${ZOT_PATH} ]; then
-        echo "you need to build ${ZOT_PATH} before running tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v curl ] &>/dev/null; then
-        echo "you need to install curl as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v jq ] &>/dev/null; then
-        echo "you need to install jq as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v skopeo ] &>/dev/null; then
-        echo "you need to install skopeo as a prerequisite to running the tests" >&3
-        return 1
-    fi
-    return 0
-}
-
-function zot_serve() {
-    local zot_path=${1}
-    local config_file=${2}
-    local pid_dir=${3}
-    ${zot_path} serve ${config_file} &
-    echo $! >>${pid_dir}/zot.pid
-}
-
-function zot_stop() {
-    local pid_dir=${1}
-    cat ${pid_dir}/zot.pid
-    kill $(cat ${pid_dir}/zot.pid)
-    rm ${pid_dir}/zot.pid
-}
-
-function setup_zot_file_level() {
-    local config_file=${1}
-    zot_serve ${ZOT_PATH} ${config_file} ${BATS_FILE_TMPDIR}
-}
-
-function teardown_zot_file_level() {
-    zot_stop ${BATS_FILE_TMPDIR}
-}
-
-function wait_zot_reachable() {
-    zot_url=${1}
-    curl --connect-timeout 3 \
-        --max-time 3 \
-        --retry 10 \
-        --retry-delay 0 \
-        --retry-max-time 60 \
-        --retry-connrefused \
-        ${zot_url}
-}
-
 function add_test_files() {
     local zot_root_dir=${BATS_FILE_TMPDIR}/zot
     echo ${zot_root_dir}
-    cp -r ${TEST_DATA_DIR}golang ${zot_root_dir}
+    cp -r ${TEST_DATA_DIR}/golang ${zot_root_dir}
     ls -al ${zot_root_dir}/golang
 }
 

test/blackbox/helpers_sync.bash

@@ -1,129 +0,0 @@
-ROOT_DIR=$(git rev-parse --show-toplevel)
-TEST_DATA_DIR=${ROOT_DIR}/test/data/
-OS="${OS:-linux}"
-ARCH="${ARCH:-amd64}"
-ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}
-
-ZOT_MINIMAL_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}-minimal
-ZB_PATH=${ROOT_DIR}/bin/zb-${OS}-${ARCH}
-
-mkdir -p ${TEST_DATA_DIR}
-
-function verify_prerequisites {
-    if [ ! -f ${BATS_RUN_TMPDIR}/.firstrun ]; then
-        env | grep proxy >&3
-        touch ${BATS_RUN_TMPDIR}/.firstrun
-    fi
-
-    if [ ! -f ${ZOT_PATH} ]; then
-        echo "you need to build ${ZOT_PATH} before running the tests" >&3
-        return 1
-    fi
-
-    if [ ! -f ${ZB_PATH} ]; then
-        echo "you need to build ${ZB_PATH} before running the tests" >&3
-        return 1
-    fi
-
-    if [ ! -f ${ZOT_MINIMAL_PATH} ]; then
-        echo "you need to build ${ZOT_MINIMAL_PATH} before running tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v curl ] &>/dev/null; then
-        echo "you need to install curl as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v jq ] &>/dev/null; then
-        echo "you need to install jq as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! command -v skopeo ] &>/dev/null; then
-        echo "you need to install skopeo as a prerequisite to running the tests" >&3
-        return 1
-    fi
-    return 0
-}
-
-function zot_serve() {
-    local zot_path=${1}
-    local config_file=${2}
-    local pid_dir=${3}
-    ${zot_path} serve ${config_file} &
-
-    echo $! >>${pid_dir}/zot.pid
-}
-
-function zb_run() {
-    local zot_address=${1}
-    ${ZB_PATH} -c 10 -n 30 -o stdout ${zot_address} --skip-cleanup
-}
-
-function wait_str() {
-    local filepath="$1"
-    local search_term="$2"
-    local wait_time="${3:-2m}"
-
-    (timeout $wait_time tail -F -n0 "$filepath" &) | grep -q "$search_term" && return 0
-
-    echo "timeout of $wait_time reached. unable to find '$search_term' in '$filepath'"
-    
-    return 1
-}
-
-function wait_for_string() {
-    local search_term="$1"
-    local filepath="$2"
-    local wait_time="${3:-2m}"
-
-    wait_file "$filepath" 60 || { echo "server log file missing: '$filepath'"; return 1; }
-
-    wait_str "$filepath" "$search_term" "$wait_time"
-}
-
-function wait_file() {
-    local file="$1"; shift
-    local wait_seconds="${1:-60}"; shift
-
-    until test $((wait_seconds--)) -eq 0 -o -f "$file" ; do sleep 1; done
-}
-
-function zot_stop() {
-    local pid_dir=${1}
-    cat ${pid_dir}/zot.pid
-    kill $(cat ${pid_dir}/zot.pid)
-    rm ${pid_dir}/zot.pid
-}
-
-function zot_minimal_stop() {
-    local pid_dir=${1}
-    kill $(cat ${pid_dir}/zot-minimal.pid)
-    rm ${pid_dir}/zot-minimal.pid
-}
-
-function setup_zot_file_level() {
-    local config_file=${1}
-    zot_serve ${ZOT_PATH} ${config_file} ${BATS_FILE_TMPDIR}
-}
-
-function setup_zot_minimal_file_level() {
-    local config_file=${1}
-    zot_serve ${ZOT_MINIMAL_PATH} ${config_file} ${BATS_FILE_TMPDIR}
-}
-
-function teardown_zot_file_level() {
-    zot_stop ${BATS_FILE_TMPDIR}
-}
-
-function wait_zot_reachable() {
-    zot_url=${1}
-    curl --connect-timeout 3 \
-        --max-time 3 \
-        --retry 10 \
-        --retry-delay 0 \
-        --retry-max-time 60 \
-        --retry-connrefused \
-        ${zot_url}
-}

test/blackbox/helpers_wait.bash

@@ -0,0 +1,28 @@
+function wait_for_string() {
+    local search_term="$1"
+    local filepath="$2"
+    local wait_time="${3:-2m}"
+
+    wait_file "$filepath" 60 || { echo "server log file missing: '$filepath'"; return 1; }
+
+    wait_str "$filepath" "$search_term" "$wait_time"
+}
+
+function wait_str() {
+    local filepath="$1"
+    local search_term="$2"
+    local wait_time="${3:-2m}"
+
+    (timeout $wait_time tail -F -n0 "$filepath" &) | grep -q "$search_term" && return 0
+
+    echo "timeout of $wait_time reached. unable to find '$search_term' in '$filepath'"
+
+    return 1
+}
+
+function wait_file() {
+    local file="$1"; shift
+    local wait_seconds="${1:-60}"; shift
+
+    until test $((wait_seconds--)) -eq 0 -o -f "$file" ; do sleep 1; done
+}

test/blackbox/helpers_zot.bash

@@ -0,0 +1,48 @@
+ROOT_DIR=$(git rev-parse --show-toplevel)
+TEST_DATA_DIR=${ROOT_DIR}/test/data/
+OS=$(go env GOOS)
+ARCH=$(go env GOARCH)
+ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}
+ZLI_PATH=${ROOT_DIR}/bin/zli-${OS}-${ARCH}
+ZOT_MINIMAL_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}-minimal
+ZB_PATH=${ROOT_DIR}/bin/zb-${OS}-${ARCH}
+
+mkdir -p ${TEST_DATA_DIR}
+
+function zot_serve() {
+    local zot_path=${1}
+    local config_file=${2}
+    ${zot_path} serve ${config_file} &
+    # zot.pid file keeps a list of zot server PIDs (in case multiple zot servers are started)
+    echo -n "$! " >> ${BATS_FILE_TMPDIR}/zot.pid
+}
+
+# stops all zot instances started by the test
+function zot_stop_all() {
+    kill $(cat ${BATS_FILE_TMPDIR}/zot.pid)
+}
+
+function wait_zot_reachable() {
+    local zot_port=${1}
+    local zot_url=http://127.0.0.1:${zot_port}/v2/_catalog
+    curl --connect-timeout 3 \
+        --max-time 3 \
+        --retry 10 \
+        --retry-delay 0 \
+        --retry-max-time 60 \
+        --retry-connrefused \
+        ${zot_url}
+}
+
+function zli_add_config() {
+    local registry_name=${1}
+    local registry_url=${2}
+    if ! ${ZLI_PATH} config --list | grep -q main; then
+        ${ZLI_PATH} config add ${registry_name} ${registry_url}
+    fi
+}
+
+function zb_run() {
+    local zot_address=${1}
+    ${ZB_PATH} -c 10 -n 30 -o stdout ${zot_address} --skip-cleanup
+}

test/blackbox/metadata.bats

@@ -1,8 +1,26 @@
-load helpers_pushpull
+# Note: Intended to be run as "make test-bats-metadata"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
+load helpers_zot
+
+function verify_prerequisites {
+    if [ ! $(command -v curl) ]; then
+        echo "you need to install curl as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    if [ ! $(command -v jq) ]; then
+        echo "you need to install jq as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    return 0
+}
 
 function setup_file() {
     # Verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
         exit 1
     fi
 
@@ -18,7 +36,7 @@ function setup_file() {
     echo 'test:$2a$10$EIIoeCnvsIDAJeDL4T1sEOnL2fWOvsq7ACZbs3RT40BBBXg.Ih7V.' >> ${htpasswordFile}
     cat > ${zot_config_file}<<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_root_dir}"
     },
@@ -64,16 +82,12 @@ function setup_file() {
 }
 EOF
     git -C ${BATS_FILE_TMPDIR} clone https://github.com/project-zot/helm-charts.git
-    setup_zot_file_level ${zot_config_file}
+    zot_serve ${ZOT_PATH} ${zot_config_file}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
+    wait_zot_reachable 8080
 }
 
 function teardown_file() {
-    local zot_root_dir=${BATS_FILE_TMPDIR}/zot
+    zot_stop_all
-    local oci_data_dir=${BATS_FILE_TMPDIR}/oci
-    teardown_zot_file_level
-    rm -rf ${zot_root_dir}
-    rm -rf ${oci_data_dir}
 }
 
 

test/blackbox/metrics.bats

@@ -1,9 +1,21 @@
-load helpers_metrics
+# Note: Intended to be run as "make test-bats-metrics" or "make test-bats-metrics-verbose"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
+load helpers_zot
+
+function verify_prerequisites() {
+    if [ ! $(command -v curl) ]; then
+        echo "you need to install curl as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    return 0
+}
 
 function setup_file() {
     # verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
-        echo "oh noooooo"
         exit 1
     fi
 
@@ -19,7 +31,7 @@ function setup_file() {
     touch ${zot_log_file}
     cat >${zot_config_file} <<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_root_dir}"
     },
@@ -42,15 +54,13 @@ function setup_file() {
 }
 EOF
 
-    setup_zot_file_level ${zot_config_file}
+    zot_serve ${ZOT_PATH} ${zot_config_file}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
+    wait_zot_reachable 8080
 
 }
 
 function teardown_file() {
-    local zot_root_dir=${BATS_FILE_TMPDIR}/zot
+    zot_stop_all
-    teardown_zot_file_level
-    rm -rf ${zot_root_dir}
 }
 
 @test "metric enabled" {

test/blackbox/pushpull.bats

@@ -1,8 +1,26 @@
-load helpers_pushpull
+# Note: Intended to be run as "make test-push-pull" or "test-push-pull-verbose"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
+load helpers_zot
+
+function verify_prerequisites {
+    if [ ! $(command -v curl) ]; then
+        echo "you need to install curl as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    if [ ! $(command -v jq) ]; then
+        echo "you need to install jq as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    return 0
+}
 
 function setup_file() {
     # Verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
         exit 1
     fi
     # Download test data to folder common for the entire suite, not just this file
@@ -15,7 +33,7 @@ function setup_file() {
     mkdir -p ${oci_data_dir}
     cat > ${zot_config_file}<<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_root_dir}"
     },
@@ -29,16 +47,12 @@ function setup_file() {
 }
 EOF
     git -C ${BATS_FILE_TMPDIR} clone https://github.com/project-zot/helm-charts.git
-    setup_zot_file_level ${zot_config_file}
+    zot_serve ${ZOT_PATH} ${zot_config_file}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
+    wait_zot_reachable 8080
 }
 
 function teardown_file() {
-    local zot_root_dir=${BATS_FILE_TMPDIR}/zot
+    zot_stop_all
-    local oci_data_dir=${BATS_FILE_TMPDIR}/oci
-    teardown_zot_file_level
-    rm -rf ${zot_root_dir}
-    rm -rf ${oci_data_dir}
 }
 
 @test "push image" {
@@ -119,12 +133,12 @@ function teardown_file() {
 
 @test "attach oras artifacts" {
     # attach signature
-    echo "{\"artifact\": \"\", \"signature\": \"pat hancock\"}" > signature.json
+    echo "{\"artifact\": \"\", \"signature\": \"pat hancock\"}" > ${BATS_FILE_TMPDIR}/signature.json
-    run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --image-spec v1.1-image --artifact-type 'signature/example' ./signature.json:application/json
+    run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --image-spec v1.1-image --artifact-type 'signature/example' ${BATS_FILE_TMPDIR}/signature.json:application/json
     [ "$status" -eq 0 ]
     # attach sbom
-    echo "{\"version\": \"0.0.0.0\", \"artifact\": \"'127.0.0.1:8080/golang:1.20'\", \"contents\": \"good\"}" > sbom.json
+    echo "{\"version\": \"0.0.0.0\", \"artifact\": \"'127.0.0.1:8080/golang:1.20'\", \"contents\": \"good\"}" > ${BATS_FILE_TMPDIR}/sbom.json
-    run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --image-spec v1.1-image --artifact-type 'sbom/example' ./sbom.json:application/json
+    run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --image-spec v1.1-image --artifact-type 'sbom/example' ${BATS_FILE_TMPDIR}/sbom.json:application/json
     [ "$status" -eq 0 ]
 }
 
@@ -135,16 +149,16 @@ function teardown_file() {
 }
 
 @test "push helm chart" {
-    run helm package ${BATS_FILE_TMPDIR}/helm-charts/charts/zot
+    run helm package ${BATS_FILE_TMPDIR}/helm-charts/charts/zot -d ${BATS_FILE_TMPDIR}
     [ "$status" -eq 0 ]
     local chart_version=$(awk '/version/{printf $2}' ${BATS_FILE_TMPDIR}/helm-charts/charts/zot/Chart.yaml)
-    run helm push zot-${chart_version}.tgz oci://localhost:8080/zot-chart
+    run helm push ${BATS_FILE_TMPDIR}/zot-${chart_version}.tgz oci://localhost:8080/zot-chart
     [ "$status" -eq 0 ]
 }
 
 @test "pull helm chart" {
     local chart_version=$(awk '/version/{printf $2}' ${BATS_FILE_TMPDIR}/helm-charts/charts/zot/Chart.yaml)
-    run helm pull oci://localhost:8080/zot-chart/zot --version ${chart_version}
+    run helm pull oci://localhost:8080/zot-chart/zot --version ${chart_version} -d ${BATS_FILE_TMPDIR}
     [ "$status" -eq 0 ]
 }
 

test/blackbox/pushpull_running_dedupe.bats

@@ -1,8 +1,26 @@
-load helpers_pushpull
+# Note: Intended to be run as "make test-push-pull-running-dedupe" or "test-push-pull-running-dedupe-verbose"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
+load helpers_zot
+
+function verify_prerequisites {
+    if [ ! $(command -v curl) ]; then
+        echo "you need to install curl as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    if [ ! $(command -v jq) ]; then
+        echo "you need to install jq as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    return 0
+}
 
 function setup_file() {
     # Verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
         exit 1
     fi
     # Download test data to folder common for the entire suite, not just this file
@@ -15,7 +33,7 @@ function setup_file() {
     mkdir -p ${oci_data_dir}
     cat > ${zot_config_file}<<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_root_dir}",
         "dedupe": false,
@@ -32,16 +50,12 @@ function setup_file() {
 }
 EOF
     git -C ${BATS_FILE_TMPDIR} clone https://github.com/project-zot/helm-charts.git
-    setup_zot_file_level ${zot_config_file}
+    zot_serve ${ZOT_PATH} ${zot_config_file}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
+    wait_zot_reachable 8080
 }
 
 function teardown_file() {
-    local zot_root_dir=${BATS_FILE_TMPDIR}/zot
+    zot_stop_all
-    local oci_data_dir=${BATS_FILE_TMPDIR}/oci
-    teardown_zot_file_level
-    rm -rf ${zot_root_dir}
-    rm -rf ${oci_data_dir}
 }
 
 @test "push image - dedupe not running" {
@@ -93,13 +107,13 @@ function teardown_file() {
     local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json
 
     # stop server
-    teardown_zot_file_level
+    zot_stop_all
 
     # enable dedupe
     sed -i 's/false/true/g' ${zot_config_file}
 
-    setup_zot_file_level ${zot_config_file}
+    zot_serve ${ZOT_PATH} ${zot_config_file}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
+    wait_zot_reachable 8080
     # deduping will now run in background (task scheduler) while we push images, shouldn't interfere
 }
 
@@ -118,7 +132,7 @@ function teardown_file() {
 @test "pull image - dedupe running" {
     local oci_data_dir=${BATS_FILE_TMPDIR}/oci
 
-    mkdir -p ${oci_data_dir}/dedupe/    
+    mkdir -p ${oci_data_dir}/dedupe/
 
     start=`date +%s`
     run skopeo --insecure-policy copy --src-tls-verify=false \
@@ -134,7 +148,7 @@ function teardown_file() {
 @test "pull deduped image - dedupe running" {
     local oci_data_dir=${BATS_FILE_TMPDIR}/oci
 
-    mkdir -p ${oci_data_dir}/dedupe/    
+    mkdir -p ${oci_data_dir}/dedupe/
 
     start=`date +%s`
     run skopeo --insecure-policy copy --src-tls-verify=false \
@@ -221,18 +235,18 @@ function teardown_file() {
 
 @test "attach oras artifacts - dedupe running" {
     # attach signature
-    echo "{\"artifact\": \"\", \"signature\": \"pat hancock\"}" > signature.json
+    echo "{\"artifact\": \"\", \"signature\": \"pat hancock\"}" > ${BATS_FILE_TMPDIR}/signature.json
     start=`date +%s`
-    run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --image-spec v1.1-image --artifact-type 'signature/example' ./signature.json:application/json
+    run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --image-spec v1.1-image --artifact-type 'signature/example' ${BATS_FILE_TMPDIR}/signature.json:application/json
     [ "$status" -eq 0 ]
     end=`date +%s`
     runtime=$((end-start))
 
     echo "attach signature exec time: $runtime sec" >&3
     # attach sbom
-    echo "{\"version\": \"0.0.0.0\", \"artifact\": \"'127.0.0.1:8080/golang:1.20'\", \"contents\": \"good\"}" > sbom.json
+    echo "{\"version\": \"0.0.0.0\", \"artifact\": \"'127.0.0.1:8080/golang:1.20'\", \"contents\": \"good\"}" > ${BATS_FILE_TMPDIR}/sbom.json
     start=`date +%s`
-    run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --image-spec v1.1-image --artifact-type 'sbom/example' ./sbom.json:application/json
+    run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --image-spec v1.1-image --artifact-type 'sbom/example' ${BATS_FILE_TMPDIR}/sbom.json:application/json
     [ "$status" -eq 0 ]
     end=`date +%s`
     runtime=$((end-start))
@@ -252,11 +266,11 @@ function teardown_file() {
 }
 
 @test "push helm chart - dedupe running" {
-    run helm package ${BATS_FILE_TMPDIR}/helm-charts/charts/zot
+    run helm package ${BATS_FILE_TMPDIR}/helm-charts/charts/zot -d ${BATS_FILE_TMPDIR}
     [ "$status" -eq 0 ]
     local chart_version=$(awk '/version/{printf $2}' ${BATS_FILE_TMPDIR}/helm-charts/charts/zot/Chart.yaml)
     start=`date +%s`
-    run helm push zot-${chart_version}.tgz oci://localhost:8080/zot-chart
+    run helm push ${BATS_FILE_TMPDIR}/zot-${chart_version}.tgz oci://localhost:8080/zot-chart
     [ "$status" -eq 0 ]
     end=`date +%s`
     runtime=$((end-start))
@@ -267,7 +281,7 @@ function teardown_file() {
 @test "pull helm chart - dedupe running" {
     local chart_version=$(awk '/version/{printf $2}' ${BATS_FILE_TMPDIR}/helm-charts/charts/zot/Chart.yaml)
     start=`date +%s`
-    run helm pull oci://localhost:8080/zot-chart/zot --version ${chart_version}
+    run helm pull oci://localhost:8080/zot-chart/zot --version ${chart_version} -d ${BATS_FILE_TMPDIR}
     [ "$status" -eq 0 ]
     end=`date +%s`
     runtime=$((end-start))

test/blackbox/referrers.bats

@@ -1,8 +1,26 @@
-load helpers_referrers
+# Note: Intended to be run as "make test-bats-referrers"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
+load helpers_zot
+
+function verify_prerequisites() {
+    if [ ! $(command -v curl) ]; then
+        echo "you need to install curl as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    if [ ! $(command -v jq) ] &>/dev/null; then
+        echo "you need to install jq as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    return 0
+}
 
 function setup() {
     # Verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
         exit 1
     fi
 
@@ -18,7 +36,7 @@ function setup() {
     touch ${ZOT_LOG_FILE}
     cat >${ZOT_CONFIG_FILE} <<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${ZOT_ROOT_DIR}"
     },
@@ -49,8 +67,8 @@ EOF
         This artifact is represented as an ispec image manifest, this is the layer inside the manifest.
 EOF
 
-    setup_zot_file_level ${ZOT_CONFIG_FILE}
+    zot_serve ${ZOT_PATH} ${ZOT_CONFIG_FILE}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
+    wait_zot_reachable 8080
 
     run skopeo --insecure-policy copy --dest-tls-verify=false \
         oci:${TEST_DATA_DIR}/golang:1.20 \
@@ -69,10 +87,8 @@ EOF
     curl -X GET http://127.0.0.1:8080/v2/golang/referrers/${MANIFEST_DIGEST}?artifactType=image.artifact/type
 }
 
-function teardown() {
+function teardown_file() {
-    local ZOT_ROOT_DIR=${BATS_FILE_TMPDIR}/zot
+    zot_stop_all
-    zot_stop ${BATS_FILE_TMPDIR}
-    rm -rf ${ZOT_ROOT_DIR}
 }
 
 @test "add referrers, one artifact and one image" {

test/blackbox/restore_s3_blobs.bats

@@ -1,8 +1,9 @@
 load helpers_cloud
+load helpers_wait
 
 function setup_file() {
     # Verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
         exit 1
     fi
 
@@ -77,7 +78,7 @@ EOF
     awslocal s3 --region "us-east-2" mb s3://zot-storage
     awslocal dynamodb --region "us-east-2" create-table --table-name "BlobTable" --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
     zot_serve ${zot_config_file_dedupe}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
+    wait_zot_reachable 8080
 }
 
 function teardown_file() {
@@ -110,7 +111,7 @@ function teardown_file() {
 
     # start with dedupe disabled
     zot_serve ${zot_config_file_nodedupe}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/"
+    wait_zot_reachable 8080
     start=`date +%s`
     echo "waiting for restoring blobs task to finish" >&3
     run wait_for_string "dedupe rebuild: finished" ${ZOT_LOG_FILE} "10m"

test/blackbox/scrub.bats

@@ -1,14 +1,21 @@
+# Note: Intended to be run as "make test-bats-scrub" or "make test-bats-scrub-verbose"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
+load helpers_zot
 load helpers_scrub
 
+function verify_prerequisites() {
+    return 0
+}
+
 function setup_file(){
     skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20
 }
 
 function setup() {
-
     # verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
-        echo "oh noooooo"
         exit 1
     fi
 
@@ -21,7 +28,7 @@ function setup() {
     touch ${ZOT_LOG_FILE}
     cat >${ZOT_CONFIG_FILE} <<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${ZOT_ROOT_DIR}",
         "dedupe": false
@@ -42,23 +49,20 @@ function setup() {
     }
 }
 EOF
-    
+
 }
 
 function teardown() {
-    local zot_root_dir=${BATS_FILE_TMPDIR}/zot
+    zot_stop_all
-    teardown_zot_file_level
-    rm -rf ${zot_root_dir}
 }
 
 
 
 @test "blobs/manifest integrity not affected" {
-
     add_test_files
     echo ${ZOT_CONFIG_FILE}
-    setup_zot_file_level ${ZOT_CONFIG_FILE}
+    zot_serve ${ZOT_PATH} ${ZOT_CONFIG_FILE}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
+    wait_zot_reachable 8080
 
     # wait for scrub to be done and logs to get populated
     run sleep 15s
@@ -68,18 +72,16 @@ function teardown() {
 }
 
 @test "blobs/manifest integrity affected" {
-
     add_test_files
     delete_blob
     echo ${ZOT_CONFIG_FILE}
-    setup_zot_file_level ${ZOT_CONFIG_FILE}
+    zot_serve ${ZOT_PATH} ${ZOT_CONFIG_FILE}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
+    wait_zot_reachable 8080
 
     # wait for scrub to be done and logs to get populated
     run sleep 15s
     run affected
     [ "$status" -eq 0 ]
     [ $(echo "${lines[0]}" ) = 'true' ]
-    # [ $(echo "${lines[-1]}" | jq .) ]
 }
 

test/blackbox/sync.bats

@@ -1,8 +1,30 @@
-load helpers_sync
+# Note: Intended to be run as "make test-bats-sync" or "make test-bats-sync-verbose"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
+load helpers_zot
+load helpers_wait
+
+
+function verify_prerequisites() {
+    if [ ! $(command -v curl) ]; then
+        echo "you need to install curl as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    if [ ! $(command -v jq) ]; then
+        echo "you need to install jq as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    return 0
+}
 
 function setup_file() {
+    export COSIGN_PASSWORD=""
+
     # Verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
         exit 1
     fi
 
@@ -26,7 +48,7 @@ function setup_file() {
 
     cat >${zot_sync_per_config_file} <<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_sync_per_root_dir}"
     },
@@ -61,7 +83,7 @@ EOF
 
     cat >${zot_sync_ondemand_config_file} <<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_sync_ondemand_root_dir}"
     },
@@ -94,7 +116,7 @@ EOF
 EOF
     cat >${zot_minimal_config_file} <<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_minimal_root_dir}"
     },
@@ -109,26 +131,19 @@ EOF
 EOF
     git -C ${BATS_FILE_TMPDIR} clone https://github.com/project-zot/helm-charts.git
 
-    setup_zot_file_level ${zot_sync_per_config_file}
+    zot_serve ${ZOT_PATH} ${zot_sync_per_config_file}
-    wait_zot_reachable "http://127.0.0.1:8081/v2/_catalog"
+    wait_zot_reachable 8081
 
-    setup_zot_file_level ${zot_sync_ondemand_config_file}
+    zot_serve ${ZOT_PATH} ${zot_sync_ondemand_config_file}
-    wait_zot_reachable "http://127.0.0.1:8082/v2/_catalog"
+    wait_zot_reachable 8082
 
-    setup_zot_minimal_file_level ${zot_minimal_config_file}
+    zot_serve ${ZOT_MINIMAL_PATH} ${zot_minimal_config_file}
-    wait_zot_reachable "http://127.0.0.1:9000/v2/_catalog"
+    wait_zot_reachable 9000
 }
 
 function teardown_file() {
-    local zot_sync_per_root_dir=${BATS_FILE_TMPDIR}/zot-per
+    zot_stop_all
-    local zot_sync_ondemand_root_dir=${BATS_FILE_TMPDIR}/zot-ondemand
+    run rm -rf ${HOME}/.config/notation
-    local oci_data_dir=${BATS_FILE_TMPDIR}/oci
-    local zot_minimal_root_dir=${BATS_FILE_TMPDIR}/zot-minimal
-    teardown_zot_file_level
-    rm -rf ${zot_sync_per_root_dir}
-    rm -rf ${zot_sync_ondemand_root_dir}
-    rm -rf ${zot_minimal_root_dir}
-    rm -rf ${oci_data_dir}
 }
 
 # sync image
@@ -144,9 +159,9 @@ function teardown_file() {
     run curl http://127.0.0.1:9000/v2/golang/tags/list
     [ "$status" -eq 0 ]
     [ $(echo "${lines[-1]}" | jq '.tags[]') = '"1.20"' ]
-    
+
     run sleep 20s
-    
+
     run curl http://127.0.0.1:8081/v2/_catalog
     [ "$status" -eq 0 ]
     [ $(echo "${lines[-1]}" | jq '.repositories[]') = '"golang"' ]
@@ -239,11 +254,11 @@ function teardown_file() {
 @test "sign/verify with cosign" {
     run cosign initialize
     [ "$status" -eq 0 ]
-    run cosign generate-key-pair --output-key-prefix "cosign-sign-sync-test"
+    run cosign generate-key-pair --output-key-prefix "${BATS_FILE_TMPDIR}/cosign-sign-sync-test"
     [ "$status" -eq 0 ]
-    run cosign sign --key cosign-sign-sync-test.key localhost:9000/golang:1.20 --yes
+    run cosign sign --key ${BATS_FILE_TMPDIR}/cosign-sign-sync-test.key localhost:9000/golang:1.20 --yes
     [ "$status" -eq 0 ]
-    run cosign verify --key cosign-sign-sync-test.pub localhost:9000/golang:1.20
+    run cosign verify --key ${BATS_FILE_TMPDIR}/cosign-sign-sync-test.pub localhost:9000/golang:1.20
     [ "$status" -eq 0 ]
 }
 
@@ -261,7 +276,7 @@ function teardown_file() {
             "name": "notation-sign-sync-test",
             "registryScopes": [ "*" ],
             "signatureVerification": {
-                "level" : "strict" 
+                "level" : "strict"
             },
             "trustStores": [ "ca:notation-sign-sync-test" ],
             "trustedIdentities": [
@@ -287,7 +302,7 @@ EOF
     run notation verify --plain-http localhost:8081/golang:1.20
     [ "$status" -eq 0 ]
 
-    run cosign verify --key cosign-sign-sync-test.pub localhost:8081/golang:1.20
+    run cosign verify --key ${BATS_FILE_TMPDIR}/cosign-sign-sync-test.pub localhost:8081/golang:1.20
     [ "$status" -eq 0 ]
 }
 
@@ -295,7 +310,7 @@ EOF
     run notation verify --plain-http localhost:8082/golang:1.20
     [ "$status" -eq 0 ]
 
-    run cosign verify --key cosign-sign-sync-test.pub localhost:8082/golang:1.20
+    run cosign verify --key ${BATS_FILE_TMPDIR}/cosign-sign-sync-test.pub localhost:8082/golang:1.20
     [ "$status" -eq 0 ]
 }
 
@@ -328,10 +343,10 @@ EOF
 
 # sync helm chart
 @test "push helm chart" {
-    run helm package ${BATS_FILE_TMPDIR}/helm-charts/charts/zot
+    run helm package ${BATS_FILE_TMPDIR}/helm-charts/charts/zot -d ${BATS_FILE_TMPDIR}
     [ "$status" -eq 0 ]
     local chart_version=$(awk '/version/{printf $2}' ${BATS_FILE_TMPDIR}/helm-charts/charts/zot/Chart.yaml)
-    run helm push zot-${chart_version}.tgz oci://localhost:9000/zot-chart
+    run helm push ${BATS_FILE_TMPDIR}/zot-${chart_version}.tgz oci://localhost:9000/zot-chart
     [ "$status" -eq 0 ]
 }
 
@@ -340,13 +355,13 @@ EOF
     run sleep 15s
 
     local chart_version=$(awk '/version/{printf $2}' ${BATS_FILE_TMPDIR}/helm-charts/charts/zot/Chart.yaml)
-    run helm pull oci://localhost:8081/zot-chart/zot --version ${chart_version}
+    run helm pull oci://localhost:8081/zot-chart/zot --version ${chart_version} -d ${BATS_FILE_TMPDIR}
     [ "$status" -eq 0 ]
 }
 
 @test "sync helm chart on demand" {
     local chart_version=$(awk '/version/{printf $2}' ${BATS_FILE_TMPDIR}/helm-charts/charts/zot/Chart.yaml)
-    run helm pull oci://localhost:8082/zot-chart/zot --version ${chart_version}
+    run helm pull oci://localhost:8082/zot-chart/zot --version ${chart_version} -d ${BATS_FILE_TMPDIR}
     [ "$status" -eq 0 ]
 }
 

test/blackbox/sync_docker.bats

@@ -1,8 +1,31 @@
-load helpers_sync
+# Note: Intended to be run as "make test-bats-sync" or "make test-bats-sync-verbose"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
+load helpers_zot
+
+function verify_prerequisites() {
+    if [ ! $(command -v curl) ]; then
+        echo "you need to install curl as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    if [ ! $(command -v jq) ]; then
+        echo "you need to install jq as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    if [ ! $(command -v docker) ]; then
+        echo "you need to install docker as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    return 0
+}
 
 function setup_file() {
     # Verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
         exit 1
     fi
 
@@ -14,7 +37,7 @@ function setup_file() {
 
     cat >${zot_sync_ondemand_config_file} <<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_root_dir}"
     },
@@ -34,13 +57,13 @@ function setup_file() {
                         "https://docker.io/library"
                     ],
                     "content": [
-						{
-							"prefix": "registry"
-						},
                         {
-							"prefix": "archlinux"
+                            "prefix": "registry"
-						}
+                        },
-					],
+                        {
+                            "prefix": "archlinux"
+                        }
+                    ],
                     "onDemand": true,
                     "tlsVerify": true
                 },
@@ -49,16 +72,16 @@ function setup_file() {
                         "https://registry.k8s.io"
                     ],
                     "content": [
-						{
-							"prefix": "kube-apiserver"
-						},
                         {
-							"prefix": "pause"
+                            "prefix": "kube-apiserver"
-						},
+                        },
+                        {
+                            "prefix": "pause"
+                        },
                         {
                             "prefix": "kube-apiserver-amd64"
                         }
-					],
+                    ],
                     "onDemand": true,
                     "tlsVerify": true
                 },
@@ -67,23 +90,22 @@ function setup_file() {
                         "https://public.ecr.aws"
                     ],
                     "content": [
-						{
+                        {
-							"prefix": "amazonlinux/amazonlinux"
+                            "prefix": "amazonlinux/amazonlinux"
-						}
+                        }
-					],
+                    ],
                     "onDemand": true,
                     "tlsVerify": true
                 },
                 {
                     "urls": [
                         "https://gcr.io"
-
                     ],
                     "content": [
-						{
+                        {
-							"prefix": "google-containers/kube-proxy-amd64"
+                            "prefix": "google-containers/kube-proxy-amd64"
-						}
+                        }
-					],
+                    ],
                     "onDemand": true,
                     "tlsVerify": true
                 },
@@ -92,10 +114,10 @@ function setup_file() {
                         "https://mcr.microsoft.com"
                     ],
                     "content": [
-						{
+                        {
-							"prefix": "azure-cognitive-services/vision/spatial-analysis/diagnostics"
+                            "prefix": "azure-cognitive-services/vision/spatial-analysis/diagnostics"
-						}
+                        }
-					],
+                    ],
                     "onDemand": true,
                     "tlsVerify": true
                 },
@@ -141,15 +163,12 @@ function setup_file() {
 }
 EOF
 
-    setup_zot_file_level ${zot_sync_ondemand_config_file}
+    zot_serve ${ZOT_PATH} ${zot_sync_ondemand_config_file}
-    wait_zot_reachable "http://127.0.0.1:8090/v2/_catalog"
+    wait_zot_reachable 8090
 }
 
 function teardown_file() {
-    local zot_root_dir=${BATS_FILE_TMPDIR}/zot
+    zot_stop_all
-
-    teardown_zot_file_level
-    rm -rf ${zot_root_dir}
 }
 
 # sync image
@@ -312,10 +331,10 @@ function teardown_file() {
 }
 
 @test "run docker with image synced from docker.io/library" {
-    local zot_root_dir=${BATS_FILE_TMPDIR}/zot 
+    local zot_root_dir=${BATS_FILE_TMPDIR}/zot
     run rm -rf ${zot_root_dir}
     [ "$status" -eq 0 ]
-    
+
     run docker run -d 127.0.0.1:8090/archlinux:latest
     [ "$status" -eq 0 ]
 

test/blackbox/sync_harness.bats

@@ -1,8 +1,17 @@
-load helpers_sync
+# Note: Intended to be run as "make test-sync-harness" or "make test-sync-harness-verbose"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
+load helpers_zot
+load helpers_wait
+
+function verify_prerequisites() {
+    return 0
+}
 
 function setup_file() {
     # Verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
         exit 1
     fi
 
@@ -25,7 +34,7 @@ function setup_file() {
 
     cat >${zot_sync_per_config_file} <<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_sync_per_root_dir}"
     },
@@ -61,7 +70,7 @@ EOF
 
     cat >${zot_minimal_config_file} <<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_minimal_root_dir}"
     },
@@ -75,18 +84,12 @@ EOF
     }
 }
 EOF
-    setup_zot_minimal_file_level ${zot_minimal_config_file}
+    zot_serve ${ZOT_MINIMAL_PATH} ${zot_minimal_config_file}
-    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
+    wait_zot_reachable 8080
 }
 
 function teardown_file() {
-    local zot_sync_per_root_dir=${BATS_FILE_TMPDIR}/zot-per
+    zot_stop_all
-    local oci_data_dir=${BATS_FILE_TMPDIR}/oci
-    local zot_minimal_root_dir=${BATS_FILE_TMPDIR}/zot-minimal
-    teardown_zot_file_level
-    rm -rf ${zot_sync_per_root_dir}
-    rm -rf ${zot_minimal_root_dir}
-    rm -rf ${oci_data_dir}
 }
 
 # sync zb images
@@ -99,8 +102,8 @@ function teardown_file() {
     zb_run "http://127.0.0.1:8080"
 
     # start zot sync server
-    setup_zot_file_level ${zot_sync_per_config_file}
+    zot_start ${ZOT_PATH} ${zot_sync_per_config_file}
-    wait_zot_reachable "http://127.0.0.1:8081/v2/_catalog"
+    wait_zot_reachable 8081
 
     start=`date +%s`
     echo "waiting for sync to finish" >&3

test/blackbox/sync_replica_cluster.bats

@@ -1,8 +1,31 @@
-load helpers_sync
+# Note: Intended to be run as "make test-bats-sync" or "make test-bats-sync-verbose"
+#       Makefile target installs & checks all necessary tooling
+#       Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
+
+load helpers_zot
+
+function verify_prerequisites() {
+    if [ ! $(command -v curl) ]; then
+        echo "you need to install curl as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    if [ ! $(command -v jq) ]; then
+        echo "you need to install jq as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    if [ ! $(command -v docker) ]; then
+        echo "you need to install docker as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    return 0
+}
 
 function setup_file() {
     # Verify prerequisites are available
-    if ! verify_prerequisites; then
+    if ! $(verify_prerequisites); then
         exit 1
     fi
 
@@ -20,7 +43,7 @@ function setup_file() {
 
     cat >${zot_sync_one_config_file} <<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_sync_one_root_dir}"
     },
@@ -56,7 +79,7 @@ EOF
 
     cat >${zot_sync_two_config_file} <<EOF
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "${zot_sync_two_root_dir}"
     },
@@ -92,19 +115,15 @@ EOF
 
     git -C ${BATS_FILE_TMPDIR} clone https://github.com/project-zot/helm-charts.git
 
-    setup_zot_file_level ${zot_sync_one_config_file}
+    zot_serve ${ZOT_PATH} ${zot_sync_one_config_file}
-    wait_zot_reachable "http://127.0.0.1:8081/v2/_catalog"
+    wait_zot_reachable 8081
 
-    setup_zot_file_level ${zot_sync_two_config_file}
+    zot_serve ${ZOT_PATH} ${zot_sync_two_config_file}
-    wait_zot_reachable "http://127.0.0.1:8082/v2/_catalog"
+    wait_zot_reachable 8082
 }
 
 function teardown_file() {
-    local zot_sync_one_root_dir=${BATS_FILE_TMPDIR}/zot-per
+    zot_stop_all
-    local zot_sync_two_root_dir=${BATS_FILE_TMPDIR}/zot-ondemand
-    teardown_zot_file_level
-    rm -rf ${zot_sync_one_root_dir}
-    rm -rf ${zot_sync_two_root_dir}
 }
 
 # sync image
@@ -119,9 +138,9 @@ function teardown_file() {
     run curl http://127.0.0.1:8081/v2/golang/tags/list
     [ "$status" -eq 0 ]
     [ $(echo "${lines[-1]}" | jq '.tags[]') = '"1.20"' ]
-    
+
     run sleep 30s
-    
+
     run curl http://127.0.0.1:8082/v2/_catalog
     [ "$status" -eq 0 ]
     [ $(echo "${lines[-1]}" | jq '.repositories[]') = '"golang"' ]
@@ -142,9 +161,9 @@ function teardown_file() {
     run curl http://127.0.0.1:8082/v2/anothergolang/tags/list
     [ "$status" -eq 0 ]
     [ $(echo "${lines[-1]}" | jq '.tags[]') = '"1.20"' ]
-    
+
     run sleep 30s
-    
+
     run curl http://127.0.0.1:8081/v2/_catalog
     [ "$status" -eq 0 ]
     [ $(echo "${lines[-1]}" | jq '.repositories[0]') = '"anothergolang"' ]

test/cluster/config-minio.json

@@ -1,5 +1,5 @@
 {
-    "distSpecVersion": "1.1.0",
+    "distSpecVersion": "1.1.0-dev",
     "storage": {
         "rootDirectory": "/tmp/zot",
         "gc": false,