Refactor the push/pull tests to use the bats test framework (#467)

This is a follow up to #444.

Signed-off-by: Andrei Aaron <andaaron@cisco.com>

Makefile

@@ -5,11 +5,12 @@ GO_VERSION=$(shell go version | awk '{print $$3}')
 COMMIT ?= $(if $(shell git status --porcelain --untracked-files=no),$(COMMIT_HASH)-dirty,$(COMMIT_HASH))
 CONTAINER_RUNTIME := $(shell command -v podman 2> /dev/null || echo docker)
 TMPDIR := $(shell mktemp -d)
-TOOLSDIR := hack/tools
+TOOLSDIR := $(shell pwd)/hack/tools
-PATH := bin:$(shell pwd)/$(TOOLSDIR)/bin:$(PATH)
+PATH := bin:$(TOOLSDIR)/bin:$(PATH)
 STACKER := $(shell which stacker)
 GOLINTER := $(TOOLSDIR)/bin/golangci-lint
 NOTATION := $(TOOLSDIR)/bin/notation
+BATS := $(TOOLSDIR)/bin/bats
 OS ?= linux
 ARCH ?= amd64
 BENCH_OUTPUT ?= stdout
@@ -63,28 +64,6 @@ run-bench: binary bench
 	bin/zb-$(OS)-$(ARCH) -c 10 -n 100 -o $(BENCH_OUTPUT) http://localhost:8080
 	killall -r zot-*
 
-.PHONY: push-pull
-push-pull: binary check-skopeo
-	bin/zot-$(OS)-$(ARCH) serve examples/config-minimal.json &
-	sleep 5
-	# skopeo push/pull
-	skopeo --debug copy --format=oci --dest-tls-verify=false docker://ghcr.io/project-zot/golang:1.17 docker://localhost:8080/golang:1.17
-	skopeo --debug copy --src-tls-verify=false docker://localhost:8080/golang:1.17 oci:golang:1.17
-	# oras artifacts
-	echo "{\"name\":\"foo\",\"value\":\"bar\"}" > config.json
-	echo "hello world" > artifact.txt
-	oras push localhost:8080/hello-artifact:v2 \
-	    --manifest-config config.json:application/vnd.acme.rocket.config.v1+json \
-		      artifact.txt:text/plain -d -v
-	rm -f artifact.txt # first delete the file
-	oras pull localhost:8080/hello-artifact:v2 -d -v -a
-	grep -q "hello world" artifact.txt  # should print "hello world"
-	if [ $? -ne 0 ]; then \
-		killall -r zot-*; \
-		exit 1; \
-	fi
-	killall -r zot-*
-
 .PHONY: test-clean
 test-clean:
 	$(shell sudo rm -rf /etc/containers/certs.d/127.0.0.1:8089/)
@@ -186,3 +165,17 @@ binary-stacker:
 .PHONY: image
 image:
 	${CONTAINER_RUNTIME} build ${BUILD_ARGS} -f Dockerfile -t zot:latest .
+
+$(BATS):
+	rm -rf bats-core; \
+	git clone https://github.com/bats-core/bats-core.git; \
+	cd bats-core; ./install.sh $(TOOLSDIR); cd ..; \
+	rm -rf bats-core
+
+.PHONY: push-pull
+push-pull: binary check-skopeo $(BATS)
+	$(BATS) --trace --print-output-on-failure test/blackbox
+
+.PHONY: push-pull-verbose
+push-pull-verbose: binary check-skopeo $(BATS)
+	$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox

test/blackbox/helpers.bash

@@ -0,0 +1,74 @@
+ROOT_DIR=$(git rev-parse --show-toplevel)
+TEST_DATA_DIR=${ROOT_DIR}/test/data/
+OS="${OS:-linux}"
+ARCH="${ARCH:-amd64}"
+ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}
+
+mkdir -p ${TEST_DATA_DIR}
+
+function verify_prerequisites {
+    if [ ! -f ${BATS_RUN_TMPDIR}/.firstrun ]; then
+        env | grep proxy >&3
+        touch ${BATS_RUN_TMPDIR}/.firstrun
+    fi
+
+    if [ ! -f ${ZOT_PATH} ]; then
+        echo "you need to build ${ZOT_PATH} before running the tests" >&3
+        return 1
+    fi
+
+    if [ ! command -v curl &> /dev/null ]; then
+        echo "you need to install curl as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    if [ ! command -v jq &> /dev/null ]; then
+        echo "you need to install jq as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    if [ ! command -v skopeo &> /dev/null ]; then
+        echo "you need to install skopeo as a prerequisite to running the tests" >&3
+        return 1
+    fi
+
+    if [ ! command -v oras &> /dev/null ]; then
+        echo "you need to install oras as a prerequisite to running the tests" >&3
+        return 1
+    fi
+    return 0
+}
+
+function zot_serve() {
+    local zot_path=${1}
+    local config_file=${2}
+    local pid_dir=${3}
+    ${zot_path} serve ${config_file} &
+    echo $! > ${pid_dir}/zot.pid
+}
+
+function zot_stop() {
+    local pid_dir=${1}
+    kill $(cat ${pid_dir}/zot.pid)
+    rm ${pid_dir}/zot.pid
+}
+
+function setup_zot_file_level() {
+    local config_file=${1}
+    zot_serve ${ZOT_PATH} ${config_file} ${BATS_FILE_TMPDIR}
+}
+
+function teardown_zot_file_level() {
+    zot_stop ${BATS_FILE_TMPDIR}
+}
+
+function wait_zot_reachable() {
+    zot_url=${1}
+    curl --connect-timeout 3 \
+        --max-time 3 \
+        --retry 10 \
+        --retry-delay 0 \
+        --retry-max-time 60 \
+        --retry-connrefused \
+        ${zot_url}
+}

test/blackbox/pushpull.bats

@@ -0,0 +1,81 @@
+load helpers
+
+function setup_file() {
+    # Verify prerequisites are available
+    if ! verify_prerequisites; then
+        exit 1
+    fi
+    # Download test data to folder common for the entire suite, not just this file
+    skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.17 oci:${TEST_DATA_DIR}/golang:1.17
+    # Setup zot server
+    local zot_root_dir=${BATS_FILE_TMPDIR}/zot
+    local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json
+    local oci_data_dir=${BATS_FILE_TMPDIR}/oci
+    mkdir -p ${zot_root_dir}
+    mkdir -p ${oci_data_dir}
+    cat > ${zot_config_file}<<EOF
+{
+    "distSpecVersion": "1.0.1",
+    "storage": {
+        "rootDirectory": "${zot_root_dir}"
+    },
+    "http": {
+        "address": "0.0.0.0",
+        "port": "8080",
+        "ReadOnly": false
+    },
+    "log": {
+        "level": "debug"
+    }
+}
+EOF
+    setup_zot_file_level ${zot_config_file}
+    wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
+}
+
+function teardown_file() {
+    local zot_root_dir=${BATS_FILE_TMPDIR}/zot
+    local oci_data_dir=${BATS_FILE_TMPDIR}/oci
+    teardown_zot_file_level
+    rm -rf ${zot_root_dir}
+    rm -rf ${oci_data_dir}
+}
+
+@test "push image" {
+    run skopeo --insecure-policy copy --dest-tls-verify=false \
+        oci:${TEST_DATA_DIR}/golang:1.17 \
+        docker://127.0.0.1:8080/golang:1.17
+    [ "$status" -eq 0 ]
+    run curl http://127.0.0.1:8080/v2/_catalog
+    [ "$status" -eq 0 ]
+    [ $(echo "${lines[-1]}" | jq '.repositories[]') = '"golang"' ]
+    run curl http://127.0.0.1:8080/v2/golang/tags/list
+    [ "$status" -eq 0 ]
+    [ $(echo "${lines[-1]}" | jq '.tags[]') = '"1.17"' ]
+}
+
+@test "pull image" {
+    local oci_data_dir=${BATS_FILE_TMPDIR}/oci
+    run skopeo --insecure-policy copy --src-tls-verify=false \
+        docker://127.0.0.1:8080/golang:1.17 \
+        oci:${oci_data_dir}/golang:1.17
+    [ "$status" -eq 0 ]
+    run cat ${BATS_FILE_TMPDIR}/oci/golang/index.json
+    [ "$status" -eq 0 ]
+    [ $(echo "${lines[-1]}" | jq '.manifests[].annotations."org.opencontainers.image.ref.name"') = '"1.17"' ]
+}
+
+@test "push oras artifact" {
+    echo "{\"name\":\"foo\",\"value\":\"bar\"}" > config.json
+    echo "hello world" > artifact.txt
+    oras push 127.0.0.1:8080/hello-artifact:v2 \
+        --manifest-config config.json:application/vnd.acme.rocket.config.v1+json artifact.txt:text/plain -d -v
+	rm -f artifact.txt
+    rm -f config.json
+}
+
+@test "pull oras artifact" {
+    oras pull 127.0.0.1:8080/hello-artifact:v2 -d -v -a
+    grep -q "hello world" artifact.txt
+    rm -f artifact.txt
+}