mirror of
https://github.com/project-zot/zot.git
synced 2024-12-16 21:56:37 -05:00
refactor(makefile): consolidate the make targets used for bats tests (#1746)
New examples of running tests: 1. To run a specific bats file (with and without verbose output): make run-blackbox-tests BATS_TEST_FILE_PATH=test/blackbox/delete_images.bats make run-blackbox-tests BATS_TEST_FILE_PATH=test/blackbox/delete_images.bats BATS_VERBOSITY=2 2. To run the CI tests (with and without verbose output) make run-blackbox-ci make run-blackbox-ci BATS_VERBOSITY=2 BATS_TEST_FILE_PATH is used to pass on the test file to run using `run-blackbox-tests` BATS_VERBOSITY controls the verbosity of the bats framework output, if unspecified the output only contains test results and failure message in case of failures. If BATS_VERBOSITY is 1, then also show commands as they are executed. If BATS_VERBOSITY is 2, on top of the above it also shows output of passed tests. Other changes in this PR: - Update some of the tests to show logs after the run ends. - Run the linters before the tests, as it saves time on failures when running in GH Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
This commit is contained in:
Andrei Aaron
GitHub
parent
f5b63963be
commit
6115eed4ec
22 changed files with 199 additions and 211 deletions
39
.github/workflows/ecosystem-tools.yaml
vendored
39
.github/workflows/ecosystem-tools.yaml
vendored
|
|
@ -69,42 +69,9 @@ jobs:
|
|||
sudo du -sh /var/lib/docker/
|
||||
du -sh /home/runner/work/
|
||||
set +x
|
||||
- name: Run referrers tests
|
||||
- name: Run CI tests
|
||||
run: |
|
||||
make test-bats-referrers
|
||||
- name: Run metadata tests
|
||||
run: |
|
||||
make test-bats-metadata
|
||||
- name: Run push-pull tests
|
||||
run: |
|
||||
make test-push-pull
|
||||
- name: Run push-pull-authn tests
|
||||
run: |
|
||||
make test-push-pull-authn
|
||||
- name: Run metrics tests
|
||||
run: |
|
||||
make test-bats-metrics
|
||||
- name: Run cve tests
|
||||
run: |
|
||||
make test-bats-cve
|
||||
- name: Run sync test
|
||||
run: |
|
||||
make test-bats-sync
|
||||
- name: Run scrub tests
|
||||
run: |
|
||||
make test-bats-scrub
|
||||
- name: Run anonymous-push-pull tests
|
||||
run: |
|
||||
make test-anonymous-push-pull
|
||||
- name: Run detect-manifest-collision tests
|
||||
run: |
|
||||
make test-detect-manifest-collision
|
||||
- name: Run annotations tests
|
||||
run: |
|
||||
make test-annotations
|
||||
- name: Run garbage collect tests
|
||||
run: |
|
||||
make test-garbage-collect
|
||||
make run-blackbox-ci
|
||||
- name: Install localstack
|
||||
run: |
|
||||
pip install --upgrade pyopenssl
|
||||
|
|
@ -117,7 +84,7 @@ jobs:
|
|||
echo "Startup complete"
|
||||
- name: Run cloud-only tests
|
||||
run: |
|
||||
make test-cloud-only
|
||||
make run-blackbox-cloud-ci
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: fake
|
||||
AWS_SECRET_ACCESS_KEY: fake
|
||||
|
|
|
|||
15
.github/workflows/nightly.yaml
vendored
15
.github/workflows/nightly.yaml
vendored
|
|
@ -44,15 +44,11 @@ jobs:
|
|||
echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container
|
||||
localstack wait -t 30 # to become ready before timing out
|
||||
echo "Startup complete"
|
||||
- name: Run restore s3 blobs after cache is deleted
|
||||
- name: Run blackbox nightly dedupe tests
|
||||
run: |
|
||||
make test-restore-s3-blobs
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: fake
|
||||
AWS_SECRET_ACCESS_KEY: fake
|
||||
- name: Run dedupe filesystem blobs after switching dedupe to enable.
|
||||
run: |
|
||||
make test-push-pull-running-dedupe
|
||||
# test restoring s3 blobs after cache is deleted
|
||||
# test deduping filesystem blobs after switching dedupe to enable
|
||||
make run-blackbox-dedupe-nightly
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: fake
|
||||
AWS_SECRET_ACCESS_KEY: fake
|
||||
|
|
@ -73,7 +69,8 @@ jobs:
|
|||
go mod download
|
||||
- name: Run sync harness
|
||||
run: |
|
||||
make test-sync-harness
|
||||
make run-blackbox-sync-nightly
|
||||
|
||||
gc-referrers-stress-s3:
|
||||
name: GC(with referrers) on S3 with short interval
|
||||
runs-on: ubuntu-latest
|
||||
|
|
|
|||
177
Makefile
177
Makefile
|
|
@ -54,8 +54,17 @@ filter-valid = $(foreach ext, $(merged-extensions), $(if $(findstring $(ext),$(A
|
|||
add-extensions = $(subst $(1),$(2),$(sort $(filter-valid)))
|
||||
BUILD_LABELS = $(call add-extensions,$(space),$(comma))
|
||||
|
||||
BATS_TEST_FILE_PATH ?= replace_me
|
||||
ifeq ($(BATS_VERBOSITY),2)
|
||||
BATS_FLAGS = --trace --verbose-run --show-output-of-passing-tests --print-output-on-failure
|
||||
else ifeq ($(BATS_VERBOSITY),1)
|
||||
BATS_FLAGS = --trace --verbose-run --print-output-on-failure
|
||||
else
|
||||
BATS_FLAGS = --print-output-on-failure
|
||||
endif
|
||||
|
||||
.PHONY: all
|
||||
all: modcheck swaggercheck binary binary-minimal binary-debug cli bench exporter-minimal verify-config test covhtml check check-gh-actions
|
||||
all: modcheck swaggercheck binary binary-minimal binary-debug cli bench exporter-minimal verify-config check check-gh-actions test covhtml
|
||||
|
||||
.PHONY: modtidy
|
||||
modtidy:
|
||||
|
|
@ -162,6 +171,10 @@ run-bench: binary bench
|
|||
check-skopeo:
|
||||
skopeo -v || (echo "You need skopeo to be installed in order to run tests"; exit 1)
|
||||
|
||||
.PHONY: check-awslocal
|
||||
check-awslocal:
|
||||
awslocal --version || (echo "You need awslocal to be installed in order to run tests"; exit 1)
|
||||
|
||||
$(NOTATION):
|
||||
mkdir -p $(TOOLSDIR)/bin
|
||||
curl -Lo notation.tar.gz https://github.com/notaryproject/notation/releases/download/v$(NOTATION_VERSION)/notation_$(NOTATION_VERSION)_linux_amd64.tar.gz
|
||||
|
|
@ -349,127 +362,57 @@ $(BATS):
|
|||
cd bats-core; ./install.sh $(TOOLSDIR); cd ..; \
|
||||
rm -rf bats-core
|
||||
|
||||
.PHONY: test-push-pull
|
||||
test-push-pull: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/pushpull.bats
|
||||
.PHONY: check-blackbox-prerequisites
|
||||
check-blackbox-prerequisites: check-linux check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL) $(NOTATION) $(COSIGN) $(STACKER)
|
||||
which skopeo && skopeo --version; \
|
||||
which stacker && stacker --version; \
|
||||
which regctl && regctl version; \
|
||||
which oras && oras version; \
|
||||
which helm && helm version; \
|
||||
which crictl && crictl version; \
|
||||
which notation && notation version; \
|
||||
which cosign && cosign version;
|
||||
|
||||
.PHONY: test-push-pull-verbose
|
||||
test-push-pull-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL)
|
||||
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull.bats
|
||||
.PHONY: run-blackbox-tests
|
||||
run-blackbox-tests: $(BATS_TEST_FILE_PATH) check-blackbox-prerequisites binary binary-minimal cli bench
|
||||
echo running bats test "$(BATS_TEST_FILE_PATH)"; \
|
||||
$(BATS) $(BATS_FLAGS) $(BATS_TEST_FILE_PATH)
|
||||
|
||||
.PHONY: test-garbage-collect
|
||||
test-garbage-collect: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/garbage_collect.bats
|
||||
.PHONY: run-blackbox-ci
|
||||
run-blackbox-ci: check-blackbox-prerequisites binary binary-minimal cli
|
||||
# ideally we would run a single bats command but too much disk space would be used at once
|
||||
echo running CI bats tests; \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/pushpull.bats && \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/pushpull_authn.bats && \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/delete_images.bats && \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/referrers.bats && \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/metadata.bats && \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/anonymous_policy.bats && \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/annotations.bats && \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/detect_manifest_collision.bats && \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/cve.bats && \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/sync.bats && \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/sync_docker.bats && \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/sync_replica_cluster.bats && \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/scrub.bats && \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/garbage_collect.bats && \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/metrics.bats
|
||||
|
||||
.PHONY: test-garbage-collect-verbose
|
||||
test-garbage-collect-verbose: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS)
|
||||
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/garbage_collect.bats
|
||||
.PHONY: run-blackbox-cloud-ci
|
||||
run-blackbox-cloud-ci: check-blackbox-prerequisites check-awslocal binary $(BATS)
|
||||
echo running cloud CI bats tests; \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/cloud_only.bats
|
||||
|
||||
.PHONY: test-push-pull-running-dedupe
|
||||
test-push-pull-running-dedupe: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/pushpull_running_dedupe.bats
|
||||
.PHONY: run-blackbox-dedupe-nightly
|
||||
run-blackbox-dedupe-nightly: check-blackbox-prerequisites check-awslocal binary binary-minimal
|
||||
echo running nightly dedupe tests; \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/restore_s3_blobs.bats && \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/pushpull_running_dedupe.bats
|
||||
|
||||
.PHONY: test-push-pull-running-dedupe-verbose
|
||||
test-push-pull-running-dedupe-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
|
||||
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull_running_dedupe.bats
|
||||
|
||||
.PHONY: test-push-pull-authn
|
||||
test-push-pull-authn: check-linux binary check-skopeo $(BATS) $(REGCLIENT)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/pushpull_authn.bats
|
||||
|
||||
.PHONY: test-sync-harness
|
||||
test-sync-harness: check-linux binary binary-minimal bench check-skopeo $(BATS)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/sync_harness.bats
|
||||
|
||||
.PHONY: test-sync-harness-verbose
|
||||
test-sync-harness-verbose: check-linux binary binary-minimal bench check-skopeo $(BATS)
|
||||
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_harness.bats
|
||||
|
||||
.PHONY: test-restore-s3-blobs
|
||||
test-restore-s3-blobs: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/restore_s3_blobs.bats
|
||||
|
||||
.PHONY: test-restore-s3-blobs-verbose
|
||||
test-restore-s3-blobs-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
|
||||
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/restore_s3_blobs.bats
|
||||
|
||||
.PHONY: test-bats-referrers
|
||||
test-bats-referrers: BUILD_LABELS=search
|
||||
test-bats-referrers: check-linux binary check-skopeo $(BATS) $(ORAS)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/referrers.bats
|
||||
|
||||
.PHONY: test-bats-metadata
|
||||
test-bats-metadata: BUILD_LABELS=search,userprefs
|
||||
test-bats-metadata: check-linux binary check-skopeo $(BATS)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/metadata.bats
|
||||
|
||||
.PHONY: test-cloud-only
|
||||
test-cloud-only: check-linux binary check-skopeo $(BATS)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/cloud_only.bats
|
||||
|
||||
.PHONY: test-cloud-only-verbose
|
||||
test-cloud-only-verbose: check-linux binary check-skopeo $(BATS)
|
||||
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/cloud_only.bats
|
||||
|
||||
.PHONY: test-bats-sync
|
||||
test-bats-sync: BUILD_LABELS=sync
|
||||
test-bats-sync: check-linux binary binary-minimal bench check-skopeo $(BATS) $(NOTATION) $(COSIGN) $(HELM)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/sync.bats
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/sync_docker.bats
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/sync_replica_cluster.bats
|
||||
|
||||
.PHONY: test-bats-delete-image
|
||||
test-bats-delete-image: check-linux binary binary-minimal bench check-skopeo $(BATS)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/delete_images.bats
|
||||
|
||||
.PHONY: test-bats-sync-verbose
|
||||
test-bats-sync-verbose: BUILD_LABELS=sync
|
||||
test-bats-sync-verbose: check-linux binary binary-minimal bench check-skopeo $(BATS) $(NOTATION) $(COSIGN) $(HELM)
|
||||
$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync.bats
|
||||
$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_docker.bats
|
||||
$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_replica_cluster.bats
|
||||
|
||||
.PHONY: test-bats-cve
|
||||
test-bats-cve: BUILD_LABELS=search
|
||||
test-bats-cve: check-linux binary cli check-skopeo $(BATS)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/cve.bats
|
||||
|
||||
.PHONY: test-bats-cve-verbose
|
||||
test-bats-cve-verbose: BUILD_LABELS=search
|
||||
test-bats-cve-verbose: check-linux binary cli check-skopeo $(BATS)
|
||||
$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/cve.bats
|
||||
|
||||
.PHONY: test-bats-scrub
|
||||
test-bats-scrub: BUILD_LABELS=scrub
|
||||
test-bats-scrub: check-linux binary check-skopeo $(BATS)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/scrub.bats
|
||||
|
||||
.PHONY: test-bats-scrub-verbose
|
||||
test-bats-scrub-verbose: BUILD_LABELS=scrub
|
||||
test-bats-scrub-verbose: check-linux binary check-skopeo $(BATS)
|
||||
$(BATS) --trace -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/scrub.bats
|
||||
|
||||
.PHONY: test-bats-metrics
|
||||
test-bats-metrics: BUILD_LABELS=metrics
|
||||
test-bats-metrics: check-linux binary check-skopeo $(BATS)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/metrics.bats
|
||||
|
||||
.PHONY: test-bats-metrics-verbose
|
||||
test-bats-metrics-verbose: BUILD_LABELS=metrics
|
||||
test-bats-metrics-verbose: check-linux binary check-skopeo $(BATS)
|
||||
$(BATS) --trace -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/metrics.bats
|
||||
|
||||
.PHONY: test-anonymous-push-pull
|
||||
test-anonymous-push-pull: check-linux binary check-skopeo $(BATS)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/anonymous_policy.bats
|
||||
|
||||
.PHONY: test-annotations
|
||||
test-annotations: check-linux binary check-skopeo $(BATS) $(STACKER) $(NOTATION) $(COSIGN)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/annotations.bats
|
||||
|
||||
.PHONY: test-detect-manifest-collision
|
||||
test-detect-manifest-collision: check-linux binary check-skopeo $(BATS) $(REGCLIENT)
|
||||
$(BATS) --trace --print-output-on-failure test/blackbox/detect_manifest_collision.bats
|
||||
.PHONY: run-blackbox-sync-nightly
|
||||
run-blackbox-sync-nightly: check-blackbox-prerequisites binary binary-minimal bench
|
||||
echo running nightly sync tests; \
|
||||
$(BATS) $(BATS_FLAGS) test/blackbox/sync_harness.bats
|
||||
|
||||
.PHONY: fuzz-all
|
||||
fuzz-all: fuzztime=${1}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Note: Intended to be run as "make test-annotations"
|
||||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
|
|
@ -79,8 +79,12 @@ EOF
|
|||
wait_zot_reachable 8080
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
# conditionally printing on failure is possible from teardown but not from from teardown_file
|
||||
cat ${BATS_FILE_TMPDIR}/zot.log
|
||||
}
|
||||
|
||||
function teardown_file() {
|
||||
cat ${BATS_FILE_TMPDIR}/zot.log >&3
|
||||
zot_stop_all
|
||||
run rm -rf ${HOME}/.config/notation
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Note: Intended to be run as "make test-anonymous-push-pull"
|
||||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
|
|
@ -59,7 +59,8 @@ function setup_file() {
|
|||
}
|
||||
},
|
||||
"log": {
|
||||
"level": "debug"
|
||||
"level": "debug",
|
||||
"output": "${BATS_FILE_TMPDIR}/zot.log"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
|
@ -67,6 +68,11 @@ EOF
|
|||
wait_zot_reachable 8080
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
# conditionally printing on failure is possible from teardown but not from from teardown_file
|
||||
cat ${BATS_FILE_TMPDIR}/zot.log
|
||||
}
|
||||
|
||||
function teardown_file() {
|
||||
zot_stop_all
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-cloud-ci"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
load helpers_cloud
|
||||
load helpers_wait
|
||||
|
||||
|
|
@ -17,8 +21,8 @@ function setup() {
|
|||
|
||||
cat > ${zot_config_file}<<EOF
|
||||
{
|
||||
"distSpecVersion": "1.1.0-dev",
|
||||
"storage": {
|
||||
"distSpecVersion": "1.1.0-dev",
|
||||
"storage": {
|
||||
"rootDirectory": "${zot_root_dir}",
|
||||
"dedupe": true,
|
||||
"remoteCache": true,
|
||||
|
|
@ -43,10 +47,10 @@ function setup() {
|
|||
"apiKeyTablename":"ApiKeyTable",
|
||||
"versionTablename": "Version"
|
||||
}
|
||||
},
|
||||
"http": {
|
||||
"address": "127.0.0.1",
|
||||
"port": "8080",
|
||||
},
|
||||
"http": {
|
||||
"address": "127.0.0.1",
|
||||
"port": "8080",
|
||||
"realm": "zot",
|
||||
"auth": {
|
||||
"openid": {
|
||||
|
|
@ -68,25 +72,25 @@ function setup() {
|
|||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"log": {
|
||||
"level": "debug"
|
||||
},
|
||||
"extensions": {
|
||||
"metrics": {
|
||||
},
|
||||
"log": {
|
||||
"level": "debug"
|
||||
},
|
||||
"extensions": {
|
||||
"metrics": {
|
||||
"enable": true,
|
||||
"prometheus": {
|
||||
"path": "/metrics"
|
||||
}
|
||||
},
|
||||
"search": {
|
||||
"search": {
|
||||
"enable": true
|
||||
},
|
||||
"scrub": {
|
||||
"enable": true,
|
||||
"interval": "24h"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scrub": {
|
||||
"enable": true,
|
||||
"interval": "24h"
|
||||
}
|
||||
}
|
||||
}
|
||||
EOF
|
||||
awslocal s3 --region "us-east-2" mb s3://zot-storage
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Note: Intended to be run as "make test-bats-cve" or "make test-bats-cve-verbose"
|
||||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
|
|
@ -42,7 +42,8 @@ function setup_file() {
|
|||
"port": "8080"
|
||||
},
|
||||
"log": {
|
||||
"level": "debug"
|
||||
"level": "debug",
|
||||
"output": "${BATS_FILE_TMPDIR}/zot.log"
|
||||
},
|
||||
"extensions": {
|
||||
"search": {
|
||||
|
|
@ -63,6 +64,11 @@ EOF
|
|||
zli_add_config ${registry_name} ${registry_url}
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
# conditionally printing on failure is possible from teardown but not from from teardown_file
|
||||
cat ${BATS_FILE_TMPDIR}/zot.log
|
||||
}
|
||||
|
||||
function teardown_file() {
|
||||
zot_stop_all
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
load helpers_zot
|
||||
|
||||
function verify_prerequisites {
|
||||
|
|
@ -14,7 +18,7 @@ function verify_prerequisites {
|
|||
|
||||
function setup_file() {
|
||||
# Verify prerequisites are available
|
||||
if ! (verify_prerequisites); then
|
||||
if ! verify_prerequisites; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
@ -22,10 +26,10 @@ function setup_file() {
|
|||
skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/test-images/alpine:3.17.3 oci:${TEST_DATA_DIR}/alpine:3.17.3
|
||||
|
||||
# Setup zot server
|
||||
ZOT_ROOT_DIR=${BATS_RUN_TMPDIR}/zot
|
||||
ZOT_ROOT_DIR=${BATS_FILE_TMPDIR}/zot
|
||||
echo ${ZOT_ROOT_DIR}
|
||||
local zot_log_file=${BATS_RUN_TMPDIR}/zot-log.json
|
||||
local zot_config_file=${BATS_RUN_TMPDIR}/zot_config.json
|
||||
local zot_log_file=${BATS_FILE_TMPDIR}/zot-log.json
|
||||
local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json
|
||||
mkdir -p ${ZOT_ROOT_DIR}
|
||||
touch ${zot_log_file}
|
||||
cat >${zot_config_file} <<EOF
|
||||
|
|
@ -65,6 +69,11 @@ EOF
|
|||
echo ${MANIFEST_DIGEST}
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
# conditionally printing on failure is possible from teardown but not from from teardown_file
|
||||
cat ${BATS_FILE_TMPDIR}/zot-log.json
|
||||
}
|
||||
|
||||
function teardown_file() {
|
||||
zot_stop_all
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Note: Intended to be run as "make test-detect-manifest-collision"
|
||||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
|
|
@ -64,7 +64,8 @@ function setup_file() {
|
|||
}
|
||||
},
|
||||
"log": {
|
||||
"level": "debug"
|
||||
"level": "debug",
|
||||
"output": "${BATS_FILE_TMPDIR}/zot.log"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
|
@ -72,6 +73,11 @@ EOF
|
|||
wait_zot_reachable 8080
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
# conditionally printing on failure is possible from teardown but not from from teardown_file
|
||||
cat ${BATS_FILE_TMPDIR}/zot.log
|
||||
}
|
||||
|
||||
function teardown_file() {
|
||||
zot_stop_all
|
||||
}
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ function setup_file() {
|
|||
},
|
||||
"log": {
|
||||
"level": "debug",
|
||||
"output": "/tmp/gc.log"
|
||||
"output": "${BATS_FILE_TMPDIR}/gc.log"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
|
@ -53,6 +53,11 @@ EOF
|
|||
wait_zot_reachable 8080
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
# conditionally printing on failure is possible from teardown but not from from teardown_file
|
||||
cat ${BATS_FILE_TMPDIR}/gc.log
|
||||
}
|
||||
|
||||
function teardown_file() {
|
||||
zot_stop_all
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Note: Intended to be run as "make test-bats-metadata"
|
||||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
|
|
@ -77,7 +77,8 @@ function setup_file() {
|
|||
}
|
||||
},
|
||||
"log": {
|
||||
"level": "debug"
|
||||
"level": "debug",
|
||||
"output": "${BATS_FILE_TMPDIR}/zot.log"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
|
@ -86,11 +87,15 @@ EOF
|
|||
wait_zot_reachable 8080
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
# conditionally printing on failure is possible from teardown but not from from teardown_file
|
||||
cat ${BATS_FILE_TMPDIR}/zot.log
|
||||
}
|
||||
|
||||
function teardown_file() {
|
||||
zot_stop_all
|
||||
}
|
||||
|
||||
|
||||
@test "push image user policy" {
|
||||
run skopeo --insecure-policy copy --dest-creds test:test --dest-tls-verify=false \
|
||||
oci:${TEST_DATA_DIR}/golang:1.18 \
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Note: Intended to be run as "make test-bats-metrics" or "make test-bats-metrics-verbose"
|
||||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
|
|
@ -59,6 +59,11 @@ EOF
|
|||
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
# conditionally printing on failure is possible from teardown but not from from teardown_file
|
||||
cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
|
||||
}
|
||||
|
||||
function teardown_file() {
|
||||
zot_stop_all
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Note: Intended to be run as "make test-push-pull" or "test-push-pull-verbose"
|
||||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
|
|
@ -42,7 +42,8 @@ function setup_file() {
|
|||
"port": "8080"
|
||||
},
|
||||
"log": {
|
||||
"level": "debug"
|
||||
"level": "debug",
|
||||
"output": "${BATS_FILE_TMPDIR}/zot.log"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
|
@ -51,6 +52,11 @@ EOF
|
|||
wait_zot_reachable 8080
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
# conditionally printing on failure is possible from teardown but not from from teardown_file
|
||||
cat ${BATS_FILE_TMPDIR}/zot.log
|
||||
}
|
||||
|
||||
function teardown_file() {
|
||||
zot_stop_all
|
||||
}
|
||||
|
|
|
|||
|
|
@ -69,7 +69,8 @@ function setup_file() {
|
|||
}
|
||||
},
|
||||
"log":{
|
||||
"level":"debug"
|
||||
"level":"debug",
|
||||
"output": "${BATS_FILE_TMPDIR}/zot.log"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
|
@ -77,6 +78,11 @@ EOF
|
|||
wait_zot_reachable 8080
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
# conditionally printing on failure is possible from teardown but not from from teardown_file
|
||||
cat ${BATS_FILE_TMPDIR}/zot.log
|
||||
}
|
||||
|
||||
function teardown_file() {
|
||||
zot_stop_all
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Note: Intended to be run as "make test-push-pull-running-dedupe" or "test-push-pull-running-dedupe-verbose"
|
||||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-dedupe-nightly"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
|
|
@ -45,7 +45,8 @@ function setup_file() {
|
|||
"port": "8080"
|
||||
},
|
||||
"log": {
|
||||
"level": "debug"
|
||||
"level": "debug",
|
||||
"output": "${BATS_FILE_TMPDIR}/zot.log"
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
|
@ -54,6 +55,11 @@ EOF
|
|||
wait_zot_reachable 8080
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
# conditionally printing on failure is possible from teardown but not from from teardown_file
|
||||
cat ${BATS_FILE_TMPDIR}/zot.log
|
||||
}
|
||||
|
||||
function teardown_file() {
|
||||
zot_stop_all
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Note: Intended to be run as "make test-bats-referrers"
|
||||
# Note: Intended to be run as "make run-blackbox-ci"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
|
|
@ -87,6 +87,11 @@ EOF
|
|||
curl -X GET http://127.0.0.1:8080/v2/golang/referrers/${MANIFEST_DIGEST}?artifactType=image.artifact/type
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
# conditionally printing on failure is possible from teardown but not from from teardown_file
|
||||
cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
|
||||
}
|
||||
|
||||
function teardown_file() {
|
||||
zot_stop_all
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-dedupe-nightly"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
load helpers_cloud
|
||||
load helpers_wait
|
||||
|
||||
|
|
@ -81,6 +85,11 @@ EOF
|
|||
wait_zot_reachable 8080
|
||||
}
|
||||
|
||||
function teardown() {
|
||||
# conditionally printing on failure is possible from teardown but not from from teardown_file
|
||||
cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
|
||||
}
|
||||
|
||||
function teardown_file() {
|
||||
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
|
||||
zot_stop
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Note: Intended to be run as "make test-bats-scrub" or "make test-bats-scrub-verbose"
|
||||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
|
|
@ -53,11 +53,10 @@ EOF
|
|||
}
|
||||
|
||||
function teardown() {
|
||||
cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
|
||||
zot_stop_all
|
||||
}
|
||||
|
||||
|
||||
|
||||
@test "blobs/manifest integrity not affected" {
|
||||
add_test_files
|
||||
echo ${ZOT_CONFIG_FILE}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Note: Intended to be run as "make test-bats-sync" or "make test-bats-sync-verbose"
|
||||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Note: Intended to be run as "make test-bats-sync" or "make test-bats-sync-verbose"
|
||||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Note: Intended to be run as "make test-sync-harness" or "make test-sync-harness-verbose"
|
||||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-sync-nightly"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
# Note: Intended to be run as "make test-bats-sync" or "make test-bats-sync-verbose"
|
||||
# Note: Intended to be run as "make run-blackbox-tests" or "make run-blackbox-ci"
|
||||
# Makefile target installs & checks all necessary tooling
|
||||
# Extra tools that are not covered in Makefile target needs to be added in verify_prerequisites()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue