Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00
Code Issues Activity

Manage builds with different combinations of extensions

Browse source 
Files were added to be built whether an extension is on or off.
New build tags were added for each extension, while minimal and extended disappeared.

added custom binary naming depending on extensions used and changed references from binary to binary-extended

added automated blackbox tests for sync, search, scrub, metrics

added contributor guidelines

Signed-off-by: Alex Stan <alexandrustan96@yahoo.ro>
This commit is contained in:
Alex Stan 2022-04-27 09:00:20 +03:00 committed by Ramkumar Chinchani
parent 616d5f8a6d
commit ada21ed842
67 changed files with 1332 additions and 266 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/codeql-analysis.yml vendored
View file

@ -37,7 +37,7 @@ jobs:
# https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
env:
CGO_ENABLED: 0
GOFLAGS: "-tags=extended,containers_image_openpgp"
GOFLAGS: "-tags=sync,search,scrub,metrics,ui_base,containers_image_openpgp"
steps:
- name: Checkout repository

12
.github/workflows/ecosystem-tools.yaml vendored
View file

@ -44,3 +44,15 @@ jobs:
- name: Run push-pull tests
run: |
make push-pull
- name: Run metrics tests
run: |
make bats-metrics
- name: Run cve tests
run: |
make bats-cve
- name: Run sync test
run: |
make bats-sync
- name: Run scrub tests
run: |
make bats-scrub

2
.github/workflows/golangci-lint.yaml vendored
View file

@ -31,7 +31,7 @@ jobs:
# Optional: golangci-lint command line arguments.
# args: --issues-exit-code=0
args: --config ./golangcilint.yaml --enable-all --build-tags extended,containers_image_openpgp ./cmd/... ./pkg/...
args: --config ./golangcilint.yaml --enable-all --build-tags sync,scrub,search,metrics,ui_base,containers_image_openpgp ./cmd/... ./pkg/...
# Optional: show only new issues if it's a pull request. The default value is `false`.
# only-new-issues: true

3
.gitignore vendored
View file

@ -20,4 +20,5 @@ test/data/
coverage.html
tags
vendor/
.vscode/
.vscode/
examples/config-sync-localhost.json

9
CONTRIBUTING.md
View file

@ -44,6 +44,15 @@ For a minimal dist-spec only zot,
make binary-minimal
```
For a zot that includes only the extensions that you specify,
the available extensions that can be used at the moment are: sync, scrub, metrics, search, ui_base .
NOTES: When multiple extensions are used, they should be enlisted in the above presented order.
```
make binary EXTENSIONS=a,b,c
```
For a node exporter used by minimal dist-spec only zot,
```

102
Makefile
View file

@ -16,6 +16,10 @@ TESTDATA := $(TOP_LEVEL)/test/data
OS ?= linux
ARCH ?= amd64
BENCH_OUTPUT ?= stdout
EXTENSIONS ?= sync,search,scrub,metrics,ui_base
comma:= ,
hyphen:= -
extended-name:=
.PHONY: all
all: modcheck swagger binary binary-minimal binary-debug cli bench exporter-minimal verify-config test covhtml check
@ -24,42 +28,48 @@ all: modcheck swagger binary binary-minimal binary-debug cli bench exporter-mini
modcheck:
go mod tidy
.PHONY: create-name
create-name:
ifdef EXTENSIONS
$(eval extended-name=-$(subst $(comma),$(hyphen),$(EXTENSIONS)))
endif
.PHONY: binary
binary: modcheck swagger create-name
env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zot-$(OS)-$(ARCH) -buildmode=pie -tags $(EXTENSIONS),containers_image_openpgp -v -trimpath -ldflags "-X zotregistry.io/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.io/zot/pkg/api/config.BinaryType=$(extended-name) -X zotregistry.io/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zot
.PHONY: binary-debug
binary-debug: modcheck swagger
env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zot-$(OS)-$(ARCH)-debug -buildmode=pie -tags extended,containers_image_openpgp -v -gcflags all='-N -l' -ldflags "-X zotregistry.io/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.io/zot/pkg/api/config.BinaryType=extended -X zotregistry.io/zot/pkg/api/config.GoVersion=${GO_VERSION}" ./cmd/zot
binary-debug: modcheck swagger create-name
env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zot-$(OS)-$(ARCH)-debug -buildmode=pie -tags $(EXTENSIONS),containers_image_openpgp -v -gcflags all='-N -l' -ldflags "-X zotregistry.io/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.io/zot/pkg/api/config.BinaryType=$(extended-name) -X zotregistry.io/zot/pkg/api/config.GoVersion=${GO_VERSION}" ./cmd/zot
.PHONY: binary-minimal
binary-minimal: modcheck swagger
env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zot-$(OS)-$(ARCH)-minimal -buildmode=pie -tags minimal,containers_image_openpgp -v -trimpath -ldflags "-X zotregistry.io/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.io/zot/pkg/api/config.BinaryType=minimal -X zotregistry.io/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zot
.PHONY: binary
binary: modcheck swagger
env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zot-$(OS)-$(ARCH) -buildmode=pie -tags extended,containers_image_openpgp -v -trimpath -ldflags "-X zotregistry.io/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.io/zot/pkg/api/config.BinaryType=extended -X zotregistry.io/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zot
env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zot-$(OS)-$(ARCH)-minimal -buildmode=pie -tags containers_image_openpgp -v -trimpath -ldflags "-X zotregistry.io/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.io/zot/pkg/api/config.BinaryType=minimal -X zotregistry.io/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zot
.PHONY: cli
cli: modcheck
env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zli-$(OS)-$(ARCH) -buildmode=pie -tags extended,containers_image_openpgp -v -trimpath -ldflags "-X zotregistry.io/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.io/zot/pkg/api/config.BinaryType=extended -X zotregistry.io/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zli
cli: modcheck create-name
env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zli-$(OS)-$(ARCH) -buildmode=pie -tags $(EXTENSIONS),ui_base,containers_image_openpgp -v -trimpath -ldflags "-X zotregistry.io/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.io/zot/pkg/api/config.BinaryType=$(extended-name) -X zotregistry.io/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zli
.PHONY: bench
bench: modcheck
env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zb-$(OS)-$(ARCH) -buildmode=pie -tags extended,containers_image_openpgp -v -trimpath -ldflags "-X zotregistry.io/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.io/zot/pkg/api/config.BinaryType=extended -X zotregistry.io/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zb
bench: modcheck create-name
env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zb-$(OS)-$(ARCH) -buildmode=pie -tags $(EXTENSIONS),containers_image_openpgp -v -trimpath -ldflags "-X zotregistry.io/zot/pkg/api/config.Commit=${COMMIT} -X zotregistry.io/zot/pkg/api/config.BinaryType=$(extended-name) -X zotregistry.io/zot/pkg/api/config.GoVersion=${GO_VERSION} -s -w" ./cmd/zb
.PHONY: exporter-minimal
exporter-minimal: modcheck
env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zxp-$(OS)-$(ARCH) -buildmode=pie -tags minimal,containers_image_openpgp -v -trimpath ./cmd/zxp
env CGO_ENABLED=0 GOOS=$(OS) GOARCH=$(ARCH) go build -o bin/zxp-$(OS)-$(ARCH) -buildmode=pie -tags containers_image_openpgp -v -trimpath ./cmd/zxp
.PHONY: test
test: check-skopeo $(TESTDATA) $(NOTATION)
go test -failfast -tags extended,containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-extended.txt -covermode=atomic ./...
go test -failfast -tags minimal,containers_image_openpgp -v -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-minimal.txt -covermode=atomic ./...
go test -failfast -tags $(EXTENSIONS),containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-extended.txt -covermode=atomic ./...
go test -failfast -tags containers_image_openpgp -v -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-minimal.txt -covermode=atomic ./...
# development-mode unit tests possibly using failure injection
go test -failfast -tags dev,extended,containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-extended.txt -covermode=atomic ./pkg/test/... ./pkg/api/... ./pkg/storage/... ./pkg/extensions/sync/... -run ^TestInject
go test -failfast -tags dev,minimal,containers_image_openpgp -v -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-dev-minimal.txt -covermode=atomic ./pkg/test/... ./pkg/storage/... ./pkg/extensions/sync/... -run ^TestInject
go test -failfast -tags stress,extended,containers_image_openpgp -v -trimpath -race -timeout 15m ./pkg/cli/stress_test.go
go test -failfast -tags dev,$(EXTENSIONS),containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-extended.txt -covermode=atomic ./pkg/test/... ./pkg/api/... ./pkg/storage/... ./pkg/extensions/sync/... -run ^TestInject
go test -failfast -tags dev,containers_image_openpgp -v -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-dev-minimal.txt -covermode=atomic ./pkg/test/... ./pkg/storage/... ./pkg/extensions/sync/... -run ^TestInject
go test -failfast -tags stress,$(EXTENSIONS),containers_image_openpgp -v -trimpath -race -timeout 15m ./pkg/cli/stress_test.go
.PHONY: privileged-test
privileged-test: check-skopeo $(TESTDATA) $(NOTATION)
go test -failfast -tags needprivileges,extended,containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-needprivileges.txt -covermode=atomic ./pkg/storage/... ./pkg/cli/... -run ^TestElevatedPrivileges
go test -failfast -tags needprivileges,$(EXTENSIONS),containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-needprivileges.txt -covermode=atomic ./pkg/storage/... ./pkg/cli/... -run ^TestElevatedPrivileges
$(TESTDATA): check-skopeo
$(shell mkdir -p ${TESTDATA}; cd ${TESTDATA}; ../scripts/gen_certs.sh; cd ${TOP_LEVEL}; skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:7 oci:${TESTDATA}/zot-test:0.0.1;skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:8 oci:${TESTDATA}/zot-cve-test:0.0.1)
@ -95,11 +105,11 @@ $(GOLINTER):
.PHONY: check
check: ./golangcilint.yaml $(GOLINTER)
$(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags minimal,containers_image_openpgp ./...
$(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags extended,containers_image_openpgp ./...
$(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags dev,minimal,containers_image_openpgp ./...
$(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags dev,extended,containers_image_openpgp ./...
$(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags stress,extended,containers_image_openpgp ./...
$(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags containers_image_openpgp ./...
$(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags $(EXTENSIONS),containers_image_openpgp ./...
$(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags dev,containers_image_openpgp ./...
$(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags dev,$(EXTENSIONS),containers_image_openpgp ./...
$(GOLINTER) --config ./golangcilint.yaml run --enable-all --out-format=colored-line-number --build-tags stress,$(EXTENSIONS),containers_image_openpgp ./...
swagger/docs.go:
swag -v || go install github.com/swaggo/swag/cmd/swag@1.6.3
@ -117,7 +127,7 @@ update-licenses:
.PHONY: check-licenses
check-licenses:
go install github.com/google/go-licenses@latest
@for tag in "extended,containers_image_openpgp" "minimal,containers_image_openpgp"; do \
@for tag in "$(EXTENSIONS),containers_image_openpgp" "$(EXTENSIONS),containers_image_openpgp"; do \
echo Evaluating tag: $$tag;\
for mod in $$(go list -m -f '{{if not (or .Indirect .Main)}}{{.Path}}{{end}}' all); do \
while [ x$$mod != x ]; do \
@ -204,8 +214,48 @@ $(BATS):
.PHONY: push-pull
push-pull: binary check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox
$(BATS) --trace --print-output-on-failure test/blackbox/pushpull.bats
.PHONY: push-pull-verbose
push-pull-verbose: binary check-skopeo $(BATS)
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull.bats
.PHONY: bats-sync
bats-sync: EXTENSIONS=sync
bats-sync: binary binary-minimal check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox/sync.bats
.PHONY: bats-sync-verbose
bats-sync-verbose: EXTENSIONS=sync
bats-sync-verbose: binary binary-minimal check-skopeo $(BATS)
$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync.bats
.PHONY: bats-cve
bats-cve: EXTENSIONS=ui_base
bats-cve: binary cli check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox/cve.bats
.PHONY: bats-cve-verbose
bats-cve-verbose: EXTENSIONS=ui_base
bats-cve-verbose: binary cli check-skopeo $(BATS)
$(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/cve.bats
.PHONY: bats-scrub
bats-scrub: EXTENSIONS=scrub
bats-scrub: binary check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox/scrub.bats
.PHONY: bats-scrub-verbose
bats-scrub-verbose: EXTENSIONS=scrub
bats-scrub-verbose: binary check-skopeo $(BATS)
$(BATS) --trace -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/scrub.bats
.PHONY: bats-metrics
bats-metrics: EXTENSIONS=metrics
bats-metrics: binary check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox/metrics.bats
.PHONY: bats-metrics-verbose
bats-metrics-verbose: EXTENSIONS=metrics
bats-metrics-verbose: binary check-skopeo $(BATS)
$(BATS) --trace -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/metrics.bats

10
README.md
View file

@ -10,9 +10,13 @@ https://zotregistry.io
**Check the [package repository](https://github.com/orgs/project-zot/packages?repo_name=zot) for your os/arch**
The following document refers on the **core dist-spec**, see also the [zot-specific extensions spec](pkg/extensions/README.md)
## [**Why zot?**](COMPARISON.md)
## What's new?
* Selectively add extensions on top of minimal build
* Supports container image signatures - [cosign](https://github.com/sigstore/cosign) and [notation](https://github.com/notaryproject/notation)
* Multi-arch support
* Clustering support
@ -96,6 +100,12 @@ make binary-stacker
make
```
* Build zot with specified extensions
```
make binary EXTENSIONS=extension1,extension2,extension3
# e.g. make binary EXTENSIONS=sync,search,metrics,scrub
```
Build artifacts are in bin/
# Serving

4
cmd/zxp/main.go
View file

@ -1,5 +1,5 @@
//go:build minimal
// +build minimal
//go:build !metrics
// +build !metrics
package main

37
examples/config-sync-localhost.json Normal file
View file

@ -0,0 +1,37 @@
{
"distspecversion":"1.0.1-dev",
"storage": {
"rootDirectory": "/tmp/zot_to_sync",
"dedupe": false,
"gc": false
},
"http": {
"address": "127.0.0.1",
"port": "8081"
},
"log": {
"level": "debug"
},
"extensions": {
"sync": {
"registries": [
{
"urls": [
"http://localhost:8080"
],
"onDemand": true,
"tlsVerify": false,
"PollInterval": "30s",
"content": [
{
"prefix": "**"
}
]
}
]
},
"scrub": {
"interval": "24h"
}
}
}

8
pkg/api/controller.go
View file

@ -353,21 +353,23 @@ func (c *Controller) Shutdown() {
func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
// Enable extensions if extension config is provided for DefaultStore
if c.Config != nil && c.Config.Extensions != nil {
ext.EnableExtensions(c.Config, c.Log, c.Config.Storage.RootDirectory)
ext.EnableMetricsExtension(c.Config, c.Log, c.Config.Storage.RootDirectory)
ext.EnableSearchExtension(c.Config, c.Log, c.Config.Storage.RootDirectory)
}
if c.Config.Storage.SubPaths != nil {
for _, storageConfig := range c.Config.Storage.SubPaths {
// Enable extensions if extension config is provided for subImageStore
if c.Config != nil && c.Config.Extensions != nil {
ext.EnableExtensions(c.Config, c.Log, storageConfig.RootDirectory)
ext.EnableMetricsExtension(c.Config, c.Log, storageConfig.RootDirectory)
ext.EnableSearchExtension(c.Config, c.Log, storageConfig.RootDirectory)
}
}
}
// Enable extensions if extension config is provided for storeController
if c.Config.Extensions != nil {
if c.Config.Extensions.Sync != nil && *c.Config.Extensions.Sync.Enable {
if c.Config.Extensions.Sync != nil {
ext.EnableSyncExtension(reloadCtx, c.Config, c.wgShutDown, c.StoreController, c.Log)
}
}

4
pkg/api/controller_test.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build sync && scrub && metrics && search && ui_base
// +build sync,scrub,metrics,search,ui_base
package api_test

3
pkg/api/routes.go
View file

@ -121,7 +121,8 @@ func (rh *RouteHandler) SetupRoutes() {
prefixedRouter.HandleFunc("/metrics", rh.GetMetrics).Methods("GET")
} else {
// extended build
ext.SetupRoutes(rh.c.Config, rh.c.Router, rh.c.StoreController, rh.c.Log)
ext.SetupMetricsRoutes(rh.c.Config, rh.c.Router, rh.c.StoreController, rh.c.Log)
ext.SetupSearchRoutes(rh.c.Config, rh.c.Router, rh.c.StoreController, rh.c.Log)
}
}
}

4
pkg/api/routes_test.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build sync && scrub && metrics && search && ui_base
// +build sync,scrub,metrics,search,ui_base
package api_test

4
pkg/cli/cli.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build ui_base || search
// +build ui_base search
package cli

4
pkg/cli/client.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build search || ui_base
// +build search ui_base
package cli

4
pkg/cli/client_elevated_test.go
View file

@ -1,5 +1,5 @@
//go:build extended && needprivileges
// +build extended,needprivileges
//go:build ui_base && needprivileges
// +build ui_base,needprivileges
package cli //nolint:testpackage

4
pkg/cli/client_test.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build ui_base
// +build ui_base
package cli //nolint:testpackage

4
pkg/cli/config_cmd.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build search || ui_base
// +build search ui_base
package cli

4
pkg/cli/config_cmd_test.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build ui_base
// +build ui_base
package cli //nolint:testpackage

4
pkg/cli/cve_cmd.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build search || ui_base
// +build search ui_base
package cli

4
pkg/cli/cve_cmd_test.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build ui_base
// +build ui_base
package cli //nolint:testpackage

4
pkg/cli/extensions_test.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build sync && scrub && metrics && search && ui_base
// +build sync,scrub,metrics,search,ui_base
package cli_test

4
pkg/cli/image_cmd.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build search || ui_base
// +build search ui_base
package cli

4
pkg/cli/image_cmd_test.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build ui_base
// +build ui_base
package cli //nolint:testpackage

4
pkg/cli/minimal.go
View file

@ -1,5 +1,5 @@
//go:build minimal
// +build minimal
//go:build !search && !ui_base
// +build !search,!ui_base
package cli

4
pkg/cli/repo_cmd.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build search || ui_base
// +build search ui_base
package cli

4
pkg/cli/searcher.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build search || ui_base
// +build search ui_base
package cli

4
pkg/cli/service.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build ui_base || search
// +build ui_base search
package cli

4
pkg/exporter/api/config.go
View file

@ -1,5 +1,5 @@
//go:build minimal
// +build minimal
//go:build !metrics
// +build !metrics
package api

4
pkg/exporter/api/controller.go
View file

@ -1,5 +1,5 @@
//go:build minimal
// +build minimal
//go:build !metrics
// +build !metrics
package api

4
pkg/exporter/api/controller_test.go
View file

@ -1,5 +1,5 @@
//go:build minimal
// +build minimal
//go:build !metrics
// +build !metrics
package api_test

4
pkg/exporter/api/exporter.go
View file

@ -1,5 +1,5 @@
//go:build minimal
// +build minimal
//go:build !metrics
// +build !metrics
// nolint: varnamelen
package api

4
pkg/exporter/cli/cli.go
View file

@ -1,5 +1,5 @@
//go:build minimal
// +build minimal
//go:build !metrics
// +build !metrics
package cli

31
pkg/extensions/README.md Normal file
View file

@ -0,0 +1,31 @@
## Adding new extensions
As new requirements come and build time extensions need to be added, there are a few things that you have to make sure are present before commiting :
- files that should be included in the binary only with a specific extension must contain the following syntax at the beginning of the file :
//go:build sync will be added automatically by the linter, so only the second line is mandatory .
NOTE: the third line in the example should be blank, otherwise the build tag would be just another comment.
```
//go:build sync
// +build sync
package extensions
...................
```
- when adding a new tag, specify the new order in which multiple tags should be used (bottom of this page)
- for each and every new file that contains functions (functionalities) specific to an extension, one should create a corresponding file that <b>must contain the exact same functions, but no functionalities included</b>. This file must begin with an "anti-tag" (e.g. // +build !sync) which will include this file in binaries that don't include this extension ( in this example, the file won't be used in binaries that include sync extension ). See [extension-sync-disabled.go](extension-sync-disabled.go) for an example.
- when a new extension comes out, the developer should also write some blackbox tests, where a binary that contains the new extension should be tested in a real usage scenario. See [test/blackbox](test/blackbox/sync.bats) folder for multiple extensions examples.
- newly added blackbox tests should have targets in Makefile. You should also add them as Github Workflows, in [.github/workflows/ecosystem-tools.yaml](.github/workflows/ecosystem-tools.yaml)
- with every new extension, you should modify the EXTENSIONS variable in Makefile by adding the new extension. The EXTENSIONS variable represents all extensions and is used in Make targets that require them all (e.g make test).
- the available extensions that can be used at the moment are: <b>sync, scrub, metrics, search, ui_base </b>.
NOTE: When multiple extensions are used, they should be enlisted in the above presented order.

25
pkg/extensions/extension-metrics-disabled.go Normal file
View file

@ -0,0 +1,25 @@
//go:build !metrics
// +build !metrics
package extensions
import (
"github.com/gorilla/mux"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
// EnableMetricsExtension ...
func EnableMetricsExtension(config *config.Config, log log.Logger, rootDir string) {
log.Warn().Msg("skipping enabling metrics extension because given zot binary doesn't include this feature," +
"please build a binary that does so")
}
// SetupMetricsRoutes ...
func SetupMetricsRoutes(conf *config.Config, router *mux.Router,
storeController storage.StoreController, log log.Logger,
) {
log.Warn().Msg("skipping setting up metrics routes because given zot binary doesn't include this feature," +
"please build a binary that does so")
}

39
pkg/extensions/extension-metrics.go Normal file
View file

@ -0,0 +1,39 @@
//go:build metrics
// +build metrics
package extensions
import (
"github.com/gorilla/mux"
"github.com/prometheus/client_golang/prometheus/promhttp"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
func EnableMetricsExtension(config *config.Config, log log.Logger, rootDir string) {
if config.Extensions.Metrics != nil &&
*config.Extensions.Metrics.Enable &&
config.Extensions.Metrics.Prometheus != nil {
if config.Extensions.Metrics.Prometheus.Path == "" {
config.Extensions.Metrics.Prometheus.Path = "/metrics"
log.Warn().Msg("Prometheus instrumentation Path not set, changing to '/metrics'.")
}
} else {
log.Info().Msg("Metrics config not provided, skipping Metrics config update")
}
}
func SetupMetricsRoutes(config *config.Config, router *mux.Router, storeController storage.StoreController,
l log.Logger,
) {
// fork a new zerolog child to avoid data race
log := log.Logger{Logger: l.With().Caller().Timestamp().Logger()}
log.Info().Msg("setting up metrics routes")
if config.Extensions.Metrics != nil && *config.Extensions.Metrics.Enable {
router.PathPrefix(config.Extensions.Metrics.Prometheus.Path).
Handler(promhttp.Handler())
}
}

19
pkg/extensions/extension-scrub-disabled.go Normal file
View file

@ -0,0 +1,19 @@
//go:build !scrub
// +build !scrub
package extensions
import (
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
// EnableScrubExtension ...
func EnableScrubExtension(config *config.Config,
log log.Logger, run bool,
imgStore storage.ImageStore, repo string,
) {
log.Warn().Msg("skipping enabling scrub extension because given zot binary doesn't include this feature," +
"please build a binary that does so")
}

33
pkg/extensions/extension-scrub.go Normal file
View file

@ -0,0 +1,33 @@
//go:build scrub
// +build scrub
package extensions
import (
"time"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/extensions/scrub"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
// EnableScrubExtension enables scrub extension.
func EnableScrubExtension(config *config.Config, log log.Logger, run bool, imgStore storage.ImageStore, repo string) {
if !run {
if config.Extensions.Scrub != nil &&
config.Extensions.Scrub.Interval != 0 {
minScrubInterval, _ := time.ParseDuration("2h")
if config.Extensions.Scrub.Interval < minScrubInterval {
config.Extensions.Scrub.Interval = minScrubInterval
log.Warn().Msg("Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") //nolint:lll // gofumpt conflicts with lll
}
} else {
log.Info().Msg("Scrub config not provided, skipping scrub")
}
} else {
scrub.RunScrubRepo(imgStore, repo, log)
}
}

31
pkg/extensions/extension-search-disabled.go Normal file
View file

@ -0,0 +1,31 @@
//go:build !search && !ui_base
// +build !search,!ui_base
package extensions
import (
"github.com/gorilla/mux"
distext "github.com/opencontainers/distribution-spec/specs-go/v1/extensions"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
// EnableSearchExtension ...
func EnableSearchExtension(config *config.Config, log log.Logger, rootDir string) {
log.Warn().Msg("skipping enabling search extension because given zot binary doesn't include this feature," +
"please build a binary that does so")
}
// SetupSearchRoutes ...
func SetupSearchRoutes(conf *config.Config, router *mux.Router,
storeController storage.StoreController, log log.Logger,
) {
log.Warn().Msg("skipping setting up search routes because given zot binary doesn't include this feature," +
"please build a binary that does so")
}
// GetExtensions...
func GetExtensions(config *config.Config) distext.ExtensionList {
return distext.ExtensionList{}
}

123
pkg/extensions/extensions.go → pkg/extensions/extension-search.go
View file

@ -1,45 +1,23 @@
//go:build extended
// +build extended
//go:build search || ui_base
// +build search ui_base
package extensions
import (
"context"
"fmt"
goSync "sync"
"time"
gqlHandler "github.com/99designs/gqlgen/graphql/handler"
"github.com/gorilla/mux"
distext "github.com/opencontainers/distribution-spec/specs-go/v1/extensions"
"github.com/prometheus/client_golang/prometheus/promhttp"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/api/constants"
"zotregistry.io/zot/pkg/extensions/scrub"
"zotregistry.io/zot/pkg/extensions/search"
cveinfo "zotregistry.io/zot/pkg/extensions/search/cve"
"zotregistry.io/zot/pkg/extensions/sync"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
// DownloadTrivyDB ...
func downloadTrivyDB(dbDir string, log log.Logger, updateInterval time.Duration) error {
for {
log.Info().Msg("updating the CVE database")
err := cveinfo.UpdateCVEDb(dbDir, log)
if err != nil {
return err
}
log.Info().Str("DB update completed, next update scheduled after", updateInterval.String()).Msg("")
time.Sleep(updateInterval)
}
}
func EnableExtensions(config *config.Config, log log.Logger, rootDir string) {
func EnableSearchExtension(config *config.Config, log log.Logger, rootDir string) {
if config.Extensions.Search != nil && *config.Extensions.Search.Enable && config.Extensions.Search.CVE != nil {
defaultUpdateInterval, _ := time.ParseDuration("2h")
@ -59,51 +37,41 @@ func EnableExtensions(config *config.Config, log log.Logger, rootDir string) {
} else {
log.Info().Msg("CVE config not provided, skipping CVE update")
}
}
if config.Extensions.Metrics != nil &&
*config.Extensions.Metrics.Enable &&
config.Extensions.Metrics.Prometheus != nil {
if config.Extensions.Metrics.Prometheus.Path == "" {
config.Extensions.Metrics.Prometheus.Path = constants.DefaultMetricsExtensionRoute
func downloadTrivyDB(dbDir string, log log.Logger, updateInterval time.Duration) error {
for {
log.Info().Msg("updating the CVE database")
log.Warn().Msg(fmt.Sprintf("Prometheus instrumentation Path not set, changing to %s.",
constants.DefaultMetricsExtensionRoute))
err := cveinfo.UpdateCVEDb(dbDir, log)
if err != nil {
return err
}
} else {
log.Info().Msg("Metrics config not provided, skipping Metrics config update")
log.Info().Str("DB update completed, next update scheduled after", updateInterval.String()).Msg("")
time.Sleep(updateInterval)
}
}
// EnableSyncExtension enables sync extension.
func EnableSyncExtension(ctx context.Context, config *config.Config, wg *goSync.WaitGroup,
storeController storage.StoreController, log log.Logger,
func SetupSearchRoutes(config *config.Config, router *mux.Router, storeController storage.StoreController,
l log.Logger,
) {
if config.Extensions.Sync != nil && *config.Extensions.Sync.Enable {
if err := sync.Run(ctx, *config.Extensions.Sync, storeController, wg, log); err != nil {
log.Error().Err(err).Msg("Error encountered while setting up syncing")
}
} else {
log.Info().Msg("Sync registries config not provided or disabled, skipping sync")
}
}
// fork a new zerolog child to avoid data race
log := log.Logger{Logger: l.With().Caller().Timestamp().Logger()}
log.Info().Msg("setting up search routes")
// EnableScrubExtension enables scrub extension.
func EnableScrubExtension(config *config.Config, log log.Logger, run bool, imgStore storage.ImageStore, repo string) {
if !run {
if config.Extensions.Scrub != nil &&
config.Extensions.Scrub.Interval != 0 {
minScrubInterval, _ := time.ParseDuration("2h")
if config.Extensions.Search != nil && *config.Extensions.Search.Enable {
var resConfig search.Config
if config.Extensions.Scrub.Interval < minScrubInterval {
config.Extensions.Scrub.Interval = minScrubInterval
log.Warn().Msg("Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") //nolint:lll // gofumpt conflicts with lll
}
if config.Extensions.Search.CVE != nil {
resConfig = search.GetResolverConfig(log, storeController, true)
} else {
log.Info().Msg("Scrub config not provided, skipping scrub")
resConfig = search.GetResolverConfig(log, storeController, false)
}
} else {
scrub.RunScrubRepo(imgStore, repo, log)
router.PathPrefix(constants.ExtSearchPrefix).Methods("OPTIONS", "GET", "POST").
Handler(gqlHandler.NewDefaultServer(search.NewExecutableSchema(resConfig)))
}
}
@ -135,40 +103,3 @@ func GetExtensions(config *config.Config) distext.ExtensionList {
return extensionList
}
// SetupRoutes ...
func SetupRoutes(config *config.Config, router *mux.Router, storeController storage.StoreController, l log.Logger,
) {
// fork a new zerolog child to avoid data race
log := log.Logger{Logger: l.With().Caller().Timestamp().Logger()}
log.Info().Msg("setting up extensions routes")
if config.Extensions.Search != nil && *config.Extensions.Search.Enable {
var resConfig search.Config
if config.Extensions.Search.CVE != nil {
resConfig = search.GetResolverConfig(log, storeController, true)
} else {
resConfig = search.GetResolverConfig(log, storeController, false)
}
router.PathPrefix(constants.ExtSearchPrefix).Methods("OPTIONS", "GET", "POST").
Handler(gqlHandler.NewDefaultServer(search.NewExecutableSchema(resConfig)))
}
if config.Extensions.Metrics != nil && *config.Extensions.Metrics.Enable {
router.PathPrefix(config.Extensions.Metrics.Prometheus.Path).
Handler(promhttp.Handler())
}
}
// SyncOneImage syncs one image.
func SyncOneImage(config *config.Config, storeController storage.StoreController,
repoName, reference string, isArtifact bool, log log.Logger,
) error {
log.Info().Msgf("syncing image %s:%s", repoName, reference)
err := sync.OneImage(*config.Extensions.Sync, storeController, repoName, reference, isArtifact, log)
return err
}

32
pkg/extensions/extension-sync-disabled.go Normal file
View file

@ -0,0 +1,32 @@
//go:build !sync
// +build !sync
package extensions
import (
"context"
goSync "sync"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
// EnableSyncExtension ...
func EnableSyncExtension(ctx context.Context,
config *config.Config, wg *goSync.WaitGroup,
storeController storage.StoreController, log log.Logger,
) {
log.Warn().Msg("skipping enabling sync extension because given zot binary doesn't include this feature," +
"please build a binary that does so")
}
// SyncOneImage ...
func SyncOneImage(config *config.Config, storeController storage.StoreController,
repoName, reference string, isArtifact bool, log log.Logger,
) error {
log.Warn().Msg("skipping syncing on demand because given zot binary doesn't include this feature," +
"please build a binary that does so")
return nil
}

36
pkg/extensions/extension-sync.go Normal file
View file

@ -0,0 +1,36 @@
//go:build sync
// +build sync
package extensions
import (
"context"
goSync "sync"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/extensions/sync"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
func EnableSyncExtension(ctx context.Context, config *config.Config, wg *goSync.WaitGroup,
storeController storage.StoreController, log log.Logger,
) {
if config.Extensions.Sync != nil && *config.Extensions.Sync.Enable {
if err := sync.Run(ctx, *config.Extensions.Sync, storeController, wg, log); err != nil {
log.Error().Err(err).Msg("Error encountered while setting up syncing")
}
} else {
log.Info().Msg("Sync registries config not provided or disabled, skipping sync")
}
}
func SyncOneImage(config *config.Config, storeController storage.StoreController,
repoName, reference string, isArtifact bool, log log.Logger,
) error {
log.Info().Msgf("syncing image %s:%s", repoName, reference)
err := sync.OneImage(*config.Extensions.Sync, storeController, repoName, reference, isArtifact, log)
return err
}

109
pkg/extensions/extensions_test.go Normal file
View file

@ -0,0 +1,109 @@
//go:build sync || metrics
// +build sync metrics
package extensions_test
import (
"context"
"io/ioutil"
"os"
"testing"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/extensions/sync"
"zotregistry.io/zot/pkg/test"
)
func TestEnableExtension(t *testing.T) {
Convey("Verify log if sync disabled in config", t, func() {
globalDir := t.TempDir()
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
falseValue := false
syncConfig := &sync.Config{
Enable: &falseValue,
Registries: []sync.RegistryConfig{},
}
// conf.Extensions.Sync.Enable = &falseValue
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Sync = syncConfig
conf.HTTP.Port = port
logFile, err := ioutil.TempFile(globalDir, "zot-log*.txt")
So(err, ShouldBeNil)
conf.Log.Level = "info"
conf.Log.Output = logFile.Name()
defer os.Remove(logFile.Name()) // cleanup
ctlr := api.NewController(conf)
defer func() {
ctx := context.Background()
_ = ctlr.Server.Shutdown(ctx)
}()
ctlr.Config.Storage.RootDirectory = globalDir
go func() {
if err := ctlr.Run(context.Background()); err != nil {
return
}
}()
test.WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
"Sync registries config not provided or disabled, skipping sync")
})
}
func TestMetricsExtension(t *testing.T) {
Convey("Verify Metrics enabled for storage subpaths", t, func() {
globalDir := t.TempDir()
conf := config.New()
port := test.GetFreePort()
conf.HTTP.Port = port
baseURL := test.GetBaseURL(port)
logFile, err := ioutil.TempFile(globalDir, "zot-log*.txt")
So(err, ShouldBeNil)
defaultValue := true
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Metrics = &extconf.MetricsConfig{
Enable: &defaultValue,
Prometheus: &extconf.PrometheusConfig{},
}
conf.Log.Level = "info"
conf.Log.Output = logFile.Name()
defer os.Remove(logFile.Name()) // cleanup
ctlr := api.NewController(conf)
subPaths := make(map[string]config.StorageConfig)
subPaths["/a"] = config.StorageConfig{}
ctlr.Config.Storage.RootDirectory = globalDir
ctlr.Config.Storage.SubPaths = subPaths
go func() {
if err := ctlr.Run(context.Background()); err != nil {
return
}
}()
test.WaitTillServerReady(baseURL)
data, _ := os.ReadFile(logFile.Name())
So(string(data), ShouldContainSubstring,
"Prometheus instrumentation Path not set, changing to '/metrics'.")
})
}

63
pkg/extensions/minimal.go
View file

@ -1,63 +0,0 @@
//go:build minimal
// +build minimal
package extensions
import (
"context"
goSync "sync"
"time"
"github.com/gorilla/mux"
distext "github.com/opencontainers/distribution-spec/specs-go/v1/extensions"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
// nolint: deadcode,unused
func downloadTrivyDB(dbDir string, log log.Logger, updateInterval time.Duration) error {
return nil
}
// EnableExtensions ...
func EnableExtensions(config *config.Config, log log.Logger, rootDir string) {
log.Warn().Msg("skipping enabling extensions because given zot binary doesn't support " +
"any extensions, please build zot full binary for this feature")
}
// GetExtensions...
func GetExtensions(config *config.Config) distext.ExtensionList {
return distext.ExtensionList{}
}
// EnableSyncExtension ...
func EnableSyncExtension(ctx context.Context, config *config.Config, wg *goSync.WaitGroup,
storeController storage.StoreController, log log.Logger,
) {
log.Warn().Msg("skipping enabling sync extension because given zot binary doesn't support any extensions," +
"please build zot full binary for this feature")
}
// EnableScrubExtension ...
func EnableScrubExtension(config *config.Config, log log.Logger, run bool, imgStore storage.ImageStore, repo string) {
log.Warn().Msg("skipping enabling scrub extension because given zot binary doesn't support any extensions," +
"please build zot full binary for this feature")
}
// SetupRoutes ...
func SetupRoutes(conf *config.Config, router *mux.Router, storeController storage.StoreController, log log.Logger,
) {
log.Warn().Msg("skipping setting up extensions routes because given zot binary doesn't support " +
"any extensions, please build zot full binary for this feature")
}
// SyncOneImage ...
func SyncOneImage(config *config.Config, storeController storage.StoreController,
repoName, reference string, isArtifact bool, log log.Logger,
) error {
log.Warn().Msg("skipping syncing on demand because given zot binary doesn't support any extensions," +
"please build zot full binary for this feature")
return nil
}

4
pkg/extensions/monitoring/extension.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build metrics
// +build metrics
package monitoring

4
pkg/extensions/monitoring/minimal.go
View file

@ -1,5 +1,5 @@
//go:build minimal
// +build minimal
//go:build !metrics
// +build !metrics
// nolint: varnamelen,forcetypeassert
package monitoring

4
pkg/extensions/monitoring/minimal_client.go
View file

@ -1,5 +1,5 @@
//go:build minimal
// +build minimal
//go:build !metrics
// +build !metrics
package monitoring

4
pkg/extensions/monitoring/monitoring_test.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build metrics
// +build metrics
package monitoring_test

4
pkg/extensions/scrub/scrub.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build scrub
// +build scrub
package scrub

4
pkg/extensions/scrub/scrub_test.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build scrub
// +build scrub
package scrub_test

4
pkg/extensions/search/common/common_test.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build search
// +build search
package common_test

4
pkg/extensions/search/cve/cve_test.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build search
// +build search
// nolint:lll,gosimple
package cveinfo_test

4
pkg/extensions/search/digest/digest_test.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build search
// +build search
// nolint: gochecknoinits
package digestinfo_test

71
pkg/extensions/sync/sync_disabled_test.go Normal file
View file

@ -0,0 +1,71 @@
//go:build !sync
// +build !sync
package sync_test
import (
"context"
"io/ioutil"
"os"
"testing"
. "github.com/smartystreets/goconvey/convey"
"gopkg.in/resty.v1"
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/extensions/sync"
"zotregistry.io/zot/pkg/test"
)
func TestSyncExtension(t *testing.T) {
Convey("Make a new controller", t, func() {
conf := config.New()
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
globalDir := t.TempDir()
defaultValue := true
logFile, err := ioutil.TempFile(globalDir, "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name())
conf.HTTP.Port = port
conf.Storage.RootDirectory = globalDir
conf.Storage.Commit = true
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Sync = &sync.Config{
Enable: &defaultValue,
}
conf.Log.Level = "warn"
conf.Log.Output = logFile.Name()
ctlr := api.NewController(conf)
go func() {
if err := ctlr.Run(context.Background()); err != nil {
return
}
}()
defer func() {
_ = ctlr.Server.Shutdown(context.Background())
}()
test.WaitTillServerReady(baseURL)
Convey("verify sync is skipped when binary doesn't include it", func() {
resp, err := resty.R().
Head(baseURL + "/v2/" + "invalid" + "/manifests/invalid:0.0.2")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
"skipping syncing on demand because given zot binary doesn't include "+
"this feature,please build a binary that does so")
})
})
}

4
pkg/extensions/sync/sync_test.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build sync
// +build sync
package sync_test

4
pkg/log/log_test.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build sync && scrub && metrics && search && ui_base
// +build sync,scrub,metrics,search,ui_base
package log_test

4
pkg/test/common_test.go
View file

@ -1,5 +1,5 @@
//go:build extended
// +build extended
//go:build sync && scrub && metrics && search && ui_base
// +build sync,scrub,metrics,search,ui_base
package test_test

68
test/blackbox/cve.bats Normal file
View file

@ -0,0 +1,68 @@
load helpers_cve
function setup_file() {
# Verify prerequisites are available
if ! verify_prerequisites; then
exit 1
fi
# Download test data to folder common for the entire suite, not just this file
skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.17 oci:${TEST_DATA_DIR}/golang:1.17
# Setup zot server
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json
mkdir -p ${zot_root_dir}
cat >${zot_config_file} <<EOF
{
"distSpecVersion": "1.0.1",
"storage": {
"rootDirectory": "${zot_root_dir}"
},
"http": {
"address": "0.0.0.0",
"port": "8080",
"ReadOnly": false
},
"log": {
"level": "debug"
},
"extensions": {
"search": {
"enable": true,
"cve": {
"updateInterval": "24h"
}
}
}
}
EOF
setup_zot_file_level ${zot_config_file}
wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
# setup zli to add zot registry to configs
local registry_name=main
local registry_url="http://127.0.0.1:8080/"
zli_add_config ${registry_name} ${registry_url}
}
function teardown_file() {
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
teardown_zot_file_level
rm -rf ${zot_root_dir}
}
@test "cve by image name and tag" {
run skopeo --insecure-policy copy --dest-tls-verify=false \
oci:${TEST_DATA_DIR}/golang:1.17 \
docker://127.0.0.1:8080/golang:1.17
[ "$status" -eq 0 ]
run curl http://127.0.0.1:8080/v2/_catalog
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.repositories[]') = '"golang"' ]
run curl http://127.0.0.1:8080/v2/golang/tags/list
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.tags[]') = '"1.17"' ]
run ${ZLI_PATH} cve ${REGISTRY_NAME} -I golang:1.17
[ "$status" -eq 0 ]
}

84
test/blackbox/helpers_cve.bash Normal file
View file

@ -0,0 +1,84 @@
ROOT_DIR=$(git rev-parse --show-toplevel)
TEST_DATA_DIR=${ROOT_DIR}/test/data/
OS="${OS:-linux}"
ARCH="${ARCH:-amd64}"
ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}
ZLI_PATH=${ROOT_DIR}/bin/zli-${OS}-${ARCH}
REGISTRY_NAME=main
function verify_prerequisites {
if [ ! -f ${BATS_RUN_TMPDIR}/.firstrun ]; then
env | grep proxy >&3
touch ${BATS_RUN_TMPDIR}/.firstrun
fi
if [ ! -f ${ZOT_PATH} ]; then
echo "you need to build ${ZOT_PATH} before running the tests" >&3
return 1
fi
if [ ! -f ${ZLI_PATH} ]; then
echo "you need to build ${ZLI} before running tests" >&3
return 1
fi
if [ ! command -v curl ] &>/dev/null; then
echo "you need to install curl as a prerequisite to running the tests" >&3
return 1
fi
if [ ! command -v jq ] &>/dev/null; then
echo "you need to install jq as a prerequisite to running the tests" >&3
return 1
fi
if [ ! command -v skopeo ] &>/dev/null; then
echo "you need to install skopeo as a prerequisite to running the tests" >&3
return 1
fi
return 0
}
function zot_serve() {
local zot_path=${1}
local config_file=${2}
local pid_dir=${3}
${zot_path} serve ${config_file} &
echo $! >>${pid_dir}/zot.pid
}
function zot_stop() {
local pid_dir=${1}
cat ${pid_dir}/zot.pid
kill $(cat ${pid_dir}/zot.pid)
rm ${pid_dir}/zot.pid
}
function setup_zot_file_level() {
local config_file=${1}
zot_serve ${ZOT_PATH} ${config_file} ${BATS_FILE_TMPDIR}
}
function teardown_zot_file_level() {
zot_stop ${BATS_FILE_TMPDIR}
}
function wait_zot_reachable() {
zot_url=${1}
curl --connect-timeout 3 \
--max-time 3 \
--retry 10 \
--retry-delay 0 \
--retry-max-time 60 \
--retry-connrefused \
${zot_url}
}
function zli_add_config() {
local registry_name=${1}
local registry_url=${2}
if ! ${ZLI_PATH} config --list | grep -q main; then
${ZLI_PATH} config add ${registry_name} ${registry_url}
fi
}

63
test/blackbox/helpers_metrics.bash Normal file
View file

@ -0,0 +1,63 @@
ROOT_DIR=$(git rev-parse --show-toplevel)
TEST_DATA_DIR=${ROOT_DIR}/test/data/
OS="${OS:-linux}"
ARCH="${ARCH:-amd64}"
ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}
function verify_prerequisites() {
if [ ! -f ${BATS_RUN_TMPDIR}/.firstrun ]; then
env | grep proxy >&3
touch ${BATS_RUN_TMPDIR}/.firstrun
fi
if [ ! command -v curl ] &>/dev/null; then
echo "you need to install curl as a prerequisite to running the tests" >&3
return 1
fi
if [ ! command -v jq ] &>/dev/null; then
echo "you need to install jq as a prerequisite to running the tests" >&3
return 1
fi
if [ ! command -v skopeo ] &>/dev/null; then
echo "you need to install skopeo as a prerequisite to running the tests" >&3
return 1
fi
return 0
}
function zot_serve() {
local zot_path=${1}
local config_file=${2}
local pid_dir=${3}
${zot_path} serve ${config_file} &
echo $! >>${pid_dir}/zot.pid
}
function zot_stop() {
local pid_dir=${1}
cat ${pid_dir}/zot.pid
kill $(cat ${pid_dir}/zot.pid)
rm ${pid_dir}/zot.pid
}
function setup_zot_file_level() {
local config_file=${1}
zot_serve ${ZOT_PATH} ${config_file} ${BATS_FILE_TMPDIR}
}
function teardown_zot_file_level() {
zot_stop ${BATS_FILE_TMPDIR}
}
function wait_zot_reachable() {
zot_url=${1}
curl --connect-timeout 3 \
--max-time 3 \
--retry 10 \
--retry-delay 0 \
--retry-max-time 60 \
--retry-connrefused \
${zot_url}
}

6
test/blackbox/helpers.bash → test/blackbox/helpers_pushpull.bash
View file

@ -22,17 +22,17 @@ function verify_prerequisites {
return 1
fi
if [ ! command -v jq &> /dev/null ]; then
if [ ! command -v jq ] &>/dev/null; then
echo "you need to install jq as a prerequisite to running the tests" >&3
return 1
fi
if [ ! command -v skopeo &> /dev/null ]; then
if [ ! command -v skopeo ] &>/dev/null; then
echo "you need to install skopeo as a prerequisite to running the tests" >&3
return 1
fi
if [ ! command -v oras &> /dev/null ]; then
if [ ! command -v oras ] &>/dev/null; then
echo "you need to install oras as a prerequisite to running the tests" >&3
return 1
fi

99
test/blackbox/helpers_scrub.bash Normal file
View file

@ -0,0 +1,99 @@
ROOT_DIR=$(git rev-parse --show-toplevel)
TEST_DATA_DIR=${ROOT_DIR}/test/data/
OS="${OS:-linux}"
ARCH="${ARCH:-amd64}"
ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}
ZOT_ROOT_DIR=?
ZOT_LOG_FILE=?
ZOT_CONFIG_FILE=
function verify_prerequisites() {
if [ ! -f ${BATS_RUN_TMPDIR}/.firstrun ]; then
env | grep proxy >&3
touch ${BATS_RUN_TMPDIR}/.firstrun
fi
if [ ! -f ${ZOT_PATH} ]; then
echo "you need to build ${ZOT_PATH} before running tests" >&3
return 1
fi
if [ ! command -v curl ] &>/dev/null; then
echo "you need to install curl as a prerequisite to running the tests" >&3
return 1
fi
if [ ! command -v jq ] &>/dev/null; then
echo "you need to install jq as a prerequisite to running the tests" >&3
return 1
fi
if [ ! command -v skopeo ] &>/dev/null; then
echo "you need to install skopeo as a prerequisite to running the tests" >&3
return 1
fi
return 0
}
function zot_serve() {
local zot_path=${1}
local config_file=${2}
local pid_dir=${3}
${zot_path} serve ${config_file} &
echo $! >>${pid_dir}/zot.pid
}
function zot_stop() {
local pid_dir=${1}
cat ${pid_dir}/zot.pid
kill $(cat ${pid_dir}/zot.pid)
rm ${pid_dir}/zot.pid
}
function setup_zot_file_level() {
local config_file=${1}
zot_serve ${ZOT_PATH} ${config_file} ${BATS_FILE_TMPDIR}
}
function teardown_zot_file_level() {
zot_stop ${BATS_FILE_TMPDIR}
}
function wait_zot_reachable() {
zot_url=${1}
curl --connect-timeout 3 \
--max-time 3 \
--retry 10 \
--retry-delay 0 \
--retry-max-time 60 \
--retry-connrefused \
${zot_url}
}
function add_test_files() {
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
echo ${zot_root_dir}
cp -r ${TEST_DATA_DIR}golang ${zot_root_dir}
ls -al ${zot_root_dir}/golang
}
function delete_blob() {
local zot_test_files=${BATS_FILE_TMPDIR}/zot/golang
find ${zot_test_files}/blobs/sha256 -maxdepth 1 -type f -name "*" -print0 |
sort -z -R |
head -z -n 1 | xargs -0 rm
ls -al ${zot_test_files}/blobs/sha256/
}
function log_output() {
local zot_log_file=${BATS_FILE_TMPDIR}/zot/zot-log.json
cat ${zot_log_file} | jq ' .["message"] '
}
function affected() {
log_output | jq 'contains("blobs/manifest affected")?' | grep true
}
function not_affected() {
log_output | jq 'contains("blobs/manifest ok")?' | grep true
}

87
test/blackbox/helpers_sync.bash Normal file
View file

@ -0,0 +1,87 @@
ROOT_DIR=$(git rev-parse --show-toplevel)
TEST_DATA_DIR=${ROOT_DIR}/test/data/
OS="${OS:-linux}"
ARCH="${ARCH:-amd64}"
ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}
ZOT_MINIMAL_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}-minimal
mkdir -p ${TEST_DATA_DIR}
function verify_prerequisites {
if [ ! -f ${BATS_RUN_TMPDIR}/.firstrun ]; then
env | grep proxy >&3
touch ${BATS_RUN_TMPDIR}/.firstrun
fi
if [ ! -f ${ZOT_PATH} ]; then
echo "you need to build ${ZOT_PATH} before running the tests" >&3
return 1
fi
if [ ! -f ${ZOT_MINIMAL_PATH} ]; then
echo "you need to build ${ZOT_MINIMAL_PATH} before running tests" >&3
return 1
fi
if [ ! command -v curl ] &>/dev/null; then
echo "you need to install curl as a prerequisite to running the tests" >&3
return 1
fi
if [ ! command -v jq ] &>/dev/null; then
echo "you need to install jq as a prerequisite to running the tests" >&3
return 1
fi
if [ ! command -v skopeo ] &>/dev/null; then
echo "you need to install skopeo as a prerequisite to running the tests" >&3
return 1
fi
return 0
}
function zot_serve() {
local zot_path=${1}
local config_file=${2}
local pid_dir=${3}
${zot_path} serve ${config_file} &
echo $! >>${pid_dir}/zot.pid
}
function zot_stop() {
local pid_dir=${1}
cat ${pid_dir}/zot.pid
kill $(cat ${pid_dir}/zot.pid)
rm ${pid_dir}/zot.pid
}
function zot_minimal_stop() {
local pid_dir=${1}
kill $(cat ${pid_dir}/zot-minimal.pid)
rm ${pid_dir}/zot-minimal.pid
}
function setup_zot_file_level() {
local config_file=${1}
zot_serve ${ZOT_PATH} ${config_file} ${BATS_FILE_TMPDIR}
}
function setup_zot_minimal_file_level() {
local config_file=${1}
zot_serve ${ZOT_MINIMAL_PATH} ${config_file} ${BATS_FILE_TMPDIR}
}
function teardown_zot_file_level() {
zot_stop ${BATS_FILE_TMPDIR}
}
function wait_zot_reachable() {
zot_url=${1}
curl --connect-timeout 3 \
--max-time 3 \
--retry 10 \
--retry-delay 0 \
--retry-max-time 60 \
--retry-connrefused \
${zot_url}
}

61
test/blackbox/metrics.bats Normal file
View file

@ -0,0 +1,61 @@
load helpers_metrics
function setup_file() {
# verify prerequisites are available
if ! verify_prerequisites; then
echo "oh noooooo"
exit 1
fi
# Download test data to folder common for the entire suite, not just this file
skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.17 oci:${TEST_DATA_DIR}/golang:1.17
# Setup zot server
zot_root_dir=${BATS_FILE_TMPDIR}/zot
echo ${zot_root_dir}
zot_log_file=${zot_root_dir}/zot-log.json
zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json
mkdir -p ${zot_root_dir}
touch ${zot_log_file}
cat >${zot_config_file} <<EOF
{
"distSpecVersion": "1.0.1",
"storage": {
"rootDirectory": "${zot_root_dir}"
},
"http": {
"address": "0.0.0.0",
"port": "8080",
"ReadOnly": false
},
"log": {
"level": "debug",
"output": "${zot_log_file}"
},
"extensions": {
"metrics": {
"enable": true,
"prometheus": {
"path": "/metrics"
}
}
}
}
EOF
setup_zot_file_level ${zot_config_file}
wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
}
function teardown_file() {
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
teardown_zot_file_level
rm -rf ${zot_root_dir}
}
@test "metric enabled" {
local servername="http://127.0.0.1:8080/metrics"
status_code=$(curl --write-out '%{http_code}' --silent --output /dev/null ${servername})
[ "$status_code" -eq 200 ]
}

2
test/blackbox/pushpull.bats
View file

@ -1,4 +1,4 @@
load helpers
load helpers_pushpull
function setup_file() {
# Verify prerequisites are available

84
test/blackbox/scrub.bats Normal file
View file

@ -0,0 +1,84 @@
load helpers_scrub
function setup_file(){
skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.17 oci:${TEST_DATA_DIR}/golang:1.17
}
function setup() {
# verify prerequisites are available
if ! verify_prerequisites; then
echo "oh noooooo"
exit 1
fi
# Setup zot server
ZOT_ROOT_DIR=${BATS_FILE_TMPDIR}/zot
echo ${ZOT_ROOT_DIR}
ZOT_LOG_FILE=${ZOT_ROOT_DIR}/zot-log.json
ZOT_CONFIG_FILE=${BATS_FILE_TMPDIR}/zot_config.json
mkdir -p ${ZOT_ROOT_DIR}
touch ${ZOT_LOG_FILE}
cat >${ZOT_CONFIG_FILE} <<EOF
{
"distSpecVersion": "1.0.1",
"storage": {
"rootDirectory": "${ZOT_ROOT_DIR}"
},
"http": {
"address": "0.0.0.0",
"port": "8080",
"ReadOnly": false
},
"log": {
"level": "debug",
"output": "${ZOT_LOG_FILE}"
},
"extensions": {
"scrub": {
"interval": "2h"
}
}
}
EOF
}
function teardown() {
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
teardown_zot_file_level
rm -rf ${zot_root_dir}
}
@test "blobs/manifest integrity not affected" {
add_test_files
echo ${ZOT_CONFIG_FILE}
setup_zot_file_level ${ZOT_CONFIG_FILE}
wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
# wait for scrub to be done and logs to get populated
run sleep 5s
run not_affected
[ "$status" -eq 0 ]
[ $(echo "${lines[0]}" ) = 'true' ]
}
@test "blobs/manifest integrity affected" {
add_test_files
delete_blob
echo ${ZOT_CONFIG_FILE}
setup_zot_file_level ${ZOT_CONFIG_FILE}
wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
# wait for scrub to be done and logs to get populated
run sleep 5s
run affected
[ "$status" -eq 0 ]
[ $(echo "${lines[0]}" ) = 'true' ]
# [ $(echo "${lines[-1]}" | jq .) ]
}

104
test/blackbox/sync.bats Normal file
View file

@ -0,0 +1,104 @@
load helpers_sync
function setup_file() {
# Verify prerequisites are available
if ! verify_prerequisites; then
exit 1
fi
# Download test data to folder common for the entire suite, not just this file
skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.17 oci:${TEST_DATA_DIR}/golang:1.17
# Setup zot server
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
local zot_minimal_root_dir=${BATS_FILE_TMPDIR}/zot-minimal
local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json
local zot_minimal_config_file=${BATS_FILE_TMPDIR}/zot_minimal_config.json
local oci_data_dir=${BATS_FILE_TMPDIR}/oci
mkdir -p ${zot_root_dir}
mkdir -p ${zot_minimal_root_dir}
mkdir -p ${oci_data_dir}
cat >${zot_config_file} <<EOF
{
"distSpecVersion": "1.0.1",
"storage": {
"rootDirectory": "${zot_root_dir}"
},
"http": {
"address": "0.0.0.0",
"port": "8080",
"ReadOnly": false
},
"log": {
"level": "debug"
},
"extensions": {
"sync": {
"registries": [
{
"urls": [
"http://localhost:9000"
],
"onDemand": true,
"tlsVerify": false,
"PollInterval": "30s",
"content": [
{
"prefix": "**"
}
]
}
]
}
}
}
EOF
cat >${zot_minimal_config_file} <<EOF
{
"distSpecVersion": "1.0.1",
"storage": {
"rootDirectory": "${zot_minimal_root_dir}"
},
"http": {
"address": "0.0.0.0",
"port": "9000",
"ReadOnly": false
},
"log": {
"level": "debug"
}
}
EOF
setup_zot_file_level ${zot_config_file}
wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
setup_zot_minimal_file_level ${zot_minimal_config_file}
wait_zot_reachable "http://127.0.0.1:9000/v2/_catalog"
}
function teardown_file() {
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
local oci_data_dir=${BATS_FILE_TMPDIR}/oci
local zot_minimal_root_dir=${BATS_FILE_TMPDIR}/zot-minimal
teardown_zot_file_level
rm -rf ${zot_root_dir}
rm -rf ${zot_minimal_root_dir}
rm -rf ${oci_data_dir}
}
@test "sync registry" {
run skopeo --insecure-policy copy --dest-tls-verify=false \
oci:${TEST_DATA_DIR}/golang:1.17 \
docker://127.0.0.1:9000/golang:1.17
[ "$status" -eq 0 ]
run curl http://127.0.0.1:9000/v2/_catalog
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.repositories[]') = '"golang"' ]
run curl http://127.0.0.1:8080/v2/_catalog
run curl http://127.0.0.1:9000/v2/golang/tags/list
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.tags[]') = '"1.17"' ]
run sleep 30s
run curl http://127.0.0.1:8080/v2/_catalog
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.repositories[]') = '"golang"' ]
}