From 19003e8a717228502ac3c5acc46234cb5d1237f1 Mon Sep 17 00:00:00 2001 From: Petu Eusebiu Date: Tue, 8 Jun 2021 23:11:18 +0300 Subject: [PATCH] Added new extension "sync" Periodically poll registries and pull images according to sync's config Added sync on demand, syncing when clients asks for an image which zot doesn't have. Signed-off-by: Petu Eusebiu --- .github/workflows/codeql-analysis.yml | 2 +- Makefile | 10 +- README.md | 14 + cmd/zot/main_test.go | 5 +- errors/errors.go | 2 + examples/config-sync.json | 57 + examples/sync-auth-filepath.json | 10 + go.mod | 6 +- go.sum | 71 +- pkg/api/authn.go | 5 +- pkg/api/authz.go | 24 +- pkg/api/{ => config}/config.go | 39 +- pkg/api/controller.go | 9 +- pkg/api/controller_test.go | 341 ++-- pkg/api/routes.go | 51 +- pkg/cli/client_test.go | 37 +- pkg/cli/cve_cmd_test.go | 15 +- pkg/cli/image_cmd_test.go | 13 +- pkg/cli/root.go | 36 +- pkg/cli/root_test.go | 4 +- pkg/compliance/v1_0_0/check_test.go | 15 +- pkg/extensions/{ => config}/config.go | 9 +- pkg/extensions/extensions.go | 97 +- pkg/extensions/minimal.go | 11 +- pkg/extensions/search/common/common_test.go | 21 +- pkg/extensions/search/cve/cve_test.go | 33 +- pkg/extensions/search/digest/digest_test.go | 41 +- pkg/extensions/sync/http_handler.go | 60 + pkg/extensions/sync/on_demand.go | 90 ++ pkg/extensions/sync/sync.go | 469 ++++++ pkg/extensions/sync/sync_internal_test.go | 109 ++ pkg/extensions/sync/sync_test.go | 1612 +++++++++++++++++++ pkg/extensions/sync/utils.go | 166 ++ pkg/log/log_test.go | 13 +- 34 files changed, 3158 insertions(+), 339 deletions(-) create mode 100644 examples/config-sync.json create mode 100644 examples/sync-auth-filepath.json rename pkg/api/{ => config}/config.go (88%) rename pkg/extensions/{ => config}/config.go (72%) create mode 100644 pkg/extensions/sync/http_handler.go create mode 100644 pkg/extensions/sync/on_demand.go create mode 100644 pkg/extensions/sync/sync.go create mode 100644 pkg/extensions/sync/sync_internal_test.go create mode 100644 pkg/extensions/sync/sync_test.go create mode 100644 pkg/extensions/sync/utils.go diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index f83df08d..2706a155 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -68,4 +68,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v1 \ No newline at end of file diff --git a/Makefile b/Makefile index 20687286..e7e0c787 100644 --- a/Makefile +++ b/Makefile @@ -12,22 +12,22 @@ all: doc binary binary-minimal debug test test-clean check .PHONY: binary-minimal binary-minimal: doc - go build -tags minimal -v -ldflags "-X github.com/anuvu/zot/pkg/api.Commit=${COMMIT} -X github.com/anuvu/zot/pkg/api.BinaryType=minimal" -o bin/zot-minimal ./cmd/zot + go build -tags minimal,containers_image_openpgp -v -ldflags "-X github.com/anuvu/zot/pkg/api.Commit=${COMMIT} -X github.com/anuvu/zot/pkg/api.BinaryType=minimal" -o bin/zot-minimal ./cmd/zot .PHONY: binary binary: doc - go build -tags extended -v -ldflags "-X github.com/anuvu/zot/pkg/api.Commit=${COMMIT} -X github.com/anuvu/zot/pkg/api.BinaryType=extended" -o bin/zot ./cmd/zot + go build -tags extended,containers_image_openpgp -v -ldflags "-X github.com/anuvu/zot/pkg/api.Commit=${COMMIT} -X github.com/anuvu/zot/pkg/api.BinaryType=extended" -o bin/zot ./cmd/zot .PHONY: debug debug: doc - go build -tags extended -v -gcflags all='-N -l' -ldflags "-X github.com/anuvu/zot/pkg/api.Commit=${COMMIT} -X github.com/anuvu/zot/pkg/api.BinaryType=extended" -o bin/zot-debug ./cmd/zot + go build -tags extended,containers_image_openpgp -v -gcflags all='-N -l' -ldflags "-X github.com/anuvu/zot/pkg/api.Commit=${COMMIT} -X github.com/anuvu/zot/pkg/api.BinaryType=extended" -o bin/zot-debug ./cmd/zot .PHONY: test test: $(shell mkdir -p test/data; cd test/data; ../scripts/gen_certs.sh; cd ${TOP_LEVEL}; sudo skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:7 oci:${TOP_LEVEL}/test/data/zot-test:0.0.1;sudo skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:8 oci:${TOP_LEVEL}/test/data/zot-cve-test:0.0.1) $(shell sudo mkdir -p /etc/containers/certs.d/127.0.0.1:8089/; sudo cp test/data/client.* /etc/containers/certs.d/127.0.0.1:8089/; sudo cp test/data/ca.* /etc/containers/certs.d/127.0.0.1:8089/;) $(shell sudo chmod a=rwx /etc/containers/certs.d/127.0.0.1:8089/*.key) - go test -tags extended -v -race -cover -coverpkg ./... -coverprofile=coverage.txt -covermode=atomic ./... + go test -tags extended,containers_image_openpgp -v -race -cover -coverpkg ./... -coverprofile=coverage.txt -covermode=atomic ./... .PHONY: test-clean test-clean: @@ -40,7 +40,7 @@ covhtml: .PHONY: check check: ./golangcilint.yaml golangci-lint --version || curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.26.0 - golangci-lint --config ./golangcilint.yaml run --enable-all --build-tags extended ./cmd/... ./pkg/... + golangci-lint --config ./golangcilint.yaml run --enable-all --build-tags extended,containers_image_openpgp ./cmd/... ./pkg/... docs/docs.go: swag -v || go install github.com/swaggo/swag/cmd/swag diff --git a/README.md b/README.md index 6530708a..534b203a 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,7 @@ https://anuvu.github.io/zot/ * Automatic garbage collection of orphaned blobs * Layer deduplication using hard links when content is identical * Serve [multiple storage paths (and backends)](./examples/config-multiple.json) using a single zot server +* Pull and synchronize with other zot registries [sync](#sync) * Swagger based documentation * Single binary for _all_ the above features * Released under Apache 2.0 License @@ -226,6 +227,19 @@ c3/openjdk-dev commit-2674e8a-squashfs b545b8ba 321MB c3/openjdk-dev commit-d5024ec-squashfs cd45f8cf 321MB ``` +# Sync +Periodically pull and synchronize images between zot registries. +The synchronization is achieved by copying all the images found at source to destination. +To use it see [sync-config](examples/config-sync.json) +Supports: + - TLS verification + - Prefix filtering (can contain multiple repos, eg repo1/repoX/repoZ) + - Tags regex filtering + - Tags semver compliance filtering (the 'v' prefix is optional) + - BASIC auth + - Trigger sync with a POST call to http://registry:port/sync + + # Ecosystem diff --git a/cmd/zot/main_test.go b/cmd/zot/main_test.go index e719828b..7034ce50 100644 --- a/cmd/zot/main_test.go +++ b/cmd/zot/main_test.go @@ -4,14 +4,15 @@ import ( "testing" "github.com/anuvu/zot/pkg/api" + "github.com/anuvu/zot/pkg/api/config" "github.com/anuvu/zot/pkg/cli" . "github.com/smartystreets/goconvey/convey" ) func TestIntegration(t *testing.T) { Convey("Make a new controller", t, func() { - config := api.NewConfig() - c := api.NewController(config) + conf := config.New() + c := api.NewController(conf) So(c, ShouldNotBeNil) cl := cli.NewRootCmd() diff --git a/errors/errors.go b/errors/errors.go index dd3f82e5..d86b29b1 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -42,4 +42,6 @@ var ( ErrImgStoreNotFound = errors.New("routes: image store not found corresponding to given route") ErrEmptyValue = errors.New("cache: empty value") ErrEmptyRepoList = errors.New("search: no repository found") + ErrInvalidRepositoryName = errors.New("routes: not a repository name") + ErrSyncMissingCatalog = errors.New("sync: couldn't fetch upstream registry's catalog") ) diff --git a/examples/config-sync.json b/examples/config-sync.json new file mode 100644 index 00000000..3a3a3a3c --- /dev/null +++ b/examples/config-sync.json @@ -0,0 +1,57 @@ +{ + "version":"0.1.0-dev", + "storage":{ + "rootDirectory":"/tmp/zot" + }, + "http":{ + "address":"127.0.0.1", + "port":"5000" + }, + "log":{ + "level":"debug" + }, + "extensions":{ + "sync": { + "credentialsFile": "./examples/sync-auth-filepath.json", + "registries": [{ + "url": "https://registry1:5000", + "onDemand": false, + "pollInterval": "6h", + "tlsVerify": true, + "certDir": "/home/user/certs", + "content":[ + { + "prefix":"/repo1/repo", + "tags":{ + "regex":"4.*", + "semver":true + } + }, + { + "prefix":"/repo2/repo" + } + ] + }, + { + "url": "https://registry2:5000", + "pollInterval": "12h", + "tlsVerify": false, + "onDemand": false, + "content":[ + { + "prefix":"/repo2", + "tags":{ + "semver":true + } + } + ] + }, + { + "url": "https://docker.io/library", + "onDemand": true, + "tlsVerify": true + } + ] + } + } +} \ No newline at end of file diff --git a/examples/sync-auth-filepath.json b/examples/sync-auth-filepath.json new file mode 100644 index 00000000..e146d41a --- /dev/null +++ b/examples/sync-auth-filepath.json @@ -0,0 +1,10 @@ +{ + "localhost:8080": { + "username": "user", + "password": "pass" + }, + "registry2:5000": { + "username": "user2", + "password": "pass2" + } +} \ No newline at end of file diff --git a/go.mod b/go.mod index 2eb466f2..22c31840 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.16 require ( github.com/99designs/gqlgen v0.13.0 + github.com/AdaLogics/go-fuzz-headers v0.0.0-20210921152813-f50b76b2163b // indirect + github.com/Masterminds/semver v1.5.0 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 github.com/apex/log v1.9.0 github.com/aquasecurity/trivy v0.0.0-00010101000000-000000000000 @@ -11,6 +13,8 @@ require ( github.com/briandowns/spinner v1.16.0 github.com/chartmuseum/auth v0.5.0 github.com/containerd/containerd v1.5.7 // indirect + github.com/containers/common v0.26.0 + github.com/containers/image/v5 v5.13.2 github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect github.com/dustin/go-humanize v1.0.0 github.com/fsnotify/fsnotify v1.5.1 @@ -49,6 +53,6 @@ require ( replace ( github.com/aquasecurity/fanal => github.com/anuvu/fanal v0.0.0-20211007194926-d0c577a014df github.com/aquasecurity/trivy => github.com/anuvu/trivy v0.9.2-0.20211013001708-27408aa50da3 - github.com/aquasecurity/trivy-db => github.com/anuvu/trivy-db v0.0.0-20211007191113-44f7e57b689c + github.com/containers/image/v5 => github.com/anuvu/image/v5 v5.0.0-20210310195111-044dd755e25e ) diff --git a/go.sum b/go.sum index a364c300..959ca98d 100644 --- a/go.sum +++ b/go.sum @@ -56,10 +56,12 @@ contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcig dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= git.apache.org/thrift.git v0.12.0/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0= github.com/99designs/gqlgen v0.13.0 h1:haLTcUp3Vwp80xMVEg5KRNwzfUrgFdRmtBY8fuB8scA= github.com/99designs/gqlgen v0.13.0/go.mod h1:NV130r6f4tpRWuAI+zsrSdooO/eWUv+Gyyoi3rEfXIk= -github.com/AdaLogics/go-fuzz-headers v0.0.0-20210401092550-0a8691dafd0d h1:oXbCfnomBQyeTWN6RNHmMUrmzyUGLYoF2OOcfkpoCHE= github.com/AdaLogics/go-fuzz-headers v0.0.0-20210401092550-0a8691dafd0d/go.mod h1:CzsSbkDixRphAF5hS6wbMKq0eI6ccJRb7/A0M6JBnwg= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20210921152813-f50b76b2163b h1:eFb6EtuUv+MRaLzNRldt6ofzuxallkY+5mqT2cdakjs= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20210921152813-f50b76b2163b/go.mod h1:WpB7kf89yJUETZxQnP1kgYPNwlT2jjdDYUCoxVggM3g= github.com/AkihiroSuda/containerd-fuse-overlayfs v1.0.0/go.mod h1:0mMDvQFeLbbn1Wy8P2j3hwFhqBq+FKn8OZPno8WLmp8= github.com/Azure/azure-amqp-common-go/v2 v2.1.0/go.mod h1:R8rea+gJRuJR6QxTir/XuEd+YuKoUiazDC/N96FiDEU= github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= @@ -176,6 +178,8 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk= github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= @@ -203,6 +207,8 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/anuvu/fanal v0.0.0-20211007194926-d0c577a014df h1:fq3+E3RolIKi+QRxASNEzCXTDMEHRl0PORQipfEyiTo= github.com/anuvu/fanal v0.0.0-20211007194926-d0c577a014df/go.mod h1:nXdCM1C89phZEkn/sHQ6S5IjcvxdTnXLSKcftmhFodg= +github.com/anuvu/image/v5 v5.0.0-20210310195111-044dd755e25e h1:sBcyXcj9dUsppI06+a4QV3p+CvGUiCbQ/Y2PaPJQUhw= +github.com/anuvu/image/v5 v5.0.0-20210310195111-044dd755e25e/go.mod h1:Zbv8d/viXvzhCbGrqiepkKgW9IkEQE9gigbCg0e8ay0= github.com/anuvu/trivy v0.9.2-0.20211013001708-27408aa50da3 h1:XpCWTtU94xfLX/Stjx60MoCSW8uGjndLJ6ONBqkXXKQ= github.com/anuvu/trivy v0.9.2-0.20211013001708-27408aa50da3/go.mod h1:AxK9ngsc2DchN5ayVETGZ6Cmne0SPsjsFIIL6AxHFKM= github.com/anuvu/trivy-db v0.0.0-20211007191113-44f7e57b689c h1:LTN7B8PyGULa/w3/8VY/rGAlrMsqan6zLN5uk2MZTQ4= @@ -271,6 +277,7 @@ github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1UJrqV3uuy861HCTo708pDMbjHHdCas= github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= @@ -317,6 +324,7 @@ github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cb github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chartmuseum/auth v0.5.0 h1:ENNmoxvjxcR/JR0HrghAEtGQe7hToMNj16+UoS5CK9Y= github.com/chartmuseum/auth v0.5.0/go.mod h1:BvoSXHyvbsq+/bbhNgVTDQsModM+HERBTNY5o9Vyrig= +github.com/checkpoint-restore/go-criu/v4 v4.0.2/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/cheggaaa/pb v1.0.27 h1:wIkZHkNfC7R6GI5w7l/PdAdzXzlrbcI3p8OAlnkTsnc= @@ -327,6 +335,7 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= +github.com/cilium/ebpf v0.0.0-20200507155900-a9f01edf17e3/go.mod h1:XT+cAw5wfvsodedcijoh1l9cf7v1x9FlFB/3VmF/O8s= github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= @@ -427,9 +436,17 @@ github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= +github.com/containers/common v0.26.0 h1:BCo/S5Dl8aRRG7vze+hoWdCd5xuThIP/tCB5NjTIn6g= +github.com/containers/common v0.26.0/go.mod h1:BCK8f8Ye1gvUVGcokJngJG4YC80c2Bjx/F9GyoIAVMc= +github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE= +github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= +github.com/containers/ocicrypt v1.1.1 h1:prL8l9w3ntVqXvNH1CiNn5ENjcCnr38JqpSyvKKB4GI= github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= +github.com/containers/storage v1.23.5/go.mod h1:ha26Q6ngehFNhf3AWoXldvAvwI4jFe3ETQAf/CeZPyM= +github.com/containers/storage v1.27.0 h1:3r4yWPNCoqZa8ptvVhljoDJyBYKrJq/tmHyODD7Lv2Y= +github.com/containers/storage v1.27.0/go.mod h1:o7PtlRZpFleYVu0TRAFSb/dPJHZnEk5GMFbVLsf0NOI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -500,6 +517,7 @@ github.com/docker/docker v0.0.0-20200511152416-a93e9eb0e95c/go.mod h1:eEKB0N0r5N github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v1.4.2-0.20180531152204-71cd53e4a197/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v1.4.2-0.20191219165747-a9416c67da9f/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v17.12.0-ce-rc1.0.20200730172259-9f28837c1d93+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.0-beta1.0.20201110211921-af34b94a78a1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= @@ -512,17 +530,21 @@ github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5Xh github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= +github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libnetwork v0.8.0-dev.2.0.20200917202933-d0951081b35f/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dvyukov/go-fuzz v0.0.0-20210914135545-4980593459a1/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -886,6 +908,7 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.6.4/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= @@ -976,6 +999,7 @@ github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfE github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 h1:DowS9hvgyYSX4TO5NpyC606/Z4SxnNYbT+WX27or6Ck= @@ -986,8 +1010,11 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.0/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= @@ -1038,6 +1065,8 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= +github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= @@ -1048,6 +1077,7 @@ github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/manifoldco/promptui v0.8.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ= github.com/maratori/testpackage v1.0.1/go.mod h1:ddKdw+XG0Phzhx8BFDTKgpWP4i7MpApTE5fXSKAqwDU= github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/masahiro331/go-mvn-version v0.0.0-20210429150710-d3157d602a08 h1:AevUBW4cc99rAF8q8vmddIP8qd/0J5s/UyltGbp66dg= @@ -1085,15 +1115,19 @@ github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxm github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/mattn/go-shellwords v1.0.11/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/pkcs11 v1.0.3 h1:iMwmD7I5225wv84WxIG/bmxz9AXjWvTWIbM/TYHvWtw= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mistifyio/go-zfs v2.1.1+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.1.1 h1:Bp6x9R1Wn16SIz3OfeDr0b7RnCG2OB66Y7PQyC/cvq4= @@ -1129,6 +1163,7 @@ github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7s github.com/moby/sys/mountinfo v0.1.0/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o= github.com/moby/sys/mountinfo v0.1.3/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o= github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.4.1 h1:1O+1cHA1aujwEwwVMa2Xm2l+gIpUHyd3+D+d7LZh1kM= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= @@ -1149,8 +1184,11 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mozilla/tls-observatory v0.0.0-20190404164649-a3c1b6cfecfd/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= github.com/mozilla/tls-observatory v0.0.0-20200317151703-4fa42e1c2dee/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk= +github.com/mrunalp/fileutils v0.0.0-20171103030105-7d4729fb3618/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0= github.com/mrunalp/fileutils v0.0.0-20200520151820-abd8a0e76976/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/mtrmac/gpgme v0.1.2 h1:dNOmvYmsrakgW7LcgiprD0yfRuQQe8/C8F6Z+zogO3s= +github.com/mtrmac/gpgme v0.1.2/go.mod h1:GYYHnGSuS7HK3zVS2n3y73y0okK/BeKzwnn5jgiVFNI= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -1181,6 +1219,7 @@ github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -1215,6 +1254,7 @@ github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59P github.com/opencontainers/runc v1.0.0-rc10/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc91/go.mod h1:3Sm6Dt7OT8z88EbdQqqcRN2oCT54jbi72tT/HqgflT8= github.com/opencontainers/runc v1.0.0-rc92/go.mod h1:X1zlU4p7wOlX4+WRCz+hvlRv8phdL7UqbYD+vQwNMmE= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= github.com/opencontainers/runc v1.0.2 h1:opHZMaswlyxz1OuGpBE53Dwe4/xF7EZTY0A2L/FpCOg= @@ -1223,11 +1263,14 @@ github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.m github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20200520003142-237cc4f519e2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20200728170252-4d89ac9fbff6/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= +github.com/opencontainers/runtime-tools v0.9.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= +github.com/opencontainers/selinux v1.5.1/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= @@ -1241,6 +1284,7 @@ github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYr github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= github.com/owenrumney/go-sarif v1.0.10/go.mod h1:sgJM0ZaZ28jT8t8Iq3/mUCFBW9cX09EobIBXYOhiYBc= github.com/owenrumney/go-sarif v1.0.11/go.mod h1:hTBFbxU7GuVRUvwMx+eStp9M/Oun4xHCS3vqpPvket8= github.com/owenrumney/squealer v0.2.28 h1:LYsqUHal+5QlANjbZ+h44SN5kIZSfHCWKUzBAS1KwB0= @@ -1272,6 +1316,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= +github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -1280,12 +1326,14 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= @@ -1296,6 +1344,7 @@ github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.29.0 h1:3jqPBvKT4OHAbje2Ql7KeaaSicDBCxMYwEJU1zRJceE= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1309,6 +1358,7 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= @@ -1349,6 +1399,7 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/seccomp/libseccomp-golang v0.9.2-0.20200616122406-847368b35ebf/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/securego/gosec v0.0.0-20200103095621-79fbf3af8d83/go.mod h1:vvbZ2Ae7AzSq3/kywjUDxSNq2SJ27RxCz2un0H3ePqE= github.com/securego/gosec v0.0.0-20200401082031-e946c8c39989/go.mod h1:i9l/TNj+yDFh9SZXUTvspXTjbFXgZGP/UvhU1S65A4A= github.com/securego/gosec/v2 v2.3.0/go.mod h1:UzeVyUXbxukhLeHKV3VVqo7HdoQR9MrRfFmZYotn8ME= @@ -1372,6 +1423,7 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.0/go.mod h1:4GuYW9TZmE769R5STWrRakJc4UqQ3+QQ95fyz7ENv1A= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -1419,6 +1471,7 @@ github.com/spf13/viper v1.6.1/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfD github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= +github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 h1:lIOOHPEbXzO3vnmx2gok1Tfs31Q8GQqKLc8vVqyQq/I= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -1446,9 +1499,11 @@ github.com/swaggo/swag v1.7.0 h1:5bCA/MTLQoIqDXXyHfOpMeDvL9j68OY/udlK4pQoo4E= github.com/swaggo/swag v1.7.0/go.mod h1:BdPIL73gvS9NBsdi7M1JOxLvlbfvNRaBP8m6WT6Aajo= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/tdakkota/asciicheck v0.0.0-20200416190851-d7f85be797a2/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= github.com/tdakkota/asciicheck v0.0.0-20200416200610-e657995f937b/go.mod h1:yHp0ai0Z9gUljN3o0xMhYJnH/IcvkdTBOX2fmJ93JEM= github.com/testcontainers/testcontainers-go v0.11.1/go.mod h1:/V0UVq+1e7NWYoqTPog179clf0Qp9TOyp4EcXaEFQz8= @@ -1479,8 +1534,9 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.8 h1:ERv8V6GKqVi23rgu5cj9pVfVzJbOqAY2Ntl88O6c2nQ= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= +github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA= github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -1500,6 +1556,9 @@ github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOV github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= github.com/vbatts/go-mtree v0.5.0 h1:dM+5XZdqH0j9CSZeerhoN/tAySdwnmevaZHO1XGW2Vc= github.com/vbatts/go-mtree v0.5.0/go.mod h1:7JbaNHyBMng+RP8C3Q4E+4Ca8JnGQA2R/MB+jb4tSOk= +github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g= +github.com/vbauerster/mpb/v6 v6.0.2 h1:DWFnBOcgHi9GUNduC1MbQ936Z7B77wvOnZexP9Hjzcw= +github.com/vbauerster/mpb/v6 v6.0.2/go.mod h1:JDNVbdx4oAMMxZNXodDH2DeDY5xBJC8bDGHNFZwRqQM= github.com/vdemeester/k8s-pkg-credentialprovider v1.17.4/go.mod h1:inCTmtUdr5KJbreVojo06krnTgaeAz/Z7lynpPk/Q2c= github.com/vektah/dataloaden v0.2.1-0.20190515034641-a19b9a6e7c9e/go.mod h1:/HUdMve7rvxZma+2ZELQeNh88+003LL7Pf/CZ089j8U= github.com/vektah/gqlparser/v2 v2.1.0/go.mod h1:SyUiHgLATUR8BiYURfTirrTcGpcE+4XkV2se04Px1Ms= @@ -1522,11 +1581,14 @@ github.com/xanzy/go-gitlab v0.32.0/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfD github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -1563,6 +1625,7 @@ go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3C go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= +go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 h1:A/5uWzF44DlIgdm/PQFwfMkW0JX+cIcQi/SwLAmZP5M= go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= @@ -1831,10 +1894,12 @@ golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1858,6 +1923,7 @@ golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2179,6 +2245,7 @@ gopkg.in/resty.v1 v1.12.0 h1:CuXP0Pjfw9rOuY6EP+UvtNvt5DSqHpIxILZKT/quCZI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/pkg/api/authn.go b/pkg/api/authn.go index f9c12c75..b35c08fd 100644 --- a/pkg/api/authn.go +++ b/pkg/api/authn.go @@ -13,6 +13,7 @@ import ( "time" "github.com/anuvu/zot/errors" + "github.com/anuvu/zot/pkg/api/config" "github.com/chartmuseum/auth" "github.com/gorilla/mux" "golang.org/x/crypto/bcrypt" @@ -237,7 +238,7 @@ func basicAuthHandler(c *Controller) mux.MiddlewareFunc { } } -func isAuthnEnabled(config *Config) bool { +func isAuthnEnabled(config *config.Config) bool { if config.HTTP.Auth != nil && (config.HTTP.Auth.HTPasswd.Path != "" || config.HTTP.Auth.LDAP != nil) { return true @@ -246,7 +247,7 @@ func isAuthnEnabled(config *Config) bool { return false } -func isBearerAuthEnabled(config *Config) bool { +func isBearerAuthEnabled(config *config.Config) bool { if config.HTTP.Auth != nil && config.HTTP.Auth.Bearer != nil && config.HTTP.Auth.Bearer.Cert != "" && diff --git a/pkg/api/authz.go b/pkg/api/authz.go index cafcc9bd..60134649 100755 --- a/pkg/api/authz.go +++ b/pkg/api/authz.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/anuvu/zot/pkg/api/config" "github.com/anuvu/zot/pkg/log" "github.com/gorilla/mux" ) @@ -24,26 +25,9 @@ const ( authzCtxKey contextKey = 0 ) -type AccessControlConfig struct { - Repositories Repositories - AdminPolicy Policy -} - -type Repositories map[string]PolicyGroup - -type PolicyGroup struct { - Policies []Policy - DefaultPolicy []string -} - -type Policy struct { - Users []string - Actions []string -} - // AccessController authorizes users to act on resources. type AccessController struct { - Config *AccessControlConfig + Config *config.AccessControlConfig Log log.Logger } @@ -53,7 +37,7 @@ type AccessControlContext struct { isAdmin bool } -func NewAccessController(config *Config) *AccessController { +func NewAccessController(config *config.Config) *AccessController { return &AccessController{ Config: config.AccessControl, Log: log.NewLogger(config.Log.Level, config.Log.Output), @@ -117,7 +101,7 @@ func (ac *AccessController) getContext(username string, r *http.Request) context } // isPermitted returns true if username can do action on a repository policy. -func isPermitted(username, action string, pg PolicyGroup) bool { +func isPermitted(username, action string, pg config.PolicyGroup) bool { var result bool // check repo/system based policies for _, p := range pg.Policies { diff --git a/pkg/api/config.go b/pkg/api/config/config.go similarity index 88% rename from pkg/api/config.go rename to pkg/api/config/config.go index f5c6d492..2152dbd3 100644 --- a/pkg/api/config.go +++ b/pkg/api/config/config.go @@ -1,10 +1,10 @@ -package api +package config import ( "fmt" "github.com/anuvu/zot/errors" - ext "github.com/anuvu/zot/pkg/extensions" + extconf "github.com/anuvu/zot/pkg/extensions/config" "github.com/anuvu/zot/pkg/log" "github.com/getlantern/deepcopy" distspec "github.com/opencontainers/distribution-spec/specs-go" @@ -83,6 +83,23 @@ type GlobalStorageConfig struct { SubPaths map[string]StorageConfig } +type AccessControlConfig struct { + Repositories Repositories + AdminPolicy Policy +} + +type Repositories map[string]PolicyGroup + +type PolicyGroup struct { + Policies []Policy + DefaultPolicy []string +} + +type Policy struct { + Users []string + Actions []string +} + type Config struct { Version string Commit string @@ -91,10 +108,10 @@ type Config struct { Storage GlobalStorageConfig HTTP HTTPConfig Log *LogConfig - Extensions *ext.ExtensionConfig + Extensions *extconf.ExtensionConfig } -func NewConfig() *Config { +func New() *Config { return &Config{ Version: distspec.Version, Commit: Commit, @@ -107,12 +124,12 @@ func NewConfig() *Config { // Sanitize makes a sanitized copy of the config removing any secrets. func (c *Config) Sanitize() *Config { - if c.HTTP.Auth != nil && c.HTTP.Auth.LDAP != nil && c.HTTP.Auth.LDAP.BindPassword != "" { - s := &Config{} - if err := deepcopy.Copy(s, c); err != nil { - panic(err) - } + s := &Config{} + if err := deepcopy.Copy(s, c); err != nil { + panic(err) + } + if c.HTTP.Auth != nil && c.HTTP.Auth.LDAP != nil && c.HTTP.Auth.LDAP.BindPassword != "" { s.HTTP.Auth.LDAP = &LDAPConfig{} if err := deepcopy.Copy(s.HTTP.Auth.LDAP, c.HTTP.Auth.LDAP); err != nil { @@ -120,11 +137,9 @@ func (c *Config) Sanitize() *Config { } s.HTTP.Auth.LDAP.BindPassword = "******" - - return s } - return c + return s } func (c *Config) Validate(log log.Logger) error { diff --git a/pkg/api/controller.go b/pkg/api/controller.go index 804c4452..de397110 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -10,6 +10,7 @@ import ( "time" "github.com/anuvu/zot/errors" + "github.com/anuvu/zot/pkg/api/config" ext "github.com/anuvu/zot/pkg/extensions" "github.com/anuvu/zot/pkg/log" "github.com/anuvu/zot/pkg/storage" @@ -22,7 +23,7 @@ const ( ) type Controller struct { - Config *Config + Config *config.Config Router *mux.Router StoreController storage.StoreController Log log.Logger @@ -30,7 +31,7 @@ type Controller struct { Server *http.Server } -func NewController(config *Config) *Controller { +func NewController(config *config.Config) *Controller { var controller Controller logger := log.NewLogger(config.Log.Level, config.Log.Output) @@ -102,7 +103,7 @@ func (c *Controller) Run() error { // Enable extensions if extension config is provided if c.Config != nil && c.Config.Extensions != nil { - ext.EnableExtensions(c.Config.Extensions, c.Log, c.Config.Storage.RootDirectory) + ext.EnableExtensions(c.Config, c.Log, c.Config.Storage.RootDirectory) } } else { // we can't proceed without global storage @@ -134,7 +135,7 @@ func (c *Controller) Run() error { // Enable extensions if extension config is provided if c.Config != nil && c.Config.Extensions != nil { - ext.EnableExtensions(c.Config.Extensions, c.Log, storageConfig.RootDirectory) + ext.EnableExtensions(c.Config, c.Log, storageConfig.RootDirectory) } } diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 44a24c9c..9ff0bdf6 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -26,6 +26,7 @@ import ( "github.com/anuvu/zot/errors" "github.com/anuvu/zot/pkg/api" + "github.com/anuvu/zot/pkg/api/config" "github.com/chartmuseum/auth" "github.com/mitchellh/mapstructure" godigest "github.com/opencontainers/go-digest" @@ -124,9 +125,9 @@ func getCredString(username, password string) string { func TestNew(t *testing.T) { Convey("Make a new controller", t, func() { - config := api.NewConfig() - So(config, ShouldNotBeNil) - So(api.NewController(config), ShouldNotBeNil) + conf := config.New() + So(conf, ShouldNotBeNil) + So(api.NewController(conf), ShouldNotBeNil) }) } @@ -142,17 +143,17 @@ func TestHtpasswdSingleCred(t *testing.T) { for _, testString := range singleCredtests { func() { - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port htpasswdPath := makeHtpasswdFileFromString(testString) defer os.Remove(htpasswdPath) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -211,16 +212,16 @@ func TestHtpasswdTwoCreds(t *testing.T) { func() { port := getFreePort() baseURL := getBaseURL(port, false) - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port htpasswdPath := makeHtpasswdFileFromString(testString) defer os.Remove(htpasswdPath) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -280,16 +281,16 @@ func TestHtpasswdFiveCreds(t *testing.T) { func() { port := getFreePort() baseURL := getBaseURL(port, false) - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port htpasswdPath := makeHtpasswdFileFromString(credString.String()) defer os.Remove(htpasswdPath) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -333,17 +334,17 @@ func TestBasicAuth(t *testing.T) { Convey("Make a new controller", t, func() { port := getFreePort() baseURL := getBaseURL(port, false) - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port htpasswdPath := makeHtpasswdFile() defer os.Remove(htpasswdPath) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -395,10 +396,10 @@ func TestInterruptedBlobUpload(t *testing.T) { Convey("Successfully cleaning interrupted blob uploads", t, func() { port := getFreePort() baseURL := getBaseURL(port, false) - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -635,17 +636,17 @@ func TestMultipleInstance(t *testing.T) { Convey("Negative test zot multiple instance", t, func() { port := getFreePort() baseURL := getBaseURL(port, false) - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port htpasswdPath := makeHtpasswdFile() defer os.Remove(htpasswdPath) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) err := c.Run() So(err, ShouldEqual, errors.ErrImgStoreNotFound) @@ -662,9 +663,9 @@ func TestMultipleInstance(t *testing.T) { defer os.RemoveAll(subDir) c.Config.Storage.RootDirectory = globalDir - subPathMap := make(map[string]api.StorageConfig) + subPathMap := make(map[string]config.StorageConfig) - subPathMap["/a"] = api.StorageConfig{RootDirectory: subDir} + subPathMap["/a"] = config.StorageConfig{RootDirectory: subDir} go func() { if err := c.Run(); err != nil { @@ -697,17 +698,17 @@ func TestMultipleInstance(t *testing.T) { Convey("Test zot multiple instance", t, func() { port := getFreePort() baseURL := getBaseURL(port, false) - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port htpasswdPath := makeHtpasswdFile() defer os.Remove(htpasswdPath) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) globalDir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -721,9 +722,9 @@ func TestMultipleInstance(t *testing.T) { defer os.RemoveAll(subDir) c.Config.Storage.RootDirectory = globalDir - subPathMap := make(map[string]api.StorageConfig) + subPathMap := make(map[string]config.StorageConfig) - subPathMap["/a"] = api.StorageConfig{RootDirectory: subDir} + subPathMap["/a"] = config.StorageConfig{RootDirectory: subDir} go func() { // this blocks if err := c.Run(); err != nil { @@ -780,19 +781,19 @@ func TestTLSWithBasicAuth(t *testing.T) { resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() - config := api.NewConfig() - config.HTTP.Port = port - config.HTTP.TLS = &api.TLSConfig{ + conf := config.New() + conf.HTTP.Port = port + conf.HTTP.TLS = &config.TLSConfig{ Cert: ServerCert, Key: ServerKey, } - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -861,20 +862,20 @@ func TestTLSWithBasicAuthAllowReadAccess(t *testing.T) { resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() - config := api.NewConfig() - config.HTTP.Port = port - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf := config.New() + conf.HTTP.Port = port + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - config.HTTP.TLS = &api.TLSConfig{ + conf.HTTP.TLS = &config.TLSConfig{ Cert: ServerCert, Key: ServerKey, } - config.HTTP.AllowReadAccess = true + conf.HTTP.AllowReadAccess = true - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -942,15 +943,15 @@ func TestTLSMutualAuth(t *testing.T) { resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() - config := api.NewConfig() - config.HTTP.Port = port - config.HTTP.TLS = &api.TLSConfig{ + conf := config.New() + conf.HTTP.Port = port + conf.HTTP.TLS = &config.TLSConfig{ Cert: ServerCert, Key: ServerKey, CACert: CACert, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -1030,16 +1031,16 @@ func TestTLSMutualAuthAllowReadAccess(t *testing.T) { resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() - config := api.NewConfig() - config.HTTP.Port = port - config.HTTP.TLS = &api.TLSConfig{ + conf := config.New() + conf.HTTP.Port = port + conf.HTTP.TLS = &config.TLSConfig{ Cert: ServerCert, Key: ServerKey, CACert: CACert, } - config.HTTP.AllowReadAccess = true + conf.HTTP.AllowReadAccess = true - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -1128,20 +1129,20 @@ func TestTLSMutualAndBasicAuth(t *testing.T) { resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() - config := api.NewConfig() - config.HTTP.Port = port - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf := config.New() + conf.HTTP.Port = port + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - config.HTTP.TLS = &api.TLSConfig{ + conf.HTTP.TLS = &config.TLSConfig{ Cert: ServerCert, Key: ServerKey, CACert: CACert, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -1226,21 +1227,21 @@ func TestTLSMutualAndBasicAuthAllowReadAccess(t *testing.T) { resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() - config := api.NewConfig() - config.HTTP.Port = port - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf := config.New() + conf.HTTP.Port = port + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - config.HTTP.TLS = &api.TLSConfig{ + conf.HTTP.TLS = &config.TLSConfig{ Cert: ServerCert, Key: ServerKey, CACert: CACert, } - config.HTTP.AllowReadAccess = true + conf.HTTP.AllowReadAccess = true - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -1400,10 +1401,10 @@ func TestBasicAuthWithLDAP(t *testing.T) { port := getFreePort() baseURL := getBaseURL(port, false) - config := api.NewConfig() - config.HTTP.Port = port - config.HTTP.Auth = &api.AuthConfig{ - LDAP: &api.LDAPConfig{ + conf := config.New() + conf.HTTP.Port = port + conf.HTTP.Auth = &config.AuthConfig{ + LDAP: &config.LDAPConfig{ Insecure: true, Address: LDAPAddress, Port: LDAPPort, @@ -1413,7 +1414,7 @@ func TestBasicAuthWithLDAP(t *testing.T) { UserAttribute: "uid", }, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -1469,20 +1470,20 @@ func TestBearerAuth(t *testing.T) { port := getFreePort() baseURL := getBaseURL(port, false) - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port u, err := url.Parse(authTestServer.URL) So(err, ShouldBeNil) - config.HTTP.Auth = &api.AuthConfig{ - Bearer: &api.BearerConfig{ + conf.HTTP.Auth = &config.AuthConfig{ + Bearer: &config.BearerConfig{ Cert: ServerCert, Realm: authTestServer.URL + "/auth/token", Service: u.Host, }, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") So(err, ShouldBeNil) defer os.RemoveAll(dir) @@ -1651,21 +1652,21 @@ func TestBearerAuthWithAllowReadAccess(t *testing.T) { port := getFreePort() baseURL := getBaseURL(port, false) - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port u, err := url.Parse(authTestServer.URL) So(err, ShouldBeNil) - config.HTTP.Auth = &api.AuthConfig{ - Bearer: &api.BearerConfig{ + conf.HTTP.Auth = &config.AuthConfig{ + Bearer: &config.BearerConfig{ Cert: ServerCert, Realm: authTestServer.URL + "/auth/token", Service: u.Host, }, } - config.HTTP.AllowReadAccess = true - c := api.NewController(config) + conf.HTTP.AllowReadAccess = true + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") So(err, ShouldBeNil) defer os.RemoveAll(dir) @@ -1885,20 +1886,20 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { port := getFreePort() baseURL := getBaseURL(port, false) - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port htpasswdPath := makeHtpasswdFile() defer os.Remove(htpasswdPath) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - config.AccessControl = &api.AccessControlConfig{ - Repositories: api.Repositories{ - AuthorizationNamespace: api.PolicyGroup{ - Policies: []api.Policy{ + conf.AccessControl = &config.AccessControlConfig{ + Repositories: config.Repositories{ + AuthorizationNamespace: config.PolicyGroup{ + Policies: []config.Policy{ { Users: []string{}, Actions: []string{}, @@ -1907,13 +1908,13 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { DefaultPolicy: []string{}, }, }, - AdminPolicy: api.Policy{ + AdminPolicy: config.Policy{ Users: []string{}, Actions: []string{}, }, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -1974,10 +1975,10 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { So(resp.StatusCode(), ShouldEqual, 403) // add test user to repo's policy with create perm - config.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Users = - append(config.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Users, "test") - config.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions = - append(config.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions, "create") + conf.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Users = + append(conf.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Users, "test") + conf.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions = + append(conf.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions, "create") // now it should get 202 resp, err = resty.R().SetBasicAuth(username, passphrase). @@ -2020,8 +2021,8 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { So(resp.StatusCode(), ShouldEqual, 403) // get tags with read access should get 200 - config.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions = - append(config.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions, "read") + conf.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions = + append(conf.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions, "read") resp, err = resty.R().SetBasicAuth(username, passphrase). Get(baseURL + "/v2/" + AuthorizationNamespace + "/tags/list") So(err, ShouldBeNil) @@ -2050,8 +2051,8 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { So(resp.StatusCode(), ShouldEqual, 403) // add delete perm on repo - config.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions = - append(config.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions, "delete") + conf.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions = + append(conf.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions, "delete") // delete blob should get 202 resp, err = resty.R().SetBasicAuth(username, passphrase). @@ -2068,10 +2069,10 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { So(resp.StatusCode(), ShouldEqual, 403) // add read perm on repo - config.AccessControl.Repositories["zot-test"] = api.PolicyGroup{Policies: []api.Policy{ + conf.AccessControl.Repositories["zot-test"] = config.PolicyGroup{Policies: []config.Policy{ { - []string{"test"}, - []string{"read"}, + Users: []string{"test"}, + Actions: []string{"read"}, }, }, DefaultPolicy: []string{}} @@ -2092,8 +2093,8 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { So(resp.StatusCode(), ShouldEqual, 403) // add create perm on repo - config.AccessControl.Repositories["zot-test"].Policies[0].Actions = - append(config.AccessControl.Repositories["zot-test"].Policies[0].Actions, "create") + conf.AccessControl.Repositories["zot-test"].Policies[0].Actions = + append(conf.AccessControl.Repositories["zot-test"].Policies[0].Actions, "create") // should get 201 with create perm resp, err = resty.R().SetBasicAuth(username, passphrase). @@ -2112,8 +2113,8 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { So(resp.StatusCode(), ShouldEqual, 403) // add update perm on repo - config.AccessControl.Repositories["zot-test"].Policies[0].Actions = - append(config.AccessControl.Repositories["zot-test"].Policies[0].Actions, "update") + conf.AccessControl.Repositories["zot-test"].Policies[0].Actions = + append(conf.AccessControl.Repositories["zot-test"].Policies[0].Actions, "update") // update manifest should get 201 with update perm resp, err = resty.R().SetBasicAuth(username, passphrase). @@ -2125,10 +2126,10 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { So(resp.StatusCode(), ShouldEqual, 201) // now use default repo policy - config.AccessControl.Repositories["zot-test"].Policies[0].Actions = []string{} - repoPolicy := config.AccessControl.Repositories["zot-test"] + conf.AccessControl.Repositories["zot-test"].Policies[0].Actions = []string{} + repoPolicy := conf.AccessControl.Repositories["zot-test"] repoPolicy.DefaultPolicy = []string{"update"} - config.AccessControl.Repositories["zot-test"] = repoPolicy + conf.AccessControl.Repositories["zot-test"] = repoPolicy // update manifest should get 201 with update perm on repo's default policy resp, err = resty.R().SetBasicAuth(username, passphrase). @@ -2140,10 +2141,10 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { So(resp.StatusCode(), ShouldEqual, 201) // with default read on repo should still get 200 - config.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions = []string{} - repoPolicy = config.AccessControl.Repositories[AuthorizationNamespace] + conf.AccessControl.Repositories[AuthorizationNamespace].Policies[0].Actions = []string{} + repoPolicy = conf.AccessControl.Repositories[AuthorizationNamespace] repoPolicy.DefaultPolicy = []string{"read"} - config.AccessControl.Repositories[AuthorizationNamespace] = repoPolicy + conf.AccessControl.Repositories[AuthorizationNamespace] = repoPolicy resp, err = resty.R().SetBasicAuth(username, passphrase). Get(baseURL + "/v2/" + AuthorizationNamespace + "/tags/list") @@ -2153,7 +2154,7 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { // upload blob without user create but with default create should get 200 repoPolicy.DefaultPolicy = append(repoPolicy.DefaultPolicy, "create") - config.AccessControl.Repositories[AuthorizationNamespace] = repoPolicy + conf.AccessControl.Repositories[AuthorizationNamespace] = repoPolicy resp, err = resty.R().SetBasicAuth(username, passphrase). Post(baseURL + "/v2/" + AuthorizationNamespace + "/blobs/uploads/") @@ -2162,10 +2163,10 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { So(resp.StatusCode(), ShouldEqual, 202) //remove per repo policy - repoPolicy = config.AccessControl.Repositories[AuthorizationNamespace] - repoPolicy.Policies = []api.Policy{} + repoPolicy = conf.AccessControl.Repositories[AuthorizationNamespace] + repoPolicy.Policies = []config.Policy{} repoPolicy.DefaultPolicy = []string{} - config.AccessControl.Repositories[AuthorizationNamespace] = repoPolicy + conf.AccessControl.Repositories[AuthorizationNamespace] = repoPolicy resp, err = resty.R().SetBasicAuth(username, passphrase). Post(baseURL + "/v2/" + AuthorizationNamespace + "/blobs/uploads/") @@ -2175,8 +2176,8 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { // let's use admin policy // remove all repo based policy - delete(config.AccessControl.Repositories, AuthorizationNamespace) - delete(config.AccessControl.Repositories, "zot-test") + delete(conf.AccessControl.Repositories, AuthorizationNamespace) + delete(conf.AccessControl.Repositories, "zot-test") // whithout any perm should get 403 resp, err = resty.R().SetBasicAuth(username, passphrase). @@ -2186,8 +2187,8 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { So(resp.StatusCode(), ShouldEqual, 403) // add read perm - config.AccessControl.AdminPolicy.Users = append(config.AccessControl.AdminPolicy.Users, "test") - config.AccessControl.AdminPolicy.Actions = append(config.AccessControl.AdminPolicy.Actions, "read") + conf.AccessControl.AdminPolicy.Users = append(conf.AccessControl.AdminPolicy.Users, "test") + conf.AccessControl.AdminPolicy.Actions = append(conf.AccessControl.AdminPolicy.Actions, "read") // with read perm should get 200 resp, err = resty.R().SetBasicAuth(username, passphrase). Get(baseURL + "/v2/" + AuthorizationNamespace + "/tags/list") @@ -2203,7 +2204,7 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { So(resp.StatusCode(), ShouldEqual, 403) // add create perm - config.AccessControl.AdminPolicy.Actions = append(config.AccessControl.AdminPolicy.Actions, "create") + conf.AccessControl.AdminPolicy.Actions = append(conf.AccessControl.AdminPolicy.Actions, "create") // with create perm should get 202 resp, err = resty.R().SetBasicAuth(username, passphrase). Post(baseURL + "/v2/" + AuthorizationNamespace + "/blobs/uploads/") @@ -2231,7 +2232,7 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { So(resp.StatusCode(), ShouldEqual, 403) // add delete perm - config.AccessControl.AdminPolicy.Actions = append(config.AccessControl.AdminPolicy.Actions, "delete") + conf.AccessControl.AdminPolicy.Actions = append(conf.AccessControl.AdminPolicy.Actions, "delete") // with delete perm should get 202 resp, err = resty.R().SetBasicAuth(username, passphrase). Delete(baseURL + "/v2/" + AuthorizationNamespace + "/blobs/" + digest) @@ -2247,7 +2248,7 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { So(resp.StatusCode(), ShouldEqual, 403) // add update perm - config.AccessControl.AdminPolicy.Actions = append(config.AccessControl.AdminPolicy.Actions, "update") + conf.AccessControl.AdminPolicy.Actions = append(conf.AccessControl.AdminPolicy.Actions, "update") // update manifest should get 201 with update perm resp, err = resty.R().SetBasicAuth(username, passphrase). SetHeader("Content-type", "application/vnd.oci.image.manifest.v1+json"). @@ -2257,7 +2258,7 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 201) - config.AccessControl = &api.AccessControlConfig{} + conf.AccessControl = &config.AccessControlConfig{} resp, err = resty.R().SetBasicAuth(username, passphrase). SetHeader("Content-type", "application/vnd.oci.image.manifest.v1+json"). @@ -2274,19 +2275,19 @@ func TestInvalidCases(t *testing.T) { port := getFreePort() baseURL := getBaseURL(port, false) - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase)) defer os.Remove(htpasswdPath) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) err := os.Mkdir("oci-repo-test", 0000) if err != nil { @@ -2343,19 +2344,19 @@ func TestHTTPReadOnly(t *testing.T) { for _, testString := range singleCredtests { func() { - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port // enable read-only mode - config.HTTP.ReadOnly = true + conf.HTTP.ReadOnly = true htpasswdPath := makeHtpasswdFileFromString(testString) defer os.Remove(htpasswdPath) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -2408,19 +2409,19 @@ func TestCrossRepoMount(t *testing.T) { port := getFreePort() baseURL := getBaseURL(port, false) - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase)) defer os.Remove(htpasswdPath) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { @@ -2611,19 +2612,19 @@ func TestCrossRepoMount(t *testing.T) { port := getFreePort() baseURL := getBaseURL(port, false) - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase)) defer os.Remove(htpasswdPath) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) //defer stopServer(c) @@ -2768,17 +2769,17 @@ func TestParallelRequests(t *testing.T) { port := getFreePort() baseURL := getBaseURL(port, false) - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase)) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { @@ -2795,10 +2796,10 @@ func TestParallelRequests(t *testing.T) { panic(err) } - subPaths := make(map[string]api.StorageConfig) + subPaths := make(map[string]config.StorageConfig) - subPaths["/a"] = api.StorageConfig{RootDirectory: firstSubDir} - subPaths["/b"] = api.StorageConfig{RootDirectory: secondSubDir} + subPaths["/a"] = config.StorageConfig{RootDirectory: firstSubDir} + subPaths["/b"] = config.StorageConfig{RootDirectory: secondSubDir} c.Config.Storage.SubPaths = subPaths @@ -3162,17 +3163,17 @@ func TestHardLink(t *testing.T) { port := getFreePort() baseURL := getBaseURL(port, false) - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase)) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "hard-link-test") if err != nil { @@ -3197,9 +3198,9 @@ func TestHardLink(t *testing.T) { } c.Config.Storage.RootDirectory = dir - subPaths := make(map[string]api.StorageConfig) + subPaths := make(map[string]config.StorageConfig) - subPaths["/a"] = api.StorageConfig{RootDirectory: subDir, Dedupe: true} + subPaths["/a"] = config.StorageConfig{RootDirectory: subDir, Dedupe: true} c.Config.Storage.SubPaths = subPaths diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 627ed701..5b310ca4 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -97,7 +97,7 @@ func (rh *RouteHandler) SetupRoutes() { rh.c.Router.PathPrefix("/swagger/v2/").Methods("GET").Handler(httpSwagger.WrapHandler) // Setup Extensions Routes if rh.c.Config != nil && rh.c.Config.Extensions != nil { - ext.SetupRoutes(rh.c.Config.Extensions, rh.c.Router, rh.c.StoreController, rh.c.Log) + ext.SetupRoutes(rh.c.Config, rh.c.Router, rh.c.StoreController, rh.c.Log) } } @@ -273,7 +273,7 @@ func (rh *RouteHandler) CheckManifest(w http.ResponseWriter, r *http.Request) { return } - _, digest, mediaType, err := is.GetImageManifest(name, reference) + _, digest, mediaType, err := getImageManifest(rh, is, name, reference) if err != nil { switch err { case errors.ErrRepoNotFound: @@ -331,7 +331,8 @@ func (rh *RouteHandler) GetManifest(w http.ResponseWriter, r *http.Request) { return } - content, digest, mediaType, err := is.GetImageManifest(name, reference) + content, digest, mediaType, err := getImageManifest(rh, is, name, reference) + if err != nil { switch err { case errors.ErrRepoNotFound: @@ -1240,3 +1241,47 @@ func WriteDataFromReader(w http.ResponseWriter, status int, length int64, mediaT func (rh *RouteHandler) getImageStore(name string) storage.ImageStore { return rh.c.StoreController.GetImageStore(name) } + +// will sync on demand if an image is not found, in case sync extensions is enabled. +func getImageManifest(rh *RouteHandler, is storage.ImageStore, name, + reference string) ([]byte, string, string, error) { + content, digest, mediaType, err := is.GetImageManifest(name, reference) + + if err != nil { + switch err { + case errors.ErrRepoNotFound: + if rh.c.Config.Extensions != nil && rh.c.Config.Extensions.Sync != nil { + rh.c.Log.Info().Msgf("image not found, trying to get image %s:%s by syncing on demand", name, reference) + ok, errSync := ext.SyncOneImage(rh.c.Config, rh.c.Log, name, reference) + + switch ok { + case true: + content, digest, mediaType, err = is.GetImageManifest(name, reference) + case false && errSync == nil: + rh.c.Log.Info().Msgf("couldn't find image %s:%s in sync registries", name, reference) + case false && errSync != nil: + rh.c.Log.Err(err).Msgf("error encounter while syncing image %s:%s", name, reference) + } + } + + case errors.ErrManifestNotFound: + if rh.c.Config.Extensions != nil && rh.c.Config.Extensions.Sync != nil { + rh.c.Log.Info().Msgf("manifest not found, trying to get image %s:%s by syncing on demand", name, reference) + ok, errSync := ext.SyncOneImage(rh.c.Config, rh.c.Log, name, reference) + + switch ok { + case true: + content, digest, mediaType, err = is.GetImageManifest(name, reference) + case false && errSync == nil: + rh.c.Log.Info().Msgf("couldn't find image %s:%s in sync registries", name, reference) + case false && errSync != nil: + rh.c.Log.Err(err).Msgf("error encounter while syncing image %s:%s", name, reference) + } + } + default: + return []byte{}, "", "", err + } + } + + return content, digest, mediaType, err +} diff --git a/pkg/cli/client_test.go b/pkg/cli/client_test.go index ed887e31..a5fbd822 100644 --- a/pkg/cli/client_test.go +++ b/pkg/cli/client_test.go @@ -20,6 +20,7 @@ import ( "time" "github.com/anuvu/zot/pkg/api" + "github.com/anuvu/zot/pkg/api/config" . "github.com/smartystreets/goconvey/convey" ) @@ -67,24 +68,24 @@ func TestTLSWithAuth(t *testing.T) { resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() - config := api.NewConfig() - config.HTTP.Port = SecurePort1 + conf := config.New() + conf.HTTP.Port = SecurePort1 htpasswdPath := makeHtpasswdFile() defer os.Remove(htpasswdPath) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - config.HTTP.TLS = &api.TLSConfig{ + conf.HTTP.TLS = &config.TLSConfig{ Cert: ServerCert, Key: ServerKey, CACert: CACert, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -173,15 +174,15 @@ func TestTLSWithoutAuth(t *testing.T) { resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() - config := api.NewConfig() - config.HTTP.Port = SecurePort1 - config.HTTP.TLS = &api.TLSConfig{ + conf := config.New() + conf.HTTP.Port = SecurePort1 + conf.HTTP.TLS = &config.TLSConfig{ Cert: ServerCert, Key: ServerKey, CACert: CACert, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -241,15 +242,15 @@ func TestTLSWithoutAuth(t *testing.T) { resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() - config := api.NewConfig() - config.HTTP.Port = SecurePort2 - config.HTTP.TLS = &api.TLSConfig{ + conf := config.New() + conf.HTTP.Port = SecurePort2 + conf.HTTP.TLS = &config.TLSConfig{ Cert: ServerCert, Key: ServerKey, CACert: CACert, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -304,15 +305,15 @@ func TestTLSBadCerts(t *testing.T) { resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() - config := api.NewConfig() - config.HTTP.Port = SecurePort3 - config.HTTP.TLS = &api.TLSConfig{ + conf := config.New() + conf.HTTP.Port = SecurePort3 + conf.HTTP.TLS = &config.TLSConfig{ Cert: ServerCert, Key: ServerKey, CACert: CACert, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) diff --git a/pkg/cli/cve_cmd_test.go b/pkg/cli/cve_cmd_test.go index 9eb89329..fc566efb 100644 --- a/pkg/cli/cve_cmd_test.go +++ b/pkg/cli/cve_cmd_test.go @@ -16,7 +16,8 @@ import ( zotErrors "github.com/anuvu/zot/errors" "github.com/anuvu/zot/pkg/api" - ext "github.com/anuvu/zot/pkg/extensions" + "github.com/anuvu/zot/pkg/api/config" + extconf "github.com/anuvu/zot/pkg/extensions/config" "gopkg.in/resty.v1" . "github.com/smartystreets/goconvey/convey" @@ -286,9 +287,9 @@ func TestSearchCVECmd(t *testing.T) { func TestServerCVEResponse(t *testing.T) { port := getFreePort() url := getBaseURL(port) - config := api.NewConfig() - config.HTTP.Port = port - c := api.NewController(config) + conf := config.New() + conf.HTTP.Port = port + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { @@ -303,14 +304,14 @@ func TestServerCVEResponse(t *testing.T) { defer os.RemoveAll(dir) c.Config.Storage.RootDirectory = dir - cveConfig := &ext.CVEConfig{ + cveConfig := &extconf.CVEConfig{ UpdateInterval: 2, } - searchConfig := &ext.SearchConfig{ + searchConfig := &extconf.SearchConfig{ CVE: cveConfig, Enable: true, } - c.Config.Extensions = &ext.ExtensionConfig{ + c.Config.Extensions = &extconf.ExtensionConfig{ Search: searchConfig, } diff --git a/pkg/cli/image_cmd_test.go b/pkg/cli/image_cmd_test.go index 27ba09af..063deb45 100644 --- a/pkg/cli/image_cmd_test.go +++ b/pkg/cli/image_cmd_test.go @@ -21,8 +21,9 @@ import ( zotErrors "github.com/anuvu/zot/errors" "github.com/anuvu/zot/pkg/api" + "github.com/anuvu/zot/pkg/api/config" "github.com/anuvu/zot/pkg/compliance/v1_0_0" - "github.com/anuvu/zot/pkg/extensions" + extconf "github.com/anuvu/zot/pkg/extensions/config" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/phayes/freeport" @@ -302,12 +303,12 @@ func TestServerResponse(t *testing.T) { Convey("Test from real server", t, func() { port := getFreePort() url := getBaseURL(port) - config := api.NewConfig() - config.HTTP.Port = port - config.Extensions = &extensions.ExtensionConfig{ - Search: &extensions.SearchConfig{Enable: true}, + conf := config.New() + conf.HTTP.Port = port + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{Enable: true}, } - c := api.NewController(config) + c := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) diff --git a/pkg/cli/root.go b/pkg/cli/root.go index 0e5c8000..614b2cde 100644 --- a/pkg/cli/root.go +++ b/pkg/cli/root.go @@ -3,6 +3,7 @@ package cli import ( "github.com/anuvu/zot/errors" "github.com/anuvu/zot/pkg/api" + "github.com/anuvu/zot/pkg/api/config" "github.com/anuvu/zot/pkg/storage" "github.com/fsnotify/fsnotify" "github.com/mitchellh/mapstructure" @@ -22,7 +23,7 @@ func metadataConfig(md *mapstructure.Metadata) viper.DecoderConfigOption { func NewRootCmd() *cobra.Command { showVersion := false - config := api.NewConfig() + conf := config.New() // "serve" serveCmd := &cobra.Command{ @@ -32,9 +33,9 @@ func NewRootCmd() *cobra.Command { Long: "`serve` stores and distributes OCI images", Run: func(cmd *cobra.Command, args []string) { if len(args) > 0 { - LoadConfiguration(config, args[0]) + LoadConfiguration(conf, args[0]) } - c := api.NewController(config) + c := api.NewController(conf) // creates a new file watcher watcher, err := fsnotify.NewWatcher() @@ -53,7 +54,7 @@ func NewRootCmd() *cobra.Command { case event := <-watcher.Events: if event.Op == fsnotify.Write { log.Info().Msg("Config file changed, trying to reload accessControl config") - newConfig := api.NewConfig() + newConfig := config.New() LoadConfiguration(newConfig, args[0]) c.Config.AccessControl = newConfig.AccessControl } @@ -85,7 +86,7 @@ func NewRootCmd() *cobra.Command { Long: "`verify` validates a zot config file", Run: func(cmd *cobra.Command, args []string) { if len(args) > 0 { - config := api.NewConfig() + config := config.New() LoadConfiguration(config, args[0]) log.Info().Msgf("Config file %s is valid", args[0]) } @@ -102,16 +103,16 @@ func NewRootCmd() *cobra.Command { Short: "`garbage-collect` deletes layers not referenced by any manifests", Long: "`garbage-collect` deletes layers not referenced by any manifests", Run: func(cmd *cobra.Command, args []string) { - log.Info().Interface("values", config).Msg("configuration settings") - if config.Storage.RootDirectory != "" { - if err := storage.Scrub(config.Storage.RootDirectory, gcDryRun); err != nil { + log.Info().Interface("values", conf).Msg("configuration settings") + if conf.Storage.RootDirectory != "" { + if err := storage.Scrub(conf.Storage.RootDirectory, gcDryRun); err != nil { panic(err) } } }, } - gcCmd.Flags().StringVarP(&config.Storage.RootDirectory, "storage-root-dir", "r", "", + gcCmd.Flags().StringVarP(&conf.Storage.RootDirectory, "storage-root-dir", "r", "", "Use specified directory for filestore backing image data") _ = gcCmd.MarkFlagRequired("storage-root-dir") @@ -126,8 +127,8 @@ func NewRootCmd() *cobra.Command { Long: "`zot`", Run: func(cmd *cobra.Command, args []string) { if showVersion { - log.Info().Str("distribution-spec", distspec.Version).Str("commit", api.Commit). - Str("binary-type", api.BinaryType).Msg("version") + log.Info().Str("distribution-spec", distspec.Version).Str("commit", config.Commit). + Str("binary-type", config.BinaryType).Msg("version") } _ = cmd.Usage() cmd.SilenceErrors = false @@ -145,7 +146,7 @@ func NewRootCmd() *cobra.Command { return rootCmd } -func LoadConfiguration(config *api.Config, configPath string) { +func LoadConfiguration(config *config.Config, configPath string) { viper.SetConfigFile(configPath) if err := viper.ReadInConfig(); err != nil { @@ -178,4 +179,15 @@ func LoadConfiguration(config *api.Config, configPath string) { log.Error().Err(errors.ErrBadConfig).Msg("Unable to unmarshal http.accessControl.key.policies") panic(err) } + + // defaults + defualtTLSVerify := true + + if config.Extensions != nil && config.Extensions.Sync != nil { + for id, regCfg := range config.Extensions.Sync.Registries { + if regCfg.TLSVerify == nil { + config.Extensions.Sync.Registries[id].TLSVerify = &defualtTLSVerify + } + } + } } diff --git a/pkg/cli/root_test.go b/pkg/cli/root_test.go index 66cd0656..a7fac07f 100644 --- a/pkg/cli/root_test.go +++ b/pkg/cli/root_test.go @@ -6,7 +6,7 @@ import ( "path" "testing" - "github.com/anuvu/zot/pkg/api" + "github.com/anuvu/zot/pkg/api/config" "github.com/anuvu/zot/pkg/cli" . "github.com/smartystreets/goconvey/convey" "github.com/spf13/viper" @@ -137,7 +137,7 @@ func TestVerify(t *testing.T) { func TestLoadConfig(t *testing.T) { Convey("Test viper load config", t, func(c C) { - config := api.NewConfig() + config := config.New() So(func() { cli.LoadConfiguration(config, "../../examples/config-policy.json") }, ShouldNotPanic) adminPolicy := viper.GetStringMapStringSlice("http.accessControl.adminPolicy") So(config.AccessControl.AdminPolicy.Actions, ShouldResemble, adminPolicy["actions"]) diff --git a/pkg/compliance/v1_0_0/check_test.go b/pkg/compliance/v1_0_0/check_test.go index f7ecfc66..74d59df7 100644 --- a/pkg/compliance/v1_0_0/check_test.go +++ b/pkg/compliance/v1_0_0/check_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/anuvu/zot/pkg/api" + "github.com/anuvu/zot/pkg/api/config" "github.com/anuvu/zot/pkg/compliance" "github.com/anuvu/zot/pkg/compliance/v1_0_0" "github.com/phayes/freeport" @@ -60,10 +61,10 @@ func startServer() (*api.Controller, string) { randomPort := fmt.Sprintf("%d", portInt) fmt.Println(randomPort) - config := api.NewConfig() - config.HTTP.Address = listenAddress - config.HTTP.Port = randomPort - ctrl := api.NewController(config) + conf := config.New() + conf.HTTP.Address = listenAddress + conf.HTTP.Port = randomPort + ctrl := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { @@ -86,10 +87,10 @@ func startServer() (*api.Controller, string) { secondDir = secondSubDir - subPaths := make(map[string]api.StorageConfig) + subPaths := make(map[string]config.StorageConfig) - subPaths["/firsttest"] = api.StorageConfig{RootDirectory: firstSubDir} - subPaths["/secondtest"] = api.StorageConfig{RootDirectory: secondSubDir} + subPaths["/firsttest"] = config.StorageConfig{RootDirectory: firstSubDir} + subPaths["/secondtest"] = config.StorageConfig{RootDirectory: secondSubDir} ctrl.Config.Storage.RootDirectory = dir diff --git a/pkg/extensions/config.go b/pkg/extensions/config/config.go similarity index 72% rename from pkg/extensions/config.go rename to pkg/extensions/config/config.go index 730b2e3e..3076a6fc 100644 --- a/pkg/extensions/config.go +++ b/pkg/extensions/config/config.go @@ -1,9 +1,14 @@ -package extensions +package config -import "time" +import ( + "time" + + "github.com/anuvu/zot/pkg/extensions/sync" +) type ExtensionConfig struct { Search *SearchConfig + Sync *sync.Config } type SearchConfig struct { diff --git a/pkg/extensions/extensions.go b/pkg/extensions/extensions.go index a263d333..92605a65 100644 --- a/pkg/extensions/extensions.go +++ b/pkg/extensions/extensions.go @@ -3,7 +3,9 @@ package extensions import ( + "github.com/anuvu/zot/pkg/api/config" "github.com/anuvu/zot/pkg/extensions/search" + "github.com/anuvu/zot/pkg/extensions/sync" "github.com/anuvu/zot/pkg/storage" "github.com/gorilla/mux" @@ -31,20 +33,19 @@ func downloadTrivyDB(dbDir string, log log.Logger, updateInterval time.Duration) } } -// EnableExtensions ... -func EnableExtensions(extension *ExtensionConfig, log log.Logger, rootDir string) { - if extension.Search != nil && extension.Search.Enable && extension.Search.CVE != nil { +func EnableExtensions(config *config.Config, log log.Logger, rootDir string) { + if config.Extensions.Search != nil && config.Extensions.Search.Enable && config.Extensions.Search.CVE != nil { defaultUpdateInterval, _ := time.ParseDuration("2h") - if extension.Search.CVE.UpdateInterval < defaultUpdateInterval { - extension.Search.CVE.UpdateInterval = defaultUpdateInterval + if config.Extensions.Search.CVE.UpdateInterval < defaultUpdateInterval { + config.Extensions.Search.CVE.UpdateInterval = defaultUpdateInterval log.Warn().Msg("CVE update interval set to too-short interval <= 1, changing update duration to 2 hours and continuing.") // nolint: lll } go func() { err := downloadTrivyDB(rootDir, log, - extension.Search.CVE.UpdateInterval) + config.Extensions.Search.CVE.UpdateInterval) if err != nil { log.Error().Err(err).Msg("error while downloading TrivyDB") } @@ -52,17 +53,47 @@ func EnableExtensions(extension *ExtensionConfig, log log.Logger, rootDir string } else { log.Info().Msg("CVE config not provided, skipping CVE update") } + + if config.Extensions.Sync != nil { + defaultPollInterval, _ := time.ParseDuration("1h") + for id, registryCfg := range config.Extensions.Sync.Registries { + if registryCfg.PollInterval < defaultPollInterval { + config.Extensions.Sync.Registries[id].PollInterval = defaultPollInterval + + log.Warn().Msg("Sync registries interval set to too-short interval <= 1h, changing update duration to 1 hour and continuing.") // nolint: lll + } + } + + var serverCert string + + var serverKey string + + var CACert string + + if config.HTTP.TLS != nil { + serverCert = config.HTTP.TLS.Cert + serverKey = config.HTTP.TLS.Key + CACert = config.HTTP.TLS.CACert + } + + if err := sync.Run(*config.Extensions.Sync, log, config.HTTP.Address, + config.HTTP.Port, serverCert, serverKey, CACert); err != nil { + log.Error().Err(err).Msg("Error encountered while setting up syncing") + } + } else { + log.Info().Msg("Sync registries config not provided, skipping sync") + } } // SetupRoutes ... -func SetupRoutes(extension *ExtensionConfig, router *mux.Router, storeController storage.StoreController, +func SetupRoutes(config *config.Config, router *mux.Router, storeController storage.StoreController, log log.Logger) { log.Info().Msg("setting up extensions routes") - if extension.Search != nil && extension.Search.Enable { + if config.Extensions.Search != nil && config.Extensions.Search.Enable { var resConfig search.Config - if extension.Search.CVE != nil { + if config.Extensions.Search.CVE != nil { resConfig = search.GetResolverConfig(log, storeController, true) } else { resConfig = search.GetResolverConfig(log, storeController, false) @@ -71,4 +102,52 @@ func SetupRoutes(extension *ExtensionConfig, router *mux.Router, storeController router.PathPrefix("/query").Methods("GET", "POST"). Handler(gqlHandler.NewDefaultServer(search.NewExecutableSchema(resConfig))) } + + var serverCert string + + var serverKey string + + var CACert string + + if config.HTTP.TLS != nil { + serverCert = config.HTTP.TLS.Cert + serverKey = config.HTTP.TLS.Key + CACert = config.HTTP.TLS.CACert + } + + if config.Extensions.Sync != nil { + postSyncer := sync.PostHandler{ + Address: config.HTTP.Address, + Port: config.HTTP.Port, + ServerCert: serverCert, + ServerKey: serverKey, + CACert: CACert, + Cfg: *config.Extensions.Sync, + Log: log, + } + + router.HandleFunc("/sync", postSyncer.Handler).Methods("POST") + } +} + +// SyncOneImage syncs one image. +func SyncOneImage(config *config.Config, log log.Logger, repoName, reference string) (bool, error) { + log.Info().Msgf("syncing image %s:%s", repoName, reference) + + var serverCert string + + var serverKey string + + var CACert string + + if config.HTTP.TLS != nil { + serverCert = config.HTTP.TLS.Cert + serverKey = config.HTTP.TLS.Key + CACert = config.HTTP.TLS.CACert + } + + ok, err := sync.OneImage(*config.Extensions.Sync, log, config.HTTP.Address, config.HTTP.Port, + serverCert, serverKey, CACert, repoName, reference) + + return ok, err } diff --git a/pkg/extensions/minimal.go b/pkg/extensions/minimal.go index 2101f978..a889c30e 100644 --- a/pkg/extensions/minimal.go +++ b/pkg/extensions/minimal.go @@ -5,6 +5,7 @@ package extensions import ( "time" + "github.com/anuvu/zot/pkg/api/config" "github.com/anuvu/zot/pkg/log" "github.com/anuvu/zot/pkg/storage" "github.com/gorilla/mux" @@ -16,11 +17,17 @@ func downloadTrivyDB(dbDir string, log log.Logger, updateInterval time.Duration) } // EnableExtensions ... -func EnableExtensions(extension *ExtensionConfig, log log.Logger, rootDir string) { +func EnableExtensions(config *config.Config, log log.Logger, rootDir string) { log.Warn().Msg("skipping enabling extensions because given zot binary doesn't support any extensions, please build zot full binary for this feature") } // SetupRoutes ... -func SetupRoutes(extension *ExtensionConfig, router *mux.Router, storeController storage.StoreController, log log.Logger) { +func SetupRoutes(conf *config.Config, router *mux.Router, storeController storage.StoreController, log log.Logger) { log.Warn().Msg("skipping setting up extensions routes because given zot binary doesn't support any extensions, please build zot full binary for this feature") } + +// SyncOneImage... +func SyncOneImage(config *config.Config, log log.Logger, repoName, reference string) (bool, error) { + log.Warn().Msg("skipping syncing on demand because given zot binary doesn't support any extensions, please build zot full binary for this feature") + return false, nil +} diff --git a/pkg/extensions/search/common/common_test.go b/pkg/extensions/search/common/common_test.go index c98db392..ba249d50 100644 --- a/pkg/extensions/search/common/common_test.go +++ b/pkg/extensions/search/common/common_test.go @@ -11,7 +11,8 @@ import ( "time" "github.com/anuvu/zot/pkg/api" - ext "github.com/anuvu/zot/pkg/extensions" + "github.com/anuvu/zot/pkg/api/config" + extconf "github.com/anuvu/zot/pkg/extensions/config" "github.com/anuvu/zot/pkg/extensions/search/common" "github.com/anuvu/zot/pkg/log" "github.com/anuvu/zot/pkg/storage" @@ -214,18 +215,18 @@ func TestLatestTagSearchHTTP(t *testing.T) { if err != nil { panic(err) } - config := api.NewConfig() - config.HTTP.Port = Port1 - config.Storage.RootDirectory = rootDir - config.Storage.SubPaths = make(map[string]api.StorageConfig) - config.Storage.SubPaths["/a"] = api.StorageConfig{RootDirectory: subRootDir} - config.Extensions = &ext.ExtensionConfig{ - Search: &ext.SearchConfig{Enable: true}, + conf := config.New() + conf.HTTP.Port = Port1 + conf.Storage.RootDirectory = rootDir + conf.Storage.SubPaths = make(map[string]config.StorageConfig) + conf.Storage.SubPaths["/a"] = config.StorageConfig{RootDirectory: subRootDir} + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{Enable: true}, } - config.Extensions.Search.CVE = nil + conf.Extensions.Search.CVE = nil - c := api.NewController(config) + c := api.NewController(conf) go func() { // this blocks diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 4df89698..66ced2bf 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -13,7 +13,8 @@ import ( "time" "github.com/anuvu/zot/pkg/api" - ext "github.com/anuvu/zot/pkg/extensions" + "github.com/anuvu/zot/pkg/api/config" + extconf "github.com/anuvu/zot/pkg/extensions/config" "github.com/anuvu/zot/pkg/extensions/search/common" cveinfo "github.com/anuvu/zot/pkg/extensions/search/cve" "github.com/anuvu/zot/pkg/log" @@ -448,26 +449,26 @@ func TestCVESearch(t *testing.T) { updateDuration, _ = time.ParseDuration("1h") port := getFreePort() baseURL := getBaseURL(port) - config := api.NewConfig() - config.HTTP.Port = port + conf := config.New() + conf.HTTP.Port = port htpasswdPath := makeHtpasswdFile() defer os.Remove(htpasswdPath) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) c.Config.Storage.RootDirectory = dbDir - cveConfig := &ext.CVEConfig{ + cveConfig := &extconf.CVEConfig{ UpdateInterval: updateDuration, } - searchConfig := &ext.SearchConfig{ + searchConfig := &extconf.SearchConfig{ CVE: cveConfig, Enable: true, } - c.Config.Extensions = &ext.ExtensionConfig{ + c.Config.Extensions = &extconf.ExtensionConfig{ Search: searchConfig, } go func() { @@ -673,18 +674,18 @@ func TestCVESearch(t *testing.T) { func TestCVEConfig(t *testing.T) { Convey("Verify CVE config", t, func() { - config := api.NewConfig() - port := config.HTTP.Port + conf := config.New() + port := conf.HTTP.Port baseURL := getBaseURL(port) htpasswdPath := makeHtpasswdFile() defer os.Remove(htpasswdPath) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) firstDir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -703,8 +704,8 @@ func TestCVEConfig(t *testing.T) { } c.Config.Storage.RootDirectory = firstDir - subPaths := make(map[string]api.StorageConfig) - subPaths["/a"] = api.StorageConfig{ + subPaths := make(map[string]config.StorageConfig) + subPaths["/a"] = config.StorageConfig{ RootDirectory: secondDir, } c.Config.Storage.SubPaths = subPaths diff --git a/pkg/extensions/search/digest/digest_test.go b/pkg/extensions/search/digest/digest_test.go index eb7ae0fe..19b0bfc3 100644 --- a/pkg/extensions/search/digest/digest_test.go +++ b/pkg/extensions/search/digest/digest_test.go @@ -12,7 +12,8 @@ import ( "time" "github.com/anuvu/zot/pkg/api" - ext "github.com/anuvu/zot/pkg/extensions" + "github.com/anuvu/zot/pkg/api/config" + extconf "github.com/anuvu/zot/pkg/extensions/config" digestinfo "github.com/anuvu/zot/pkg/extensions/search/digest" "github.com/anuvu/zot/pkg/log" "github.com/anuvu/zot/pkg/storage" @@ -183,14 +184,14 @@ func TestDigestInfo(t *testing.T) { func TestDigestSearchHTTP(t *testing.T) { Convey("Test image search by digest scanning", t, func() { - config := api.NewConfig() - config.HTTP.Port = Port1 - config.Storage.RootDirectory = rootDir - config.Extensions = &ext.ExtensionConfig{ - Search: &ext.SearchConfig{Enable: true}, + conf := config.New() + conf.HTTP.Port = Port1 + conf.Storage.RootDirectory = rootDir + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{Enable: true}, } - c := api.NewController(config) + c := api.NewController(conf) go func() { // this blocks @@ -309,13 +310,13 @@ func TestDigestSearchHTTP(t *testing.T) { func TestDigestSearchHTTPSubPaths(t *testing.T) { Convey("Test image search by digest scanning using storage subpaths", t, func() { - config := api.NewConfig() - config.HTTP.Port = Port1 - config.Extensions = &ext.ExtensionConfig{ - Search: &ext.SearchConfig{Enable: true}, + conf := config.New() + conf.HTTP.Port = Port1 + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{Enable: true}, } - c := api.NewController(config) + c := api.NewController(conf) globalDir, err := ioutil.TempDir("", "digest_test") if err != nil { @@ -325,9 +326,9 @@ func TestDigestSearchHTTPSubPaths(t *testing.T) { c.Config.Storage.RootDirectory = globalDir - subPathMap := make(map[string]api.StorageConfig) + subPathMap := make(map[string]config.StorageConfig) - subPathMap["/a"] = api.StorageConfig{RootDirectory: subRootDir} + subPathMap["/a"] = config.StorageConfig{RootDirectory: subRootDir} c.Config.Storage.SubPaths = subPathMap @@ -380,14 +381,14 @@ func TestDigestSearchDisabled(t *testing.T) { Convey("Test disabling image search", t, func() { dir, err := ioutil.TempDir("", "digest_test") So(err, ShouldBeNil) - config := api.NewConfig() - config.HTTP.Port = Port1 - config.Storage.RootDirectory = dir - config.Extensions = &ext.ExtensionConfig{ - Search: &ext.SearchConfig{Enable: false}, + conf := config.New() + conf.HTTP.Port = Port1 + conf.Storage.RootDirectory = dir + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{Enable: false}, } - c := api.NewController(config) + c := api.NewController(conf) go func() { // this blocks diff --git a/pkg/extensions/sync/http_handler.go b/pkg/extensions/sync/http_handler.go new file mode 100644 index 00000000..39b88215 --- /dev/null +++ b/pkg/extensions/sync/http_handler.go @@ -0,0 +1,60 @@ +package sync + +import ( + "fmt" + "net/http" + "strings" + + "github.com/anuvu/zot/pkg/log" +) + +type PostHandler struct { + Address string + Port string + ServerCert string + ServerKey string + CACert string + Cfg Config + Log log.Logger +} + +func (h *PostHandler) Handler(w http.ResponseWriter, r *http.Request) { + upstreamCtx, policyCtx, err := getLocalContexts(h.ServerCert, h.ServerKey, h.CACert, h.Log) + if err != nil { + WriteData(w, http.StatusInternalServerError, err.Error()) + + return + } + + defer policyCtx.Destroy() //nolint: errcheck + + var credentialsFile CredentialsFile + + if h.Cfg.CredentialsFile != "" { + credentialsFile, err = getFileCredentials(h.Cfg.CredentialsFile) + if err != nil { + h.Log.Error().Err(err).Msgf("couldn't get registry credentials from %s", h.Cfg.CredentialsFile) + WriteData(w, http.StatusInternalServerError, err.Error()) + } + } + + localRegistryName := strings.Replace(fmt.Sprintf("%s:%s", h.Address, h.Port), "0.0.0.0", "127.0.0.1", 1) + + for _, regCfg := range h.Cfg.Registries { + upstreamRegistryName := strings.Replace(strings.Replace(regCfg.URL, "http://", "", 1), "https://", "", 1) + + if err := syncRegistry(regCfg, h.Log, localRegistryName, upstreamCtx, policyCtx, + credentialsFile[upstreamRegistryName]); err != nil { + h.Log.Err(err).Msg("error while syncing") + WriteData(w, http.StatusInternalServerError, err.Error()) + } + } + + WriteData(w, http.StatusOK, "") +} + +func WriteData(w http.ResponseWriter, status int, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(msg)) +} diff --git a/pkg/extensions/sync/on_demand.go b/pkg/extensions/sync/on_demand.go new file mode 100644 index 00000000..b06595b2 --- /dev/null +++ b/pkg/extensions/sync/on_demand.go @@ -0,0 +1,90 @@ +package sync + +import ( + "context" + "fmt" + "strings" + + "github.com/anuvu/zot/pkg/log" + "github.com/containers/common/pkg/retry" + "github.com/containers/image/v5/copy" + "github.com/containers/image/v5/docker" + "github.com/containers/image/v5/docker/reference" +) + +func OneImage(cfg Config, log log.Logger, + address, port, serverCert, serverKey, caCert, repoName, tag string) (bool, error) { + localCtx, policyCtx, err := getLocalContexts(serverCert, serverKey, caCert, log) + if err != nil { + return false, err + } + + localRegistryName := strings.Replace(fmt.Sprintf("%s:%s", address, port), "0.0.0.0", "127.0.0.1", 1) + + var credentialsFile CredentialsFile + + if cfg.CredentialsFile != "" { + credentialsFile, err = getFileCredentials(cfg.CredentialsFile) + if err != nil { + log.Error().Err(err).Msgf("couldn't get registry credentials from %s", cfg.CredentialsFile) + return false, err + } + } + + var synced bool + + for _, regCfg := range cfg.Registries { + if !regCfg.OnDemand { + log.Info().Msgf("skipping syncing on demand from %s, onDemand flag is false", regCfg.URL) + continue + } + + registryConfig := regCfg + log.Info().Msgf("syncing on demand with %s", registryConfig.URL) + + upstreamRegistryName := strings.Replace(strings.Replace(regCfg.URL, "http://", "", 1), "https://", "", 1) + + upstreamCtx := getUpstreamContext(®istryConfig, credentialsFile[upstreamRegistryName]) + + upstreamRepoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", upstreamRegistryName, repoName)) + + upstreamTaggedRef, err := reference.WithTag(upstreamRepoRef, tag) + if err != nil { + log.Err(err).Msgf("error creating a reference for repository %s and tag %q", upstreamRepoRef.Name(), tag) + return synced, err + } + + upstreamRef, err := docker.NewReference(upstreamTaggedRef) + ref := strings.Replace(upstreamRef.DockerReference().String(), upstreamRegistryName, "", 1) + + localRef, err := docker.Transport.ParseReference( + fmt.Sprintf("//%s%s", localRegistryName, ref), + ) + if err != nil { + return synced, err + } + + log.Info().Msgf("copying image %s to %s", upstreamRef.DockerReference().Name(), localRef.DockerReference().Name()) + + options := getCopyOptions(upstreamCtx, localCtx) + + retryOptions := &retry.RetryOptions{ + MaxRetry: maxRetries, + } + + if err = retry.RetryIfNecessary(context.Background(), func() error { + _, err = copy.Image(context.Background(), policyCtx, localRef, upstreamRef, &options) + return err + }, retryOptions); err != nil { + log.Error().Err(err).Msgf("error while copying image %s to %s", + upstreamRef.DockerReference().Name(), localRef.DockerReference().Name()) + } else { + log.Info().Msgf("successfully synced %s", upstreamRef.DockerReference().Name()) + synced = true + + return synced, nil + } + } + + return synced, nil +} diff --git a/pkg/extensions/sync/sync.go b/pkg/extensions/sync/sync.go new file mode 100644 index 00000000..c700fa92 --- /dev/null +++ b/pkg/extensions/sync/sync.go @@ -0,0 +1,469 @@ +package sync + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "regexp" + "strings" + "time" + + "github.com/Masterminds/semver" + "github.com/anuvu/zot/errors" + "github.com/anuvu/zot/pkg/log" + "github.com/containers/common/pkg/retry" + "github.com/containers/image/v5/copy" + "github.com/containers/image/v5/docker" + "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/signature" + "github.com/containers/image/v5/types" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + "gopkg.in/resty.v1" +) + +const ( + maxRetries = 3 + delay = 5 * time.Minute +) + +// /v2/_catalog struct. +type catalog struct { + Repositories []string `json:"repositories"` +} + +// key is registry address. +type CredentialsFile map[string]Credentials + +type Credentials struct { + Username string + Password string +} + +type Config struct { + CredentialsFile string + Registries []RegistryConfig +} + +type RegistryConfig struct { + URL string + PollInterval time.Duration + Content []Content + TLSVerify *bool + OnDemand bool + CertDir string +} + +type Content struct { + Prefix string + Tags *Tags +} + +type Tags struct { + Regex *string + Semver *bool +} + +// getUpstreamCatalog gets all repos from a registry. +func getUpstreamCatalog(regCfg *RegistryConfig, credentials Credentials, log log.Logger) (catalog, error) { + var c catalog + + registryCatalogURL := fmt.Sprintf("%s%s", regCfg.URL, "/v2/_catalog") + client := resty.New() + + if regCfg.CertDir != "" { + log.Debug().Msgf("sync: using certs directory: %s", regCfg.CertDir) + clientCert := fmt.Sprintf("%s/client.cert", regCfg.CertDir) + clientKey := fmt.Sprintf("%s/client.key", regCfg.CertDir) + caCertPath := fmt.Sprintf("%s/ca.crt", regCfg.CertDir) + + caCert, err := ioutil.ReadFile(caCertPath) + if err != nil { + return c, err + } + + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + + client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) + + cert, err := tls.LoadX509KeyPair(clientCert, clientKey) + if err != nil { + return c, err + } + + client.SetCertificates(cert) + } + + if credentials.Username != "" && credentials.Password != "" { + log.Debug().Msgf("sync: using basic auth") + client.SetBasicAuth(credentials.Username, credentials.Password) + } + + resp, err := client.R().SetHeader("Content-Type", "application/json").Get(registryCatalogURL) + if err != nil { + log.Err(err).Msgf("couldn't query %s", registryCatalogURL) + return c, err + } + + if resp.IsError() { + log.Error().Msgf("couldn't query %s, status code: %d, body: %s", registryCatalogURL, + resp.StatusCode(), resp.Body()) + return c, errors.ErrSyncMissingCatalog + } + + err = json.Unmarshal(resp.Body(), &c) + if err != nil { + log.Err(err).Str("body", string(resp.Body())).Msg("couldn't unmarshal registry's catalog") + return c, err + } + + return c, nil +} + +// getImageTags lists all tags in a repository. +// It returns a string slice of tags and any error encountered. +func getImageTags(ctx context.Context, sysCtx *types.SystemContext, repoRef reference.Named) ([]string, error) { + dockerRef, err := docker.NewReference(reference.TagNameOnly(repoRef)) + if err != nil { + return nil, err // Should never happen for a reference with tag and no digest + } + + tags, err := docker.GetRepositoryTags(ctx, sysCtx, dockerRef) + if err != nil { + return nil, err + } + + return tags, nil +} + +// filterImagesByTagRegex filters images by tag regex give in the config. +func filterImagesByTagRegex(upstreamReferences *[]types.ImageReference, content Content, log log.Logger) error { + refs := *upstreamReferences + + if content.Tags == nil { + // no need to filter anything + return nil + } + + if content.Tags.Regex != nil { + log.Info().Msgf("start filtering using the regular expression: %s", *content.Tags.Regex) + + tagReg, err := regexp.Compile(*content.Tags.Regex) + if err != nil { + return err + } + + n := 0 + + for _, ref := range refs { + tagged := getTagFromRef(ref, log) + if tagged != nil { + if tagReg.MatchString(tagged.Tag()) { + refs[n] = ref + n++ + } + } + } + + refs = refs[:n] + } + + *upstreamReferences = refs + + return nil +} + +// filterImagesBySemver filters images by checking if their tags are semver compliant. +func filterImagesBySemver(upstreamReferences *[]types.ImageReference, content Content, log log.Logger) { + refs := *upstreamReferences + + if content.Tags == nil { + return + } + + if content.Tags.Semver != nil && *content.Tags.Semver { + log.Info().Msg("start filtering using semver compliant rule") + + n := 0 + + for _, ref := range refs { + tagged := getTagFromRef(ref, log) + if tagged != nil { + _, ok := semver.NewVersion(tagged.Tag()) + if ok == nil { + refs[n] = ref + n++ + } + } + } + + refs = refs[:n] + } + + *upstreamReferences = refs +} + +// imagesToCopyFromRepos lists all images given a registry name and its repos. +func imagesToCopyFromUpstream(registryName string, repos []string, sourceCtx *types.SystemContext, + content Content, log log.Logger) ([]types.ImageReference, error) { + var upstreamReferences []types.ImageReference + + for _, repoName := range repos { + repoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", registryName, repoName)) + if err != nil { + return nil, err + } + + tags, err := getImageTags(context.Background(), sourceCtx, repoRef) + if err != nil { + return nil, err + } + + for _, tag := range tags { + taggedRef, err := reference.WithTag(repoRef, tag) + if err != nil { + log.Err(err).Msgf("error creating a reference for repository %s and tag %q", repoRef.Name(), tag) + return nil, err + } + + ref, err := docker.NewReference(taggedRef) + if err != nil { + log.Err(err).Msgf("cannot obtain a valid image reference for transport %q and reference %s", + docker.Transport.Name(), taggedRef.String()) + return nil, err + } + + upstreamReferences = append(upstreamReferences, ref) + } + } + + log.Debug().Msgf("upstream refs to be copied: %v", upstreamReferences) + + err := filterImagesByTagRegex(&upstreamReferences, content, log) + if err != nil { + return []types.ImageReference{}, err + } + + log.Debug().Msgf("remaining upstream refs to be copied: %v", upstreamReferences) + filterImagesBySemver(&upstreamReferences, content, log) + + log.Debug().Msgf("remaining upstream refs to be copied: %v", upstreamReferences) + + return upstreamReferences, nil +} + +func getCopyOptions(upstreamCtx, localCtx *types.SystemContext) copy.Options { + options := copy.Options{ + DestinationCtx: localCtx, + SourceCtx: upstreamCtx, + // force only oci manifest MIME type + ForceManifestMIMEType: ispec.MediaTypeImageManifest, + } + + return options +} + +func getUpstreamContext(regCfg *RegistryConfig, credentials Credentials) *types.SystemContext { + upstreamCtx := &types.SystemContext{} + + upstreamCtx.DockerCertPath = regCfg.CertDir + upstreamCtx.DockerDaemonCertPath = regCfg.CertDir + + if regCfg.TLSVerify != nil && *regCfg.TLSVerify { + upstreamCtx.DockerDaemonInsecureSkipTLSVerify = false + upstreamCtx.DockerInsecureSkipTLSVerify = types.NewOptionalBool(false) + } else { + upstreamCtx.DockerDaemonInsecureSkipTLSVerify = true + upstreamCtx.DockerInsecureSkipTLSVerify = types.NewOptionalBool(true) + } + + if credentials != (Credentials{}) { + upstreamCtx.DockerAuthConfig = &types.DockerAuthConfig{ + Username: credentials.Username, + Password: credentials.Password, + } + } + + return upstreamCtx +} + +func syncRegistry(regCfg RegistryConfig, log log.Logger, localRegistryName string, localCtx *types.SystemContext, + policyCtx *signature.PolicyContext, credentials Credentials) error { + if len(regCfg.Content) == 0 { + log.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL) + return nil + } + + log.Info().Msgf("syncing registry: %s", regCfg.URL) + + var err error + + log.Debug().Msg("getting upstream context") + + upstreamCtx := getUpstreamContext(®Cfg, credentials) + options := getCopyOptions(upstreamCtx, localCtx) + + retryOptions := &retry.RetryOptions{ + MaxRetry: maxRetries, + Delay: delay, + } + + var catalog catalog + + if err = retry.RetryIfNecessary(context.Background(), func() error { + catalog, err = getUpstreamCatalog(®Cfg, credentials, log) + return err + }, retryOptions); err != nil { + log.Error().Err(err).Msg("error while getting upstream catalog, retrying...") + return err + } + + upstreamRegistryName := strings.Replace(strings.Replace(regCfg.URL, "http://", "", 1), "https://", "", 1) + + log.Info().Msg("filtering repos based on sync prefixes") + + repos := filterRepos(catalog.Repositories, regCfg.Content) + + log.Info().Msgf("got repos: %v", repos) + + var images []types.ImageReference + + for contentID, repos := range repos { + r := repos + id := contentID + + if err = retry.RetryIfNecessary(context.Background(), func() error { + refs, err := imagesToCopyFromUpstream(upstreamRegistryName, r, upstreamCtx, regCfg.Content[id], log) + images = append(images, refs...) + return err + }, retryOptions); err != nil { + log.Error().Err(err).Msg("error while getting images references from upstream, retrying...") + return err + } + } + + if len(images) == 0 { + log.Info().Msg("no images to copy, no need to sync") + return nil + } + + for _, ref := range images { + upstreamRef := ref + + suffix := strings.Replace(ref.DockerReference().String(), upstreamRegistryName, "", 1) + + localRef, err := docker.Transport.ParseReference( + fmt.Sprintf("//%s%s", localRegistryName, suffix), + ) + if err != nil { + return err + } + + log.Info().Msgf("copying image %s to %s", upstreamRef.DockerReference().Name(), localRef.DockerReference().Name()) + + if err = retry.RetryIfNecessary(context.Background(), func() error { + _, err = copy.Image(context.Background(), policyCtx, localRef, upstreamRef, &options) + return err + }, retryOptions); err != nil { + log.Error().Err(err).Msgf("error while copying image %s to %s", + upstreamRef.DockerReference().Name(), localRef.DockerReference().Name()) + return err + } + } + + log.Info().Msgf("finished syncing %s", regCfg.URL) + + return nil +} + +func getLocalContexts(serverCert, serverKey, + caCert string, log log.Logger) (*types.SystemContext, *signature.PolicyContext, error) { + log.Debug().Msg("getting local context") + + var policy *signature.Policy + + var err error + + localCtx := &types.SystemContext{} + + if serverCert != "" && serverKey != "" { + certsDir, err := copyLocalCerts(serverCert, serverKey, caCert, log) + if err != nil { + return &types.SystemContext{}, &signature.PolicyContext{}, err + } + + localCtx.DockerDaemonCertPath = certsDir + localCtx.DockerCertPath = certsDir + + policy, err = signature.DefaultPolicy(localCtx) + if err != nil { + return &types.SystemContext{}, &signature.PolicyContext{}, err + } + } else { + localCtx.DockerDaemonInsecureSkipTLSVerify = true + localCtx.DockerInsecureSkipTLSVerify = types.NewOptionalBool(true) + policy = &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}} + } + + policyContext, err := signature.NewPolicyContext(policy) + if err != nil { + return &types.SystemContext{}, &signature.PolicyContext{}, err + } + + return localCtx, policyContext, nil +} + +func Run(cfg Config, log log.Logger, address, port, serverCert, serverKey, caCert string) error { + localCtx, policyCtx, err := getLocalContexts(serverCert, serverKey, caCert, log) + if err != nil { + return err + } + + localRegistry := strings.Replace(fmt.Sprintf("%s:%s", address, port), "0.0.0.0", "127.0.0.1", 1) + + var credentialsFile CredentialsFile + + if cfg.CredentialsFile != "" { + credentialsFile, err = getFileCredentials(cfg.CredentialsFile) + if err != nil { + log.Error().Err(err).Msgf("couldn't get registry credentials from %s", cfg.CredentialsFile) + return err + } + } + + var ticker *time.Ticker + + for _, regCfg := range cfg.Registries { + // schedule each registry sync + ticker = time.NewTicker(regCfg.PollInterval) + + upstreamRegistry := strings.Replace(strings.Replace(regCfg.URL, "http://", "", 1), "https://", "", 1) + + go func(regCfg RegistryConfig) { + defer os.RemoveAll(certsDir) + // run sync first, then run on interval + if err := syncRegistry(regCfg, log, localRegistry, localCtx, policyCtx, + credentialsFile[upstreamRegistry]); err != nil { + log.Err(err).Msg("sync exited with error, stopping it...") + ticker.Stop() + } + + // run on intervals + for range ticker.C { + if err := syncRegistry(regCfg, log, localRegistry, localCtx, policyCtx, + credentialsFile[upstreamRegistry]); err != nil { + log.Err(err).Msg("sync exited with error, stopping it...") + ticker.Stop() + } + } + }(regCfg) + } + + log.Info().Msg("finished setting up sync") + + return nil +} diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go new file mode 100644 index 00000000..0839f6a7 --- /dev/null +++ b/pkg/extensions/sync/sync_internal_test.go @@ -0,0 +1,109 @@ +package sync + +import ( + "context" + "fmt" + "io/ioutil" + "testing" + "time" + + "github.com/anuvu/zot/errors" + "github.com/anuvu/zot/pkg/log" + "github.com/containers/image/v5/docker" + "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/types" + . "github.com/smartystreets/goconvey/convey" +) + +const ( + BaseURL = "http://127.0.0.1:5001" + ServerCert = "../../../test/data/server.cert" + ServerKey = "../../../test/data/server.key" + CACert = "../../../test/data/ca.crt" + + testImage = "zot-test" + testImageTag = "0.0.1" + + host = "127.0.0.1:45117" +) + +func TestSyncInternal(t *testing.T) { + Convey("test parseRepositoryReference func", t, func() { + repositoryReference := fmt.Sprintf("%s/%s", host, testImage) + ref, err := parseRepositoryReference(repositoryReference) + So(err, ShouldBeNil) + So(ref.Name(), ShouldEqual, repositoryReference) + + repositoryReference = fmt.Sprintf("%s/%s:tagged", host, testImage) + _, err = parseRepositoryReference(repositoryReference) + So(err, ShouldEqual, errors.ErrInvalidRepositoryName) + + repositoryReference = fmt.Sprintf("http://%s/%s", host, testImage) + _, err = parseRepositoryReference(repositoryReference) + So(err, ShouldNotBeNil) + + repositoryReference = fmt.Sprintf("docker://%s/%s", host, testImage) + _, err = parseRepositoryReference(repositoryReference) + So(err, ShouldNotBeNil) + + _, err = getFileCredentials("/path/to/inexistent/file") + So(err, ShouldNotBeNil) + + f, err := ioutil.TempFile("", "sync-credentials-") + if err != nil { + panic(err) + } + + content := []byte(`{`) + if err := ioutil.WriteFile(f.Name(), content, 0600); err != nil { + panic(err) + } + + _, err = getFileCredentials(f.Name()) + So(err, ShouldNotBeNil) + + srcCtx := &types.SystemContext{} + _, err = getImageTags(context.Background(), srcCtx, ref) + + So(err, ShouldNotBeNil) + + _, _, err = getLocalContexts("inexistent.cert", "inexistent.key", "inexistent.crt", log.NewLogger("", "")) + So(err, ShouldNotBeNil) + + _, _, err = getLocalContexts(ServerCert, "inexistent.key", "inexistent.crt", log.NewLogger("", "")) + So(err, ShouldNotBeNil) + + _, _, err = getLocalContexts(ServerCert, ServerKey, "inexistent.crt", log.NewLogger("", "")) + So(err, ShouldNotBeNil) + + taggedRef, err := reference.WithTag(ref, testImageTag) + So(err, ShouldBeNil) + + dockerRef, err := docker.NewReference(taggedRef) + So(err, ShouldBeNil) + + So(getTagFromRef(dockerRef, log.NewLogger("", "")), ShouldNotBeNil) + + var tlsVerify bool + updateDuration := time.Microsecond + syncRegistryConfig := RegistryConfig{ + Content: []Content{ + { + Prefix: testImage, + }, + }, + URL: BaseURL, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: "", + } + + cfg := Config{Registries: []RegistryConfig{syncRegistryConfig}, CredentialsFile: "/invalid/path/to/file"} + + So(Run(cfg, log.NewLogger("", ""), + "127.0.0.1", "5000", ServerCert, ServerKey, CACert), ShouldNotBeNil) + + _, err = getFileCredentials("/invalid/path/to/file") + So(err, ShouldNotBeNil) + }) +} diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go new file mode 100644 index 00000000..b4e282de --- /dev/null +++ b/pkg/extensions/sync/sync_test.go @@ -0,0 +1,1612 @@ +package sync_test + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "reflect" + "strings" + "testing" + "time" + + "github.com/anuvu/zot/pkg/api" + "github.com/anuvu/zot/pkg/api/config" + extconf "github.com/anuvu/zot/pkg/extensions/config" + "github.com/anuvu/zot/pkg/extensions/sync" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/phayes/freeport" + . "github.com/smartystreets/goconvey/convey" + "gopkg.in/resty.v1" +) + +const ( + BaseURL = "http://127.0.0.1:%s" + BaseSecureURL = "https://127.0.0.1:%s" + ServerCert = "../../../test/data/server.cert" + ServerKey = "../../../test/data/server.key" + CACert = "../../../test/data/ca.crt" + ClientCert = "../../../test/data/client.cert" + ClientKey = "../../../test/data/client.key" + + testImage = "zot-test" + testImageTag = "0.0.1" +) + +var errSync = errors.New("sync error, src oci repo differs from dest one") + +type TagsList struct { + Name string + Tags []string +} + +type catalog struct { + Repositories []string `json:"repositories"` +} + +func getFreePort() string { + port, err := freeport.GetFreePort() + if err != nil { + panic(err) + } + + return fmt.Sprint(port) +} + +func getBaseURL(port string, secure bool) string { + if secure { + return fmt.Sprintf(BaseSecureURL, port) + } + + return fmt.Sprintf(BaseURL, port) +} + +func copyFile(sourceFilePath, destFilePath string) error { + destFile, err := os.Create(destFilePath) + if err != nil { + return err + } + defer destFile.Close() + + sourceFile, err := os.Open(sourceFilePath) + if err != nil { + return err + } + defer sourceFile.Close() + + if _, err = io.Copy(destFile, sourceFile); err != nil { + return err + } + + return nil +} + +func copyFiles(sourceDir string, destDir string) error { + sourceMeta, err := os.Stat(sourceDir) + if err != nil { + return err + } + + if err := os.MkdirAll(destDir, sourceMeta.Mode()); err != nil { + return err + } + + files, err := ioutil.ReadDir(sourceDir) + if err != nil { + return err + } + + for _, file := range files { + sourceFilePath := path.Join(sourceDir, file.Name()) + destFilePath := path.Join(destDir, file.Name()) + + if file.IsDir() { + if err = copyFiles(sourceFilePath, destFilePath); err != nil { + return err + } + } else { + sourceFile, err := os.Open(sourceFilePath) + if err != nil { + return err + } + defer sourceFile.Close() + + destFile, err := os.Create(destFilePath) + if err != nil { + return err + } + defer destFile.Close() + + if _, err = io.Copy(destFile, sourceFile); err != nil { + return err + } + } + } + + return nil +} + +func makeHtpasswdFile() string { + f, err := ioutil.TempFile("", "htpasswd-") + if err != nil { + panic(err) + } + + // bcrypt(username="test", passwd="test") + content := []byte("test:$2y$05$hlbSXDp6hzDLu6VwACS39ORvVRpr3OMR4RlJ31jtlaOEGnPjKZI1m\n") + if err := ioutil.WriteFile(f.Name(), content, 0600); err != nil { + panic(err) + } + + return f.Name() +} + +func TestSyncOnDemand(t *testing.T) { + Convey("Verify sync on demand feature", t, func() { + updateDuration, _ := time.ParseDuration("30m") + + srcPort := getFreePort() + srcBaseURL := getBaseURL(srcPort, false) + + srcConfig := config.New() + srcConfig.HTTP.Port = srcPort + + srcDir, err := ioutil.TempDir("", "oci-src-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(srcDir) + + err = copyFiles("../../../test/data", srcDir) + if err != nil { + panic(err) + } + + srcConfig.Storage.RootDirectory = srcDir + + sc := api.NewController(srcConfig) + + go func() { + // this blocks + if err := sc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(srcBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + destPort := getFreePort() + destBaseURL := getBaseURL(destPort, false) + + destConfig := config.New() + destConfig.HTTP.Port = destPort + + destDir, err := ioutil.TempDir("", "oci-dest-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(destDir) + + destConfig.Storage.RootDirectory = destDir + + regex := ".*" + var semver bool + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + // won't match any image on source registry, we will sync on demand + Prefix: "dummy", + Tags: &sync.Tags{ + Regex: ®ex, + Semver: &semver, + }, + }, + }, + URL: srcBaseURL, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, + } + + destConfig.Extensions = &extconf.ExtensionConfig{} + destConfig.Extensions.Search = nil + destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc := api.NewController(destConfig) + + go func() { + // this blocks + if err := dc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(destBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + var srcTagsList TagsList + var destTagsList TagsList + + resp, _ := resty.R().Get(srcBaseURL + "/v2/" + testImage + "/tags/list") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + err = json.Unmarshal(resp.Body(), &srcTagsList) + if err != nil { + panic(err) + } + + resp, err = resty.R().Get(destBaseURL + "/v2/" + "inexistent" + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 404) + + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "inexistent") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 404) + + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "1.1.1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 404) + + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") + if err != nil { + panic(err) + } + + err = json.Unmarshal(resp.Body(), &destTagsList) + if err != nil { + panic(err) + } + + if eq := reflect.DeepEqual(destTagsList.Tags, srcTagsList.Tags); eq == false { + panic(errSync) + } + + defer func() { + ctx := context.Background() + _ = dc.Server.Shutdown(ctx) + _ = sc.Server.Shutdown(ctx) + }() + }) +} + +func TestSync(t *testing.T) { + Convey("Verify sync feature", t, func() { + updateDuration, _ := time.ParseDuration("30m") + + srcPort := getFreePort() + srcBaseURL := getBaseURL(srcPort, false) + + srcConfig := config.New() + srcConfig.HTTP.Port = srcPort + + srcDir, err := ioutil.TempDir("", "oci-src-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(srcDir) + + err = copyFiles("../../../test/data", srcDir) + if err != nil { + panic(err) + } + + srcConfig.Storage.RootDirectory = srcDir + + sc := api.NewController(srcConfig) + + go func() { + // this blocks + if err := sc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(srcBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + destPort := getFreePort() + destBaseURL := getBaseURL(destPort, false) + + destConfig := config.New() + destConfig.HTTP.Port = destPort + + destDir, err := ioutil.TempDir("", "oci-dest-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(destDir) + + destConfig.Storage.RootDirectory = destDir + + regex := ".*" + semver := true + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + Prefix: testImage, + Tags: &sync.Tags{ + Regex: ®ex, + Semver: &semver, + }, + }, + }, + URL: srcBaseURL, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: "", + } + + destConfig.Extensions = &extconf.ExtensionConfig{} + destConfig.Extensions.Search = nil + destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc := api.NewController(destConfig) + + go func() { + // this blocks + if err := dc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(destBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + var srcTagsList TagsList + var destTagsList TagsList + + resp, _ := resty.R().Get(srcBaseURL + "/v2/" + testImage + "/tags/list") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + err = json.Unmarshal(resp.Body(), &srcTagsList) + if err != nil { + panic(err) + } + + for { + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") + if err != nil { + panic(err) + } + + err = json.Unmarshal(resp.Body(), &destTagsList) + if err != nil { + panic(err) + } + + if len(destTagsList.Tags) > 0 { + break + } + + time.Sleep(1 * time.Second) + } + + if eq := reflect.DeepEqual(destTagsList.Tags, srcTagsList.Tags); eq == false { + panic(errSync) + } + + Convey("Test sync on POST request on /sync", func() { + resp, _ := resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + }) + + defer func() { + ctx := context.Background() + _ = dc.Server.Shutdown(ctx) + _ = sc.Server.Shutdown(ctx) + }() + }) +} + +func TestSyncTLS(t *testing.T) { + Convey("Verify sync TLS feature", t, func() { + caCert, err := ioutil.ReadFile(CACert) + So(err, ShouldBeNil) + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + + client := resty.New() + + client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) + defer func() { client.SetTLSClientConfig(nil) }() + + var srcIndex ispec.Index + var destIndex ispec.Index + + updateDuration, _ := time.ParseDuration("1h") + + srcPort := getFreePort() + srcBaseURL := getBaseURL(srcPort, true) + + srcConfig := config.New() + srcConfig.HTTP.Port = srcPort + + srcConfig.HTTP.TLS = &config.TLSConfig{ + Cert: ServerCert, + Key: ServerKey, + CACert: CACert, + } + + srcDir, err := ioutil.TempDir("", "oci-src-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(srcDir) + + err = copyFiles("../../../test/data", srcDir) + if err != nil { + panic(err) + } + + srcConfig.Storage.RootDirectory = srcDir + + sc := api.NewController(srcConfig) + + go func() { + // this blocks + if err := sc.Run(); err != nil { + return + } + }() + + cert, err := tls.LoadX509KeyPair("../../../test/data/client.cert", "../../../test/data/client.key") + if err != nil { + panic(err) + } + + client.SetCertificates(cert) + // wait till ready + for { + _, err := client.R().Get(srcBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + srcBuf, err := ioutil.ReadFile(path.Join(srcDir, testImage, "index.json")) + if err != nil { + panic(err) + } + + if err := json.Unmarshal(srcBuf, &srcIndex); err != nil { + panic(err) + } + + destPort := getFreePort() + destConfig := config.New() + destConfig.HTTP.Port = destPort + + destConfig.HTTP.TLS = &config.TLSConfig{ + Cert: ServerCert, + Key: ServerKey, + CACert: CACert, + } + + destDir, err := ioutil.TempDir("", "oci-dest-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(destDir) + + destConfig.Storage.RootDirectory = destDir + + // copy client certs, use them in sync config + clientCertDir, err := ioutil.TempDir("", "certs") + if err != nil { + panic(err) + } + + destFilePath := path.Join(clientCertDir, "ca.crt") + err = copyFile(CACert, destFilePath) + if err != nil { + panic(err) + } + + destFilePath = path.Join(clientCertDir, "client.cert") + err = copyFile(ClientCert, destFilePath) + if err != nil { + panic(err) + } + + destFilePath = path.Join(clientCertDir, "client.key") + err = copyFile(ClientKey, destFilePath) + if err != nil { + panic(err) + } + + regex := ".*" + var semver bool + tlsVerify := true + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + Prefix: testImage, + Tags: &sync.Tags{ + Regex: ®ex, + Semver: &semver, + }, + }, + }, + URL: srcBaseURL, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: clientCertDir, + } + + destConfig.Extensions = &extconf.ExtensionConfig{} + destConfig.Extensions.Search = nil + destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc := api.NewController(destConfig) + + go func() { + // this blocks + if err := dc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + destBuf, _ := ioutil.ReadFile(path.Join(destDir, testImage, "index.json")) + _ = json.Unmarshal(destBuf, &destIndex) + time.Sleep(1 * time.Second) + if len(destIndex.Manifests) > 0 { + break + } + } + + var found bool + for _, manifest := range srcIndex.Manifests { + if reflect.DeepEqual(manifest.Annotations, destIndex.Manifests[0].Annotations) { + found = true + } + } + + if !found { + panic(errSync) + } + + defer func() { + ctx := context.Background() + _ = dc.Server.Shutdown(ctx) + _ = sc.Server.Shutdown(ctx) + }() + }) +} + +func TestSyncBasicAuth(t *testing.T) { + Convey("Verify sync basic auth", t, func() { + updateDuration, _ := time.ParseDuration("1h") + + srcPort := getFreePort() + srcBaseURL := getBaseURL(srcPort, false) + + srcConfig := config.New() + srcConfig.HTTP.Port = srcPort + + htpasswdPath := makeHtpasswdFile() + defer os.Remove(htpasswdPath) + + srcConfig.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ + Path: htpasswdPath, + }, + } + + srcDir, err := ioutil.TempDir("", "oci-src-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(srcDir) + + err = copyFiles("../../../test/data", srcDir) + if err != nil { + panic(err) + } + + srcConfig.Storage.RootDirectory = srcDir + + sc := api.NewController(srcConfig) + + go func() { + // this blocks + if err := sc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(srcBaseURL) + t.Logf("err %v", err) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + Convey("Verify sync basic auth with file credentials", func() { + destPort := getFreePort() + destBaseURL := getBaseURL(destPort, false) + + destConfig := config.New() + destConfig.HTTP.Port = destPort + + destDir, err := ioutil.TempDir("", "oci-dest-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(destDir) + + destConfig.Storage.RootDirectory = destDir + + registryName := strings.Replace(strings.Replace(srcBaseURL, "http://", "", 1), "https://", "", 1) + + credentialsFile := makeCredentialsFile(fmt.Sprintf(`{"%s":{"username": "test", "password": "test"}}`, registryName)) + + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + Prefix: testImage, + }, + }, + URL: srcBaseURL, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: "", + } + + destConfig.Extensions = &extconf.ExtensionConfig{} + destConfig.Extensions.Search = nil + destConfig.Extensions.Sync = &sync.Config{CredentialsFile: credentialsFile, + Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc := api.NewController(destConfig) + + go func() { + // this blocks + if err := dc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(destBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + var srcTagsList TagsList + var destTagsList TagsList + + resp, _ := resty.R().SetBasicAuth("test", "test").Get(srcBaseURL + "/v2/" + testImage + "/tags/list") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + err = json.Unmarshal(resp.Body(), &srcTagsList) + if err != nil { + panic(err) + } + + for { + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") + if err != nil { + panic(err) + } + + err = json.Unmarshal(resp.Body(), &destTagsList) + if err != nil { + panic(err) + } + + if len(destTagsList.Tags) > 0 { + break + } + + time.Sleep(1 * time.Second) + } + + if eq := reflect.DeepEqual(destTagsList.Tags, srcTagsList.Tags); eq == false { + panic(errSync) + } + + defer func() { + ctx := context.Background() + _ = dc.Server.Shutdown(ctx) + _ = sc.Server.Shutdown(ctx) + }() + }) + + Convey("Verify sync basic auth with bad file credentials", func() { + destPort := getFreePort() + destBaseURL := getBaseURL(destPort, false) + + destConfig := config.New() + destConfig.HTTP.Port = destPort + + destDir, err := ioutil.TempDir("", "oci-dest-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(destDir) + + destConfig.Storage.RootDirectory = destDir + + regex := ".*" + var semver bool + + registryName := strings.Replace(strings.Replace(srcBaseURL, "http://", "", 1), "https://", "", 1) + + credentialsFile := makeCredentialsFile(fmt.Sprintf(`{"%s":{"username": "test", "password": "invalid"}}`, + registryName)) + + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + Prefix: testImage, + Tags: &sync.Tags{ + Regex: ®ex, + Semver: &semver, + }, + }, + }, + URL: srcBaseURL, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: "", + } + + destConfig.Extensions = &extconf.ExtensionConfig{} + destConfig.Extensions.Search = nil + destConfig.Extensions.Sync = &sync.Config{CredentialsFile: credentialsFile, + Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc := api.NewController(destConfig) + + go func() { + // this blocks + if err := dc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(destBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + Convey("Test sync on POST request on /sync", func() { + resp, _ := resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(string(resp.Body()), ShouldContainSubstring, "sync: couldn't fetch upstream registry's catalog") + So(resp.StatusCode(), ShouldEqual, 500) + }) + + defer func() { + ctx := context.Background() + _ = dc.Server.Shutdown(ctx) + _ = sc.Server.Shutdown(ctx) + }() + }) + + Convey("Verify on demand sync with basic auth", func() { + destPort := getFreePort() + destBaseURL := getBaseURL(destPort, false) + + destConfig := config.New() + destConfig.HTTP.Port = destPort + + destDir, err := ioutil.TempDir("", "oci-dest-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(destDir) + + destConfig.Storage.RootDirectory = destDir + + registryName := strings.Replace(strings.Replace(srcBaseURL, "http://", "", 1), "https://", "", 1) + credentialsFile := makeCredentialsFile(fmt.Sprintf(`{"%s":{"username": "test", "password": "test"}}`, registryName)) + + syncRegistryConfig := sync.RegistryConfig{ + URL: srcBaseURL, + OnDemand: true, + } + + unreacheableSyncRegistryConfig1 := sync.RegistryConfig{ + URL: "localhost:999999", + OnDemand: true, + } + + unreacheableSyncRegistryConfig2 := sync.RegistryConfig{ + URL: "localhost:999999", + OnDemand: false, + } + + destConfig.Extensions = &extconf.ExtensionConfig{} + destConfig.Extensions.Search = nil + // add file path to the credentials + destConfig.Extensions.Sync = &sync.Config{CredentialsFile: credentialsFile, + Registries: []sync.RegistryConfig{unreacheableSyncRegistryConfig1, + unreacheableSyncRegistryConfig2, + syncRegistryConfig}} + + dc := api.NewController(destConfig) + + go func() { + // this blocks + if err := dc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(destBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + var srcTagsList TagsList + var destTagsList TagsList + + resp, _ := resty.R().SetBasicAuth("test", "test").Get(srcBaseURL + "/v2/" + testImage + "/tags/list") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + err = json.Unmarshal(resp.Body(), &srcTagsList) + if err != nil { + panic(err) + } + + resp, err = resty.R().Get(destBaseURL + "/v2/" + "inexistent" + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 404) + + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "inexistent") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 404) + + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + err = dc.StoreController.DefaultStore.DeleteImageManifest(testImage, testImageTag) + So(err, ShouldBeNil) + + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "1.1.1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 404) + + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") + if err != nil { + panic(err) + } + + err = json.Unmarshal(resp.Body(), &destTagsList) + if err != nil { + panic(err) + } + + if eq := reflect.DeepEqual(destTagsList.Tags, srcTagsList.Tags); eq == false { + panic(errSync) + } + + Convey("Test sync on POST request on /sync", func() { + resp, _ := resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + }) + + defer func() { + ctx := context.Background() + _ = dc.Server.Shutdown(ctx) + _ = sc.Server.Shutdown(ctx) + }() + }) + }) +} + +func TestSyncBadUrl(t *testing.T) { + Convey("Verify sync with bad url", t, func() { + updateDuration, _ := time.ParseDuration("1h") + + destPort := getFreePort() + destBaseURL := getBaseURL(destPort, false) + + destConfig := config.New() + destConfig.HTTP.Port = destPort + + destDir, err := ioutil.TempDir("", "oci-dest-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(destDir) + + destConfig.Storage.RootDirectory = destDir + + regex := ".*" + var semver bool + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + Prefix: testImage, + Tags: &sync.Tags{ + Regex: ®ex, + Semver: &semver, + }, + }, + }, + URL: "bad-registry-url", + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: "", + } + + destConfig.Extensions = &extconf.ExtensionConfig{} + destConfig.Extensions.Search = nil + destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + dc := api.NewController(destConfig) + + go func() { + // this blocks + if err := dc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(destBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + Convey("Test sync on POST request on /sync", func() { + resp, _ := resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(string(resp.Body()), ShouldContainSubstring, "unsupported protocol scheme") + So(resp.StatusCode(), ShouldEqual, 500) + }) + + defer func() { + ctx := context.Background() + _ = dc.Server.Shutdown(ctx) + }() + }) +} + +func TestSyncNoImagesByRegex(t *testing.T) { + Convey("Verify sync with no images on source based on regex", t, func() { + updateDuration, _ := time.ParseDuration("1h") + + srcPort := getFreePort() + srcBaseURL := getBaseURL(srcPort, false) + + srcConfig := config.New() + srcConfig.HTTP.Port = srcPort + + srcDir, err := ioutil.TempDir("", "oci-src-repo-test") + if err != nil { + panic(err) + } + + err = copyFiles("../../../test/data", srcDir) + if err != nil { + panic(err) + } + + defer os.RemoveAll(srcDir) + + srcConfig.Storage.RootDirectory = srcDir + + sc := api.NewController(srcConfig) + + go func() { + // this blocks + if err := sc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(srcBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + destPort := getFreePort() + destBaseURL := getBaseURL(destPort, false) + + destConfig := config.New() + destConfig.HTTP.Port = destPort + + destDir, err := ioutil.TempDir("", "oci-dest-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(destDir) + + destConfig.Storage.RootDirectory = destDir + + regex := "9.9.9" + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + Prefix: testImage, + Tags: &sync.Tags{ + Regex: ®ex, + }, + }, + }, + URL: srcBaseURL, + TLSVerify: &tlsVerify, + PollInterval: updateDuration, + CertDir: "", + } + + destConfig.Extensions = &extconf.ExtensionConfig{} + destConfig.Extensions.Search = nil + destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc := api.NewController(destConfig) + + go func() { + // this blocks + if err := dc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(destBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + Convey("Test sync on POST request on /sync", func() { + resp, _ := resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + So(err, ShouldBeNil) + + resp, err := resty.R().Get(destBaseURL + "/v2/_catalog") + if err != nil { + panic(err) + } + + var c catalog + err = json.Unmarshal(resp.Body(), &c) + if err != nil { + panic(err) + } + + So(c.Repositories, ShouldResemble, []string{}) + }) + + defer func() { + ctx := context.Background() + _ = dc.Server.Shutdown(ctx) + _ = sc.Server.Shutdown(ctx) + }() + }) +} + +func TestSyncInvalidRegex(t *testing.T) { + Convey("Verify sync with invalid regex", t, func() { + updateDuration, _ := time.ParseDuration("1h") + + srcPort := getFreePort() + srcBaseURL := getBaseURL(srcPort, false) + + srcConfig := config.New() + srcConfig.HTTP.Port = srcPort + + srcDir, err := ioutil.TempDir("", "oci-src-repo-test") + if err != nil { + panic(err) + } + + err = copyFiles("../../../test/data", srcDir) + if err != nil { + panic(err) + } + + defer os.RemoveAll(srcDir) + + srcConfig.Storage.RootDirectory = srcDir + + sc := api.NewController(srcConfig) + + go func() { + // this blocks + if err := sc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(srcBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + destPort := getFreePort() + destBaseURL := getBaseURL(destPort, false) + + destConfig := config.New() + destConfig.HTTP.Port = destPort + + destDir, err := ioutil.TempDir("", "oci-dest-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(destDir) + + destConfig.Storage.RootDirectory = destDir + + regex := "[" + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + Prefix: testImage, + Tags: &sync.Tags{ + Regex: ®ex, + }, + }, + }, + URL: srcBaseURL, + TLSVerify: &tlsVerify, + PollInterval: updateDuration, + CertDir: "", + } + + destConfig.Extensions = &extconf.ExtensionConfig{} + destConfig.Extensions.Search = nil + destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc := api.NewController(destConfig) + + go func() { + // this blocks + if err := dc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(destBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + Convey("Test sync on POST request on /sync", func() { + resp, _ := resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(string(resp.Body()), ShouldContainSubstring, "error parsing regexp") + So(resp.StatusCode(), ShouldEqual, 500) + }) + + defer func() { + ctx := context.Background() + _ = dc.Server.Shutdown(ctx) + _ = sc.Server.Shutdown(ctx) + }() + }) +} + +func TestSyncNotSemver(t *testing.T) { + Convey("Verify sync feature semver compliant", t, func() { + updateDuration, _ := time.ParseDuration("30m") + + srcPort := getFreePort() + srcBaseURL := getBaseURL(srcPort, false) + + srcConfig := config.New() + srcConfig.HTTP.Port = srcPort + + srcDir, err := ioutil.TempDir("", "oci-src-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(srcDir) + + err = copyFiles("../../../test/data", srcDir) + if err != nil { + panic(err) + } + + srcConfig.Storage.RootDirectory = srcDir + + sc := api.NewController(srcConfig) + + go func() { + // this blocks + if err := sc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(srcBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + // get manifest so we can update it with a semver non compliant tag + resp, err := resty.R().Get(srcBaseURL + "/v2/zot-test/manifests/0.0.1") + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + manifestBlob := resp.Body() + + resp, err = resty.R().SetHeader("Content-type", "application/vnd.oci.image.manifest.v1+json"). + SetBody(manifestBlob). + Put(srcBaseURL + "/v2/" + testImage + "/manifests/notSemverTag") + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 201) + + destPort := getFreePort() + destBaseURL := getBaseURL(destPort, false) + + destConfig := config.New() + destConfig.HTTP.Port = destPort + + destDir, err := ioutil.TempDir("", "oci-dest-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(destDir) + + destConfig.Storage.RootDirectory = destDir + + semver := true + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + Prefix: testImage, + Tags: &sync.Tags{ + Semver: &semver, + }, + }, + }, + URL: srcBaseURL, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: "", + } + + destConfig.Extensions = &extconf.ExtensionConfig{} + destConfig.Extensions.Search = nil + destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc := api.NewController(destConfig) + + go func() { + // this blocks + if err := dc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(destBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + Convey("Test sync on POST request on /sync", func() { + resp, _ := resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + So(err, ShouldBeNil) + + var destTagsList TagsList + + resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") + if err != nil { + panic(err) + } + + err = json.Unmarshal(resp.Body(), &destTagsList) + if err != nil { + panic(err) + } + + So(len(destTagsList.Tags), ShouldEqual, 1) + So(destTagsList.Tags[0], ShouldEqual, testImageTag) + }) + + defer func() { + ctx := context.Background() + _ = dc.Server.Shutdown(ctx) + _ = sc.Server.Shutdown(ctx) + }() + }) +} + +func TestSyncInvalidCerts(t *testing.T) { + Convey("Verify sync with bad certs", t, func() { + caCert, err := ioutil.ReadFile(CACert) + So(err, ShouldBeNil) + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + + client := resty.New() + + client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) + defer func() { client.SetTLSClientConfig(nil) }() + updateDuration, _ := time.ParseDuration("1h") + + srcPort := getFreePort() + srcBaseURL := getBaseURL(srcPort, true) + + srcConfig := config.New() + srcConfig.HTTP.Port = srcPort + + srcConfig.HTTP.TLS = &config.TLSConfig{ + Cert: ServerCert, + Key: ServerKey, + CACert: CACert, + } + + srcDir, err := ioutil.TempDir("", "oci-src-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(srcDir) + + err = copyFiles("../../../test/data", srcDir) + if err != nil { + panic(err) + } + + srcConfig.Storage.RootDirectory = srcDir + + sc := api.NewController(srcConfig) + + go func() { + // this blocks + if err := sc.Run(); err != nil { + return + } + }() + + cert, err := tls.LoadX509KeyPair("../../../test/data/client.cert", "../../../test/data/client.key") + if err != nil { + panic(err) + } + + client.SetCertificates(cert) + // wait till ready + for { + _, err := client.R().Get(srcBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + destPort := getFreePort() + destBaseURL := getBaseURL(destPort, false) + destConfig := config.New() + destConfig.HTTP.Port = destPort + + os.RemoveAll("/tmp/zot-certs-dir") + + destDir, err := ioutil.TempDir("", "oci-dest-repo-test") + if err != nil { + panic(err) + } + + defer os.RemoveAll(destDir) + + destConfig.Storage.RootDirectory = destDir + + // copy client certs, use them in sync config + clientCertDir, err := ioutil.TempDir("", "certs") + if err != nil { + panic(err) + } + + destFilePath := path.Join(clientCertDir, "ca.crt") + err = copyFile(CACert, destFilePath) + if err != nil { + panic(err) + } + + f, err := os.OpenFile(destFilePath, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0600) + if err != nil { + panic(err) + } + + defer f.Close() + + if _, err = f.WriteString("Add Invalid Text In Cert"); err != nil { + panic(err) + } + + destFilePath = path.Join(clientCertDir, "client.cert") + err = copyFile(ClientCert, destFilePath) + if err != nil { + panic(err) + } + + destFilePath = path.Join(clientCertDir, "client.key") + err = copyFile(ClientKey, destFilePath) + if err != nil { + panic(err) + } + + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + Prefix: "", + }, + }, + URL: srcBaseURL, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: clientCertDir, + } + + destConfig.Extensions = &extconf.ExtensionConfig{} + destConfig.Extensions.Search = nil + destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc := api.NewController(destConfig) + + go func() { + // this blocks + if err := dc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(destBaseURL) + if err == nil { + break + } + time.Sleep(100 * time.Millisecond) + } + + Convey("Test sync on POST request on /sync", func() { + resp, _ := resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(string(resp.Body()), ShouldContainSubstring, "signed by unknown authority") + So(resp.StatusCode(), ShouldEqual, 500) + }) + + defer func() { + ctx := context.Background() + _ = sc.Server.Shutdown(ctx) + _ = dc.Server.Shutdown(ctx) + }() + }) +} + +func makeCredentialsFile(fileContent string) string { + f, err := ioutil.TempFile("", "sync-credentials-") + if err != nil { + panic(err) + } + + content := []byte(fileContent) + if err := ioutil.WriteFile(f.Name(), content, 0600); err != nil { + panic(err) + } + + return f.Name() +} diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go new file mode 100644 index 00000000..64f22efa --- /dev/null +++ b/pkg/extensions/sync/utils.go @@ -0,0 +1,166 @@ +package sync + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "strings" + + "github.com/anuvu/zot/errors" + "github.com/anuvu/zot/pkg/log" + "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/types" +) + +var certsDir = fmt.Sprintf("%s/zot-certs-dir/", os.TempDir()) //nolint: gochecknoglobals + +func copyFile(sourceFilePath, destFilePath string) error { + destFile, err := os.Create(destFilePath) + if err != nil { + return err + } + defer destFile.Close() + + // should never get error because server certs are already handled by zot, by the time + // it gets here + sourceFile, _ := os.Open(sourceFilePath) + defer sourceFile.Close() + + if _, err := io.Copy(destFile, sourceFile); err != nil { + return err + } + + return nil +} + +func copyLocalCerts(serverCert, serverKey, caCert string, log log.Logger) (string, error) { + log.Debug().Msgf("Creating certs directory: %s", certsDir) + + err := os.Mkdir(certsDir, 0755) + if err != nil && !os.IsExist(err) { + return "", err + } + + if serverCert != "" { + log.Debug().Msgf("Copying server cert: %s", serverCert) + + err := copyFile(serverCert, path.Join(certsDir, "server.cert")) + if err != nil { + return "", err + } + } + + if serverKey != "" { + log.Debug().Msgf("Copying server key: %s", serverKey) + + err := copyFile(serverKey, path.Join(certsDir, "server.key")) + if err != nil { + return "", err + } + } + + if caCert != "" { + log.Debug().Msgf("Copying CA cert: %s", caCert) + + err := copyFile(caCert, path.Join(certsDir, "ca.crt")) + if err != nil { + return "", err + } + } + + return certsDir, nil +} + +// getTagFromRef returns a tagged reference from an image reference. +func getTagFromRef(ref types.ImageReference, log log.Logger) reference.Tagged { + tagged, isTagged := ref.DockerReference().(reference.Tagged) + if !isTagged { + log.Warn().Msgf("internal server error, reference %s does not have a tag, skipping", ref.DockerReference()) + return nil + } + + return tagged +} + +// parseRepositoryReference parses input into a reference.Named, and verifies that it names a repository, not an image. +func parseRepositoryReference(input string) (reference.Named, error) { + ref, err := reference.ParseNormalizedNamed(input) + if err != nil { + return nil, err + } + + if !reference.IsNameOnly(ref) { + return nil, errors.ErrInvalidRepositoryName + } + + return ref, nil +} + +// filterRepos filters repos based on prefix given in the config. +func filterRepos(repos []string, content []Content) map[int][]string { + // prefix: repo + filtered := make(map[int][]string) + + for _, repo := range repos { + matched := false + // we use contentID to figure out tags filtering + for contentID, c := range content { + // handle prefixes starting with '/' + var prefix string + if strings.HasPrefix(c.Prefix, "/") { + prefix = c.Prefix[1:] + } else { + prefix = c.Prefix + } + + // split both prefix and repository and compare each part + splittedPrefix := strings.Split(prefix, "/") + // split at most n + 1 + splittedRepo := strings.SplitN(repo, "/", len(splittedPrefix)+1) + + // if prefix is longer than a repository, no match + if len(splittedPrefix) > len(splittedRepo) { + continue + } + + // check if matched each part of prefix and repository + for i := 0; i < len(splittedPrefix); i++ { + if splittedRepo[i] == splittedPrefix[i] { + matched = true + } else { + // if a part doesn't match, check next prefix + matched = false + break + } + } + + // if matched no need to check the next prefixes + if matched { + filtered[contentID] = append(filtered[contentID], repo) + break + } + } + } + + return filtered +} + +// Get sync.FileCredentials from file. +func getFileCredentials(filepath string) (CredentialsFile, error) { + f, err := ioutil.ReadFile(filepath) + if err != nil { + return nil, err + } + + var creds CredentialsFile + + err = json.Unmarshal(f, &creds) + if err != nil { + return nil, err + } + + return creds, nil +} diff --git a/pkg/log/log_test.go b/pkg/log/log_test.go index dc767406..fbc80819 100644 --- a/pkg/log/log_test.go +++ b/pkg/log/log_test.go @@ -17,6 +17,7 @@ import ( "time" "github.com/anuvu/zot/pkg/api" + "github.com/anuvu/zot/pkg/api/config" godigest "github.com/opencontainers/go-digest" . "github.com/smartystreets/goconvey/convey" "gopkg.in/resty.v1" @@ -129,23 +130,23 @@ func TestAuditLogMessages(t *testing.T) { panic(err) } - config := api.NewConfig() + conf := config.New() outputPath := dir + "/zot.log" auditPath := dir + "/zot-audit.log" - config.Log = &api.LogConfig{Level: "debug", Output: outputPath, Audit: auditPath} + conf.Log = &config.LogConfig{Level: "debug", Output: outputPath, Audit: auditPath} - config.HTTP.Port = SecurePort + conf.HTTP.Port = SecurePort htpasswdPath := makeHtpasswdFile() defer os.Remove(htpasswdPath) - config.HTTP.Auth = &api.AuthConfig{ - HTPasswd: api.AuthHTPasswd{ + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ Path: htpasswdPath, }, } - c := api.NewController(config) + c := api.NewController(conf) c.Config.Storage.RootDirectory = dir go func() { // this blocks