diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index 6ca1c23e..e52fde11 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -31,7 +31,7 @@ jobs: # Optional: golangci-lint command line arguments. # args: --issues-exit-code=0 - args: --config ./golangcilint.yaml --enable-all --build-tags sync,scrub,search,metrics,ui_base,containers_image_openpgp ./cmd/... ./pkg/... + args: --config ./golangcilint.yaml --enable-all --build-tags sync,scrub,search,metrics,ui_base,containers_image_openpgp,lint ./cmd/... ./pkg/... # Optional: show only new issues if it's a pull request. The default value is `false`. # only-new-issues: true diff --git a/Makefile b/Makefile index 4d5de652..01dc9ca2 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ TESTDATA := $(TOP_LEVEL)/test/data OS ?= linux ARCH ?= amd64 BENCH_OUTPUT ?= stdout -EXTENSIONS ?= sync,search,scrub,metrics,ui_base +EXTENSIONS ?= sync,search,scrub,metrics,ui_base,lint comma:= , hyphen:= - extended-name:= diff --git a/README.md b/README.md index 42f069c3..9ffcc30e 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ The following document refers on the **core dist-spec**, see also the [zot-speci * Supports container image signatures - [cosign](https://github.com/sigstore/cosign) and [notation](https://github.com/notaryproject/notation) * Multi-arch support * Clustering support +* Image linting support ## [Demos](demos/README.md) @@ -381,6 +382,11 @@ bin/zxp config _config-file_ ## Enable Metrics In the zot with all extensions case see [configuration example](./examples/config-metrics.json) for enabling metrics +## Image linting + +# Mandatory Annotations +When pushing an image, if the mandatory annotations option is enabled, linter will verify if the mandatory annotations list present in the config is also found in the manifest's annotations list. If there are any missing annotations, the push will not take place. + ## Clustering zot supports clustering by using multiple stateless zot with shared s3 storage and a haproxy (with sticky session) in front of them. diff --git a/errors/errors.go b/errors/errors.go index 220be1de..c1f95a47 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -53,4 +53,5 @@ var ( ErrRegistryNoContent = errors.New("sync: could not find a Content that matches localRepo") ErrSyncSignatureNotFound = errors.New("sync: couldn't find any upstream notary/cosign signatures") ErrSyncSignature = errors.New("sync: couldn't get upstream notary/cosign signatures") + ErrImageLintAnnotations = errors.New("routes: lint checks failed") ) diff --git a/examples/config-lint.json b/examples/config-lint.json new file mode 100644 index 00000000..d46e5680 --- /dev/null +++ b/examples/config-lint.json @@ -0,0 +1,21 @@ +{ + "distSpecVersion": "1.0.1-dev", + "storage": { + "rootDirectory": "/tmp/zot" + }, + "http": { + "address": "127.0.0.1", + "port": "8080", + "ReadOnly": false + }, + "log": { + "level": "debug" + }, + "extensions": { + "lint": { + "enabled": true, + "mandatoryAnnotations": ["annot1", "annot2", "annot3"] + + } + } +} \ No newline at end of file diff --git a/pkg/api/controller.go b/pkg/api/controller.go index ba70e752..c4e34ef2 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -218,6 +218,8 @@ func (c *Controller) Run(reloadCtx context.Context) error { func (c *Controller) InitImageStore(reloadCtx context.Context) error { c.StoreController = storage.StoreController{} + linter := ext.GetLinter(c.Config, c.Log) + if c.Config.Storage.RootDirectory != "" { // no need to validate hard links work on s3 if c.Config.Storage.Dedupe && c.Config.Storage.StorageDriver == nil { @@ -232,8 +234,12 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error { var defaultStore storage.ImageStore if c.Config.Storage.StorageDriver == nil { + // false positive lint - linter does not implement Lint method + // nolint: typecheck defaultStore = storage.NewImageStore(c.Config.Storage.RootDirectory, - c.Config.Storage.GC, c.Config.Storage.GCDelay, c.Config.Storage.Dedupe, c.Config.Storage.Commit, c.Log, c.Metrics) + c.Config.Storage.GC, c.Config.Storage.GCDelay, + c.Config.Storage.Dedupe, c.Config.Storage.Commit, c.Log, c.Metrics, linter, + ) } else { storeName := fmt.Sprintf("%v", c.Config.Storage.StorageDriver["name"]) if storeName != storage.S3StorageDriverName { @@ -255,9 +261,11 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error { rootDir = fmt.Sprintf("%v", c.Config.Storage.StorageDriver["rootdirectory"]) } + // false positive lint - linter does not implement Lint method + // nolint: typecheck defaultStore = s3.NewImageStore(rootDir, c.Config.Storage.RootDirectory, c.Config.Storage.GC, c.Config.Storage.GCDelay, c.Config.Storage.Dedupe, - c.Config.Storage.Commit, c.Log, c.Metrics, store) + c.Config.Storage.Commit, c.Log, c.Metrics, linter, store) } c.StoreController.DefaultStore = defaultStore @@ -288,8 +296,10 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error { } if storageConfig.StorageDriver == nil { + // false positive lint - linter does not implement Lint method + // nolint: typecheck subImageStore[route] = storage.NewImageStore(storageConfig.RootDirectory, - storageConfig.GC, storageConfig.GCDelay, storageConfig.Dedupe, storageConfig.Commit, c.Log, c.Metrics) + storageConfig.GC, storageConfig.GCDelay, storageConfig.Dedupe, storageConfig.Commit, c.Log, c.Metrics, linter) } else { storeName := fmt.Sprintf("%v", storageConfig.StorageDriver["name"]) if storeName != storage.S3StorageDriverName { @@ -311,8 +321,12 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error { rootDir = fmt.Sprintf("%v", c.Config.Storage.StorageDriver["rootdirectory"]) } + // false positive lint - linter does not implement Lint method + // nolint: typecheck subImageStore[route] = s3.NewImageStore(rootDir, storageConfig.RootDirectory, - storageConfig.GC, storageConfig.GCDelay, storageConfig.Dedupe, storageConfig.Commit, c.Log, c.Metrics, store) + storageConfig.GC, storageConfig.GCDelay, + storageConfig.Dedupe, storageConfig.Commit, c.Log, c.Metrics, linter, store, + ) } } diff --git a/pkg/api/routes.go b/pkg/api/routes.go index a62ffaed..2eddb91c 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -470,6 +470,9 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht } else if errors.Is(err, zerr.ErrRepoBadVersion) { WriteJSON(response, http.StatusInternalServerError, NewErrorList(NewError(INVALID_INDEX, map[string]string{"name": name}))) + } else if errors.Is(err, zerr.ErrImageLintAnnotations) { + WriteJSON(response, http.StatusBadRequest, + NewErrorList(NewError(MANIFEST_INVALID, map[string]string{"reference": reference}))) } else { // could be syscall.EMFILE (Err:0x18 too many opened files), etc rh.c.Log.Error().Err(err).Msg("unexpected error: performing cleanup") diff --git a/pkg/api/routes_test.go b/pkg/api/routes_test.go index b287f246..459f69af 100644 --- a/pkg/api/routes_test.go +++ b/pkg/api/routes_test.go @@ -1,5 +1,5 @@ -//go:build sync && scrub && metrics && search && ui_base -// +build sync,scrub,metrics,search,ui_base +//go:build sync && scrub && metrics && search && ui_base && lint +// +build sync,scrub,metrics,search,ui_base,lint package api_test diff --git a/pkg/cli/extensions_test.go b/pkg/cli/extensions_test.go index 33256cc6..4003884a 100644 --- a/pkg/cli/extensions_test.go +++ b/pkg/cli/extensions_test.go @@ -102,7 +102,7 @@ func TestServeExtensions(t *testing.T) { WaitTillServerReady(baseURL) data, err := os.ReadFile(logFile.Name()) So(err, ShouldBeNil) - So(string(data), ShouldContainSubstring, "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null") //nolint:lll // gofumpt conflicts with lll + So(string(data), ShouldContainSubstring, "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null") //nolint:lll // gofumpt conflicts with lll }) } @@ -143,7 +143,7 @@ func testWithMetricsEnabled(cfgContentFormat string) { data, err := os.ReadFile(logFile.Name()) So(err, ShouldBeNil) So(string(data), ShouldContainSubstring, - "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":true,\"Prometheus\":{\"Path\":\"/metrics\"}},\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll + "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":true,\"Prometheus\":{\"Path\":\"/metrics\"}},\"Scrub\":null,\"Lint\":null}") //nolint:lll // gofumpt conflicts with lll } func TestServeMetricsExtension(t *testing.T) { @@ -267,7 +267,7 @@ func TestServeMetricsExtension(t *testing.T) { data, err := os.ReadFile(logFile.Name()) So(err, ShouldBeNil) So(string(data), ShouldContainSubstring, - "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":false,\"Prometheus\":{\"Path\":\"/metrics\"}},\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll + "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":false,\"Prometheus\":{\"Path\":\"/metrics\"}},\"Scrub\":null,\"Lint\":null}}") //nolint:lll // gofumpt conflicts with lll }) } @@ -424,7 +424,7 @@ func TestServeScrubExtension(t *testing.T) { So(err, ShouldBeNil) // Even if in config we specified scrub interval=1h, the minimum interval is 2h So(data, ShouldContainSubstring, - "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":{\"Interval\":3600000000000}") //nolint:lll // gofumpt conflicts with lll + "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":{\"Interval\":3600000000000},\"Lint\":null") //nolint:lll // gofumpt conflicts with lll So(data, ShouldContainSubstring, "Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") So(data, ShouldContainSubstring, "Starting periodic background tasks for") @@ -453,13 +453,72 @@ func TestServeScrubExtension(t *testing.T) { data, err := runCLIWithConfig(t.TempDir(), content) So(err, ShouldBeNil) So(data, ShouldContainSubstring, - "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") + "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}") So(data, ShouldContainSubstring, "Scrub config not provided, skipping scrub") So(data, ShouldNotContainSubstring, "Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") }) } +func TestServeLintExtension(t *testing.T) { + oldArgs := os.Args + + defer func() { os.Args = oldArgs }() + + Convey("lint enabled", t, func(c C) { + content := `{ + "storage": { + "rootDirectory": "%s" + }, + "http": { + "address": "127.0.0.1", + "port": "%s" + }, + "log": { + "level": "debug", + "output": "%s" + }, + "extensions": { + "lint": { + "enabled": "true", + "mandatoryAnnotations": ["annot1"] + } + } + }` + + data, err := runCLIWithConfig(t.TempDir(), content) + So(err, ShouldBeNil) + So(data, ShouldContainSubstring, + "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":{\"Enabled\":true,\"MandatoryAnnotations\":") //nolint:lll // gofumpt conflicts with lll + }) + + Convey("lint enabled", t, func(c C) { + content := `{ + "storage": { + "rootDirectory": "%s" + }, + "http": { + "address": "127.0.0.1", + "port": "%s" + }, + "log": { + "level": "debug", + "output": "%s" + }, + "extensions": { + "lint": { + "enabled": "false" + } + } + }` + + data, err := runCLIWithConfig(t.TempDir(), content) + So(err, ShouldBeNil) + So(data, ShouldContainSubstring, + "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":{\"Enabled\":false,\"MandatoryAnnotations\":null}") //nolint:lll // gofumpt conflicts with lll + }) +} + func TestServeSearchEnabled(t *testing.T) { oldArgs := os.Args @@ -490,7 +549,7 @@ func TestServeSearchEnabled(t *testing.T) { WaitTillTrivyDBDownloadStarted(tempDir) So(err, ShouldBeNil) So(data, ShouldContainSubstring, - "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll + "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}") //nolint:lll // gofumpt conflicts with lll So(data, ShouldContainSubstring, "updating the CVE database") }) } @@ -529,7 +588,7 @@ func TestServeSearchEnabledCVE(t *testing.T) { So(err, ShouldBeNil) // Even if in config we specified updateInterval=1h, the minimum interval is 2h So(data, ShouldContainSubstring, - "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":3600000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll + "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":3600000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}") //nolint:lll // gofumpt conflicts with lll So(data, ShouldContainSubstring, "updating the CVE database") So(data, ShouldContainSubstring, "CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.") @@ -567,7 +626,7 @@ func TestServeSearchEnabledNoCVE(t *testing.T) { WaitTillTrivyDBDownloadStarted(tempDir) So(err, ShouldBeNil) So(data, ShouldContainSubstring, - "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll + "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}") //nolint:lll // gofumpt conflicts with lll So(data, ShouldContainSubstring, "updating the CVE database") }) } @@ -605,7 +664,7 @@ func TestServeSearchDisabled(t *testing.T) { So(err, ShouldBeNil) So(data, ShouldContainSubstring, - "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":10800000000000},\"Enable\":false},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll + "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":10800000000000},\"Enable\":false},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}") //nolint:lll // gofumpt conflicts with lll So(data, ShouldContainSubstring, "CVE config not provided, skipping CVE update") So(data, ShouldNotContainSubstring, "CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.") diff --git a/pkg/extensions/config/config.go b/pkg/extensions/config/config.go index 3000c313..4855ac77 100644 --- a/pkg/extensions/config/config.go +++ b/pkg/extensions/config/config.go @@ -11,6 +11,12 @@ type ExtensionConfig struct { Sync *sync.Config Metrics *MetricsConfig Scrub *ScrubConfig + Lint *LintConfig +} + +type LintConfig struct { + Enabled *bool + MandatoryAnnotations []string } type SearchConfig struct { diff --git a/pkg/extensions/extensions_lint.go b/pkg/extensions/extensions_lint.go new file mode 100644 index 00000000..d34deefa --- /dev/null +++ b/pkg/extensions/extensions_lint.go @@ -0,0 +1,18 @@ +//go:build lint +// +build lint + +package extensions + +import ( + "zotregistry.io/zot/pkg/api/config" + "zotregistry.io/zot/pkg/extensions/lint" + "zotregistry.io/zot/pkg/log" +) + +func GetLinter(config *config.Config, log log.Logger) *lint.Linter { + if config.Extensions == nil { + return lint.NewLinter(nil, log) + } + + return lint.NewLinter(config.Extensions.Lint, log) +} diff --git a/pkg/extensions/extensions_lint_disabled.go b/pkg/extensions/extensions_lint_disabled.go new file mode 100644 index 00000000..d2d803d1 --- /dev/null +++ b/pkg/extensions/extensions_lint_disabled.go @@ -0,0 +1,17 @@ +//go:build !lint +// +build !lint + +package extensions + +import ( + "zotregistry.io/zot/pkg/api/config" + "zotregistry.io/zot/pkg/extensions/lint" + "zotregistry.io/zot/pkg/log" +) + +func GetLinter(config *config.Config, log log.Logger) *lint.Linter { + log.Warn().Msg("lint extension is disabled because given zot binary doesn't" + + "include this feature please build a binary that does so") + + return nil +} diff --git a/pkg/extensions/lint/lint-disabled.go b/pkg/extensions/lint/lint-disabled.go new file mode 100644 index 00000000..33e1dc7e --- /dev/null +++ b/pkg/extensions/lint/lint-disabled.go @@ -0,0 +1,17 @@ +//go:build !lint +// +build !lint + +package lint + +import ( + godigest "github.com/opencontainers/go-digest" + "zotregistry.io/zot/pkg/storage" +) + +type Linter struct{} + +func (linter *Linter) Lint(repo string, manifestDigest godigest.Digest, + imageStore storage.ImageStore, +) (bool, error) { + return true, nil +} diff --git a/pkg/extensions/lint/lint.go b/pkg/extensions/lint/lint.go new file mode 100644 index 00000000..6ba17a38 --- /dev/null +++ b/pkg/extensions/lint/lint.go @@ -0,0 +1,76 @@ +//go:build lint +// +build lint + +package lint + +import ( + "encoding/json" + + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + "zotregistry.io/zot/pkg/extensions/config" + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/storage" +) + +type Linter struct { + config *config.LintConfig + log log.Logger +} + +func NewLinter(config *config.LintConfig, log log.Logger) *Linter { + return &Linter{ + config: config, + log: log, + } +} + +func (linter *Linter) CheckMandatoryAnnotations(repo string, manifestDigest godigest.Digest, + imgStore storage.ImageStore, +) (bool, error) { + if linter.config == nil { + return true, nil + } + + if (linter.config != nil && !*linter.config.Enabled) || len(linter.config.MandatoryAnnotations) == 0 { + return true, nil + } + + mandatoryAnnotationsList := linter.config.MandatoryAnnotations + + content, err := imgStore.GetBlobContent(repo, string(manifestDigest)) + if err != nil { + linter.log.Error().Err(err).Msg("linter: unable to get image manifest") + + return false, err + } + + var manifest ispec.Manifest + + if err := json.Unmarshal(content, &manifest); err != nil { + linter.log.Error().Err(err).Msg("linter: couldn't unmarshal manifest JSON") + + return false, err + } + + annotations := manifest.Annotations + + for _, annot := range mandatoryAnnotationsList { + _, found := annotations[annot] + + if !found { + // if annotations are not found, return false but it's not an error + linter.log.Error().Msgf("linter: missing %s annotations", annot) + + return false, nil + } + } + + return true, nil +} + +func (linter *Linter) Lint(repo string, manifestDigest godigest.Digest, + imageStore storage.ImageStore, +) (bool, error) { + return linter.CheckMandatoryAnnotations(repo, manifestDigest, imageStore) +} diff --git a/pkg/extensions/lint/lint_test.go b/pkg/extensions/lint/lint_test.go new file mode 100644 index 00000000..9cbc304b --- /dev/null +++ b/pkg/extensions/lint/lint_test.go @@ -0,0 +1,645 @@ +//go:build lint +// +build lint + +package lint_test + +import ( + "context" + "encoding/json" + "io/ioutil" + "net/http" + "os" + "path" + "testing" + + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + . "github.com/smartystreets/goconvey/convey" + "gopkg.in/resty.v1" + "zotregistry.io/zot/pkg/api" + "zotregistry.io/zot/pkg/api/config" + extconf "zotregistry.io/zot/pkg/extensions/config" + "zotregistry.io/zot/pkg/extensions/lint" + "zotregistry.io/zot/pkg/extensions/monitoring" + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/storage" + "zotregistry.io/zot/pkg/test" +) + +const ( + username = "test" + passphrase = "test" + ServerCert = "../../test/data/server.cert" + ServerKey = "../../test/data/server.key" + CACert = "../../test/data/ca.crt" + AuthorizedNamespace = "everyone/isallowed" + UnauthorizedNamespace = "fortknox/notallowed" + ALICE = "alice" + AuthorizationNamespace = "authz/image" + AuthorizationAllRepos = "**" +) + +func TestVerifyMandatoryAnnotations(t *testing.T) { + // nolint: dupl + Convey("Mandatory annotations disabled", t, func() { + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + + conf := config.New() + conf.HTTP.Port = port + enabled := false + conf.Extensions = &extconf.ExtensionConfig{Lint: &extconf.LintConfig{}} + conf.Extensions.Lint.MandatoryAnnotations = []string{} + conf.Extensions.Lint.Enabled = &enabled + + ctlr := api.NewController(conf) + dir := t.TempDir() + + err := test.CopyFiles("../../../test/data", dir) + if err != nil { + panic(err) + } + + ctlr.Config.Storage.RootDirectory = dir + + go startServer(ctlr) + defer stopServer(ctlr) + test.WaitTillServerReady(baseURL) + + resp, err := resty.R().SetBasicAuth(username, passphrase). + Get(baseURL + "/v2/zot-test/manifests/0.0.1") + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + manifestBlob := resp.Body() + var manifest ispec.Manifest + err = json.Unmarshal(manifestBlob, &manifest) + So(err, ShouldBeNil) + + manifest.SchemaVersion = 2 + content, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). + SetBody(content).Put(baseURL + "/v2/zot-test/manifests/0.0.1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + }) + + // nolint: dupl + Convey("Mandatory annotations enabled, but no list in config", t, func() { + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + + conf := config.New() + conf.HTTP.Port = port + enabled := true + conf.Extensions = &extconf.ExtensionConfig{Lint: &extconf.LintConfig{}} + conf.Extensions.Lint.MandatoryAnnotations = []string{} + + conf.Extensions.Lint.Enabled = &enabled + + ctlr := api.NewController(conf) + dir := t.TempDir() + + err := test.CopyFiles("../../../test/data", dir) + if err != nil { + panic(err) + } + + ctlr.Config.Storage.RootDirectory = dir + + go startServer(ctlr) + defer stopServer(ctlr) + test.WaitTillServerReady(baseURL) + + resp, err := resty.R().SetBasicAuth(username, passphrase). + Get(baseURL + "/v2/zot-test/manifests/0.0.1") + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + manifestBlob := resp.Body() + var manifest ispec.Manifest + err = json.Unmarshal(manifestBlob, &manifest) + So(err, ShouldBeNil) + + manifest.SchemaVersion = 2 + content, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). + SetBody(content).Put(baseURL + "/v2/zot-test/manifests/0.0.1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + }) + + Convey("Mandatory annotations verification passing", t, func() { + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + + conf := config.New() + conf.HTTP.Port = port + enabled := true + conf.Extensions = &extconf.ExtensionConfig{Lint: &extconf.LintConfig{}} + conf.Extensions.Lint.MandatoryAnnotations = []string{} + + conf.Extensions.Lint.Enabled = &enabled + conf.Extensions.Lint.MandatoryAnnotations = []string{"annotation1", "annotation2", "annotation3"} + + ctlr := api.NewController(conf) + dir := t.TempDir() + + err := test.CopyFiles("../../../test/data", dir) + if err != nil { + panic(err) + } + + ctlr.Config.Storage.RootDirectory = dir + + go startServer(ctlr) + defer stopServer(ctlr) + test.WaitTillServerReady(baseURL) + + resp, err := resty.R().SetBasicAuth(username, passphrase). + Get(baseURL + "/v2/zot-test/manifests/0.0.1") + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + manifestBlob := resp.Body() + var manifest ispec.Manifest + err = json.Unmarshal(manifestBlob, &manifest) + So(err, ShouldBeNil) + + manifest.Annotations = make(map[string]string) + + manifest.Annotations["annotation1"] = "testPass1" + manifest.Annotations["annotation2"] = "testPass2" + manifest.Annotations["annotation3"] = "testPass3" + + manifest.SchemaVersion = 2 + content, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). + SetBody(content).Put(baseURL + "/v2/zot-test/manifests/0.0.1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + }) + + Convey("Mandatory annotations incomplete in manifest", t, func() { + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + + conf := config.New() + conf.HTTP.Port = port + enabled := true + conf.Extensions = &extconf.ExtensionConfig{Lint: &extconf.LintConfig{}} + conf.Extensions.Lint.MandatoryAnnotations = []string{} + + conf.Extensions.Lint.Enabled = &enabled + conf.Extensions.Lint.MandatoryAnnotations = []string{"annotation1", "annotation2", "annotation3"} + + ctlr := api.NewController(conf) + dir := t.TempDir() + + err := test.CopyFiles("../../../test/data", dir) + if err != nil { + panic(err) + } + + ctlr.Config.Storage.RootDirectory = dir + + go startServer(ctlr) + defer stopServer(ctlr) + test.WaitTillServerReady(baseURL) + + resp, err := resty.R().SetBasicAuth(username, passphrase). + Get(baseURL + "/v2/zot-test/manifests/0.0.1") + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + manifestBlob := resp.Body() + var manifest ispec.Manifest + err = json.Unmarshal(manifestBlob, &manifest) + So(err, ShouldBeNil) + + manifest.Annotations = make(map[string]string) + + manifest.Annotations["annotation1"] = "testFail1" + manifest.Annotations["annotation3"] = "testFail3" + + manifest.SchemaVersion = 2 + content, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). + SetBody(content).Put(baseURL + "/v2/zot-test/manifests/0.0.1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusBadRequest) + }) + + Convey("Mandatory annotations verification passing - more annotations than the mandatory list", t, func() { + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + + conf := config.New() + conf.HTTP.Port = port + enabled := true + conf.Extensions = &extconf.ExtensionConfig{Lint: &extconf.LintConfig{}} + conf.Extensions.Lint.MandatoryAnnotations = []string{} + conf.Extensions.Lint.Enabled = &enabled + conf.Extensions.Lint.MandatoryAnnotations = []string{"annotation1", "annotation2", "annotation3"} + + ctlr := api.NewController(conf) + dir := t.TempDir() + + err := test.CopyFiles("../../../test/data", dir) + if err != nil { + panic(err) + } + + ctlr.Config.Storage.RootDirectory = dir + + go startServer(ctlr) + defer stopServer(ctlr) + test.WaitTillServerReady(baseURL) + + resp, err := resty.R().SetBasicAuth(username, passphrase). + Get(baseURL + "/v2/zot-test/manifests/0.0.1") + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + manifestBlob := resp.Body() + var manifest ispec.Manifest + err = json.Unmarshal(manifestBlob, &manifest) + So(err, ShouldBeNil) + + manifest.Annotations = make(map[string]string) + + manifest.Annotations["annotation1"] = "testPassMore1" + manifest.Annotations["annotation2"] = "testPassMore2" + manifest.Annotations["annotation3"] = "testPassMore3" + manifest.Annotations["annotation4"] = "testPassMore4" + + manifest.SchemaVersion = 2 + content, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). + SetBody(content).Put(baseURL + "/v2/zot-test/manifests/0.0.1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + }) +} + +func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { + Convey("Mandatory annotations disabled", t, func() { + enabled := false + + lintConfig := &extconf.LintConfig{ + Enabled: &enabled, + MandatoryAnnotations: []string{}, + } + + dir := t.TempDir() + + err := test.CopyFiles("../../../test/data", dir) + if err != nil { + panic(err) + } + + var index ispec.Index + + linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) + imgStore := storage.NewImageStore(dir, false, 0, false, false, + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter) + + indexContent, err := imgStore.GetIndexContent("zot-test") + So(err, ShouldBeNil) + err = json.Unmarshal(indexContent, &index) + So(err, ShouldBeNil) + + manifestDigest := index.Manifests[0].Digest + + pass, err := linter.CheckMandatoryAnnotations("zot-test", manifestDigest, imgStore) + So(err, ShouldBeNil) + So(pass, ShouldBeTrue) + }) + + Convey("Mandatory annotations enabled, but no list in config", t, func() { + enabled := true + + lintConfig := &extconf.LintConfig{ + Enabled: &enabled, + MandatoryAnnotations: []string{}, + } + + dir := t.TempDir() + + err := test.CopyFiles("../../../test/data", dir) + if err != nil { + panic(err) + } + + var index ispec.Index + + linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) + imgStore := storage.NewImageStore(dir, false, 0, false, false, + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter) + + indexContent, err := imgStore.GetIndexContent("zot-test") + So(err, ShouldBeNil) + err = json.Unmarshal(indexContent, &index) + So(err, ShouldBeNil) + + manifestDigest := index.Manifests[0].Digest + + pass, err := linter.CheckMandatoryAnnotations("zot-test", manifestDigest, imgStore) + So(err, ShouldBeNil) + So(pass, ShouldBeTrue) + }) + + Convey("Mandatory annotations verification passing", t, func() { + enabled := true + + lintConfig := &extconf.LintConfig{ + Enabled: &enabled, + MandatoryAnnotations: []string{"annotation1", "annotation2", "annotation3"}, + } + + dir := t.TempDir() + + err := test.CopyFiles("../../../test/data", dir) + if err != nil { + panic(err) + } + + var index ispec.Index + buf, err := ioutil.ReadFile(path.Join(dir, "zot-test", "index.json")) + So(err, ShouldBeNil) + err = json.Unmarshal(buf, &index) + So(err, ShouldBeNil) + + manifestDigest := index.Manifests[0].Digest + + var manifest ispec.Manifest + buf, err = ioutil.ReadFile(path.Join(dir, "zot-test", "blobs", + manifestDigest.Algorithm().String(), manifestDigest.Encoded())) + So(err, ShouldBeNil) + err = json.Unmarshal(buf, &manifest) + So(err, ShouldBeNil) + + manifest.Annotations = make(map[string]string) + + manifest.Annotations["annotation1"] = "testPass1" + manifest.Annotations["annotation2"] = "testPass2" + manifest.Annotations["annotation3"] = "testPass3" + + manifest.SchemaVersion = 2 + content, err := json.Marshal(manifest) + So(err, ShouldBeNil) + So(content, ShouldNotBeNil) + + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + + err = ioutil.WriteFile(path.Join(dir, "zot-test", "blobs", + digest.Algorithm().String(), digest.Encoded()), content, 0o600) + So(err, ShouldBeNil) + + manifestDesc := ispec.Descriptor{ + Size: int64(len(content)), + Digest: digest, + } + + index.Manifests = append(index.Manifests, manifestDesc) + + linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) + imgStore := storage.NewImageStore(dir, false, 0, false, false, + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter) + + pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) + So(err, ShouldBeNil) + So(pass, ShouldBeTrue) + }) + + Convey("Mandatory annotations incomplete in manifest", t, func() { + enabled := true + + lintConfig := &extconf.LintConfig{ + Enabled: &enabled, + MandatoryAnnotations: []string{"annotation1", "annotation2", "annotation3"}, + } + + dir := t.TempDir() + + err := test.CopyFiles("../../../test/data", dir) + if err != nil { + panic(err) + } + + var index ispec.Index + buf, err := ioutil.ReadFile(path.Join(dir, "zot-test", "index.json")) + So(err, ShouldBeNil) + err = json.Unmarshal(buf, &index) + So(err, ShouldBeNil) + + manifestDigest := index.Manifests[0].Digest + + var manifest ispec.Manifest + buf, err = ioutil.ReadFile(path.Join(dir, "zot-test", "blobs", + manifestDigest.Algorithm().String(), manifestDigest.Encoded())) + So(err, ShouldBeNil) + err = json.Unmarshal(buf, &manifest) + So(err, ShouldBeNil) + + manifest.Annotations = make(map[string]string) + + manifest.Annotations["annotation1"] = "test1" + manifest.Annotations["annotation3"] = "test3" + + manifest.SchemaVersion = 2 + content, err := json.Marshal(manifest) + So(err, ShouldBeNil) + So(content, ShouldNotBeNil) + + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + + err = ioutil.WriteFile(path.Join(dir, "zot-test", "blobs", + digest.Algorithm().String(), digest.Encoded()), content, 0o600) + So(err, ShouldBeNil) + + manifestDesc := ispec.Descriptor{ + Size: int64(len(content)), + Digest: digest, + } + + index.Manifests = append(index.Manifests, manifestDesc) + + linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) + imgStore := storage.NewImageStore(dir, false, 0, false, false, + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter) + + pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) + So(err, ShouldBeNil) + So(pass, ShouldBeFalse) + }) + + Convey("Mandatory annotations verification passing - more annotations than the mandatory list", t, func() { + enabled := true + + lintConfig := &extconf.LintConfig{ + Enabled: &enabled, + MandatoryAnnotations: []string{"annotation1", "annotation2", "annotation3"}, + } + + dir := t.TempDir() + + err := test.CopyFiles("../../../test/data", dir) + if err != nil { + panic(err) + } + + var index ispec.Index + buf, err := ioutil.ReadFile(path.Join(dir, "zot-test", "index.json")) + So(err, ShouldBeNil) + err = json.Unmarshal(buf, &index) + So(err, ShouldBeNil) + + manifestDigest := index.Manifests[0].Digest + + var manifest ispec.Manifest + buf, err = ioutil.ReadFile(path.Join(dir, "zot-test", "blobs", + manifestDigest.Algorithm().String(), manifestDigest.Encoded())) + So(err, ShouldBeNil) + err = json.Unmarshal(buf, &manifest) + So(err, ShouldBeNil) + + manifest.Annotations = make(map[string]string) + + manifest.Annotations["annotation1"] = "testPassMore1" + manifest.Annotations["annotation2"] = "testPassMore2" + manifest.Annotations["annotation3"] = "testPassMore3" + manifest.Annotations["annotation4"] = "testPassMore4" + + manifest.SchemaVersion = 2 + content, err := json.Marshal(manifest) + So(err, ShouldBeNil) + So(content, ShouldNotBeNil) + + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + + err = ioutil.WriteFile(path.Join(dir, "zot-test", "blobs", + digest.Algorithm().String(), digest.Encoded()), content, 0o600) + So(err, ShouldBeNil) + + manifestDesc := ispec.Descriptor{ + Size: int64(len(content)), + Digest: digest, + } + + index.Manifests = append(index.Manifests, manifestDesc) + + linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) + imgStore := storage.NewImageStore(dir, false, 0, false, false, + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter) + + pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) + So(err, ShouldBeNil) + So(pass, ShouldBeTrue) + }) + + Convey("Cannot unmarshal manifest", t, func() { + enabled := true + + lintConfig := &extconf.LintConfig{ + Enabled: &enabled, + MandatoryAnnotations: []string{"annotation1", "annotation2", "annotation3"}, + } + + dir := t.TempDir() + + err := test.CopyFiles("../../../test/data", dir) + if err != nil { + panic(err) + } + + var index ispec.Index + buf, err := ioutil.ReadFile(path.Join(dir, "zot-test", "index.json")) + So(err, ShouldBeNil) + err = json.Unmarshal(buf, &index) + So(err, ShouldBeNil) + + manifestDigest := index.Manifests[0].Digest + + var manifest ispec.Manifest + buf, err = ioutil.ReadFile(path.Join(dir, "zot-test", "blobs", + manifestDigest.Algorithm().String(), manifestDigest.Encoded())) + So(err, ShouldBeNil) + err = json.Unmarshal(buf, &manifest) + So(err, ShouldBeNil) + + manifest.Annotations = make(map[string]string) + + manifest.Annotations["annotation1"] = "testUnmarshal1" + manifest.Annotations["annotation2"] = "testUnmarshal2" + manifest.Annotations["annotation3"] = "testUnmarshal3" + + manifest.SchemaVersion = 2 + content, err := json.Marshal(manifest) + So(err, ShouldBeNil) + So(content, ShouldNotBeNil) + + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + + err = ioutil.WriteFile(path.Join(dir, "zot-test", "blobs", + digest.Algorithm().String(), digest.Encoded()), content, 0o600) + So(err, ShouldBeNil) + + manifestDesc := ispec.Descriptor{ + Size: int64(len(content)), + Digest: digest, + } + + index.Manifests = append(index.Manifests, manifestDesc) + + linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) + imgStore := storage.NewImageStore(dir, false, 0, false, false, + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter) + + err = os.Chmod(path.Join(dir, "zot-test", "blobs"), 0o000) + if err != nil { + panic(err) + } + + pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) + So(err, ShouldNotBeNil) + So(pass, ShouldBeFalse) + + err = os.Chmod(path.Join(dir, "zot-test", "blobs"), 0o755) + if err != nil { + panic(err) + } + }) +} + +func startServer(c *api.Controller) { + // this blocks + ctx := context.Background() + if err := c.Run(ctx); err != nil { + return + } +} + +func stopServer(c *api.Controller) { + ctx := context.Background() + _ = c.Server.Shutdown(ctx) +} diff --git a/pkg/extensions/scrub/scrub_test.go b/pkg/extensions/scrub/scrub_test.go index d0dd5163..23b38ffb 100644 --- a/pkg/extensions/scrub/scrub_test.go +++ b/pkg/extensions/scrub/scrub_test.go @@ -224,10 +224,15 @@ func TestRunScrubRepo(t *testing.T) { defer os.Remove(logFile.Name()) // clean up + conf := config.New() + conf.Extensions = &extconf.ExtensionConfig{} + conf.Extensions.Lint = &extconf.LintConfig{} + dir := t.TempDir() log := log.NewLogger("debug", logFile.Name()) metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, + true, log, metrics, nil) err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName)) if err != nil { @@ -247,10 +252,16 @@ func TestRunScrubRepo(t *testing.T) { defer os.Remove(logFile.Name()) // clean up + conf := config.New() + + conf.Extensions = &extconf.ExtensionConfig{} + conf.Extensions.Lint = &extconf.LintConfig{} + dir := t.TempDir() log := log.NewLogger("debug", logFile.Name()) metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, + true, log, metrics, nil) err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName)) if err != nil { @@ -277,10 +288,15 @@ func TestRunScrubRepo(t *testing.T) { defer os.Remove(logFile.Name()) // clean up + conf := config.New() + conf.Extensions = &extconf.ExtensionConfig{} + conf.Extensions.Lint = &extconf.LintConfig{} + dir := t.TempDir() log := log.NewLogger("debug", logFile.Name()) metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, 1*time.Second, + true, true, log, metrics, nil) err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName)) if err != nil { diff --git a/pkg/extensions/search/common/common_test.go b/pkg/extensions/search/common/common_test.go index c0c9a6de..88ee3d78 100644 --- a/pkg/extensions/search/common/common_test.go +++ b/pkg/extensions/search/common/common_test.go @@ -264,8 +264,13 @@ func TestImageFormat(t *testing.T) { log := log.NewLogger("debug", "") dbDir := "../../../../test/data" + conf := config.New() + conf.Extensions = &extconf.ExtensionConfig{} + conf.Extensions.Lint = &extconf.LintConfig{} + metrics := monitoring.NewMetricsServer(false, log) - defaultStore := storage.NewImageStore(dbDir, false, storage.DefaultGCDelay, false, false, log, metrics) + defaultStore := storage.NewImageStore(dbDir, false, storage.DefaultGCDelay, + false, false, log, metrics, nil) storeController := storage.StoreController{DefaultStore: defaultStore} olu := common.NewBaseOciLayoutUtils(storeController, log) @@ -708,10 +713,16 @@ func TestUtilsMethod(t *testing.T) { subRootDir := t.TempDir() - metrics := monitoring.NewMetricsServer(false, log) - defaultStore := storage.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics) + conf := config.New() + conf.Extensions = &extconf.ExtensionConfig{} + conf.Extensions.Lint = &extconf.LintConfig{} - subStore := storage.NewImageStore(subRootDir, false, storage.DefaultGCDelay, false, false, log, metrics) + metrics := monitoring.NewMetricsServer(false, log) + defaultStore := storage.NewImageStore(rootDir, false, + storage.DefaultGCDelay, false, false, log, metrics, nil) + + subStore := storage.NewImageStore(subRootDir, false, + storage.DefaultGCDelay, false, false, log, metrics, nil) subStoreMap := make(map[string]storage.ImageStore) diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 0ece828b..98d48d9d 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -92,7 +92,11 @@ func testSetup() error { log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storeController := storage.StoreController{DefaultStore: storage.NewImageStore(dir, false, storage.DefaultGCDelay, false, false, log, metrics)} + conf := config.New() + conf.Extensions = &extconf.ExtensionConfig{} + conf.Extensions.Lint = &extconf.LintConfig{} + + storeController := storage.StoreController{DefaultStore: storage.NewImageStore(dir, false, storage.DefaultGCDelay, false, false, log, metrics, nil)} layoutUtils := common.NewBaseOciLayoutUtils(storeController, log) @@ -334,12 +338,16 @@ func TestMultipleStoragePath(t *testing.T) { log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) + conf := config.New() + conf.Extensions = &extconf.ExtensionConfig{} + conf.Extensions.Lint = &extconf.LintConfig{} + // Create ImageStore - firstStore := storage.NewImageStore(firstRootDir, false, storage.DefaultGCDelay, false, false, log, metrics) + firstStore := storage.NewImageStore(firstRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil) - secondStore := storage.NewImageStore(secondRootDir, false, storage.DefaultGCDelay, false, false, log, metrics) + secondStore := storage.NewImageStore(secondRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil) - thirdStore := storage.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay, false, false, log, metrics) + thirdStore := storage.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil) storeController := storage.StoreController{} diff --git a/pkg/extensions/search/digest/digest_test.go b/pkg/extensions/search/digest/digest_test.go index 5afc36a7..dd22132e 100644 --- a/pkg/extensions/search/digest/digest_test.go +++ b/pkg/extensions/search/digest/digest_test.go @@ -97,10 +97,15 @@ func testSetup() error { return err } + conf := config.New() + conf.Extensions = &extconf.ExtensionConfig{} + conf.Extensions.Lint = &extconf.LintConfig{} + log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) storeController := storage.StoreController{ - DefaultStore: storage.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics), + DefaultStore: storage.NewImageStore(rootDir, false, storage.DefaultGCDelay, + false, false, log, metrics, nil), } digestInfo = digestinfo.NewDigestInfo(storeController, log) diff --git a/pkg/extensions/sync/on_demand.go b/pkg/extensions/sync/on_demand.go index 9fa2a88d..29eb250c 100644 --- a/pkg/extensions/sync/on_demand.go +++ b/pkg/extensions/sync/on_demand.go @@ -274,7 +274,8 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S imageChannel <- nil } -func syncRun(regCfg RegistryConfig, localRepo, remoteRepo, tag string, utils syncContextUtils, +func syncRun(regCfg RegistryConfig, + localRepo, remoteRepo, tag string, utils syncContextUtils, log log.Logger, ) (bool, error) { upstreamImageRef, err := getImageRef(utils.upstreamAddr, remoteRepo, tag) diff --git a/pkg/extensions/sync/signatures.go b/pkg/extensions/sync/signatures.go index 11fa936b..1ac41b0e 100644 --- a/pkg/extensions/sync/signatures.go +++ b/pkg/extensions/sync/signatures.go @@ -196,7 +196,8 @@ func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore, } // push manifest - _, err = imageStore.PutImageManifest(localRepo, cosignTag, ispec.MediaTypeImageManifest, cosignManifestBuf) + _, err = imageStore.PutImageManifest(localRepo, cosignTag, + ispec.MediaTypeImageManifest, cosignManifestBuf) if err != nil { log.Error().Str("errorType", TypeOf(err)). Err(err).Msg("couldn't upload cosign manifest") diff --git a/pkg/extensions/sync/sync.go b/pkg/extensions/sync/sync.go index c602237b..42a7d906 100644 --- a/pkg/extensions/sync/sync.go +++ b/pkg/extensions/sync/sync.go @@ -296,7 +296,8 @@ func getUpstreamContext(regCfg *RegistryConfig, credentials Credentials) *types. } // nolint:gocyclo // offloading some of the functionalities from here would make the code harder to follow -func syncRegistry(ctx context.Context, regCfg RegistryConfig, upstreamURL string, +func syncRegistry(ctx context.Context, regCfg RegistryConfig, + upstreamURL string, storeController storage.StoreController, localCtx *types.SystemContext, policyCtx *signature.PolicyContext, credentials Credentials, retryOptions *retry.RetryOptions, log log.Logger, @@ -509,7 +510,6 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig, upstreamURL string } // push from cache to repo err = pushSyncedLocalImage(localRepo, tag, localCachePath, imageStore, log) - if err != nil { log.Error().Str("errorType", TypeOf(err)). Err(err).Msgf("error while pushing synced cached image %s", @@ -573,7 +573,8 @@ func getLocalContexts(log log.Logger) (*types.SystemContext, *signature.PolicyCo return localCtx, policyContext, nil } -func Run(ctx context.Context, cfg Config, storeController storage.StoreController, +func Run(ctx context.Context, cfg Config, + storeController storage.StoreController, wtgrp *goSync.WaitGroup, logger log.Logger, ) error { var credentialsFile CredentialsFile diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index e9d29682..19a6874f 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -61,7 +61,9 @@ func TestInjectSyncUtils(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imageStore := storage.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay, false, false, log, metrics) + imageStore := storage.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay, + false, false, log, metrics, nil, + ) injected = test.InjectFailure(0) _, err = getLocalCachePath(imageStore, testImage) @@ -147,7 +149,8 @@ func TestSyncInternal(t *testing.T) { } ctx := context.Background() - So(Run(ctx, cfg, storage.StoreController{}, new(goSync.WaitGroup), log.NewLogger("debug", "")), ShouldNotBeNil) + So(Run(ctx, cfg, storage.StoreController{}, + new(goSync.WaitGroup), log.NewLogger("debug", "")), ShouldNotBeNil) _, err = getFileCredentials("/invalid/path/to/file") So(err, ShouldNotBeNil) @@ -157,7 +160,8 @@ func TestSyncInternal(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imageStore := storage.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay, false, false, log, metrics) + imageStore := storage.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay, + false, false, log, metrics, nil) err := os.Chmod(imageStore.RootDir(), 0o000) So(err, ShouldBeNil) @@ -340,7 +344,8 @@ func TestSyncInternal(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imageStore := storage.NewImageStore(storageDir, false, storage.DefaultGCDelay, false, false, log, metrics) + imageStore := storage.NewImageStore(storageDir, false, storage.DefaultGCDelay, + false, false, log, metrics, nil) refs := ReferenceList{[]artifactspec.Descriptor{ { @@ -422,7 +427,8 @@ func TestSyncInternal(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imageStore := storage.NewImageStore(storageDir, false, storage.DefaultGCDelay, false, false, log, metrics) + imageStore := storage.NewImageStore(storageDir, false, storage.DefaultGCDelay, + false, false, log, metrics, nil) storeController := storage.StoreController{} storeController.DefaultStore = imageStore @@ -443,7 +449,8 @@ func TestSyncInternal(t *testing.T) { panic(err) } - testImageStore := storage.NewImageStore(testRootDir, false, storage.DefaultGCDelay, false, false, log, metrics) + testImageStore := storage.NewImageStore(testRootDir, false, + storage.DefaultGCDelay, false, false, log, metrics, nil) manifestContent, _, _, err := testImageStore.GetImageManifest(testImage, testImageTag) So(err, ShouldBeNil) diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 847b5f3c..90e6edd1 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -862,6 +862,107 @@ func TestConfigReloader(t *testing.T) { }) } +func TestMandatoryAnnotations(t *testing.T) { + Convey("Verify mandatory annotations failing - on demand disabled", t, func() { + updateDuration, _ := time.ParseDuration("30m") + + sctlr, srcBaseURL, _, _, _ := startUpstreamServer(t, false, false) + + defer func() { + sctlr.Shutdown() + }() + + regex := ".*" + var semver bool + tlsVerify := false + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + Prefix: testImage, + Tags: &sync.Tags{ + Regex: ®ex, + Semver: &semver, + }, + }, + }, + URLs: []string{srcBaseURL}, + OnDemand: false, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + } + + defaultVal := true + syncConfig := &sync.Config{ + Enable: &defaultVal, + Registries: []sync.RegistryConfig{syncRegistryConfig}, + } + + destPort := test.GetFreePort() + destConfig := config.New() + destClient := resty.New() + + destBaseURL := test.GetBaseURL(destPort) + + destConfig.HTTP.Port = destPort + + destDir := t.TempDir() + + destConfig.Storage.RootDirectory = destDir + destConfig.Storage.Dedupe = false + destConfig.Storage.GC = false + + destConfig.Extensions = &extconf.ExtensionConfig{} + destConfig.Extensions.Sync = syncConfig + + logFile, err := ioutil.TempFile("", "zot-log*.txt") + So(err, ShouldBeNil) + + destConfig.Log.Output = logFile.Name() + + lintEnabled := true + destConfig.Extensions.Lint = &extconf.LintConfig{} + destConfig.Extensions.Lint.Enabled = &lintEnabled + destConfig.Extensions.Lint.MandatoryAnnotations = []string{"annot1", "annot2", "annot3"} + + dctlr := api.NewController(destConfig) + + go func() { + // this blocks + if err := dctlr.Run(context.Background()); err != nil { + return + } + }() + + // wait till ready + for { + _, err := destClient.R().Get(destBaseURL) + if err == nil { + break + } + + time.Sleep(100 * time.Millisecond) + } + + defer func() { + dctlr.Shutdown() + }() + + // give it time to set up sync + time.Sleep(3 * time.Second) + + resp, err := destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifest/0.0.1") + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 404) + + data, err := os.ReadFile(logFile.Name()) + t.Logf("downstream log: %s", string(data)) + So(err, ShouldBeNil) + So(string(data), ShouldContainSubstring, "couldn't upload manifest because of missing annotations") + }) +} + func TestBadTLS(t *testing.T) { Convey("Verify sync TLS feature", t, func() { updateDuration, _ := time.ParseDuration("30m") diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go index 1b009bc3..6d461fd8 100644 --- a/pkg/extensions/sync/utils.go +++ b/pkg/extensions/sync/utils.go @@ -270,7 +270,9 @@ func pushSyncedLocalImage(localRepo, tag, localCachePath string, log.Info().Msgf("pushing synced local image %s/%s:%s to local registry", localCachePath, localRepo, tag) metrics := monitoring.NewMetricsServer(false, log) - cacheImageStore := storage.NewImageStore(localCachePath, false, storage.DefaultGCDelay, false, false, log, metrics) + + cacheImageStore := storage.NewImageStore(localCachePath, false, + storage.DefaultGCDelay, false, false, log, metrics, nil) manifestContent, _, _, err := cacheImageStore.GetImageManifest(localRepo, tag) if err != nil { @@ -331,8 +333,16 @@ func pushSyncedLocalImage(localRepo, tag, localCachePath string, } } - _, err = imageStore.PutImageManifest(localRepo, tag, ispec.MediaTypeImageManifest, manifestContent) + _, err = imageStore.PutImageManifest(localRepo, tag, + ispec.MediaTypeImageManifest, manifestContent) if err != nil { + if errors.Is(err, zerr.ErrImageLintAnnotations) { + log.Error().Str("errorType", TypeOf(err)). + Err(err).Msg("couldn't upload manifest because of missing annotations") + + return nil + } + log.Error().Str("errorType", TypeOf(err)). Err(err).Msg("couldn't upload manifest") diff --git a/pkg/storage/lint-interface.go b/pkg/storage/lint-interface.go new file mode 100644 index 00000000..f679a27e --- /dev/null +++ b/pkg/storage/lint-interface.go @@ -0,0 +1,9 @@ +package storage + +import ( + godigest "github.com/opencontainers/go-digest" +) + +type Lint interface { + Lint(repo string, manifestDigest godigest.Digest, imageStore ImageStore) (bool, error) +} diff --git a/pkg/storage/local.go b/pkg/storage/local.go index 2261f4d8..7a768494 100644 --- a/pkg/storage/local.go +++ b/pkg/storage/local.go @@ -25,6 +25,7 @@ import ( "github.com/opencontainers/umoci/oci/casext" artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" "github.com/rs/zerolog" + "github.com/sigstore/cosign/pkg/oci/remote" zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/extensions/monitoring" zlog "zotregistry.io/zot/pkg/log" @@ -64,6 +65,7 @@ type ImageStoreLocal struct { gcDelay time.Duration log zerolog.Logger metrics monitoring.MetricServer + linter Lint } func (is *ImageStoreLocal) RootDir() string { @@ -105,7 +107,7 @@ func (sc StoreController) GetImageStore(name string) ImageStore { // NewImageStore returns a new image store backed by a file storage. func NewImageStore(rootDir string, gc bool, gcDelay time.Duration, dedupe, commit bool, - log zlog.Logger, metrics monitoring.MetricServer, + log zlog.Logger, metrics monitoring.MetricServer, linter Lint, ) ImageStore { if _, err := os.Stat(rootDir); os.IsNotExist(err) { if err := os.MkdirAll(rootDir, DefaultDirPerms); err != nil { @@ -125,6 +127,7 @@ func NewImageStore(rootDir string, gc bool, gcDelay time.Duration, dedupe, commi commit: commit, log: log.With().Caller().Logger(), metrics: metrics, + linter: linter, } if dedupe { @@ -549,29 +552,9 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, return "", zerr.ErrBadManifest } - if mediaType == ispec.MediaTypeImageManifest { - var manifest ispec.Manifest - if err := json.Unmarshal(body, &manifest); err != nil { - is.log.Error().Err(err).Msg("unable to unmarshal JSON") - - return "", zerr.ErrBadManifest - } - - if manifest.Config.MediaType == ispec.MediaTypeImageConfig { - digest, err := is.validateOCIManifest(repo, reference, &manifest) - if err != nil { - is.log.Error().Err(err).Msg("invalid oci image manifest") - - return digest, err - } - } - } else if mediaType == artifactspec.MediaTypeArtifactManifest { - var m notation.Descriptor - if err := json.Unmarshal(body, &m); err != nil { - is.log.Error().Err(err).Msg("unable to unmarshal JSON") - - return "", zerr.ErrBadManifest - } + dig, err := validateManifest(is, repo, reference, mediaType, body) + if err != nil { + return dig, err } mDigest := godigest.FromBytes(body) @@ -666,6 +649,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, _ = ensureDir(dir, is.log) file := path.Join(dir, mDigest.Encoded()) + // in case the linter will not pass, it will be garbage collected if err := is.writeFile(file, body); err != nil { is.log.Error().Err(err).Str("file", file).Msg("unable to write") @@ -684,6 +668,28 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, return "", err } + // apply linter only on images, not signatures + if is.linter != nil { + if mediaType == ispec.MediaTypeImageManifest && + // check that image manifest is not cosign signature + !strings.HasPrefix(reference, "sha256-") && + !strings.HasSuffix(reference, remote.SignatureTagSuffix) { + // lint new index with new manifest before writing to disk + is.Unlock(&lockLatency) + pass, err := is.linter.Lint(repo, mDigest, is) + is.Lock(&lockLatency) + if err != nil { + is.log.Error().Err(err).Msg("linter error") + + return "", err + } + + if !pass { + return "", zerr.ErrImageLintAnnotations + } + } + } + err = is.writeFile(file, buf) if err := test.Error(err); err != nil { is.log.Error().Err(err).Str("file", file).Msg("unable to write") @@ -1616,6 +1622,37 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error { return nil } +func validateManifest(imgStore *ImageStoreLocal, repo, reference, + mediaType string, body []byte, +) (string, error) { + if mediaType == ispec.MediaTypeImageManifest { + var manifest ispec.Manifest + if err := json.Unmarshal(body, &manifest); err != nil { + imgStore.log.Error().Err(err).Msg("unable to unmarshal JSON") + + return "", zerr.ErrBadManifest + } + + if manifest.Config.MediaType == ispec.MediaTypeImageConfig { + digest, err := imgStore.validateOCIManifest(repo, reference, &manifest) + if err != nil { + imgStore.log.Error().Err(err).Msg("invalid oci image manifest") + + return digest, err + } + } + } else if mediaType == artifactspec.MediaTypeArtifactManifest { + var m notation.Descriptor + if err := json.Unmarshal(body, &m); err != nil { + imgStore.log.Error().Err(err).Msg("unable to unmarshal JSON") + + return "", zerr.ErrBadManifest + } + } + + return "", nil +} + func ifOlderThan(imgStore *ImageStoreLocal, repo string, delay time.Duration) casext.GCPolicy { return func(ctx context.Context, digest godigest.Digest) (bool, error) { blobPath := imgStore.BlobPath(repo, digest) diff --git a/pkg/storage/local_elevated_test.go b/pkg/storage/local_elevated_test.go index bd7ebd15..66244bfb 100644 --- a/pkg/storage/local_elevated_test.go +++ b/pkg/storage/local_elevated_test.go @@ -27,7 +27,7 @@ func TestElevatedPrivilegesInvalidDedupe(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil) upload, err := imgStore.NewBlobUpload("dedupe1") So(err, ShouldBeNil) diff --git a/pkg/storage/local_test.go b/pkg/storage/local_test.go index ff8bc3c2..23acfcf4 100644 --- a/pkg/storage/local_test.go +++ b/pkg/storage/local_test.go @@ -36,7 +36,8 @@ func TestStorageFSAPIs(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, + true, log, metrics, nil) Convey("Repo layout", t, func(c C) { repoName := "test" @@ -169,7 +170,7 @@ func TestGetReferrers(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil) Convey("Get referrers", t, func(c C) { err := test.CopyFiles("../../test/data/zot-test", path.Join(dir, "zot-test")) @@ -218,7 +219,8 @@ func TestDedupeLinks(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, + true, true, log, metrics, nil) Convey("Dedupe", t, func(c C) { // manifest1 @@ -272,7 +274,8 @@ func TestDedupeLinks(t *testing.T) { manifestBuf, err := json.Marshal(manifest) So(err, ShouldBeNil) digest = godigest.FromBytes(manifestBuf) - _, err = imgStore.PutImageManifest("dedupe1", digest.String(), ispec.MediaTypeImageManifest, manifestBuf) + _, err = imgStore.PutImageManifest("dedupe1", digest.String(), + ispec.MediaTypeImageManifest, manifestBuf) So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("dedupe1", digest.String()) @@ -356,7 +359,7 @@ func TestDedupe(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - il := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + il := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil) So(il.DedupeBlob("", "", ""), ShouldNotBeNil) }) @@ -371,9 +374,11 @@ func TestNegativeCases(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - So(storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics), ShouldNotBeNil) + So(storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, + true, log, metrics, nil), ShouldNotBeNil) if os.Geteuid() != 0 { - So(storage.NewImageStore("/deadBEEF", true, storage.DefaultGCDelay, true, true, log, metrics), ShouldBeNil) + So(storage.NewImageStore("/deadBEEF", true, storage.DefaultGCDelay, + true, true, log, metrics, nil), ShouldBeNil) } }) @@ -382,7 +387,8 @@ func TestNegativeCases(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, + true, true, log, metrics, nil) err := os.Chmod(dir, 0o000) // remove all perms if err != nil { @@ -417,7 +423,8 @@ func TestNegativeCases(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, + true, log, metrics, nil) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -531,7 +538,8 @@ func TestNegativeCases(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, + true, true, log, metrics, nil) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -554,7 +562,8 @@ func TestNegativeCases(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, + true, log, metrics, nil) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -595,7 +604,8 @@ func TestNegativeCases(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, + true, true, log, metrics, nil) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -739,7 +749,8 @@ func TestInjectWriteFile(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, + true, true, log, metrics, nil) Convey("Failure path1", func() { injected := test.InjectFailure(0) @@ -769,7 +780,8 @@ func TestInjectWriteFile(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, false, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, + true, false, log, metrics, nil) Convey("Failure path not reached", func() { err := imgStore.InitRepo("repo1") @@ -786,7 +798,8 @@ func TestGarbageCollect(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) Convey("Garbage collect with default/long delay", func() { - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, + true, true, log, metrics, nil) repoName := "gc-long" upload, err := imgStore.NewBlobUpload(repoName) @@ -853,7 +866,7 @@ func TestGarbageCollect(t *testing.T) { }) Convey("Garbage collect with short delay", func() { - imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil) repoName := "gc-short" // upload orphan blob @@ -949,7 +962,7 @@ func TestGarbageCollect(t *testing.T) { Convey("Garbage collect with dedupe", func() { // garbage-collect is repo-local and dedupe is global and they can interact in strange ways - imgStore := storage.NewImageStore(dir, true, 5*time.Second, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, 5*time.Second, true, true, log, metrics, nil) // first upload an image to the first repo and wait for GC timeout @@ -1150,7 +1163,7 @@ func TestGarbageCollectForImageStore(t *testing.T) { log := log.NewLogger("debug", logFile.Name()) metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil) repoName := "gc-all-repos-short" err := test.CopyFiles("../../test/data/zot-test", path.Join(dir, repoName)) @@ -1182,7 +1195,7 @@ func TestGarbageCollectForImageStore(t *testing.T) { log := log.NewLogger("debug", logFile.Name()) metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil) repoName := "gc-all-repos-short" err := test.CopyFiles("../../test/data/zot-test", path.Join(dir, repoName)) @@ -1227,7 +1240,8 @@ func TestInitRepo(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, + true, true, log, metrics, nil) err := os.Mkdir(path.Join(dir, "test-dir"), 0o000) So(err, ShouldBeNil) @@ -1243,7 +1257,8 @@ func TestValidateRepo(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, + true, true, log, metrics, nil) err := os.Mkdir(path.Join(dir, "test-dir"), 0o000) So(err, ShouldBeNil) @@ -1259,7 +1274,9 @@ func TestGetRepositoriesError(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, + true, true, log, metrics, nil, + ) // create valid directory with permissions err := os.Mkdir(path.Join(dir, "test-dir"), 0o755) @@ -1279,7 +1296,8 @@ func TestPutBlobChunkStreamed(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, + true, true, log, metrics, nil) uuid, err := imgStore.NewBlobUpload("test") So(err, ShouldBeNil) diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index 7ac67d11..90533319 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -11,6 +11,7 @@ import ( "os" "path" "path/filepath" + "strings" "sync" "time" @@ -23,6 +24,7 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" "github.com/rs/zerolog" + "github.com/sigstore/cosign/pkg/oci/remote" zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/extensions/monitoring" zlog "zotregistry.io/zot/pkg/log" @@ -50,6 +52,7 @@ type ObjectStorage struct { metrics monitoring.MetricServer cache *storage.Cache dedupe bool + linter storage.Lint } func (is *ObjectStorage) RootDir() string { @@ -67,7 +70,7 @@ func (is *ObjectStorage) DirExists(d string) bool { // NewObjectStorage returns a new image store backed by cloud storages. // see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers func NewImageStore(rootDir string, cacheDir string, gc bool, gcDelay time.Duration, dedupe, commit bool, - log zlog.Logger, metrics monitoring.MetricServer, + log zlog.Logger, metrics monitoring.MetricServer, linter storage.Lint, store driver.StorageDriver, ) storage.ImageStore { imgStore := &ObjectStorage{ @@ -79,6 +82,7 @@ func NewImageStore(rootDir string, cacheDir string, gc bool, gcDelay time.Durati multiPartUploads: sync.Map{}, metrics: metrics, dedupe: dedupe, + linter: linter, } cachePath := path.Join(cacheDir, CacheDBName+storage.DBExtensionName) @@ -395,8 +399,8 @@ func (is *ObjectStorage) GetImageManifest(repo, reference string) ([]byte, strin // PutImageManifest adds an image manifest to the repository. func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, - body []byte, -) (string, error) { + body []byte) (string, error, +) { if err := is.InitRepo(repo); err != nil { is.log.Debug().Err(err).Msg("init repo") @@ -549,6 +553,26 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, return "", err } + // apply linter only on images, not signatures + if is.linter != nil { + if mediaType == ispec.MediaTypeImageManifest && + // check that image manifest is not cosign signature + !strings.HasPrefix(reference, "sha256-") && + !strings.HasSuffix(reference, remote.SignatureTagSuffix) { + // lint new index with new manifest before writing to disk + pass, err := is.linter.Lint(repo, mDigest, is) + if err != nil { + is.log.Error().Err(err).Msg("linter error") + + return "", err + } + + if !pass { + return "", zerr.ErrImageLintAnnotations + } + } + } + if err = is.store.PutContent(context.Background(), indexPath, buf); err != nil { is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write") diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index a68e3d8b..c9adf650 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -56,7 +56,9 @@ func skipIt(t *testing.T) { func createMockStorage(rootDir string, cacheDir string, dedupe bool, store driver.StorageDriver) storage.ImageStore { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, dedupe, false, log, metrics, store) + il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, + dedupe, false, log, metrics, nil, store, + ) return il } @@ -95,7 +97,8 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) ( log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, dedupe, false, log, metrics, store) + il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, + dedupe, false, log, metrics, nil, store) return store, il, err } @@ -894,7 +897,8 @@ func TestS3Dedupe(t *testing.T) { manifestBuf, err := json.Marshal(manifest) So(err, ShouldBeNil) digest = godigest.FromBytes(manifestBuf) - _, err = imgStore.PutImageManifest("dedupe1", digest.String(), ispec.MediaTypeImageManifest, manifestBuf) + _, err = imgStore.PutImageManifest("dedupe1", digest.String(), + ispec.MediaTypeImageManifest, manifestBuf) So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("dedupe1", digest.String()) @@ -956,7 +960,8 @@ func TestS3Dedupe(t *testing.T) { manifestBuf, err = json.Marshal(manifest) So(err, ShouldBeNil) digest = godigest.FromBytes(manifestBuf) - _, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest, manifestBuf) + _, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest, + manifestBuf) So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("dedupe2", digest.String()) @@ -1078,7 +1083,8 @@ func TestS3Dedupe(t *testing.T) { manifestBuf, err = json.Marshal(manifest) So(err, ShouldBeNil) digest = godigest.FromBytes(manifestBuf) - _, err = imgStore.PutImageManifest("dedupe3", "1.0", ispec.MediaTypeImageManifest, manifestBuf) + _, err = imgStore.PutImageManifest("dedupe3", "1.0", ispec.MediaTypeImageManifest, + manifestBuf) So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("dedupe3", digest.String()) diff --git a/pkg/storage/scrub_test.go b/pkg/storage/scrub_test.go index e4cc848e..9b9365dc 100644 --- a/pkg/storage/scrub_test.go +++ b/pkg/storage/scrub_test.go @@ -31,7 +31,8 @@ func TestCheckAllBlobsIntegrity(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) - imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, + true, true, log, metrics, nil) Convey("Scrub only one repo", t, func(c C) { // initialize repo @@ -117,10 +118,11 @@ func TestCheckAllBlobsIntegrity(t *testing.T) { } mnfst.SchemaVersion = 2 - mb, err := json.Marshal(mnfst) + mbytes, err := json.Marshal(mnfst) So(err, ShouldBeNil) - manifest, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, mb) + manifest, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, + mbytes) So(err, ShouldBeNil) Convey("Blobs integrity not affected", func() { diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 4f781a23..f0d8be4e 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -5,6 +5,7 @@ import ( "context" _ "crypto/sha256" "encoding/json" + "errors" "fmt" "os" "path" @@ -28,6 +29,7 @@ import ( "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/s3" "zotregistry.io/zot/pkg/test" + "zotregistry.io/zot/pkg/test/mocks" ) func cleanupStorage(store driver.StorageDriver, name string) { @@ -73,7 +75,9 @@ func createObjectsStore(rootDir string, cacheDir string) (driver.StorageDriver, log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, true, false, log, metrics, store) + il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, + true, false, log, metrics, nil, store, + ) return store, il, err } @@ -117,7 +121,8 @@ func TestStorageAPIs(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imgStore = storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics) + imgStore = storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, + true, log, metrics, nil) } Convey("Repo layout", t, func(c C) { @@ -446,10 +451,12 @@ func TestStorageAPIs(t *testing.T) { }) Convey("Bad image manifest", func() { - _, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, manifestBuf) + _, err = imgStore.PutImageManifest("test", digest.String(), + ispec.MediaTypeImageManifest, manifestBuf) So(err, ShouldNotBeNil) - _, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, []byte("bad json")) + _, err = imgStore.PutImageManifest("test", digest.String(), + ispec.MediaTypeImageManifest, []byte("bad json")) So(err, ShouldNotBeNil) _, _, _, err = imgStore.GetImageManifest("test", digest.String()) @@ -483,11 +490,13 @@ func TestStorageAPIs(t *testing.T) { manifestBuf, err = json.Marshal(manifest) So(err, ShouldBeNil) digest := godigest.FromBytes(manifestBuf) - _, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, manifestBuf) + _, err = imgStore.PutImageManifest("test", digest.String(), + ispec.MediaTypeImageManifest, manifestBuf) So(err, ShouldBeNil) // same manifest for coverage - _, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, manifestBuf) + _, err = imgStore.PutImageManifest("test", digest.String(), + ispec.MediaTypeImageManifest, manifestBuf) So(err, ShouldBeNil) _, _, _, err = imgStore.GetImageManifest("test", digest.String()) @@ -661,6 +670,117 @@ func TestStorageAPIs(t *testing.T) { } } +func TestMandatoryAnnotations(t *testing.T) { + for _, testcase := range testCases { + testcase := testcase + t.Run(testcase.testCaseName, func(t *testing.T) { + var imgStore storage.ImageStore + var testDir, tdir string + var store driver.StorageDriver + + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + + if testcase.storageType == "s3" { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir = path.Join("/oci-repo-test", uuid.String()) + tdir = t.TempDir() + + store, _, _ = createObjectsStore(testDir, tdir) + imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics, + &mocks.MockedLint{ + LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) { + return false, nil + }, + }, store) + + defer cleanupStorage(store, testDir) + } else { + tdir = t.TempDir() + + imgStore = storage.NewImageStore(tdir, true, storage.DefaultGCDelay, true, + true, log, metrics, &mocks.MockedLint{ + LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) { + return false, nil + }, + }) + } + + Convey("Setup manifest", t, func() { + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + digest := godigest.FromBytes(content) + + _, _, err := imgStore.FullBlobUpload("test", bytes.NewReader(buf.Bytes()), digest.String()) + So(err, ShouldBeNil) + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload("test", bytes.NewReader(cblob), cdigest.String()) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + + annotationsMap := make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = "1.0" + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + Convey("Missing mandatory annotations", func() { + _, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldNotBeNil) + }) + + Convey("Error on mandatory annotations", func() { + if testcase.storageType == "s3" { + imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics, + &mocks.MockedLint{ + LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) { + // nolint: goerr113 + return false, errors.New("linter error") + }, + }, store) + } else { + imgStore = storage.NewImageStore(tdir, true, storage.DefaultGCDelay, true, + true, log, metrics, &mocks.MockedLint{ + LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) { + // nolint: goerr113 + return false, errors.New("linter error") + }, + }) + } + + _, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldNotBeNil) + }) + }) + }) + } +} + func TestStorageHandler(t *testing.T) { for _, testcase := range testCases { testcase := testcase @@ -700,11 +820,14 @@ func TestStorageHandler(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) // Create ImageStore - firstStore = storage.NewImageStore(firstRootDir, false, storage.DefaultGCDelay, false, false, log, metrics) + firstStore = storage.NewImageStore(firstRootDir, false, storage.DefaultGCDelay, + false, false, log, metrics, nil) - secondStore = storage.NewImageStore(secondRootDir, false, storage.DefaultGCDelay, false, false, log, metrics) + secondStore = storage.NewImageStore(secondRootDir, false, + storage.DefaultGCDelay, false, false, log, metrics, nil) - thirdStore = storage.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay, false, false, log, metrics) + thirdStore = storage.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay, + false, false, log, metrics, nil) } Convey("Test storage handler", t, func() { diff --git a/pkg/test/mocks/lint_mock.go b/pkg/test/mocks/lint_mock.go new file mode 100644 index 00000000..0fe112af --- /dev/null +++ b/pkg/test/mocks/lint_mock.go @@ -0,0 +1,18 @@ +package mocks + +import ( + godigest "github.com/opencontainers/go-digest" + "zotregistry.io/zot/pkg/storage" +) + +type MockedLint struct { + LintFn func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) +} + +func (lint MockedLint) Lint(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) { + if lint.LintFn != nil { + return lint.LintFn(repo, manifestDigest, imageStore) + } + + return false, nil +}