0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2025-01-06 22:40:28 -05:00

image level lint: enforce manifest mandatory annotations

closes 

Signed-off-by: Lisca Ana-Roberta <ana.kagome@yahoo.com>
This commit is contained in:
Lisca Ana-Roberta 2022-06-24 16:08:47 +03:00 committed by Andrei Aaron
parent 3d72dad507
commit 87fc941b3c
34 changed files with 1391 additions and 110 deletions

View file

@ -31,7 +31,7 @@ jobs:
# Optional: golangci-lint command line arguments.
# args: --issues-exit-code=0
args: --config ./golangcilint.yaml --enable-all --build-tags sync,scrub,search,metrics,ui_base,containers_image_openpgp ./cmd/... ./pkg/...
args: --config ./golangcilint.yaml --enable-all --build-tags sync,scrub,search,metrics,ui_base,containers_image_openpgp,lint ./cmd/... ./pkg/...
# Optional: show only new issues if it's a pull request. The default value is `false`.
# only-new-issues: true

View file

@ -16,7 +16,7 @@ TESTDATA := $(TOP_LEVEL)/test/data
OS ?= linux
ARCH ?= amd64
BENCH_OUTPUT ?= stdout
EXTENSIONS ?= sync,search,scrub,metrics,ui_base
EXTENSIONS ?= sync,search,scrub,metrics,ui_base,lint
comma:= ,
hyphen:= -
extended-name:=

View file

@ -20,6 +20,7 @@ The following document refers on the **core dist-spec**, see also the [zot-speci
* Supports container image signatures - [cosign](https://github.com/sigstore/cosign) and [notation](https://github.com/notaryproject/notation)
* Multi-arch support
* Clustering support
* Image linting support
## [Demos](demos/README.md)
@ -381,6 +382,11 @@ bin/zxp config _config-file_
## Enable Metrics
In the zot with all extensions case see [configuration example](./examples/config-metrics.json) for enabling metrics
## Image linting
# Mandatory Annotations
When pushing an image, if the mandatory annotations option is enabled, linter will verify if the mandatory annotations list present in the config is also found in the manifest's annotations list. If there are any missing annotations, the push will not take place.
## Clustering
zot supports clustering by using multiple stateless zot with shared s3 storage and a haproxy (with sticky session) in front of them.

View file

@ -53,4 +53,5 @@ var (
ErrRegistryNoContent = errors.New("sync: could not find a Content that matches localRepo")
ErrSyncSignatureNotFound = errors.New("sync: couldn't find any upstream notary/cosign signatures")
ErrSyncSignature = errors.New("sync: couldn't get upstream notary/cosign signatures")
ErrImageLintAnnotations = errors.New("routes: lint checks failed")
)

21
examples/config-lint.json Normal file
View file

@ -0,0 +1,21 @@
{
"distSpecVersion": "1.0.1-dev",
"storage": {
"rootDirectory": "/tmp/zot"
},
"http": {
"address": "127.0.0.1",
"port": "8080",
"ReadOnly": false
},
"log": {
"level": "debug"
},
"extensions": {
"lint": {
"enabled": true,
"mandatoryAnnotations": ["annot1", "annot2", "annot3"]
}
}
}

View file

@ -218,6 +218,8 @@ func (c *Controller) Run(reloadCtx context.Context) error {
func (c *Controller) InitImageStore(reloadCtx context.Context) error {
c.StoreController = storage.StoreController{}
linter := ext.GetLinter(c.Config, c.Log)
if c.Config.Storage.RootDirectory != "" {
// no need to validate hard links work on s3
if c.Config.Storage.Dedupe && c.Config.Storage.StorageDriver == nil {
@ -232,8 +234,12 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error {
var defaultStore storage.ImageStore
if c.Config.Storage.StorageDriver == nil {
// false positive lint - linter does not implement Lint method
// nolint: typecheck
defaultStore = storage.NewImageStore(c.Config.Storage.RootDirectory,
c.Config.Storage.GC, c.Config.Storage.GCDelay, c.Config.Storage.Dedupe, c.Config.Storage.Commit, c.Log, c.Metrics)
c.Config.Storage.GC, c.Config.Storage.GCDelay,
c.Config.Storage.Dedupe, c.Config.Storage.Commit, c.Log, c.Metrics, linter,
)
} else {
storeName := fmt.Sprintf("%v", c.Config.Storage.StorageDriver["name"])
if storeName != storage.S3StorageDriverName {
@ -255,9 +261,11 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error {
rootDir = fmt.Sprintf("%v", c.Config.Storage.StorageDriver["rootdirectory"])
}
// false positive lint - linter does not implement Lint method
// nolint: typecheck
defaultStore = s3.NewImageStore(rootDir, c.Config.Storage.RootDirectory,
c.Config.Storage.GC, c.Config.Storage.GCDelay, c.Config.Storage.Dedupe,
c.Config.Storage.Commit, c.Log, c.Metrics, store)
c.Config.Storage.Commit, c.Log, c.Metrics, linter, store)
}
c.StoreController.DefaultStore = defaultStore
@ -288,8 +296,10 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error {
}
if storageConfig.StorageDriver == nil {
// false positive lint - linter does not implement Lint method
// nolint: typecheck
subImageStore[route] = storage.NewImageStore(storageConfig.RootDirectory,
storageConfig.GC, storageConfig.GCDelay, storageConfig.Dedupe, storageConfig.Commit, c.Log, c.Metrics)
storageConfig.GC, storageConfig.GCDelay, storageConfig.Dedupe, storageConfig.Commit, c.Log, c.Metrics, linter)
} else {
storeName := fmt.Sprintf("%v", storageConfig.StorageDriver["name"])
if storeName != storage.S3StorageDriverName {
@ -311,8 +321,12 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error {
rootDir = fmt.Sprintf("%v", c.Config.Storage.StorageDriver["rootdirectory"])
}
// false positive lint - linter does not implement Lint method
// nolint: typecheck
subImageStore[route] = s3.NewImageStore(rootDir, storageConfig.RootDirectory,
storageConfig.GC, storageConfig.GCDelay, storageConfig.Dedupe, storageConfig.Commit, c.Log, c.Metrics, store)
storageConfig.GC, storageConfig.GCDelay,
storageConfig.Dedupe, storageConfig.Commit, c.Log, c.Metrics, linter, store,
)
}
}

View file

@ -470,6 +470,9 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht
} else if errors.Is(err, zerr.ErrRepoBadVersion) {
WriteJSON(response, http.StatusInternalServerError,
NewErrorList(NewError(INVALID_INDEX, map[string]string{"name": name})))
} else if errors.Is(err, zerr.ErrImageLintAnnotations) {
WriteJSON(response, http.StatusBadRequest,
NewErrorList(NewError(MANIFEST_INVALID, map[string]string{"reference": reference})))
} else {
// could be syscall.EMFILE (Err:0x18 too many opened files), etc
rh.c.Log.Error().Err(err).Msg("unexpected error: performing cleanup")

View file

@ -1,5 +1,5 @@
//go:build sync && scrub && metrics && search && ui_base
// +build sync,scrub,metrics,search,ui_base
//go:build sync && scrub && metrics && search && ui_base && lint
// +build sync,scrub,metrics,search,ui_base,lint
package api_test

View file

@ -102,7 +102,7 @@ func TestServeExtensions(t *testing.T) {
WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring, "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null") //nolint:lll // gofumpt conflicts with lll
So(string(data), ShouldContainSubstring, "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null") //nolint:lll // gofumpt conflicts with lll
})
}
@ -143,7 +143,7 @@ func testWithMetricsEnabled(cfgContentFormat string) {
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":true,\"Prometheus\":{\"Path\":\"/metrics\"}},\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":true,\"Prometheus\":{\"Path\":\"/metrics\"}},\"Scrub\":null,\"Lint\":null}") //nolint:lll // gofumpt conflicts with lll
}
func TestServeMetricsExtension(t *testing.T) {
@ -267,7 +267,7 @@ func TestServeMetricsExtension(t *testing.T) {
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":false,\"Prometheus\":{\"Path\":\"/metrics\"}},\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":false,\"Prometheus\":{\"Path\":\"/metrics\"}},\"Scrub\":null,\"Lint\":null}}") //nolint:lll // gofumpt conflicts with lll
})
}
@ -424,7 +424,7 @@ func TestServeScrubExtension(t *testing.T) {
So(err, ShouldBeNil)
// Even if in config we specified scrub interval=1h, the minimum interval is 2h
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":{\"Interval\":3600000000000}") //nolint:lll // gofumpt conflicts with lll
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":{\"Interval\":3600000000000},\"Lint\":null") //nolint:lll // gofumpt conflicts with lll
So(data, ShouldContainSubstring,
"Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.")
So(data, ShouldContainSubstring, "Starting periodic background tasks for")
@ -453,13 +453,72 @@ func TestServeScrubExtension(t *testing.T) {
data, err := runCLIWithConfig(t.TempDir(), content)
So(err, ShouldBeNil)
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null}")
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}")
So(data, ShouldContainSubstring, "Scrub config not provided, skipping scrub")
So(data, ShouldNotContainSubstring,
"Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.")
})
}
func TestServeLintExtension(t *testing.T) {
oldArgs := os.Args
defer func() { os.Args = oldArgs }()
Convey("lint enabled", t, func(c C) {
content := `{
"storage": {
"rootDirectory": "%s"
},
"http": {
"address": "127.0.0.1",
"port": "%s"
},
"log": {
"level": "debug",
"output": "%s"
},
"extensions": {
"lint": {
"enabled": "true",
"mandatoryAnnotations": ["annot1"]
}
}
}`
data, err := runCLIWithConfig(t.TempDir(), content)
So(err, ShouldBeNil)
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":{\"Enabled\":true,\"MandatoryAnnotations\":") //nolint:lll // gofumpt conflicts with lll
})
Convey("lint enabled", t, func(c C) {
content := `{
"storage": {
"rootDirectory": "%s"
},
"http": {
"address": "127.0.0.1",
"port": "%s"
},
"log": {
"level": "debug",
"output": "%s"
},
"extensions": {
"lint": {
"enabled": "false"
}
}
}`
data, err := runCLIWithConfig(t.TempDir(), content)
So(err, ShouldBeNil)
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":{\"Enabled\":false,\"MandatoryAnnotations\":null}") //nolint:lll // gofumpt conflicts with lll
})
}
func TestServeSearchEnabled(t *testing.T) {
oldArgs := os.Args
@ -490,7 +549,7 @@ func TestServeSearchEnabled(t *testing.T) {
WaitTillTrivyDBDownloadStarted(tempDir)
So(err, ShouldBeNil)
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}") //nolint:lll // gofumpt conflicts with lll
So(data, ShouldContainSubstring, "updating the CVE database")
})
}
@ -529,7 +588,7 @@ func TestServeSearchEnabledCVE(t *testing.T) {
So(err, ShouldBeNil)
// Even if in config we specified updateInterval=1h, the minimum interval is 2h
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":3600000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":3600000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}") //nolint:lll // gofumpt conflicts with lll
So(data, ShouldContainSubstring, "updating the CVE database")
So(data, ShouldContainSubstring,
"CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.")
@ -567,7 +626,7 @@ func TestServeSearchEnabledNoCVE(t *testing.T) {
WaitTillTrivyDBDownloadStarted(tempDir)
So(err, ShouldBeNil)
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}") //nolint:lll // gofumpt conflicts with lll
So(data, ShouldContainSubstring, "updating the CVE database")
})
}
@ -605,7 +664,7 @@ func TestServeSearchDisabled(t *testing.T) {
So(err, ShouldBeNil)
So(data, ShouldContainSubstring,
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":10800000000000},\"Enable\":false},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") //nolint:lll // gofumpt conflicts with lll
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":10800000000000},\"Enable\":false},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}") //nolint:lll // gofumpt conflicts with lll
So(data, ShouldContainSubstring, "CVE config not provided, skipping CVE update")
So(data, ShouldNotContainSubstring,
"CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.")

View file

@ -11,6 +11,12 @@ type ExtensionConfig struct {
Sync *sync.Config
Metrics *MetricsConfig
Scrub *ScrubConfig
Lint *LintConfig
}
type LintConfig struct {
Enabled *bool
MandatoryAnnotations []string
}
type SearchConfig struct {

View file

@ -0,0 +1,18 @@
//go:build lint
// +build lint
package extensions
import (
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/extensions/lint"
"zotregistry.io/zot/pkg/log"
)
func GetLinter(config *config.Config, log log.Logger) *lint.Linter {
if config.Extensions == nil {
return lint.NewLinter(nil, log)
}
return lint.NewLinter(config.Extensions.Lint, log)
}

View file

@ -0,0 +1,17 @@
//go:build !lint
// +build !lint
package extensions
import (
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/extensions/lint"
"zotregistry.io/zot/pkg/log"
)
func GetLinter(config *config.Config, log log.Logger) *lint.Linter {
log.Warn().Msg("lint extension is disabled because given zot binary doesn't" +
"include this feature please build a binary that does so")
return nil
}

View file

@ -0,0 +1,17 @@
//go:build !lint
// +build !lint
package lint
import (
godigest "github.com/opencontainers/go-digest"
"zotregistry.io/zot/pkg/storage"
)
type Linter struct{}
func (linter *Linter) Lint(repo string, manifestDigest godigest.Digest,
imageStore storage.ImageStore,
) (bool, error) {
return true, nil
}

View file

@ -0,0 +1,76 @@
//go:build lint
// +build lint
package lint
import (
"encoding/json"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
type Linter struct {
config *config.LintConfig
log log.Logger
}
func NewLinter(config *config.LintConfig, log log.Logger) *Linter {
return &Linter{
config: config,
log: log,
}
}
func (linter *Linter) CheckMandatoryAnnotations(repo string, manifestDigest godigest.Digest,
imgStore storage.ImageStore,
) (bool, error) {
if linter.config == nil {
return true, nil
}
if (linter.config != nil && !*linter.config.Enabled) || len(linter.config.MandatoryAnnotations) == 0 {
return true, nil
}
mandatoryAnnotationsList := linter.config.MandatoryAnnotations
content, err := imgStore.GetBlobContent(repo, string(manifestDigest))
if err != nil {
linter.log.Error().Err(err).Msg("linter: unable to get image manifest")
return false, err
}
var manifest ispec.Manifest
if err := json.Unmarshal(content, &manifest); err != nil {
linter.log.Error().Err(err).Msg("linter: couldn't unmarshal manifest JSON")
return false, err
}
annotations := manifest.Annotations
for _, annot := range mandatoryAnnotationsList {
_, found := annotations[annot]
if !found {
// if annotations are not found, return false but it's not an error
linter.log.Error().Msgf("linter: missing %s annotations", annot)
return false, nil
}
}
return true, nil
}
func (linter *Linter) Lint(repo string, manifestDigest godigest.Digest,
imageStore storage.ImageStore,
) (bool, error) {
return linter.CheckMandatoryAnnotations(repo, manifestDigest, imageStore)
}

View file

@ -0,0 +1,645 @@
//go:build lint
// +build lint
package lint_test
import (
"context"
"encoding/json"
"io/ioutil"
"net/http"
"os"
"path"
"testing"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"
"gopkg.in/resty.v1"
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/extensions/lint"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/test"
)
const (
username = "test"
passphrase = "test"
ServerCert = "../../test/data/server.cert"
ServerKey = "../../test/data/server.key"
CACert = "../../test/data/ca.crt"
AuthorizedNamespace = "everyone/isallowed"
UnauthorizedNamespace = "fortknox/notallowed"
ALICE = "alice"
AuthorizationNamespace = "authz/image"
AuthorizationAllRepos = "**"
)
func TestVerifyMandatoryAnnotations(t *testing.T) {
// nolint: dupl
Convey("Mandatory annotations disabled", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
enabled := false
conf.Extensions = &extconf.ExtensionConfig{Lint: &extconf.LintConfig{}}
conf.Extensions.Lint.MandatoryAnnotations = []string{}
conf.Extensions.Lint.Enabled = &enabled
ctlr := api.NewController(conf)
dir := t.TempDir()
err := test.CopyFiles("../../../test/data", dir)
if err != nil {
panic(err)
}
ctlr.Config.Storage.RootDirectory = dir
go startServer(ctlr)
defer stopServer(ctlr)
test.WaitTillServerReady(baseURL)
resp, err := resty.R().SetBasicAuth(username, passphrase).
Get(baseURL + "/v2/zot-test/manifests/0.0.1")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
manifestBlob := resp.Body()
var manifest ispec.Manifest
err = json.Unmarshal(manifestBlob, &manifest)
So(err, ShouldBeNil)
manifest.SchemaVersion = 2
content, err := json.Marshal(manifest)
So(err, ShouldBeNil)
resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(content).Put(baseURL + "/v2/zot-test/manifests/0.0.1")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
})
// nolint: dupl
Convey("Mandatory annotations enabled, but no list in config", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
enabled := true
conf.Extensions = &extconf.ExtensionConfig{Lint: &extconf.LintConfig{}}
conf.Extensions.Lint.MandatoryAnnotations = []string{}
conf.Extensions.Lint.Enabled = &enabled
ctlr := api.NewController(conf)
dir := t.TempDir()
err := test.CopyFiles("../../../test/data", dir)
if err != nil {
panic(err)
}
ctlr.Config.Storage.RootDirectory = dir
go startServer(ctlr)
defer stopServer(ctlr)
test.WaitTillServerReady(baseURL)
resp, err := resty.R().SetBasicAuth(username, passphrase).
Get(baseURL + "/v2/zot-test/manifests/0.0.1")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
manifestBlob := resp.Body()
var manifest ispec.Manifest
err = json.Unmarshal(manifestBlob, &manifest)
So(err, ShouldBeNil)
manifest.SchemaVersion = 2
content, err := json.Marshal(manifest)
So(err, ShouldBeNil)
resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(content).Put(baseURL + "/v2/zot-test/manifests/0.0.1")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
})
Convey("Mandatory annotations verification passing", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
enabled := true
conf.Extensions = &extconf.ExtensionConfig{Lint: &extconf.LintConfig{}}
conf.Extensions.Lint.MandatoryAnnotations = []string{}
conf.Extensions.Lint.Enabled = &enabled
conf.Extensions.Lint.MandatoryAnnotations = []string{"annotation1", "annotation2", "annotation3"}
ctlr := api.NewController(conf)
dir := t.TempDir()
err := test.CopyFiles("../../../test/data", dir)
if err != nil {
panic(err)
}
ctlr.Config.Storage.RootDirectory = dir
go startServer(ctlr)
defer stopServer(ctlr)
test.WaitTillServerReady(baseURL)
resp, err := resty.R().SetBasicAuth(username, passphrase).
Get(baseURL + "/v2/zot-test/manifests/0.0.1")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
manifestBlob := resp.Body()
var manifest ispec.Manifest
err = json.Unmarshal(manifestBlob, &manifest)
So(err, ShouldBeNil)
manifest.Annotations = make(map[string]string)
manifest.Annotations["annotation1"] = "testPass1"
manifest.Annotations["annotation2"] = "testPass2"
manifest.Annotations["annotation3"] = "testPass3"
manifest.SchemaVersion = 2
content, err := json.Marshal(manifest)
So(err, ShouldBeNil)
resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(content).Put(baseURL + "/v2/zot-test/manifests/0.0.1")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
})
Convey("Mandatory annotations incomplete in manifest", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
enabled := true
conf.Extensions = &extconf.ExtensionConfig{Lint: &extconf.LintConfig{}}
conf.Extensions.Lint.MandatoryAnnotations = []string{}
conf.Extensions.Lint.Enabled = &enabled
conf.Extensions.Lint.MandatoryAnnotations = []string{"annotation1", "annotation2", "annotation3"}
ctlr := api.NewController(conf)
dir := t.TempDir()
err := test.CopyFiles("../../../test/data", dir)
if err != nil {
panic(err)
}
ctlr.Config.Storage.RootDirectory = dir
go startServer(ctlr)
defer stopServer(ctlr)
test.WaitTillServerReady(baseURL)
resp, err := resty.R().SetBasicAuth(username, passphrase).
Get(baseURL + "/v2/zot-test/manifests/0.0.1")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
manifestBlob := resp.Body()
var manifest ispec.Manifest
err = json.Unmarshal(manifestBlob, &manifest)
So(err, ShouldBeNil)
manifest.Annotations = make(map[string]string)
manifest.Annotations["annotation1"] = "testFail1"
manifest.Annotations["annotation3"] = "testFail3"
manifest.SchemaVersion = 2
content, err := json.Marshal(manifest)
So(err, ShouldBeNil)
resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(content).Put(baseURL + "/v2/zot-test/manifests/0.0.1")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusBadRequest)
})
Convey("Mandatory annotations verification passing - more annotations than the mandatory list", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
enabled := true
conf.Extensions = &extconf.ExtensionConfig{Lint: &extconf.LintConfig{}}
conf.Extensions.Lint.MandatoryAnnotations = []string{}
conf.Extensions.Lint.Enabled = &enabled
conf.Extensions.Lint.MandatoryAnnotations = []string{"annotation1", "annotation2", "annotation3"}
ctlr := api.NewController(conf)
dir := t.TempDir()
err := test.CopyFiles("../../../test/data", dir)
if err != nil {
panic(err)
}
ctlr.Config.Storage.RootDirectory = dir
go startServer(ctlr)
defer stopServer(ctlr)
test.WaitTillServerReady(baseURL)
resp, err := resty.R().SetBasicAuth(username, passphrase).
Get(baseURL + "/v2/zot-test/manifests/0.0.1")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
manifestBlob := resp.Body()
var manifest ispec.Manifest
err = json.Unmarshal(manifestBlob, &manifest)
So(err, ShouldBeNil)
manifest.Annotations = make(map[string]string)
manifest.Annotations["annotation1"] = "testPassMore1"
manifest.Annotations["annotation2"] = "testPassMore2"
manifest.Annotations["annotation3"] = "testPassMore3"
manifest.Annotations["annotation4"] = "testPassMore4"
manifest.SchemaVersion = 2
content, err := json.Marshal(manifest)
So(err, ShouldBeNil)
resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetBody(content).Put(baseURL + "/v2/zot-test/manifests/0.0.1")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
})
}
func TestVerifyMandatoryAnnotationsFunction(t *testing.T) {
Convey("Mandatory annotations disabled", t, func() {
enabled := false
lintConfig := &extconf.LintConfig{
Enabled: &enabled,
MandatoryAnnotations: []string{},
}
dir := t.TempDir()
err := test.CopyFiles("../../../test/data", dir)
if err != nil {
panic(err)
}
var index ispec.Index
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := storage.NewImageStore(dir, false, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter)
indexContent, err := imgStore.GetIndexContent("zot-test")
So(err, ShouldBeNil)
err = json.Unmarshal(indexContent, &index)
So(err, ShouldBeNil)
manifestDigest := index.Manifests[0].Digest
pass, err := linter.CheckMandatoryAnnotations("zot-test", manifestDigest, imgStore)
So(err, ShouldBeNil)
So(pass, ShouldBeTrue)
})
Convey("Mandatory annotations enabled, but no list in config", t, func() {
enabled := true
lintConfig := &extconf.LintConfig{
Enabled: &enabled,
MandatoryAnnotations: []string{},
}
dir := t.TempDir()
err := test.CopyFiles("../../../test/data", dir)
if err != nil {
panic(err)
}
var index ispec.Index
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := storage.NewImageStore(dir, false, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter)
indexContent, err := imgStore.GetIndexContent("zot-test")
So(err, ShouldBeNil)
err = json.Unmarshal(indexContent, &index)
So(err, ShouldBeNil)
manifestDigest := index.Manifests[0].Digest
pass, err := linter.CheckMandatoryAnnotations("zot-test", manifestDigest, imgStore)
So(err, ShouldBeNil)
So(pass, ShouldBeTrue)
})
Convey("Mandatory annotations verification passing", t, func() {
enabled := true
lintConfig := &extconf.LintConfig{
Enabled: &enabled,
MandatoryAnnotations: []string{"annotation1", "annotation2", "annotation3"},
}
dir := t.TempDir()
err := test.CopyFiles("../../../test/data", dir)
if err != nil {
panic(err)
}
var index ispec.Index
buf, err := ioutil.ReadFile(path.Join(dir, "zot-test", "index.json"))
So(err, ShouldBeNil)
err = json.Unmarshal(buf, &index)
So(err, ShouldBeNil)
manifestDigest := index.Manifests[0].Digest
var manifest ispec.Manifest
buf, err = ioutil.ReadFile(path.Join(dir, "zot-test", "blobs",
manifestDigest.Algorithm().String(), manifestDigest.Encoded()))
So(err, ShouldBeNil)
err = json.Unmarshal(buf, &manifest)
So(err, ShouldBeNil)
manifest.Annotations = make(map[string]string)
manifest.Annotations["annotation1"] = "testPass1"
manifest.Annotations["annotation2"] = "testPass2"
manifest.Annotations["annotation3"] = "testPass3"
manifest.SchemaVersion = 2
content, err := json.Marshal(manifest)
So(err, ShouldBeNil)
So(content, ShouldNotBeNil)
digest := godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
err = ioutil.WriteFile(path.Join(dir, "zot-test", "blobs",
digest.Algorithm().String(), digest.Encoded()), content, 0o600)
So(err, ShouldBeNil)
manifestDesc := ispec.Descriptor{
Size: int64(len(content)),
Digest: digest,
}
index.Manifests = append(index.Manifests, manifestDesc)
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := storage.NewImageStore(dir, false, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter)
pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore)
So(err, ShouldBeNil)
So(pass, ShouldBeTrue)
})
Convey("Mandatory annotations incomplete in manifest", t, func() {
enabled := true
lintConfig := &extconf.LintConfig{
Enabled: &enabled,
MandatoryAnnotations: []string{"annotation1", "annotation2", "annotation3"},
}
dir := t.TempDir()
err := test.CopyFiles("../../../test/data", dir)
if err != nil {
panic(err)
}
var index ispec.Index
buf, err := ioutil.ReadFile(path.Join(dir, "zot-test", "index.json"))
So(err, ShouldBeNil)
err = json.Unmarshal(buf, &index)
So(err, ShouldBeNil)
manifestDigest := index.Manifests[0].Digest
var manifest ispec.Manifest
buf, err = ioutil.ReadFile(path.Join(dir, "zot-test", "blobs",
manifestDigest.Algorithm().String(), manifestDigest.Encoded()))
So(err, ShouldBeNil)
err = json.Unmarshal(buf, &manifest)
So(err, ShouldBeNil)
manifest.Annotations = make(map[string]string)
manifest.Annotations["annotation1"] = "test1"
manifest.Annotations["annotation3"] = "test3"
manifest.SchemaVersion = 2
content, err := json.Marshal(manifest)
So(err, ShouldBeNil)
So(content, ShouldNotBeNil)
digest := godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
err = ioutil.WriteFile(path.Join(dir, "zot-test", "blobs",
digest.Algorithm().String(), digest.Encoded()), content, 0o600)
So(err, ShouldBeNil)
manifestDesc := ispec.Descriptor{
Size: int64(len(content)),
Digest: digest,
}
index.Manifests = append(index.Manifests, manifestDesc)
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := storage.NewImageStore(dir, false, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter)
pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore)
So(err, ShouldBeNil)
So(pass, ShouldBeFalse)
})
Convey("Mandatory annotations verification passing - more annotations than the mandatory list", t, func() {
enabled := true
lintConfig := &extconf.LintConfig{
Enabled: &enabled,
MandatoryAnnotations: []string{"annotation1", "annotation2", "annotation3"},
}
dir := t.TempDir()
err := test.CopyFiles("../../../test/data", dir)
if err != nil {
panic(err)
}
var index ispec.Index
buf, err := ioutil.ReadFile(path.Join(dir, "zot-test", "index.json"))
So(err, ShouldBeNil)
err = json.Unmarshal(buf, &index)
So(err, ShouldBeNil)
manifestDigest := index.Manifests[0].Digest
var manifest ispec.Manifest
buf, err = ioutil.ReadFile(path.Join(dir, "zot-test", "blobs",
manifestDigest.Algorithm().String(), manifestDigest.Encoded()))
So(err, ShouldBeNil)
err = json.Unmarshal(buf, &manifest)
So(err, ShouldBeNil)
manifest.Annotations = make(map[string]string)
manifest.Annotations["annotation1"] = "testPassMore1"
manifest.Annotations["annotation2"] = "testPassMore2"
manifest.Annotations["annotation3"] = "testPassMore3"
manifest.Annotations["annotation4"] = "testPassMore4"
manifest.SchemaVersion = 2
content, err := json.Marshal(manifest)
So(err, ShouldBeNil)
So(content, ShouldNotBeNil)
digest := godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
err = ioutil.WriteFile(path.Join(dir, "zot-test", "blobs",
digest.Algorithm().String(), digest.Encoded()), content, 0o600)
So(err, ShouldBeNil)
manifestDesc := ispec.Descriptor{
Size: int64(len(content)),
Digest: digest,
}
index.Manifests = append(index.Manifests, manifestDesc)
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := storage.NewImageStore(dir, false, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter)
pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore)
So(err, ShouldBeNil)
So(pass, ShouldBeTrue)
})
Convey("Cannot unmarshal manifest", t, func() {
enabled := true
lintConfig := &extconf.LintConfig{
Enabled: &enabled,
MandatoryAnnotations: []string{"annotation1", "annotation2", "annotation3"},
}
dir := t.TempDir()
err := test.CopyFiles("../../../test/data", dir)
if err != nil {
panic(err)
}
var index ispec.Index
buf, err := ioutil.ReadFile(path.Join(dir, "zot-test", "index.json"))
So(err, ShouldBeNil)
err = json.Unmarshal(buf, &index)
So(err, ShouldBeNil)
manifestDigest := index.Manifests[0].Digest
var manifest ispec.Manifest
buf, err = ioutil.ReadFile(path.Join(dir, "zot-test", "blobs",
manifestDigest.Algorithm().String(), manifestDigest.Encoded()))
So(err, ShouldBeNil)
err = json.Unmarshal(buf, &manifest)
So(err, ShouldBeNil)
manifest.Annotations = make(map[string]string)
manifest.Annotations["annotation1"] = "testUnmarshal1"
manifest.Annotations["annotation2"] = "testUnmarshal2"
manifest.Annotations["annotation3"] = "testUnmarshal3"
manifest.SchemaVersion = 2
content, err := json.Marshal(manifest)
So(err, ShouldBeNil)
So(content, ShouldNotBeNil)
digest := godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
err = ioutil.WriteFile(path.Join(dir, "zot-test", "blobs",
digest.Algorithm().String(), digest.Encoded()), content, 0o600)
So(err, ShouldBeNil)
manifestDesc := ispec.Descriptor{
Size: int64(len(content)),
Digest: digest,
}
index.Manifests = append(index.Manifests, manifestDesc)
linter := lint.NewLinter(lintConfig, log.NewLogger("debug", ""))
imgStore := storage.NewImageStore(dir, false, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter)
err = os.Chmod(path.Join(dir, "zot-test", "blobs"), 0o000)
if err != nil {
panic(err)
}
pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore)
So(err, ShouldNotBeNil)
So(pass, ShouldBeFalse)
err = os.Chmod(path.Join(dir, "zot-test", "blobs"), 0o755)
if err != nil {
panic(err)
}
})
}
func startServer(c *api.Controller) {
// this blocks
ctx := context.Background()
if err := c.Run(ctx); err != nil {
return
}
}
func stopServer(c *api.Controller) {
ctx := context.Background()
_ = c.Server.Shutdown(ctx)
}

View file

@ -224,10 +224,15 @@ func TestRunScrubRepo(t *testing.T) {
defer os.Remove(logFile.Name()) // clean up
conf := config.New()
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Lint = &extconf.LintConfig{}
dir := t.TempDir()
log := log.NewLogger("debug", logFile.Name())
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true,
true, log, metrics, nil)
err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName))
if err != nil {
@ -247,10 +252,16 @@ func TestRunScrubRepo(t *testing.T) {
defer os.Remove(logFile.Name()) // clean up
conf := config.New()
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Lint = &extconf.LintConfig{}
dir := t.TempDir()
log := log.NewLogger("debug", logFile.Name())
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true,
true, log, metrics, nil)
err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName))
if err != nil {
@ -277,10 +288,15 @@ func TestRunScrubRepo(t *testing.T) {
defer os.Remove(logFile.Name()) // clean up
conf := config.New()
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Lint = &extconf.LintConfig{}
dir := t.TempDir()
log := log.NewLogger("debug", logFile.Name())
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, 1*time.Second,
true, true, log, metrics, nil)
err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName))
if err != nil {

View file

@ -264,8 +264,13 @@ func TestImageFormat(t *testing.T) {
log := log.NewLogger("debug", "")
dbDir := "../../../../test/data"
conf := config.New()
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Lint = &extconf.LintConfig{}
metrics := monitoring.NewMetricsServer(false, log)
defaultStore := storage.NewImageStore(dbDir, false, storage.DefaultGCDelay, false, false, log, metrics)
defaultStore := storage.NewImageStore(dbDir, false, storage.DefaultGCDelay,
false, false, log, metrics, nil)
storeController := storage.StoreController{DefaultStore: defaultStore}
olu := common.NewBaseOciLayoutUtils(storeController, log)
@ -708,10 +713,16 @@ func TestUtilsMethod(t *testing.T) {
subRootDir := t.TempDir()
metrics := monitoring.NewMetricsServer(false, log)
defaultStore := storage.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics)
conf := config.New()
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Lint = &extconf.LintConfig{}
subStore := storage.NewImageStore(subRootDir, false, storage.DefaultGCDelay, false, false, log, metrics)
metrics := monitoring.NewMetricsServer(false, log)
defaultStore := storage.NewImageStore(rootDir, false,
storage.DefaultGCDelay, false, false, log, metrics, nil)
subStore := storage.NewImageStore(subRootDir, false,
storage.DefaultGCDelay, false, false, log, metrics, nil)
subStoreMap := make(map[string]storage.ImageStore)

View file

@ -92,7 +92,11 @@ func testSetup() error {
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
storeController := storage.StoreController{DefaultStore: storage.NewImageStore(dir, false, storage.DefaultGCDelay, false, false, log, metrics)}
conf := config.New()
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Lint = &extconf.LintConfig{}
storeController := storage.StoreController{DefaultStore: storage.NewImageStore(dir, false, storage.DefaultGCDelay, false, false, log, metrics, nil)}
layoutUtils := common.NewBaseOciLayoutUtils(storeController, log)
@ -334,12 +338,16 @@ func TestMultipleStoragePath(t *testing.T) {
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
conf := config.New()
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Lint = &extconf.LintConfig{}
// Create ImageStore
firstStore := storage.NewImageStore(firstRootDir, false, storage.DefaultGCDelay, false, false, log, metrics)
firstStore := storage.NewImageStore(firstRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil)
secondStore := storage.NewImageStore(secondRootDir, false, storage.DefaultGCDelay, false, false, log, metrics)
secondStore := storage.NewImageStore(secondRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil)
thirdStore := storage.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay, false, false, log, metrics)
thirdStore := storage.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil)
storeController := storage.StoreController{}

View file

@ -97,10 +97,15 @@ func testSetup() error {
return err
}
conf := config.New()
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Lint = &extconf.LintConfig{}
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
storeController := storage.StoreController{
DefaultStore: storage.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics),
DefaultStore: storage.NewImageStore(rootDir, false, storage.DefaultGCDelay,
false, false, log, metrics, nil),
}
digestInfo = digestinfo.NewDigestInfo(storeController, log)

View file

@ -274,7 +274,8 @@ func syncOneImage(imageChannel chan error, cfg Config, storeController storage.S
imageChannel <- nil
}
func syncRun(regCfg RegistryConfig, localRepo, remoteRepo, tag string, utils syncContextUtils,
func syncRun(regCfg RegistryConfig,
localRepo, remoteRepo, tag string, utils syncContextUtils,
log log.Logger,
) (bool, error) {
upstreamImageRef, err := getImageRef(utils.upstreamAddr, remoteRepo, tag)

View file

@ -196,7 +196,8 @@ func syncCosignSignature(client *resty.Client, imageStore storage.ImageStore,
}
// push manifest
_, err = imageStore.PutImageManifest(localRepo, cosignTag, ispec.MediaTypeImageManifest, cosignManifestBuf)
_, err = imageStore.PutImageManifest(localRepo, cosignTag,
ispec.MediaTypeImageManifest, cosignManifestBuf)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload cosign manifest")

View file

@ -296,7 +296,8 @@ func getUpstreamContext(regCfg *RegistryConfig, credentials Credentials) *types.
}
// nolint:gocyclo // offloading some of the functionalities from here would make the code harder to follow
func syncRegistry(ctx context.Context, regCfg RegistryConfig, upstreamURL string,
func syncRegistry(ctx context.Context, regCfg RegistryConfig,
upstreamURL string,
storeController storage.StoreController, localCtx *types.SystemContext,
policyCtx *signature.PolicyContext, credentials Credentials,
retryOptions *retry.RetryOptions, log log.Logger,
@ -509,7 +510,6 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig, upstreamURL string
}
// push from cache to repo
err = pushSyncedLocalImage(localRepo, tag, localCachePath, imageStore, log)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("error while pushing synced cached image %s",
@ -573,7 +573,8 @@ func getLocalContexts(log log.Logger) (*types.SystemContext, *signature.PolicyCo
return localCtx, policyContext, nil
}
func Run(ctx context.Context, cfg Config, storeController storage.StoreController,
func Run(ctx context.Context, cfg Config,
storeController storage.StoreController,
wtgrp *goSync.WaitGroup, logger log.Logger,
) error {
var credentialsFile CredentialsFile

View file

@ -61,7 +61,9 @@ func TestInjectSyncUtils(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := storage.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay, false, false, log, metrics)
imageStore := storage.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
false, false, log, metrics, nil,
)
injected = test.InjectFailure(0)
_, err = getLocalCachePath(imageStore, testImage)
@ -147,7 +149,8 @@ func TestSyncInternal(t *testing.T) {
}
ctx := context.Background()
So(Run(ctx, cfg, storage.StoreController{}, new(goSync.WaitGroup), log.NewLogger("debug", "")), ShouldNotBeNil)
So(Run(ctx, cfg, storage.StoreController{},
new(goSync.WaitGroup), log.NewLogger("debug", "")), ShouldNotBeNil)
_, err = getFileCredentials("/invalid/path/to/file")
So(err, ShouldNotBeNil)
@ -157,7 +160,8 @@ func TestSyncInternal(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := storage.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay, false, false, log, metrics)
imageStore := storage.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
false, false, log, metrics, nil)
err := os.Chmod(imageStore.RootDir(), 0o000)
So(err, ShouldBeNil)
@ -340,7 +344,8 @@ func TestSyncInternal(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := storage.NewImageStore(storageDir, false, storage.DefaultGCDelay, false, false, log, metrics)
imageStore := storage.NewImageStore(storageDir, false, storage.DefaultGCDelay,
false, false, log, metrics, nil)
refs := ReferenceList{[]artifactspec.Descriptor{
{
@ -422,7 +427,8 @@ func TestSyncInternal(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := storage.NewImageStore(storageDir, false, storage.DefaultGCDelay, false, false, log, metrics)
imageStore := storage.NewImageStore(storageDir, false, storage.DefaultGCDelay,
false, false, log, metrics, nil)
storeController := storage.StoreController{}
storeController.DefaultStore = imageStore
@ -443,7 +449,8 @@ func TestSyncInternal(t *testing.T) {
panic(err)
}
testImageStore := storage.NewImageStore(testRootDir, false, storage.DefaultGCDelay, false, false, log, metrics)
testImageStore := storage.NewImageStore(testRootDir, false,
storage.DefaultGCDelay, false, false, log, metrics, nil)
manifestContent, _, _, err := testImageStore.GetImageManifest(testImage, testImageTag)
So(err, ShouldBeNil)

View file

@ -862,6 +862,107 @@ func TestConfigReloader(t *testing.T) {
})
}
func TestMandatoryAnnotations(t *testing.T) {
Convey("Verify mandatory annotations failing - on demand disabled", t, func() {
updateDuration, _ := time.ParseDuration("30m")
sctlr, srcBaseURL, _, _, _ := startUpstreamServer(t, false, false)
defer func() {
sctlr.Shutdown()
}()
regex := ".*"
var semver bool
tlsVerify := false
syncRegistryConfig := sync.RegistryConfig{
Content: []sync.Content{
{
Prefix: testImage,
Tags: &sync.Tags{
Regex: &regex,
Semver: &semver,
},
},
},
URLs: []string{srcBaseURL},
OnDemand: false,
PollInterval: updateDuration,
TLSVerify: &tlsVerify,
}
defaultVal := true
syncConfig := &sync.Config{
Enable: &defaultVal,
Registries: []sync.RegistryConfig{syncRegistryConfig},
}
destPort := test.GetFreePort()
destConfig := config.New()
destClient := resty.New()
destBaseURL := test.GetBaseURL(destPort)
destConfig.HTTP.Port = destPort
destDir := t.TempDir()
destConfig.Storage.RootDirectory = destDir
destConfig.Storage.Dedupe = false
destConfig.Storage.GC = false
destConfig.Extensions = &extconf.ExtensionConfig{}
destConfig.Extensions.Sync = syncConfig
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
destConfig.Log.Output = logFile.Name()
lintEnabled := true
destConfig.Extensions.Lint = &extconf.LintConfig{}
destConfig.Extensions.Lint.Enabled = &lintEnabled
destConfig.Extensions.Lint.MandatoryAnnotations = []string{"annot1", "annot2", "annot3"}
dctlr := api.NewController(destConfig)
go func() {
// this blocks
if err := dctlr.Run(context.Background()); err != nil {
return
}
}()
// wait till ready
for {
_, err := destClient.R().Get(destBaseURL)
if err == nil {
break
}
time.Sleep(100 * time.Millisecond)
}
defer func() {
dctlr.Shutdown()
}()
// give it time to set up sync
time.Sleep(3 * time.Second)
resp, err := destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifest/0.0.1")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
data, err := os.ReadFile(logFile.Name())
t.Logf("downstream log: %s", string(data))
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring, "couldn't upload manifest because of missing annotations")
})
}
func TestBadTLS(t *testing.T) {
Convey("Verify sync TLS feature", t, func() {
updateDuration, _ := time.ParseDuration("30m")

View file

@ -270,7 +270,9 @@ func pushSyncedLocalImage(localRepo, tag, localCachePath string,
log.Info().Msgf("pushing synced local image %s/%s:%s to local registry", localCachePath, localRepo, tag)
metrics := monitoring.NewMetricsServer(false, log)
cacheImageStore := storage.NewImageStore(localCachePath, false, storage.DefaultGCDelay, false, false, log, metrics)
cacheImageStore := storage.NewImageStore(localCachePath, false,
storage.DefaultGCDelay, false, false, log, metrics, nil)
manifestContent, _, _, err := cacheImageStore.GetImageManifest(localRepo, tag)
if err != nil {
@ -331,8 +333,16 @@ func pushSyncedLocalImage(localRepo, tag, localCachePath string,
}
}
_, err = imageStore.PutImageManifest(localRepo, tag, ispec.MediaTypeImageManifest, manifestContent)
_, err = imageStore.PutImageManifest(localRepo, tag,
ispec.MediaTypeImageManifest, manifestContent)
if err != nil {
if errors.Is(err, zerr.ErrImageLintAnnotations) {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload manifest because of missing annotations")
return nil
}
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload manifest")

View file

@ -0,0 +1,9 @@
package storage
import (
godigest "github.com/opencontainers/go-digest"
)
type Lint interface {
Lint(repo string, manifestDigest godigest.Digest, imageStore ImageStore) (bool, error)
}

View file

@ -25,6 +25,7 @@ import (
"github.com/opencontainers/umoci/oci/casext"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"github.com/rs/zerolog"
"github.com/sigstore/cosign/pkg/oci/remote"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/monitoring"
zlog "zotregistry.io/zot/pkg/log"
@ -64,6 +65,7 @@ type ImageStoreLocal struct {
gcDelay time.Duration
log zerolog.Logger
metrics monitoring.MetricServer
linter Lint
}
func (is *ImageStoreLocal) RootDir() string {
@ -105,7 +107,7 @@ func (sc StoreController) GetImageStore(name string) ImageStore {
// NewImageStore returns a new image store backed by a file storage.
func NewImageStore(rootDir string, gc bool, gcDelay time.Duration, dedupe, commit bool,
log zlog.Logger, metrics monitoring.MetricServer,
log zlog.Logger, metrics monitoring.MetricServer, linter Lint,
) ImageStore {
if _, err := os.Stat(rootDir); os.IsNotExist(err) {
if err := os.MkdirAll(rootDir, DefaultDirPerms); err != nil {
@ -125,6 +127,7 @@ func NewImageStore(rootDir string, gc bool, gcDelay time.Duration, dedupe, commi
commit: commit,
log: log.With().Caller().Logger(),
metrics: metrics,
linter: linter,
}
if dedupe {
@ -549,29 +552,9 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string,
return "", zerr.ErrBadManifest
}
if mediaType == ispec.MediaTypeImageManifest {
var manifest ispec.Manifest
if err := json.Unmarshal(body, &manifest); err != nil {
is.log.Error().Err(err).Msg("unable to unmarshal JSON")
return "", zerr.ErrBadManifest
}
if manifest.Config.MediaType == ispec.MediaTypeImageConfig {
digest, err := is.validateOCIManifest(repo, reference, &manifest)
if err != nil {
is.log.Error().Err(err).Msg("invalid oci image manifest")
return digest, err
}
}
} else if mediaType == artifactspec.MediaTypeArtifactManifest {
var m notation.Descriptor
if err := json.Unmarshal(body, &m); err != nil {
is.log.Error().Err(err).Msg("unable to unmarshal JSON")
return "", zerr.ErrBadManifest
}
dig, err := validateManifest(is, repo, reference, mediaType, body)
if err != nil {
return dig, err
}
mDigest := godigest.FromBytes(body)
@ -666,6 +649,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string,
_ = ensureDir(dir, is.log)
file := path.Join(dir, mDigest.Encoded())
// in case the linter will not pass, it will be garbage collected
if err := is.writeFile(file, body); err != nil {
is.log.Error().Err(err).Str("file", file).Msg("unable to write")
@ -684,6 +668,28 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string,
return "", err
}
// apply linter only on images, not signatures
if is.linter != nil {
if mediaType == ispec.MediaTypeImageManifest &&
// check that image manifest is not cosign signature
!strings.HasPrefix(reference, "sha256-") &&
!strings.HasSuffix(reference, remote.SignatureTagSuffix) {
// lint new index with new manifest before writing to disk
is.Unlock(&lockLatency)
pass, err := is.linter.Lint(repo, mDigest, is)
is.Lock(&lockLatency)
if err != nil {
is.log.Error().Err(err).Msg("linter error")
return "", err
}
if !pass {
return "", zerr.ErrImageLintAnnotations
}
}
}
err = is.writeFile(file, buf)
if err := test.Error(err); err != nil {
is.log.Error().Err(err).Str("file", file).Msg("unable to write")
@ -1616,6 +1622,37 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error {
return nil
}
func validateManifest(imgStore *ImageStoreLocal, repo, reference,
mediaType string, body []byte,
) (string, error) {
if mediaType == ispec.MediaTypeImageManifest {
var manifest ispec.Manifest
if err := json.Unmarshal(body, &manifest); err != nil {
imgStore.log.Error().Err(err).Msg("unable to unmarshal JSON")
return "", zerr.ErrBadManifest
}
if manifest.Config.MediaType == ispec.MediaTypeImageConfig {
digest, err := imgStore.validateOCIManifest(repo, reference, &manifest)
if err != nil {
imgStore.log.Error().Err(err).Msg("invalid oci image manifest")
return digest, err
}
}
} else if mediaType == artifactspec.MediaTypeArtifactManifest {
var m notation.Descriptor
if err := json.Unmarshal(body, &m); err != nil {
imgStore.log.Error().Err(err).Msg("unable to unmarshal JSON")
return "", zerr.ErrBadManifest
}
}
return "", nil
}
func ifOlderThan(imgStore *ImageStoreLocal, repo string, delay time.Duration) casext.GCPolicy {
return func(ctx context.Context, digest godigest.Digest) (bool, error) {
blobPath := imgStore.BlobPath(repo, digest)

View file

@ -27,7 +27,7 @@ func TestElevatedPrivilegesInvalidDedupe(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil)
upload, err := imgStore.NewBlobUpload("dedupe1")
So(err, ShouldBeNil)

View file

@ -36,7 +36,8 @@ func TestStorageFSAPIs(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true,
true, log, metrics, nil)
Convey("Repo layout", t, func(c C) {
repoName := "test"
@ -169,7 +170,7 @@ func TestGetReferrers(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil)
Convey("Get referrers", t, func(c C) {
err := test.CopyFiles("../../test/data/zot-test", path.Join(dir, "zot-test"))
@ -218,7 +219,8 @@ func TestDedupeLinks(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
Convey("Dedupe", t, func(c C) {
// manifest1
@ -272,7 +274,8 @@ func TestDedupeLinks(t *testing.T) {
manifestBuf, err := json.Marshal(manifest)
So(err, ShouldBeNil)
digest = godigest.FromBytes(manifestBuf)
_, err = imgStore.PutImageManifest("dedupe1", digest.String(), ispec.MediaTypeImageManifest, manifestBuf)
_, err = imgStore.PutImageManifest("dedupe1", digest.String(),
ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldBeNil)
_, _, _, err = imgStore.GetImageManifest("dedupe1", digest.String())
@ -356,7 +359,7 @@ func TestDedupe(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
il := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
il := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil)
So(il.DedupeBlob("", "", ""), ShouldNotBeNil)
})
@ -371,9 +374,11 @@ func TestNegativeCases(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
So(storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics), ShouldNotBeNil)
So(storage.NewImageStore(dir, true, storage.DefaultGCDelay, true,
true, log, metrics, nil), ShouldNotBeNil)
if os.Geteuid() != 0 {
So(storage.NewImageStore("/deadBEEF", true, storage.DefaultGCDelay, true, true, log, metrics), ShouldBeNil)
So(storage.NewImageStore("/deadBEEF", true, storage.DefaultGCDelay,
true, true, log, metrics, nil), ShouldBeNil)
}
})
@ -382,7 +387,8 @@ func TestNegativeCases(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
err := os.Chmod(dir, 0o000) // remove all perms
if err != nil {
@ -417,7 +423,8 @@ func TestNegativeCases(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true,
true, log, metrics, nil)
So(imgStore, ShouldNotBeNil)
So(imgStore.InitRepo("test"), ShouldBeNil)
@ -531,7 +538,8 @@ func TestNegativeCases(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
So(imgStore, ShouldNotBeNil)
So(imgStore.InitRepo("test"), ShouldBeNil)
@ -554,7 +562,8 @@ func TestNegativeCases(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true,
true, log, metrics, nil)
So(imgStore, ShouldNotBeNil)
So(imgStore.InitRepo("test"), ShouldBeNil)
@ -595,7 +604,8 @@ func TestNegativeCases(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
So(imgStore, ShouldNotBeNil)
So(imgStore.InitRepo("test"), ShouldBeNil)
@ -739,7 +749,8 @@ func TestInjectWriteFile(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
Convey("Failure path1", func() {
injected := test.InjectFailure(0)
@ -769,7 +780,8 @@ func TestInjectWriteFile(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, false, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, false, log, metrics, nil)
Convey("Failure path not reached", func() {
err := imgStore.InitRepo("repo1")
@ -786,7 +798,8 @@ func TestGarbageCollect(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
Convey("Garbage collect with default/long delay", func() {
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
repoName := "gc-long"
upload, err := imgStore.NewBlobUpload(repoName)
@ -853,7 +866,7 @@ func TestGarbageCollect(t *testing.T) {
})
Convey("Garbage collect with short delay", func() {
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil)
repoName := "gc-short"
// upload orphan blob
@ -949,7 +962,7 @@ func TestGarbageCollect(t *testing.T) {
Convey("Garbage collect with dedupe", func() {
// garbage-collect is repo-local and dedupe is global and they can interact in strange ways
imgStore := storage.NewImageStore(dir, true, 5*time.Second, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, 5*time.Second, true, true, log, metrics, nil)
// first upload an image to the first repo and wait for GC timeout
@ -1150,7 +1163,7 @@ func TestGarbageCollectForImageStore(t *testing.T) {
log := log.NewLogger("debug", logFile.Name())
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil)
repoName := "gc-all-repos-short"
err := test.CopyFiles("../../test/data/zot-test", path.Join(dir, repoName))
@ -1182,7 +1195,7 @@ func TestGarbageCollectForImageStore(t *testing.T) {
log := log.NewLogger("debug", logFile.Name())
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil)
repoName := "gc-all-repos-short"
err := test.CopyFiles("../../test/data/zot-test", path.Join(dir, repoName))
@ -1227,7 +1240,8 @@ func TestInitRepo(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
err := os.Mkdir(path.Join(dir, "test-dir"), 0o000)
So(err, ShouldBeNil)
@ -1243,7 +1257,8 @@ func TestValidateRepo(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
err := os.Mkdir(path.Join(dir, "test-dir"), 0o000)
So(err, ShouldBeNil)
@ -1259,7 +1274,9 @@ func TestGetRepositoriesError(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil,
)
// create valid directory with permissions
err := os.Mkdir(path.Join(dir, "test-dir"), 0o755)
@ -1279,7 +1296,8 @@ func TestPutBlobChunkStreamed(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
uuid, err := imgStore.NewBlobUpload("test")
So(err, ShouldBeNil)

View file

@ -11,6 +11,7 @@ import (
"os"
"path"
"path/filepath"
"strings"
"sync"
"time"
@ -23,6 +24,7 @@ import (
ispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"github.com/rs/zerolog"
"github.com/sigstore/cosign/pkg/oci/remote"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/monitoring"
zlog "zotregistry.io/zot/pkg/log"
@ -50,6 +52,7 @@ type ObjectStorage struct {
metrics monitoring.MetricServer
cache *storage.Cache
dedupe bool
linter storage.Lint
}
func (is *ObjectStorage) RootDir() string {
@ -67,7 +70,7 @@ func (is *ObjectStorage) DirExists(d string) bool {
// NewObjectStorage returns a new image store backed by cloud storages.
// see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers
func NewImageStore(rootDir string, cacheDir string, gc bool, gcDelay time.Duration, dedupe, commit bool,
log zlog.Logger, metrics monitoring.MetricServer,
log zlog.Logger, metrics monitoring.MetricServer, linter storage.Lint,
store driver.StorageDriver,
) storage.ImageStore {
imgStore := &ObjectStorage{
@ -79,6 +82,7 @@ func NewImageStore(rootDir string, cacheDir string, gc bool, gcDelay time.Durati
multiPartUploads: sync.Map{},
metrics: metrics,
dedupe: dedupe,
linter: linter,
}
cachePath := path.Join(cacheDir, CacheDBName+storage.DBExtensionName)
@ -395,8 +399,8 @@ func (is *ObjectStorage) GetImageManifest(repo, reference string) ([]byte, strin
// PutImageManifest adds an image manifest to the repository.
func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string,
body []byte,
) (string, error) {
body []byte) (string, error,
) {
if err := is.InitRepo(repo); err != nil {
is.log.Debug().Err(err).Msg("init repo")
@ -549,6 +553,26 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string,
return "", err
}
// apply linter only on images, not signatures
if is.linter != nil {
if mediaType == ispec.MediaTypeImageManifest &&
// check that image manifest is not cosign signature
!strings.HasPrefix(reference, "sha256-") &&
!strings.HasSuffix(reference, remote.SignatureTagSuffix) {
// lint new index with new manifest before writing to disk
pass, err := is.linter.Lint(repo, mDigest, is)
if err != nil {
is.log.Error().Err(err).Msg("linter error")
return "", err
}
if !pass {
return "", zerr.ErrImageLintAnnotations
}
}
}
if err = is.store.PutContent(context.Background(), indexPath, buf); err != nil {
is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write")

View file

@ -56,7 +56,9 @@ func skipIt(t *testing.T) {
func createMockStorage(rootDir string, cacheDir string, dedupe bool, store driver.StorageDriver) storage.ImageStore {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, dedupe, false, log, metrics, store)
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay,
dedupe, false, log, metrics, nil, store,
)
return il
}
@ -95,7 +97,8 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) (
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, dedupe, false, log, metrics, store)
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay,
dedupe, false, log, metrics, nil, store)
return store, il, err
}
@ -894,7 +897,8 @@ func TestS3Dedupe(t *testing.T) {
manifestBuf, err := json.Marshal(manifest)
So(err, ShouldBeNil)
digest = godigest.FromBytes(manifestBuf)
_, err = imgStore.PutImageManifest("dedupe1", digest.String(), ispec.MediaTypeImageManifest, manifestBuf)
_, err = imgStore.PutImageManifest("dedupe1", digest.String(),
ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldBeNil)
_, _, _, err = imgStore.GetImageManifest("dedupe1", digest.String())
@ -956,7 +960,8 @@ func TestS3Dedupe(t *testing.T) {
manifestBuf, err = json.Marshal(manifest)
So(err, ShouldBeNil)
digest = godigest.FromBytes(manifestBuf)
_, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest, manifestBuf)
_, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest,
manifestBuf)
So(err, ShouldBeNil)
_, _, _, err = imgStore.GetImageManifest("dedupe2", digest.String())
@ -1078,7 +1083,8 @@ func TestS3Dedupe(t *testing.T) {
manifestBuf, err = json.Marshal(manifest)
So(err, ShouldBeNil)
digest = godigest.FromBytes(manifestBuf)
_, err = imgStore.PutImageManifest("dedupe3", "1.0", ispec.MediaTypeImageManifest, manifestBuf)
_, err = imgStore.PutImageManifest("dedupe3", "1.0", ispec.MediaTypeImageManifest,
manifestBuf)
So(err, ShouldBeNil)
_, _, _, err = imgStore.GetImageManifest("dedupe3", digest.String())

View file

@ -31,7 +31,8 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore := storage.NewImageStore(dir, true, storage.DefaultGCDelay,
true, true, log, metrics, nil)
Convey("Scrub only one repo", t, func(c C) {
// initialize repo
@ -117,10 +118,11 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
}
mnfst.SchemaVersion = 2
mb, err := json.Marshal(mnfst)
mbytes, err := json.Marshal(mnfst)
So(err, ShouldBeNil)
manifest, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, mb)
manifest, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest,
mbytes)
So(err, ShouldBeNil)
Convey("Blobs integrity not affected", func() {

View file

@ -5,6 +5,7 @@ import (
"context"
_ "crypto/sha256"
"encoding/json"
"errors"
"fmt"
"os"
"path"
@ -28,6 +29,7 @@ import (
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/s3"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
)
func cleanupStorage(store driver.StorageDriver, name string) {
@ -73,7 +75,9 @@ func createObjectsStore(rootDir string, cacheDir string) (driver.StorageDriver,
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, true, false, log, metrics, store)
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay,
true, false, log, metrics, nil, store,
)
return store, il, err
}
@ -117,7 +121,8 @@ func TestStorageAPIs(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imgStore = storage.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics)
imgStore = storage.NewImageStore(dir, true, storage.DefaultGCDelay, true,
true, log, metrics, nil)
}
Convey("Repo layout", t, func(c C) {
@ -446,10 +451,12 @@ func TestStorageAPIs(t *testing.T) {
})
Convey("Bad image manifest", func() {
_, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, manifestBuf)
_, err = imgStore.PutImageManifest("test", digest.String(),
ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldNotBeNil)
_, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, []byte("bad json"))
_, err = imgStore.PutImageManifest("test", digest.String(),
ispec.MediaTypeImageManifest, []byte("bad json"))
So(err, ShouldNotBeNil)
_, _, _, err = imgStore.GetImageManifest("test", digest.String())
@ -483,11 +490,13 @@ func TestStorageAPIs(t *testing.T) {
manifestBuf, err = json.Marshal(manifest)
So(err, ShouldBeNil)
digest := godigest.FromBytes(manifestBuf)
_, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, manifestBuf)
_, err = imgStore.PutImageManifest("test", digest.String(),
ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldBeNil)
// same manifest for coverage
_, err = imgStore.PutImageManifest("test", digest.String(), ispec.MediaTypeImageManifest, manifestBuf)
_, err = imgStore.PutImageManifest("test", digest.String(),
ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldBeNil)
_, _, _, err = imgStore.GetImageManifest("test", digest.String())
@ -661,6 +670,117 @@ func TestStorageAPIs(t *testing.T) {
}
}
func TestMandatoryAnnotations(t *testing.T) {
for _, testcase := range testCases {
testcase := testcase
t.Run(testcase.testCaseName, func(t *testing.T) {
var imgStore storage.ImageStore
var testDir, tdir string
var store driver.StorageDriver
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
if testcase.storageType == "s3" {
skipIt(t)
uuid, err := guuid.NewV4()
if err != nil {
panic(err)
}
testDir = path.Join("/oci-repo-test", uuid.String())
tdir = t.TempDir()
store, _, _ = createObjectsStore(testDir, tdir)
imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics,
&mocks.MockedLint{
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
return false, nil
},
}, store)
defer cleanupStorage(store, testDir)
} else {
tdir = t.TempDir()
imgStore = storage.NewImageStore(tdir, true, storage.DefaultGCDelay, true,
true, log, metrics, &mocks.MockedLint{
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
return false, nil
},
})
}
Convey("Setup manifest", t, func() {
content := []byte("test-data1")
buf := bytes.NewBuffer(content)
buflen := buf.Len()
digest := godigest.FromBytes(content)
_, _, err := imgStore.FullBlobUpload("test", bytes.NewReader(buf.Bytes()), digest.String())
So(err, ShouldBeNil)
cblob, cdigest := test.GetRandomImageConfig()
_, clen, err := imgStore.FullBlobUpload("test", bytes.NewReader(cblob), cdigest.String())
So(err, ShouldBeNil)
So(clen, ShouldEqual, len(cblob))
annotationsMap := make(map[string]string)
annotationsMap[ispec.AnnotationRefName] = "1.0"
manifest := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: digest,
Size: int64(buflen),
},
},
Annotations: annotationsMap,
}
manifest.SchemaVersion = 2
manifestBuf, err := json.Marshal(manifest)
So(err, ShouldBeNil)
Convey("Missing mandatory annotations", func() {
_, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldNotBeNil)
})
Convey("Error on mandatory annotations", func() {
if testcase.storageType == "s3" {
imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics,
&mocks.MockedLint{
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
// nolint: goerr113
return false, errors.New("linter error")
},
}, store)
} else {
imgStore = storage.NewImageStore(tdir, true, storage.DefaultGCDelay, true,
true, log, metrics, &mocks.MockedLint{
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
// nolint: goerr113
return false, errors.New("linter error")
},
})
}
_, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldNotBeNil)
})
})
})
}
}
func TestStorageHandler(t *testing.T) {
for _, testcase := range testCases {
testcase := testcase
@ -700,11 +820,14 @@ func TestStorageHandler(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
// Create ImageStore
firstStore = storage.NewImageStore(firstRootDir, false, storage.DefaultGCDelay, false, false, log, metrics)
firstStore = storage.NewImageStore(firstRootDir, false, storage.DefaultGCDelay,
false, false, log, metrics, nil)
secondStore = storage.NewImageStore(secondRootDir, false, storage.DefaultGCDelay, false, false, log, metrics)
secondStore = storage.NewImageStore(secondRootDir, false,
storage.DefaultGCDelay, false, false, log, metrics, nil)
thirdStore = storage.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay, false, false, log, metrics)
thirdStore = storage.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay,
false, false, log, metrics, nil)
}
Convey("Test storage handler", t, func() {

View file

@ -0,0 +1,18 @@
package mocks
import (
godigest "github.com/opencontainers/go-digest"
"zotregistry.io/zot/pkg/storage"
)
type MockedLint struct {
LintFn func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error)
}
func (lint MockedLint) Lint(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
if lint.LintFn != nil {
return lint.LintFn(repo, manifestDigest, imageStore)
}
return false, nil
}