Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00
Code Issues Activity

refactor(storage): refactoring storage (#1459)

Browse source 
Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae 2023-05-26 21:08:19 +03:00 committed by GitHub
parent 9acd19f7ea
commit a3f355c278
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
45 changed files with 850 additions and 751 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
pkg/api/config/config.go
View file

@ -8,7 +8,7 @@ import (
distspec "github.com/opencontainers/distribution-spec/specs-go"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/storage"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
)
var (
@ -147,7 +147,7 @@ func New() *Config {
ReleaseTag: ReleaseTag,
BinaryType: BinaryType,
Storage: GlobalStorageConfig{
StorageConfig: StorageConfig{GC: true, GCDelay: storage.DefaultGCDelay, Dedupe: true},
StorageConfig: StorageConfig{GC: true, GCDelay: storageConstants.DefaultGCDelay, Dedupe: true},
},
HTTP: HTTPConfig{Address: "127.0.0.1", Port: "8080", Auth: &AuthConfig{FailDelay: 0}},
Log: &LogConfig{Level: "debug"},

235
pkg/api/controller.go
View file

@ -14,7 +14,6 @@ import (
"syscall"
"time"
"github.com/docker/distribution/registry/storage/driver/factory"
"github.com/gorilla/handlers"
"github.com/gorilla/mux"
@ -27,10 +26,6 @@ import (
"zotregistry.io/zot/pkg/meta/repodb/repodbfactory"
"zotregistry.io/zot/pkg/scheduler"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
"zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/storage/s3"
)
const (
@ -224,7 +219,7 @@ func (c *Controller) Init(reloadCtx context.Context) error {
c.Metrics = monitoring.NewMetricsServer(enabled, c.Log)
if err := c.InitImageStore(reloadCtx); err != nil {
if err := c.InitImageStore(); err != nil { //nolint:contextcheck
return err
}
@ -244,235 +239,15 @@ func (c *Controller) InitCVEInfo() {
}
}
func (c *Controller) InitImageStore(ctx context.Context) error {
c.StoreController = storage.StoreController{}
func (c *Controller) InitImageStore() error {
linter := ext.GetLinter(c.Config, c.Log)
if c.Config.Storage.RootDirectory != "" {
// no need to validate hard links work on s3
if c.Config.Storage.Dedupe && c.Config.Storage.StorageDriver == nil {
err := local.ValidateHardLink(c.Config.Storage.RootDirectory)
if err != nil {
c.Log.Warn().Msg("input storage root directory filesystem does not supports hardlinking," +
"disabling dedupe functionality")
c.Config.Storage.Dedupe = false
}
}
var defaultStore storage.ImageStore
if c.Config.Storage.StorageDriver == nil {
// false positive lint - linter does not implement Lint method
//nolint:typecheck,contextcheck
defaultStore = local.NewImageStore(c.Config.Storage.RootDirectory,
c.Config.Storage.GC, c.Config.Storage.GCDelay,
c.Config.Storage.Dedupe, c.Config.Storage.Commit, c.Log, c.Metrics, linter,
CreateCacheDatabaseDriver(c.Config.Storage.StorageConfig, c.Log),
)
} else {
storeName := fmt.Sprintf("%v", c.Config.Storage.StorageDriver["name"])
if storeName != storage.S3StorageDriverName {
c.Log.Fatal().Err(errors.ErrBadConfig).Str("storageDriver", storeName).
Msg("unsupported storage driver")
}
// Init a Storager from connection string.
store, err := factory.Create(storeName, c.Config.Storage.StorageDriver)
if err != nil {
c.Log.Error().Err(err).Str("rootDir", c.Config.Storage.RootDirectory).Msg("unable to create s3 service")
return err
}
/* in the case of s3 c.Config.Storage.RootDirectory is used for caching blobs locally and
c.Config.Storage.StorageDriver["rootdirectory"] is the actual rootDir in s3 */
rootDir := "/"
if c.Config.Storage.StorageDriver["rootdirectory"] != nil {
rootDir = fmt.Sprintf("%v", c.Config.Storage.StorageDriver["rootdirectory"])
}
// false positive lint - linter does not implement Lint method
//nolint: typecheck,contextcheck
defaultStore = s3.NewImageStore(rootDir, c.Config.Storage.RootDirectory,
c.Config.Storage.GC, c.Config.Storage.GCDelay, c.Config.Storage.Dedupe,
c.Config.Storage.Commit, c.Log, c.Metrics, linter, store,
CreateCacheDatabaseDriver(c.Config.Storage.StorageConfig, c.Log))
}
c.StoreController.DefaultStore = defaultStore
} else {
// we can't proceed without global storage
c.Log.Error().Err(errors.ErrImgStoreNotFound).Msg("controller: no storage config provided")
return errors.ErrImgStoreNotFound
}
if c.Config.Storage.SubPaths != nil {
if len(c.Config.Storage.SubPaths) > 0 {
subPaths := c.Config.Storage.SubPaths
//nolint: contextcheck
subImageStore, err := c.getSubStore(subPaths, linter)
if err != nil {
c.Log.Error().Err(err).Msg("controller: error getting sub image store")
return err
}
c.StoreController.SubStore = subImageStore
}
}
return nil
}
func (c *Controller) getSubStore(subPaths map[string]config.StorageConfig,
linter storage.Lint,
) (map[string]storage.ImageStore, error) {
imgStoreMap := make(map[string]storage.ImageStore, 0)
subImageStore := make(map[string]storage.ImageStore)
// creating image store per subpaths
for route, storageConfig := range subPaths {
// no need to validate hard links work on s3
if storageConfig.Dedupe && storageConfig.StorageDriver == nil {
err := local.ValidateHardLink(storageConfig.RootDirectory)
if err != nil {
c.Log.Warn().Msg("input storage root directory filesystem does not supports hardlinking, " +
"disabling dedupe functionality")
storageConfig.Dedupe = false
}
}
if storageConfig.StorageDriver == nil {
// Compare if subpath root dir is same as default root dir
isSame, _ := config.SameFile(c.Config.Storage.RootDirectory, storageConfig.RootDirectory)
if isSame {
c.Log.Error().Err(errors.ErrBadConfig).Msg("sub path storage directory is same as root directory")
return nil, errors.ErrBadConfig
}
isUnique := true
// Compare subpath unique files
for file := range imgStoreMap {
// We already have image storage for this file
if compareImageStore(file, storageConfig.RootDirectory) {
subImageStore[route] = imgStoreMap[file]
isUnique = true
}
}
// subpath root directory is unique
// add it to uniqueSubFiles
// Create a new image store and assign it to imgStoreMap
if isUnique {
imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(storageConfig.RootDirectory,
storageConfig.GC, storageConfig.GCDelay, storageConfig.Dedupe,
storageConfig.Commit, c.Log, c.Metrics, linter, CreateCacheDatabaseDriver(storageConfig, c.Log))
subImageStore[route] = imgStoreMap[storageConfig.RootDirectory]
}
} else {
storeName := fmt.Sprintf("%v", storageConfig.StorageDriver["name"])
if storeName != storage.S3StorageDriverName {
c.Log.Fatal().Err(errors.ErrBadConfig).Str("storageDriver", storeName).
Msg("unsupported storage driver")
}
// Init a Storager from connection string.
store, err := factory.Create(storeName, storageConfig.StorageDriver)
if err != nil {
c.Log.Error().Err(err).Str("rootDir", storageConfig.RootDirectory).Msg("Unable to create s3 service")
return nil, err
}
/* in the case of s3 c.Config.Storage.RootDirectory is used for caching blobs locally and
c.Config.Storage.StorageDriver["rootdirectory"] is the actual rootDir in s3 */
rootDir := "/"
if c.Config.Storage.StorageDriver["rootdirectory"] != nil {
rootDir = fmt.Sprintf("%v", c.Config.Storage.StorageDriver["rootdirectory"])
}
// false positive lint - linter does not implement Lint method
//nolint: typecheck
subImageStore[route] = s3.NewImageStore(rootDir, storageConfig.RootDirectory,
storageConfig.GC, storageConfig.GCDelay,
storageConfig.Dedupe, storageConfig.Commit, c.Log, c.Metrics, linter, store,
CreateCacheDatabaseDriver(storageConfig, c.Log),
)
}
}
return subImageStore, nil
}
func compareImageStore(root1, root2 string) bool {
isSameFile, err := config.SameFile(root1, root2)
// This error is path error that means either of root directory doesn't exist, in that case do string match
storeController, err := storage.New(c.Config, linter, c.Metrics, c.Log)
if err != nil {
return strings.EqualFold(root1, root2)
return err
}
return isSameFile
}
func getUseRelPaths(storageConfig *config.StorageConfig) bool {
return storageConfig.StorageDriver == nil
}
func CreateCacheDatabaseDriver(storageConfig config.StorageConfig, log log.Logger) cache.Cache {
if storageConfig.Dedupe || storageConfig.StorageDriver != nil {
if !storageConfig.RemoteCache {
params := cache.BoltDBDriverParameters{}
params.RootDir = storageConfig.RootDirectory
params.Name = constants.BoltdbName
if storageConfig.StorageDriver != nil {
params.Name = s3.CacheDBName
}
params.UseRelPaths = getUseRelPaths(&storageConfig)
driver, _ := storage.Create("boltdb", params, log)
return driver
}
// remote cache
if storageConfig.CacheDriver != nil {
name, ok := storageConfig.CacheDriver["name"].(string)
if !ok {
log.Warn().Msg("remote cache driver name missing!")
return nil
}
if name != constants.DynamoDBDriverName {
log.Warn().Str("driver", name).Msg("remote cache driver unsupported!")
return nil
}
// dynamodb
dynamoParams := cache.DynamoDBDriverParameters{}
dynamoParams.Endpoint, _ = storageConfig.CacheDriver["endpoint"].(string)
dynamoParams.Region, _ = storageConfig.CacheDriver["region"].(string)
dynamoParams.TableName, _ = storageConfig.CacheDriver["cachetablename"].(string)
driver, _ := storage.Create("dynamodb", dynamoParams, log)
return driver
}
return nil
}
c.StoreController = storeController
return nil
}

35
pkg/api/controller_test.go
View file

@ -51,8 +51,9 @@ import (
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb/repodbfactory"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/local"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/inject"
)
const (
@ -121,13 +122,13 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
panic(err)
}
driver := api.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
driver := storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
So(driver, ShouldBeNil)
conf.Storage.RemoteCache = true
conf.Storage.RootDirectory = t.TempDir()
driver = api.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
So(driver, ShouldBeNil)
})
skipDynamo(t)
@ -160,7 +161,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
"versionTablename": "Version",
}
driver := api.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
driver := storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
So(driver, ShouldNotBeNil)
// negative test cases
@ -175,7 +176,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
"versionTablename": "Version",
}
driver = api.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
So(driver, ShouldBeNil)
conf.Storage.CacheDriver = map[string]interface{}{
@ -189,7 +190,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
"versionTablename": "Version",
}
driver = api.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log)
So(driver, ShouldBeNil)
})
}
@ -360,7 +361,7 @@ func TestObjectStorageController(t *testing.T) {
conf.HTTP.Port = port
storageDriverParams := map[string]interface{}{
"rootdirectory": "zot",
"name": storage.S3StorageDriverName,
"name": storageConstants.S3StorageDriverName,
}
conf.Storage.StorageDriver = storageDriverParams
ctlr := makeController(conf, "zot", "")
@ -380,7 +381,7 @@ func TestObjectStorageController(t *testing.T) {
storageDriverParams := map[string]interface{}{
"rootdirectory": "zot",
"name": storage.S3StorageDriverName,
"name": storageConstants.S3StorageDriverName,
"region": "us-east-2",
"bucket": bucket,
"regionendpoint": endpoint,
@ -409,7 +410,7 @@ func TestObjectStorageControllerSubPaths(t *testing.T) {
storageDriverParams := map[string]interface{}{
"rootdirectory": "zot",
"name": storage.S3StorageDriverName,
"name": storageConstants.S3StorageDriverName,
"region": "us-east-2",
"bucket": bucket,
"regionendpoint": endpoint,
@ -5533,7 +5534,7 @@ func TestManifestImageIndex(t *testing.T) {
Convey("Corrupt index", func() {
err = os.WriteFile(path.Join(dir, "index", "blobs", index1dgst.Algorithm().String(), index1dgst.Encoded()),
[]byte("deadbeef"), local.DefaultFilePerms)
[]byte("deadbeef"), storageConstants.DefaultFilePerms)
So(err, ShouldBeNil)
resp, err = resty.R().Delete(baseURL + fmt.Sprintf("/v2/index/manifests/%s", index1dgst))
So(err, ShouldBeNil)
@ -5906,7 +5907,7 @@ func TestInjectInterruptedImageManifest(t *testing.T) {
// Testing router path: @Router /v2/{name}/manifests/{reference} [put]
Convey("Uploading an image manifest blob (when injected simulates an interrupted image manifest upload)", func() {
injected := test.InjectFailure(0)
injected := inject.InjectFailure(0)
request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content))
request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"})
@ -5967,7 +5968,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) {
So(digest, ShouldNotBeNil)
// monolithic blob upload
injected := test.InjectFailure(0)
injected := inject.InjectFailure(0)
if injected {
request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, loc, bytes.NewReader(content))
tokens := strings.Split(loc, "/")
@ -6040,7 +6041,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) {
// Testing router path: @Router /v2/{name}/manifests/{reference} [put]
//nolint:lll // gofumpt conflicts with lll
Convey("Uploading an image manifest blob (when injected simulates that PutImageManifest failed due to 'too many open files' error)", func() {
injected := test.InjectFailure(1)
injected := inject.InjectFailure(1)
request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content))
request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"})
@ -6060,7 +6061,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) {
}
})
Convey("when injected simulates a `too many open files` error inside PutImageManifest method of img store", func() {
injected := test.InjectFailure(2)
injected := inject.InjectFailure(2)
request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content))
request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"})
@ -6081,7 +6082,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) {
}
})
Convey("code coverage: error inside PutImageManifest method of img store (unable to marshal JSON)", func() {
injected := test.InjectFailure(1)
injected := inject.InjectFailure(1)
request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content))
request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"})
@ -6102,7 +6103,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) {
}
})
Convey("code coverage: error inside PutImageManifest method of img store (umoci.OpenLayout error)", func() {
injected := test.InjectFailure(3)
injected := inject.InjectFailure(3)
request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content))
request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"})
@ -6123,7 +6124,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) {
}
})
Convey("code coverage: error inside PutImageManifest method of img store (oci.GC)", func() {
injected := test.InjectFailure(4)
injected := inject.InjectFailure(4)
request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content))
request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"})

17
pkg/api/routes.go
View file

@ -39,8 +39,9 @@ import (
"zotregistry.io/zot/pkg/meta"
zreg "zotregistry.io/zot/pkg/regexp"
localCtx "zotregistry.io/zot/pkg/requestcontext"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/test" //nolint:goimports
storageCommon "zotregistry.io/zot/pkg/storage/common"
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test/inject"
)
type RouteHandler struct {
@ -488,7 +489,7 @@ type ImageIndex struct {
}
func getReferrers(ctx context.Context, routeHandler *RouteHandler,
imgStore storage.ImageStore, name string, digest godigest.Digest,
imgStore storageTypes.ImageStore, name string, digest godigest.Digest,
artifactTypes []string,
) (ispec.Index, error) {
references, err := imgStore.GetReferrers(name, digest, artifactTypes)
@ -621,7 +622,7 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht
}
mediaType := request.Header.Get("Content-Type")
if !storage.IsSupportedMediaType(mediaType) {
if !storageCommon.IsSupportedMediaType(mediaType) {
// response.WriteHeader(http.StatusUnsupportedMediaType)
WriteJSON(response, http.StatusUnsupportedMediaType,
NewErrorList(NewError(MANIFEST_INVALID, map[string]string{"mediaType": mediaType})))
@ -632,7 +633,7 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht
body, err := io.ReadAll(request.Body)
// hard to reach test case, injected error (simulates an interrupted image manifest upload)
// err could be io.ErrUnexpectedEOF
if err := test.Error(err); err != nil {
if err := inject.Error(err); err != nil {
rh.c.Log.Error().Err(err).Msg("unexpected error")
response.WriteHeader(http.StatusInternalServerError)
@ -1716,12 +1717,12 @@ func WriteDataFromReader(response http.ResponseWriter, status int, length int64,
}
// will return image storage corresponding to subpath provided in config.
func (rh *RouteHandler) getImageStore(name string) storage.ImageStore {
func (rh *RouteHandler) getImageStore(name string) storageTypes.ImageStore {
return rh.c.StoreController.GetImageStore(name)
}
// will sync on demand if an image is not found, in case sync extensions is enabled.
func getImageManifest(ctx context.Context, routeHandler *RouteHandler, imgStore storage.ImageStore,
func getImageManifest(ctx context.Context, routeHandler *RouteHandler, imgStore storageTypes.ImageStore,
name, reference string,
) ([]byte, godigest.Digest, string, error) {
syncEnabled := false
@ -1757,7 +1758,7 @@ func getImageManifest(ctx context.Context, routeHandler *RouteHandler, imgStore
// will sync referrers on demand if they are not found, in case sync extensions is enabled.
func getOrasReferrers(ctx context.Context, routeHandler *RouteHandler,
imgStore storage.ImageStore, name string, digest godigest.Digest,
imgStore storageTypes.ImageStore, name string, digest godigest.Digest,
artifactType string,
) ([]artifactspec.Descriptor, error) {
refs, err := imgStore.GetOrasReferrers(name, digest, artifactType)

8
pkg/api/routes_test.go
View file

@ -22,7 +22,7 @@ import (
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/api/constants"
localCtx "zotregistry.io/zot/pkg/requestcontext"
"zotregistry.io/zot/pkg/storage"
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
)
@ -1200,7 +1200,7 @@ func TestRoutes(t *testing.T) {
ism *mocks.MockedImageStore,
) int {
ctlr.StoreController.DefaultStore = ism
ctlr.StoreController.SubStore = map[string]storage.ImageStore{
ctlr.StoreController.SubStore = map[string]storageTypes.ImageStore{
"test": &mocks.MockedImageStore{
GetRepositoriesFn: func() ([]string, error) {
return []string{}, ErrUnexpectedError
@ -1238,7 +1238,7 @@ func TestRoutes(t *testing.T) {
ism *mocks.MockedImageStore,
) int {
ctlr.StoreController.DefaultStore = ism
ctlr.StoreController.SubStore = map[string]storage.ImageStore{}
ctlr.StoreController.SubStore = map[string]storageTypes.ImageStore{}
request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPatch, baseURL, nil)
request = mux.SetURLVars(request, vars)
@ -1300,7 +1300,7 @@ func TestRoutes(t *testing.T) {
return []string{"repo"}, nil
},
}
ctlr.StoreController.SubStore = map[string]storage.ImageStore{
ctlr.StoreController.SubStore = map[string]storageTypes.ImageStore{
"test1": &mocks.MockedImageStore{
GetRepositoriesFn: func() ([]string, error) {
return []string{"repo1"}, nil

9
pkg/cli/root.go
View file

@ -24,7 +24,6 @@ import (
"zotregistry.io/zot/pkg/api/constants"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/storage"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/s3"
)
@ -116,7 +115,7 @@ func newScrubCmd(conf *config.Config) *cobra.Command {
ctlr := api.NewController(conf)
ctlr.Metrics = monitoring.NewMetricsServer(false, ctlr.Log)
if err := ctlr.InitImageStore(context.Background()); err != nil {
if err := ctlr.InitImageStore(); err != nil {
panic(err)
}
@ -388,7 +387,7 @@ func validateConfiguration(config *config.Config) error {
if len(config.Storage.StorageDriver) != 0 {
// enforce s3 driver in case of using storage driver
if config.Storage.StorageDriver["name"] != storage.S3StorageDriverName {
if config.Storage.StorageDriver["name"] != storageConstants.S3StorageDriverName {
log.Error().Err(errors.ErrBadConfig).Interface("cacheDriver", config.Storage.StorageDriver["name"]).
Msg("unsupported storage driver")
@ -410,7 +409,7 @@ func validateConfiguration(config *config.Config) error {
for route, storageConfig := range subPaths {
if len(storageConfig.StorageDriver) != 0 {
if storageConfig.StorageDriver["name"] != storage.S3StorageDriverName {
if storageConfig.StorageDriver["name"] != storageConstants.S3StorageDriverName {
log.Error().Err(errors.ErrBadConfig).Str("subpath", route).Interface("storageDriver",
storageConfig.StorageDriver["name"]).Msg("unsupported storage driver")
@ -583,7 +582,7 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper) {
// if gc is enabled and gcDelay is not set, it is set to default value
if storageConfig.GC && !viperInstance.IsSet("storage::subpaths::"+name+"::gcdelay") {
storageConfig.GCDelay = storage.DefaultGCDelay
storageConfig.GCDelay = storageConstants.DefaultGCDelay
config.Storage.SubPaths[name] = storageConfig
}
}

5
pkg/cli/root_test.go
View file

@ -13,7 +13,6 @@ import (
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/cli"
"zotregistry.io/zot/pkg/storage"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/s3"
. "zotregistry.io/zot/pkg/test"
@ -1227,10 +1226,10 @@ func TestGC(t *testing.T) {
config := config.New()
err := cli.LoadConfiguration(config, "../../examples/config-multiple.json")
So(err, ShouldBeNil)
So(config.Storage.GCDelay, ShouldEqual, storage.DefaultGCDelay)
So(config.Storage.GCDelay, ShouldEqual, storageConstants.DefaultGCDelay)
err = cli.LoadConfiguration(config, "../../examples/config-gc.json")
So(err, ShouldBeNil)
So(config.Storage.GCDelay, ShouldNotEqual, storage.DefaultGCDelay)
So(config.Storage.GCDelay, ShouldNotEqual, storageConstants.DefaultGCDelay)
err = cli.LoadConfiguration(config, "../../examples/config-gc-periodic.json")
So(err, ShouldBeNil)
})

3
pkg/extensions/extension_scrub.go
View file

@ -13,6 +13,7 @@ import (
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/scheduler"
"zotregistry.io/zot/pkg/storage"
storageTypes "zotregistry.io/zot/pkg/storage/types"
)
// EnableScrubExtension enables scrub extension.
@ -50,7 +51,7 @@ func EnableScrubExtension(config *config.Config, log log.Logger, storeController
}
type taskGenerator struct {
imgStore storage.ImageStore
imgStore storageTypes.ImageStore
log log.Logger
lastRepo string
done bool

4
pkg/extensions/lint/lint-disabled.go
View file

@ -6,13 +6,13 @@ package lint
import (
godigest "github.com/opencontainers/go-digest"
"zotregistry.io/zot/pkg/storage"
storageTypes "zotregistry.io/zot/pkg/storage/types"
)
type Linter struct{}
func (linter *Linter) Lint(repo string, manifestDigest godigest.Digest,
imageStore storage.ImageStore,
imageStore storageTypes.ImageStore,
) (bool, error) {
return true, nil
}

6
pkg/extensions/lint/lint.go
View file

@ -13,7 +13,7 @@ import (
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
storageTypes "zotregistry.io/zot/pkg/storage/types"
)
type Linter struct {
@ -29,7 +29,7 @@ func NewLinter(config *config.LintConfig, log log.Logger) *Linter {
}
func (linter *Linter) CheckMandatoryAnnotations(repo string, manifestDigest godigest.Digest,
imgStore storage.ImageStore,
imgStore storageTypes.ImageStore,
) (bool, error) {
if linter.config == nil {
return true, nil
@ -112,7 +112,7 @@ func (linter *Linter) CheckMandatoryAnnotations(repo string, manifestDigest godi
}
func (linter *Linter) Lint(repo string, manifestDigest godigest.Digest,
imageStore storage.ImageStore,
imageStore storageTypes.ImageStore,
) (bool, error) {
return linter.CheckMandatoryAnnotations(repo, manifestDigest, imageStore)
}

7
pkg/extensions/scrub/scrub.go
View file

@ -9,10 +9,11 @@ import (
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
storageTypes "zotregistry.io/zot/pkg/storage/types"
)
// Scrub Extension for repo...
func RunScrubRepo(imgStore storage.ImageStore, repo string, log log.Logger) error {
func RunScrubRepo(imgStore storageTypes.ImageStore, repo string, log log.Logger) error {
execMsg := fmt.Sprintf("executing scrub to check manifest/blob integrity for %s", path.Join(imgStore.RootDir(), repo))
log.Info().Msg(execMsg)
@ -48,12 +49,12 @@ func RunScrubRepo(imgStore storage.ImageStore, repo string, log log.Logger) erro
}
type Task struct {
imgStore storage.ImageStore
imgStore storageTypes.ImageStore
repo string
log log.Logger
}
func NewTask(imgStore storage.ImageStore, repo string, log log.Logger) *Task {
func NewTask(imgStore storageTypes.ImageStore, repo string, log log.Logger) *Task {
return &Task{imgStore, repo, log}
}

3
pkg/extensions/search/cve/cve_test.go
View file

@ -33,6 +33,7 @@ import (
"zotregistry.io/zot/pkg/meta/repodb"
boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper"
"zotregistry.io/zot/pkg/storage"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
. "zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
@ -310,7 +311,7 @@ func TestImageFormat(t *testing.T) {
dbDir := t.TempDir()
metrics := monitoring.NewMetricsServer(false, log)
defaultStore := local.NewImageStore(imgDir, false, storage.DefaultGCDelay,
defaultStore := local.NewImageStore(imgDir, false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil)
storeController := storage.StoreController{DefaultStore: defaultStore}

20
pkg/extensions/search/cve/trivy/scanner_internal_test.go
View file

@ -22,7 +22,9 @@ import (
"zotregistry.io/zot/pkg/meta/repodb"
boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper"
"zotregistry.io/zot/pkg/storage"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test"
)
@ -67,17 +69,21 @@ func TestMultipleStoragePath(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
// Create ImageStore
firstStore := local.NewImageStore(firstRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
secondStore := local.NewImageStore(secondRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
firstStore := local.NewImageStore(firstRootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics,
nil, nil)
thirdStore := local.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
secondStore := local.NewImageStore(secondRootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics,
nil, nil)
thirdStore := local.NewImageStore(thirdRootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics,
nil, nil)
storeController := storage.StoreController{}
storeController.DefaultStore = firstStore
subStore := make(map[string]storage.ImageStore)
subStore := make(map[string]storageTypes.ImageStore)
subStore["/a"] = secondStore
subStore["/b"] = thirdStore
@ -173,7 +179,7 @@ func TestTrivyLibraryErrors(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
// Create ImageStore
store := local.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
store := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil)
storeController := storage.StoreController{}
storeController.DefaultStore = store
@ -370,7 +376,7 @@ func TestImageScannable(t *testing.T) {
// Continue with initializing the objects the scanner depends on
metrics := monitoring.NewMetricsServer(false, log)
store := local.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
store := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil)
storeController := storage.StoreController{}
storeController.DefaultStore = store
@ -432,7 +438,7 @@ func TestDefaultTrivyDBUrl(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
// Create ImageStore
store := local.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
store := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil)
storeController := storage.StoreController{}
storeController.DefaultStore = store

12
pkg/extensions/search/search_test.go
View file

@ -40,7 +40,9 @@ import (
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
storageTypes "zotregistry.io/zot/pkg/storage/types"
. "zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
ocilayout "zotregistry.io/zot/pkg/test/oci-layout"
@ -1218,7 +1220,7 @@ func TestExpandedRepoInfo(t *testing.T) {
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
testStorage := local.NewImageStore(rootDir, false, storage.DefaultGCDelay,
testStorage := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil)
resp, err := resty.R().Get(baseURL + "/v2/")
@ -2821,7 +2823,7 @@ func TestGetRepositories(t *testing.T) {
storeController := storage.StoreController{
DefaultStore: mockImageStore,
SubStore: map[string]storage.ImageStore{"test": mockImageStore},
SubStore: map[string]storageTypes.ImageStore{"test": mockImageStore},
}
olu := ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", ""))
@ -2831,7 +2833,7 @@ func TestGetRepositories(t *testing.T) {
storeController = storage.StoreController{
DefaultStore: mocks.MockedImageStore{},
SubStore: map[string]storage.ImageStore{"test": mockImageStore},
SubStore: map[string]storageTypes.ImageStore{"test": mockImageStore},
}
olu = ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", ""))
@ -5446,7 +5448,7 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
// get signatur digest
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
storage := local.NewImageStore(dir, false, storage.DefaultGCDelay,
storage := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil)
indexBlob, err := storage.GetIndexContent(repo)
@ -5523,7 +5525,7 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
// get signatur digest
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
storage := local.NewImageStore(dir, false, storage.DefaultGCDelay,
storage := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil)
indexBlob, err := storage.GetIndexContent(repo)

3
pkg/extensions/sync/signatures.go
View file

@ -24,6 +24,7 @@ import (
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/meta/signatures"
"zotregistry.io/zot/pkg/storage"
storageTypes "zotregistry.io/zot/pkg/storage/types"
)
type signaturesCopier struct {
@ -520,7 +521,7 @@ func (sig *signaturesCopier) canSkipOCIRefs(localRepo, digestStr string, index i
return true, nil
}
func syncBlob(sig *signaturesCopier, imageStore storage.ImageStore, localRepo, remoteRepo string,
func syncBlob(sig *signaturesCopier, imageStore storageTypes.ImageStore, localRepo, remoteRepo string,
digest godigest.Digest,
) error {
getBlobURL := sig.upstreamURL

4
pkg/extensions/sync/sync.go
View file

@ -24,7 +24,7 @@ import (
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/inject"
)
const (
@ -315,7 +315,7 @@ func getLocalContexts(log log.Logger) (*types.SystemContext, *signature.PolicyCo
policy = &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}}
policyContext, err := signature.NewPolicyContext(policy)
if err := test.Error(err); err != nil {
if err := inject.Error(err); err != nil {
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't create policy context")

33
pkg/extensions/sync/sync_internal_test.go
View file

@ -27,8 +27,11 @@ import (
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/inject"
"zotregistry.io/zot/pkg/test/mocks"
)
@ -51,13 +54,13 @@ func TestInjectSyncUtils(t *testing.T) {
taggedRef, err := reference.WithTag(ref, "tag")
So(err, ShouldBeNil)
injected := test.InjectFailure(0)
injected := inject.InjectFailure(0)
if injected {
_, err = getImageTags(context.Background(), &types.SystemContext{}, taggedRef)
So(err, ShouldNotBeNil)
}
injected = test.InjectFailure(0)
injected = inject.InjectFailure(0)
_, _, err = getLocalContexts(log.NewLogger("debug", ""))
if injected {
So(err, ShouldNotBeNil)
@ -67,10 +70,10 @@ func TestInjectSyncUtils(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
imageStore := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil,
)
injected = test.InjectFailure(0)
injected = inject.InjectFailure(0)
_, err = getLocalCachePath(imageStore, testImage)
if injected {
@ -183,7 +186,7 @@ func TestSyncInternal(t *testing.T) {
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
imageStore := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil,
)
@ -200,7 +203,7 @@ func TestSyncInternal(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
imageStore := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil)
err := os.Chmod(imageStore.RootDir(), 0o000)
@ -375,7 +378,7 @@ func TestSyncInternal(t *testing.T) {
}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
imageStore := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil,
)
mockRepoDB := mocks.RepoDBMock{}
@ -401,7 +404,7 @@ func TestSyncInternal(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(storageDir, false, storage.DefaultGCDelay,
imageStore := local.NewImageStore(storageDir, false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil)
refs := ispec.Index{Manifests: []ispec.Descriptor{
@ -642,7 +645,7 @@ func TestSyncInternal(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(storageDir, false, storage.DefaultGCDelay,
imageStore := local.NewImageStore(storageDir, false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil)
storeController := storage.StoreController{}
@ -662,7 +665,7 @@ func TestSyncInternal(t *testing.T) {
test.CopyTestFiles("../../../test/data", testRootDir)
testImageStore := local.NewImageStore(testRootDir, false,
storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil)
manifestContent, _, _, err := testImageStore.GetImageManifest(testImage, testImageTag)
So(err, ShouldBeNil)
@ -729,14 +732,14 @@ func TestSyncInternal(t *testing.T) {
err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStore, log)
So(err, ShouldNotBeNil)
err = os.Chmod(path.Join(testRootDir, repo, "blobs",
index.Manifests[0].Digest.Algorithm().String(), index.Manifests[0].Digest.Encoded()), local.DefaultDirPerms)
err = os.Chmod(path.Join(testRootDir, repo, "blobs", index.Manifests[0].Digest.Algorithm().String(),
index.Manifests[0].Digest.Encoded()), storageConstants.DefaultDirPerms)
So(err, ShouldBeNil)
// trigger linter error on manifest push
imageStoreWithLinter := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
imageStoreWithLinter := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay,
false, false, log, metrics, &mocks.MockedLint{
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) {
return false, nil
},
}, nil,
@ -769,7 +772,7 @@ func TestSyncInternal(t *testing.T) {
err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStore, log)
So(err, ShouldNotBeNil)
err = os.Chmod(configBlobPath, local.DefaultDirPerms)
err = os.Chmod(configBlobPath, storageConstants.DefaultDirPerms)
So(err, ShouldBeNil)
err = os.RemoveAll(path.Join(imageStore.RootDir(), repo, "index.json"))

9
pkg/extensions/sync/sync_test.go
View file

@ -42,8 +42,7 @@ import (
"zotregistry.io/zot/pkg/extensions/sync"
logger "zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/signatures"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/local"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
)
@ -1660,7 +1659,7 @@ func TestBasicAuth(t *testing.T) {
"a": {
RootDirectory: destDir,
GC: true,
GCDelay: storage.DefaultGCDelay,
GCDelay: storageConstants.DefaultGCDelay,
Dedupe: true,
},
}
@ -2506,7 +2505,7 @@ func TestSubPaths(t *testing.T) {
subpath: {
RootDirectory: subPathDestDir,
GC: true,
GCDelay: storage.DefaultGCDelay,
GCDelay: storageConstants.DefaultGCDelay,
Dedupe: true,
},
}
@ -5203,7 +5202,7 @@ func TestSyncWithDestination(t *testing.T) {
test.CopyTestFiles("../../../test/data", srcDir)
err := os.MkdirAll(path.Join(sctlr.Config.Storage.RootDirectory, "/zot-fold"), local.DefaultDirPerms)
err := os.MkdirAll(path.Join(sctlr.Config.Storage.RootDirectory, "/zot-fold"), storageConstants.DefaultDirPerms)
So(err, ShouldBeNil)
// move upstream images under /zot-fold

28
pkg/extensions/sync/utils.go
View file

@ -32,9 +32,11 @@ import (
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
storageCommon "zotregistry.io/zot/pkg/storage/common"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/test"
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test/inject"
)
type ReferenceList struct {
@ -57,7 +59,7 @@ func getTagFromRef(ref types.ImageReference, log log.Logger) reference.Tagged {
func getImageTags(ctx context.Context, sysCtx *types.SystemContext, repoRef reference.Named) ([]string, error) {
dockerRef, err := docker.NewReference(reference.TagNameOnly(repoRef))
// hard to reach test case, injected error, see pkg/test/dev.go
if err = test.Error(err); err != nil {
if err = inject.Error(err); err != nil {
return nil, err // Should never happen for a reference with tag and no digest
}
@ -266,7 +268,7 @@ func getFileCredentials(filepath string) (syncconf.CredentialsFile, error) {
}
func pushSyncedLocalImage(localRepo, reference, localCachePath string,
repoDB repodb.RepoDB, imageStore storage.ImageStore, log log.Logger,
repoDB repodb.RepoDB, imageStore storageTypes.ImageStore, log log.Logger,
) error {
log.Info().Str("image", localCachePath+"/"+localRepo+":"+reference).Msg("pushing synced local image to local registry")
@ -275,7 +277,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
metrics := monitoring.NewMetricsServer(false, log)
cacheImageStore := local.NewImageStore(localCachePath, false,
storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil)
manifestBlob, manifestDigest, mediaType, err := cacheImageStore.GetImageManifest(localRepo, reference)
if err != nil {
@ -361,7 +363,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
}
func copyManifest(localRepo string, manifestContent []byte, reference string, repoDB repodb.RepoDB,
cacheImageStore, imageStore storage.ImageStore, log log.Logger,
cacheImageStore, imageStore storageTypes.ImageStore, log log.Logger,
) error {
var manifest ispec.Manifest
@ -376,7 +378,7 @@ func copyManifest(localRepo string, manifestContent []byte, reference string, re
}
for _, blob := range manifest.Layers {
if storage.IsNonDistributable(blob.MediaType) {
if storageCommon.IsNonDistributable(blob.MediaType) {
continue
}
@ -421,7 +423,7 @@ func copyManifest(localRepo string, manifestContent []byte, reference string, re
// Copy a blob from one image store to another image store.
func copyBlob(localRepo string, blobDigest godigest.Digest, blobMediaType string,
souceImageStore, destinationImageStore storage.ImageStore, log log.Logger,
souceImageStore, destinationImageStore storageTypes.ImageStore, log log.Logger,
) error {
if found, _, _ := destinationImageStore.CheckBlob(localRepo, blobDigest); found {
// Blob is already at destination, nothing to do
@ -508,12 +510,12 @@ func getLocalImageRef(localCachePath, repo, reference string) (types.ImageRefere
}
// Returns the localCachePath with an UUID at the end. Only to be called once per repo.
func getLocalCachePath(imageStore storage.ImageStore, repo string) (string, error) {
func getLocalCachePath(imageStore storageTypes.ImageStore, repo string) (string, error) {
localRepoPath := path.Join(imageStore.RootDir(), repo, SyncBlobUploadDir)
// check if SyncBlobUploadDir exists, create if not
var err error
if _, err = os.ReadDir(localRepoPath); os.IsNotExist(err) {
if err = os.MkdirAll(localRepoPath, local.DefaultDirPerms); err != nil {
if err = os.MkdirAll(localRepoPath, storageConstants.DefaultDirPerms); err != nil {
return "", err
}
}
@ -525,14 +527,14 @@ func getLocalCachePath(imageStore storage.ImageStore, repo string) (string, erro
// create uuid folder
uuid, err := guuid.NewV4()
// hard to reach test case, injected error, see pkg/test/dev.go
if err := test.Error(err); err != nil {
if err := inject.Error(err); err != nil {
return "", err
}
localCachePath := path.Join(localRepoPath, uuid.String())
cachedRepoPath := path.Join(localCachePath, repo)
if err = os.MkdirAll(cachedRepoPath, local.DefaultDirPerms); err != nil {
if err = os.MkdirAll(cachedRepoPath, storageConstants.DefaultDirPerms); err != nil {
return "", err
}
@ -540,7 +542,7 @@ func getLocalCachePath(imageStore storage.ImageStore, repo string) (string, erro
}
// canSkipImage returns whether or not we already synced this image.
func canSkipImage(repo, tag string, digest godigest.Digest, imageStore storage.ImageStore, log log.Logger,
func canSkipImage(repo, tag string, digest godigest.Digest, imageStore storageTypes.ImageStore, log log.Logger,
) (bool, error) {
// check image already synced
_, localImageManifestDigest, _, err := imageStore.GetImageManifest(repo, tag)

15
pkg/meta/repodb/storage_parsing.go
View file

@ -13,6 +13,7 @@ import (
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/signatures"
"zotregistry.io/zot/pkg/storage"
storageTypes "zotregistry.io/zot/pkg/storage/types"
)
// ParseStorage will sync all repos found in the rootdirectory of the oci layout that zot was deployed on with the
@ -215,8 +216,8 @@ func isManifestMetaPresent(repo string, manifest ispec.Descriptor, repoDB RepoDB
return true, nil
}
func GetSignatureLayersInfo(
repo, tag, manifestDigest, signatureType string, manifestBlob []byte, imageStore storage.ImageStore, log log.Logger,
func GetSignatureLayersInfo(repo, tag, manifestDigest, signatureType string, manifestBlob []byte,
imageStore storageTypes.ImageStore, log log.Logger,
) ([]LayerInfo, error) {
switch signatureType {
case signatures.CosignSignature:
@ -229,7 +230,7 @@ func GetSignatureLayersInfo(
}
func getCosignSignatureLayersInfo(
repo, tag, manifestDigest string, manifestBlob []byte, imageStore storage.ImageStore, log log.Logger,
repo, tag, manifestDigest string, manifestBlob []byte, imageStore storageTypes.ImageStore, log log.Logger,
) ([]LayerInfo, error) {
layers := []LayerInfo{}
@ -267,7 +268,7 @@ func getCosignSignatureLayersInfo(
}
func getNotationSignatureLayersInfo(
repo, manifestDigest string, manifestBlob []byte, imageStore storage.ImageStore, log log.Logger,
repo, manifestDigest string, manifestBlob []byte, imageStore storageTypes.ImageStore, log log.Logger,
) ([]LayerInfo, error) {
layers := []LayerInfo{}
@ -308,7 +309,7 @@ func getNotationSignatureLayersInfo(
}
// NewManifestMeta takes raw data about an image and createa a new ManifestMetadate object.
func NewManifestData(repoName string, manifestBlob []byte, imageStore storage.ImageStore,
func NewManifestData(repoName string, manifestBlob []byte, imageStore storageTypes.ImageStore,
) (ManifestData, error) {
var (
manifestContent ispec.Manifest
@ -337,7 +338,7 @@ func NewManifestData(repoName string, manifestBlob []byte, imageStore storage.Im
return manifestData, nil
}
func NewIndexData(repoName string, indexBlob []byte, imageStore storage.ImageStore,
func NewIndexData(repoName string, indexBlob []byte, imageStore storageTypes.ImageStore,
) IndexData {
indexData := IndexData{}
@ -349,7 +350,7 @@ func NewIndexData(repoName string, indexBlob []byte, imageStore storage.ImageSto
// SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag
// (in case the reference is a tag). The function expects image manifests and indexes (multi arch images).
func SetImageMetaFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte,
imageStore storage.ImageStore, repoDB RepoDB, log log.Logger,
imageStore storageTypes.ImageStore, repoDB RepoDB, log log.Logger,
) error {
switch mediaType {
case ispec.MediaTypeImageManifest:

3
pkg/meta/repodb/storage_parsing_test.go
View file

@ -25,6 +25,7 @@ import (
"zotregistry.io/zot/pkg/meta/signatures"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/local"
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
)
@ -63,7 +64,7 @@ func TestParseStorageErrors(t *testing.T) {
}
storeController := storage.StoreController{
DefaultStore: imageStore1,
SubStore: map[string]storage.ImageStore{
SubStore: map[string]storageTypes.ImageStore{
"a": imageStore2,
},
}

52
pkg/storage/cache.go
View file

@ -2,10 +2,58 @@ package storage
import (
"zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/api/config"
zlog "zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage/cache"
"zotregistry.io/zot/pkg/storage/constants"
)
func CreateCacheDatabaseDriver(storageConfig config.StorageConfig, log zlog.Logger) cache.Cache {
if !storageConfig.Dedupe && storageConfig.StorageDriver == nil {
return nil
}
// local cache
if !storageConfig.RemoteCache {
params := cache.BoltDBDriverParameters{}
params.RootDir = storageConfig.RootDirectory
params.Name = constants.BoltdbName
params.UseRelPaths = getUseRelPaths(&storageConfig)
driver, _ := Create("boltdb", params, log)
return driver
}
// remote cache
if storageConfig.CacheDriver != nil {
name, ok := storageConfig.CacheDriver["name"].(string)
if !ok {
log.Warn().Msg("remote cache driver name missing!")
return nil
}
if name != constants.DynamoDBDriverName {
log.Warn().Str("driver", name).Msg("remote cache driver unsupported!")
return nil
}
// dynamodb
dynamoParams := cache.DynamoDBDriverParameters{}
dynamoParams.Endpoint, _ = storageConfig.CacheDriver["endpoint"].(string)
dynamoParams.Region, _ = storageConfig.CacheDriver["region"].(string)
dynamoParams.TableName, _ = storageConfig.CacheDriver["cachetablename"].(string)
driver, _ := Create("dynamodb", dynamoParams, log)
return driver
}
return nil
}
func Create(dbtype string, parameters interface{}, log zlog.Logger) (cache.Cache, error) {
switch dbtype {
case "boltdb":
@ -22,3 +70,7 @@ func Create(dbtype string, parameters interface{}, log zlog.Logger) (cache.Cache
}
}
}
func getUseRelPaths(storageConfig *config.StorageConfig) bool {
return storageConfig.StorageDriver == nil
}

2
pkg/storage/cache/dynamodb.go vendored
View file

@ -80,7 +80,7 @@ func NewDynamoDBCache(parameters interface{}, log zlog.Logger) Cache {
// Using the SDK's default configuration, loading additional config
// and credentials values from the environment variables, shared
// credentials, and shared configuration files
cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(properParameters.Region),
cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(properParameters.Region),
config.WithEndpointResolverWithOptions(customResolver))
if err != nil {
log.Error().Err(err).Msg("unable to load AWS SDK config for dynamodb")

86
pkg/storage/common.go → pkg/storage/common/common.go
View file

@ -6,7 +6,6 @@ import (
"path"
"strings"
"github.com/gobwas/glob"
notreg "github.com/notaryproject/notation-go/registry"
godigest "github.com/opencontainers/go-digest"
imeta "github.com/opencontainers/image-spec/specs-go"
@ -19,11 +18,7 @@ import (
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/scheduler"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
)
const (
CosignType = "cosign"
NotationType = "notation"
storageTypes "zotregistry.io/zot/pkg/storage/types"
)
func GetTagsByIndex(index ispec.Index) []string {
@ -56,7 +51,7 @@ func GetManifestDescByReference(index ispec.Index, reference string) (ispec.Desc
return manifestDesc, false
}
func ValidateManifest(imgStore ImageStore, repo, reference, mediaType string, body []byte,
func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaType string, body []byte,
log zerolog.Logger,
) (godigest.Digest, error) {
// validate the manifest
@ -111,8 +106,8 @@ func ValidateManifest(imgStore ImageStore, repo, reference, mediaType string, bo
return "", nil
}
func validateOCIManifest(imgStore ImageStore, repo, reference string, manifest *ispec.Manifest, //nolint:unparam
log zerolog.Logger,
func validateOCIManifest(imgStore storageTypes.ImageStore, repo, reference string, //nolint:unparam
manifest *ispec.Manifest, log zerolog.Logger,
) (godigest.Digest, error) {
if manifest.SchemaVersion != storageConstants.SchemaVersion {
log.Error().Int("SchemaVersion", manifest.SchemaVersion).Msg("invalid manifest")
@ -245,7 +240,7 @@ func CheckIfIndexNeedsUpdate(index *ispec.Index, desc *ispec.Descriptor,
}
// GetIndex returns the contents of index.json.
func GetIndex(imgStore ImageStore, repo string, log zerolog.Logger) (ispec.Index, error) {
func GetIndex(imgStore storageTypes.ImageStore, repo string, log zerolog.Logger) (ispec.Index, error) {
var index ispec.Index
buf, err := imgStore.GetIndexContent(repo)
@ -263,7 +258,8 @@ func GetIndex(imgStore ImageStore, repo string, log zerolog.Logger) (ispec.Index
}
// GetImageIndex returns a multiarch type image.
func GetImageIndex(imgStore ImageStore, repo string, digest godigest.Digest, log zerolog.Logger) (ispec.Index, error) {
func GetImageIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger,
) (ispec.Index, error) {
var imageIndex ispec.Index
if err := digest.Validate(); err != nil {
@ -287,7 +283,7 @@ func GetImageIndex(imgStore ImageStore, repo string, digest godigest.Digest, log
return imageIndex, nil
}
func GetImageManifest(imgStore ImageStore, repo string, digest godigest.Digest, log zerolog.Logger,
func GetImageManifest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger,
) (ispec.Manifest, error) {
var manifestContent ispec.Manifest
@ -353,7 +349,7 @@ Unmarshal an image index and for all manifests in that
index, ensure that they do not have a name or they are not in other
manifest indexes else GC can never clean them.
*/
func UpdateIndexWithPrunedImageManifests(imgStore ImageStore, index *ispec.Index, repo string,
func UpdateIndexWithPrunedImageManifests(imgStore storageTypes.ImageStore, index *ispec.Index, repo string,
desc ispec.Descriptor, oldDgst godigest.Digest, log zerolog.Logger,
) error {
if (desc.MediaType == ispec.MediaTypeImageIndex) && (oldDgst != "") {
@ -386,7 +382,7 @@ same constitutent manifests so that they can be garbage-collected correctly
PruneImageManifestsFromIndex is a helper routine to achieve this.
*/
func PruneImageManifestsFromIndex(imgStore ImageStore, repo string, digest godigest.Digest, //nolint:gocyclo
func PruneImageManifestsFromIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, //nolint:gocyclo,lll
outIndex ispec.Index, otherImgIndexes []ispec.Descriptor, log zerolog.Logger,
) ([]ispec.Descriptor, error) {
dir := path.Join(imgStore.RootDir(), repo)
@ -461,7 +457,8 @@ func PruneImageManifestsFromIndex(imgStore ImageStore, repo string, digest godig
return prunedManifests, nil
}
func ApplyLinter(imgStore ImageStore, linter Lint, repo string, descriptor ispec.Descriptor) (bool, error) {
func ApplyLinter(imgStore storageTypes.ImageStore, linter Lint, repo string, descriptor ispec.Descriptor,
) (bool, error) {
pass := true
// we'll skip anything that's not a image manifest
@ -505,7 +502,7 @@ func IsSignature(descriptor ispec.Descriptor) bool {
return false
}
func GetOrasReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, artifactType string,
func GetOrasReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godigest.Digest, artifactType string,
log zerolog.Logger,
) ([]oras.Descriptor, error) {
if err := gdigest.Validate(); err != nil {
@ -583,7 +580,7 @@ func getReferrerFilterAnnotation(artifactTypes []string) string {
return annotation
}
func GetReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, artifactTypes []string,
func GetReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godigest.Digest, artifactTypes []string,
log zerolog.Logger,
) (ispec.Index, error) {
nilIndex := ispec.Index{}
@ -665,7 +662,7 @@ func GetReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, art
return index, nil
}
func GetOrasManifestByDigest(imgStore ImageStore, repo string, digest godigest.Digest, log zerolog.Logger,
func GetOrasManifestByDigest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger,
) (oras.Manifest, error) {
var artManifest oras.Manifest
@ -703,57 +700,14 @@ func IsNonDistributable(mediaType string) bool {
mediaType == ispec.MediaTypeImageLayerNonDistributableZstd //nolint:staticcheck
}
// CheckIsImageSignature checks if the given image (repo:tag) represents a signature. The function
// returns:
//
// - bool: if the image is a signature or not
//
// - string: the type of signature
//
// - string: the digest of the image it signs
//
// - error: any errors that occur.
func CheckIsImageSignature(repoName string, manifestBlob []byte, reference string,
) (bool, string, godigest.Digest, error) {
var manifestContent ispec.Manifest
err := json.Unmarshal(manifestBlob, &manifestContent)
if err != nil {
return false, "", "", err
}
manifestArtifactType := zcommon.GetManifestArtifactType(manifestContent)
// check notation signature
if manifestArtifactType == notreg.ArtifactTypeNotation && manifestContent.Subject != nil {
return true, NotationType, manifestContent.Subject.Digest, nil
}
// check cosign
cosignTagRule := glob.MustCompile("sha256-*.sig")
if tag := reference; cosignTagRule.Match(reference) {
prefixLen := len("sha256-")
digestLen := 64
signedImageManifestDigestEncoded := tag[prefixLen : prefixLen+digestLen]
signedImageManifestDigest := godigest.NewDigestFromEncoded(godigest.SHA256,
signedImageManifestDigestEncoded)
return true, CosignType, signedImageManifestDigest, nil
}
return false, "", "", nil
}
/*
DedupeTaskGenerator takes all blobs paths found in the imagestore and groups them by digest
DedupeTaskGenerator takes all blobs paths found in the storage.imagestore and groups them by digest
for each digest and based on the dedupe value it will dedupe or restore deduped blobs to the original state(undeduped)\
by creating a task for each digest and pushing it to the task scheduler.
*/
type DedupeTaskGenerator struct {
ImgStore ImageStore
ImgStore storageTypes.ImageStore
// storage dedupe value
Dedupe bool
// store blobs paths grouped by digest
@ -769,7 +723,7 @@ type DedupeTaskGenerator struct {
func (gen *DedupeTaskGenerator) GenerateTask() (scheduler.Task, error) {
var err error
// get all blobs from imageStore and group them by digest
// get all blobs from storage.imageStore and group them by digest
gen.digest, gen.duplicateBlobs, err = gen.ImgStore.GetNextDigestWithBlobPaths(gen.lastDigests)
if err != nil {
gen.Log.Error().Err(err).Msg("dedupe rebuild: failed to get next digest")
@ -805,7 +759,7 @@ func (gen *DedupeTaskGenerator) Reset() {
}
type dedupeTask struct {
imgStore ImageStore
imgStore storageTypes.ImageStore
// digest of duplicateBLobs
digest godigest.Digest
// blobs paths with the same digest ^
@ -814,7 +768,7 @@ type dedupeTask struct {
log zerolog.Logger
}
func newDedupeTask(imgStore ImageStore, digest godigest.Digest, dedupe bool,
func newDedupeTask(imgStore storageTypes.ImageStore, digest godigest.Digest, dedupe bool,
duplicateBlobs []string, log zerolog.Logger,
) *dedupeTask {
return &dedupeTask{imgStore, digest, duplicateBlobs, dedupe, log}

42
pkg/storage/common_test.go → pkg/storage/common/common_test.go
View file

@ -18,6 +18,8 @@ import (
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
common "zotregistry.io/zot/pkg/storage/common"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
@ -34,7 +36,7 @@ func TestValidateManifest(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true,
true, log, metrics, nil, cacheDriver)
content := []byte("this is a blob")
@ -148,33 +150,33 @@ func TestGetReferrersErrors(t *testing.T) {
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, false,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, false,
true, log, metrics, nil, cacheDriver)
artifactType := "application/vnd.example.icecream.v1"
validDigest := godigest.FromBytes([]byte("blob"))
Convey("Trigger invalid digest error", func(c C) {
_, err := storage.GetReferrers(imgStore, "zot-test", "invalidDigest",
_, err := common.GetReferrers(imgStore, "zot-test", "invalidDigest",
[]string{artifactType}, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
_, err = storage.GetOrasReferrers(imgStore, "zot-test", "invalidDigest",
_, err = common.GetOrasReferrers(imgStore, "zot-test", "invalidDigest",
artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
Convey("Trigger repo not found error", func(c C) {
_, err := storage.GetReferrers(imgStore, "zot-test", validDigest,
_, err := common.GetReferrers(imgStore, "zot-test", validDigest,
[]string{artifactType}, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
_, err = storage.GetOrasReferrers(imgStore, "zot-test", validDigest,
_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
err := test.CopyFiles("../../test/data/zot-test", path.Join(dir, "zot-test"))
err := test.CopyFiles("../../../test/data/zot-test", path.Join(dir, "zot-test"))
So(err, ShouldBeNil)
digest := godigest.FromBytes([]byte("{}"))
@ -201,11 +203,11 @@ func TestGetReferrersErrors(t *testing.T) {
},
}
_, err = storage.GetReferrers(imgStore, "zot-test", validDigest,
_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
[]string{artifactType}, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
_, err = storage.GetOrasReferrers(imgStore, "zot-test", validDigest,
_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
@ -220,11 +222,11 @@ func TestGetReferrersErrors(t *testing.T) {
},
}
_, err = storage.GetReferrers(imgStore, "zot-test", validDigest,
_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
[]string{artifactType}, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
_, err = storage.GetOrasReferrers(imgStore, "zot-test", validDigest,
_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
@ -249,11 +251,11 @@ func TestGetReferrersErrors(t *testing.T) {
},
}
_, err = storage.GetOrasReferrers(imgStore, "zot-test", validDigest,
_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
_, err = storage.GetOrasReferrers(imgStore, "zot-test", digest,
_, err = common.GetOrasReferrers(imgStore, "zot-test", digest,
artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
@ -268,7 +270,7 @@ func TestGetReferrersErrors(t *testing.T) {
},
}
_, err = storage.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger())
_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
@ -294,7 +296,7 @@ func TestGetReferrersErrors(t *testing.T) {
},
}
_, err = storage.GetReferrers(imgStore, "zot-test", validDigest,
_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
[]string{artifactType}, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
@ -328,7 +330,7 @@ func TestGetReferrersErrors(t *testing.T) {
},
}
_, err = storage.GetReferrers(imgStore, "zot-test", validDigest,
_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
[]string{artifactType}, log.With().Caller().Logger())
So(err, ShouldBeNil)
})
@ -341,7 +343,7 @@ func TestGetImageIndexErrors(t *testing.T) {
Convey("Trigger invalid digest error", t, func(c C) {
imgStore := &mocks.MockedImageStore{}
_, err := storage.GetImageIndex(imgStore, "zot-test", "invalidDigest", log)
_, err := common.GetImageIndex(imgStore, "zot-test", "invalidDigest", log)
So(err, ShouldNotBeNil)
})
@ -354,7 +356,7 @@ func TestGetImageIndexErrors(t *testing.T) {
validDigest := godigest.FromBytes([]byte("blob"))
_, err := storage.GetImageIndex(imgStore, "zot-test", validDigest, log)
_, err := common.GetImageIndex(imgStore, "zot-test", validDigest, log)
So(err, ShouldNotBeNil)
})
@ -367,14 +369,14 @@ func TestGetImageIndexErrors(t *testing.T) {
validDigest := godigest.FromBytes([]byte("blob"))
_, err := storage.GetImageIndex(imgStore, "zot-test", validDigest, log)
_, err := common.GetImageIndex(imgStore, "zot-test", validDigest, log)
So(err, ShouldNotBeNil)
})
}
func TestIsSignature(t *testing.T) {
Convey("Unknown media type", t, func(c C) {
isSingature := storage.IsSignature(ispec.Descriptor{
isSingature := common.IsSignature(ispec.Descriptor{
MediaType: "unknown media type",
})
So(isSingature, ShouldBeFalse)

11
pkg/storage/common/lint-interface.go Normal file
View file

@ -0,0 +1,11 @@
package storage
import (
godigest "github.com/opencontainers/go-digest"
storageTypes "zotregistry.io/zot/pkg/storage/types"
)
type Lint interface {
Lint(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error)
}

2
pkg/storage/constants/constants.go
View file

@ -20,4 +20,6 @@ const (
BoltdbName = "cache"
ReferrerFilterAnnotation = "org.opencontainers.referrers.filtersApplied"
DynamoDBDriverName = "dynamodb"
DefaultGCDelay = 1 * time.Hour
S3StorageDriverName = "s3"
)

9
pkg/storage/lint-interface.go
View file

@ -1,9 +0,0 @@
package storage
import (
godigest "github.com/opencontainers/go-digest"
)
type Lint interface {
Lint(repo string, manifestDigest godigest.Digest, imageStore ImageStore) (bool, error)
}

136
pkg/storage/local/local.go
View file

@ -34,31 +34,25 @@ import (
zlog "zotregistry.io/zot/pkg/log"
zreg "zotregistry.io/zot/pkg/regexp"
"zotregistry.io/zot/pkg/scheduler"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
common "zotregistry.io/zot/pkg/storage/common"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/test"
)
const (
DefaultFilePerms = 0o600
DefaultDirPerms = 0o700
defaultSchemaVersion = 2
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test/inject"
)
// ImageStoreLocal provides the image storage operations.
type ImageStoreLocal struct {
rootDir string
lock *sync.RWMutex
blobUploads map[string]storage.BlobUpload
cache cache.Cache
gc bool
dedupe bool
commit bool
gcDelay time.Duration
log zerolog.Logger
metrics monitoring.MetricServer
linter storage.Lint
rootDir string
lock *sync.RWMutex
cache cache.Cache
gc bool
dedupe bool
commit bool
gcDelay time.Duration
log zerolog.Logger
metrics monitoring.MetricServer
linter common.Lint
}
func (is *ImageStoreLocal) RootDir() string {
@ -72,10 +66,10 @@ func (is *ImageStoreLocal) DirExists(d string) bool {
// NewImageStore returns a new image store backed by a file storage.
// Use the last argument to properly set a cache database, or it will default to boltDB local storage.
func NewImageStore(rootDir string, gc bool, gcDelay time.Duration, dedupe, commit bool,
log zlog.Logger, metrics monitoring.MetricServer, linter storage.Lint, cacheDriver cache.Cache,
) storage.ImageStore {
log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, cacheDriver cache.Cache,
) storageTypes.ImageStore {
if _, err := os.Stat(rootDir); os.IsNotExist(err) {
if err := os.MkdirAll(rootDir, DefaultDirPerms); err != nil {
if err := os.MkdirAll(rootDir, storageConstants.DefaultDirPerms); err != nil {
log.Error().Err(err).Str("rootDir", rootDir).Msg("unable to create root dir")
return nil
@ -83,16 +77,15 @@ func NewImageStore(rootDir string, gc bool, gcDelay time.Duration, dedupe, commi
}
imgStore := &ImageStoreLocal{
rootDir: rootDir,
lock: &sync.RWMutex{},
blobUploads: make(map[string]storage.BlobUpload),
gc: gc,
gcDelay: gcDelay,
dedupe: dedupe,
commit: commit,
log: log.With().Caller().Logger(),
metrics: metrics,
linter: linter,
rootDir: rootDir,
lock: &sync.RWMutex{},
gc: gc,
gcDelay: gcDelay,
dedupe: dedupe,
commit: commit,
log: log.With().Caller().Logger(),
metrics: metrics,
linter: linter,
}
imgStore.cache = cacheDriver
@ -197,7 +190,7 @@ func (is *ImageStoreLocal) initRepo(name string) error {
// "index.json" file - create if it doesn't exist
indexPath := path.Join(repoDir, "index.json")
if _, err := os.Stat(indexPath); err != nil {
index := ispec.Index{Versioned: imeta.Versioned{SchemaVersion: defaultSchemaVersion}}
index := ispec.Index{Versioned: imeta.Versioned{SchemaVersion: storageConstants.SchemaVersion}}
buf, err := json.Marshal(index)
if err != nil {
@ -401,12 +394,12 @@ func (is *ImageStoreLocal) GetImageTags(repo string) ([]string, error) {
is.RLock(&lockLatency)
defer is.RUnlock(&lockLatency)
index, err := storage.GetIndex(is, repo, is.log)
index, err := common.GetIndex(is, repo, is.log)
if err != nil {
return nil, err
}
return storage.GetTagsByIndex(index), nil
return common.GetTagsByIndex(index), nil
}
// GetImageManifest returns the image manifest of an image in the specific repository.
@ -421,12 +414,12 @@ func (is *ImageStoreLocal) GetImageManifest(repo, reference string) ([]byte, god
is.RLock(&lockLatency)
defer is.RUnlock(&lockLatency)
index, err := storage.GetIndex(is, repo, is.log)
index, err := common.GetIndex(is, repo, is.log)
if err != nil {
return nil, "", "", err
}
manifestDesc, found := storage.GetManifestDescByReference(index, reference)
manifestDesc, found := common.GetManifestDescByReference(index, reference)
if !found {
return nil, "", "", zerr.ErrManifestNotFound
}
@ -467,14 +460,14 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, /
is.Lock(&lockLatency)
defer is.Unlock(&lockLatency)
digest, err := storage.ValidateManifest(is, repo, reference, mediaType, body, is.log)
digest, err := common.ValidateManifest(is, repo, reference, mediaType, body, is.log)
if err != nil {
return digest, "", err
}
refIsDigest := true
mDigest, err := storage.GetAndValidateRequestDigest(body, reference, is.log)
mDigest, err := common.GetAndValidateRequestDigest(body, reference, is.log)
if err != nil {
if errors.Is(err, zerr.ErrBadManifest) {
return mDigest, "", err
@ -483,7 +476,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, /
refIsDigest = false
}
index, err := storage.GetIndex(is, repo, is.log)
index, err := common.GetIndex(is, repo, is.log)
if err != nil {
return "", "", err
}
@ -516,7 +509,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, /
artifactType = zcommon.GetManifestArtifactType(manifest)
}
updateIndex, oldDgst, err := storage.CheckIfIndexNeedsUpdate(&index, &desc, is.log)
updateIndex, oldDgst, err := common.CheckIfIndexNeedsUpdate(&index, &desc, is.log)
if err != nil {
return "", "", err
}
@ -537,7 +530,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, /
return "", "", err
}
err = storage.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log)
err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log)
if err != nil {
return "", "", err
}
@ -548,7 +541,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, /
file = path.Join(dir, "index.json")
buf, err := json.Marshal(index)
if err := test.Error(err); err != nil {
if err := inject.Error(err); err != nil {
is.log.Error().Err(err).Str("file", file).Msg("unable to marshal JSON")
return "", "", err
@ -558,7 +551,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, /
desc.ArtifactType = artifactType
// apply linter only on images, not signatures or indexes
pass, err := storage.ApplyLinter(is, is.linter, repo, desc)
pass, err := common.ApplyLinter(is, is.linter, repo, desc)
if !pass {
is.log.Error().Err(err).Str("repository", repo).Str("reference", reference).Msg("linter didn't pass")
@ -566,7 +559,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, /
}
err = is.writeFile(file, buf)
if err := test.Error(err); err != nil {
if err := inject.Error(err); err != nil {
is.log.Error().Err(err).Str("file", file).Msg("unable to write")
return "", "", err
@ -596,17 +589,17 @@ func (is *ImageStoreLocal) DeleteImageManifest(repo, reference string, detectCol
is.Lock(&lockLatency)
defer is.Unlock(&lockLatency)
index, err := storage.GetIndex(is, repo, is.log)
index, err := common.GetIndex(is, repo, is.log)
if err != nil {
return err
}
manifestDesc, err := storage.RemoveManifestDescByReference(&index, reference, detectCollision)
manifestDesc, err := common.RemoveManifestDescByReference(&index, reference, detectCollision)
if err != nil {
return err
}
err = storage.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log)
err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log)
if err != nil {
return err
}
@ -678,7 +671,7 @@ func (is *ImageStoreLocal) NewBlobUpload(repo string) (string, error) {
blobUploadPath := is.BlobUploadPath(repo, uid)
file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, DefaultFilePerms)
file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, storageConstants.DefaultFilePerms)
if err != nil {
return "", zerr.ErrRepoNotFound
}
@ -724,7 +717,7 @@ func (is *ImageStoreLocal) PutBlobChunkStreamed(repo, uuid string, body io.Reade
return -1, zerr.ErrUploadNotFound
}
file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, DefaultFilePerms)
file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, storageConstants.DefaultFilePerms)
if err != nil {
is.log.Error().Err(err).Msg("failed to open file")
@ -773,7 +766,7 @@ func (is *ImageStoreLocal) PutBlobChunk(repo, uuid string, from, to int64,
return -1, zerr.ErrBadUploadRange
}
file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, DefaultFilePerms)
file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, storageConstants.DefaultFilePerms)
if err != nil {
is.log.Error().Err(err).Msg("failed to open file")
@ -876,7 +869,7 @@ func (is *ImageStoreLocal) FinishBlobUpload(repo, uuid string, body io.Reader, d
if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) {
err = is.DedupeBlob(src, dstDigest, dst)
if err := test.Error(err); err != nil {
if err := inject.Error(err); err != nil {
is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()).
Str("dst", dst).Msg("unable to dedupe blob")
@ -1390,7 +1383,7 @@ func (is *ImageStoreLocal) GetReferrers(repo string, gdigest godigest.Digest, ar
is.RLock(&lockLatency)
defer is.RUnlock(&lockLatency)
return storage.GetReferrers(is, repo, gdigest, artifactTypes, is.log)
return common.GetReferrers(is, repo, gdigest, artifactTypes, is.log)
}
func (is *ImageStoreLocal) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string,
@ -1400,42 +1393,40 @@ func (is *ImageStoreLocal) GetOrasReferrers(repo string, gdigest godigest.Digest
is.RLock(&lockLatency)
defer is.RUnlock(&lockLatency)
return storage.GetOrasReferrers(is, repo, gdigest, artifactType, is.log)
return common.GetOrasReferrers(is, repo, gdigest, artifactType, is.log)
}
func (is *ImageStoreLocal) writeFile(filename string, data []byte) error {
if !is.commit {
return os.WriteFile(filename, data, DefaultFilePerms)
return os.WriteFile(filename, data, storageConstants.DefaultFilePerms)
}
fhandle, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, DefaultFilePerms)
fhandle, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, storageConstants.DefaultFilePerms)
if err != nil {
return err
}
_, err = fhandle.Write(data)
if err1 := test.Error(fhandle.Sync()); err1 != nil && err == nil {
if err1 := inject.Error(fhandle.Sync()); err1 != nil && err == nil {
err = err1
is.log.Error().Err(err).Str("filename", filename).Msg("unable to sync file")
}
if err1 := test.Error(fhandle.Close()); err1 != nil && err == nil {
if err1 := inject.Error(fhandle.Close()); err1 != nil && err == nil {
err = err1
}
return err
}
// utility routines
func ValidateHardLink(rootDir string) error {
if err := os.MkdirAll(rootDir, DefaultDirPerms); err != nil {
if err := os.MkdirAll(rootDir, storageConstants.DefaultDirPerms); err != nil {
return err
}
err := os.WriteFile(path.Join(rootDir, "hardlinkcheck.txt"),
[]byte("check whether hardlinks work on filesystem"), DefaultFilePerms)
[]byte("check whether hardlinks work on filesystem"), storageConstants.DefaultFilePerms)
if err != nil {
return err
}
@ -1459,8 +1450,9 @@ func ValidateHardLink(rootDir string) error {
return os.RemoveAll(path.Join(rootDir, "duphardlinkcheck.txt"))
}
// utility routines.
func ensureDir(dir string, log zerolog.Logger) error {
if err := os.MkdirAll(dir, DefaultDirPerms); err != nil {
if err := os.MkdirAll(dir, storageConstants.DefaultDirPerms); err != nil {
log.Error().Err(err).Str("dir", dir).Msg("unable to create dir")
return err
@ -1477,7 +1469,7 @@ type extendedManifest struct {
func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error {
oci, err := umoci.OpenLayout(dir)
if err := test.Error(err); err != nil {
if err := inject.Error(err); err != nil {
return err
}
defer oci.Close()
@ -1497,7 +1489,7 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error {
for _, desc := range index.Manifests {
switch desc.MediaType {
case ispec.MediaTypeImageIndex:
indexImage, err := storage.GetImageIndex(is, repo, desc.Digest, is.log)
indexImage, err := common.GetImageIndex(is, repo, desc.Digest, is.log)
if err != nil {
is.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()).
Msg("gc: failed to read multiarch(index) image")
@ -1519,7 +1511,7 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error {
}
}
manifestContent, err := storage.GetImageManifest(is, repo, desc.Digest, is.log)
manifestContent, err := common.GetImageManifest(is, repo, desc.Digest, is.log)
if err != nil {
is.log.Error().Err(err).Str("repo", repo).Str("digest", desc.Digest.String()).
Msg("gc: failed to read manifest image")
@ -1559,7 +1551,7 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error {
is.log.Info().Msg("gc: blobs")
err = oci.GC(context.Background(), ifOlderThan(is, repo, is.gcDelay))
if err := test.Error(err); err != nil {
if err := inject.Error(err); err != nil {
return err
}
@ -1616,7 +1608,7 @@ func gcUntaggedManifests(imgStore *ImageStoreLocal, oci casext.Engine, index *is
imgStore.log.Info().Str("repository", repo).Str("digest", desc.Digest.String()).
Msg("gc: removing manifest without tag")
_, err = storage.RemoveManifestDescByReference(index, desc.Digest.String(), true)
_, err = common.RemoveManifestDescByReference(index, desc.Digest.String(), true)
if errors.Is(err, zerr.ErrManifestConflict) {
imgStore.log.Info().Str("repository", repo).Str("digest", desc.Digest.String()).
Msg("gc: skipping removing manifest due to conflict")
@ -1655,7 +1647,7 @@ func gcCosignSignatures(imgStore *ImageStoreLocal, oci casext.Engine, index *isp
Msg("gc: removing cosign signature without subject")
// no need to check for manifest conflict, if one doesn't have a subject, then none with same digest will have
_, _ = storage.RemoveManifestDescByReference(index, cosignDesc.Digest.String(), false)
_, _ = common.RemoveManifestDescByReference(index, cosignDesc.Digest.String(), false)
err := oci.PutIndex(context.Background(), *index)
if err != nil {
@ -1685,7 +1677,7 @@ func gcNotationSignatures(imgStore *ImageStoreLocal, oci casext.Engine, index *i
Msg("gc: removing notation signature without subject")
// no need to check for manifest conflict, if one doesn't have a subject, then none with same digest will have
_, _ = storage.RemoveManifestDescByReference(index, notationManifest.Digest.String(), false)
_, _ = common.RemoveManifestDescByReference(index, notationManifest.Digest.String(), false)
err := oci.PutIndex(context.Background(), *index)
if err != nil {
@ -1910,7 +1902,7 @@ func (is *ImageStoreLocal) dedupeBlobs(digest godigest.Digest, duplicateBlobs []
tempLinkBlobDir := path.Join(strings.Replace(blobPath, path.Join("blobs/sha256", binfo.Name()), "", 1),
storageConstants.BlobUploadDir)
if err := os.MkdirAll(tempLinkBlobDir, DefaultDirPerms); err != nil {
if err := os.MkdirAll(tempLinkBlobDir, storageConstants.DefaultDirPerms); err != nil {
is.log.Error().Err(err).Str("dir", tempLinkBlobDir).Msg("rebuild dedupe: unable to mkdir")
return err
@ -1962,7 +1954,7 @@ func (is *ImageStoreLocal) RunDedupeForDigest(digest godigest.Digest, dedupe boo
func (is *ImageStoreLocal) RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) {
// for local storage no need to undedupe blobs
if is.dedupe {
generator := &storage.DedupeTaskGenerator{
generator := &common.DedupeTaskGenerator{
ImgStore: is,
Dedupe: is.dedupe,
Log: is.log,

4
pkg/storage/local/local_elevated_test.go
View file

@ -20,6 +20,7 @@ import (
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
)
@ -35,7 +36,8 @@ func TestElevatedPrivilegesInvalidDedupe(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log,
metrics, nil, cacheDriver)
upload, err := imgStore.NewBlobUpload("dedupe1")
So(err, ShouldBeNil)

147
pkg/storage/local/local_test.go
View file

@ -34,7 +34,9 @@ import (
"zotregistry.io/zot/pkg/storage/cache"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/inject"
"zotregistry.io/zot/pkg/test/mocks"
)
@ -65,7 +67,7 @@ func TestStorageFSAPIs(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true,
true, log, metrics, nil, cacheDriver)
Convey("Repo layout", t, func(c C) {
@ -202,7 +204,7 @@ func TestGetOrasReferrers(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver)
Convey("Get referrers", t, func(c C) {
err := test.CopyFiles("../../../test/data/zot-test", path.Join(dir, "zot-test"))
@ -257,7 +259,8 @@ func FuzzNewBlobUpload(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil,
cacheDriver)
_, err := imgStore.NewBlobUpload(data)
if err != nil {
@ -282,7 +285,8 @@ func FuzzPutBlobChunk(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil,
cacheDriver)
repoName := data
uuid, err := imgStore.NewBlobUpload(repoName)
@ -315,7 +319,8 @@ func FuzzPutBlobChunkStreamed(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil,
cacheDriver)
repoName := data
@ -347,7 +352,8 @@ func FuzzGetBlobUpload(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil,
cacheDriver)
_, err := imgStore.GetBlobUpload(data1, data2)
if err != nil {
@ -372,7 +378,8 @@ func FuzzTestPutGetImageManifest(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
cblob, cdigest := test.GetRandomImageConfig()
@ -423,7 +430,8 @@ func FuzzTestPutDeleteImageManifest(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
cblob, cdigest := test.GetRandomImageConfig()
@ -481,7 +489,8 @@ func FuzzTestDeleteImageManifest(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
digest, _, err := newRandomBlobForFuzz(data)
if err != nil {
@ -516,7 +525,8 @@ func FuzzInitRepo(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
err := imgStore.InitRepo(data)
if err != nil {
if isKnownErr(err) {
@ -540,7 +550,8 @@ func FuzzInitValidateRepo(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
err := imgStore.InitRepo(data)
if err != nil {
if isKnownErr(err) {
@ -571,7 +582,8 @@ func FuzzGetImageTags(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
_, err := imgStore.GetImageTags(data)
if err != nil {
if errors.Is(err, zerr.ErrRepoNotFound) || isKnownErr(err) {
@ -595,7 +607,8 @@ func FuzzBlobUploadPath(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
_ = imgStore.BlobUploadPath(repo, uuid)
})
@ -614,7 +627,8 @@ func FuzzBlobUploadInfo(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
repo := data
_, err := imgStore.BlobUploadInfo(repo, uuid)
@ -639,7 +653,8 @@ func FuzzTestGetImageManifest(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil,
cacheDriver)
repoName := data
@ -667,7 +682,8 @@ func FuzzFinishBlobUpload(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil,
cacheDriver)
repoName := data
@ -716,7 +732,8 @@ func FuzzFullBlobUpload(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
ldigest, lblob, err := newRandomBlobForFuzz(data)
if err != nil {
@ -746,7 +763,7 @@ func TestStorageCacheErrors(t *testing.T) {
cblob, cdigest := test.GetRandomImageConfig()
getBlobPath := ""
imgStore := local.NewImageStore(dir, false, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, &mocks.CacheMock{
PutBlobFn: func(digest godigest.Digest, path string) error {
if strings.Contains(path, dedupedRepo) {
@ -788,7 +805,8 @@ func FuzzDedupeBlob(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
blobDigest := godigest.FromString(data)
@ -829,7 +847,8 @@ func FuzzDeleteBlobUpload(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
uuid, err := imgStore.NewBlobUpload(repoName)
if err != nil {
@ -860,7 +879,8 @@ func FuzzBlobPath(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
digest := godigest.FromString(data)
_ = imgStore.BlobPath(repoName, digest)
@ -881,7 +901,8 @@ func FuzzCheckBlob(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
digest := godigest.FromString(data)
_, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest)
@ -912,7 +933,8 @@ func FuzzGetBlob(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
digest := godigest.FromString(data)
_, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest)
@ -950,7 +972,8 @@ func FuzzDeleteBlob(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
digest := godigest.FromString(data)
_, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest)
@ -985,7 +1008,8 @@ func FuzzGetIndexContent(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
digest := godigest.FromString(data)
_, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest)
@ -1020,7 +1044,8 @@ func FuzzGetBlobContent(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
digest := godigest.FromString(data)
_, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest)
@ -1054,7 +1079,8 @@ func FuzzGetOrasReferrers(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
err := test.CopyFiles("../../../test/data/zot-test", path.Join(dir, "zot-test"))
if err != nil {
@ -1114,7 +1140,8 @@ func FuzzRunGCRepo(f *testing.F) {
Name: "cache",
UseRelPaths: true,
}, *log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver)
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil,
cacheDriver)
if err := imgStore.RunGCRepo(data); err != nil {
t.Error(err)
@ -1149,13 +1176,13 @@ func TestDedupeLinks(t *testing.T) {
UseRelPaths: true,
}, log)
var imgStore storage.ImageStore
var imgStore storageTypes.ImageStore
if testCase.dedupe {
imgStore = local.NewImageStore(dir, false, storage.DefaultGCDelay,
imgStore = local.NewImageStore(dir, false, storageConstants.DefaultGCDelay,
testCase.dedupe, true, log, metrics, nil, cacheDriver)
} else {
imgStore = local.NewImageStore(dir, false, storage.DefaultGCDelay,
imgStore = local.NewImageStore(dir, false, storageConstants.DefaultGCDelay,
testCase.dedupe, true, log, metrics, nil, nil)
}
@ -1299,7 +1326,7 @@ func TestDedupeLinks(t *testing.T) {
Convey("test RunDedupeForDigest directly, trigger stat error on original blob", func() {
// rebuild with dedupe true
imgStore := local.NewImageStore(dir, false, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver)
duplicateBlobs := []string{
@ -1319,7 +1346,7 @@ func TestDedupeLinks(t *testing.T) {
for i := 0; i < 10; i++ {
taskScheduler, cancel := runAndGetScheduler()
// rebuild with dedupe true
imgStore := local.NewImageStore(dir, false, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver)
imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
@ -1332,7 +1359,7 @@ func TestDedupeLinks(t *testing.T) {
taskScheduler, cancel := runAndGetScheduler()
// rebuild with dedupe true
imgStore := local.NewImageStore(dir, false, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver)
imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
@ -1352,7 +1379,7 @@ func TestDedupeLinks(t *testing.T) {
// switch dedupe to true from false
taskScheduler, cancel := runAndGetScheduler()
imgStore := local.NewImageStore(dir, false, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, nil)
// rebuild with dedupe true
@ -1375,7 +1402,7 @@ func TestDedupeLinks(t *testing.T) {
// switch dedupe to true from false
taskScheduler, cancel := runAndGetScheduler()
imgStore := local.NewImageStore(dir, false, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, &mocks.CacheMock{
HasBlobFn: func(digest godigest.Digest, path string) bool {
return false
@ -1404,7 +1431,7 @@ func TestDedupeLinks(t *testing.T) {
// switch dedupe to true from false
taskScheduler, cancel := runAndGetScheduler()
imgStore := local.NewImageStore(dir, false, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, &mocks.CacheMock{
HasBlobFn: func(digest godigest.Digest, path string) bool {
return false
@ -1479,7 +1506,7 @@ func TestDedupeLinks(t *testing.T) {
func TestDedupe(t *testing.T) {
Convey("Dedupe", t, func(c C) {
Convey("Nil ImageStore", func() {
var is storage.ImageStore
var is storageTypes.ImageStore
So(func() { _ = is.DedupeBlob("", "", "") }, ShouldPanic)
})
@ -1493,7 +1520,7 @@ func TestDedupe(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
il := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver)
il := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver)
So(il.DedupeBlob("", "", ""), ShouldNotBeNil)
})
@ -1512,7 +1539,7 @@ func TestNegativeCases(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
So(local.NewImageStore(dir, true, storage.DefaultGCDelay, true,
So(local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true,
true, log, metrics, nil, cacheDriver), ShouldNotBeNil)
if os.Geteuid() != 0 {
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
@ -1520,7 +1547,7 @@ func TestNegativeCases(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
So(local.NewImageStore("/deadBEEF", true, storage.DefaultGCDelay,
So(local.NewImageStore("/deadBEEF", true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver), ShouldBeNil)
}
})
@ -1535,7 +1562,7 @@ func TestNegativeCases(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver)
err := os.Chmod(dir, 0o000) // remove all perms
@ -1585,7 +1612,7 @@ func TestNegativeCases(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true,
true, log, metrics, nil, cacheDriver)
So(imgStore, ShouldNotBeNil)
@ -1705,7 +1732,7 @@ func TestNegativeCases(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver)
So(imgStore, ShouldNotBeNil)
@ -1734,7 +1761,7 @@ func TestNegativeCases(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true,
true, log, metrics, nil, cacheDriver)
So(imgStore, ShouldNotBeNil)
@ -1781,7 +1808,7 @@ func TestNegativeCases(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver)
So(imgStore, ShouldNotBeNil)
@ -1952,11 +1979,11 @@ func TestInjectWriteFile(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver)
Convey("Failure path1", func() {
injected := test.InjectFailure(0)
injected := inject.InjectFailure(0)
err := imgStore.InitRepo("repo1")
if injected {
@ -1967,7 +1994,7 @@ func TestInjectWriteFile(t *testing.T) {
})
Convey("Failure path2", func() {
injected := test.InjectFailure(1)
injected := inject.InjectFailure(1)
err := imgStore.InitRepo("repo2")
if injected {
@ -1988,7 +2015,7 @@ func TestInjectWriteFile(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
true, false, log, metrics, nil, cacheDriver)
Convey("Failure path not reached", func() {
@ -2011,7 +2038,7 @@ func TestGarbageCollect(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver)
repoName := "gc-long"
@ -2693,7 +2720,7 @@ func TestInitRepo(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver)
err := os.Mkdir(path.Join(dir, "test-dir"), 0o000)
@ -2715,7 +2742,7 @@ func TestValidateRepo(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver)
err := os.Mkdir(path.Join(dir, "test-dir"), 0o000)
@ -2735,7 +2762,7 @@ func TestValidateRepo(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver)
_, err := imgStore.ValidateRepo(".")
@ -2780,7 +2807,7 @@ func TestGetRepositories(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver,
)
@ -2877,7 +2904,7 @@ func TestGetRepositories(t *testing.T) {
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver,
)
@ -2925,7 +2952,7 @@ func TestGetRepositories(t *testing.T) {
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(rootDir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(rootDir, true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver,
)
@ -2968,7 +2995,7 @@ func TestGetNextRepository(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver,
)
firstRepoName := "repo1"
@ -3013,7 +3040,7 @@ func TestPutBlobChunkStreamed(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver)
uuid, err := imgStore.NewBlobUpload("test")
@ -3042,7 +3069,7 @@ func TestPullRange(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver)
repoName := "pull-range"

77
pkg/storage/s3/s3.go
View file

@ -29,10 +29,11 @@ import (
zlog "zotregistry.io/zot/pkg/log"
zreg "zotregistry.io/zot/pkg/regexp"
"zotregistry.io/zot/pkg/scheduler"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
common "zotregistry.io/zot/pkg/storage/common"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/test"
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test/inject"
)
const (
@ -41,15 +42,14 @@ const (
// ObjectStorage provides the image storage operations.
type ObjectStorage struct {
rootDir string
store driver.StorageDriver
lock *sync.RWMutex
blobUploads map[string]storage.BlobUpload
log zerolog.Logger
metrics monitoring.MetricServer
cache cache.Cache
dedupe bool
linter storage.Lint
rootDir string
store driver.StorageDriver
lock *sync.RWMutex
log zerolog.Logger
metrics monitoring.MetricServer
cache cache.Cache
dedupe bool
linter common.Lint
}
func (is *ObjectStorage) RootDir() string {
@ -68,18 +68,17 @@ func (is *ObjectStorage) DirExists(d string) bool {
// see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers
// Use the last argument to properly set a cache database, or it will default to boltDB local storage.
func NewImageStore(rootDir string, cacheDir string, gc bool, gcDelay time.Duration, dedupe, commit bool,
log zlog.Logger, metrics monitoring.MetricServer, linter storage.Lint,
log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint,
store driver.StorageDriver, cacheDriver cache.Cache,
) storage.ImageStore {
) storageTypes.ImageStore {
imgStore := &ObjectStorage{
rootDir: rootDir,
store: store,
lock: &sync.RWMutex{},
blobUploads: make(map[string]storage.BlobUpload),
log: log.With().Caller().Logger(),
metrics: metrics,
dedupe: dedupe,
linter: linter,
rootDir: rootDir,
store: store,
lock: &sync.RWMutex{},
log: log.With().Caller().Logger(),
metrics: metrics,
dedupe: dedupe,
linter: linter,
}
imgStore.cache = cacheDriver
@ -306,12 +305,12 @@ func (is *ObjectStorage) GetImageTags(repo string) ([]string, error) {
is.RLock(&lockLatency)
defer is.RUnlock(&lockLatency)
index, err := storage.GetIndex(is, repo, is.log)
index, err := common.GetIndex(is, repo, is.log)
if err != nil {
return nil, err
}
return storage.GetTagsByIndex(index), nil
return common.GetTagsByIndex(index), nil
}
// GetImageManifest returns the image manifest of an image in the specific repository.
@ -326,12 +325,12 @@ func (is *ObjectStorage) GetImageManifest(repo, reference string) ([]byte, godig
is.RLock(&lockLatency)
defer is.RUnlock(&lockLatency)
index, err := storage.GetIndex(is, repo, is.log)
index, err := common.GetIndex(is, repo, is.log)
if err != nil {
return nil, "", "", zerr.ErrRepoNotFound
}
manifestDesc, found := storage.GetManifestDescByReference(index, reference)
manifestDesc, found := common.GetManifestDescByReference(index, reference)
if !found {
return nil, "", "", zerr.ErrManifestNotFound
}
@ -372,14 +371,14 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n
is.Lock(&lockLatency)
defer is.Unlock(&lockLatency)
dig, err := storage.ValidateManifest(is, repo, reference, mediaType, body, is.log)
dig, err := common.ValidateManifest(is, repo, reference, mediaType, body, is.log)
if err != nil {
return dig, "", err
}
refIsDigest := true
mDigest, err := storage.GetAndValidateRequestDigest(body, reference, is.log)
mDigest, err := common.GetAndValidateRequestDigest(body, reference, is.log)
if err != nil {
if errors.Is(err, zerr.ErrBadManifest) {
return mDigest, "", err
@ -388,7 +387,7 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n
refIsDigest = false
}
index, err := storage.GetIndex(is, repo, is.log)
index, err := common.GetIndex(is, repo, is.log)
if err != nil {
return "", "", err
}
@ -421,7 +420,7 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n
artifactType = zcommon.GetManifestArtifactType(manifest)
}
updateIndex, oldDgst, err := storage.CheckIfIndexNeedsUpdate(&index, &desc, is.log)
updateIndex, oldDgst, err := common.CheckIfIndexNeedsUpdate(&index, &desc, is.log)
if err != nil {
return "", "", err
}
@ -440,7 +439,7 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n
return "", "", err
}
err = storage.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log)
err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log)
if err != nil {
return "", "", err
}
@ -461,7 +460,7 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n
desc.ArtifactType = artifactType
// apply linter only on images, not signatures
pass, err := storage.ApplyLinter(is, is.linter, repo, desc)
pass, err := common.ApplyLinter(is, is.linter, repo, desc)
if !pass {
is.log.Error().Err(err).Str("repository", repo).Str("reference", reference).Msg("linter didn't pass")
@ -492,17 +491,17 @@ func (is *ObjectStorage) DeleteImageManifest(repo, reference string, detectColli
is.Lock(&lockLatency)
defer is.Unlock(&lockLatency)
index, err := storage.GetIndex(is, repo, is.log)
index, err := common.GetIndex(is, repo, is.log)
if err != nil {
return err
}
manifestDesc, err := storage.RemoveManifestDescByReference(&index, reference, detectCollisions)
manifestDesc, err := common.RemoveManifestDescByReference(&index, reference, detectCollisions)
if err != nil {
return err
}
err = storage.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log)
err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log)
if err != nil {
return err
}
@ -862,7 +861,7 @@ retry:
is.log.Debug().Str("src", src).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: enter")
dstRecord, err := is.cache.GetBlob(dstDigest)
if err := test.Error(err); err != nil && !errors.Is(err, zerr.ErrCacheMiss) {
if err := inject.Error(err); err != nil && !errors.Is(err, zerr.ErrCacheMiss) {
is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to lookup blob record")
return err
@ -892,7 +891,7 @@ retry:
is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat")
// the actual blob on disk may have been removed by GC, so sync the cache
err := is.cache.DeleteBlob(dstDigest, dstRecord)
if err = test.Error(err); err != nil {
if err = inject.Error(err); err != nil {
//nolint:lll
is.log.Error().Err(err).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: unable to delete blob record")
@ -1296,7 +1295,7 @@ func (is *ObjectStorage) GetReferrers(repo string, gdigest godigest.Digest, arti
is.RLock(&lockLatency)
defer is.RUnlock(&lockLatency)
return storage.GetReferrers(is, repo, gdigest, artifactTypes, is.log)
return common.GetReferrers(is, repo, gdigest, artifactTypes, is.log)
}
func (is *ObjectStorage) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string,
@ -1306,7 +1305,7 @@ func (is *ObjectStorage) GetOrasReferrers(repo string, gdigest godigest.Digest,
is.RLock(&lockLatency)
defer is.RUnlock(&lockLatency)
return storage.GetOrasReferrers(is, repo, gdigest, artifactType, is.log)
return common.GetOrasReferrers(is, repo, gdigest, artifactType, is.log)
}
// GetIndexContent returns index.json contents, SHOULD lock from outside.
@ -1639,7 +1638,7 @@ func (is *ObjectStorage) RunDedupeForDigest(digest godigest.Digest, dedupe bool,
}
func (is *ObjectStorage) RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) {
generator := &storage.DedupeTaskGenerator{
generator := &common.DedupeTaskGenerator{
ImgStore: is,
Dedupe: is.dedupe,
Log: is.log,

25
pkg/storage/s3/s3_test.go
View file

@ -35,7 +35,9 @@ import (
"zotregistry.io/zot/pkg/storage/cache"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/s3"
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/inject"
"zotregistry.io/zot/pkg/test/mocks"
)
@ -67,7 +69,8 @@ func skipIt(t *testing.T) {
}
}
func createMockStorage(rootDir string, cacheDir string, dedupe bool, store driver.StorageDriver) storage.ImageStore {
func createMockStorage(rootDir string, cacheDir string, dedupe bool, store driver.StorageDriver,
) storageTypes.ImageStore {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
@ -81,7 +84,7 @@ func createMockStorage(rootDir string, cacheDir string, dedupe bool, store drive
UseRelPaths: false,
}, log)
}
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay,
il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay,
dedupe, false, log, metrics, nil, store, cacheDriver,
)
@ -90,11 +93,11 @@ func createMockStorage(rootDir string, cacheDir string, dedupe bool, store drive
func createMockStorageWithMockCache(rootDir string, dedupe bool, store driver.StorageDriver,
cacheDriver cache.Cache,
) storage.ImageStore {
) storageTypes.ImageStore {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
il := s3.NewImageStore(rootDir, "", false, storage.DefaultGCDelay,
il := s3.NewImageStore(rootDir, "", false, storageConstants.DefaultGCDelay,
dedupe, false, log, metrics, nil, store, cacheDriver,
)
@ -134,7 +137,7 @@ func createStoreDriver(rootDir string) driver.StorageDriver {
func createObjectsStore(rootDir string, cacheDir string, dedupe bool) (
driver.StorageDriver,
storage.ImageStore,
storageTypes.ImageStore,
error,
) {
store := createStoreDriver(rootDir)
@ -156,7 +159,7 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) (
}, log)
}
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay,
il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay,
dedupe, false, log, metrics, nil, store, cacheDriver)
return store, il, err
@ -164,7 +167,7 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) (
func createObjectsStoreDynamo(rootDir string, cacheDir string, dedupe bool, tableName string) (
driver.StorageDriver,
storage.ImageStore,
storageTypes.ImageStore,
error,
) {
store := createStoreDriver(rootDir)
@ -191,7 +194,7 @@ func createObjectsStoreDynamo(rootDir string, cacheDir string, dedupe bool, tabl
panic(err)
}
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay,
il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay,
dedupe, false, log, metrics, nil, store, cacheDriver)
return store, il, err
@ -730,7 +733,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) {
controller := api.NewController(conf)
So(controller, ShouldNotBeNil)
err = controller.InitImageStore(context.Background())
err = controller.InitImageStore()
So(err, ShouldBeNil)
})
@ -3781,7 +3784,7 @@ func TestInjectDedupe(t *testing.T) {
err := imgStore.DedupeBlob("blob", "digest", "newblob")
So(err, ShouldBeNil)
injected := test.InjectFailure(0)
injected := inject.InjectFailure(0)
err = imgStore.DedupeBlob("blob", "digest", "newblob")
if injected {
So(err, ShouldNotBeNil)
@ -3789,7 +3792,7 @@ func TestInjectDedupe(t *testing.T) {
So(err, ShouldBeNil)
}
injected = test.InjectFailure(1)
injected = inject.InjectFailure(1)
err = imgStore.DedupeBlob("blob", "digest", "newblob")
if injected {
So(err, ShouldNotBeNil)

7
pkg/storage/scrub.go
View file

@ -17,6 +17,7 @@ import (
"github.com/opencontainers/umoci/oci/casext"
"zotregistry.io/zot/errors"
storageTypes "zotregistry.io/zot/pkg/storage/types"
)
const (
@ -45,7 +46,7 @@ type ScrubResults struct {
func (sc StoreController) CheckAllBlobsIntegrity() (ScrubResults, error) {
results := ScrubResults{}
imageStoreList := make(map[string]ImageStore)
imageStoreList := make(map[string]storageTypes.ImageStore)
if sc.SubStore != nil {
imageStoreList = sc.SubStore
}
@ -64,7 +65,7 @@ func (sc StoreController) CheckAllBlobsIntegrity() (ScrubResults, error) {
return results, nil
}
func CheckImageStoreBlobsIntegrity(imgStore ImageStore) ([]ScrubImageResult, error) {
func CheckImageStoreBlobsIntegrity(imgStore storageTypes.ImageStore) ([]ScrubImageResult, error) {
results := []ScrubImageResult{}
repos, err := imgStore.GetRepositories()
@ -84,7 +85,7 @@ func CheckImageStoreBlobsIntegrity(imgStore ImageStore) ([]ScrubImageResult, err
return results, nil
}
func CheckRepo(imageName string, imgStore ImageStore) ([]ScrubImageResult, error) {
func CheckRepo(imageName string, imgStore storageTypes.ImageStore) ([]ScrubImageResult, error) {
results := []ScrubImageResult{}
dir := path.Join(imgStore.RootDir(), imageName)

10
pkg/storage/scrub_test.go
View file

@ -17,6 +17,8 @@ import (
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
common "zotregistry.io/zot/pkg/storage/common"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/test"
)
@ -37,7 +39,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay,
imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay,
true, true, log, metrics, nil, cacheDriver)
Convey("Scrub only one repo", t, func(c C) {
@ -111,7 +113,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
// verify error message
So(actual, ShouldContainSubstring, "test 1.0 affected parse application/vnd.oci.image.manifest.v1+json")
index, err := storage.GetIndex(imgStore, repoName, log.With().Caller().Logger())
index, err := common.GetIndex(imgStore, repoName, log.With().Caller().Logger())
So(err, ShouldBeNil)
So(len(index.Manifests), ShouldEqual, 1)
@ -191,7 +193,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
err = os.Chmod(layerFile, 0x0200)
So(err, ShouldBeNil)
index, err := storage.GetIndex(imgStore, repoName, log.With().Caller().Logger())
index, err := common.GetIndex(imgStore, repoName, log.With().Caller().Logger())
So(err, ShouldBeNil)
So(len(index.Manifests), ShouldEqual, 1)
@ -325,7 +327,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
So(actual, ShouldContainSubstring, "test 1.0 affected")
So(actual, ShouldContainSubstring, "no such file or directory")
index, err := storage.GetIndex(imgStore, repoName, log.With().Caller().Logger())
index, err := common.GetIndex(imgStore, repoName, log.With().Caller().Logger())
So(err, ShouldBeNil)
So(len(index.Manifests), ShouldEqual, 2)

284
pkg/storage/storage.go
View file

@ -1,59 +1,247 @@
package storage
import (
"io"
"time"
"encoding/json"
"fmt"
"strings"
"github.com/docker/distribution/registry/storage/driver/factory"
"github.com/gobwas/glob"
notreg "github.com/notaryproject/notation-go/registry"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"zotregistry.io/zot/pkg/scheduler"
"zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/api/config"
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
common "zotregistry.io/zot/pkg/storage/common"
"zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/storage/s3"
storageTypes "zotregistry.io/zot/pkg/storage/types"
)
const (
S3StorageDriverName = "s3"
DefaultGCDelay = 1 * time.Hour
)
func New(config *config.Config, linter common.Lint, metrics monitoring.MetricServer,
log log.Logger,
) (StoreController, error) {
storeController := StoreController{}
type ImageStore interface { //nolint:interfacebloat
DirExists(d string) bool
RootDir() string
RLock(*time.Time)
RUnlock(*time.Time)
Lock(*time.Time)
Unlock(*time.Time)
InitRepo(name string) error
ValidateRepo(name string) (bool, error)
GetRepositories() ([]string, error)
GetNextRepository(repo string) (string, error)
GetImageTags(repo string) ([]string, error)
GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error)
PutImageManifest(repo, reference, mediaType string, body []byte) (godigest.Digest, godigest.Digest, error)
DeleteImageManifest(repo, reference string, detectCollision bool) error
BlobUploadPath(repo, uuid string) string
NewBlobUpload(repo string) (string, error)
GetBlobUpload(repo, uuid string) (int64, error)
PutBlobChunkStreamed(repo, uuid string, body io.Reader) (int64, error)
PutBlobChunk(repo, uuid string, from, to int64, body io.Reader) (int64, error)
BlobUploadInfo(repo, uuid string) (int64, error)
FinishBlobUpload(repo, uuid string, body io.Reader, digest godigest.Digest) error
FullBlobUpload(repo string, body io.Reader, digest godigest.Digest) (string, int64, error)
DedupeBlob(src string, dstDigest godigest.Digest, dst string) error
DeleteBlobUpload(repo, uuid string) error
BlobPath(repo string, digest godigest.Digest) string
CheckBlob(repo string, digest godigest.Digest) (bool, int64, error)
GetBlob(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error)
GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64,
) (io.ReadCloser, int64, int64, error)
DeleteBlob(repo string, digest godigest.Digest) error
GetIndexContent(repo string) ([]byte, error)
GetBlobContent(repo string, digest godigest.Digest) ([]byte, error)
GetReferrers(repo string, digest godigest.Digest, artifactTypes []string) (ispec.Index, error)
GetOrasReferrers(repo string, digest godigest.Digest, artifactType string) ([]artifactspec.Descriptor, error)
RunGCRepo(repo string) error
RunGCPeriodically(interval time.Duration, sch *scheduler.Scheduler)
RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler)
RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error
GetNextDigestWithBlobPaths(lastDigests []godigest.Digest) (godigest.Digest, []string, error)
if config.Storage.RootDirectory == "" {
// we can't proceed without global storage
log.Error().Err(errors.ErrImgStoreNotFound).Msg("controller: no storage config provided")
return storeController, errors.ErrImgStoreNotFound
}
// no need to validate hard links work on s3
if config.Storage.Dedupe && config.Storage.StorageDriver == nil {
err := local.ValidateHardLink(config.Storage.RootDirectory)
if err != nil {
log.Warn().Msg("input storage root directory filesystem does not supports hardlinking," +
"disabling dedupe functionality")
config.Storage.Dedupe = false
}
}
var defaultStore storageTypes.ImageStore
if config.Storage.StorageDriver == nil {
// false positive lint - linter does not implement Lint method
//nolint:typecheck,contextcheck
defaultStore = local.NewImageStore(config.Storage.RootDirectory,
config.Storage.GC, config.Storage.GCDelay,
config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter,
CreateCacheDatabaseDriver(config.Storage.StorageConfig, log),
)
} else {
storeName := fmt.Sprintf("%v", config.Storage.StorageDriver["name"])
if storeName != constants.S3StorageDriverName {
log.Fatal().Err(errors.ErrBadConfig).Str("storageDriver", storeName).
Msg("unsupported storage driver")
}
// Init a Storager from connection string.
store, err := factory.Create(storeName, config.Storage.StorageDriver)
if err != nil {
log.Error().Err(err).Str("rootDir", config.Storage.RootDirectory).Msg("unable to create s3 service")
return storeController, err
}
/* in the case of s3 config.Storage.RootDirectory is used for caching blobs locally and
config.Storage.StorageDriver["rootdirectory"] is the actual rootDir in s3 */
rootDir := "/"
if config.Storage.StorageDriver["rootdirectory"] != nil {
rootDir = fmt.Sprintf("%v", config.Storage.StorageDriver["rootdirectory"])
}
// false positive lint - linter does not implement Lint method
//nolint: typecheck,contextcheck
defaultStore = s3.NewImageStore(rootDir, config.Storage.RootDirectory,
config.Storage.GC, config.Storage.GCDelay, config.Storage.Dedupe,
config.Storage.Commit, log, metrics, linter, store,
CreateCacheDatabaseDriver(config.Storage.StorageConfig, log))
}
storeController.DefaultStore = defaultStore
if config.Storage.SubPaths != nil {
if len(config.Storage.SubPaths) > 0 {
subPaths := config.Storage.SubPaths
//nolint: contextcheck
subImageStore, err := getSubStore(config, subPaths, linter, metrics, log)
if err != nil {
log.Error().Err(err).Msg("controller: error getting sub image store")
return storeController, err
}
storeController.SubStore = subImageStore
}
}
return storeController, nil
}
func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig,
linter common.Lint, metrics monitoring.MetricServer, log log.Logger,
) (map[string]storageTypes.ImageStore, error) {
imgStoreMap := make(map[string]storageTypes.ImageStore, 0)
subImageStore := make(map[string]storageTypes.ImageStore)
// creating image store per subpaths
for route, storageConfig := range subPaths {
// no need to validate hard links work on s3
if storageConfig.Dedupe && storageConfig.StorageDriver == nil {
err := local.ValidateHardLink(storageConfig.RootDirectory)
if err != nil {
log.Warn().Msg("input storage root directory filesystem does not supports hardlinking, " +
"disabling dedupe functionality")
storageConfig.Dedupe = false
}
}
if storageConfig.StorageDriver == nil {
// Compare if subpath root dir is same as default root dir
isSame, _ := config.SameFile(cfg.Storage.RootDirectory, storageConfig.RootDirectory)
if isSame {
log.Error().Err(errors.ErrBadConfig).Msg("sub path storage directory is same as root directory")
return nil, errors.ErrBadConfig
}
isUnique := true
// Compare subpath unique files
for file := range imgStoreMap {
// We already have image storage for this file
if compareImageStore(file, storageConfig.RootDirectory) {
subImageStore[route] = imgStoreMap[file]
isUnique = false
}
}
// subpath root directory is unique
// add it to uniqueSubFiles
// Create a new image store and assign it to imgStoreMap
if isUnique {
imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(storageConfig.RootDirectory,
storageConfig.GC, storageConfig.GCDelay, storageConfig.Dedupe,
storageConfig.Commit, log, metrics, linter, CreateCacheDatabaseDriver(storageConfig, log))
subImageStore[route] = imgStoreMap[storageConfig.RootDirectory]
}
} else {
storeName := fmt.Sprintf("%v", storageConfig.StorageDriver["name"])
if storeName != constants.S3StorageDriverName {
log.Fatal().Err(errors.ErrBadConfig).Str("storageDriver", storeName).
Msg("unsupported storage driver")
}
// Init a Storager from connection string.
store, err := factory.Create(storeName, storageConfig.StorageDriver)
if err != nil {
log.Error().Err(err).Str("rootDir", storageConfig.RootDirectory).Msg("Unable to create s3 service")
return nil, err
}
/* in the case of s3 c.Config.Storage.RootDirectory is used for caching blobs locally and
c.Config.Storage.StorageDriver["rootdirectory"] is the actual rootDir in s3 */
rootDir := "/"
if cfg.Storage.StorageDriver["rootdirectory"] != nil {
rootDir = fmt.Sprintf("%v", cfg.Storage.StorageDriver["rootdirectory"])
}
// false positive lint - linter does not implement Lint method
//nolint: typecheck
subImageStore[route] = s3.NewImageStore(rootDir, storageConfig.RootDirectory,
storageConfig.GC, storageConfig.GCDelay,
storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store,
CreateCacheDatabaseDriver(storageConfig, log),
)
}
}
return subImageStore, nil
}
func compareImageStore(root1, root2 string) bool {
isSameFile, err := config.SameFile(root1, root2)
// This error is path error that means either of root directory doesn't exist, in that case do string match
if err != nil {
return strings.EqualFold(root1, root2)
}
return isSameFile
}
// CheckIsImageSignature checks if the given image (repo:tag) represents a signature. The function
// returns:
//
// - bool: if the image is a signature or not
//
// - string: the type of signature
//
// - string: the digest of the image it signs
//
// - error: any errors that occur.
func CheckIsImageSignature(repoName string, manifestBlob []byte, reference string,
) (bool, string, godigest.Digest, error) {
var manifestContent ispec.Manifest
err := json.Unmarshal(manifestBlob, &manifestContent)
if err != nil {
return false, "", "", err
}
manifestArtifactType := zcommon.GetManifestArtifactType(manifestContent)
// check notation signature
if manifestArtifactType == notreg.ArtifactTypeNotation && manifestContent.Subject != nil {
return true, NotationType, manifestContent.Subject.Digest, nil
}
// check cosign
cosignTagRule := glob.MustCompile("sha256-*.sig")
if tag := reference; cosignTagRule.Match(reference) {
prefixLen := len("sha256-")
digestLen := 64
signedImageManifestDigestEncoded := tag[prefixLen : prefixLen+digestLen]
signedImageManifestDigest := godigest.NewDigestFromEncoded(godigest.SHA256,
signedImageManifestDigestEncoded)
return true, CosignType, signedImageManifestDigest, nil
}
return false, "", "", nil
}

19
pkg/storage/storage_controller.go
View file

@ -3,17 +3,18 @@ package storage
import (
"fmt"
"strings"
storageTypes "zotregistry.io/zot/pkg/storage/types"
)
const (
CosignType = "cosign"
NotationType = "notation"
)
type StoreController struct {
DefaultStore ImageStore
SubStore map[string]ImageStore
}
// BlobUpload models and upload request.
type BlobUpload struct {
StoreName string
ID string
DefaultStore storageTypes.ImageStore
SubStore map[string]storageTypes.ImageStore
}
func GetRoutePrefix(name string) string {
@ -29,7 +30,7 @@ func GetRoutePrefix(name string) string {
return fmt.Sprintf("/%s", names[0])
}
func (sc StoreController) GetImageStore(name string) ImageStore {
func (sc StoreController) GetImageStore(name string) storageTypes.ImageStore {
if sc.SubStore != nil {
// SubStore is being provided, now we need to find equivalent image store and this will be found by splitting name
prefixName := GetRoutePrefix(name)

38
pkg/storage/storage_test.go
View file

@ -30,8 +30,10 @@ import (
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/storage/s3"
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
)
@ -48,7 +50,7 @@ func skipIt(t *testing.T) {
}
}
func createObjectsStore(rootDir string, cacheDir string) (driver.StorageDriver, storage.ImageStore, error) {
func createObjectsStore(rootDir string, cacheDir string) (driver.StorageDriver, storageTypes.ImageStore, error) {
bucket := "zot-storage-test"
endpoint := os.Getenv("S3MOCK_ENDPOINT")
storageDriverParams := map[string]interface{}{
@ -85,7 +87,7 @@ func createObjectsStore(rootDir string, cacheDir string) (driver.StorageDriver,
UseRelPaths: false,
}, log)
il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay,
il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay,
true, false, log, metrics, nil, store, cacheDriver,
)
@ -111,7 +113,7 @@ func TestStorageAPIs(t *testing.T) {
for _, testcase := range testCases {
testcase := testcase
t.Run(testcase.testCaseName, func(t *testing.T) {
var imgStore storage.ImageStore
var imgStore storageTypes.ImageStore
if testcase.storageType == "s3" {
skipIt(t)
@ -136,7 +138,7 @@ func TestStorageAPIs(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore = local.NewImageStore(dir, true, storage.DefaultGCDelay, true,
imgStore = local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true,
true, log, metrics, nil, cacheDriver)
}
@ -710,7 +712,7 @@ func TestMandatoryAnnotations(t *testing.T) {
for _, testcase := range testCases {
testcase := testcase
t.Run(testcase.testCaseName, func(t *testing.T) {
var imgStore storage.ImageStore
var imgStore storageTypes.ImageStore
var testDir, tdir string
var store driver.StorageDriver
@ -731,7 +733,7 @@ func TestMandatoryAnnotations(t *testing.T) {
store, _, _ = createObjectsStore(testDir, tdir)
imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics,
&mocks.MockedLint{
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) {
return false, nil
},
}, store, nil)
@ -744,9 +746,9 @@ func TestMandatoryAnnotations(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore = local.NewImageStore(tdir, true, storage.DefaultGCDelay, true,
imgStore = local.NewImageStore(tdir, true, storageConstants.DefaultGCDelay, true,
true, log, metrics, &mocks.MockedLint{
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) {
return false, nil
},
}, cacheDriver)
@ -798,7 +800,7 @@ func TestMandatoryAnnotations(t *testing.T) {
if testcase.storageType == "s3" {
imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics,
&mocks.MockedLint{
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) {
//nolint: goerr113
return false, errors.New("linter error")
},
@ -809,9 +811,9 @@ func TestMandatoryAnnotations(t *testing.T) {
Name: "cache",
UseRelPaths: true,
}, log)
imgStore = local.NewImageStore(tdir, true, storage.DefaultGCDelay, true,
imgStore = local.NewImageStore(tdir, true, storageConstants.DefaultGCDelay, true,
true, log, metrics, &mocks.MockedLint{
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) {
//nolint: goerr113
return false, errors.New("linter error")
},
@ -830,9 +832,9 @@ func TestStorageHandler(t *testing.T) {
for _, testcase := range testCases {
testcase := testcase
t.Run(testcase.testCaseName, func(t *testing.T) {
var firstStore storage.ImageStore
var secondStore storage.ImageStore
var thirdStore storage.ImageStore
var firstStore storageTypes.ImageStore
var secondStore storageTypes.ImageStore
var thirdStore storageTypes.ImageStore
var firstRootDir string
var secondRootDir string
var thirdRootDir string
@ -865,13 +867,13 @@ func TestStorageHandler(t *testing.T) {
metrics := monitoring.NewMetricsServer(false, log)
// Create ImageStore
firstStore = local.NewImageStore(firstRootDir, false, storage.DefaultGCDelay,
firstStore = local.NewImageStore(firstRootDir, false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil)
secondStore = local.NewImageStore(secondRootDir, false,
storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil)
thirdStore = local.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay,
thirdStore = local.NewImageStore(thirdRootDir, false, storageConstants.DefaultGCDelay,
false, false, log, metrics, nil, nil)
}
@ -880,7 +882,7 @@ func TestStorageHandler(t *testing.T) {
storeController.DefaultStore = firstStore
subStore := make(map[string]storage.ImageStore)
subStore := make(map[string]storageTypes.ImageStore)
subStore["/a"] = secondStore
subStore["/b"] = thirdStore

54
pkg/storage/types/types.go Normal file
View file

@ -0,0 +1,54 @@
package types
import (
"io"
"time"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"zotregistry.io/zot/pkg/scheduler"
)
type ImageStore interface { //nolint:interfacebloat
DirExists(d string) bool
RootDir() string
RLock(*time.Time)
RUnlock(*time.Time)
Lock(*time.Time)
Unlock(*time.Time)
InitRepo(name string) error
ValidateRepo(name string) (bool, error)
GetRepositories() ([]string, error)
GetNextRepository(repo string) (string, error)
GetImageTags(repo string) ([]string, error)
GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error)
PutImageManifest(repo, reference, mediaType string, body []byte) (godigest.Digest, godigest.Digest, error)
DeleteImageManifest(repo, reference string, detectCollision bool) error
BlobUploadPath(repo, uuid string) string
NewBlobUpload(repo string) (string, error)
GetBlobUpload(repo, uuid string) (int64, error)
PutBlobChunkStreamed(repo, uuid string, body io.Reader) (int64, error)
PutBlobChunk(repo, uuid string, from, to int64, body io.Reader) (int64, error)
BlobUploadInfo(repo, uuid string) (int64, error)
FinishBlobUpload(repo, uuid string, body io.Reader, digest godigest.Digest) error
FullBlobUpload(repo string, body io.Reader, digest godigest.Digest) (string, int64, error)
DedupeBlob(src string, dstDigest godigest.Digest, dst string) error
DeleteBlobUpload(repo, uuid string) error
BlobPath(repo string, digest godigest.Digest) string
CheckBlob(repo string, digest godigest.Digest) (bool, int64, error)
GetBlob(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error)
GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64,
) (io.ReadCloser, int64, int64, error)
DeleteBlob(repo string, digest godigest.Digest) error
GetIndexContent(repo string) ([]byte, error)
GetBlobContent(repo string, digest godigest.Digest) ([]byte, error)
GetReferrers(repo string, digest godigest.Digest, artifactTypes []string) (ispec.Index, error)
GetOrasReferrers(repo string, digest godigest.Digest, artifactType string) ([]artifactspec.Descriptor, error)
RunGCRepo(repo string) error
RunGCPeriodically(interval time.Duration, sch *scheduler.Scheduler)
RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler)
RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error
GetNextDigestWithBlobPaths(lastDigests []godigest.Digest) (godigest.Digest, []string, error)
}

37
pkg/test/common.go
View file

@ -47,6 +47,7 @@ import (
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/test/inject"
)
const (
@ -549,7 +550,7 @@ func GetImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manifest, e
}
configBlob, err := json.Marshal(config)
if err = Error(err); err != nil {
if err = inject.Error(err); err != nil {
return ispec.Image{}, [][]byte{}, ispec.Manifest{}, err
}
@ -597,7 +598,7 @@ func GetRandomImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manif
}
configBlob, err := json.Marshal(config)
if err = Error(err); err != nil {
if err = inject.Error(err); err != nil {
return ispec.Image{}, [][]byte{}, ispec.Manifest{}, err
}
@ -665,7 +666,7 @@ func GetRandomImage(reference string) (Image, error) {
func GetImageComponentsWithConfig(conf ispec.Image) (ispec.Image, [][]byte, ispec.Manifest, error) {
configBlob, err := json.Marshal(conf)
if err = Error(err); err != nil {
if err = inject.Error(err); err != nil {
return ispec.Image{}, [][]byte{}, ispec.Manifest{}, err
}
@ -843,7 +844,7 @@ func UploadImage(img Image, baseURL, repo string) error {
}
// upload config
cblob, err := json.Marshal(img.Config)
if err = Error(err); err != nil {
if err = inject.Error(err); err != nil {
return err
}
@ -856,11 +857,11 @@ func UploadImage(img Image, baseURL, repo string) error {
resp, err := resty.R().
Post(baseURL + "/v2/" + repo + "/blobs/uploads/")
if err = Error(err); err != nil {
if err = inject.Error(err); err != nil {
return err
}
if ErrStatusCode(resp.StatusCode()) != http.StatusAccepted || ErrStatusCode(resp.StatusCode()) == -1 {
if inject.ErrStatusCode(resp.StatusCode()) != http.StatusAccepted || inject.ErrStatusCode(resp.StatusCode()) == -1 {
return ErrPostBlob
}
@ -873,17 +874,17 @@ func UploadImage(img Image, baseURL, repo string) error {
SetQueryParam("digest", cdigest.String()).
SetBody(cblob).
Put(loc)
if err = Error(err); err != nil {
if err = inject.Error(err); err != nil {
return err
}
if ErrStatusCode(resp.StatusCode()) != http.StatusCreated || ErrStatusCode(resp.StatusCode()) == -1 {
if inject.ErrStatusCode(resp.StatusCode()) != http.StatusCreated || inject.ErrStatusCode(resp.StatusCode()) == -1 {
return ErrPostBlob
}
// put manifest
manifestBlob, err := json.Marshal(img.Manifest)
if err = Error(err); err != nil {
if err = inject.Error(err); err != nil {
return err
}
@ -892,11 +893,11 @@ func UploadImage(img Image, baseURL, repo string) error {
SetBody(manifestBlob).
Put(baseURL + "/v2/" + repo + "/manifests/" + img.Reference)
if ErrStatusCode(resp.StatusCode()) != http.StatusCreated {
if inject.ErrStatusCode(resp.StatusCode()) != http.StatusCreated {
return ErrPutBlob
}
if ErrStatusCode(resp.StatusCode()) != http.StatusCreated {
if inject.ErrStatusCode(resp.StatusCode()) != http.StatusCreated {
return ErrPutBlob
}
@ -1512,7 +1513,7 @@ func UploadImageWithBasicAuth(img Image, baseURL, repo, user, password string) e
}
// upload config
cblob, err := json.Marshal(img.Config)
if err = Error(err); err != nil {
if err = inject.Error(err); err != nil {
return err
}
@ -1526,11 +1527,11 @@ func UploadImageWithBasicAuth(img Image, baseURL, repo, user, password string) e
resp, err := resty.R().
SetBasicAuth(user, password).
Post(baseURL + "/v2/" + repo + "/blobs/uploads/")
if err = Error(err); err != nil {
if err = inject.Error(err); err != nil {
return err
}
if ErrStatusCode(resp.StatusCode()) != http.StatusAccepted || ErrStatusCode(resp.StatusCode()) == -1 {
if inject.ErrStatusCode(resp.StatusCode()) != http.StatusAccepted || inject.ErrStatusCode(resp.StatusCode()) == -1 {
return ErrPostBlob
}
@ -1544,17 +1545,17 @@ func UploadImageWithBasicAuth(img Image, baseURL, repo, user, password string) e
SetQueryParam("digest", cdigest.String()).
SetBody(cblob).
Put(loc)
if err = Error(err); err != nil {
if err = inject.Error(err); err != nil {
return err
}
if ErrStatusCode(resp.StatusCode()) != http.StatusCreated || ErrStatusCode(resp.StatusCode()) == -1 {
if inject.ErrStatusCode(resp.StatusCode()) != http.StatusCreated || inject.ErrStatusCode(resp.StatusCode()) == -1 {
return ErrPostBlob
}
// put manifest
manifestBlob, err := json.Marshal(img.Manifest)
if err = Error(err); err != nil {
if err = inject.Error(err); err != nil {
return err
}
@ -1795,7 +1796,7 @@ func UploadMultiarchImage(multiImage MultiarchImage, baseURL string, repo string
// put manifest
indexBlob, err := json.Marshal(multiImage.Index)
if err = Error(err); err != nil {
if err = inject.Error(err); err != nil {
return err
}

43
pkg/test/common_test.go
View file

@ -24,6 +24,7 @@ import (
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/inject"
"zotregistry.io/zot/pkg/test/mocks"
)
@ -174,7 +175,7 @@ func TestGetOciLayoutDigests(t *testing.T) {
func TestGetImageComponents(t *testing.T) {
Convey("Inject failures for unreachable lines", t, func() {
injected := test.InjectFailure(0)
injected := inject.InjectFailure(0)
if injected {
_, _, _, err := test.GetImageComponents(100)
So(err, ShouldNotBeNil)
@ -186,6 +187,26 @@ func TestGetImageComponents(t *testing.T) {
})
}
func TestGetRandomImageComponents(t *testing.T) {
Convey("Inject failures for unreachable lines", t, func() {
injected := inject.InjectFailure(0)
if injected {
_, _, _, err := test.GetRandomImageComponents(100)
So(err, ShouldNotBeNil)
}
})
}
func TestGetImageComponentsWithConfig(t *testing.T) {
Convey("Inject failures for unreachable lines", t, func() {
injected := inject.InjectFailure(0)
if injected {
_, _, _, err := test.GetImageComponentsWithConfig(ispec.Image{})
So(err, ShouldNotBeNil)
}
})
}
func TestWaitTillTrivyDBDownloadStarted(t *testing.T) {
Convey("finishes successfully", t, func() {
tempDir := t.TempDir()
@ -601,14 +622,14 @@ func TestUploadImage(t *testing.T) {
}
Convey("CreateBlobUpload", func() {
injected := test.InjectFailure(2)
injected := inject.InjectFailure(2)
if injected {
err := test.UploadImage(img, baseURL, "test")
So(err, ShouldNotBeNil)
}
})
Convey("UpdateBlobUpload", func() {
injected := test.InjectFailure(4)
injected := inject.InjectFailure(4)
if injected {
err := test.UploadImage(img, baseURL, "test")
So(err, ShouldNotBeNil)
@ -662,28 +683,28 @@ func TestInjectUploadImage(t *testing.T) {
}
Convey("first marshal", func() {
injected := test.InjectFailure(0)
injected := inject.InjectFailure(0)
if injected {
err := test.UploadImage(img, baseURL, "test")
So(err, ShouldNotBeNil)
}
})
Convey("CreateBlobUpload POST call", func() {
injected := test.InjectFailure(1)
injected := inject.InjectFailure(1)
if injected {
err := test.UploadImage(img, baseURL, "test")
So(err, ShouldNotBeNil)
}
})
Convey("UpdateBlobUpload PUT call", func() {
injected := test.InjectFailure(3)
injected := inject.InjectFailure(3)
if injected {
err := test.UploadImage(img, baseURL, "test")
So(err, ShouldNotBeNil)
}
})
Convey("second marshal", func() {
injected := test.InjectFailure(5)
injected := inject.InjectFailure(5)
if injected {
err := test.UploadImage(img, baseURL, "test")
So(err, ShouldNotBeNil)
@ -791,28 +812,28 @@ func TestInjectUploadImageWithBasicAuth(t *testing.T) {
}
Convey("first marshal", func() {
injected := test.InjectFailure(0)
injected := inject.InjectFailure(0)
if injected {
err := test.UploadImageWithBasicAuth(img, baseURL, "test", "user", "password")
So(err, ShouldNotBeNil)
}
})
Convey("CreateBlobUpload POST call", func() {
injected := test.InjectFailure(1)
injected := inject.InjectFailure(1)
if injected {
err := test.UploadImageWithBasicAuth(img, baseURL, "test", "user", "password")
So(err, ShouldNotBeNil)
}
})
Convey("UpdateBlobUpload PUT call", func() {
injected := test.InjectFailure(3)
injected := inject.InjectFailure(3)
if injected {
err := test.UploadImageWithBasicAuth(img, baseURL, "test", "user", "password")
So(err, ShouldNotBeNil)
}
})
Convey("second marshal", func() {
injected := test.InjectFailure(5)
injected := inject.InjectFailure(5)
if injected {
err := test.UploadImageWithBasicAuth(img, baseURL, "test", "user", "password")
So(err, ShouldNotBeNil)

2
pkg/test/dev.go → pkg/test/inject/dev.go
View file

@ -3,7 +3,7 @@
// This file should be linked only in **development** mode.
package test
package inject
import (
"net/http"

42
pkg/test/inject_test.go → pkg/test/inject/inject_test.go
View file

@ -3,7 +3,7 @@
// This file should be linked only in **development** mode.
package test_test
package inject_test
import (
"errors"
@ -11,7 +11,7 @@ import (
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/inject"
)
var (
@ -26,12 +26,12 @@ func foo() error {
fmap := map[string]string{"key1": "val1", "key2": "val2"}
_, ok := fmap["key1"] // should never fail
if !test.Ok(ok) {
if !inject.Ok(ok) {
return errKey1
}
_, ok = fmap["key2"] // should never fail
if !test.Ok(ok) {
if !inject.Ok(ok) {
return errKey2
}
@ -48,12 +48,12 @@ func errgen(i int) error {
func bar() error {
err := errgen(0) // should never fail
if test.Error(err) != nil {
if inject.Error(err) != nil {
return errCall1
}
err = errgen(0) // should never fail
if test.Error(err) != nil {
if inject.Error(err) != nil {
return errCall2
}
@ -61,11 +61,11 @@ func bar() error {
}
func baz() error {
if test.ErrStatusCode(0) != 0 {
if inject.ErrStatusCode(0) != 0 {
return errCall1
}
if test.ErrStatusCode(0) != 0 {
if inject.ErrStatusCode(0) != 0 {
return errCall2
}
@ -88,15 +88,15 @@ func TestInject(t *testing.T) {
Convey("Check Ok", func() {
Convey("Without skipping", func() {
test.InjectFailure(0) // inject a failure
inject.InjectFailure(0) // inject a failure
err := foo() // should be a failure
So(err, ShouldNotBeNil) // should be a failure
So(errors.Is(err, errKey1), ShouldBeTrue)
})
Convey("With skipping", func() {
test.InjectFailure(1) // inject a failure but skip first one
err := foo() // should be a failure
inject.InjectFailure(1) // inject a failure but skip first one
err := foo() // should be a failure
So(errors.Is(err, errKey1), ShouldBeFalse)
So(errors.Is(err, errKey2), ShouldBeTrue)
})
@ -108,15 +108,15 @@ func TestInject(t *testing.T) {
Convey("Check Err", func() {
Convey("Without skipping", func() {
test.InjectFailure(0) // inject a failure
inject.InjectFailure(0) // inject a failure
err := bar() // should be a failure
So(err, ShouldNotBeNil) // should be a failure
So(errors.Is(err, errCall1), ShouldBeTrue)
})
Convey("With skipping", func() {
test.InjectFailure(1) // inject a failure but skip first one
err := bar() // should be a failure
inject.InjectFailure(1) // inject a failure but skip first one
err := bar() // should be a failure
So(errors.Is(err, errCall1), ShouldBeFalse)
So(errors.Is(err, errCall2), ShouldBeTrue)
})
@ -124,15 +124,15 @@ func TestInject(t *testing.T) {
Convey("Check ErrStatusCode", func() {
Convey("Without skipping", func() {
test.InjectFailure(0) // inject a failure
inject.InjectFailure(0) // inject a failure
err := baz() // should be a failure
So(err, ShouldNotBeNil) // should be a failure
So(errors.Is(err, errCall1), ShouldBeTrue)
})
Convey("With skipping", func() {
test.InjectFailure(1) // inject a failure but skip first one
err := baz() // should be a failure
inject.InjectFailure(1) // inject a failure but skip first one
err := baz() // should be a failure
So(errors.Is(err, errCall1), ShouldBeFalse)
So(errors.Is(err, errCall2), ShouldBeTrue)
})
@ -141,14 +141,14 @@ func TestInject(t *testing.T) {
Convey("Without injected failure", t, func(c C) {
err := alwaysErr()
So(test.Error(err), ShouldNotBeNil)
So(inject.Error(err), ShouldNotBeNil)
ok := alwaysNotOk()
So(test.Ok(ok), ShouldBeFalse)
So(inject.Ok(ok), ShouldBeFalse)
})
Convey("Incomplete injected failure", t, func(c C) {
test.InjectFailure(0) // inject a failure
So(func() { test.InjectFailure(0) }, ShouldPanic)
inject.InjectFailure(0) // inject a failure
So(func() { inject.InjectFailure(0) }, ShouldPanic)
})
}

2
pkg/test/prod.go → pkg/test/inject/prod.go
View file

@ -1,7 +1,7 @@
//go:build !dev
// +build !dev
package test
package inject
func Error(err error) error {
return err

7
pkg/test/mocks/lint_mock.go
View file

@ -3,14 +3,15 @@ package mocks
import (
godigest "github.com/opencontainers/go-digest"
"zotregistry.io/zot/pkg/storage"
storageTypes "zotregistry.io/zot/pkg/storage/types"
)
type MockedLint struct {
LintFn func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error)
LintFn func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error)
}
func (lint MockedLint) Lint(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) {
func (lint MockedLint) Lint(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore,
) (bool, error) {
if lint.LintFn != nil {
return lint.LintFn(repo, manifestDigest, imageStore)
}