diff --git a/pkg/api/config/config.go b/pkg/api/config/config.go index 0fa7fc2b..7edb5727 100644 --- a/pkg/api/config/config.go +++ b/pkg/api/config/config.go @@ -8,7 +8,7 @@ import ( distspec "github.com/opencontainers/distribution-spec/specs-go" extconf "zotregistry.io/zot/pkg/extensions/config" - "zotregistry.io/zot/pkg/storage" + storageConstants "zotregistry.io/zot/pkg/storage/constants" ) var ( @@ -147,7 +147,7 @@ func New() *Config { ReleaseTag: ReleaseTag, BinaryType: BinaryType, Storage: GlobalStorageConfig{ - StorageConfig: StorageConfig{GC: true, GCDelay: storage.DefaultGCDelay, Dedupe: true}, + StorageConfig: StorageConfig{GC: true, GCDelay: storageConstants.DefaultGCDelay, Dedupe: true}, }, HTTP: HTTPConfig{Address: "127.0.0.1", Port: "8080", Auth: &AuthConfig{FailDelay: 0}}, Log: &LogConfig{Level: "debug"}, diff --git a/pkg/api/controller.go b/pkg/api/controller.go index 909d20fd..1040b5e3 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -14,7 +14,6 @@ import ( "syscall" "time" - "github.com/docker/distribution/registry/storage/driver/factory" "github.com/gorilla/handlers" "github.com/gorilla/mux" @@ -27,10 +26,6 @@ import ( "zotregistry.io/zot/pkg/meta/repodb/repodbfactory" "zotregistry.io/zot/pkg/scheduler" "zotregistry.io/zot/pkg/storage" - "zotregistry.io/zot/pkg/storage/cache" - "zotregistry.io/zot/pkg/storage/constants" - "zotregistry.io/zot/pkg/storage/local" - "zotregistry.io/zot/pkg/storage/s3" ) const ( @@ -224,7 +219,7 @@ func (c *Controller) Init(reloadCtx context.Context) error { c.Metrics = monitoring.NewMetricsServer(enabled, c.Log) - if err := c.InitImageStore(reloadCtx); err != nil { + if err := c.InitImageStore(); err != nil { //nolint:contextcheck return err } @@ -244,235 +239,15 @@ func (c *Controller) InitCVEInfo() { } } -func (c *Controller) InitImageStore(ctx context.Context) error { - c.StoreController = storage.StoreController{} - +func (c *Controller) InitImageStore() error { linter := ext.GetLinter(c.Config, c.Log) - if c.Config.Storage.RootDirectory != "" { - // no need to validate hard links work on s3 - if c.Config.Storage.Dedupe && c.Config.Storage.StorageDriver == nil { - err := local.ValidateHardLink(c.Config.Storage.RootDirectory) - if err != nil { - c.Log.Warn().Msg("input storage root directory filesystem does not supports hardlinking," + - "disabling dedupe functionality") - - c.Config.Storage.Dedupe = false - } - } - - var defaultStore storage.ImageStore - if c.Config.Storage.StorageDriver == nil { - // false positive lint - linter does not implement Lint method - //nolint:typecheck,contextcheck - defaultStore = local.NewImageStore(c.Config.Storage.RootDirectory, - c.Config.Storage.GC, c.Config.Storage.GCDelay, - c.Config.Storage.Dedupe, c.Config.Storage.Commit, c.Log, c.Metrics, linter, - CreateCacheDatabaseDriver(c.Config.Storage.StorageConfig, c.Log), - ) - } else { - storeName := fmt.Sprintf("%v", c.Config.Storage.StorageDriver["name"]) - if storeName != storage.S3StorageDriverName { - c.Log.Fatal().Err(errors.ErrBadConfig).Str("storageDriver", storeName). - Msg("unsupported storage driver") - } - // Init a Storager from connection string. - store, err := factory.Create(storeName, c.Config.Storage.StorageDriver) - if err != nil { - c.Log.Error().Err(err).Str("rootDir", c.Config.Storage.RootDirectory).Msg("unable to create s3 service") - - return err - } - - /* in the case of s3 c.Config.Storage.RootDirectory is used for caching blobs locally and - c.Config.Storage.StorageDriver["rootdirectory"] is the actual rootDir in s3 */ - rootDir := "/" - if c.Config.Storage.StorageDriver["rootdirectory"] != nil { - rootDir = fmt.Sprintf("%v", c.Config.Storage.StorageDriver["rootdirectory"]) - } - - // false positive lint - linter does not implement Lint method - //nolint: typecheck,contextcheck - defaultStore = s3.NewImageStore(rootDir, c.Config.Storage.RootDirectory, - c.Config.Storage.GC, c.Config.Storage.GCDelay, c.Config.Storage.Dedupe, - c.Config.Storage.Commit, c.Log, c.Metrics, linter, store, - CreateCacheDatabaseDriver(c.Config.Storage.StorageConfig, c.Log)) - } - - c.StoreController.DefaultStore = defaultStore - } else { - // we can't proceed without global storage - c.Log.Error().Err(errors.ErrImgStoreNotFound).Msg("controller: no storage config provided") - - return errors.ErrImgStoreNotFound - } - - if c.Config.Storage.SubPaths != nil { - if len(c.Config.Storage.SubPaths) > 0 { - subPaths := c.Config.Storage.SubPaths - - //nolint: contextcheck - subImageStore, err := c.getSubStore(subPaths, linter) - if err != nil { - c.Log.Error().Err(err).Msg("controller: error getting sub image store") - - return err - } - - c.StoreController.SubStore = subImageStore - } - } - - return nil -} - -func (c *Controller) getSubStore(subPaths map[string]config.StorageConfig, - linter storage.Lint, -) (map[string]storage.ImageStore, error) { - imgStoreMap := make(map[string]storage.ImageStore, 0) - - subImageStore := make(map[string]storage.ImageStore) - - // creating image store per subpaths - for route, storageConfig := range subPaths { - // no need to validate hard links work on s3 - if storageConfig.Dedupe && storageConfig.StorageDriver == nil { - err := local.ValidateHardLink(storageConfig.RootDirectory) - if err != nil { - c.Log.Warn().Msg("input storage root directory filesystem does not supports hardlinking, " + - "disabling dedupe functionality") - - storageConfig.Dedupe = false - } - } - - if storageConfig.StorageDriver == nil { - // Compare if subpath root dir is same as default root dir - isSame, _ := config.SameFile(c.Config.Storage.RootDirectory, storageConfig.RootDirectory) - - if isSame { - c.Log.Error().Err(errors.ErrBadConfig).Msg("sub path storage directory is same as root directory") - - return nil, errors.ErrBadConfig - } - - isUnique := true - - // Compare subpath unique files - for file := range imgStoreMap { - // We already have image storage for this file - if compareImageStore(file, storageConfig.RootDirectory) { - subImageStore[route] = imgStoreMap[file] - - isUnique = true - } - } - - // subpath root directory is unique - // add it to uniqueSubFiles - // Create a new image store and assign it to imgStoreMap - if isUnique { - imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(storageConfig.RootDirectory, - storageConfig.GC, storageConfig.GCDelay, storageConfig.Dedupe, - storageConfig.Commit, c.Log, c.Metrics, linter, CreateCacheDatabaseDriver(storageConfig, c.Log)) - - subImageStore[route] = imgStoreMap[storageConfig.RootDirectory] - } - } else { - storeName := fmt.Sprintf("%v", storageConfig.StorageDriver["name"]) - if storeName != storage.S3StorageDriverName { - c.Log.Fatal().Err(errors.ErrBadConfig).Str("storageDriver", storeName). - Msg("unsupported storage driver") - } - - // Init a Storager from connection string. - store, err := factory.Create(storeName, storageConfig.StorageDriver) - if err != nil { - c.Log.Error().Err(err).Str("rootDir", storageConfig.RootDirectory).Msg("Unable to create s3 service") - - return nil, err - } - - /* in the case of s3 c.Config.Storage.RootDirectory is used for caching blobs locally and - c.Config.Storage.StorageDriver["rootdirectory"] is the actual rootDir in s3 */ - rootDir := "/" - if c.Config.Storage.StorageDriver["rootdirectory"] != nil { - rootDir = fmt.Sprintf("%v", c.Config.Storage.StorageDriver["rootdirectory"]) - } - - // false positive lint - linter does not implement Lint method - //nolint: typecheck - subImageStore[route] = s3.NewImageStore(rootDir, storageConfig.RootDirectory, - storageConfig.GC, storageConfig.GCDelay, - storageConfig.Dedupe, storageConfig.Commit, c.Log, c.Metrics, linter, store, - CreateCacheDatabaseDriver(storageConfig, c.Log), - ) - } - } - - return subImageStore, nil -} - -func compareImageStore(root1, root2 string) bool { - isSameFile, err := config.SameFile(root1, root2) - // This error is path error that means either of root directory doesn't exist, in that case do string match + storeController, err := storage.New(c.Config, linter, c.Metrics, c.Log) if err != nil { - return strings.EqualFold(root1, root2) + return err } - return isSameFile -} - -func getUseRelPaths(storageConfig *config.StorageConfig) bool { - return storageConfig.StorageDriver == nil -} - -func CreateCacheDatabaseDriver(storageConfig config.StorageConfig, log log.Logger) cache.Cache { - if storageConfig.Dedupe || storageConfig.StorageDriver != nil { - if !storageConfig.RemoteCache { - params := cache.BoltDBDriverParameters{} - params.RootDir = storageConfig.RootDirectory - params.Name = constants.BoltdbName - - if storageConfig.StorageDriver != nil { - params.Name = s3.CacheDBName - } - - params.UseRelPaths = getUseRelPaths(&storageConfig) - - driver, _ := storage.Create("boltdb", params, log) - - return driver - } - - // remote cache - if storageConfig.CacheDriver != nil { - name, ok := storageConfig.CacheDriver["name"].(string) - if !ok { - log.Warn().Msg("remote cache driver name missing!") - - return nil - } - - if name != constants.DynamoDBDriverName { - log.Warn().Str("driver", name).Msg("remote cache driver unsupported!") - - return nil - } - - // dynamodb - dynamoParams := cache.DynamoDBDriverParameters{} - dynamoParams.Endpoint, _ = storageConfig.CacheDriver["endpoint"].(string) - dynamoParams.Region, _ = storageConfig.CacheDriver["region"].(string) - dynamoParams.TableName, _ = storageConfig.CacheDriver["cachetablename"].(string) - - driver, _ := storage.Create("dynamodb", dynamoParams, log) - - return driver - } - - return nil - } + c.StoreController = storeController return nil } diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index faa538ad..0eb97d48 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -51,8 +51,9 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/repodb/repodbfactory" "zotregistry.io/zot/pkg/storage" - "zotregistry.io/zot/pkg/storage/local" + storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/test" + "zotregistry.io/zot/pkg/test/inject" ) const ( @@ -121,13 +122,13 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { panic(err) } - driver := api.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) + driver := storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) So(driver, ShouldBeNil) conf.Storage.RemoteCache = true conf.Storage.RootDirectory = t.TempDir() - driver = api.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) + driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) So(driver, ShouldBeNil) }) skipDynamo(t) @@ -160,7 +161,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { "versionTablename": "Version", } - driver := api.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) + driver := storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) So(driver, ShouldNotBeNil) // negative test cases @@ -175,7 +176,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { "versionTablename": "Version", } - driver = api.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) + driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) So(driver, ShouldBeNil) conf.Storage.CacheDriver = map[string]interface{}{ @@ -189,7 +190,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { "versionTablename": "Version", } - driver = api.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) + driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) So(driver, ShouldBeNil) }) } @@ -360,7 +361,7 @@ func TestObjectStorageController(t *testing.T) { conf.HTTP.Port = port storageDriverParams := map[string]interface{}{ "rootdirectory": "zot", - "name": storage.S3StorageDriverName, + "name": storageConstants.S3StorageDriverName, } conf.Storage.StorageDriver = storageDriverParams ctlr := makeController(conf, "zot", "") @@ -380,7 +381,7 @@ func TestObjectStorageController(t *testing.T) { storageDriverParams := map[string]interface{}{ "rootdirectory": "zot", - "name": storage.S3StorageDriverName, + "name": storageConstants.S3StorageDriverName, "region": "us-east-2", "bucket": bucket, "regionendpoint": endpoint, @@ -409,7 +410,7 @@ func TestObjectStorageControllerSubPaths(t *testing.T) { storageDriverParams := map[string]interface{}{ "rootdirectory": "zot", - "name": storage.S3StorageDriverName, + "name": storageConstants.S3StorageDriverName, "region": "us-east-2", "bucket": bucket, "regionendpoint": endpoint, @@ -5533,7 +5534,7 @@ func TestManifestImageIndex(t *testing.T) { Convey("Corrupt index", func() { err = os.WriteFile(path.Join(dir, "index", "blobs", index1dgst.Algorithm().String(), index1dgst.Encoded()), - []byte("deadbeef"), local.DefaultFilePerms) + []byte("deadbeef"), storageConstants.DefaultFilePerms) So(err, ShouldBeNil) resp, err = resty.R().Delete(baseURL + fmt.Sprintf("/v2/index/manifests/%s", index1dgst)) So(err, ShouldBeNil) @@ -5906,7 +5907,7 @@ func TestInjectInterruptedImageManifest(t *testing.T) { // Testing router path: @Router /v2/{name}/manifests/{reference} [put] Convey("Uploading an image manifest blob (when injected simulates an interrupted image manifest upload)", func() { - injected := test.InjectFailure(0) + injected := inject.InjectFailure(0) request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content)) request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"}) @@ -5967,7 +5968,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) { So(digest, ShouldNotBeNil) // monolithic blob upload - injected := test.InjectFailure(0) + injected := inject.InjectFailure(0) if injected { request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, loc, bytes.NewReader(content)) tokens := strings.Split(loc, "/") @@ -6040,7 +6041,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) { // Testing router path: @Router /v2/{name}/manifests/{reference} [put] //nolint:lll // gofumpt conflicts with lll Convey("Uploading an image manifest blob (when injected simulates that PutImageManifest failed due to 'too many open files' error)", func() { - injected := test.InjectFailure(1) + injected := inject.InjectFailure(1) request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content)) request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"}) @@ -6060,7 +6061,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) { } }) Convey("when injected simulates a `too many open files` error inside PutImageManifest method of img store", func() { - injected := test.InjectFailure(2) + injected := inject.InjectFailure(2) request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content)) request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"}) @@ -6081,7 +6082,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) { } }) Convey("code coverage: error inside PutImageManifest method of img store (unable to marshal JSON)", func() { - injected := test.InjectFailure(1) + injected := inject.InjectFailure(1) request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content)) request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"}) @@ -6102,7 +6103,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) { } }) Convey("code coverage: error inside PutImageManifest method of img store (umoci.OpenLayout error)", func() { - injected := test.InjectFailure(3) + injected := inject.InjectFailure(3) request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content)) request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"}) @@ -6123,7 +6124,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) { } }) Convey("code coverage: error inside PutImageManifest method of img store (oci.GC)", func() { - injected := test.InjectFailure(4) + injected := inject.InjectFailure(4) request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content)) request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"}) diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 9f4fd104..36e3b03d 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -39,8 +39,9 @@ import ( "zotregistry.io/zot/pkg/meta" zreg "zotregistry.io/zot/pkg/regexp" localCtx "zotregistry.io/zot/pkg/requestcontext" - "zotregistry.io/zot/pkg/storage" - "zotregistry.io/zot/pkg/test" //nolint:goimports + storageCommon "zotregistry.io/zot/pkg/storage/common" + storageTypes "zotregistry.io/zot/pkg/storage/types" + "zotregistry.io/zot/pkg/test/inject" ) type RouteHandler struct { @@ -488,7 +489,7 @@ type ImageIndex struct { } func getReferrers(ctx context.Context, routeHandler *RouteHandler, - imgStore storage.ImageStore, name string, digest godigest.Digest, + imgStore storageTypes.ImageStore, name string, digest godigest.Digest, artifactTypes []string, ) (ispec.Index, error) { references, err := imgStore.GetReferrers(name, digest, artifactTypes) @@ -621,7 +622,7 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht } mediaType := request.Header.Get("Content-Type") - if !storage.IsSupportedMediaType(mediaType) { + if !storageCommon.IsSupportedMediaType(mediaType) { // response.WriteHeader(http.StatusUnsupportedMediaType) WriteJSON(response, http.StatusUnsupportedMediaType, NewErrorList(NewError(MANIFEST_INVALID, map[string]string{"mediaType": mediaType}))) @@ -632,7 +633,7 @@ func (rh *RouteHandler) UpdateManifest(response http.ResponseWriter, request *ht body, err := io.ReadAll(request.Body) // hard to reach test case, injected error (simulates an interrupted image manifest upload) // err could be io.ErrUnexpectedEOF - if err := test.Error(err); err != nil { + if err := inject.Error(err); err != nil { rh.c.Log.Error().Err(err).Msg("unexpected error") response.WriteHeader(http.StatusInternalServerError) @@ -1716,12 +1717,12 @@ func WriteDataFromReader(response http.ResponseWriter, status int, length int64, } // will return image storage corresponding to subpath provided in config. -func (rh *RouteHandler) getImageStore(name string) storage.ImageStore { +func (rh *RouteHandler) getImageStore(name string) storageTypes.ImageStore { return rh.c.StoreController.GetImageStore(name) } // will sync on demand if an image is not found, in case sync extensions is enabled. -func getImageManifest(ctx context.Context, routeHandler *RouteHandler, imgStore storage.ImageStore, +func getImageManifest(ctx context.Context, routeHandler *RouteHandler, imgStore storageTypes.ImageStore, name, reference string, ) ([]byte, godigest.Digest, string, error) { syncEnabled := false @@ -1757,7 +1758,7 @@ func getImageManifest(ctx context.Context, routeHandler *RouteHandler, imgStore // will sync referrers on demand if they are not found, in case sync extensions is enabled. func getOrasReferrers(ctx context.Context, routeHandler *RouteHandler, - imgStore storage.ImageStore, name string, digest godigest.Digest, + imgStore storageTypes.ImageStore, name string, digest godigest.Digest, artifactType string, ) ([]artifactspec.Descriptor, error) { refs, err := imgStore.GetOrasReferrers(name, digest, artifactType) diff --git a/pkg/api/routes_test.go b/pkg/api/routes_test.go index c21c805c..76b0b93a 100644 --- a/pkg/api/routes_test.go +++ b/pkg/api/routes_test.go @@ -22,7 +22,7 @@ import ( "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/api/constants" localCtx "zotregistry.io/zot/pkg/requestcontext" - "zotregistry.io/zot/pkg/storage" + storageTypes "zotregistry.io/zot/pkg/storage/types" "zotregistry.io/zot/pkg/test" "zotregistry.io/zot/pkg/test/mocks" ) @@ -1200,7 +1200,7 @@ func TestRoutes(t *testing.T) { ism *mocks.MockedImageStore, ) int { ctlr.StoreController.DefaultStore = ism - ctlr.StoreController.SubStore = map[string]storage.ImageStore{ + ctlr.StoreController.SubStore = map[string]storageTypes.ImageStore{ "test": &mocks.MockedImageStore{ GetRepositoriesFn: func() ([]string, error) { return []string{}, ErrUnexpectedError @@ -1238,7 +1238,7 @@ func TestRoutes(t *testing.T) { ism *mocks.MockedImageStore, ) int { ctlr.StoreController.DefaultStore = ism - ctlr.StoreController.SubStore = map[string]storage.ImageStore{} + ctlr.StoreController.SubStore = map[string]storageTypes.ImageStore{} request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPatch, baseURL, nil) request = mux.SetURLVars(request, vars) @@ -1300,7 +1300,7 @@ func TestRoutes(t *testing.T) { return []string{"repo"}, nil }, } - ctlr.StoreController.SubStore = map[string]storage.ImageStore{ + ctlr.StoreController.SubStore = map[string]storageTypes.ImageStore{ "test1": &mocks.MockedImageStore{ GetRepositoriesFn: func() ([]string, error) { return []string{"repo1"}, nil diff --git a/pkg/cli/root.go b/pkg/cli/root.go index d3a47d71..a1bb4bf4 100644 --- a/pkg/cli/root.go +++ b/pkg/cli/root.go @@ -24,7 +24,6 @@ import ( "zotregistry.io/zot/pkg/api/constants" extconf "zotregistry.io/zot/pkg/extensions/config" "zotregistry.io/zot/pkg/extensions/monitoring" - "zotregistry.io/zot/pkg/storage" storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/s3" ) @@ -116,7 +115,7 @@ func newScrubCmd(conf *config.Config) *cobra.Command { ctlr := api.NewController(conf) ctlr.Metrics = monitoring.NewMetricsServer(false, ctlr.Log) - if err := ctlr.InitImageStore(context.Background()); err != nil { + if err := ctlr.InitImageStore(); err != nil { panic(err) } @@ -388,7 +387,7 @@ func validateConfiguration(config *config.Config) error { if len(config.Storage.StorageDriver) != 0 { // enforce s3 driver in case of using storage driver - if config.Storage.StorageDriver["name"] != storage.S3StorageDriverName { + if config.Storage.StorageDriver["name"] != storageConstants.S3StorageDriverName { log.Error().Err(errors.ErrBadConfig).Interface("cacheDriver", config.Storage.StorageDriver["name"]). Msg("unsupported storage driver") @@ -410,7 +409,7 @@ func validateConfiguration(config *config.Config) error { for route, storageConfig := range subPaths { if len(storageConfig.StorageDriver) != 0 { - if storageConfig.StorageDriver["name"] != storage.S3StorageDriverName { + if storageConfig.StorageDriver["name"] != storageConstants.S3StorageDriverName { log.Error().Err(errors.ErrBadConfig).Str("subpath", route).Interface("storageDriver", storageConfig.StorageDriver["name"]).Msg("unsupported storage driver") @@ -583,7 +582,7 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper) { // if gc is enabled and gcDelay is not set, it is set to default value if storageConfig.GC && !viperInstance.IsSet("storage::subpaths::"+name+"::gcdelay") { - storageConfig.GCDelay = storage.DefaultGCDelay + storageConfig.GCDelay = storageConstants.DefaultGCDelay config.Storage.SubPaths[name] = storageConfig } } diff --git a/pkg/cli/root_test.go b/pkg/cli/root_test.go index 6827eb5a..599ae903 100644 --- a/pkg/cli/root_test.go +++ b/pkg/cli/root_test.go @@ -13,7 +13,6 @@ import ( "zotregistry.io/zot/pkg/api" "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/cli" - "zotregistry.io/zot/pkg/storage" storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/s3" . "zotregistry.io/zot/pkg/test" @@ -1227,10 +1226,10 @@ func TestGC(t *testing.T) { config := config.New() err := cli.LoadConfiguration(config, "../../examples/config-multiple.json") So(err, ShouldBeNil) - So(config.Storage.GCDelay, ShouldEqual, storage.DefaultGCDelay) + So(config.Storage.GCDelay, ShouldEqual, storageConstants.DefaultGCDelay) err = cli.LoadConfiguration(config, "../../examples/config-gc.json") So(err, ShouldBeNil) - So(config.Storage.GCDelay, ShouldNotEqual, storage.DefaultGCDelay) + So(config.Storage.GCDelay, ShouldNotEqual, storageConstants.DefaultGCDelay) err = cli.LoadConfiguration(config, "../../examples/config-gc-periodic.json") So(err, ShouldBeNil) }) diff --git a/pkg/extensions/extension_scrub.go b/pkg/extensions/extension_scrub.go index 5d0fc37e..af49a096 100644 --- a/pkg/extensions/extension_scrub.go +++ b/pkg/extensions/extension_scrub.go @@ -13,6 +13,7 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/scheduler" "zotregistry.io/zot/pkg/storage" + storageTypes "zotregistry.io/zot/pkg/storage/types" ) // EnableScrubExtension enables scrub extension. @@ -50,7 +51,7 @@ func EnableScrubExtension(config *config.Config, log log.Logger, storeController } type taskGenerator struct { - imgStore storage.ImageStore + imgStore storageTypes.ImageStore log log.Logger lastRepo string done bool diff --git a/pkg/extensions/lint/lint-disabled.go b/pkg/extensions/lint/lint-disabled.go index 3c33bc39..bebd6be1 100644 --- a/pkg/extensions/lint/lint-disabled.go +++ b/pkg/extensions/lint/lint-disabled.go @@ -6,13 +6,13 @@ package lint import ( godigest "github.com/opencontainers/go-digest" - "zotregistry.io/zot/pkg/storage" + storageTypes "zotregistry.io/zot/pkg/storage/types" ) type Linter struct{} func (linter *Linter) Lint(repo string, manifestDigest godigest.Digest, - imageStore storage.ImageStore, + imageStore storageTypes.ImageStore, ) (bool, error) { return true, nil } diff --git a/pkg/extensions/lint/lint.go b/pkg/extensions/lint/lint.go index 2e7cf7c3..e093ef30 100644 --- a/pkg/extensions/lint/lint.go +++ b/pkg/extensions/lint/lint.go @@ -13,7 +13,7 @@ import ( zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/extensions/config" "zotregistry.io/zot/pkg/log" - "zotregistry.io/zot/pkg/storage" + storageTypes "zotregistry.io/zot/pkg/storage/types" ) type Linter struct { @@ -29,7 +29,7 @@ func NewLinter(config *config.LintConfig, log log.Logger) *Linter { } func (linter *Linter) CheckMandatoryAnnotations(repo string, manifestDigest godigest.Digest, - imgStore storage.ImageStore, + imgStore storageTypes.ImageStore, ) (bool, error) { if linter.config == nil { return true, nil @@ -112,7 +112,7 @@ func (linter *Linter) CheckMandatoryAnnotations(repo string, manifestDigest godi } func (linter *Linter) Lint(repo string, manifestDigest godigest.Digest, - imageStore storage.ImageStore, + imageStore storageTypes.ImageStore, ) (bool, error) { return linter.CheckMandatoryAnnotations(repo, manifestDigest, imageStore) } diff --git a/pkg/extensions/scrub/scrub.go b/pkg/extensions/scrub/scrub.go index 9d7aa4a5..9a94d05d 100644 --- a/pkg/extensions/scrub/scrub.go +++ b/pkg/extensions/scrub/scrub.go @@ -9,10 +9,11 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" + storageTypes "zotregistry.io/zot/pkg/storage/types" ) // Scrub Extension for repo... -func RunScrubRepo(imgStore storage.ImageStore, repo string, log log.Logger) error { +func RunScrubRepo(imgStore storageTypes.ImageStore, repo string, log log.Logger) error { execMsg := fmt.Sprintf("executing scrub to check manifest/blob integrity for %s", path.Join(imgStore.RootDir(), repo)) log.Info().Msg(execMsg) @@ -48,12 +49,12 @@ func RunScrubRepo(imgStore storage.ImageStore, repo string, log log.Logger) erro } type Task struct { - imgStore storage.ImageStore + imgStore storageTypes.ImageStore repo string log log.Logger } -func NewTask(imgStore storage.ImageStore, repo string, log log.Logger) *Task { +func NewTask(imgStore storageTypes.ImageStore, repo string, log log.Logger) *Task { return &Task{imgStore, repo, log} } diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 9ca7142f..1317c0ce 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -33,6 +33,7 @@ import ( "zotregistry.io/zot/pkg/meta/repodb" boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" "zotregistry.io/zot/pkg/storage" + storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/local" . "zotregistry.io/zot/pkg/test" "zotregistry.io/zot/pkg/test/mocks" @@ -310,7 +311,7 @@ func TestImageFormat(t *testing.T) { dbDir := t.TempDir() metrics := monitoring.NewMetricsServer(false, log) - defaultStore := local.NewImageStore(imgDir, false, storage.DefaultGCDelay, + defaultStore := local.NewImageStore(imgDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) storeController := storage.StoreController{DefaultStore: defaultStore} diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index dc27365a..c73dce06 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -22,7 +22,9 @@ import ( "zotregistry.io/zot/pkg/meta/repodb" boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" "zotregistry.io/zot/pkg/storage" + storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/local" + storageTypes "zotregistry.io/zot/pkg/storage/types" "zotregistry.io/zot/pkg/test" ) @@ -67,17 +69,21 @@ func TestMultipleStoragePath(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) // Create ImageStore - firstStore := local.NewImageStore(firstRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil) - secondStore := local.NewImageStore(secondRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil) + firstStore := local.NewImageStore(firstRootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, + nil, nil) - thirdStore := local.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil) + secondStore := local.NewImageStore(secondRootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, + nil, nil) + + thirdStore := local.NewImageStore(thirdRootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, + nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = firstStore - subStore := make(map[string]storage.ImageStore) + subStore := make(map[string]storageTypes.ImageStore) subStore["/a"] = secondStore subStore["/b"] = thirdStore @@ -173,7 +179,7 @@ func TestTrivyLibraryErrors(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) // Create ImageStore - store := local.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil) + store := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = store @@ -370,7 +376,7 @@ func TestImageScannable(t *testing.T) { // Continue with initializing the objects the scanner depends on metrics := monitoring.NewMetricsServer(false, log) - store := local.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil) + store := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = store @@ -432,7 +438,7 @@ func TestDefaultTrivyDBUrl(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) // Create ImageStore - store := local.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil) + store := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = store diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index 1b46113f..d76791b6 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -40,7 +40,9 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/repodb" "zotregistry.io/zot/pkg/storage" + storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/local" + storageTypes "zotregistry.io/zot/pkg/storage/types" . "zotregistry.io/zot/pkg/test" "zotregistry.io/zot/pkg/test/mocks" ocilayout "zotregistry.io/zot/pkg/test/oci-layout" @@ -1218,7 +1220,7 @@ func TestExpandedRepoInfo(t *testing.T) { log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - testStorage := local.NewImageStore(rootDir, false, storage.DefaultGCDelay, + testStorage := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) resp, err := resty.R().Get(baseURL + "/v2/") @@ -2821,7 +2823,7 @@ func TestGetRepositories(t *testing.T) { storeController := storage.StoreController{ DefaultStore: mockImageStore, - SubStore: map[string]storage.ImageStore{"test": mockImageStore}, + SubStore: map[string]storageTypes.ImageStore{"test": mockImageStore}, } olu := ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) @@ -2831,7 +2833,7 @@ func TestGetRepositories(t *testing.T) { storeController = storage.StoreController{ DefaultStore: mocks.MockedImageStore{}, - SubStore: map[string]storage.ImageStore{"test": mockImageStore}, + SubStore: map[string]storageTypes.ImageStore{"test": mockImageStore}, } olu = ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) @@ -5446,7 +5448,7 @@ func TestRepoDBWhenDeletingImages(t *testing.T) { // get signatur digest log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storage := local.NewImageStore(dir, false, storage.DefaultGCDelay, + storage := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) indexBlob, err := storage.GetIndexContent(repo) @@ -5523,7 +5525,7 @@ func TestRepoDBWhenDeletingImages(t *testing.T) { // get signatur digest log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storage := local.NewImageStore(dir, false, storage.DefaultGCDelay, + storage := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) indexBlob, err := storage.GetIndexContent(repo) diff --git a/pkg/extensions/sync/signatures.go b/pkg/extensions/sync/signatures.go index ca9400ff..1ee7065e 100644 --- a/pkg/extensions/sync/signatures.go +++ b/pkg/extensions/sync/signatures.go @@ -24,6 +24,7 @@ import ( "zotregistry.io/zot/pkg/meta/repodb" "zotregistry.io/zot/pkg/meta/signatures" "zotregistry.io/zot/pkg/storage" + storageTypes "zotregistry.io/zot/pkg/storage/types" ) type signaturesCopier struct { @@ -520,7 +521,7 @@ func (sig *signaturesCopier) canSkipOCIRefs(localRepo, digestStr string, index i return true, nil } -func syncBlob(sig *signaturesCopier, imageStore storage.ImageStore, localRepo, remoteRepo string, +func syncBlob(sig *signaturesCopier, imageStore storageTypes.ImageStore, localRepo, remoteRepo string, digest godigest.Digest, ) error { getBlobURL := sig.upstreamURL diff --git a/pkg/extensions/sync/sync.go b/pkg/extensions/sync/sync.go index b74769c8..b2c6320d 100644 --- a/pkg/extensions/sync/sync.go +++ b/pkg/extensions/sync/sync.go @@ -24,7 +24,7 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/repodb" "zotregistry.io/zot/pkg/storage" - "zotregistry.io/zot/pkg/test" + "zotregistry.io/zot/pkg/test/inject" ) const ( @@ -315,7 +315,7 @@ func getLocalContexts(log log.Logger) (*types.SystemContext, *signature.PolicyCo policy = &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}} policyContext, err := signature.NewPolicyContext(policy) - if err := test.Error(err); err != nil { + if err := inject.Error(err); err != nil { log.Error().Str("errorType", common.TypeOf(err)). Err(err).Msg("couldn't create policy context") diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index fd9da321..528c42fa 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -27,8 +27,11 @@ import ( "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" + storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/local" + storageTypes "zotregistry.io/zot/pkg/storage/types" "zotregistry.io/zot/pkg/test" + "zotregistry.io/zot/pkg/test/inject" "zotregistry.io/zot/pkg/test/mocks" ) @@ -51,13 +54,13 @@ func TestInjectSyncUtils(t *testing.T) { taggedRef, err := reference.WithTag(ref, "tag") So(err, ShouldBeNil) - injected := test.InjectFailure(0) + injected := inject.InjectFailure(0) if injected { _, err = getImageTags(context.Background(), &types.SystemContext{}, taggedRef) So(err, ShouldNotBeNil) } - injected = test.InjectFailure(0) + injected = inject.InjectFailure(0) _, _, err = getLocalContexts(log.NewLogger("debug", "")) if injected { So(err, ShouldNotBeNil) @@ -67,10 +70,10 @@ func TestInjectSyncUtils(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay, + imageStore := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil, ) - injected = test.InjectFailure(0) + injected = inject.InjectFailure(0) _, err = getLocalCachePath(imageStore, testImage) if injected { @@ -183,7 +186,7 @@ func TestSyncInternal(t *testing.T) { log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay, + imageStore := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil, ) @@ -200,7 +203,7 @@ func TestSyncInternal(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay, + imageStore := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) err := os.Chmod(imageStore.RootDir(), 0o000) @@ -375,7 +378,7 @@ func TestSyncInternal(t *testing.T) { } metrics := monitoring.NewMetricsServer(false, log) - imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay, + imageStore := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil, ) mockRepoDB := mocks.RepoDBMock{} @@ -401,7 +404,7 @@ func TestSyncInternal(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imageStore := local.NewImageStore(storageDir, false, storage.DefaultGCDelay, + imageStore := local.NewImageStore(storageDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) refs := ispec.Index{Manifests: []ispec.Descriptor{ @@ -642,7 +645,7 @@ func TestSyncInternal(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imageStore := local.NewImageStore(storageDir, false, storage.DefaultGCDelay, + imageStore := local.NewImageStore(storageDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) storeController := storage.StoreController{} @@ -662,7 +665,7 @@ func TestSyncInternal(t *testing.T) { test.CopyTestFiles("../../../test/data", testRootDir) testImageStore := local.NewImageStore(testRootDir, false, - storage.DefaultGCDelay, false, false, log, metrics, nil, nil) + storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) manifestContent, _, _, err := testImageStore.GetImageManifest(testImage, testImageTag) So(err, ShouldBeNil) @@ -729,14 +732,14 @@ func TestSyncInternal(t *testing.T) { err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStore, log) So(err, ShouldNotBeNil) - err = os.Chmod(path.Join(testRootDir, repo, "blobs", - index.Manifests[0].Digest.Algorithm().String(), index.Manifests[0].Digest.Encoded()), local.DefaultDirPerms) + err = os.Chmod(path.Join(testRootDir, repo, "blobs", index.Manifests[0].Digest.Algorithm().String(), + index.Manifests[0].Digest.Encoded()), storageConstants.DefaultDirPerms) So(err, ShouldBeNil) // trigger linter error on manifest push - imageStoreWithLinter := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay, + imageStoreWithLinter := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay, false, false, log, metrics, &mocks.MockedLint{ - LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) { + LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { return false, nil }, }, nil, @@ -769,7 +772,7 @@ func TestSyncInternal(t *testing.T) { err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStore, log) So(err, ShouldNotBeNil) - err = os.Chmod(configBlobPath, local.DefaultDirPerms) + err = os.Chmod(configBlobPath, storageConstants.DefaultDirPerms) So(err, ShouldBeNil) err = os.RemoveAll(path.Join(imageStore.RootDir(), repo, "index.json")) diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index adbfb715..c75aa31c 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -42,8 +42,7 @@ import ( "zotregistry.io/zot/pkg/extensions/sync" logger "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/signatures" - "zotregistry.io/zot/pkg/storage" - "zotregistry.io/zot/pkg/storage/local" + storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/test" "zotregistry.io/zot/pkg/test/mocks" ) @@ -1660,7 +1659,7 @@ func TestBasicAuth(t *testing.T) { "a": { RootDirectory: destDir, GC: true, - GCDelay: storage.DefaultGCDelay, + GCDelay: storageConstants.DefaultGCDelay, Dedupe: true, }, } @@ -2506,7 +2505,7 @@ func TestSubPaths(t *testing.T) { subpath: { RootDirectory: subPathDestDir, GC: true, - GCDelay: storage.DefaultGCDelay, + GCDelay: storageConstants.DefaultGCDelay, Dedupe: true, }, } @@ -5203,7 +5202,7 @@ func TestSyncWithDestination(t *testing.T) { test.CopyTestFiles("../../../test/data", srcDir) - err := os.MkdirAll(path.Join(sctlr.Config.Storage.RootDirectory, "/zot-fold"), local.DefaultDirPerms) + err := os.MkdirAll(path.Join(sctlr.Config.Storage.RootDirectory, "/zot-fold"), storageConstants.DefaultDirPerms) So(err, ShouldBeNil) // move upstream images under /zot-fold diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go index dc6a962f..369a84cc 100644 --- a/pkg/extensions/sync/utils.go +++ b/pkg/extensions/sync/utils.go @@ -32,9 +32,11 @@ import ( "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/repodb" - "zotregistry.io/zot/pkg/storage" + storageCommon "zotregistry.io/zot/pkg/storage/common" + storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/local" - "zotregistry.io/zot/pkg/test" + storageTypes "zotregistry.io/zot/pkg/storage/types" + "zotregistry.io/zot/pkg/test/inject" ) type ReferenceList struct { @@ -57,7 +59,7 @@ func getTagFromRef(ref types.ImageReference, log log.Logger) reference.Tagged { func getImageTags(ctx context.Context, sysCtx *types.SystemContext, repoRef reference.Named) ([]string, error) { dockerRef, err := docker.NewReference(reference.TagNameOnly(repoRef)) // hard to reach test case, injected error, see pkg/test/dev.go - if err = test.Error(err); err != nil { + if err = inject.Error(err); err != nil { return nil, err // Should never happen for a reference with tag and no digest } @@ -266,7 +268,7 @@ func getFileCredentials(filepath string) (syncconf.CredentialsFile, error) { } func pushSyncedLocalImage(localRepo, reference, localCachePath string, - repoDB repodb.RepoDB, imageStore storage.ImageStore, log log.Logger, + repoDB repodb.RepoDB, imageStore storageTypes.ImageStore, log log.Logger, ) error { log.Info().Str("image", localCachePath+"/"+localRepo+":"+reference).Msg("pushing synced local image to local registry") @@ -275,7 +277,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string, metrics := monitoring.NewMetricsServer(false, log) cacheImageStore := local.NewImageStore(localCachePath, false, - storage.DefaultGCDelay, false, false, log, metrics, nil, nil) + storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) manifestBlob, manifestDigest, mediaType, err := cacheImageStore.GetImageManifest(localRepo, reference) if err != nil { @@ -361,7 +363,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string, } func copyManifest(localRepo string, manifestContent []byte, reference string, repoDB repodb.RepoDB, - cacheImageStore, imageStore storage.ImageStore, log log.Logger, + cacheImageStore, imageStore storageTypes.ImageStore, log log.Logger, ) error { var manifest ispec.Manifest @@ -376,7 +378,7 @@ func copyManifest(localRepo string, manifestContent []byte, reference string, re } for _, blob := range manifest.Layers { - if storage.IsNonDistributable(blob.MediaType) { + if storageCommon.IsNonDistributable(blob.MediaType) { continue } @@ -421,7 +423,7 @@ func copyManifest(localRepo string, manifestContent []byte, reference string, re // Copy a blob from one image store to another image store. func copyBlob(localRepo string, blobDigest godigest.Digest, blobMediaType string, - souceImageStore, destinationImageStore storage.ImageStore, log log.Logger, + souceImageStore, destinationImageStore storageTypes.ImageStore, log log.Logger, ) error { if found, _, _ := destinationImageStore.CheckBlob(localRepo, blobDigest); found { // Blob is already at destination, nothing to do @@ -508,12 +510,12 @@ func getLocalImageRef(localCachePath, repo, reference string) (types.ImageRefere } // Returns the localCachePath with an UUID at the end. Only to be called once per repo. -func getLocalCachePath(imageStore storage.ImageStore, repo string) (string, error) { +func getLocalCachePath(imageStore storageTypes.ImageStore, repo string) (string, error) { localRepoPath := path.Join(imageStore.RootDir(), repo, SyncBlobUploadDir) // check if SyncBlobUploadDir exists, create if not var err error if _, err = os.ReadDir(localRepoPath); os.IsNotExist(err) { - if err = os.MkdirAll(localRepoPath, local.DefaultDirPerms); err != nil { + if err = os.MkdirAll(localRepoPath, storageConstants.DefaultDirPerms); err != nil { return "", err } } @@ -525,14 +527,14 @@ func getLocalCachePath(imageStore storage.ImageStore, repo string) (string, erro // create uuid folder uuid, err := guuid.NewV4() // hard to reach test case, injected error, see pkg/test/dev.go - if err := test.Error(err); err != nil { + if err := inject.Error(err); err != nil { return "", err } localCachePath := path.Join(localRepoPath, uuid.String()) cachedRepoPath := path.Join(localCachePath, repo) - if err = os.MkdirAll(cachedRepoPath, local.DefaultDirPerms); err != nil { + if err = os.MkdirAll(cachedRepoPath, storageConstants.DefaultDirPerms); err != nil { return "", err } @@ -540,7 +542,7 @@ func getLocalCachePath(imageStore storage.ImageStore, repo string) (string, erro } // canSkipImage returns whether or not we already synced this image. -func canSkipImage(repo, tag string, digest godigest.Digest, imageStore storage.ImageStore, log log.Logger, +func canSkipImage(repo, tag string, digest godigest.Digest, imageStore storageTypes.ImageStore, log log.Logger, ) (bool, error) { // check image already synced _, localImageManifestDigest, _, err := imageStore.GetImageManifest(repo, tag) diff --git a/pkg/meta/repodb/storage_parsing.go b/pkg/meta/repodb/storage_parsing.go index 4c8893f7..6f77997d 100644 --- a/pkg/meta/repodb/storage_parsing.go +++ b/pkg/meta/repodb/storage_parsing.go @@ -13,6 +13,7 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/signatures" "zotregistry.io/zot/pkg/storage" + storageTypes "zotregistry.io/zot/pkg/storage/types" ) // ParseStorage will sync all repos found in the rootdirectory of the oci layout that zot was deployed on with the @@ -215,8 +216,8 @@ func isManifestMetaPresent(repo string, manifest ispec.Descriptor, repoDB RepoDB return true, nil } -func GetSignatureLayersInfo( - repo, tag, manifestDigest, signatureType string, manifestBlob []byte, imageStore storage.ImageStore, log log.Logger, +func GetSignatureLayersInfo(repo, tag, manifestDigest, signatureType string, manifestBlob []byte, + imageStore storageTypes.ImageStore, log log.Logger, ) ([]LayerInfo, error) { switch signatureType { case signatures.CosignSignature: @@ -229,7 +230,7 @@ func GetSignatureLayersInfo( } func getCosignSignatureLayersInfo( - repo, tag, manifestDigest string, manifestBlob []byte, imageStore storage.ImageStore, log log.Logger, + repo, tag, manifestDigest string, manifestBlob []byte, imageStore storageTypes.ImageStore, log log.Logger, ) ([]LayerInfo, error) { layers := []LayerInfo{} @@ -267,7 +268,7 @@ func getCosignSignatureLayersInfo( } func getNotationSignatureLayersInfo( - repo, manifestDigest string, manifestBlob []byte, imageStore storage.ImageStore, log log.Logger, + repo, manifestDigest string, manifestBlob []byte, imageStore storageTypes.ImageStore, log log.Logger, ) ([]LayerInfo, error) { layers := []LayerInfo{} @@ -308,7 +309,7 @@ func getNotationSignatureLayersInfo( } // NewManifestMeta takes raw data about an image and createa a new ManifestMetadate object. -func NewManifestData(repoName string, manifestBlob []byte, imageStore storage.ImageStore, +func NewManifestData(repoName string, manifestBlob []byte, imageStore storageTypes.ImageStore, ) (ManifestData, error) { var ( manifestContent ispec.Manifest @@ -337,7 +338,7 @@ func NewManifestData(repoName string, manifestBlob []byte, imageStore storage.Im return manifestData, nil } -func NewIndexData(repoName string, indexBlob []byte, imageStore storage.ImageStore, +func NewIndexData(repoName string, indexBlob []byte, imageStore storageTypes.ImageStore, ) IndexData { indexData := IndexData{} @@ -349,7 +350,7 @@ func NewIndexData(repoName string, indexBlob []byte, imageStore storage.ImageSto // SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag // (in case the reference is a tag). The function expects image manifests and indexes (multi arch images). func SetImageMetaFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte, - imageStore storage.ImageStore, repoDB RepoDB, log log.Logger, + imageStore storageTypes.ImageStore, repoDB RepoDB, log log.Logger, ) error { switch mediaType { case ispec.MediaTypeImageManifest: diff --git a/pkg/meta/repodb/storage_parsing_test.go b/pkg/meta/repodb/storage_parsing_test.go index 26be67a8..709a4c92 100644 --- a/pkg/meta/repodb/storage_parsing_test.go +++ b/pkg/meta/repodb/storage_parsing_test.go @@ -25,6 +25,7 @@ import ( "zotregistry.io/zot/pkg/meta/signatures" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/local" + storageTypes "zotregistry.io/zot/pkg/storage/types" "zotregistry.io/zot/pkg/test" "zotregistry.io/zot/pkg/test/mocks" ) @@ -63,7 +64,7 @@ func TestParseStorageErrors(t *testing.T) { } storeController := storage.StoreController{ DefaultStore: imageStore1, - SubStore: map[string]storage.ImageStore{ + SubStore: map[string]storageTypes.ImageStore{ "a": imageStore2, }, } diff --git a/pkg/storage/cache.go b/pkg/storage/cache.go index 2ddb072c..9f4ba357 100644 --- a/pkg/storage/cache.go +++ b/pkg/storage/cache.go @@ -2,10 +2,58 @@ package storage import ( "zotregistry.io/zot/errors" + "zotregistry.io/zot/pkg/api/config" zlog "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage/cache" + "zotregistry.io/zot/pkg/storage/constants" ) +func CreateCacheDatabaseDriver(storageConfig config.StorageConfig, log zlog.Logger) cache.Cache { + if !storageConfig.Dedupe && storageConfig.StorageDriver == nil { + return nil + } + + // local cache + if !storageConfig.RemoteCache { + params := cache.BoltDBDriverParameters{} + params.RootDir = storageConfig.RootDirectory + params.Name = constants.BoltdbName + params.UseRelPaths = getUseRelPaths(&storageConfig) + + driver, _ := Create("boltdb", params, log) + + return driver + } + + // remote cache + if storageConfig.CacheDriver != nil { + name, ok := storageConfig.CacheDriver["name"].(string) + if !ok { + log.Warn().Msg("remote cache driver name missing!") + + return nil + } + + if name != constants.DynamoDBDriverName { + log.Warn().Str("driver", name).Msg("remote cache driver unsupported!") + + return nil + } + + // dynamodb + dynamoParams := cache.DynamoDBDriverParameters{} + dynamoParams.Endpoint, _ = storageConfig.CacheDriver["endpoint"].(string) + dynamoParams.Region, _ = storageConfig.CacheDriver["region"].(string) + dynamoParams.TableName, _ = storageConfig.CacheDriver["cachetablename"].(string) + + driver, _ := Create("dynamodb", dynamoParams, log) + + return driver + } + + return nil +} + func Create(dbtype string, parameters interface{}, log zlog.Logger) (cache.Cache, error) { switch dbtype { case "boltdb": @@ -22,3 +70,7 @@ func Create(dbtype string, parameters interface{}, log zlog.Logger) (cache.Cache } } } + +func getUseRelPaths(storageConfig *config.StorageConfig) bool { + return storageConfig.StorageDriver == nil +} diff --git a/pkg/storage/cache/dynamodb.go b/pkg/storage/cache/dynamodb.go index 48b059dd..890d5177 100644 --- a/pkg/storage/cache/dynamodb.go +++ b/pkg/storage/cache/dynamodb.go @@ -80,7 +80,7 @@ func NewDynamoDBCache(parameters interface{}, log zlog.Logger) Cache { // Using the SDK's default configuration, loading additional config // and credentials values from the environment variables, shared // credentials, and shared configuration files - cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(properParameters.Region), + cfg, err := config.LoadDefaultConfig(context.Background(), config.WithRegion(properParameters.Region), config.WithEndpointResolverWithOptions(customResolver)) if err != nil { log.Error().Err(err).Msg("unable to load AWS SDK config for dynamodb") diff --git a/pkg/storage/common.go b/pkg/storage/common/common.go similarity index 87% rename from pkg/storage/common.go rename to pkg/storage/common/common.go index dfa19077..558fa4f0 100644 --- a/pkg/storage/common.go +++ b/pkg/storage/common/common.go @@ -6,7 +6,6 @@ import ( "path" "strings" - "github.com/gobwas/glob" notreg "github.com/notaryproject/notation-go/registry" godigest "github.com/opencontainers/go-digest" imeta "github.com/opencontainers/image-spec/specs-go" @@ -19,11 +18,7 @@ import ( zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/scheduler" storageConstants "zotregistry.io/zot/pkg/storage/constants" -) - -const ( - CosignType = "cosign" - NotationType = "notation" + storageTypes "zotregistry.io/zot/pkg/storage/types" ) func GetTagsByIndex(index ispec.Index) []string { @@ -56,7 +51,7 @@ func GetManifestDescByReference(index ispec.Index, reference string) (ispec.Desc return manifestDesc, false } -func ValidateManifest(imgStore ImageStore, repo, reference, mediaType string, body []byte, +func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaType string, body []byte, log zerolog.Logger, ) (godigest.Digest, error) { // validate the manifest @@ -111,8 +106,8 @@ func ValidateManifest(imgStore ImageStore, repo, reference, mediaType string, bo return "", nil } -func validateOCIManifest(imgStore ImageStore, repo, reference string, manifest *ispec.Manifest, //nolint:unparam - log zerolog.Logger, +func validateOCIManifest(imgStore storageTypes.ImageStore, repo, reference string, //nolint:unparam + manifest *ispec.Manifest, log zerolog.Logger, ) (godigest.Digest, error) { if manifest.SchemaVersion != storageConstants.SchemaVersion { log.Error().Int("SchemaVersion", manifest.SchemaVersion).Msg("invalid manifest") @@ -245,7 +240,7 @@ func CheckIfIndexNeedsUpdate(index *ispec.Index, desc *ispec.Descriptor, } // GetIndex returns the contents of index.json. -func GetIndex(imgStore ImageStore, repo string, log zerolog.Logger) (ispec.Index, error) { +func GetIndex(imgStore storageTypes.ImageStore, repo string, log zerolog.Logger) (ispec.Index, error) { var index ispec.Index buf, err := imgStore.GetIndexContent(repo) @@ -263,7 +258,8 @@ func GetIndex(imgStore ImageStore, repo string, log zerolog.Logger) (ispec.Index } // GetImageIndex returns a multiarch type image. -func GetImageIndex(imgStore ImageStore, repo string, digest godigest.Digest, log zerolog.Logger) (ispec.Index, error) { +func GetImageIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger, +) (ispec.Index, error) { var imageIndex ispec.Index if err := digest.Validate(); err != nil { @@ -287,7 +283,7 @@ func GetImageIndex(imgStore ImageStore, repo string, digest godigest.Digest, log return imageIndex, nil } -func GetImageManifest(imgStore ImageStore, repo string, digest godigest.Digest, log zerolog.Logger, +func GetImageManifest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger, ) (ispec.Manifest, error) { var manifestContent ispec.Manifest @@ -353,7 +349,7 @@ Unmarshal an image index and for all manifests in that index, ensure that they do not have a name or they are not in other manifest indexes else GC can never clean them. */ -func UpdateIndexWithPrunedImageManifests(imgStore ImageStore, index *ispec.Index, repo string, +func UpdateIndexWithPrunedImageManifests(imgStore storageTypes.ImageStore, index *ispec.Index, repo string, desc ispec.Descriptor, oldDgst godigest.Digest, log zerolog.Logger, ) error { if (desc.MediaType == ispec.MediaTypeImageIndex) && (oldDgst != "") { @@ -386,7 +382,7 @@ same constitutent manifests so that they can be garbage-collected correctly PruneImageManifestsFromIndex is a helper routine to achieve this. */ -func PruneImageManifestsFromIndex(imgStore ImageStore, repo string, digest godigest.Digest, //nolint:gocyclo +func PruneImageManifestsFromIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, //nolint:gocyclo,lll outIndex ispec.Index, otherImgIndexes []ispec.Descriptor, log zerolog.Logger, ) ([]ispec.Descriptor, error) { dir := path.Join(imgStore.RootDir(), repo) @@ -461,7 +457,8 @@ func PruneImageManifestsFromIndex(imgStore ImageStore, repo string, digest godig return prunedManifests, nil } -func ApplyLinter(imgStore ImageStore, linter Lint, repo string, descriptor ispec.Descriptor) (bool, error) { +func ApplyLinter(imgStore storageTypes.ImageStore, linter Lint, repo string, descriptor ispec.Descriptor, +) (bool, error) { pass := true // we'll skip anything that's not a image manifest @@ -505,7 +502,7 @@ func IsSignature(descriptor ispec.Descriptor) bool { return false } -func GetOrasReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, artifactType string, +func GetOrasReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godigest.Digest, artifactType string, log zerolog.Logger, ) ([]oras.Descriptor, error) { if err := gdigest.Validate(); err != nil { @@ -583,7 +580,7 @@ func getReferrerFilterAnnotation(artifactTypes []string) string { return annotation } -func GetReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, artifactTypes []string, +func GetReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godigest.Digest, artifactTypes []string, log zerolog.Logger, ) (ispec.Index, error) { nilIndex := ispec.Index{} @@ -665,7 +662,7 @@ func GetReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, art return index, nil } -func GetOrasManifestByDigest(imgStore ImageStore, repo string, digest godigest.Digest, log zerolog.Logger, +func GetOrasManifestByDigest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger, ) (oras.Manifest, error) { var artManifest oras.Manifest @@ -703,57 +700,14 @@ func IsNonDistributable(mediaType string) bool { mediaType == ispec.MediaTypeImageLayerNonDistributableZstd //nolint:staticcheck } -// CheckIsImageSignature checks if the given image (repo:tag) represents a signature. The function -// returns: -// -// - bool: if the image is a signature or not -// -// - string: the type of signature -// -// - string: the digest of the image it signs -// -// - error: any errors that occur. -func CheckIsImageSignature(repoName string, manifestBlob []byte, reference string, -) (bool, string, godigest.Digest, error) { - var manifestContent ispec.Manifest - - err := json.Unmarshal(manifestBlob, &manifestContent) - if err != nil { - return false, "", "", err - } - - manifestArtifactType := zcommon.GetManifestArtifactType(manifestContent) - - // check notation signature - if manifestArtifactType == notreg.ArtifactTypeNotation && manifestContent.Subject != nil { - return true, NotationType, manifestContent.Subject.Digest, nil - } - - // check cosign - cosignTagRule := glob.MustCompile("sha256-*.sig") - - if tag := reference; cosignTagRule.Match(reference) { - prefixLen := len("sha256-") - digestLen := 64 - signedImageManifestDigestEncoded := tag[prefixLen : prefixLen+digestLen] - - signedImageManifestDigest := godigest.NewDigestFromEncoded(godigest.SHA256, - signedImageManifestDigestEncoded) - - return true, CosignType, signedImageManifestDigest, nil - } - - return false, "", "", nil -} - /* - DedupeTaskGenerator takes all blobs paths found in the imagestore and groups them by digest + DedupeTaskGenerator takes all blobs paths found in the storage.imagestore and groups them by digest for each digest and based on the dedupe value it will dedupe or restore deduped blobs to the original state(undeduped)\ by creating a task for each digest and pushing it to the task scheduler. */ type DedupeTaskGenerator struct { - ImgStore ImageStore + ImgStore storageTypes.ImageStore // storage dedupe value Dedupe bool // store blobs paths grouped by digest @@ -769,7 +723,7 @@ type DedupeTaskGenerator struct { func (gen *DedupeTaskGenerator) GenerateTask() (scheduler.Task, error) { var err error - // get all blobs from imageStore and group them by digest + // get all blobs from storage.imageStore and group them by digest gen.digest, gen.duplicateBlobs, err = gen.ImgStore.GetNextDigestWithBlobPaths(gen.lastDigests) if err != nil { gen.Log.Error().Err(err).Msg("dedupe rebuild: failed to get next digest") @@ -805,7 +759,7 @@ func (gen *DedupeTaskGenerator) Reset() { } type dedupeTask struct { - imgStore ImageStore + imgStore storageTypes.ImageStore // digest of duplicateBLobs digest godigest.Digest // blobs paths with the same digest ^ @@ -814,7 +768,7 @@ type dedupeTask struct { log zerolog.Logger } -func newDedupeTask(imgStore ImageStore, digest godigest.Digest, dedupe bool, +func newDedupeTask(imgStore storageTypes.ImageStore, digest godigest.Digest, dedupe bool, duplicateBlobs []string, log zerolog.Logger, ) *dedupeTask { return &dedupeTask{imgStore, digest, duplicateBlobs, dedupe, log} diff --git a/pkg/storage/common_test.go b/pkg/storage/common/common_test.go similarity index 85% rename from pkg/storage/common_test.go rename to pkg/storage/common/common_test.go index f35d3b09..43aefbcb 100644 --- a/pkg/storage/common_test.go +++ b/pkg/storage/common/common_test.go @@ -18,6 +18,8 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/cache" + common "zotregistry.io/zot/pkg/storage/common" + storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/test" "zotregistry.io/zot/pkg/test/mocks" @@ -34,7 +36,7 @@ func TestValidateManifest(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) content := []byte("this is a blob") @@ -148,33 +150,33 @@ func TestGetReferrersErrors(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, false, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, false, true, log, metrics, nil, cacheDriver) artifactType := "application/vnd.example.icecream.v1" validDigest := godigest.FromBytes([]byte("blob")) Convey("Trigger invalid digest error", func(c C) { - _, err := storage.GetReferrers(imgStore, "zot-test", "invalidDigest", + _, err := common.GetReferrers(imgStore, "zot-test", "invalidDigest", []string{artifactType}, log.With().Caller().Logger()) So(err, ShouldNotBeNil) - _, err = storage.GetOrasReferrers(imgStore, "zot-test", "invalidDigest", + _, err = common.GetOrasReferrers(imgStore, "zot-test", "invalidDigest", artifactType, log.With().Caller().Logger()) So(err, ShouldNotBeNil) }) Convey("Trigger repo not found error", func(c C) { - _, err := storage.GetReferrers(imgStore, "zot-test", validDigest, + _, err := common.GetReferrers(imgStore, "zot-test", validDigest, []string{artifactType}, log.With().Caller().Logger()) So(err, ShouldNotBeNil) - _, err = storage.GetOrasReferrers(imgStore, "zot-test", validDigest, + _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger()) So(err, ShouldNotBeNil) }) - err := test.CopyFiles("../../test/data/zot-test", path.Join(dir, "zot-test")) + err := test.CopyFiles("../../../test/data/zot-test", path.Join(dir, "zot-test")) So(err, ShouldBeNil) digest := godigest.FromBytes([]byte("{}")) @@ -201,11 +203,11 @@ func TestGetReferrersErrors(t *testing.T) { }, } - _, err = storage.GetReferrers(imgStore, "zot-test", validDigest, + _, err = common.GetReferrers(imgStore, "zot-test", validDigest, []string{artifactType}, log.With().Caller().Logger()) So(err, ShouldNotBeNil) - _, err = storage.GetOrasReferrers(imgStore, "zot-test", validDigest, + _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger()) So(err, ShouldNotBeNil) }) @@ -220,11 +222,11 @@ func TestGetReferrersErrors(t *testing.T) { }, } - _, err = storage.GetReferrers(imgStore, "zot-test", validDigest, + _, err = common.GetReferrers(imgStore, "zot-test", validDigest, []string{artifactType}, log.With().Caller().Logger()) So(err, ShouldNotBeNil) - _, err = storage.GetOrasReferrers(imgStore, "zot-test", validDigest, + _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger()) So(err, ShouldNotBeNil) }) @@ -249,11 +251,11 @@ func TestGetReferrersErrors(t *testing.T) { }, } - _, err = storage.GetOrasReferrers(imgStore, "zot-test", validDigest, + _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger()) So(err, ShouldNotBeNil) - _, err = storage.GetOrasReferrers(imgStore, "zot-test", digest, + _, err = common.GetOrasReferrers(imgStore, "zot-test", digest, artifactType, log.With().Caller().Logger()) So(err, ShouldNotBeNil) }) @@ -268,7 +270,7 @@ func TestGetReferrersErrors(t *testing.T) { }, } - _, err = storage.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger()) + _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger()) So(err, ShouldNotBeNil) }) @@ -294,7 +296,7 @@ func TestGetReferrersErrors(t *testing.T) { }, } - _, err = storage.GetReferrers(imgStore, "zot-test", validDigest, + _, err = common.GetReferrers(imgStore, "zot-test", validDigest, []string{artifactType}, log.With().Caller().Logger()) So(err, ShouldNotBeNil) }) @@ -328,7 +330,7 @@ func TestGetReferrersErrors(t *testing.T) { }, } - _, err = storage.GetReferrers(imgStore, "zot-test", validDigest, + _, err = common.GetReferrers(imgStore, "zot-test", validDigest, []string{artifactType}, log.With().Caller().Logger()) So(err, ShouldBeNil) }) @@ -341,7 +343,7 @@ func TestGetImageIndexErrors(t *testing.T) { Convey("Trigger invalid digest error", t, func(c C) { imgStore := &mocks.MockedImageStore{} - _, err := storage.GetImageIndex(imgStore, "zot-test", "invalidDigest", log) + _, err := common.GetImageIndex(imgStore, "zot-test", "invalidDigest", log) So(err, ShouldNotBeNil) }) @@ -354,7 +356,7 @@ func TestGetImageIndexErrors(t *testing.T) { validDigest := godigest.FromBytes([]byte("blob")) - _, err := storage.GetImageIndex(imgStore, "zot-test", validDigest, log) + _, err := common.GetImageIndex(imgStore, "zot-test", validDigest, log) So(err, ShouldNotBeNil) }) @@ -367,14 +369,14 @@ func TestGetImageIndexErrors(t *testing.T) { validDigest := godigest.FromBytes([]byte("blob")) - _, err := storage.GetImageIndex(imgStore, "zot-test", validDigest, log) + _, err := common.GetImageIndex(imgStore, "zot-test", validDigest, log) So(err, ShouldNotBeNil) }) } func TestIsSignature(t *testing.T) { Convey("Unknown media type", t, func(c C) { - isSingature := storage.IsSignature(ispec.Descriptor{ + isSingature := common.IsSignature(ispec.Descriptor{ MediaType: "unknown media type", }) So(isSingature, ShouldBeFalse) diff --git a/pkg/storage/common/lint-interface.go b/pkg/storage/common/lint-interface.go new file mode 100644 index 00000000..e880fa4d --- /dev/null +++ b/pkg/storage/common/lint-interface.go @@ -0,0 +1,11 @@ +package storage + +import ( + godigest "github.com/opencontainers/go-digest" + + storageTypes "zotregistry.io/zot/pkg/storage/types" +) + +type Lint interface { + Lint(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) +} diff --git a/pkg/storage/constants/constants.go b/pkg/storage/constants/constants.go index 85af6b62..06a0c41d 100644 --- a/pkg/storage/constants/constants.go +++ b/pkg/storage/constants/constants.go @@ -20,4 +20,6 @@ const ( BoltdbName = "cache" ReferrerFilterAnnotation = "org.opencontainers.referrers.filtersApplied" DynamoDBDriverName = "dynamodb" + DefaultGCDelay = 1 * time.Hour + S3StorageDriverName = "s3" ) diff --git a/pkg/storage/lint-interface.go b/pkg/storage/lint-interface.go deleted file mode 100644 index f679a27e..00000000 --- a/pkg/storage/lint-interface.go +++ /dev/null @@ -1,9 +0,0 @@ -package storage - -import ( - godigest "github.com/opencontainers/go-digest" -) - -type Lint interface { - Lint(repo string, manifestDigest godigest.Digest, imageStore ImageStore) (bool, error) -} diff --git a/pkg/storage/local/local.go b/pkg/storage/local/local.go index c2b2c4db..6651baf9 100644 --- a/pkg/storage/local/local.go +++ b/pkg/storage/local/local.go @@ -34,31 +34,25 @@ import ( zlog "zotregistry.io/zot/pkg/log" zreg "zotregistry.io/zot/pkg/regexp" "zotregistry.io/zot/pkg/scheduler" - "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/cache" + common "zotregistry.io/zot/pkg/storage/common" storageConstants "zotregistry.io/zot/pkg/storage/constants" - "zotregistry.io/zot/pkg/test" -) - -const ( - DefaultFilePerms = 0o600 - DefaultDirPerms = 0o700 - defaultSchemaVersion = 2 + storageTypes "zotregistry.io/zot/pkg/storage/types" + "zotregistry.io/zot/pkg/test/inject" ) // ImageStoreLocal provides the image storage operations. type ImageStoreLocal struct { - rootDir string - lock *sync.RWMutex - blobUploads map[string]storage.BlobUpload - cache cache.Cache - gc bool - dedupe bool - commit bool - gcDelay time.Duration - log zerolog.Logger - metrics monitoring.MetricServer - linter storage.Lint + rootDir string + lock *sync.RWMutex + cache cache.Cache + gc bool + dedupe bool + commit bool + gcDelay time.Duration + log zerolog.Logger + metrics monitoring.MetricServer + linter common.Lint } func (is *ImageStoreLocal) RootDir() string { @@ -72,10 +66,10 @@ func (is *ImageStoreLocal) DirExists(d string) bool { // NewImageStore returns a new image store backed by a file storage. // Use the last argument to properly set a cache database, or it will default to boltDB local storage. func NewImageStore(rootDir string, gc bool, gcDelay time.Duration, dedupe, commit bool, - log zlog.Logger, metrics monitoring.MetricServer, linter storage.Lint, cacheDriver cache.Cache, -) storage.ImageStore { + log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, cacheDriver cache.Cache, +) storageTypes.ImageStore { if _, err := os.Stat(rootDir); os.IsNotExist(err) { - if err := os.MkdirAll(rootDir, DefaultDirPerms); err != nil { + if err := os.MkdirAll(rootDir, storageConstants.DefaultDirPerms); err != nil { log.Error().Err(err).Str("rootDir", rootDir).Msg("unable to create root dir") return nil @@ -83,16 +77,15 @@ func NewImageStore(rootDir string, gc bool, gcDelay time.Duration, dedupe, commi } imgStore := &ImageStoreLocal{ - rootDir: rootDir, - lock: &sync.RWMutex{}, - blobUploads: make(map[string]storage.BlobUpload), - gc: gc, - gcDelay: gcDelay, - dedupe: dedupe, - commit: commit, - log: log.With().Caller().Logger(), - metrics: metrics, - linter: linter, + rootDir: rootDir, + lock: &sync.RWMutex{}, + gc: gc, + gcDelay: gcDelay, + dedupe: dedupe, + commit: commit, + log: log.With().Caller().Logger(), + metrics: metrics, + linter: linter, } imgStore.cache = cacheDriver @@ -197,7 +190,7 @@ func (is *ImageStoreLocal) initRepo(name string) error { // "index.json" file - create if it doesn't exist indexPath := path.Join(repoDir, "index.json") if _, err := os.Stat(indexPath); err != nil { - index := ispec.Index{Versioned: imeta.Versioned{SchemaVersion: defaultSchemaVersion}} + index := ispec.Index{Versioned: imeta.Versioned{SchemaVersion: storageConstants.SchemaVersion}} buf, err := json.Marshal(index) if err != nil { @@ -401,12 +394,12 @@ func (is *ImageStoreLocal) GetImageTags(repo string) ([]string, error) { is.RLock(&lockLatency) defer is.RUnlock(&lockLatency) - index, err := storage.GetIndex(is, repo, is.log) + index, err := common.GetIndex(is, repo, is.log) if err != nil { return nil, err } - return storage.GetTagsByIndex(index), nil + return common.GetTagsByIndex(index), nil } // GetImageManifest returns the image manifest of an image in the specific repository. @@ -421,12 +414,12 @@ func (is *ImageStoreLocal) GetImageManifest(repo, reference string) ([]byte, god is.RLock(&lockLatency) defer is.RUnlock(&lockLatency) - index, err := storage.GetIndex(is, repo, is.log) + index, err := common.GetIndex(is, repo, is.log) if err != nil { return nil, "", "", err } - manifestDesc, found := storage.GetManifestDescByReference(index, reference) + manifestDesc, found := common.GetManifestDescByReference(index, reference) if !found { return nil, "", "", zerr.ErrManifestNotFound } @@ -467,14 +460,14 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, / is.Lock(&lockLatency) defer is.Unlock(&lockLatency) - digest, err := storage.ValidateManifest(is, repo, reference, mediaType, body, is.log) + digest, err := common.ValidateManifest(is, repo, reference, mediaType, body, is.log) if err != nil { return digest, "", err } refIsDigest := true - mDigest, err := storage.GetAndValidateRequestDigest(body, reference, is.log) + mDigest, err := common.GetAndValidateRequestDigest(body, reference, is.log) if err != nil { if errors.Is(err, zerr.ErrBadManifest) { return mDigest, "", err @@ -483,7 +476,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, / refIsDigest = false } - index, err := storage.GetIndex(is, repo, is.log) + index, err := common.GetIndex(is, repo, is.log) if err != nil { return "", "", err } @@ -516,7 +509,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, / artifactType = zcommon.GetManifestArtifactType(manifest) } - updateIndex, oldDgst, err := storage.CheckIfIndexNeedsUpdate(&index, &desc, is.log) + updateIndex, oldDgst, err := common.CheckIfIndexNeedsUpdate(&index, &desc, is.log) if err != nil { return "", "", err } @@ -537,7 +530,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, / return "", "", err } - err = storage.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) + err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) if err != nil { return "", "", err } @@ -548,7 +541,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, / file = path.Join(dir, "index.json") buf, err := json.Marshal(index) - if err := test.Error(err); err != nil { + if err := inject.Error(err); err != nil { is.log.Error().Err(err).Str("file", file).Msg("unable to marshal JSON") return "", "", err @@ -558,7 +551,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, / desc.ArtifactType = artifactType // apply linter only on images, not signatures or indexes - pass, err := storage.ApplyLinter(is, is.linter, repo, desc) + pass, err := common.ApplyLinter(is, is.linter, repo, desc) if !pass { is.log.Error().Err(err).Str("repository", repo).Str("reference", reference).Msg("linter didn't pass") @@ -566,7 +559,7 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, / } err = is.writeFile(file, buf) - if err := test.Error(err); err != nil { + if err := inject.Error(err); err != nil { is.log.Error().Err(err).Str("file", file).Msg("unable to write") return "", "", err @@ -596,17 +589,17 @@ func (is *ImageStoreLocal) DeleteImageManifest(repo, reference string, detectCol is.Lock(&lockLatency) defer is.Unlock(&lockLatency) - index, err := storage.GetIndex(is, repo, is.log) + index, err := common.GetIndex(is, repo, is.log) if err != nil { return err } - manifestDesc, err := storage.RemoveManifestDescByReference(&index, reference, detectCollision) + manifestDesc, err := common.RemoveManifestDescByReference(&index, reference, detectCollision) if err != nil { return err } - err = storage.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log) + err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log) if err != nil { return err } @@ -678,7 +671,7 @@ func (is *ImageStoreLocal) NewBlobUpload(repo string) (string, error) { blobUploadPath := is.BlobUploadPath(repo, uid) - file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, DefaultFilePerms) + file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, storageConstants.DefaultFilePerms) if err != nil { return "", zerr.ErrRepoNotFound } @@ -724,7 +717,7 @@ func (is *ImageStoreLocal) PutBlobChunkStreamed(repo, uuid string, body io.Reade return -1, zerr.ErrUploadNotFound } - file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, DefaultFilePerms) + file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, storageConstants.DefaultFilePerms) if err != nil { is.log.Error().Err(err).Msg("failed to open file") @@ -773,7 +766,7 @@ func (is *ImageStoreLocal) PutBlobChunk(repo, uuid string, from, to int64, return -1, zerr.ErrBadUploadRange } - file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, DefaultFilePerms) + file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, storageConstants.DefaultFilePerms) if err != nil { is.log.Error().Err(err).Msg("failed to open file") @@ -876,7 +869,7 @@ func (is *ImageStoreLocal) FinishBlobUpload(repo, uuid string, body io.Reader, d if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { err = is.DedupeBlob(src, dstDigest, dst) - if err := test.Error(err); err != nil { + if err := inject.Error(err); err != nil { is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). Str("dst", dst).Msg("unable to dedupe blob") @@ -1390,7 +1383,7 @@ func (is *ImageStoreLocal) GetReferrers(repo string, gdigest godigest.Digest, ar is.RLock(&lockLatency) defer is.RUnlock(&lockLatency) - return storage.GetReferrers(is, repo, gdigest, artifactTypes, is.log) + return common.GetReferrers(is, repo, gdigest, artifactTypes, is.log) } func (is *ImageStoreLocal) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string, @@ -1400,42 +1393,40 @@ func (is *ImageStoreLocal) GetOrasReferrers(repo string, gdigest godigest.Digest is.RLock(&lockLatency) defer is.RUnlock(&lockLatency) - return storage.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) + return common.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) } func (is *ImageStoreLocal) writeFile(filename string, data []byte) error { if !is.commit { - return os.WriteFile(filename, data, DefaultFilePerms) + return os.WriteFile(filename, data, storageConstants.DefaultFilePerms) } - fhandle, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, DefaultFilePerms) + fhandle, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, storageConstants.DefaultFilePerms) if err != nil { return err } _, err = fhandle.Write(data) - if err1 := test.Error(fhandle.Sync()); err1 != nil && err == nil { + if err1 := inject.Error(fhandle.Sync()); err1 != nil && err == nil { err = err1 is.log.Error().Err(err).Str("filename", filename).Msg("unable to sync file") } - if err1 := test.Error(fhandle.Close()); err1 != nil && err == nil { + if err1 := inject.Error(fhandle.Close()); err1 != nil && err == nil { err = err1 } return err } -// utility routines - func ValidateHardLink(rootDir string) error { - if err := os.MkdirAll(rootDir, DefaultDirPerms); err != nil { + if err := os.MkdirAll(rootDir, storageConstants.DefaultDirPerms); err != nil { return err } err := os.WriteFile(path.Join(rootDir, "hardlinkcheck.txt"), - []byte("check whether hardlinks work on filesystem"), DefaultFilePerms) + []byte("check whether hardlinks work on filesystem"), storageConstants.DefaultFilePerms) if err != nil { return err } @@ -1459,8 +1450,9 @@ func ValidateHardLink(rootDir string) error { return os.RemoveAll(path.Join(rootDir, "duphardlinkcheck.txt")) } +// utility routines. func ensureDir(dir string, log zerolog.Logger) error { - if err := os.MkdirAll(dir, DefaultDirPerms); err != nil { + if err := os.MkdirAll(dir, storageConstants.DefaultDirPerms); err != nil { log.Error().Err(err).Str("dir", dir).Msg("unable to create dir") return err @@ -1477,7 +1469,7 @@ type extendedManifest struct { func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error { oci, err := umoci.OpenLayout(dir) - if err := test.Error(err); err != nil { + if err := inject.Error(err); err != nil { return err } defer oci.Close() @@ -1497,7 +1489,7 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error { for _, desc := range index.Manifests { switch desc.MediaType { case ispec.MediaTypeImageIndex: - indexImage, err := storage.GetImageIndex(is, repo, desc.Digest, is.log) + indexImage, err := common.GetImageIndex(is, repo, desc.Digest, is.log) if err != nil { is.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). Msg("gc: failed to read multiarch(index) image") @@ -1519,7 +1511,7 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error { } } - manifestContent, err := storage.GetImageManifest(is, repo, desc.Digest, is.log) + manifestContent, err := common.GetImageManifest(is, repo, desc.Digest, is.log) if err != nil { is.log.Error().Err(err).Str("repo", repo).Str("digest", desc.Digest.String()). Msg("gc: failed to read manifest image") @@ -1559,7 +1551,7 @@ func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error { is.log.Info().Msg("gc: blobs") err = oci.GC(context.Background(), ifOlderThan(is, repo, is.gcDelay)) - if err := test.Error(err); err != nil { + if err := inject.Error(err); err != nil { return err } @@ -1616,7 +1608,7 @@ func gcUntaggedManifests(imgStore *ImageStoreLocal, oci casext.Engine, index *is imgStore.log.Info().Str("repository", repo).Str("digest", desc.Digest.String()). Msg("gc: removing manifest without tag") - _, err = storage.RemoveManifestDescByReference(index, desc.Digest.String(), true) + _, err = common.RemoveManifestDescByReference(index, desc.Digest.String(), true) if errors.Is(err, zerr.ErrManifestConflict) { imgStore.log.Info().Str("repository", repo).Str("digest", desc.Digest.String()). Msg("gc: skipping removing manifest due to conflict") @@ -1655,7 +1647,7 @@ func gcCosignSignatures(imgStore *ImageStoreLocal, oci casext.Engine, index *isp Msg("gc: removing cosign signature without subject") // no need to check for manifest conflict, if one doesn't have a subject, then none with same digest will have - _, _ = storage.RemoveManifestDescByReference(index, cosignDesc.Digest.String(), false) + _, _ = common.RemoveManifestDescByReference(index, cosignDesc.Digest.String(), false) err := oci.PutIndex(context.Background(), *index) if err != nil { @@ -1685,7 +1677,7 @@ func gcNotationSignatures(imgStore *ImageStoreLocal, oci casext.Engine, index *i Msg("gc: removing notation signature without subject") // no need to check for manifest conflict, if one doesn't have a subject, then none with same digest will have - _, _ = storage.RemoveManifestDescByReference(index, notationManifest.Digest.String(), false) + _, _ = common.RemoveManifestDescByReference(index, notationManifest.Digest.String(), false) err := oci.PutIndex(context.Background(), *index) if err != nil { @@ -1910,7 +1902,7 @@ func (is *ImageStoreLocal) dedupeBlobs(digest godigest.Digest, duplicateBlobs [] tempLinkBlobDir := path.Join(strings.Replace(blobPath, path.Join("blobs/sha256", binfo.Name()), "", 1), storageConstants.BlobUploadDir) - if err := os.MkdirAll(tempLinkBlobDir, DefaultDirPerms); err != nil { + if err := os.MkdirAll(tempLinkBlobDir, storageConstants.DefaultDirPerms); err != nil { is.log.Error().Err(err).Str("dir", tempLinkBlobDir).Msg("rebuild dedupe: unable to mkdir") return err @@ -1962,7 +1954,7 @@ func (is *ImageStoreLocal) RunDedupeForDigest(digest godigest.Digest, dedupe boo func (is *ImageStoreLocal) RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) { // for local storage no need to undedupe blobs if is.dedupe { - generator := &storage.DedupeTaskGenerator{ + generator := &common.DedupeTaskGenerator{ ImgStore: is, Dedupe: is.dedupe, Log: is.log, diff --git a/pkg/storage/local/local_elevated_test.go b/pkg/storage/local/local_elevated_test.go index 7a74d14c..07568921 100644 --- a/pkg/storage/local/local_elevated_test.go +++ b/pkg/storage/local/local_elevated_test.go @@ -20,6 +20,7 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/cache" + storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/local" ) @@ -35,7 +36,8 @@ func TestElevatedPrivilegesInvalidDedupe(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, + metrics, nil, cacheDriver) upload, err := imgStore.NewBlobUpload("dedupe1") So(err, ShouldBeNil) diff --git a/pkg/storage/local/local_test.go b/pkg/storage/local/local_test.go index c2953e75..b80ac558 100644 --- a/pkg/storage/local/local_test.go +++ b/pkg/storage/local/local_test.go @@ -34,7 +34,9 @@ import ( "zotregistry.io/zot/pkg/storage/cache" storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/local" + storageTypes "zotregistry.io/zot/pkg/storage/types" "zotregistry.io/zot/pkg/test" + "zotregistry.io/zot/pkg/test/inject" "zotregistry.io/zot/pkg/test/mocks" ) @@ -65,7 +67,7 @@ func TestStorageFSAPIs(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) Convey("Repo layout", t, func(c C) { @@ -202,7 +204,7 @@ func TestGetOrasReferrers(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) Convey("Get referrers", t, func(c C) { err := test.CopyFiles("../../../test/data/zot-test", path.Join(dir, "zot-test")) @@ -257,7 +259,8 @@ func FuzzNewBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, + cacheDriver) _, err := imgStore.NewBlobUpload(data) if err != nil { @@ -282,7 +285,8 @@ func FuzzPutBlobChunk(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, + cacheDriver) repoName := data uuid, err := imgStore.NewBlobUpload(repoName) @@ -315,7 +319,8 @@ func FuzzPutBlobChunkStreamed(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, + cacheDriver) repoName := data @@ -347,7 +352,8 @@ func FuzzGetBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, + cacheDriver) _, err := imgStore.GetBlobUpload(data1, data2) if err != nil { @@ -372,7 +378,8 @@ func FuzzTestPutGetImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) cblob, cdigest := test.GetRandomImageConfig() @@ -423,7 +430,8 @@ func FuzzTestPutDeleteImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) cblob, cdigest := test.GetRandomImageConfig() @@ -481,7 +489,8 @@ func FuzzTestDeleteImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) digest, _, err := newRandomBlobForFuzz(data) if err != nil { @@ -516,7 +525,8 @@ func FuzzInitRepo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) err := imgStore.InitRepo(data) if err != nil { if isKnownErr(err) { @@ -540,7 +550,8 @@ func FuzzInitValidateRepo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) err := imgStore.InitRepo(data) if err != nil { if isKnownErr(err) { @@ -571,7 +582,8 @@ func FuzzGetImageTags(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) _, err := imgStore.GetImageTags(data) if err != nil { if errors.Is(err, zerr.ErrRepoNotFound) || isKnownErr(err) { @@ -595,7 +607,8 @@ func FuzzBlobUploadPath(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) _ = imgStore.BlobUploadPath(repo, uuid) }) @@ -614,7 +627,8 @@ func FuzzBlobUploadInfo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) repo := data _, err := imgStore.BlobUploadInfo(repo, uuid) @@ -639,7 +653,8 @@ func FuzzTestGetImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, + cacheDriver) repoName := data @@ -667,7 +682,8 @@ func FuzzFinishBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, + cacheDriver) repoName := data @@ -716,7 +732,8 @@ func FuzzFullBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) ldigest, lblob, err := newRandomBlobForFuzz(data) if err != nil { @@ -746,7 +763,7 @@ func TestStorageCacheErrors(t *testing.T) { cblob, cdigest := test.GetRandomImageConfig() getBlobPath := "" - imgStore := local.NewImageStore(dir, false, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, &mocks.CacheMock{ PutBlobFn: func(digest godigest.Digest, path string) error { if strings.Contains(path, dedupedRepo) { @@ -788,7 +805,8 @@ func FuzzDedupeBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) blobDigest := godigest.FromString(data) @@ -829,7 +847,8 @@ func FuzzDeleteBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) uuid, err := imgStore.NewBlobUpload(repoName) if err != nil { @@ -860,7 +879,8 @@ func FuzzBlobPath(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) digest := godigest.FromString(data) _ = imgStore.BlobPath(repoName, digest) @@ -881,7 +901,8 @@ func FuzzCheckBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -912,7 +933,8 @@ func FuzzGetBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -950,7 +972,8 @@ func FuzzDeleteBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -985,7 +1008,8 @@ func FuzzGetIndexContent(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -1020,7 +1044,8 @@ func FuzzGetBlobContent(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -1054,7 +1079,8 @@ func FuzzGetOrasReferrers(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) err := test.CopyFiles("../../../test/data/zot-test", path.Join(dir, "zot-test")) if err != nil { @@ -1114,7 +1140,8 @@ func FuzzRunGCRepo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, *log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, + cacheDriver) if err := imgStore.RunGCRepo(data); err != nil { t.Error(err) @@ -1149,13 +1176,13 @@ func TestDedupeLinks(t *testing.T) { UseRelPaths: true, }, log) - var imgStore storage.ImageStore + var imgStore storageTypes.ImageStore if testCase.dedupe { - imgStore = local.NewImageStore(dir, false, storage.DefaultGCDelay, + imgStore = local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, testCase.dedupe, true, log, metrics, nil, cacheDriver) } else { - imgStore = local.NewImageStore(dir, false, storage.DefaultGCDelay, + imgStore = local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, testCase.dedupe, true, log, metrics, nil, nil) } @@ -1299,7 +1326,7 @@ func TestDedupeLinks(t *testing.T) { Convey("test RunDedupeForDigest directly, trigger stat error on original blob", func() { // rebuild with dedupe true - imgStore := local.NewImageStore(dir, false, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) duplicateBlobs := []string{ @@ -1319,7 +1346,7 @@ func TestDedupeLinks(t *testing.T) { for i := 0; i < 10; i++ { taskScheduler, cancel := runAndGetScheduler() // rebuild with dedupe true - imgStore := local.NewImageStore(dir, false, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) @@ -1332,7 +1359,7 @@ func TestDedupeLinks(t *testing.T) { taskScheduler, cancel := runAndGetScheduler() // rebuild with dedupe true - imgStore := local.NewImageStore(dir, false, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) @@ -1352,7 +1379,7 @@ func TestDedupeLinks(t *testing.T) { // switch dedupe to true from false taskScheduler, cancel := runAndGetScheduler() - imgStore := local.NewImageStore(dir, false, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, nil) // rebuild with dedupe true @@ -1375,7 +1402,7 @@ func TestDedupeLinks(t *testing.T) { // switch dedupe to true from false taskScheduler, cancel := runAndGetScheduler() - imgStore := local.NewImageStore(dir, false, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, &mocks.CacheMock{ HasBlobFn: func(digest godigest.Digest, path string) bool { return false @@ -1404,7 +1431,7 @@ func TestDedupeLinks(t *testing.T) { // switch dedupe to true from false taskScheduler, cancel := runAndGetScheduler() - imgStore := local.NewImageStore(dir, false, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, &mocks.CacheMock{ HasBlobFn: func(digest godigest.Digest, path string) bool { return false @@ -1479,7 +1506,7 @@ func TestDedupeLinks(t *testing.T) { func TestDedupe(t *testing.T) { Convey("Dedupe", t, func(c C) { Convey("Nil ImageStore", func() { - var is storage.ImageStore + var is storageTypes.ImageStore So(func() { _ = is.DedupeBlob("", "", "") }, ShouldPanic) }) @@ -1493,7 +1520,7 @@ func TestDedupe(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - il := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) + il := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) So(il.DedupeBlob("", "", ""), ShouldNotBeNil) }) @@ -1512,7 +1539,7 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - So(local.NewImageStore(dir, true, storage.DefaultGCDelay, true, + So(local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver), ShouldNotBeNil) if os.Geteuid() != 0 { cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ @@ -1520,7 +1547,7 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - So(local.NewImageStore("/deadBEEF", true, storage.DefaultGCDelay, + So(local.NewImageStore("/deadBEEF", true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver), ShouldBeNil) } }) @@ -1535,7 +1562,7 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) err := os.Chmod(dir, 0o000) // remove all perms @@ -1585,7 +1612,7 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) So(imgStore, ShouldNotBeNil) @@ -1705,7 +1732,7 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) So(imgStore, ShouldNotBeNil) @@ -1734,7 +1761,7 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, true, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) So(imgStore, ShouldNotBeNil) @@ -1781,7 +1808,7 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) So(imgStore, ShouldNotBeNil) @@ -1952,11 +1979,11 @@ func TestInjectWriteFile(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) Convey("Failure path1", func() { - injected := test.InjectFailure(0) + injected := inject.InjectFailure(0) err := imgStore.InitRepo("repo1") if injected { @@ -1967,7 +1994,7 @@ func TestInjectWriteFile(t *testing.T) { }) Convey("Failure path2", func() { - injected := test.InjectFailure(1) + injected := inject.InjectFailure(1) err := imgStore.InitRepo("repo2") if injected { @@ -1988,7 +2015,7 @@ func TestInjectWriteFile(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, false, log, metrics, nil, cacheDriver) Convey("Failure path not reached", func() { @@ -2011,7 +2038,7 @@ func TestGarbageCollect(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) repoName := "gc-long" @@ -2693,7 +2720,7 @@ func TestInitRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) err := os.Mkdir(path.Join(dir, "test-dir"), 0o000) @@ -2715,7 +2742,7 @@ func TestValidateRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) err := os.Mkdir(path.Join(dir, "test-dir"), 0o000) @@ -2735,7 +2762,7 @@ func TestValidateRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) _, err := imgStore.ValidateRepo(".") @@ -2780,7 +2807,7 @@ func TestGetRepositories(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver, ) @@ -2877,7 +2904,7 @@ func TestGetRepositories(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver, ) @@ -2925,7 +2952,7 @@ func TestGetRepositories(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(rootDir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(rootDir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver, ) @@ -2968,7 +2995,7 @@ func TestGetNextRepository(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver, ) firstRepoName := "repo1" @@ -3013,7 +3040,7 @@ func TestPutBlobChunkStreamed(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) uuid, err := imgStore.NewBlobUpload("test") @@ -3042,7 +3069,7 @@ func TestPullRange(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) repoName := "pull-range" diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index faf23735..6837386e 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -29,10 +29,11 @@ import ( zlog "zotregistry.io/zot/pkg/log" zreg "zotregistry.io/zot/pkg/regexp" "zotregistry.io/zot/pkg/scheduler" - "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/cache" + common "zotregistry.io/zot/pkg/storage/common" storageConstants "zotregistry.io/zot/pkg/storage/constants" - "zotregistry.io/zot/pkg/test" + storageTypes "zotregistry.io/zot/pkg/storage/types" + "zotregistry.io/zot/pkg/test/inject" ) const ( @@ -41,15 +42,14 @@ const ( // ObjectStorage provides the image storage operations. type ObjectStorage struct { - rootDir string - store driver.StorageDriver - lock *sync.RWMutex - blobUploads map[string]storage.BlobUpload - log zerolog.Logger - metrics monitoring.MetricServer - cache cache.Cache - dedupe bool - linter storage.Lint + rootDir string + store driver.StorageDriver + lock *sync.RWMutex + log zerolog.Logger + metrics monitoring.MetricServer + cache cache.Cache + dedupe bool + linter common.Lint } func (is *ObjectStorage) RootDir() string { @@ -68,18 +68,17 @@ func (is *ObjectStorage) DirExists(d string) bool { // see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers // Use the last argument to properly set a cache database, or it will default to boltDB local storage. func NewImageStore(rootDir string, cacheDir string, gc bool, gcDelay time.Duration, dedupe, commit bool, - log zlog.Logger, metrics monitoring.MetricServer, linter storage.Lint, + log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, store driver.StorageDriver, cacheDriver cache.Cache, -) storage.ImageStore { +) storageTypes.ImageStore { imgStore := &ObjectStorage{ - rootDir: rootDir, - store: store, - lock: &sync.RWMutex{}, - blobUploads: make(map[string]storage.BlobUpload), - log: log.With().Caller().Logger(), - metrics: metrics, - dedupe: dedupe, - linter: linter, + rootDir: rootDir, + store: store, + lock: &sync.RWMutex{}, + log: log.With().Caller().Logger(), + metrics: metrics, + dedupe: dedupe, + linter: linter, } imgStore.cache = cacheDriver @@ -306,12 +305,12 @@ func (is *ObjectStorage) GetImageTags(repo string) ([]string, error) { is.RLock(&lockLatency) defer is.RUnlock(&lockLatency) - index, err := storage.GetIndex(is, repo, is.log) + index, err := common.GetIndex(is, repo, is.log) if err != nil { return nil, err } - return storage.GetTagsByIndex(index), nil + return common.GetTagsByIndex(index), nil } // GetImageManifest returns the image manifest of an image in the specific repository. @@ -326,12 +325,12 @@ func (is *ObjectStorage) GetImageManifest(repo, reference string) ([]byte, godig is.RLock(&lockLatency) defer is.RUnlock(&lockLatency) - index, err := storage.GetIndex(is, repo, is.log) + index, err := common.GetIndex(is, repo, is.log) if err != nil { return nil, "", "", zerr.ErrRepoNotFound } - manifestDesc, found := storage.GetManifestDescByReference(index, reference) + manifestDesc, found := common.GetManifestDescByReference(index, reference) if !found { return nil, "", "", zerr.ErrManifestNotFound } @@ -372,14 +371,14 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n is.Lock(&lockLatency) defer is.Unlock(&lockLatency) - dig, err := storage.ValidateManifest(is, repo, reference, mediaType, body, is.log) + dig, err := common.ValidateManifest(is, repo, reference, mediaType, body, is.log) if err != nil { return dig, "", err } refIsDigest := true - mDigest, err := storage.GetAndValidateRequestDigest(body, reference, is.log) + mDigest, err := common.GetAndValidateRequestDigest(body, reference, is.log) if err != nil { if errors.Is(err, zerr.ErrBadManifest) { return mDigest, "", err @@ -388,7 +387,7 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n refIsDigest = false } - index, err := storage.GetIndex(is, repo, is.log) + index, err := common.GetIndex(is, repo, is.log) if err != nil { return "", "", err } @@ -421,7 +420,7 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n artifactType = zcommon.GetManifestArtifactType(manifest) } - updateIndex, oldDgst, err := storage.CheckIfIndexNeedsUpdate(&index, &desc, is.log) + updateIndex, oldDgst, err := common.CheckIfIndexNeedsUpdate(&index, &desc, is.log) if err != nil { return "", "", err } @@ -440,7 +439,7 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n return "", "", err } - err = storage.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) + err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) if err != nil { return "", "", err } @@ -461,7 +460,7 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n desc.ArtifactType = artifactType // apply linter only on images, not signatures - pass, err := storage.ApplyLinter(is, is.linter, repo, desc) + pass, err := common.ApplyLinter(is, is.linter, repo, desc) if !pass { is.log.Error().Err(err).Str("repository", repo).Str("reference", reference).Msg("linter didn't pass") @@ -492,17 +491,17 @@ func (is *ObjectStorage) DeleteImageManifest(repo, reference string, detectColli is.Lock(&lockLatency) defer is.Unlock(&lockLatency) - index, err := storage.GetIndex(is, repo, is.log) + index, err := common.GetIndex(is, repo, is.log) if err != nil { return err } - manifestDesc, err := storage.RemoveManifestDescByReference(&index, reference, detectCollisions) + manifestDesc, err := common.RemoveManifestDescByReference(&index, reference, detectCollisions) if err != nil { return err } - err = storage.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log) + err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log) if err != nil { return err } @@ -862,7 +861,7 @@ retry: is.log.Debug().Str("src", src).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: enter") dstRecord, err := is.cache.GetBlob(dstDigest) - if err := test.Error(err); err != nil && !errors.Is(err, zerr.ErrCacheMiss) { + if err := inject.Error(err); err != nil && !errors.Is(err, zerr.ErrCacheMiss) { is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to lookup blob record") return err @@ -892,7 +891,7 @@ retry: is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat") // the actual blob on disk may have been removed by GC, so sync the cache err := is.cache.DeleteBlob(dstDigest, dstRecord) - if err = test.Error(err); err != nil { + if err = inject.Error(err); err != nil { //nolint:lll is.log.Error().Err(err).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: unable to delete blob record") @@ -1296,7 +1295,7 @@ func (is *ObjectStorage) GetReferrers(repo string, gdigest godigest.Digest, arti is.RLock(&lockLatency) defer is.RUnlock(&lockLatency) - return storage.GetReferrers(is, repo, gdigest, artifactTypes, is.log) + return common.GetReferrers(is, repo, gdigest, artifactTypes, is.log) } func (is *ObjectStorage) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string, @@ -1306,7 +1305,7 @@ func (is *ObjectStorage) GetOrasReferrers(repo string, gdigest godigest.Digest, is.RLock(&lockLatency) defer is.RUnlock(&lockLatency) - return storage.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) + return common.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) } // GetIndexContent returns index.json contents, SHOULD lock from outside. @@ -1639,7 +1638,7 @@ func (is *ObjectStorage) RunDedupeForDigest(digest godigest.Digest, dedupe bool, } func (is *ObjectStorage) RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) { - generator := &storage.DedupeTaskGenerator{ + generator := &common.DedupeTaskGenerator{ ImgStore: is, Dedupe: is.dedupe, Log: is.log, diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 552d41fd..5229e01e 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -35,7 +35,9 @@ import ( "zotregistry.io/zot/pkg/storage/cache" storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/s3" + storageTypes "zotregistry.io/zot/pkg/storage/types" "zotregistry.io/zot/pkg/test" + "zotregistry.io/zot/pkg/test/inject" "zotregistry.io/zot/pkg/test/mocks" ) @@ -67,7 +69,8 @@ func skipIt(t *testing.T) { } } -func createMockStorage(rootDir string, cacheDir string, dedupe bool, store driver.StorageDriver) storage.ImageStore { +func createMockStorage(rootDir string, cacheDir string, dedupe bool, store driver.StorageDriver, +) storageTypes.ImageStore { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) @@ -81,7 +84,7 @@ func createMockStorage(rootDir string, cacheDir string, dedupe bool, store drive UseRelPaths: false, }, log) } - il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, + il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay, dedupe, false, log, metrics, nil, store, cacheDriver, ) @@ -90,11 +93,11 @@ func createMockStorage(rootDir string, cacheDir string, dedupe bool, store drive func createMockStorageWithMockCache(rootDir string, dedupe bool, store driver.StorageDriver, cacheDriver cache.Cache, -) storage.ImageStore { +) storageTypes.ImageStore { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - il := s3.NewImageStore(rootDir, "", false, storage.DefaultGCDelay, + il := s3.NewImageStore(rootDir, "", false, storageConstants.DefaultGCDelay, dedupe, false, log, metrics, nil, store, cacheDriver, ) @@ -134,7 +137,7 @@ func createStoreDriver(rootDir string) driver.StorageDriver { func createObjectsStore(rootDir string, cacheDir string, dedupe bool) ( driver.StorageDriver, - storage.ImageStore, + storageTypes.ImageStore, error, ) { store := createStoreDriver(rootDir) @@ -156,7 +159,7 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) ( }, log) } - il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, + il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay, dedupe, false, log, metrics, nil, store, cacheDriver) return store, il, err @@ -164,7 +167,7 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) ( func createObjectsStoreDynamo(rootDir string, cacheDir string, dedupe bool, tableName string) ( driver.StorageDriver, - storage.ImageStore, + storageTypes.ImageStore, error, ) { store := createStoreDriver(rootDir) @@ -191,7 +194,7 @@ func createObjectsStoreDynamo(rootDir string, cacheDir string, dedupe bool, tabl panic(err) } - il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, + il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay, dedupe, false, log, metrics, nil, store, cacheDriver) return store, il, err @@ -730,7 +733,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { controller := api.NewController(conf) So(controller, ShouldNotBeNil) - err = controller.InitImageStore(context.Background()) + err = controller.InitImageStore() So(err, ShouldBeNil) }) @@ -3781,7 +3784,7 @@ func TestInjectDedupe(t *testing.T) { err := imgStore.DedupeBlob("blob", "digest", "newblob") So(err, ShouldBeNil) - injected := test.InjectFailure(0) + injected := inject.InjectFailure(0) err = imgStore.DedupeBlob("blob", "digest", "newblob") if injected { So(err, ShouldNotBeNil) @@ -3789,7 +3792,7 @@ func TestInjectDedupe(t *testing.T) { So(err, ShouldBeNil) } - injected = test.InjectFailure(1) + injected = inject.InjectFailure(1) err = imgStore.DedupeBlob("blob", "digest", "newblob") if injected { So(err, ShouldNotBeNil) diff --git a/pkg/storage/scrub.go b/pkg/storage/scrub.go index 1a8073e8..492a0d07 100644 --- a/pkg/storage/scrub.go +++ b/pkg/storage/scrub.go @@ -17,6 +17,7 @@ import ( "github.com/opencontainers/umoci/oci/casext" "zotregistry.io/zot/errors" + storageTypes "zotregistry.io/zot/pkg/storage/types" ) const ( @@ -45,7 +46,7 @@ type ScrubResults struct { func (sc StoreController) CheckAllBlobsIntegrity() (ScrubResults, error) { results := ScrubResults{} - imageStoreList := make(map[string]ImageStore) + imageStoreList := make(map[string]storageTypes.ImageStore) if sc.SubStore != nil { imageStoreList = sc.SubStore } @@ -64,7 +65,7 @@ func (sc StoreController) CheckAllBlobsIntegrity() (ScrubResults, error) { return results, nil } -func CheckImageStoreBlobsIntegrity(imgStore ImageStore) ([]ScrubImageResult, error) { +func CheckImageStoreBlobsIntegrity(imgStore storageTypes.ImageStore) ([]ScrubImageResult, error) { results := []ScrubImageResult{} repos, err := imgStore.GetRepositories() @@ -84,7 +85,7 @@ func CheckImageStoreBlobsIntegrity(imgStore ImageStore) ([]ScrubImageResult, err return results, nil } -func CheckRepo(imageName string, imgStore ImageStore) ([]ScrubImageResult, error) { +func CheckRepo(imageName string, imgStore storageTypes.ImageStore) ([]ScrubImageResult, error) { results := []ScrubImageResult{} dir := path.Join(imgStore.RootDir(), imageName) diff --git a/pkg/storage/scrub_test.go b/pkg/storage/scrub_test.go index 521434a2..e90e4a10 100644 --- a/pkg/storage/scrub_test.go +++ b/pkg/storage/scrub_test.go @@ -17,6 +17,8 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/cache" + common "zotregistry.io/zot/pkg/storage/common" + storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/test" ) @@ -37,7 +39,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, + imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) Convey("Scrub only one repo", t, func(c C) { @@ -111,7 +113,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) { // verify error message So(actual, ShouldContainSubstring, "test 1.0 affected parse application/vnd.oci.image.manifest.v1+json") - index, err := storage.GetIndex(imgStore, repoName, log.With().Caller().Logger()) + index, err := common.GetIndex(imgStore, repoName, log.With().Caller().Logger()) So(err, ShouldBeNil) So(len(index.Manifests), ShouldEqual, 1) @@ -191,7 +193,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) { err = os.Chmod(layerFile, 0x0200) So(err, ShouldBeNil) - index, err := storage.GetIndex(imgStore, repoName, log.With().Caller().Logger()) + index, err := common.GetIndex(imgStore, repoName, log.With().Caller().Logger()) So(err, ShouldBeNil) So(len(index.Manifests), ShouldEqual, 1) @@ -325,7 +327,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) { So(actual, ShouldContainSubstring, "test 1.0 affected") So(actual, ShouldContainSubstring, "no such file or directory") - index, err := storage.GetIndex(imgStore, repoName, log.With().Caller().Logger()) + index, err := common.GetIndex(imgStore, repoName, log.With().Caller().Logger()) So(err, ShouldBeNil) So(len(index.Manifests), ShouldEqual, 2) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 8be45b1b..7aab199d 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -1,59 +1,247 @@ package storage import ( - "io" - "time" + "encoding/json" + "fmt" + "strings" + "github.com/docker/distribution/registry/storage/driver/factory" + "github.com/gobwas/glob" + notreg "github.com/notaryproject/notation-go/registry" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" - artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" - "zotregistry.io/zot/pkg/scheduler" + "zotregistry.io/zot/errors" + "zotregistry.io/zot/pkg/api/config" + zcommon "zotregistry.io/zot/pkg/common" + "zotregistry.io/zot/pkg/extensions/monitoring" + "zotregistry.io/zot/pkg/log" + common "zotregistry.io/zot/pkg/storage/common" + "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/storage/local" + "zotregistry.io/zot/pkg/storage/s3" + storageTypes "zotregistry.io/zot/pkg/storage/types" ) -const ( - S3StorageDriverName = "s3" - DefaultGCDelay = 1 * time.Hour -) +func New(config *config.Config, linter common.Lint, metrics monitoring.MetricServer, + log log.Logger, +) (StoreController, error) { + storeController := StoreController{} -type ImageStore interface { //nolint:interfacebloat - DirExists(d string) bool - RootDir() string - RLock(*time.Time) - RUnlock(*time.Time) - Lock(*time.Time) - Unlock(*time.Time) - InitRepo(name string) error - ValidateRepo(name string) (bool, error) - GetRepositories() ([]string, error) - GetNextRepository(repo string) (string, error) - GetImageTags(repo string) ([]string, error) - GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) - PutImageManifest(repo, reference, mediaType string, body []byte) (godigest.Digest, godigest.Digest, error) - DeleteImageManifest(repo, reference string, detectCollision bool) error - BlobUploadPath(repo, uuid string) string - NewBlobUpload(repo string) (string, error) - GetBlobUpload(repo, uuid string) (int64, error) - PutBlobChunkStreamed(repo, uuid string, body io.Reader) (int64, error) - PutBlobChunk(repo, uuid string, from, to int64, body io.Reader) (int64, error) - BlobUploadInfo(repo, uuid string) (int64, error) - FinishBlobUpload(repo, uuid string, body io.Reader, digest godigest.Digest) error - FullBlobUpload(repo string, body io.Reader, digest godigest.Digest) (string, int64, error) - DedupeBlob(src string, dstDigest godigest.Digest, dst string) error - DeleteBlobUpload(repo, uuid string) error - BlobPath(repo string, digest godigest.Digest) string - CheckBlob(repo string, digest godigest.Digest) (bool, int64, error) - GetBlob(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) - GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, - ) (io.ReadCloser, int64, int64, error) - DeleteBlob(repo string, digest godigest.Digest) error - GetIndexContent(repo string) ([]byte, error) - GetBlobContent(repo string, digest godigest.Digest) ([]byte, error) - GetReferrers(repo string, digest godigest.Digest, artifactTypes []string) (ispec.Index, error) - GetOrasReferrers(repo string, digest godigest.Digest, artifactType string) ([]artifactspec.Descriptor, error) - RunGCRepo(repo string) error - RunGCPeriodically(interval time.Duration, sch *scheduler.Scheduler) - RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) - RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error - GetNextDigestWithBlobPaths(lastDigests []godigest.Digest) (godigest.Digest, []string, error) + if config.Storage.RootDirectory == "" { + // we can't proceed without global storage + log.Error().Err(errors.ErrImgStoreNotFound).Msg("controller: no storage config provided") + + return storeController, errors.ErrImgStoreNotFound + } + + // no need to validate hard links work on s3 + if config.Storage.Dedupe && config.Storage.StorageDriver == nil { + err := local.ValidateHardLink(config.Storage.RootDirectory) + if err != nil { + log.Warn().Msg("input storage root directory filesystem does not supports hardlinking," + + "disabling dedupe functionality") + + config.Storage.Dedupe = false + } + } + + var defaultStore storageTypes.ImageStore + + if config.Storage.StorageDriver == nil { + // false positive lint - linter does not implement Lint method + //nolint:typecheck,contextcheck + defaultStore = local.NewImageStore(config.Storage.RootDirectory, + config.Storage.GC, config.Storage.GCDelay, + config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, + CreateCacheDatabaseDriver(config.Storage.StorageConfig, log), + ) + } else { + storeName := fmt.Sprintf("%v", config.Storage.StorageDriver["name"]) + if storeName != constants.S3StorageDriverName { + log.Fatal().Err(errors.ErrBadConfig).Str("storageDriver", storeName). + Msg("unsupported storage driver") + } + // Init a Storager from connection string. + store, err := factory.Create(storeName, config.Storage.StorageDriver) + if err != nil { + log.Error().Err(err).Str("rootDir", config.Storage.RootDirectory).Msg("unable to create s3 service") + + return storeController, err + } + + /* in the case of s3 config.Storage.RootDirectory is used for caching blobs locally and + config.Storage.StorageDriver["rootdirectory"] is the actual rootDir in s3 */ + rootDir := "/" + if config.Storage.StorageDriver["rootdirectory"] != nil { + rootDir = fmt.Sprintf("%v", config.Storage.StorageDriver["rootdirectory"]) + } + + // false positive lint - linter does not implement Lint method + //nolint: typecheck,contextcheck + defaultStore = s3.NewImageStore(rootDir, config.Storage.RootDirectory, + config.Storage.GC, config.Storage.GCDelay, config.Storage.Dedupe, + config.Storage.Commit, log, metrics, linter, store, + CreateCacheDatabaseDriver(config.Storage.StorageConfig, log)) + } + + storeController.DefaultStore = defaultStore + + if config.Storage.SubPaths != nil { + if len(config.Storage.SubPaths) > 0 { + subPaths := config.Storage.SubPaths + + //nolint: contextcheck + subImageStore, err := getSubStore(config, subPaths, linter, metrics, log) + if err != nil { + log.Error().Err(err).Msg("controller: error getting sub image store") + + return storeController, err + } + + storeController.SubStore = subImageStore + } + } + + return storeController, nil +} + +func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig, + linter common.Lint, metrics monitoring.MetricServer, log log.Logger, +) (map[string]storageTypes.ImageStore, error) { + imgStoreMap := make(map[string]storageTypes.ImageStore, 0) + + subImageStore := make(map[string]storageTypes.ImageStore) + + // creating image store per subpaths + for route, storageConfig := range subPaths { + // no need to validate hard links work on s3 + if storageConfig.Dedupe && storageConfig.StorageDriver == nil { + err := local.ValidateHardLink(storageConfig.RootDirectory) + if err != nil { + log.Warn().Msg("input storage root directory filesystem does not supports hardlinking, " + + "disabling dedupe functionality") + + storageConfig.Dedupe = false + } + } + + if storageConfig.StorageDriver == nil { + // Compare if subpath root dir is same as default root dir + isSame, _ := config.SameFile(cfg.Storage.RootDirectory, storageConfig.RootDirectory) + + if isSame { + log.Error().Err(errors.ErrBadConfig).Msg("sub path storage directory is same as root directory") + + return nil, errors.ErrBadConfig + } + + isUnique := true + + // Compare subpath unique files + for file := range imgStoreMap { + // We already have image storage for this file + if compareImageStore(file, storageConfig.RootDirectory) { + subImageStore[route] = imgStoreMap[file] + + isUnique = false + } + } + + // subpath root directory is unique + // add it to uniqueSubFiles + // Create a new image store and assign it to imgStoreMap + if isUnique { + imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(storageConfig.RootDirectory, + storageConfig.GC, storageConfig.GCDelay, storageConfig.Dedupe, + storageConfig.Commit, log, metrics, linter, CreateCacheDatabaseDriver(storageConfig, log)) + + subImageStore[route] = imgStoreMap[storageConfig.RootDirectory] + } + } else { + storeName := fmt.Sprintf("%v", storageConfig.StorageDriver["name"]) + if storeName != constants.S3StorageDriverName { + log.Fatal().Err(errors.ErrBadConfig).Str("storageDriver", storeName). + Msg("unsupported storage driver") + } + + // Init a Storager from connection string. + store, err := factory.Create(storeName, storageConfig.StorageDriver) + if err != nil { + log.Error().Err(err).Str("rootDir", storageConfig.RootDirectory).Msg("Unable to create s3 service") + + return nil, err + } + + /* in the case of s3 c.Config.Storage.RootDirectory is used for caching blobs locally and + c.Config.Storage.StorageDriver["rootdirectory"] is the actual rootDir in s3 */ + rootDir := "/" + if cfg.Storage.StorageDriver["rootdirectory"] != nil { + rootDir = fmt.Sprintf("%v", cfg.Storage.StorageDriver["rootdirectory"]) + } + + // false positive lint - linter does not implement Lint method + //nolint: typecheck + subImageStore[route] = s3.NewImageStore(rootDir, storageConfig.RootDirectory, + storageConfig.GC, storageConfig.GCDelay, + storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store, + CreateCacheDatabaseDriver(storageConfig, log), + ) + } + } + + return subImageStore, nil +} + +func compareImageStore(root1, root2 string) bool { + isSameFile, err := config.SameFile(root1, root2) + // This error is path error that means either of root directory doesn't exist, in that case do string match + if err != nil { + return strings.EqualFold(root1, root2) + } + + return isSameFile +} + +// CheckIsImageSignature checks if the given image (repo:tag) represents a signature. The function +// returns: +// +// - bool: if the image is a signature or not +// +// - string: the type of signature +// +// - string: the digest of the image it signs +// +// - error: any errors that occur. +func CheckIsImageSignature(repoName string, manifestBlob []byte, reference string, +) (bool, string, godigest.Digest, error) { + var manifestContent ispec.Manifest + + err := json.Unmarshal(manifestBlob, &manifestContent) + if err != nil { + return false, "", "", err + } + + manifestArtifactType := zcommon.GetManifestArtifactType(manifestContent) + + // check notation signature + if manifestArtifactType == notreg.ArtifactTypeNotation && manifestContent.Subject != nil { + return true, NotationType, manifestContent.Subject.Digest, nil + } + + // check cosign + cosignTagRule := glob.MustCompile("sha256-*.sig") + + if tag := reference; cosignTagRule.Match(reference) { + prefixLen := len("sha256-") + digestLen := 64 + signedImageManifestDigestEncoded := tag[prefixLen : prefixLen+digestLen] + + signedImageManifestDigest := godigest.NewDigestFromEncoded(godigest.SHA256, + signedImageManifestDigestEncoded) + + return true, CosignType, signedImageManifestDigest, nil + } + + return false, "", "", nil } diff --git a/pkg/storage/storage_controller.go b/pkg/storage/storage_controller.go index 5668f847..0dfdf3d1 100644 --- a/pkg/storage/storage_controller.go +++ b/pkg/storage/storage_controller.go @@ -3,17 +3,18 @@ package storage import ( "fmt" "strings" + + storageTypes "zotregistry.io/zot/pkg/storage/types" +) + +const ( + CosignType = "cosign" + NotationType = "notation" ) type StoreController struct { - DefaultStore ImageStore - SubStore map[string]ImageStore -} - -// BlobUpload models and upload request. -type BlobUpload struct { - StoreName string - ID string + DefaultStore storageTypes.ImageStore + SubStore map[string]storageTypes.ImageStore } func GetRoutePrefix(name string) string { @@ -29,7 +30,7 @@ func GetRoutePrefix(name string) string { return fmt.Sprintf("/%s", names[0]) } -func (sc StoreController) GetImageStore(name string) ImageStore { +func (sc StoreController) GetImageStore(name string) storageTypes.ImageStore { if sc.SubStore != nil { // SubStore is being provided, now we need to find equivalent image store and this will be found by splitting name prefixName := GetRoutePrefix(name) diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index f1ff3e48..2279b564 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -30,8 +30,10 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/cache" + storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/storage/s3" + storageTypes "zotregistry.io/zot/pkg/storage/types" "zotregistry.io/zot/pkg/test" "zotregistry.io/zot/pkg/test/mocks" ) @@ -48,7 +50,7 @@ func skipIt(t *testing.T) { } } -func createObjectsStore(rootDir string, cacheDir string) (driver.StorageDriver, storage.ImageStore, error) { +func createObjectsStore(rootDir string, cacheDir string) (driver.StorageDriver, storageTypes.ImageStore, error) { bucket := "zot-storage-test" endpoint := os.Getenv("S3MOCK_ENDPOINT") storageDriverParams := map[string]interface{}{ @@ -85,7 +87,7 @@ func createObjectsStore(rootDir string, cacheDir string) (driver.StorageDriver, UseRelPaths: false, }, log) - il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, + il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay, true, false, log, metrics, nil, store, cacheDriver, ) @@ -111,7 +113,7 @@ func TestStorageAPIs(t *testing.T) { for _, testcase := range testCases { testcase := testcase t.Run(testcase.testCaseName, func(t *testing.T) { - var imgStore storage.ImageStore + var imgStore storageTypes.ImageStore if testcase.storageType == "s3" { skipIt(t) @@ -136,7 +138,7 @@ func TestStorageAPIs(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore = local.NewImageStore(dir, true, storage.DefaultGCDelay, true, + imgStore = local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) } @@ -710,7 +712,7 @@ func TestMandatoryAnnotations(t *testing.T) { for _, testcase := range testCases { testcase := testcase t.Run(testcase.testCaseName, func(t *testing.T) { - var imgStore storage.ImageStore + var imgStore storageTypes.ImageStore var testDir, tdir string var store driver.StorageDriver @@ -731,7 +733,7 @@ func TestMandatoryAnnotations(t *testing.T) { store, _, _ = createObjectsStore(testDir, tdir) imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics, &mocks.MockedLint{ - LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) { + LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { return false, nil }, }, store, nil) @@ -744,9 +746,9 @@ func TestMandatoryAnnotations(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore = local.NewImageStore(tdir, true, storage.DefaultGCDelay, true, + imgStore = local.NewImageStore(tdir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, &mocks.MockedLint{ - LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) { + LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { return false, nil }, }, cacheDriver) @@ -798,7 +800,7 @@ func TestMandatoryAnnotations(t *testing.T) { if testcase.storageType == "s3" { imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics, &mocks.MockedLint{ - LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) { + LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { //nolint: goerr113 return false, errors.New("linter error") }, @@ -809,9 +811,9 @@ func TestMandatoryAnnotations(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore = local.NewImageStore(tdir, true, storage.DefaultGCDelay, true, + imgStore = local.NewImageStore(tdir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, &mocks.MockedLint{ - LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) { + LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { //nolint: goerr113 return false, errors.New("linter error") }, @@ -830,9 +832,9 @@ func TestStorageHandler(t *testing.T) { for _, testcase := range testCases { testcase := testcase t.Run(testcase.testCaseName, func(t *testing.T) { - var firstStore storage.ImageStore - var secondStore storage.ImageStore - var thirdStore storage.ImageStore + var firstStore storageTypes.ImageStore + var secondStore storageTypes.ImageStore + var thirdStore storageTypes.ImageStore var firstRootDir string var secondRootDir string var thirdRootDir string @@ -865,13 +867,13 @@ func TestStorageHandler(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) // Create ImageStore - firstStore = local.NewImageStore(firstRootDir, false, storage.DefaultGCDelay, + firstStore = local.NewImageStore(firstRootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) secondStore = local.NewImageStore(secondRootDir, false, - storage.DefaultGCDelay, false, false, log, metrics, nil, nil) + storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) - thirdStore = local.NewImageStore(thirdRootDir, false, storage.DefaultGCDelay, + thirdStore = local.NewImageStore(thirdRootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) } @@ -880,7 +882,7 @@ func TestStorageHandler(t *testing.T) { storeController.DefaultStore = firstStore - subStore := make(map[string]storage.ImageStore) + subStore := make(map[string]storageTypes.ImageStore) subStore["/a"] = secondStore subStore["/b"] = thirdStore diff --git a/pkg/storage/types/types.go b/pkg/storage/types/types.go new file mode 100644 index 00000000..452286a3 --- /dev/null +++ b/pkg/storage/types/types.go @@ -0,0 +1,54 @@ +package types + +import ( + "io" + "time" + + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" + + "zotregistry.io/zot/pkg/scheduler" +) + +type ImageStore interface { //nolint:interfacebloat + DirExists(d string) bool + RootDir() string + RLock(*time.Time) + RUnlock(*time.Time) + Lock(*time.Time) + Unlock(*time.Time) + InitRepo(name string) error + ValidateRepo(name string) (bool, error) + GetRepositories() ([]string, error) + GetNextRepository(repo string) (string, error) + GetImageTags(repo string) ([]string, error) + GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) + PutImageManifest(repo, reference, mediaType string, body []byte) (godigest.Digest, godigest.Digest, error) + DeleteImageManifest(repo, reference string, detectCollision bool) error + BlobUploadPath(repo, uuid string) string + NewBlobUpload(repo string) (string, error) + GetBlobUpload(repo, uuid string) (int64, error) + PutBlobChunkStreamed(repo, uuid string, body io.Reader) (int64, error) + PutBlobChunk(repo, uuid string, from, to int64, body io.Reader) (int64, error) + BlobUploadInfo(repo, uuid string) (int64, error) + FinishBlobUpload(repo, uuid string, body io.Reader, digest godigest.Digest) error + FullBlobUpload(repo string, body io.Reader, digest godigest.Digest) (string, int64, error) + DedupeBlob(src string, dstDigest godigest.Digest, dst string) error + DeleteBlobUpload(repo, uuid string) error + BlobPath(repo string, digest godigest.Digest) string + CheckBlob(repo string, digest godigest.Digest) (bool, int64, error) + GetBlob(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) + GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, + ) (io.ReadCloser, int64, int64, error) + DeleteBlob(repo string, digest godigest.Digest) error + GetIndexContent(repo string) ([]byte, error) + GetBlobContent(repo string, digest godigest.Digest) ([]byte, error) + GetReferrers(repo string, digest godigest.Digest, artifactTypes []string) (ispec.Index, error) + GetOrasReferrers(repo string, digest godigest.Digest, artifactType string) ([]artifactspec.Descriptor, error) + RunGCRepo(repo string) error + RunGCPeriodically(interval time.Duration, sch *scheduler.Scheduler) + RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) + RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error + GetNextDigestWithBlobPaths(lastDigests []godigest.Digest) (godigest.Digest, []string, error) +} diff --git a/pkg/test/common.go b/pkg/test/common.go index 3d87d1b0..c79273de 100644 --- a/pkg/test/common.go +++ b/pkg/test/common.go @@ -47,6 +47,7 @@ import ( "zotregistry.io/zot/pkg/meta/repodb" "zotregistry.io/zot/pkg/storage" + "zotregistry.io/zot/pkg/test/inject" ) const ( @@ -549,7 +550,7 @@ func GetImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manifest, e } configBlob, err := json.Marshal(config) - if err = Error(err); err != nil { + if err = inject.Error(err); err != nil { return ispec.Image{}, [][]byte{}, ispec.Manifest{}, err } @@ -597,7 +598,7 @@ func GetRandomImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manif } configBlob, err := json.Marshal(config) - if err = Error(err); err != nil { + if err = inject.Error(err); err != nil { return ispec.Image{}, [][]byte{}, ispec.Manifest{}, err } @@ -665,7 +666,7 @@ func GetRandomImage(reference string) (Image, error) { func GetImageComponentsWithConfig(conf ispec.Image) (ispec.Image, [][]byte, ispec.Manifest, error) { configBlob, err := json.Marshal(conf) - if err = Error(err); err != nil { + if err = inject.Error(err); err != nil { return ispec.Image{}, [][]byte{}, ispec.Manifest{}, err } @@ -843,7 +844,7 @@ func UploadImage(img Image, baseURL, repo string) error { } // upload config cblob, err := json.Marshal(img.Config) - if err = Error(err); err != nil { + if err = inject.Error(err); err != nil { return err } @@ -856,11 +857,11 @@ func UploadImage(img Image, baseURL, repo string) error { resp, err := resty.R(). Post(baseURL + "/v2/" + repo + "/blobs/uploads/") - if err = Error(err); err != nil { + if err = inject.Error(err); err != nil { return err } - if ErrStatusCode(resp.StatusCode()) != http.StatusAccepted || ErrStatusCode(resp.StatusCode()) == -1 { + if inject.ErrStatusCode(resp.StatusCode()) != http.StatusAccepted || inject.ErrStatusCode(resp.StatusCode()) == -1 { return ErrPostBlob } @@ -873,17 +874,17 @@ func UploadImage(img Image, baseURL, repo string) error { SetQueryParam("digest", cdigest.String()). SetBody(cblob). Put(loc) - if err = Error(err); err != nil { + if err = inject.Error(err); err != nil { return err } - if ErrStatusCode(resp.StatusCode()) != http.StatusCreated || ErrStatusCode(resp.StatusCode()) == -1 { + if inject.ErrStatusCode(resp.StatusCode()) != http.StatusCreated || inject.ErrStatusCode(resp.StatusCode()) == -1 { return ErrPostBlob } // put manifest manifestBlob, err := json.Marshal(img.Manifest) - if err = Error(err); err != nil { + if err = inject.Error(err); err != nil { return err } @@ -892,11 +893,11 @@ func UploadImage(img Image, baseURL, repo string) error { SetBody(manifestBlob). Put(baseURL + "/v2/" + repo + "/manifests/" + img.Reference) - if ErrStatusCode(resp.StatusCode()) != http.StatusCreated { + if inject.ErrStatusCode(resp.StatusCode()) != http.StatusCreated { return ErrPutBlob } - if ErrStatusCode(resp.StatusCode()) != http.StatusCreated { + if inject.ErrStatusCode(resp.StatusCode()) != http.StatusCreated { return ErrPutBlob } @@ -1512,7 +1513,7 @@ func UploadImageWithBasicAuth(img Image, baseURL, repo, user, password string) e } // upload config cblob, err := json.Marshal(img.Config) - if err = Error(err); err != nil { + if err = inject.Error(err); err != nil { return err } @@ -1526,11 +1527,11 @@ func UploadImageWithBasicAuth(img Image, baseURL, repo, user, password string) e resp, err := resty.R(). SetBasicAuth(user, password). Post(baseURL + "/v2/" + repo + "/blobs/uploads/") - if err = Error(err); err != nil { + if err = inject.Error(err); err != nil { return err } - if ErrStatusCode(resp.StatusCode()) != http.StatusAccepted || ErrStatusCode(resp.StatusCode()) == -1 { + if inject.ErrStatusCode(resp.StatusCode()) != http.StatusAccepted || inject.ErrStatusCode(resp.StatusCode()) == -1 { return ErrPostBlob } @@ -1544,17 +1545,17 @@ func UploadImageWithBasicAuth(img Image, baseURL, repo, user, password string) e SetQueryParam("digest", cdigest.String()). SetBody(cblob). Put(loc) - if err = Error(err); err != nil { + if err = inject.Error(err); err != nil { return err } - if ErrStatusCode(resp.StatusCode()) != http.StatusCreated || ErrStatusCode(resp.StatusCode()) == -1 { + if inject.ErrStatusCode(resp.StatusCode()) != http.StatusCreated || inject.ErrStatusCode(resp.StatusCode()) == -1 { return ErrPostBlob } // put manifest manifestBlob, err := json.Marshal(img.Manifest) - if err = Error(err); err != nil { + if err = inject.Error(err); err != nil { return err } @@ -1795,7 +1796,7 @@ func UploadMultiarchImage(multiImage MultiarchImage, baseURL string, repo string // put manifest indexBlob, err := json.Marshal(multiImage.Index) - if err = Error(err); err != nil { + if err = inject.Error(err); err != nil { return err } diff --git a/pkg/test/common_test.go b/pkg/test/common_test.go index 3153bf98..544bcd3d 100644 --- a/pkg/test/common_test.go +++ b/pkg/test/common_test.go @@ -24,6 +24,7 @@ import ( "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/test" + "zotregistry.io/zot/pkg/test/inject" "zotregistry.io/zot/pkg/test/mocks" ) @@ -174,7 +175,7 @@ func TestGetOciLayoutDigests(t *testing.T) { func TestGetImageComponents(t *testing.T) { Convey("Inject failures for unreachable lines", t, func() { - injected := test.InjectFailure(0) + injected := inject.InjectFailure(0) if injected { _, _, _, err := test.GetImageComponents(100) So(err, ShouldNotBeNil) @@ -186,6 +187,26 @@ func TestGetImageComponents(t *testing.T) { }) } +func TestGetRandomImageComponents(t *testing.T) { + Convey("Inject failures for unreachable lines", t, func() { + injected := inject.InjectFailure(0) + if injected { + _, _, _, err := test.GetRandomImageComponents(100) + So(err, ShouldNotBeNil) + } + }) +} + +func TestGetImageComponentsWithConfig(t *testing.T) { + Convey("Inject failures for unreachable lines", t, func() { + injected := inject.InjectFailure(0) + if injected { + _, _, _, err := test.GetImageComponentsWithConfig(ispec.Image{}) + So(err, ShouldNotBeNil) + } + }) +} + func TestWaitTillTrivyDBDownloadStarted(t *testing.T) { Convey("finishes successfully", t, func() { tempDir := t.TempDir() @@ -601,14 +622,14 @@ func TestUploadImage(t *testing.T) { } Convey("CreateBlobUpload", func() { - injected := test.InjectFailure(2) + injected := inject.InjectFailure(2) if injected { err := test.UploadImage(img, baseURL, "test") So(err, ShouldNotBeNil) } }) Convey("UpdateBlobUpload", func() { - injected := test.InjectFailure(4) + injected := inject.InjectFailure(4) if injected { err := test.UploadImage(img, baseURL, "test") So(err, ShouldNotBeNil) @@ -662,28 +683,28 @@ func TestInjectUploadImage(t *testing.T) { } Convey("first marshal", func() { - injected := test.InjectFailure(0) + injected := inject.InjectFailure(0) if injected { err := test.UploadImage(img, baseURL, "test") So(err, ShouldNotBeNil) } }) Convey("CreateBlobUpload POST call", func() { - injected := test.InjectFailure(1) + injected := inject.InjectFailure(1) if injected { err := test.UploadImage(img, baseURL, "test") So(err, ShouldNotBeNil) } }) Convey("UpdateBlobUpload PUT call", func() { - injected := test.InjectFailure(3) + injected := inject.InjectFailure(3) if injected { err := test.UploadImage(img, baseURL, "test") So(err, ShouldNotBeNil) } }) Convey("second marshal", func() { - injected := test.InjectFailure(5) + injected := inject.InjectFailure(5) if injected { err := test.UploadImage(img, baseURL, "test") So(err, ShouldNotBeNil) @@ -791,28 +812,28 @@ func TestInjectUploadImageWithBasicAuth(t *testing.T) { } Convey("first marshal", func() { - injected := test.InjectFailure(0) + injected := inject.InjectFailure(0) if injected { err := test.UploadImageWithBasicAuth(img, baseURL, "test", "user", "password") So(err, ShouldNotBeNil) } }) Convey("CreateBlobUpload POST call", func() { - injected := test.InjectFailure(1) + injected := inject.InjectFailure(1) if injected { err := test.UploadImageWithBasicAuth(img, baseURL, "test", "user", "password") So(err, ShouldNotBeNil) } }) Convey("UpdateBlobUpload PUT call", func() { - injected := test.InjectFailure(3) + injected := inject.InjectFailure(3) if injected { err := test.UploadImageWithBasicAuth(img, baseURL, "test", "user", "password") So(err, ShouldNotBeNil) } }) Convey("second marshal", func() { - injected := test.InjectFailure(5) + injected := inject.InjectFailure(5) if injected { err := test.UploadImageWithBasicAuth(img, baseURL, "test", "user", "password") So(err, ShouldNotBeNil) diff --git a/pkg/test/dev.go b/pkg/test/inject/dev.go similarity index 99% rename from pkg/test/dev.go rename to pkg/test/inject/dev.go index 853e29f1..6aa117a4 100644 --- a/pkg/test/dev.go +++ b/pkg/test/inject/dev.go @@ -3,7 +3,7 @@ // This file should be linked only in **development** mode. -package test +package inject import ( "net/http" diff --git a/pkg/test/inject_test.go b/pkg/test/inject/inject_test.go similarity index 73% rename from pkg/test/inject_test.go rename to pkg/test/inject/inject_test.go index 0c8afefa..f3b5a7b3 100644 --- a/pkg/test/inject_test.go +++ b/pkg/test/inject/inject_test.go @@ -3,7 +3,7 @@ // This file should be linked only in **development** mode. -package test_test +package inject_test import ( "errors" @@ -11,7 +11,7 @@ import ( . "github.com/smartystreets/goconvey/convey" - "zotregistry.io/zot/pkg/test" + "zotregistry.io/zot/pkg/test/inject" ) var ( @@ -26,12 +26,12 @@ func foo() error { fmap := map[string]string{"key1": "val1", "key2": "val2"} _, ok := fmap["key1"] // should never fail - if !test.Ok(ok) { + if !inject.Ok(ok) { return errKey1 } _, ok = fmap["key2"] // should never fail - if !test.Ok(ok) { + if !inject.Ok(ok) { return errKey2 } @@ -48,12 +48,12 @@ func errgen(i int) error { func bar() error { err := errgen(0) // should never fail - if test.Error(err) != nil { + if inject.Error(err) != nil { return errCall1 } err = errgen(0) // should never fail - if test.Error(err) != nil { + if inject.Error(err) != nil { return errCall2 } @@ -61,11 +61,11 @@ func bar() error { } func baz() error { - if test.ErrStatusCode(0) != 0 { + if inject.ErrStatusCode(0) != 0 { return errCall1 } - if test.ErrStatusCode(0) != 0 { + if inject.ErrStatusCode(0) != 0 { return errCall2 } @@ -88,15 +88,15 @@ func TestInject(t *testing.T) { Convey("Check Ok", func() { Convey("Without skipping", func() { - test.InjectFailure(0) // inject a failure + inject.InjectFailure(0) // inject a failure err := foo() // should be a failure So(err, ShouldNotBeNil) // should be a failure So(errors.Is(err, errKey1), ShouldBeTrue) }) Convey("With skipping", func() { - test.InjectFailure(1) // inject a failure but skip first one - err := foo() // should be a failure + inject.InjectFailure(1) // inject a failure but skip first one + err := foo() // should be a failure So(errors.Is(err, errKey1), ShouldBeFalse) So(errors.Is(err, errKey2), ShouldBeTrue) }) @@ -108,15 +108,15 @@ func TestInject(t *testing.T) { Convey("Check Err", func() { Convey("Without skipping", func() { - test.InjectFailure(0) // inject a failure + inject.InjectFailure(0) // inject a failure err := bar() // should be a failure So(err, ShouldNotBeNil) // should be a failure So(errors.Is(err, errCall1), ShouldBeTrue) }) Convey("With skipping", func() { - test.InjectFailure(1) // inject a failure but skip first one - err := bar() // should be a failure + inject.InjectFailure(1) // inject a failure but skip first one + err := bar() // should be a failure So(errors.Is(err, errCall1), ShouldBeFalse) So(errors.Is(err, errCall2), ShouldBeTrue) }) @@ -124,15 +124,15 @@ func TestInject(t *testing.T) { Convey("Check ErrStatusCode", func() { Convey("Without skipping", func() { - test.InjectFailure(0) // inject a failure + inject.InjectFailure(0) // inject a failure err := baz() // should be a failure So(err, ShouldNotBeNil) // should be a failure So(errors.Is(err, errCall1), ShouldBeTrue) }) Convey("With skipping", func() { - test.InjectFailure(1) // inject a failure but skip first one - err := baz() // should be a failure + inject.InjectFailure(1) // inject a failure but skip first one + err := baz() // should be a failure So(errors.Is(err, errCall1), ShouldBeFalse) So(errors.Is(err, errCall2), ShouldBeTrue) }) @@ -141,14 +141,14 @@ func TestInject(t *testing.T) { Convey("Without injected failure", t, func(c C) { err := alwaysErr() - So(test.Error(err), ShouldNotBeNil) + So(inject.Error(err), ShouldNotBeNil) ok := alwaysNotOk() - So(test.Ok(ok), ShouldBeFalse) + So(inject.Ok(ok), ShouldBeFalse) }) Convey("Incomplete injected failure", t, func(c C) { - test.InjectFailure(0) // inject a failure - So(func() { test.InjectFailure(0) }, ShouldPanic) + inject.InjectFailure(0) // inject a failure + So(func() { inject.InjectFailure(0) }, ShouldPanic) }) } diff --git a/pkg/test/prod.go b/pkg/test/inject/prod.go similarity index 95% rename from pkg/test/prod.go rename to pkg/test/inject/prod.go index dca8811d..3e01b91e 100644 --- a/pkg/test/prod.go +++ b/pkg/test/inject/prod.go @@ -1,7 +1,7 @@ //go:build !dev // +build !dev -package test +package inject func Error(err error) error { return err diff --git a/pkg/test/mocks/lint_mock.go b/pkg/test/mocks/lint_mock.go index 7adf2064..4b400c42 100644 --- a/pkg/test/mocks/lint_mock.go +++ b/pkg/test/mocks/lint_mock.go @@ -3,14 +3,15 @@ package mocks import ( godigest "github.com/opencontainers/go-digest" - "zotregistry.io/zot/pkg/storage" + storageTypes "zotregistry.io/zot/pkg/storage/types" ) type MockedLint struct { - LintFn func(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) + LintFn func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) } -func (lint MockedLint) Lint(repo string, manifestDigest godigest.Digest, imageStore storage.ImageStore) (bool, error) { +func (lint MockedLint) Lint(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore, +) (bool, error) { if lint.LintFn != nil { return lint.LintFn(repo, manifestDigest, imageStore) }