From 4c156234cb34492ffc8a744fa773d79659cd2bd3 Mon Sep 17 00:00:00 2001 From: LaurentiuNiculae Date: Thu, 9 Mar 2023 20:41:48 +0200 Subject: [PATCH] feat(repodb): sync-repodb WIP (#1241) Signed-off-by: Laurentiu Niculae --- errors/errors.go | 2 +- pkg/api/controller.go | 4 +- pkg/api/routes.go | 6 +- pkg/extensions/extension_sync.go | 11 +- pkg/extensions/extension_sync_disabled.go | 7 +- pkg/extensions/search/common/common_test.go | 4 +- pkg/extensions/search/convert/convert_test.go | 2 +- pkg/extensions/search/cve/cve_test.go | 20 +- pkg/extensions/search/cve/pagination_test.go | 4 +- .../search/cve/trivy/scanner_internal_test.go | 11 +- pkg/extensions/search/resolver_test.go | 2 +- pkg/extensions/sync/on_demand.go | 14 +- pkg/extensions/sync/signatures.go | 77 +++++- pkg/extensions/sync/sync.go | 12 +- pkg/extensions/sync/sync_internal_test.go | 66 ++++-- pkg/extensions/sync/sync_test.go | 221 +++++++++++++++++- pkg/extensions/sync/utils.go | 42 +++- .../repodb/boltdb-wrapper/boltdb_wrapper.go | 77 +++--- .../boltdb-wrapper/boltdb_wrapper_test.go | 18 +- pkg/meta/repodb/common/common.go | 4 +- .../repodb/dynamodb-wrapper/dynamo_test.go | 49 ++-- .../repodb/dynamodb-wrapper/dynamo_wrapper.go | 79 ++++--- pkg/meta/repodb/repodb.go | 4 +- pkg/meta/repodb/repodb_test.go | 177 +++++++------- pkg/meta/repodb/sync_repodb.go | 28 +-- pkg/meta/repodb/sync_repodb_test.go | 4 +- pkg/meta/repodb/update/update.go | 4 +- pkg/meta/repodb/update/update_test.go | 5 +- pkg/test/common_test.go | 3 + pkg/test/mocks/repo_db_mock.go | 10 +- 30 files changed, 652 insertions(+), 315 deletions(-) diff --git a/errors/errors.go b/errors/errors.go index 0bbbc9d6..d1099a0b 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -61,7 +61,7 @@ var ( ErrBadRange = errors.New("storage: bad range") ErrBadLayerCount = errors.New("manifest: layers count doesn't correspond to config history") ErrManifestConflict = errors.New("manifest: multiple manifests found") - ErrManifestMetaNotFound = errors.New("repodb: image metadata not found for given manifest digest") + ErrManifestMetaNotFound = errors.New("repodb: image metadata not found for given manifest reference") ErrManifestDataNotFound = errors.New("repodb: image data not found for given manifest digest") ErrIndexDataNotFount = errors.New("repodb: index data not found for given digest") ErrRepoMetaNotFound = errors.New("repodb: repo metadata not found for given repo name") diff --git a/pkg/api/controller.go b/pkg/api/controller.go index ea752be0..dfd734e2 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -605,7 +605,7 @@ func (c *Controller) LoadNewConfig(reloadCtx context.Context, config *config.Con if config.Extensions != nil && config.Extensions.Sync != nil { // reload sync config c.Config.Extensions.Sync = config.Extensions.Sync - ext.EnableSyncExtension(reloadCtx, c.Config, c.wgShutDown, c.StoreController, c.Log) + ext.EnableSyncExtension(reloadCtx, c.Config, c.wgShutDown, c.RepoDB, c.StoreController, c.Log) } else if c.Config.Extensions != nil { c.Config.Extensions.Sync = nil } @@ -653,7 +653,7 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) { // Enable extensions if extension config is provided for storeController if c.Config.Extensions != nil { if c.Config.Extensions.Sync != nil { - ext.EnableSyncExtension(reloadCtx, c.Config, c.wgShutDown, c.StoreController, c.Log) + ext.EnableSyncExtension(reloadCtx, c.Config, c.wgShutDown, c.RepoDB, c.StoreController, c.Log) } } diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 0e340dab..b4ce6f09 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -438,7 +438,7 @@ func getReferrers(ctx context.Context, routeHandler *RouteHandler, routeHandler.c.Log.Info().Msgf("referrers not found, trying to get referrers to %s:%s by syncing on demand", name, digest) - errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.StoreController, + errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.RepoDB, routeHandler.c.StoreController, name, digest.String(), sync.OCIReference, routeHandler.c.Log) if errSync != nil { routeHandler.c.Log.Error().Err(err).Str("name", name).Str("digest", digest.String()).Msg("unable to get references") @@ -1640,7 +1640,7 @@ func getImageManifest(ctx context.Context, routeHandler *RouteHandler, imgStore routeHandler.c.Log.Info().Msgf("trying to get updated image %s:%s by syncing on demand", name, reference) - errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.StoreController, + errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.RepoDB, routeHandler.c.StoreController, name, reference, "", routeHandler.c.Log) if errSync != nil { routeHandler.c.Log.Err(errSync).Msgf("error encounter while syncing image %s:%s", @@ -1664,7 +1664,7 @@ func getOrasReferrers(ctx context.Context, routeHandler *RouteHandler, routeHandler.c.Log.Info().Msgf("artifact not found, trying to get artifact %s:%s by syncing on demand", name, digest.String()) - errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.StoreController, + errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.RepoDB, routeHandler.c.StoreController, name, digest.String(), sync.OrasArtifact, routeHandler.c.Log) if errSync != nil { routeHandler.c.Log.Error().Err(err).Str("name", name).Str("digest", digest.String()).Msg("unable to get references") diff --git a/pkg/extensions/extension_sync.go b/pkg/extensions/extension_sync.go index 4526f79f..a1545afa 100644 --- a/pkg/extensions/extension_sync.go +++ b/pkg/extensions/extension_sync.go @@ -10,14 +10,15 @@ import ( "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/extensions/sync" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/repodb" "zotregistry.io/zot/pkg/storage" ) func EnableSyncExtension(ctx context.Context, config *config.Config, wg *goSync.WaitGroup, - storeController storage.StoreController, log log.Logger, + repoDB repodb.RepoDB, storeController storage.StoreController, log log.Logger, ) { if config.Extensions.Sync != nil && *config.Extensions.Sync.Enable { - if err := sync.Run(ctx, *config.Extensions.Sync, storeController, wg, log); err != nil { + if err := sync.Run(ctx, *config.Extensions.Sync, repoDB, storeController, wg, log); err != nil { log.Error().Err(err).Msg("Error encountered while setting up syncing") } } else { @@ -25,12 +26,12 @@ func EnableSyncExtension(ctx context.Context, config *config.Config, wg *goSync. } } -func SyncOneImage(ctx context.Context, config *config.Config, storeController storage.StoreController, - repoName, reference string, artifactType string, log log.Logger, +func SyncOneImage(ctx context.Context, config *config.Config, repoDB repodb.RepoDB, + storeController storage.StoreController, repoName, reference string, artifactType string, log log.Logger, ) error { log.Info().Msgf("syncing image %s:%s", repoName, reference) - err := sync.OneImage(ctx, *config.Extensions.Sync, storeController, repoName, reference, artifactType, log) + err := sync.OneImage(ctx, *config.Extensions.Sync, repoDB, storeController, repoName, reference, artifactType, log) return err } diff --git a/pkg/extensions/extension_sync_disabled.go b/pkg/extensions/extension_sync_disabled.go index 62c50c4d..494768d7 100644 --- a/pkg/extensions/extension_sync_disabled.go +++ b/pkg/extensions/extension_sync_disabled.go @@ -9,12 +9,13 @@ import ( "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/repodb" "zotregistry.io/zot/pkg/storage" ) // EnableSyncExtension ... func EnableSyncExtension(ctx context.Context, - config *config.Config, wg *goSync.WaitGroup, + config *config.Config, wg *goSync.WaitGroup, repoDB repodb.RepoDB, storeController storage.StoreController, log log.Logger, ) { log.Warn().Msg("skipping enabling sync extension because given zot binary doesn't include this feature," + @@ -22,8 +23,8 @@ func EnableSyncExtension(ctx context.Context, } // SyncOneImage ... -func SyncOneImage(ctx context.Context, config *config.Config, storeController storage.StoreController, - repoName, reference string, artifactType string, log log.Logger, +func SyncOneImage(ctx context.Context, config *config.Config, repoDB repodb.RepoDB, + storeController storage.StoreController, repoName, reference string, artifactType string, log log.Logger, ) error { log.Warn().Msg("skipping syncing on demand because given zot binary doesn't include this feature," + "please build a binary that does so") diff --git a/pkg/extensions/search/common/common_test.go b/pkg/extensions/search/common/common_test.go index 67a0c4cd..99c0e5e9 100644 --- a/pkg/extensions/search/common/common_test.go +++ b/pkg/extensions/search/common/common_test.go @@ -4588,9 +4588,9 @@ func TestRepoDBWhenPushingImages(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("SetManifestMeta succeeds but SetRepoTag fails", func() { + Convey("SetManifestMeta succeeds but SetRepoReference fails", func() { ctlr.RepoDB = mocks.RepoDBMock{ - SetRepoTagFn: func(repo, tag string, manifestDigest godigest.Digest, mediaType string) error { + SetRepoReferenceFn: func(repo, reference string, manifestDigest godigest.Digest, mediaType string) error { return ErrTestError }, } diff --git a/pkg/extensions/search/convert/convert_test.go b/pkg/extensions/search/convert/convert_test.go index 8d64c16b..55a219d7 100644 --- a/pkg/extensions/search/convert/convert_test.go +++ b/pkg/extensions/search/convert/convert_test.go @@ -57,7 +57,7 @@ func TestConvertErrors(t *testing.T) { digest11 := godigest.FromString("abc1") err = repoDB.SetManifestMeta("repo1", digest11, repoMeta11) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repoMetas, manifestMetaMap, _, _, err := repoDB.SearchRepos(context.Background(), "", repodb.Filter{}, diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 2c948ba7..d12bdf76 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -755,7 +755,7 @@ func TestCVEStruct(t *testing.T) { digest11 := godigest.FromBytes(manifestBlob11) err = repoDB.SetManifestMeta("repo1", digest11, repoMeta11) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) timeStamp12 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC) @@ -791,7 +791,7 @@ func TestCVEStruct(t *testing.T) { digest12 := godigest.FromBytes(manifestBlob12) err = repoDB.SetManifestMeta("repo1", digest12, repoMeta12) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo1", "1.0.0", digest12, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo1", "1.0.0", digest12, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) timeStamp13 := time.Date(2010, 1, 1, 12, 0, 0, 0, time.UTC) @@ -825,7 +825,7 @@ func TestCVEStruct(t *testing.T) { digest13 := godigest.FromBytes(manifestBlob13) err = repoDB.SetManifestMeta("repo1", digest13, repoMeta13) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo1", "1.1.0", digest13, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo1", "1.1.0", digest13, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) timeStamp14 := time.Date(2011, 1, 1, 12, 0, 0, 0, time.UTC) @@ -859,7 +859,7 @@ func TestCVEStruct(t *testing.T) { digest14 := godigest.FromBytes(manifestBlob14) err = repoDB.SetManifestMeta("repo1", digest14, repoMeta14) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo1", "1.0.1", digest14, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo1", "1.0.1", digest14, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create repodb data for scannable image with no vulnerabilities @@ -894,7 +894,7 @@ func TestCVEStruct(t *testing.T) { digest61 := godigest.FromBytes(manifestBlob61) err = repoDB.SetManifestMeta("repo6", digest61, repoMeta61) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo6", "1.0.0", digest61, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo6", "1.0.0", digest61, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create repodb data for image not supporting scanning @@ -929,7 +929,7 @@ func TestCVEStruct(t *testing.T) { digest21 := godigest.FromBytes(manifestBlob21) err = repoDB.SetManifestMeta("repo2", digest21, repoMeta21) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo2", "1.0.0", digest21, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo2", "1.0.0", digest21, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // Create repodb data for invalid images/negative tests @@ -943,7 +943,7 @@ func TestCVEStruct(t *testing.T) { digest31 := godigest.FromBytes(manifestBlob31) err = repoDB.SetManifestMeta("repo3", digest31, repoMeta31) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo3", "invalid-manifest", digest31, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo3", "invalid-manifest", digest31, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) configBlob41 := []byte("invalid config blob") @@ -956,11 +956,11 @@ func TestCVEStruct(t *testing.T) { digest41 := godigest.FromString("abc7") err = repoDB.SetManifestMeta("repo4", digest41, repoMeta41) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo4", "invalid-config", digest41, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo4", "invalid-config", digest41, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) digest51 := godigest.FromString("abc8") - err = repoDB.SetRepoTag("repo5", "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo5", "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // ------ Multiarch image @@ -997,7 +997,7 @@ func TestCVEStruct(t *testing.T) { }) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repoIndex", "tagIndex", indexDigest, ispec.MediaTypeImageIndex) + err = repoDB.SetRepoReference("repoIndex", "tagIndex", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) // RepoDB loaded with initial data, mock the scanner diff --git a/pkg/extensions/search/cve/pagination_test.go b/pkg/extensions/search/cve/pagination_test.go index 15fe3e6c..7206eb33 100644 --- a/pkg/extensions/search/cve/pagination_test.go +++ b/pkg/extensions/search/cve/pagination_test.go @@ -61,7 +61,7 @@ func TestCVEPagination(t *testing.T) { digest11 := godigest.FromBytes(manifestBlob11) err = repoDB.SetManifestMeta("repo1", digest11, repoMeta11) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) timeStamp12 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC) @@ -95,7 +95,7 @@ func TestCVEPagination(t *testing.T) { digest12 := godigest.FromBytes(manifestBlob12) err = repoDB.SetManifestMeta("repo1", digest12, repoMeta12) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo1", "1.0.0", digest12, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo1", "1.0.0", digest12, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) // RepoDB loaded with initial data, mock the scanner diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index ff06f3c2..c84d1f25 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -273,7 +273,7 @@ func TestImageScannable(t *testing.T) { panic(err) } - err = repoDB.SetRepoTag("repo1", "valid", digestValidManifest, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo1", "valid", digestValidManifest, ispec.MediaTypeImageManifest) if err != nil { panic(err) } @@ -309,7 +309,8 @@ func TestImageScannable(t *testing.T) { panic(err) } - err = repoDB.SetRepoTag("repo1", "unscannable-layer", digestManifestUnscannableLayer, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo1", "unscannable-layer", digestManifestUnscannableLayer, + ispec.MediaTypeImageManifest) if err != nil { panic(err) } @@ -328,7 +329,7 @@ func TestImageScannable(t *testing.T) { panic(err) } - err = repoDB.SetRepoTag("repo1", "unmarshable", digestUnmarshableManifest, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo1", "unmarshable", digestUnmarshableManifest, ispec.MediaTypeImageManifest) if err != nil { panic(err) } @@ -336,13 +337,13 @@ func TestImageScannable(t *testing.T) { // Manifest meta cannot be found digestMissingManifest := godigest.FromBytes([]byte("Some other string")) - err = repoDB.SetRepoTag("repo1", "missing", digestMissingManifest, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo1", "missing", digestMissingManifest, ispec.MediaTypeImageManifest) if err != nil { panic(err) } // RepoMeta contains invalid digest - err = repoDB.SetRepoTag("repo1", "invalid-digest", "invalid", ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo1", "invalid-digest", "invalid", ispec.MediaTypeImageManifest) if err != nil { panic(err) } diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index dc0f9472..2bc82aa3 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -2020,7 +2020,7 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo for image, digest := range tagsMap { repo, tag := common.GetImageDirAndTag(image) - err := repoDB.SetRepoTag(repo, tag, digest, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo, tag, digest, ispec.MediaTypeImageManifest) if err != nil { panic(err) } diff --git a/pkg/extensions/sync/on_demand.go b/pkg/extensions/sync/on_demand.go index dcf4585b..84f84596 100644 --- a/pkg/extensions/sync/on_demand.go +++ b/pkg/extensions/sync/on_demand.go @@ -16,6 +16,7 @@ import ( "zotregistry.io/zot/pkg/common" syncconf "zotregistry.io/zot/pkg/extensions/config/sync" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/repodb" "zotregistry.io/zot/pkg/storage" ) @@ -59,8 +60,8 @@ func (di *demandedImages) delete(key string) { di.syncedMap.Delete(key) } -func OneImage(ctx context.Context, cfg syncconf.Config, storeController storage.StoreController, - repo, reference string, artifactType string, log log.Logger, +func OneImage(ctx context.Context, cfg syncconf.Config, repoDB repodb.RepoDB, + storeController storage.StoreController, repo, reference string, artifactType string, log log.Logger, ) error { // guard against multiple parallel requests demandedImage := fmt.Sprintf("%s:%s", repo, reference) @@ -82,7 +83,7 @@ func OneImage(ctx context.Context, cfg syncconf.Config, storeController storage. defer demandedImgs.delete(demandedImage) defer close(imageChannel) - go syncOneImage(ctx, imageChannel, cfg, storeController, repo, reference, artifactType, log) + go syncOneImage(ctx, imageChannel, cfg, repoDB, storeController, repo, reference, artifactType, log) err, ok := <-imageChannel if !ok { @@ -92,8 +93,8 @@ func OneImage(ctx context.Context, cfg syncconf.Config, storeController storage. return err } -func syncOneImage(ctx context.Context, imageChannel chan error, - cfg syncconf.Config, storeController storage.StoreController, +func syncOneImage(ctx context.Context, imageChannel chan error, cfg syncconf.Config, + repoDB repodb.RepoDB, storeController storage.StoreController, localRepo, reference string, artifactType string, log log.Logger, ) { var credentialsFile syncconf.CredentialsFile @@ -180,7 +181,8 @@ func syncOneImage(ctx context.Context, imageChannel chan error, return } - sig := newSignaturesCopier(httpClient, credentialsFile[upstreamAddr], *registryURL, storeController, log) + sig := newSignaturesCopier(httpClient, credentialsFile[upstreamAddr], *registryURL, repoDB, + storeController, log) upstreamCtx := getUpstreamContext(®Cfg, credentialsFile[upstreamAddr]) options := getCopyOptions(upstreamCtx, localCtx) diff --git a/pkg/extensions/sync/signatures.go b/pkg/extensions/sync/signatures.go index 095e1310..9e9ca464 100644 --- a/pkg/extensions/sync/signatures.go +++ b/pkg/extensions/sync/signatures.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "net/http" "net/url" "path" @@ -20,25 +21,28 @@ import ( "zotregistry.io/zot/pkg/common" syncconf "zotregistry.io/zot/pkg/extensions/config/sync" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/repodb" "zotregistry.io/zot/pkg/storage" ) type signaturesCopier struct { client *http.Client upstreamURL url.URL - storeController storage.StoreController credentials syncconf.Credentials + repoDB repodb.RepoDB + storeController storage.StoreController log log.Logger } func newSignaturesCopier(httpClient *http.Client, credentials syncconf.Credentials, - upstreamURL url.URL, + upstreamURL url.URL, repoDB repodb.RepoDB, storeController storage.StoreController, log log.Logger, ) *signaturesCopier { return &signaturesCopier{ client: httpClient, credentials: credentials, upstreamURL: upstreamURL, + repoDB: repoDB, storeController: storeController, log: log, } @@ -185,7 +189,18 @@ func (sig *signaturesCopier) syncCosignSignature(localRepo, remoteRepo, digestSt return err } - sig.log.Info().Msgf("successfully synced cosign signature for repo %s digest %s", localRepo, digestStr) + if sig.repoDB != nil { + sig.log.Debug().Msgf("trying to sync cosign signature for repo %s digest %s", localRepo, digestStr) + + err = repodb.SetMetadataFromInput(localRepo, cosignTag, ispec.MediaTypeImageManifest, + godigest.FromBytes(cosignManifestBuf), cosignManifestBuf, sig.storeController.GetImageStore(localRepo), + sig.repoDB, sig.log) + if err != nil { + return fmt.Errorf("failed to set metadata for cosign signature '%s@%s': %w", localRepo, digestStr, err) + } + + sig.log.Info().Msgf("successfully added cosign signature to RepoDB for repo %s digest %s", localRepo, digestStr) + } return nil } @@ -249,6 +264,19 @@ func (sig *signaturesCopier) syncORASRefs(localRepo, remoteRepo, digestStr strin return err } + + // this is for notation signatures + if sig.repoDB != nil { + sig.log.Debug().Msgf("trying to sync oras artifact for repo %s digest %s", localRepo, digestStr) + + err = repodb.SetMetadataFromInput(localRepo, ref.Digest.String(), ref.MediaType, + ref.Digest, body, sig.storeController.GetImageStore(localRepo), sig.repoDB, sig.log) + if err != nil { + return fmt.Errorf("failed to set metadata for oras artifact '%s@%s': %w", localRepo, digestStr, err) + } + + sig.log.Info().Msgf("successfully added oras artifacts to RepoDB for repo %s digest %s", localRepo, digestStr) + } } sig.log.Info().Msgf("successfully synced ORAS artifacts for repo %s digest %s", localRepo, digestStr) @@ -283,7 +311,7 @@ func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, digestStr string var artifactManifest oras.Manifest - body, statusCode, err := common.MakeHTTPGetRequest(sig.client, sig.credentials.Username, + OCIRefBody, statusCode, err := common.MakeHTTPGetRequest(sig.client, sig.credentials.Username, sig.credentials.Password, &artifactManifest, getRefManifestURL.String(), ref.MediaType, sig.log) if err != nil { @@ -304,7 +332,7 @@ func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, digestStr string // read manifest var manifest ispec.Manifest - err = json.Unmarshal(body, &manifest) + err = json.Unmarshal(OCIRefBody, &manifest) if err != nil { sig.log.Error().Str("errorType", common.TypeOf(err)). Err(err).Msgf("couldn't unmarshal oci reference manifest: %s", getRefManifestURL.String()) @@ -326,7 +354,7 @@ func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, digestStr string // read manifest var manifest ispec.Artifact - err = json.Unmarshal(body, &manifest) + err = json.Unmarshal(OCIRefBody, &manifest) if err != nil { sig.log.Error().Str("errorType", common.TypeOf(err)). Err(err).Msgf("couldn't unmarshal oci reference manifest: %s", getRefManifestURL.String()) @@ -341,17 +369,30 @@ func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, digestStr string } } - _, err = imageStore.PutImageManifest(localRepo, ref.Digest.String(), - ref.MediaType, body) + digest, err := imageStore.PutImageManifest(localRepo, ref.Digest.String(), + ref.MediaType, OCIRefBody) if err != nil { sig.log.Error().Str("errorType", common.TypeOf(err)). Err(err).Msg("couldn't upload oci reference manifest") return err } + + if sig.repoDB != nil { + sig.log.Debug().Msgf("trying to add OCI refs for repo %s digest %s", localRepo, digestStr) + + err = repodb.SetMetadataFromInput(localRepo, digestStr, ref.MediaType, + digest, OCIRefBody, sig.storeController.GetImageStore(localRepo), + sig.repoDB, sig.log) + if err != nil { + return fmt.Errorf("failed to set metadata for OCI ref in '%s@%s': %w", localRepo, digestStr, err) + } + + sig.log.Info().Msgf("successfully added OCI refs to RepoDB for repo %s digest %s", localRepo, digestStr) + } } - sig.log.Info().Msgf("successfully synced oci references for repo %s digest %s", localRepo, digestStr) + sig.log.Info().Msgf("successfully synced OCI refs for repo %s digest %s", localRepo, digestStr) return nil } @@ -397,7 +438,7 @@ func (sig *signaturesCopier) syncOCIArtifact(localRepo, remoteRepo, reference st } // push manifest - _, err = imageStore.PutImageManifest(localRepo, reference, + digest, err := imageStore.PutImageManifest(localRepo, reference, ispec.MediaTypeArtifactManifest, artifactManifestBuf) if err != nil { sig.log.Error().Str("errorType", common.TypeOf(err)). @@ -406,6 +447,20 @@ func (sig *signaturesCopier) syncOCIArtifact(localRepo, remoteRepo, reference st return err } + if sig.repoDB != nil { + sig.log.Debug().Msgf("trying to OCI refs for repo %s digest %s", localRepo, digest.String()) + + err = repodb.SetMetadataFromInput(localRepo, reference, ispec.MediaTypeArtifactManifest, + digest, artifactManifestBuf, sig.storeController.GetImageStore(localRepo), + sig.repoDB, sig.log) + if err != nil { + return fmt.Errorf("failed to set metadata for OCI Artifact '%s@%s': %w", localRepo, digest.String(), err) + } + + sig.log.Info().Msgf("successfully added oci artifacts to RepoDB for repo %s digest %s", localRepo, + digest.String()) + } + sig.log.Info().Msgf("successfully synced OCI artifact for repo %s tag %s", localRepo, reference) return nil @@ -450,7 +505,7 @@ func (sig *signaturesCopier) canSkipOCIArtifact(localRepo, reference string, art localArtifactBuf, _, _, err := imageStore.GetImageManifest(localRepo, reference) if err != nil { - if errors.Is(err, zerr.ErrManifestNotFound) { + if errors.Is(err, zerr.ErrManifestNotFound) || errors.Is(err, zerr.ErrRepoNotFound) { return false, nil } diff --git a/pkg/extensions/sync/sync.go b/pkg/extensions/sync/sync.go index e5be3647..b20272e6 100644 --- a/pkg/extensions/sync/sync.go +++ b/pkg/extensions/sync/sync.go @@ -23,6 +23,7 @@ import ( "zotregistry.io/zot/pkg/common" syncconf "zotregistry.io/zot/pkg/extensions/config/sync" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/repodb" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/test" ) @@ -165,7 +166,7 @@ func getUpstreamContext(regCfg *syncconf.RegistryConfig, credentials syncconf.Cr //nolint:gocyclo // offloading some of the functionalities from here would make the code harder to follow func syncRegistry(ctx context.Context, regCfg syncconf.RegistryConfig, - upstreamURL string, + upstreamURL string, repoDB repodb.RepoDB, storeController storage.StoreController, localCtx *types.SystemContext, policyCtx *signature.PolicyContext, credentials syncconf.Credentials, retryOptions *retry.RetryOptions, log log.Logger, @@ -246,7 +247,7 @@ func syncRegistry(ctx context.Context, regCfg syncconf.RegistryConfig, } } - sig := newSignaturesCopier(httpClient, credentials, *registryURL, storeController, log) + sig := newSignaturesCopier(httpClient, credentials, *registryURL, repoDB, storeController, log) for _, repoReference := range reposReferences { upstreamRepo := repoReference.name @@ -323,9 +324,8 @@ func getLocalContexts(log log.Logger) (*types.SystemContext, *signature.PolicyCo return localCtx, policyContext, nil } -func Run(ctx context.Context, cfg syncconf.Config, - storeController storage.StoreController, - wtgrp *goSync.WaitGroup, logger log.Logger, +func Run(ctx context.Context, cfg syncconf.Config, repoDB repodb.RepoDB, + storeController storage.StoreController, wtgrp *goSync.WaitGroup, logger log.Logger, ) error { var credentialsFile syncconf.CredentialsFile @@ -382,7 +382,7 @@ func Run(ctx context.Context, cfg syncconf.Config, for _, upstreamURL := range regCfg.URLs { upstreamAddr := StripRegistryTransport(upstreamURL) // first try syncing main registry - if err := syncRegistry(ctx, regCfg, upstreamURL, storeController, localCtx, policyCtx, + if err := syncRegistry(ctx, regCfg, upstreamURL, repoDB, storeController, localCtx, policyCtx, credentialsFile[upstreamAddr], retryOptions, logger); err != nil { logger.Error().Str("errortype", common.TypeOf(err)). Err(err).Str("registry", upstreamURL). diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index 6d6fd24d..85b9c74b 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -40,6 +40,8 @@ const ( host = "127.0.0.1:45117" ) +var ErrTestError = fmt.Errorf("testError") + func TestInjectSyncUtils(t *testing.T) { Convey("Inject errors in utils functions", t, func() { repositoryReference := fmt.Sprintf("%s/%s", host, testImage) @@ -154,7 +156,7 @@ func TestSyncInternal(t *testing.T) { } ctx := context.Background() - So(Run(ctx, cfg, storage.StoreController{}, + So(Run(ctx, cfg, mocks.RepoDBMock{}, storage.StoreController{}, new(goSync.WaitGroup), log.NewLogger("debug", "")), ShouldNotBeNil) _, err = getFileCredentials("/invalid/path/to/file") @@ -190,7 +192,7 @@ func TestSyncInternal(t *testing.T) { localCtx, policyCtx, err := getLocalContexts(log) So(err, ShouldBeNil) - err = syncRegistry(ctx, syncRegistryConfig, "randomUpstreamURL", + err = syncRegistry(ctx, syncRegistryConfig, "randomUpstreamURL", mocks.RepoDBMock{}, storage.StoreController{DefaultStore: imageStore}, localCtx, policyCtx, syncconf.Credentials{}, nil, log) So(err, ShouldNotBeNil) @@ -378,9 +380,10 @@ func TestSyncInternal(t *testing.T) { imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil, ) + mockRepoDB := mocks.RepoDBMock{} sig := newSignaturesCopier(client, syncconf.Credentials{}, - *regURL, storage.StoreController{DefaultStore: imageStore}, log) + *regURL, mockRepoDB, storage.StoreController{DefaultStore: imageStore}, log) err = sig.syncCosignSignature(testImage, testImage, testImageTag, &ispec.Manifest{}) So(err, ShouldNotBeNil) @@ -430,8 +433,9 @@ func TestSyncInternal(t *testing.T) { So(regURL, ShouldNotBeNil) client := &http.Client{} + mockRepoDB := mocks.RepoDBMock{} sig := newSignaturesCopier(client, syncconf.Credentials{}, - *regURL, storage.StoreController{DefaultStore: imageStore}, log) + *regURL, mockRepoDB, storage.StoreController{DefaultStore: imageStore}, log) canBeSkipped, err = sig.canSkipOCIRefs(testImage, testImageManifestDigest.String(), refs) So(err, ShouldBeNil) @@ -596,7 +600,7 @@ func TestSyncInternal(t *testing.T) { testRootDir := path.Join(imageStore.RootDir(), testImage, SyncBlobUploadDir) // testImagePath := path.Join(testRootDir, testImage) - err := pushSyncedLocalImage(testImage, testImageTag, testRootDir, imageStore, log) + err := pushSyncedLocalImage(testImage, testImageTag, testRootDir, nil, imageStore, log) So(err, ShouldNotBeNil) err = os.MkdirAll(testRootDir, 0o755) @@ -663,7 +667,7 @@ func TestSyncInternal(t *testing.T) { _, err = testImageStore.PutImageManifest(repo, "latest", ispec.MediaTypeImageIndex, content) So(err, ShouldBeNil) - err = pushSyncedLocalImage(repo, "latest", testRootDir, imageStore, log) + err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStore, log) So(err, ShouldBeNil) // trigger error on manifest pull @@ -671,7 +675,7 @@ func TestSyncInternal(t *testing.T) { index.Manifests[0].Digest.Algorithm().String(), index.Manifests[0].Digest.Encoded()), 0o000) So(err, ShouldBeNil) - err = pushSyncedLocalImage(repo, "latest", testRootDir, imageStore, log) + err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStore, log) So(err, ShouldNotBeNil) err = os.Chmod(path.Join(testRootDir, repo, "blobs", @@ -687,7 +691,7 @@ func TestSyncInternal(t *testing.T) { }, nil, ) - err = pushSyncedLocalImage(repo, "latest", testRootDir, imageStoreWithLinter, log) + err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStoreWithLinter, log) // linter error will be ignored by sync So(err, ShouldBeNil) @@ -711,7 +715,7 @@ func TestSyncInternal(t *testing.T) { manifest.Config.Digest.Algorithm().String(), manifest.Config.Digest.Encoded())) So(err, ShouldBeNil) - err = pushSyncedLocalImage(repo, "latest", testRootDir, imageStore, log) + err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStore, log) So(err, ShouldNotBeNil) err = os.Chmod(configBlobPath, local.DefaultDirPerms) @@ -729,13 +733,45 @@ func TestSyncInternal(t *testing.T) { err = os.MkdirAll(indexManifestPath, 0o000) So(err, ShouldBeNil) - err = pushSyncedLocalImage(repo, "latest", testRootDir, imageStore, log) + err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStore, log) So(err, ShouldNotBeNil) err = os.Remove(indexManifestPath) So(err, ShouldBeNil) }) + Convey("RepoDB Errors", func() { + multiArch, err := test.GetRandomMultiarchImage("bad-repodb-tag") + So(err, ShouldBeNil) + + err = test.WriteMultiArchImageToFileSystem(multiArch, "repo", storage.StoreController{ + DefaultStore: testImageStore, + }) + So(err, ShouldBeNil) + + // copyManifest Errors + err = pushSyncedLocalImage("repo", "bad-repodb-tag", testRootDir, + mocks.RepoDBMock{ + SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error { + return ErrTestError + }, + }, imageStore, log) + So(err, ShouldNotBeNil) + + // SetMetadataFromInput + err = pushSyncedLocalImage("repo", "bad-repodb-tag", testRootDir, + mocks.RepoDBMock{ + SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error { + if Reference == "bad-repodb-tag" { + return ErrTestError + } + + return nil + }, + }, imageStore, log) + So(err, ShouldNotBeNil) + }) + Convey("manifest image errors", func() { var manifest ispec.Manifest @@ -749,7 +785,7 @@ func TestSyncInternal(t *testing.T) { if os.Geteuid() != 0 { So(func() { - _ = pushSyncedLocalImage(testImage, testImageTag, testRootDir, imageStore, log) + _ = pushSyncedLocalImage(testImage, testImageTag, testRootDir, nil, imageStore, log) }, ShouldPanic) } @@ -762,7 +798,7 @@ func TestSyncInternal(t *testing.T) { panic(err) } - err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, imageStore, log) + err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, nil, imageStore, log) So(err, ShouldNotBeNil) if err := os.Chmod(path.Join(testRootDir, testImage, "blobs", "sha256", @@ -776,7 +812,7 @@ func TestSyncInternal(t *testing.T) { panic(err) } - err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, imageStore, log) + err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, nil, imageStore, log) So(err, ShouldNotBeNil) if err := os.Chmod(cachedManifestConfigPath, 0o755); err != nil { @@ -803,7 +839,7 @@ func TestSyncInternal(t *testing.T) { panic(err) } - err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, imageStore, log) + err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, nil, imageStore, log) So(err, ShouldNotBeNil) manifest.Config.Digest = configDigestBackup @@ -828,7 +864,7 @@ func TestSyncInternal(t *testing.T) { panic(err) } - err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, imageStore, log) + err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, nil, imageStore, log) So(err, ShouldNotBeNil) }) }) diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 9267f3a7..2e04864a 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -46,6 +46,7 @@ import ( "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/test" + "zotregistry.io/zot/pkg/test/mocks" ) const ( @@ -137,6 +138,12 @@ func startUpstreamServer( srcConfig.Storage.RootDirectory = srcDir + defVal := true + srcConfig.Extensions = &extconf.ExtensionConfig{} + srcConfig.Extensions.Search = &extconf.SearchConfig{ + BaseConfig: extconf.BaseConfig{Enable: &defVal}, + } + sctlr := api.NewController(srcConfig) scm := test.NewControllerManager(sctlr) @@ -193,7 +200,10 @@ func startDownstreamServer( destConfig.Storage.GC = false destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil + defVal := true + destConfig.Extensions.Search = &extconf.SearchConfig{ + BaseConfig: extconf.BaseConfig{Enable: &defVal}, + } destConfig.Extensions.Sync = syncConfig dctlr := api.NewController(destConfig) @@ -627,6 +637,186 @@ func TestOnDemand(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 500) }) + + Convey("Sync on Demand errors", t, func() { + Convey("Signature copier errors", func() { + // start upstream server + rootDir := t.TempDir() + port := test.GetFreePort() + srcBaseURL := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + conf.Storage.GC = false + ctlr := api.NewController(conf) + ctlr.Config.Storage.RootDirectory = rootDir + + cm := test.NewControllerManager(ctlr) + cm.StartAndWait(conf.HTTP.Port) + defer cm.StopServer() + + imageConfig, layers, manifest, err := test.GetRandomImageComponents(10) + So(err, ShouldBeNil) + + manifestBlob, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(manifestBlob) + + err = test.UploadImage( + test.Image{Config: imageConfig, Layers: layers, Manifest: manifest, Reference: "test"}, + srcBaseURL, + "remote-repo", + ) + So(err, ShouldBeNil) + + // sign using cosign + err = test.SignImageUsingCosign(fmt.Sprintf("remote-repo@%s", manifestDigest.String()), port) + So(err, ShouldBeNil) + + // add OCI Ref + OCIRefManifest := ispec.Artifact{ + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: manifestDigest, + }, + Blobs: []ispec.Descriptor{}, + MediaType: ispec.MediaTypeArtifactManifest, + } + + OCIRefManifestBlob, err := json.Marshal(OCIRefManifest) + So(err, ShouldBeNil) + + resp, err := resty.R(). + SetHeader("Content-type", ispec.MediaTypeArtifactManifest). + SetBody(OCIRefManifestBlob). + Put(srcBaseURL + "/v2/remote-repo/manifests/oci.ref") + + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + // add ORAS Ref + ORASRefManifest := artifactspec.Manifest{ + Subject: &artifactspec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: manifestDigest, + }, + Blobs: []artifactspec.Descriptor{}, + MediaType: artifactspec.MediaTypeArtifactManifest, + } + + ORASRefManifestBlob, err := json.Marshal(ORASRefManifest) + So(err, ShouldBeNil) + + resp, err = resty.R(). + SetHeader("Content-type", artifactspec.MediaTypeArtifactManifest). + SetBody(ORASRefManifestBlob). + Put(srcBaseURL + "/v2/remote-repo/manifests/oras.ref") + + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + //------- Start downstream server + + var tlsVerify bool + + regex := ".*" + semver := true + + syncRegistryConfig := syncconf.RegistryConfig{ + Content: []syncconf.Content{ + { + Prefix: "remote-repo", + Tags: &syncconf.Tags{ + Regex: ®ex, + Semver: &semver, + }, + }, + }, + URLs: []string{srcBaseURL}, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, + } + + defaultVal := true + syncConfig := &syncconf.Config{ + Enable: &defaultVal, + Registries: []syncconf.RegistryConfig{syncRegistryConfig}, + } + + destPort := test.GetFreePort() + destConfig := config.New() + + destBaseURL := test.GetBaseURL(destPort) + + destConfig.HTTP.Port = destPort + + destDir := t.TempDir() + + destConfig.Storage.RootDirectory = destDir + destConfig.Storage.Dedupe = false + destConfig.Storage.GC = false + + destConfig.Extensions = &extconf.ExtensionConfig{} + defVal := true + destConfig.Extensions.Search = &extconf.SearchConfig{ + BaseConfig: extconf.BaseConfig{Enable: &defVal}, + } + destConfig.Extensions.Sync = syncConfig + + dctlr := api.NewController(destConfig) + dcm := test.NewControllerManager(dctlr) + dcm.StartAndWait(destPort) + + // repodb fails for syncOCIRefs + + dctlr.RepoDB = mocks.RepoDBMock{ + SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error { + if mediaType == ispec.MediaTypeArtifactManifest { + return sync.ErrTestError + } + + return nil + }, + } + + resp, err = resty.R().Get(destBaseURL + "/v2/remote-repo/manifests/test") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + // repodb fails for syncCosignSignature" + + dctlr.RepoDB = mocks.RepoDBMock{ + SetRepoReferenceFn: func(repo, reference string, manifestDigest godigest.Digest, mediaType string) error { + if strings.HasPrefix(reference, "sha256") || strings.HasSuffix(reference, ".sig") { + return sync.ErrTestError + } + + return nil + }, + } + + resp, err = resty.R().Get(destBaseURL + "/v2/remote-repo/manifests/test") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + // repodb fails for getORASRefs + + dctlr.RepoDB = mocks.RepoDBMock{ + SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error { + if mediaType == artifactspec.MediaTypeArtifactManifest { + return sync.ErrTestError + } + + return nil + }, + } + + resp, err = resty.R().Get(destBaseURL + "/v2/remote-repo/manifests/test") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + }) + }) } func TestPeriodically(t *testing.T) { @@ -4510,11 +4700,24 @@ func TestSyncWithDestination(t *testing.T) { }, } - sctlr, srcBaseURL, _, _, _ := startUpstreamServer(t, false, false) + srcPort := test.GetFreePort() + srcConfig := config.New() + srcBaseURL := test.GetBaseURL(srcPort) - defer func() { - sctlr.Shutdown() - }() + srcConfig.HTTP.Port = srcPort + + srcDir := t.TempDir() + + srcConfig.Storage.RootDirectory = srcDir + defVal := true + srcConfig.Extensions = &extconf.ExtensionConfig{} + srcConfig.Extensions.Search = &extconf.SearchConfig{ + BaseConfig: extconf.BaseConfig{Enable: &defVal}, + } + + sctlr := api.NewController(srcConfig) + + test.CopyTestFiles("../../../test/data", srcDir) err := os.MkdirAll(path.Join(sctlr.Config.Storage.RootDirectory, "/zot-fold"), local.DefaultDirPerms) So(err, ShouldBeNil) @@ -4524,9 +4727,15 @@ func TestSyncWithDestination(t *testing.T) { path.Join(sctlr.Config.Storage.RootDirectory, "zot-test"), path.Join(sctlr.Config.Storage.RootDirectory, "/zot-fold/zot-test"), ) - So(err, ShouldBeNil) + scm := test.NewControllerManager(sctlr) + scm.StartAndWait(srcPort) + + defer func() { + sctlr.Shutdown() + }() + Convey("Test peridiocally sync", func() { for _, testCase := range testCases { updateDuration, _ := time.ParseDuration("30m") diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go index 243d6624..b5e9243e 100644 --- a/pkg/extensions/sync/utils.go +++ b/pkg/extensions/sync/utils.go @@ -31,6 +31,7 @@ import ( syncconf "zotregistry.io/zot/pkg/extensions/config/sync" "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/repodb" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/test" @@ -264,7 +265,7 @@ func getFileCredentials(filepath string) (syncconf.CredentialsFile, error) { } func pushSyncedLocalImage(localRepo, reference, localCachePath string, - imageStore storage.ImageStore, log log.Logger, + repoDB repodb.RepoDB, imageStore storage.ImageStore, log log.Logger, ) error { log.Info().Msgf("pushing synced local image %s/%s:%s to local registry", localCachePath, localRepo, reference) @@ -275,7 +276,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string, cacheImageStore := local.NewImageStore(localCachePath, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil) - manifestContent, _, mediaType, err := cacheImageStore.GetImageManifest(localRepo, reference) + manifestBlob, manifestDigest, mediaType, err := cacheImageStore.GetImageManifest(localRepo, reference) if err != nil { log.Error().Str("errorType", common.TypeOf(err)). Err(err).Str("dir", path.Join(cacheImageStore.RootDir(), localRepo)). @@ -287,7 +288,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string, // is image manifest switch mediaType { case ispec.MediaTypeImageManifest: - if err := copyManifest(localRepo, manifestContent, reference, cacheImageStore, imageStore, log); err != nil { + if err := copyManifest(localRepo, manifestBlob, reference, repoDB, cacheImageStore, imageStore, log); err != nil { if errors.Is(err, zerr.ErrImageLintAnnotations) { log.Error().Str("errorType", common.TypeOf(err)). Err(err).Msg("couldn't upload manifest because of missing annotations") @@ -301,7 +302,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string, // is image index var indexManifest ispec.Index - if err := json.Unmarshal(manifestContent, &indexManifest); err != nil { + if err := json.Unmarshal(manifestBlob, &indexManifest); err != nil { log.Error().Str("errorType", common.TypeOf(err)). Err(err).Str("dir", path.Join(cacheImageStore.RootDir(), localRepo)). Msg("invalid JSON") @@ -322,7 +323,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string, return err } - if err := copyManifest(localRepo, manifestBuf, manifest.Digest.String(), + if err := copyManifest(localRepo, manifestBuf, manifest.Digest.String(), repoDB, cacheImageStore, imageStore, log); err != nil { if errors.Is(err, zerr.ErrImageLintAnnotations) { log.Error().Str("errorType", common.TypeOf(err)). @@ -335,19 +336,29 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string, } } - _, err = imageStore.PutImageManifest(localRepo, reference, mediaType, manifestContent) + _, err = imageStore.PutImageManifest(localRepo, reference, mediaType, manifestBlob) if err != nil { log.Error().Str("errorType", common.TypeOf(err)). Err(err).Msg("couldn't upload manifest") return err } + + if repoDB != nil { + err = repodb.SetMetadataFromInput(localRepo, reference, mediaType, + manifestDigest, manifestBlob, imageStore, repoDB, log) + if err != nil { + return fmt.Errorf("failed to set metadata for image '%s %s': %w", localRepo, reference, err) + } + + log.Debug().Msgf("successfully set metadata for %s:%s", localRepo, reference) + } } return nil } -func copyManifest(localRepo string, manifestContent []byte, reference string, +func copyManifest(localRepo string, manifestContent []byte, reference string, repoDB repodb.RepoDB, cacheImageStore, imageStore storage.ImageStore, log log.Logger, ) error { var manifest ispec.Manifest @@ -376,7 +387,7 @@ func copyManifest(localRepo string, manifestContent []byte, reference string, return err } - _, err = imageStore.PutImageManifest(localRepo, reference, + digest, err := imageStore.PutImageManifest(localRepo, reference, ispec.MediaTypeImageManifest, manifestContent) if err != nil { log.Error().Str("errorType", common.TypeOf(err)). @@ -385,6 +396,19 @@ func copyManifest(localRepo string, manifestContent []byte, reference string, return err } + if repoDB != nil { + err = repodb.SetMetadataFromInput(localRepo, reference, ispec.MediaTypeImageManifest, + digest, manifestContent, imageStore, repoDB, log) + if err != nil { + log.Error().Str("errorType", common.TypeOf(err)). + Err(err).Msg("couldn't set metadata from input") + + return err + } + + log.Debug().Msgf("successfully set metadata for %s:%s", localRepo, reference) + } + return nil } @@ -720,7 +744,7 @@ func syncImageWithRefs(ctx context.Context, localRepo, upstreamRepo, reference s } // push from cache to repo - err = pushSyncedLocalImage(localRepo, reference, localCachePath, imageStore, log) + err = pushSyncedLocalImage(localRepo, reference, localCachePath, sig.repoDB, imageStore, log) if err != nil { log.Error().Str("errorType", common.TypeOf(err)). Err(err).Msgf("error while pushing synced cached image %s", diff --git a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go index 1274ec00..304c200d 100644 --- a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go +++ b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go @@ -79,7 +79,7 @@ func NewBoltDBWrapper(params DBParameters) (*DBWrapper, error) { }, nil } -func (bdw DBWrapper) SetManifestData(manifestDigest godigest.Digest, manifestData repodb.ManifestData) error { +func (bdw *DBWrapper) SetManifestData(manifestDigest godigest.Digest, manifestData repodb.ManifestData) error { err := bdw.DB.Update(func(tx *bolt.Tx) error { buck := tx.Bucket([]byte(repodb.ManifestDataBucket)) @@ -99,7 +99,7 @@ func (bdw DBWrapper) SetManifestData(manifestDigest godigest.Digest, manifestDat return err } -func (bdw DBWrapper) GetManifestData(manifestDigest godigest.Digest) (repodb.ManifestData, error) { +func (bdw *DBWrapper) GetManifestData(manifestDigest godigest.Digest) (repodb.ManifestData, error) { var manifestData repodb.ManifestData err := bdw.DB.View(func(tx *bolt.Tx) error { @@ -122,7 +122,7 @@ func (bdw DBWrapper) GetManifestData(manifestDigest godigest.Digest) (repodb.Man return manifestData, err } -func (bdw DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Digest, manifestMeta repodb.ManifestMetadata, +func (bdw *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Digest, manifestMeta repodb.ManifestMetadata, ) error { err := bdw.DB.Update(func(tx *bolt.Tx) error { dataBuck := tx.Bucket([]byte(repodb.ManifestDataBucket)) @@ -169,7 +169,7 @@ func (bdw DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Digest return err } -func (bdw DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Digest) (repodb.ManifestMetadata, error) { +func (bdw *DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Digest) (repodb.ManifestMetadata, error) { var manifestMetadata repodb.ManifestMetadata err := bdw.DB.View(func(tx *bolt.Tx) error { @@ -214,7 +214,7 @@ func (bdw DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Digest return manifestMetadata, err } -func (bdw DBWrapper) SetIndexData(indexDigest godigest.Digest, indexMetadata repodb.IndexData) error { +func (bdw *DBWrapper) SetIndexData(indexDigest godigest.Digest, indexMetadata repodb.IndexData) error { // we make the assumption that the oci layout is consistent and all manifests refferenced inside the // index are present err := bdw.DB.Update(func(tx *bolt.Tx) error { @@ -236,7 +236,7 @@ func (bdw DBWrapper) SetIndexData(indexDigest godigest.Digest, indexMetadata rep return err } -func (bdw DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData, error) { +func (bdw *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData, error) { var indexMetadata repodb.IndexData err := bdw.DB.View(func(tx *bolt.Tx) error { @@ -259,10 +259,10 @@ func (bdw DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData return indexMetadata, err } -func (bdw DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest.Digest, +func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, mediaType string, ) error { - if err := common.ValidateRepoTagInput(repo, tag, manifestDigest); err != nil { + if err := common.ValidateRepoReferenceInput(repo, reference, manifestDigest); err != nil { return err } @@ -273,29 +273,17 @@ func (bdw DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest // object not found if len(repoMetaBlob) == 0 { - // create a new object - repoMeta := repodb.RepoMetadata{ - Name: repo, - Tags: map[string]repodb.Descriptor{ - tag: { - Digest: manifestDigest.String(), - MediaType: mediaType, - }, - }, - Statistics: map[string]repodb.DescriptorStatistics{ - manifestDigest.String(): {DownloadCount: 0}, - }, - Signatures: map[string]repodb.ManifestSignatures{ - manifestDigest.String(): {}, - }, - } + var err error - repoMetaBlob, err := json.Marshal(repoMeta) + repoMetaBlob, err = json.Marshal(repodb.RepoMetadata{ + Name: repo, + Tags: map[string]repodb.Descriptor{}, + Statistics: map[string]repodb.DescriptorStatistics{}, + Signatures: map[string]repodb.ManifestSignatures{}, + }) if err != nil { return err } - - return buck.Put([]byte(repo), repoMetaBlob) } // object found @@ -306,11 +294,16 @@ func (bdw DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest return err } - repoMeta.Tags[tag] = repodb.Descriptor{ - Digest: manifestDigest.String(), - MediaType: mediaType, + if !common.ReferenceIsDigest(reference) { + repoMeta.Tags[reference] = repodb.Descriptor{ + Digest: manifestDigest.String(), + MediaType: mediaType, + } } + repoMeta.Statistics[manifestDigest.String()] = repodb.DescriptorStatistics{DownloadCount: 0} + repoMeta.Signatures[manifestDigest.String()] = repodb.ManifestSignatures{} + repoMetaBlob, err = json.Marshal(repoMeta) if err != nil { return err @@ -322,7 +315,7 @@ func (bdw DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest return err } -func (bdw DBWrapper) GetRepoMeta(repo string) (repodb.RepoMetadata, error) { +func (bdw *DBWrapper) GetRepoMeta(repo string) (repodb.RepoMetadata, error) { var repoMeta repodb.RepoMetadata err := bdw.DB.Update(func(tx *bolt.Tx) error { @@ -347,7 +340,7 @@ func (bdw DBWrapper) GetRepoMeta(repo string) (repodb.RepoMetadata, error) { return repoMeta, err } -func (bdw DBWrapper) DeleteRepoTag(repo string, tag string) error { +func (bdw *DBWrapper) DeleteRepoTag(repo string, tag string) error { err := bdw.DB.Update(func(tx *bolt.Tx) error { buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) @@ -383,7 +376,7 @@ func (bdw DBWrapper) DeleteRepoTag(repo string, tag string) error { return err } -func (bdw DBWrapper) IncrementRepoStars(repo string) error { +func (bdw *DBWrapper) IncrementRepoStars(repo string) error { err := bdw.DB.Update(func(tx *bolt.Tx) error { buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) @@ -412,7 +405,7 @@ func (bdw DBWrapper) IncrementRepoStars(repo string) error { return err } -func (bdw DBWrapper) DecrementRepoStars(repo string) error { +func (bdw *DBWrapper) DecrementRepoStars(repo string) error { err := bdw.DB.Update(func(tx *bolt.Tx) error { buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) @@ -443,7 +436,7 @@ func (bdw DBWrapper) DecrementRepoStars(repo string) error { return err } -func (bdw DBWrapper) GetRepoStars(repo string) (int, error) { +func (bdw *DBWrapper) GetRepoStars(repo string) (int, error) { stars := 0 err := bdw.DB.View(func(tx *bolt.Tx) error { @@ -470,7 +463,7 @@ func (bdw DBWrapper) GetRepoStars(repo string) (int, error) { return stars, err } -func (bdw DBWrapper) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta repodb.RepoMetadata) bool, +func (bdw *DBWrapper) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta repodb.RepoMetadata) bool, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, error) { var ( @@ -515,7 +508,7 @@ func (bdw DBWrapper) GetMultipleRepoMeta(ctx context.Context, filter func(repoMe return foundRepos, err } -func (bdw DBWrapper) IncrementImageDownloads(repo string, reference string) error { +func (bdw *DBWrapper) IncrementImageDownloads(repo string, reference string) error { err := bdw.DB.Update(func(tx *bolt.Tx) error { buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) @@ -559,7 +552,7 @@ func (bdw DBWrapper) IncrementImageDownloads(repo string, reference string) erro return err } -func (bdw DBWrapper) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, +func (bdw *DBWrapper) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, sygMeta repodb.SignatureMetadata, ) error { err := bdw.DB.Update(func(tx *bolt.Tx) error { @@ -616,7 +609,7 @@ func (bdw DBWrapper) AddManifestSignature(repo string, signedManifestDigest godi return err } -func (bdw DBWrapper) DeleteSignature(repo string, signedManifestDigest godigest.Digest, +func (bdw *DBWrapper) DeleteSignature(repo string, signedManifestDigest godigest.Digest, sigMeta repodb.SignatureMetadata, ) error { err := bdw.DB.Update(func(tx *bolt.Tx) error { @@ -670,7 +663,7 @@ func (bdw DBWrapper) DeleteSignature(repo string, signedManifestDigest godigest. return err } -func (bdw DBWrapper) SearchRepos(ctx context.Context, searchText string, filter repodb.Filter, +func (bdw *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error, @@ -1020,7 +1013,7 @@ func NewManifestMetadata(manifestDigest string, repoMeta repodb.RepoMetadata, return manifestMeta } -func (bdw DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, +func (bdw *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error, @@ -1179,7 +1172,7 @@ func (bdw DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err } -func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter, +func (bdw *DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) { var ( diff --git a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go index 24211eec..c2d80880 100644 --- a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go +++ b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go @@ -70,7 +70,7 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("SetRepoTag", func() { + Convey("SetRepoReference", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) @@ -78,7 +78,7 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - err = boltdbWrapper.SetRepoTag("repo1", "tag", "digest", ispec.MediaTypeImageManifest) + err = boltdbWrapper.SetRepoReference("repo1", "tag", "digest", ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) @@ -388,7 +388,7 @@ func TestWrapperErrors(t *testing.T) { Convey("Bad index data", func() { indexDigest := digest.FromString("indexDigest") - err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck So(err, ShouldBeNil) err = setBadIndexData(boltdbWrapper.DB, indexDigest.String()) @@ -404,7 +404,7 @@ func TestWrapperErrors(t *testing.T) { Convey("Bad indexBlob in IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck So(err, ShouldBeNil) err = boltdbWrapper.SetIndexData(indexDigest, repodb.IndexData{ @@ -426,7 +426,7 @@ func TestWrapperErrors(t *testing.T) { manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2") ) - err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck So(err, ShouldBeNil) indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{ @@ -557,7 +557,7 @@ func TestWrapperErrors(t *testing.T) { Convey("FilterTags bad IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck So(err, ShouldBeNil) err = setBadIndexData(boltdbWrapper.DB, indexDigest.String()) @@ -573,7 +573,7 @@ func TestWrapperErrors(t *testing.T) { Convey("FilterTags bad indexBlob in IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck So(err, ShouldBeNil) err = boltdbWrapper.SetIndexData(indexDigest, repodb.IndexData{ @@ -595,7 +595,7 @@ func TestWrapperErrors(t *testing.T) { manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2") ) - err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck So(err, ShouldBeNil) indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{ @@ -631,7 +631,7 @@ func TestWrapperErrors(t *testing.T) { Convey("Unsuported type", func() { digest := digest.FromString("digest") - err := boltdbWrapper.SetRepoTag("repo", "tag1", digest, "invalid type") //nolint:contextcheck + err := boltdbWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck So(err, ShouldBeNil) _, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{}) diff --git a/pkg/meta/repodb/common/common.go b/pkg/meta/repodb/common/common.go index d76492cd..7a61cdc9 100644 --- a/pkg/meta/repodb/common/common.go +++ b/pkg/meta/repodb/common/common.go @@ -45,12 +45,12 @@ func ReferenceIsDigest(reference string) bool { return err == nil } -func ValidateRepoTagInput(repo, tag string, manifestDigest godigest.Digest) error { +func ValidateRepoReferenceInput(repo, reference string, manifestDigest godigest.Digest) error { if repo == "" { return zerr.ErrEmptyRepoName } - if tag == "" { + if reference == "" { return zerr.ErrEmptyTag } diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go index 6be628e0..c8df81c0 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go @@ -55,13 +55,13 @@ func TestIterator(t *testing.T) { So(dynamoWrapper.ResetManifestDataTable(), ShouldBeNil) So(dynamoWrapper.ResetRepoMetaTable(), ShouldBeNil) - err = dynamoWrapper.SetRepoTag("repo1", "tag1", "manifestType", "manifestDigest1") + err = dynamoWrapper.SetRepoReference("repo1", "tag1", "manifestType", "manifestDigest1") So(err, ShouldBeNil) - err = dynamoWrapper.SetRepoTag("repo2", "tag2", "manifestType", "manifestDigest2") + err = dynamoWrapper.SetRepoReference("repo2", "tag2", "manifestType", "manifestDigest2") So(err, ShouldBeNil) - err = dynamoWrapper.SetRepoTag("repo3", "tag3", "manifestType", "manifestDigest3") + err = dynamoWrapper.SetRepoReference("repo3", "tag3", "manifestType", "manifestDigest3") So(err, ShouldBeNil) repoMetaAttributeIterator := iterator.NewBaseDynamoAttributesIterator( @@ -196,7 +196,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("GetManifestMeta GetManifestData not found error", func() { - err := dynamoWrapper.SetRepoTag("repo", "tag", "dig", "") + err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") So(err, ShouldBeNil) _, err = dynamoWrapper.GetManifestMeta("repo", "dig") @@ -270,7 +270,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("IncrementImageDownloads tag not found error", func() { - err := dynamoWrapper.SetRepoTag("repo", "tag", "dig", "") + err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") So(err, ShouldBeNil) err = dynamoWrapper.IncrementImageDownloads("repo", "notFoundTag") @@ -278,7 +278,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("AddManifestSignature GetRepoMeta error", func() { - err := dynamoWrapper.SetRepoTag("repo", "tag", "dig", "") + err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") So(err, ShouldBeNil) err = dynamoWrapper.AddManifestSignature("repoNotFound", "tag", repodb.SignatureMetadata{}) @@ -286,7 +286,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("AddManifestSignature ManifestSignatures signedManifestDigest not found error", func() { - err := dynamoWrapper.SetRepoTag("repo", "tag", "dig", "") + err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") So(err, ShouldBeNil) err = dynamoWrapper.AddManifestSignature("repo", "tagNotFound", repodb.SignatureMetadata{}) @@ -294,7 +294,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("AddManifestSignature SignatureType repodb.NotationType", func() { - err := dynamoWrapper.SetRepoTag("repo", "tag", "dig", "") + err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") So(err, ShouldBeNil) err = dynamoWrapper.AddManifestSignature("repo", "tagNotFound", repodb.SignatureMetadata{ @@ -349,7 +349,8 @@ func TestWrapperErrors(t *testing.T) { }) Convey("SearchRepos GetManifestMeta error", func() { - err := dynamoWrapper.SetRepoTag("repo", "tag1", "notFoundDigest", ispec.MediaTypeImageManifest) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", "notFoundDigest", //nolint:contextcheck + ispec.MediaTypeImageManifest) So(err, ShouldBeNil) _, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{}) @@ -358,7 +359,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("SearchRepos config unmarshal error", func() { - err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck So(err, ShouldBeNil) err = dynamoWrapper.SetManifestData("dig1", repodb.ManifestData{ //nolint:contextcheck @@ -375,7 +376,7 @@ func TestWrapperErrors(t *testing.T) { Convey("Unsuported type", func() { digest := digest.FromString("digest") - err := dynamoWrapper.SetRepoTag("repo", "tag1", digest, "invalid type") //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck So(err, ShouldBeNil) _, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{}) @@ -395,7 +396,7 @@ func TestWrapperErrors(t *testing.T) { Convey("SearchRepos bad index data", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck So(err, ShouldBeNil) err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck @@ -408,7 +409,7 @@ func TestWrapperErrors(t *testing.T) { Convey("SearchRepos bad indexBlob in IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck So(err, ShouldBeNil) err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck @@ -427,7 +428,7 @@ func TestWrapperErrors(t *testing.T) { manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2") ) - err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck So(err, ShouldBeNil) indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{ @@ -466,7 +467,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("SearchTags GetManifestMeta error", func() { - err := dynamoWrapper.SetRepoTag("repo", "tag1", "manifestNotFound", //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", "manifestNotFound", //nolint:contextcheck ispec.MediaTypeImageManifest) So(err, ShouldBeNil) @@ -476,7 +477,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("SearchTags config unmarshal error", func() { - err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck So(err, ShouldBeNil) err = dynamoWrapper.SetManifestData( //nolint:contextcheck @@ -496,7 +497,7 @@ func TestWrapperErrors(t *testing.T) { Convey("SearchTags bad index data", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck So(err, ShouldBeNil) err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck @@ -509,7 +510,7 @@ func TestWrapperErrors(t *testing.T) { Convey("SearchTags bad indexBlob in IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck So(err, ShouldBeNil) err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck @@ -528,7 +529,7 @@ func TestWrapperErrors(t *testing.T) { manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2") ) - err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck So(err, ShouldBeNil) indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{ @@ -573,7 +574,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("FilterTags manifestMeta not found", func() { - err := dynamoWrapper.SetRepoTag("repo", "tag1", "manifestNotFound", //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", "manifestNotFound", //nolint:contextcheck ispec.MediaTypeImageManifest) So(err, ShouldBeNil) @@ -589,7 +590,7 @@ func TestWrapperErrors(t *testing.T) { }) Convey("FilterTags manifestMeta unmarshal error", func() { - err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig", ispec.MediaTypeImageManifest) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", "dig", ispec.MediaTypeImageManifest) //nolint:contextcheck So(err, ShouldBeNil) err = setBadManifestData(dynamoWrapper.Client, manifestDataTablename, "dig") //nolint:contextcheck @@ -609,7 +610,7 @@ func TestWrapperErrors(t *testing.T) { Convey("FilterTags bad IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck So(err, ShouldBeNil) err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck @@ -625,7 +626,7 @@ func TestWrapperErrors(t *testing.T) { Convey("FilterTags bad indexBlob in IndexData", func() { indexDigest := digest.FromString("indexDigest") - err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck So(err, ShouldBeNil) err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck @@ -647,7 +648,7 @@ func TestWrapperErrors(t *testing.T) { manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2") ) - err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck + err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck So(err, ShouldBeNil) indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{ diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go index c95c92d9..c73ea3ca 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go @@ -91,7 +91,7 @@ func NewDynamoDBWrapper(params dynamoParams.DBDriverParameters) (*DBWrapper, err return &dynamoWrapper, nil } -func (dwr DBWrapper) SetManifestData(manifestDigest godigest.Digest, manifestData repodb.ManifestData) error { +func (dwr *DBWrapper) SetManifestData(manifestDigest godigest.Digest, manifestData repodb.ManifestData) error { mdAttributeValue, err := attributevalue.Marshal(manifestData) if err != nil { return err @@ -116,7 +116,7 @@ func (dwr DBWrapper) SetManifestData(manifestDigest godigest.Digest, manifestDat return err } -func (dwr DBWrapper) GetManifestData(manifestDigest godigest.Digest) (repodb.ManifestData, error) { +func (dwr *DBWrapper) GetManifestData(manifestDigest godigest.Digest) (repodb.ManifestData, error) { resp, err := dwr.Client.GetItem(context.Background(), &dynamodb.GetItemInput{ TableName: aws.String(dwr.ManifestDataTablename), Key: map[string]types.AttributeValue{ @@ -141,7 +141,7 @@ func (dwr DBWrapper) GetManifestData(manifestDigest godigest.Digest) (repodb.Man return manifestData, nil } -func (dwr DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Digest, manifestMeta repodb.ManifestMetadata, +func (dwr *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Digest, manifestMeta repodb.ManifestMetadata, ) error { if manifestMeta.Signatures == nil { manifestMeta.Signatures = repodb.ManifestSignatures{} @@ -179,7 +179,7 @@ func (dwr DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Digest return err } -func (dwr DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Digest, +func (dwr *DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Digest, ) (repodb.ManifestMetadata, error) { //nolint:contextcheck manifestData, err := dwr.GetManifestData(manifestDigest) if err != nil { @@ -218,7 +218,7 @@ func (dwr DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Digest return manifestMetadata, nil } -func (dwr DBWrapper) IncrementRepoStars(repo string) error { +func (dwr *DBWrapper) IncrementRepoStars(repo string) error { repoMeta, err := dwr.GetRepoMeta(repo) if err != nil { return err @@ -231,7 +231,7 @@ func (dwr DBWrapper) IncrementRepoStars(repo string) error { return err } -func (dwr DBWrapper) DecrementRepoStars(repo string) error { +func (dwr *DBWrapper) DecrementRepoStars(repo string) error { repoMeta, err := dwr.GetRepoMeta(repo) if err != nil { return err @@ -246,7 +246,7 @@ func (dwr DBWrapper) DecrementRepoStars(repo string) error { return err } -func (dwr DBWrapper) GetRepoStars(repo string) (int, error) { +func (dwr *DBWrapper) GetRepoStars(repo string) (int, error) { repoMeta, err := dwr.GetRepoMeta(repo) if err != nil { return 0, err @@ -255,7 +255,7 @@ func (dwr DBWrapper) GetRepoStars(repo string) (int, error) { return repoMeta.Stars, nil } -func (dwr DBWrapper) SetIndexData(indexDigest godigest.Digest, indexData repodb.IndexData) error { +func (dwr *DBWrapper) SetIndexData(indexDigest godigest.Digest, indexData repodb.IndexData) error { indexAttributeValue, err := attributevalue.Marshal(indexData) if err != nil { return err @@ -280,7 +280,7 @@ func (dwr DBWrapper) SetIndexData(indexDigest godigest.Digest, indexData repodb. return err } -func (dwr DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData, error) { +func (dwr *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData, error) { resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ TableName: aws.String(dwr.IndexDataTablename), Key: map[string]types.AttributeValue{ @@ -307,8 +307,10 @@ func (dwr DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData return indexData, nil } -func (dwr DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest.Digest, mediaType string) error { - if err := common.ValidateRepoTagInput(repo, tag, manifestDigest); err != nil { +func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, + mediaType string, +) error { + if err := common.ValidateRepoReferenceInput(repo, reference, manifestDigest); err != nil { return err } @@ -336,17 +338,22 @@ func (dwr DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest } } - repoMeta.Tags[tag] = repodb.Descriptor{ - Digest: manifestDigest.String(), - MediaType: mediaType, + if !common.ReferenceIsDigest(reference) { + repoMeta.Tags[reference] = repodb.Descriptor{ + Digest: manifestDigest.String(), + MediaType: mediaType, + } } + repoMeta.Statistics[manifestDigest.String()] = repodb.DescriptorStatistics{DownloadCount: 0} + repoMeta.Signatures[manifestDigest.String()] = repodb.ManifestSignatures{} + err = dwr.setRepoMeta(repo, repoMeta) return err } -func (dwr DBWrapper) DeleteRepoTag(repo string, tag string) error { +func (dwr *DBWrapper) DeleteRepoTag(repo string, tag string) error { resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ TableName: aws.String(dwr.RepoMetaTablename), Key: map[string]types.AttributeValue{ @@ -405,7 +412,7 @@ func (dwr DBWrapper) DeleteRepoTag(repo string, tag string) error { return err } -func (dwr DBWrapper) GetRepoMeta(repo string) (repodb.RepoMetadata, error) { +func (dwr *DBWrapper) GetRepoMeta(repo string) (repodb.RepoMetadata, error) { resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ TableName: aws.String(dwr.RepoMetaTablename), Key: map[string]types.AttributeValue{ @@ -430,7 +437,7 @@ func (dwr DBWrapper) GetRepoMeta(repo string) (repodb.RepoMetadata, error) { return repoMeta, nil } -func (dwr DBWrapper) IncrementImageDownloads(repo string, reference string) error { +func (dwr *DBWrapper) IncrementImageDownloads(repo string, reference string) error { repoMeta, err := dwr.GetRepoMeta(repo) if err != nil { return err @@ -456,7 +463,7 @@ func (dwr DBWrapper) IncrementImageDownloads(repo string, reference string) erro return dwr.setRepoMeta(repo, repoMeta) } -func (dwr DBWrapper) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, +func (dwr *DBWrapper) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, sygMeta repodb.SignatureMetadata, ) error { repoMeta, err := dwr.GetRepoMeta(repo) @@ -497,7 +504,7 @@ func (dwr DBWrapper) AddManifestSignature(repo string, signedManifestDigest godi return err } -func (dwr DBWrapper) DeleteSignature(repo string, signedManifestDigest godigest.Digest, +func (dwr *DBWrapper) DeleteSignature(repo string, signedManifestDigest godigest.Digest, sigMeta repodb.SignatureMetadata, ) error { repoMeta, err := dwr.GetRepoMeta(repo) @@ -535,7 +542,7 @@ func (dwr DBWrapper) DeleteSignature(repo string, signedManifestDigest godigest. return err } -func (dwr DBWrapper) GetMultipleRepoMeta(ctx context.Context, +func (dwr *DBWrapper) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta repodb.RepoMetadata) bool, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, error) { var ( @@ -583,7 +590,7 @@ func (dwr DBWrapper) GetMultipleRepoMeta(ctx context.Context, return foundRepos, err } -func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter repodb.Filter, +func (dwr *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) { var ( @@ -743,7 +750,7 @@ func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err } -func (dwr DBWrapper) fetchManifestMetaWithCheck(repoName string, manifestDigest string, +func (dwr *DBWrapper) fetchManifestMetaWithCheck(repoName string, manifestDigest string, manifestMetadataMap map[string]repodb.ManifestMetadata, ) (repodb.ManifestMetadata, error) { var ( @@ -796,7 +803,7 @@ func collectImageManifestFilterData(digest string, repoMeta repodb.RepoMetadata, }, nil } -func (dwr DBWrapper) fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]repodb.IndexData, +func (dwr *DBWrapper) fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]repodb.IndexData, ) (repodb.IndexData, error) { var ( indexData repodb.IndexData @@ -816,7 +823,7 @@ func (dwr DBWrapper) fetchIndexDataWithCheck(indexDigest string, indexDataMap ma return indexData, err } -func (dwr DBWrapper) collectImageIndexFilterInfo(indexDigest string, repoMeta repodb.RepoMetadata, +func (dwr *DBWrapper) collectImageIndexFilterInfo(indexDigest string, repoMeta repodb.RepoMetadata, indexData repodb.IndexData, manifestMetadataMap map[string]repodb.ManifestMetadata, ) (repodb.FilterData, error) { var indexContent ispec.Index @@ -871,7 +878,7 @@ func (dwr DBWrapper) collectImageIndexFilterInfo(indexDigest string, repoMeta re }, nil } -func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, +func (dwr *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) { var ( @@ -1009,7 +1016,7 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err } -func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter, +func (dwr *DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) { var ( @@ -1244,7 +1251,7 @@ func (dwr *DBWrapper) PatchDB() error { return nil } -func (dwr DBWrapper) setRepoMeta(repo string, repoMeta repodb.RepoMetadata) error { +func (dwr *DBWrapper) setRepoMeta(repo string, repoMeta repodb.RepoMetadata) error { repoAttributeValue, err := attributevalue.Marshal(repoMeta) if err != nil { return err @@ -1269,7 +1276,7 @@ func (dwr DBWrapper) setRepoMeta(repo string, repoMeta repodb.RepoMetadata) erro return err } -func (dwr DBWrapper) createRepoMetaTable() error { +func (dwr *DBWrapper) createRepoMetaTable() error { _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ TableName: aws.String(dwr.RepoMetaTablename), AttributeDefinitions: []types.AttributeDefinition{ @@ -1294,7 +1301,7 @@ func (dwr DBWrapper) createRepoMetaTable() error { return dwr.waitTableToBeCreated(dwr.RepoMetaTablename) } -func (dwr DBWrapper) deleteRepoMetaTable() error { +func (dwr *DBWrapper) deleteRepoMetaTable() error { _, err := dwr.Client.DeleteTable(context.Background(), &dynamodb.DeleteTableInput{ TableName: aws.String(dwr.RepoMetaTablename), }) @@ -1306,7 +1313,7 @@ func (dwr DBWrapper) deleteRepoMetaTable() error { return dwr.waitTableToBeDeleted(dwr.RepoMetaTablename) } -func (dwr DBWrapper) ResetRepoMetaTable() error { +func (dwr *DBWrapper) ResetRepoMetaTable() error { err := dwr.deleteRepoMetaTable() if err != nil { return err @@ -1315,7 +1322,7 @@ func (dwr DBWrapper) ResetRepoMetaTable() error { return dwr.createRepoMetaTable() } -func (dwr DBWrapper) waitTableToBeCreated(tableName string) error { +func (dwr *DBWrapper) waitTableToBeCreated(tableName string) error { const maxWaitTime = 20 * time.Second waiter := dynamodb.NewTableExistsWaiter(dwr.Client) @@ -1325,7 +1332,7 @@ func (dwr DBWrapper) waitTableToBeCreated(tableName string) error { }, maxWaitTime) } -func (dwr DBWrapper) waitTableToBeDeleted(tableName string) error { +func (dwr *DBWrapper) waitTableToBeDeleted(tableName string) error { const maxWaitTime = 20 * time.Second waiter := dynamodb.NewTableNotExistsWaiter(dwr.Client) @@ -1335,7 +1342,7 @@ func (dwr DBWrapper) waitTableToBeDeleted(tableName string) error { }, maxWaitTime) } -func (dwr DBWrapper) createManifestDataTable() error { +func (dwr *DBWrapper) createManifestDataTable() error { _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ TableName: aws.String(dwr.ManifestDataTablename), AttributeDefinitions: []types.AttributeDefinition{ @@ -1360,7 +1367,7 @@ func (dwr DBWrapper) createManifestDataTable() error { return dwr.waitTableToBeCreated(dwr.ManifestDataTablename) } -func (dwr DBWrapper) createIndexDataTable() error { +func (dwr *DBWrapper) createIndexDataTable() error { _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ TableName: aws.String(dwr.IndexDataTablename), AttributeDefinitions: []types.AttributeDefinition{ @@ -1470,7 +1477,7 @@ func (dwr *DBWrapper) getDBVersion() (string, error) { return version, nil } -func (dwr DBWrapper) deleteManifestDataTable() error { +func (dwr *DBWrapper) deleteManifestDataTable() error { _, err := dwr.Client.DeleteTable(context.Background(), &dynamodb.DeleteTableInput{ TableName: aws.String(dwr.ManifestDataTablename), }) @@ -1482,7 +1489,7 @@ func (dwr DBWrapper) deleteManifestDataTable() error { return dwr.waitTableToBeDeleted(dwr.ManifestDataTablename) } -func (dwr DBWrapper) ResetManifestDataTable() error { +func (dwr *DBWrapper) ResetManifestDataTable() error { err := dwr.deleteManifestDataTable() if err != nil { return err diff --git a/pkg/meta/repodb/repodb.go b/pkg/meta/repodb/repodb.go index e0495161..e3a9700f 100644 --- a/pkg/meta/repodb/repodb.go +++ b/pkg/meta/repodb/repodb.go @@ -35,8 +35,8 @@ type RepoDB interface { //nolint:interfacebloat // GetRepoStars returns the total number of stars a repo has GetRepoStars(repo string) (int, error) - // SetRepoTag sets the tag of a manifest in the tag list of a repo - SetRepoTag(repo string, tag string, manifestDigest godigest.Digest, mediaType string) error + // SetRepoReference sets the reference of a manifest in the tag list of a repo + SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, mediaType string) error // DeleteRepoTag delets the tag from the tag list of a repo DeleteRepoTag(repo string, tag string) error diff --git a/pkg/meta/repodb/repodb_test.go b/pkg/meta/repodb/repodb_test.go index b154674c..aceb8291 100644 --- a/pkg/meta/repodb/repodb_test.go +++ b/pkg/meta/repodb/repodb_test.go @@ -176,7 +176,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { }) }) - Convey("Test SetRepoTag", func() { + Convey("Test SetRepoReference", func() { // test behaviours var ( repo1 = "repo1" @@ -189,18 +189,33 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { ) Convey("Setting a good repo", func() { - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repoMeta, err := repoDB.GetRepoMeta(repo1) So(err, ShouldBeNil) + So(repoMeta.Name, ShouldResemble, repo1) So(repoMeta.Tags[tag1].Digest, ShouldEqual, manifestDigest1) }) - Convey("Set multiple tags for repo", func() { - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + Convey("Setting a good repo using a digest", func() { + _, err := repoDB.GetRepoMeta(repo1) + So(err, ShouldNotBeNil) + + digest := godigest.FromString("digest") + err = repoDB.SetRepoReference(repo1, digest.String(), digest, + ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + + repoMeta, err := repoDB.GetRepoMeta(repo1) + So(err, ShouldBeNil) + So(repoMeta.Name, ShouldResemble, repo1) + }) + + Convey("Set multiple tags for repo", func() { + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + err = repoDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repoMeta, err := repoDB.GetRepoMeta(repo1) @@ -210,9 +225,9 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { }) Convey("Set multiple repos", func() { - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repoMeta1, err := repoDB.GetRepoMeta(repo1) @@ -226,17 +241,17 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { Convey("Setting a repo with invalid fields", func() { Convey("Repo name is not valid", func() { - err := repoDB.SetRepoTag("", tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference("", tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) Convey("Tag is not valid", func() { - err := repoDB.SetRepoTag(repo1, "", manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, "", manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) Convey("Manifest Digest is not valid", func() { - err := repoDB.SetRepoTag(repo1, tag1, "", ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, "", ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) }) @@ -255,10 +270,10 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { InexistentRepo = "InexistentRepo" ) - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) Convey("Get a existent repo", func() { @@ -287,10 +302,10 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { manifestDigest2 = godigest.FromString("fake-manifest2") ) - err := repoDB.SetRepoTag(repo, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) Convey("Delete from repo a tag", func() { @@ -352,13 +367,13 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { manifestDigest2 = godigest.FromString("fake-manifest2") ) - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) Convey("Get all Repometa", func() { @@ -407,7 +422,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { manifestDigest1 = godigest.FromString("fake-manifest1") ) - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = repoDB.IncrementRepoStars(repo1) @@ -439,7 +454,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { manifestDigest1 = godigest.FromString("fake-manifest1") ) - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = repoDB.IncrementRepoStars(repo1) @@ -474,7 +489,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { manifestDigest1 = godigest.FromString("fake-manifest1") ) - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = repoDB.IncrementRepoStars(repo1) @@ -508,7 +523,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { manifestDigest := godigest.FromBytes(manifestBlob) - err = repoDB.SetRepoTag(repo1, tag1, manifestDigest, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, tag1, manifestDigest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = repoDB.SetManifestMeta(repo1, manifestDigest, repodb.ManifestMetadata{ @@ -544,7 +559,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { manifestDigest1 = godigest.FromString("fake-manifest1") ) - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = repoDB.SetManifestMeta(repo1, manifestDigest1, repodb.ManifestMetadata{}) @@ -572,7 +587,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { manifestDigest1 = godigest.FromString("fake-manifest1") ) - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = repoDB.SetManifestMeta(repo1, manifestDigest1, repodb.ManifestMetadata{}) @@ -633,11 +648,11 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { } Convey("Search all repos", func() { - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo2, tag3, manifestDigest3, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo2, tag3, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) @@ -657,7 +672,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { }) Convey("Search a repo by name", func() { - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) @@ -671,10 +686,10 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { }) Convey("Search non-existing repo by name", func() { - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repos, manifestMetaMap, _, _, err := repoDB.SearchRepos(ctx, "RepoThatDoesntExist", repodb.Filter{}, @@ -685,11 +700,11 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { }) Convey("Search with partial match", func() { - err := repoDB.SetRepoTag("alpine", tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference("alpine", tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("pine", tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("pine", tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("golang", tag3, manifestDigest3, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("golang", tag3, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = repoDB.SetManifestMeta("alpine", manifestDigest1, emptyRepoMeta) @@ -708,11 +723,11 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { }) Convey("Search multiple repos that share manifests", func() { - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) @@ -729,11 +744,11 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { }) Convey("Search repos with access control", func() { - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) @@ -789,7 +804,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { } repoName := "repo" + strconv.Itoa(i) - err = repoDB.SetRepoTag(repoName, tag1, manifestDigest, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repoName, tag1, manifestDigest, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = repoDB.SetManifestMeta(repoName, manifestDigest, manifestMeta) @@ -972,10 +987,10 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { }) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo", tag4, indexDigest, ispec.MediaTypeImageIndex) + err = repoDB.SetRepoReference("repo", tag4, indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo", tag5, manifestDigest3, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo", tag5, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repos, manifestMetaMap, indexDataMap, _, err := repoDB.SearchRepos(ctx, "repo", repodb.Filter{}, repodb.PageInput{}) @@ -1015,17 +1030,17 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { ConfigBlob: emptyConfigBlob, } - err = repoDB.SetRepoTag(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) @@ -1093,11 +1108,11 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { tag3 = "0.0.3" ) - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) config := ispec.Image{} @@ -1199,13 +1214,13 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { }) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo", tag4, indexDigest, ispec.MediaTypeImageIndex) + err = repoDB.SetRepoReference("repo", tag4, indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo", tag5, manifestDigest3, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo", tag5, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag("repo", tag6, manifestDigest4, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference("repo", tag6, manifestDigest4, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) repos, manifestMetaMap, indexDataMap, _, err := repoDB.SearchTags(ctx, "repo:0.0", repodb.Filter{}, @@ -1236,15 +1251,15 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { tag5 = "0.0.5" ) - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, tag2, manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, tag2, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, tag3, manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, tag3, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, tag4, manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, tag4, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, tag5, manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, tag5, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) config := ispec.Image{} @@ -1309,15 +1324,15 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { manifestDigest3 = godigest.FromString("fake-manifest3") ) - err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo3, tag1, manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo3, tag1, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) config1 := ispec.Image{ @@ -1433,7 +1448,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { manifestFromIndexDigest2 = godigest.FromString("fake-manifestFromIndexDigest2") ) - err := repoDB.SetRepoTag(repo1, tag3, indexDigest, ispec.MediaTypeImageIndex) + err := repoDB.SetRepoReference(repo1, tag3, indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) indexBlob, err := test.GetIndexBlobWithManifests( @@ -1449,15 +1464,15 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { }) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo3, tag1, manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo3, tag1, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) config1 := ispec.Image{ @@ -1561,7 +1576,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { ConfigBlob: emptyConfigBlob, } - err = repoDB.SetRepoTag(repo1, "2.0.0", indexDigest, ispec.MediaTypeImageIndex) + err = repoDB.SetRepoReference(repo1, "2.0.0", indexDigest, ispec.MediaTypeImageIndex) So(err, ShouldBeNil) indexBlob, err := test.GetIndexBlobWithManifests([]godigest.Digest{ @@ -1575,17 +1590,17 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { }) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyManifestMeta) @@ -1797,15 +1812,15 @@ func TestRelevanceSorting(t *testing.T) { ConfigBlob: emptyConfigBlob, } - err = repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo2, tag3, manifestDigest3, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo2, tag3, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo3, tag3, manifestDigest3, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo3, tag3, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) - err = repoDB.SetRepoTag(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest) + err = repoDB.SetRepoReference(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest) So(err, ShouldBeNil) err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) diff --git a/pkg/meta/repodb/sync_repodb.go b/pkg/meta/repodb/sync_repodb.go index d3681181..0d291ae9 100644 --- a/pkg/meta/repodb/sync_repodb.go +++ b/pkg/meta/repodb/sync_repodb.go @@ -82,7 +82,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle } if manifestMetaIsPresent && hasTag { - err = repoDB.SetRepoTag(repo, tag, manifest.Digest, manifest.MediaType) + err = repoDB.SetRepoReference(repo, tag, manifest.Digest, manifest.MediaType) if err != nil { log.Error().Err(err).Msgf("sync-repo: failed to set repo tag for %s:%s", repo, tag) @@ -132,7 +132,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle } err = SetMetadataFromInput(repo, reference, manifest.MediaType, manifest.Digest, manifestBlob, - storeController, repoDB, log) + imageStore, repoDB, log) if err != nil { log.Error().Err(err).Msgf("sync-repo: failed to set metadata for %s:%s", repo, tag) @@ -220,7 +220,7 @@ func isManifestMetaPresent(repo string, manifest ispec.Descriptor, repoDB RepoDB } // NewManifestMeta takes raw data about an image and createa a new ManifestMetadate object. -func NewManifestData(repoName string, manifestBlob []byte, storeController storage.StoreController, +func NewManifestData(repoName string, manifestBlob []byte, imgStore storage.ImageStore, ) (ManifestData, error) { var ( manifestContent ispec.Manifest @@ -228,8 +228,6 @@ func NewManifestData(repoName string, manifestBlob []byte, storeController stora manifestData ManifestData ) - imgStore := storeController.GetImageStore(repoName) - err := json.Unmarshal(manifestBlob, &manifestContent) if err != nil { return ManifestData{}, err @@ -251,7 +249,7 @@ func NewManifestData(repoName string, manifestBlob []byte, storeController stora return manifestData, nil } -func NewIndexData(repoName string, indexBlob []byte, storeController storage.StoreController, +func NewIndexData(repoName string, indexBlob []byte, ) IndexData { indexData := IndexData{} @@ -263,11 +261,11 @@ func NewIndexData(repoName string, indexBlob []byte, storeController storage.Sto // SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag // (in case the reference is a tag). The function expects image manifests and indexes (multi arch images). func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte, - storeController storage.StoreController, repoDB RepoDB, log log.Logger, + imageStore storage.ImageStore, repoDB RepoDB, log log.Logger, ) error { switch mediaType { case ispec.MediaTypeImageManifest: - imageData, err := NewManifestData(repo, descriptorBlob, storeController) + imageData, err := NewManifestData(repo, descriptorBlob, imageStore) if err != nil { return err } @@ -279,7 +277,7 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig return err } case ispec.MediaTypeImageIndex: - indexData := NewIndexData(repo, descriptorBlob, storeController) + indexData := NewIndexData(repo, descriptorBlob) err := repoDB.SetIndexData(digest, indexData) if err != nil { @@ -289,11 +287,7 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig } } - if refferenceIsDigest(reference) { - return nil - } - - err := repoDB.SetRepoTag(repo, reference, digest, mediaType) + err := repoDB.SetRepoReference(repo, reference, digest, mediaType) if err != nil { log.Error().Err(err).Msg("repodb: error while putting repo meta") @@ -302,9 +296,3 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig return nil } - -func refferenceIsDigest(reference string) bool { - _, err := godigest.Parse(reference) - - return err == nil -} diff --git a/pkg/meta/repodb/sync_repodb_test.go b/pkg/meta/repodb/sync_repodb_test.go index 1191e098..eec1544a 100644 --- a/pkg/meta/repodb/sync_repodb_test.go +++ b/pkg/meta/repodb/sync_repodb_test.go @@ -178,8 +178,8 @@ func TestSyncRepoDBErrors(t *testing.T) { return indexBlob, nil } - Convey("repoDB.SetRepoTag", func() { - repoDB.SetRepoTagFn = func(repo, tag string, manifestDigest godigest.Digest, mediaType string) error { + Convey("repoDB.SetRepoReference", func() { + repoDB.SetRepoReferenceFn = func(repo, tag string, manifestDigest godigest.Digest, mediaType string) error { return ErrTestError } diff --git a/pkg/meta/repodb/update/update.go b/pkg/meta/repodb/update/update.go index 6b2fb154..0c57c3bd 100644 --- a/pkg/meta/repodb/update/update.go +++ b/pkg/meta/repodb/update/update.go @@ -52,7 +52,7 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest, } } else { err := repodb.SetMetadataFromInput(name, reference, mediaType, digest, body, - storeController, repoDB, log) + imgStore, repoDB, log) if err != nil { metadataSuccessfullySet = false } @@ -152,7 +152,7 @@ func OnGetManifest(name, reference string, digest godigest.Digest, body []byte, if !isSignature { err := repoDB.IncrementImageDownloads(name, reference) if err != nil { - log.Error().Err(err).Msg("unexpected error") + log.Error().Err(err).Msgf("unexpected error for '%s:%s'", name, reference) return err } diff --git a/pkg/meta/repodb/update/update_test.go b/pkg/meta/repodb/update/update_test.go index 53fd21b6..8c184746 100644 --- a/pkg/meta/repodb/update/update_test.go +++ b/pkg/meta/repodb/update/update_test.go @@ -181,12 +181,11 @@ func TestUpdateErrors(t *testing.T) { Convey("SetMetadataFromInput", func() { imageStore := mocks.MockedImageStore{} - storeController := storage.StoreController{DefaultStore: &imageStore} repoDB := mocks.RepoDBMock{} log := log.NewLogger("debug", "") err := repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest", - []byte("BadManifestBlob"), storeController, repoDB, log) + []byte("BadManifestBlob"), imageStore, repoDB, log) So(err, ShouldNotBeNil) // reference is digest @@ -203,7 +202,7 @@ func TestUpdateErrors(t *testing.T) { } err = repodb.SetMetadataFromInput("repo", string(godigest.FromString("reference")), "", "digest", - manifestBlob, storeController, repoDB, log) + manifestBlob, imageStore, repoDB, log) So(err, ShouldBeNil) }) }) diff --git a/pkg/test/common_test.go b/pkg/test/common_test.go index 6b38a23f..5df53ebf 100644 --- a/pkg/test/common_test.go +++ b/pkg/test/common_test.go @@ -6,6 +6,7 @@ package test_test import ( "context" "encoding/json" + "errors" "fmt" "os" "path" @@ -23,6 +24,8 @@ import ( "zotregistry.io/zot/pkg/test" ) +var ErrTestError = errors.New("test error") + func TestCopyFiles(t *testing.T) { Convey("sourceDir does not exist", t, func() { err := test.CopyFiles("/path/to/some/unexisting/directory", os.TempDir()) diff --git a/pkg/test/mocks/repo_db_mock.go b/pkg/test/mocks/repo_db_mock.go index 73ebdde3..81526e8b 100644 --- a/pkg/test/mocks/repo_db_mock.go +++ b/pkg/test/mocks/repo_db_mock.go @@ -19,7 +19,7 @@ type RepoDBMock struct { SetRepoLogoFn func(repo string, logoPath string) error - SetRepoTagFn func(repo string, tag string, manifestDigest godigest.Digest, mediaType string) error + SetRepoReferenceFn func(repo string, Reference string, manifestDigest godigest.Digest, mediaType string) error DeleteRepoTagFn func(repo string, tag string) error @@ -111,9 +111,11 @@ func (sdm RepoDBMock) SetRepoLogo(repo string, logoPath string) error { return nil } -func (sdm RepoDBMock) SetRepoTag(repo string, tag string, manifestDigest godigest.Digest, mediaType string) error { - if sdm.SetRepoTagFn != nil { - return sdm.SetRepoTagFn(repo, tag, manifestDigest, mediaType) +func (sdm RepoDBMock) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, + mediaType string, +) error { + if sdm.SetRepoReferenceFn != nil { + return sdm.SetRepoReferenceFn(repo, reference, manifestDigest, mediaType) } return nil