Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00
Code Issues Activity

feat(repodb): sync-repodb WIP (#1241)

Browse source 
Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae 2023-03-09 20:41:48 +02:00 committed by GitHub
parent fd5a2af10b
commit 4c156234cb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
30 changed files with 652 additions and 315 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
errors/errors.go
View file

@ -61,7 +61,7 @@ var (
ErrBadRange = errors.New("storage: bad range")
ErrBadLayerCount = errors.New("manifest: layers count doesn't correspond to config history")
ErrManifestConflict = errors.New("manifest: multiple manifests found")
ErrManifestMetaNotFound = errors.New("repodb: image metadata not found for given manifest digest")
ErrManifestMetaNotFound = errors.New("repodb: image metadata not found for given manifest reference")
ErrManifestDataNotFound = errors.New("repodb: image data not found for given manifest digest")
ErrIndexDataNotFount = errors.New("repodb: index data not found for given digest")
ErrRepoMetaNotFound = errors.New("repodb: repo metadata not found for given repo name")

4
pkg/api/controller.go
View file

@ -605,7 +605,7 @@ func (c *Controller) LoadNewConfig(reloadCtx context.Context, config *config.Con
if config.Extensions != nil && config.Extensions.Sync != nil {
// reload sync config
c.Config.Extensions.Sync = config.Extensions.Sync
ext.EnableSyncExtension(reloadCtx, c.Config, c.wgShutDown, c.StoreController, c.Log)
ext.EnableSyncExtension(reloadCtx, c.Config, c.wgShutDown, c.RepoDB, c.StoreController, c.Log)
} else if c.Config.Extensions != nil {
c.Config.Extensions.Sync = nil
}
@ -653,7 +653,7 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
// Enable extensions if extension config is provided for storeController
if c.Config.Extensions != nil {
if c.Config.Extensions.Sync != nil {
ext.EnableSyncExtension(reloadCtx, c.Config, c.wgShutDown, c.StoreController, c.Log)
ext.EnableSyncExtension(reloadCtx, c.Config, c.wgShutDown, c.RepoDB, c.StoreController, c.Log)
}
}

6
pkg/api/routes.go
View file

@ -438,7 +438,7 @@ func getReferrers(ctx context.Context, routeHandler *RouteHandler,
routeHandler.c.Log.Info().Msgf("referrers not found, trying to get referrers to %s:%s by syncing on demand",
name, digest)
errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.StoreController,
errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.RepoDB, routeHandler.c.StoreController,
name, digest.String(), sync.OCIReference, routeHandler.c.Log)
if errSync != nil {
routeHandler.c.Log.Error().Err(err).Str("name", name).Str("digest", digest.String()).Msg("unable to get references")
@ -1640,7 +1640,7 @@ func getImageManifest(ctx context.Context, routeHandler *RouteHandler, imgStore
routeHandler.c.Log.Info().Msgf("trying to get updated image %s:%s by syncing on demand",
name, reference)
errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.StoreController,
errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.RepoDB, routeHandler.c.StoreController,
name, reference, "", routeHandler.c.Log)
if errSync != nil {
routeHandler.c.Log.Err(errSync).Msgf("error encounter while syncing image %s:%s",
@ -1664,7 +1664,7 @@ func getOrasReferrers(ctx context.Context, routeHandler *RouteHandler,
routeHandler.c.Log.Info().Msgf("artifact not found, trying to get artifact %s:%s by syncing on demand",
name, digest.String())
errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.StoreController,
errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.RepoDB, routeHandler.c.StoreController,
name, digest.String(), sync.OrasArtifact, routeHandler.c.Log)
if errSync != nil {
routeHandler.c.Log.Error().Err(err).Str("name", name).Str("digest", digest.String()).Msg("unable to get references")

11
pkg/extensions/extension_sync.go
View file

@ -10,14 +10,15 @@ import (
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/extensions/sync"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
)
func EnableSyncExtension(ctx context.Context, config *config.Config, wg *goSync.WaitGroup,
storeController storage.StoreController, log log.Logger,
repoDB repodb.RepoDB, storeController storage.StoreController, log log.Logger,
) {
if config.Extensions.Sync != nil && *config.Extensions.Sync.Enable {
if err := sync.Run(ctx, *config.Extensions.Sync, storeController, wg, log); err != nil {
if err := sync.Run(ctx, *config.Extensions.Sync, repoDB, storeController, wg, log); err != nil {
log.Error().Err(err).Msg("Error encountered while setting up syncing")
}
} else {
@ -25,12 +26,12 @@ func EnableSyncExtension(ctx context.Context, config *config.Config, wg *goSync.
}
}
func SyncOneImage(ctx context.Context, config *config.Config, storeController storage.StoreController,
repoName, reference string, artifactType string, log log.Logger,
func SyncOneImage(ctx context.Context, config *config.Config, repoDB repodb.RepoDB,
storeController storage.StoreController, repoName, reference string, artifactType string, log log.Logger,
) error {
log.Info().Msgf("syncing image %s:%s", repoName, reference)
err := sync.OneImage(ctx, *config.Extensions.Sync, storeController, repoName, reference, artifactType, log)
err := sync.OneImage(ctx, *config.Extensions.Sync, repoDB, storeController, repoName, reference, artifactType, log)
return err
}

7
pkg/extensions/extension_sync_disabled.go
View file

@ -9,12 +9,13 @@ import (
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
)
// EnableSyncExtension ...
func EnableSyncExtension(ctx context.Context,
config *config.Config, wg *goSync.WaitGroup,
config *config.Config, wg *goSync.WaitGroup, repoDB repodb.RepoDB,
storeController storage.StoreController, log log.Logger,
) {
log.Warn().Msg("skipping enabling sync extension because given zot binary doesn't include this feature," +
@ -22,8 +23,8 @@ func EnableSyncExtension(ctx context.Context,
}
// SyncOneImage ...
func SyncOneImage(ctx context.Context, config *config.Config, storeController storage.StoreController,
repoName, reference string, artifactType string, log log.Logger,
func SyncOneImage(ctx context.Context, config *config.Config, repoDB repodb.RepoDB,
storeController storage.StoreController, repoName, reference string, artifactType string, log log.Logger,
) error {
log.Warn().Msg("skipping syncing on demand because given zot binary doesn't include this feature," +
"please build a binary that does so")

4
pkg/extensions/search/common/common_test.go
View file

@ -4588,9 +4588,9 @@ func TestRepoDBWhenPushingImages(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("SetManifestMeta succeeds but SetRepoTag fails", func() {
Convey("SetManifestMeta succeeds but SetRepoReference fails", func() {
ctlr.RepoDB = mocks.RepoDBMock{
SetRepoTagFn: func(repo, tag string, manifestDigest godigest.Digest, mediaType string) error {
SetRepoReferenceFn: func(repo, reference string, manifestDigest godigest.Digest, mediaType string) error {
return ErrTestError
},
}

2
pkg/extensions/search/convert/convert_test.go
View file

@ -57,7 +57,7 @@ func TestConvertErrors(t *testing.T) {
digest11 := godigest.FromString("abc1")
err = repoDB.SetManifestMeta("repo1", digest11, repoMeta11)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
repoMetas, manifestMetaMap, _, _, err := repoDB.SearchRepos(context.Background(), "", repodb.Filter{},

20
pkg/extensions/search/cve/cve_test.go
View file

@ -755,7 +755,7 @@ func TestCVEStruct(t *testing.T) {
digest11 := godigest.FromBytes(manifestBlob11)
err = repoDB.SetManifestMeta("repo1", digest11, repoMeta11)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
timeStamp12 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC)
@ -791,7 +791,7 @@ func TestCVEStruct(t *testing.T) {
digest12 := godigest.FromBytes(manifestBlob12)
err = repoDB.SetManifestMeta("repo1", digest12, repoMeta12)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo1", "1.0.0", digest12, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo1", "1.0.0", digest12, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
timeStamp13 := time.Date(2010, 1, 1, 12, 0, 0, 0, time.UTC)
@ -825,7 +825,7 @@ func TestCVEStruct(t *testing.T) {
digest13 := godigest.FromBytes(manifestBlob13)
err = repoDB.SetManifestMeta("repo1", digest13, repoMeta13)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo1", "1.1.0", digest13, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo1", "1.1.0", digest13, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
timeStamp14 := time.Date(2011, 1, 1, 12, 0, 0, 0, time.UTC)
@ -859,7 +859,7 @@ func TestCVEStruct(t *testing.T) {
digest14 := godigest.FromBytes(manifestBlob14)
err = repoDB.SetManifestMeta("repo1", digest14, repoMeta14)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo1", "1.0.1", digest14, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo1", "1.0.1", digest14, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
// Create repodb data for scannable image with no vulnerabilities
@ -894,7 +894,7 @@ func TestCVEStruct(t *testing.T) {
digest61 := godigest.FromBytes(manifestBlob61)
err = repoDB.SetManifestMeta("repo6", digest61, repoMeta61)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo6", "1.0.0", digest61, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo6", "1.0.0", digest61, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
// Create repodb data for image not supporting scanning
@ -929,7 +929,7 @@ func TestCVEStruct(t *testing.T) {
digest21 := godigest.FromBytes(manifestBlob21)
err = repoDB.SetManifestMeta("repo2", digest21, repoMeta21)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo2", "1.0.0", digest21, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo2", "1.0.0", digest21, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
// Create repodb data for invalid images/negative tests
@ -943,7 +943,7 @@ func TestCVEStruct(t *testing.T) {
digest31 := godigest.FromBytes(manifestBlob31)
err = repoDB.SetManifestMeta("repo3", digest31, repoMeta31)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo3", "invalid-manifest", digest31, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo3", "invalid-manifest", digest31, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
configBlob41 := []byte("invalid config blob")
@ -956,11 +956,11 @@ func TestCVEStruct(t *testing.T) {
digest41 := godigest.FromString("abc7")
err = repoDB.SetManifestMeta("repo4", digest41, repoMeta41)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo4", "invalid-config", digest41, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo4", "invalid-config", digest41, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
digest51 := godigest.FromString("abc8")
err = repoDB.SetRepoTag("repo5", "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo5", "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
// ------ Multiarch image
@ -997,7 +997,7 @@ func TestCVEStruct(t *testing.T) {
})
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repoIndex", "tagIndex", indexDigest, ispec.MediaTypeImageIndex)
err = repoDB.SetRepoReference("repoIndex", "tagIndex", indexDigest, ispec.MediaTypeImageIndex)
So(err, ShouldBeNil)
// RepoDB loaded with initial data, mock the scanner

4
pkg/extensions/search/cve/pagination_test.go
View file

@ -61,7 +61,7 @@ func TestCVEPagination(t *testing.T) {
digest11 := godigest.FromBytes(manifestBlob11)
err = repoDB.SetManifestMeta("repo1", digest11, repoMeta11)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
timeStamp12 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC)
@ -95,7 +95,7 @@ func TestCVEPagination(t *testing.T) {
digest12 := godigest.FromBytes(manifestBlob12)
err = repoDB.SetManifestMeta("repo1", digest12, repoMeta12)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo1", "1.0.0", digest12, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo1", "1.0.0", digest12, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
// RepoDB loaded with initial data, mock the scanner

11
pkg/extensions/search/cve/trivy/scanner_internal_test.go
View file

@ -273,7 +273,7 @@ func TestImageScannable(t *testing.T) {
panic(err)
}
err = repoDB.SetRepoTag("repo1", "valid", digestValidManifest, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo1", "valid", digestValidManifest, ispec.MediaTypeImageManifest)
if err != nil {
panic(err)
}
@ -309,7 +309,8 @@ func TestImageScannable(t *testing.T) {
panic(err)
}
err = repoDB.SetRepoTag("repo1", "unscannable-layer", digestManifestUnscannableLayer, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo1", "unscannable-layer", digestManifestUnscannableLayer,
ispec.MediaTypeImageManifest)
if err != nil {
panic(err)
}
@ -328,7 +329,7 @@ func TestImageScannable(t *testing.T) {
panic(err)
}
err = repoDB.SetRepoTag("repo1", "unmarshable", digestUnmarshableManifest, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo1", "unmarshable", digestUnmarshableManifest, ispec.MediaTypeImageManifest)
if err != nil {
panic(err)
}
@ -336,13 +337,13 @@ func TestImageScannable(t *testing.T) {
// Manifest meta cannot be found
digestMissingManifest := godigest.FromBytes([]byte("Some other string"))
err = repoDB.SetRepoTag("repo1", "missing", digestMissingManifest, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo1", "missing", digestMissingManifest, ispec.MediaTypeImageManifest)
if err != nil {
panic(err)
}
// RepoMeta contains invalid digest
err = repoDB.SetRepoTag("repo1", "invalid-digest", "invalid", ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo1", "invalid-digest", "invalid", ispec.MediaTypeImageManifest)
if err != nil {
panic(err)
}

2
pkg/extensions/search/resolver_test.go
View file

@ -2020,7 +2020,7 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo
for image, digest := range tagsMap {
repo, tag := common.GetImageDirAndTag(image)
err := repoDB.SetRepoTag(repo, tag, digest, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo, tag, digest, ispec.MediaTypeImageManifest)
if err != nil {
panic(err)
}

14
pkg/extensions/sync/on_demand.go
View file

@ -16,6 +16,7 @@ import (
"zotregistry.io/zot/pkg/common"
syncconf "zotregistry.io/zot/pkg/extensions/config/sync"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
)
@ -59,8 +60,8 @@ func (di *demandedImages) delete(key string) {
di.syncedMap.Delete(key)
}
func OneImage(ctx context.Context, cfg syncconf.Config, storeController storage.StoreController,
repo, reference string, artifactType string, log log.Logger,
func OneImage(ctx context.Context, cfg syncconf.Config, repoDB repodb.RepoDB,
storeController storage.StoreController, repo, reference string, artifactType string, log log.Logger,
) error {
// guard against multiple parallel requests
demandedImage := fmt.Sprintf("%s:%s", repo, reference)
@ -82,7 +83,7 @@ func OneImage(ctx context.Context, cfg syncconf.Config, storeController storage.
defer demandedImgs.delete(demandedImage)
defer close(imageChannel)
go syncOneImage(ctx, imageChannel, cfg, storeController, repo, reference, artifactType, log)
go syncOneImage(ctx, imageChannel, cfg, repoDB, storeController, repo, reference, artifactType, log)
err, ok := <-imageChannel
if !ok {
@ -92,8 +93,8 @@ func OneImage(ctx context.Context, cfg syncconf.Config, storeController storage.
return err
}
func syncOneImage(ctx context.Context, imageChannel chan error,
cfg syncconf.Config, storeController storage.StoreController,
func syncOneImage(ctx context.Context, imageChannel chan error, cfg syncconf.Config,
repoDB repodb.RepoDB, storeController storage.StoreController,
localRepo, reference string, artifactType string, log log.Logger,
) {
var credentialsFile syncconf.CredentialsFile
@ -180,7 +181,8 @@ func syncOneImage(ctx context.Context, imageChannel chan error,
return
}
sig := newSignaturesCopier(httpClient, credentialsFile[upstreamAddr], *registryURL, storeController, log)
sig := newSignaturesCopier(httpClient, credentialsFile[upstreamAddr], *registryURL, repoDB,
storeController, log)
upstreamCtx := getUpstreamContext(&regCfg, credentialsFile[upstreamAddr])
options := getCopyOptions(upstreamCtx, localCtx)

77
pkg/extensions/sync/signatures.go
View file

@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"path"
@ -20,25 +21,28 @@ import (
"zotregistry.io/zot/pkg/common"
syncconf "zotregistry.io/zot/pkg/extensions/config/sync"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
)
type signaturesCopier struct {
client *http.Client
upstreamURL url.URL
storeController storage.StoreController
credentials syncconf.Credentials
repoDB repodb.RepoDB
storeController storage.StoreController
log log.Logger
}
func newSignaturesCopier(httpClient *http.Client, credentials syncconf.Credentials,
upstreamURL url.URL,
upstreamURL url.URL, repoDB repodb.RepoDB,
storeController storage.StoreController, log log.Logger,
) *signaturesCopier {
return &signaturesCopier{
client: httpClient,
credentials: credentials,
upstreamURL: upstreamURL,
repoDB: repoDB,
storeController: storeController,
log: log,
}
@ -185,7 +189,18 @@ func (sig *signaturesCopier) syncCosignSignature(localRepo, remoteRepo, digestSt
return err
}
sig.log.Info().Msgf("successfully synced cosign signature for repo %s digest %s", localRepo, digestStr)
if sig.repoDB != nil {
sig.log.Debug().Msgf("trying to sync cosign signature for repo %s digest %s", localRepo, digestStr)
err = repodb.SetMetadataFromInput(localRepo, cosignTag, ispec.MediaTypeImageManifest,
godigest.FromBytes(cosignManifestBuf), cosignManifestBuf, sig.storeController.GetImageStore(localRepo),
sig.repoDB, sig.log)
if err != nil {
return fmt.Errorf("failed to set metadata for cosign signature '%s@%s': %w", localRepo, digestStr, err)
}
sig.log.Info().Msgf("successfully added cosign signature to RepoDB for repo %s digest %s", localRepo, digestStr)
}
return nil
}
@ -249,6 +264,19 @@ func (sig *signaturesCopier) syncORASRefs(localRepo, remoteRepo, digestStr strin
return err
}
// this is for notation signatures
if sig.repoDB != nil {
sig.log.Debug().Msgf("trying to sync oras artifact for repo %s digest %s", localRepo, digestStr)
err = repodb.SetMetadataFromInput(localRepo, ref.Digest.String(), ref.MediaType,
ref.Digest, body, sig.storeController.GetImageStore(localRepo), sig.repoDB, sig.log)
if err != nil {
return fmt.Errorf("failed to set metadata for oras artifact '%s@%s': %w", localRepo, digestStr, err)
}
sig.log.Info().Msgf("successfully added oras artifacts to RepoDB for repo %s digest %s", localRepo, digestStr)
}
}
sig.log.Info().Msgf("successfully synced ORAS artifacts for repo %s digest %s", localRepo, digestStr)
@ -283,7 +311,7 @@ func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, digestStr string
var artifactManifest oras.Manifest
body, statusCode, err := common.MakeHTTPGetRequest(sig.client, sig.credentials.Username,
OCIRefBody, statusCode, err := common.MakeHTTPGetRequest(sig.client, sig.credentials.Username,
sig.credentials.Password, &artifactManifest,
getRefManifestURL.String(), ref.MediaType, sig.log)
if err != nil {
@ -304,7 +332,7 @@ func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, digestStr string
// read manifest
var manifest ispec.Manifest
err = json.Unmarshal(body, &manifest)
err = json.Unmarshal(OCIRefBody, &manifest)
if err != nil {
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't unmarshal oci reference manifest: %s", getRefManifestURL.String())
@ -326,7 +354,7 @@ func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, digestStr string
// read manifest
var manifest ispec.Artifact
err = json.Unmarshal(body, &manifest)
err = json.Unmarshal(OCIRefBody, &manifest)
if err != nil {
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't unmarshal oci reference manifest: %s", getRefManifestURL.String())
@ -341,17 +369,30 @@ func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, digestStr string
}
}
_, err = imageStore.PutImageManifest(localRepo, ref.Digest.String(),
ref.MediaType, body)
digest, err := imageStore.PutImageManifest(localRepo, ref.Digest.String(),
ref.MediaType, OCIRefBody)
if err != nil {
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't upload oci reference manifest")
return err
}
if sig.repoDB != nil {
sig.log.Debug().Msgf("trying to add OCI refs for repo %s digest %s", localRepo, digestStr)
err = repodb.SetMetadataFromInput(localRepo, digestStr, ref.MediaType,
digest, OCIRefBody, sig.storeController.GetImageStore(localRepo),
sig.repoDB, sig.log)
if err != nil {
return fmt.Errorf("failed to set metadata for OCI ref in '%s@%s': %w", localRepo, digestStr, err)
}
sig.log.Info().Msgf("successfully added OCI refs to RepoDB for repo %s digest %s", localRepo, digestStr)
}
}
sig.log.Info().Msgf("successfully synced oci references for repo %s digest %s", localRepo, digestStr)
sig.log.Info().Msgf("successfully synced OCI refs for repo %s digest %s", localRepo, digestStr)
return nil
}
@ -397,7 +438,7 @@ func (sig *signaturesCopier) syncOCIArtifact(localRepo, remoteRepo, reference st
}
// push manifest
_, err = imageStore.PutImageManifest(localRepo, reference,
digest, err := imageStore.PutImageManifest(localRepo, reference,
ispec.MediaTypeArtifactManifest, artifactManifestBuf)
if err != nil {
sig.log.Error().Str("errorType", common.TypeOf(err)).
@ -406,6 +447,20 @@ func (sig *signaturesCopier) syncOCIArtifact(localRepo, remoteRepo, reference st
return err
}
if sig.repoDB != nil {
sig.log.Debug().Msgf("trying to OCI refs for repo %s digest %s", localRepo, digest.String())
err = repodb.SetMetadataFromInput(localRepo, reference, ispec.MediaTypeArtifactManifest,
digest, artifactManifestBuf, sig.storeController.GetImageStore(localRepo),
sig.repoDB, sig.log)
if err != nil {
return fmt.Errorf("failed to set metadata for OCI Artifact '%s@%s': %w", localRepo, digest.String(), err)
}
sig.log.Info().Msgf("successfully added oci artifacts to RepoDB for repo %s digest %s", localRepo,
digest.String())
}
sig.log.Info().Msgf("successfully synced OCI artifact for repo %s tag %s", localRepo, reference)
return nil
@ -450,7 +505,7 @@ func (sig *signaturesCopier) canSkipOCIArtifact(localRepo, reference string, art
localArtifactBuf, _, _, err := imageStore.GetImageManifest(localRepo, reference)
if err != nil {
if errors.Is(err, zerr.ErrManifestNotFound) {
if errors.Is(err, zerr.ErrManifestNotFound) || errors.Is(err, zerr.ErrRepoNotFound) {
return false, nil
}

12
pkg/extensions/sync/sync.go
View file

@ -23,6 +23,7 @@ import (
"zotregistry.io/zot/pkg/common"
syncconf "zotregistry.io/zot/pkg/extensions/config/sync"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/test"
)
@ -165,7 +166,7 @@ func getUpstreamContext(regCfg *syncconf.RegistryConfig, credentials syncconf.Cr
//nolint:gocyclo // offloading some of the functionalities from here would make the code harder to follow
func syncRegistry(ctx context.Context, regCfg syncconf.RegistryConfig,
upstreamURL string,
upstreamURL string, repoDB repodb.RepoDB,
storeController storage.StoreController, localCtx *types.SystemContext,
policyCtx *signature.PolicyContext, credentials syncconf.Credentials,
retryOptions *retry.RetryOptions, log log.Logger,
@ -246,7 +247,7 @@ func syncRegistry(ctx context.Context, regCfg syncconf.RegistryConfig,
}
}
sig := newSignaturesCopier(httpClient, credentials, *registryURL, storeController, log)
sig := newSignaturesCopier(httpClient, credentials, *registryURL, repoDB, storeController, log)
for _, repoReference := range reposReferences {
upstreamRepo := repoReference.name
@ -323,9 +324,8 @@ func getLocalContexts(log log.Logger) (*types.SystemContext, *signature.PolicyCo
return localCtx, policyContext, nil
}
func Run(ctx context.Context, cfg syncconf.Config,
storeController storage.StoreController,
wtgrp *goSync.WaitGroup, logger log.Logger,
func Run(ctx context.Context, cfg syncconf.Config, repoDB repodb.RepoDB,
storeController storage.StoreController, wtgrp *goSync.WaitGroup, logger log.Logger,
) error {
var credentialsFile syncconf.CredentialsFile
@ -382,7 +382,7 @@ func Run(ctx context.Context, cfg syncconf.Config,
for _, upstreamURL := range regCfg.URLs {
upstreamAddr := StripRegistryTransport(upstreamURL)
// first try syncing main registry
if err := syncRegistry(ctx, regCfg, upstreamURL, storeController, localCtx, policyCtx,
if err := syncRegistry(ctx, regCfg, upstreamURL, repoDB, storeController, localCtx, policyCtx,
credentialsFile[upstreamAddr], retryOptions, logger); err != nil {
logger.Error().Str("errortype", common.TypeOf(err)).
Err(err).Str("registry", upstreamURL).

66
pkg/extensions/sync/sync_internal_test.go
View file

@ -40,6 +40,8 @@ const (
host = "127.0.0.1:45117"
)
var ErrTestError = fmt.Errorf("testError")
func TestInjectSyncUtils(t *testing.T) {
Convey("Inject errors in utils functions", t, func() {
repositoryReference := fmt.Sprintf("%s/%s", host, testImage)
@ -154,7 +156,7 @@ func TestSyncInternal(t *testing.T) {
}
ctx := context.Background()
So(Run(ctx, cfg, storage.StoreController{},
So(Run(ctx, cfg, mocks.RepoDBMock{}, storage.StoreController{},
new(goSync.WaitGroup), log.NewLogger("debug", "")), ShouldNotBeNil)
_, err = getFileCredentials("/invalid/path/to/file")
@ -190,7 +192,7 @@ func TestSyncInternal(t *testing.T) {
localCtx, policyCtx, err := getLocalContexts(log)
So(err, ShouldBeNil)
err = syncRegistry(ctx, syncRegistryConfig, "randomUpstreamURL",
err = syncRegistry(ctx, syncRegistryConfig, "randomUpstreamURL", mocks.RepoDBMock{},
storage.StoreController{DefaultStore: imageStore}, localCtx, policyCtx,
syncconf.Credentials{}, nil, log)
So(err, ShouldNotBeNil)
@ -378,9 +380,10 @@ func TestSyncInternal(t *testing.T) {
imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
false, false, log, metrics, nil, nil,
)
mockRepoDB := mocks.RepoDBMock{}
sig := newSignaturesCopier(client, syncconf.Credentials{},
*regURL, storage.StoreController{DefaultStore: imageStore}, log)
*regURL, mockRepoDB, storage.StoreController{DefaultStore: imageStore}, log)
err = sig.syncCosignSignature(testImage, testImage, testImageTag, &ispec.Manifest{})
So(err, ShouldNotBeNil)
@ -430,8 +433,9 @@ func TestSyncInternal(t *testing.T) {
So(regURL, ShouldNotBeNil)
client := &http.Client{}
mockRepoDB := mocks.RepoDBMock{}
sig := newSignaturesCopier(client, syncconf.Credentials{},
*regURL, storage.StoreController{DefaultStore: imageStore}, log)
*regURL, mockRepoDB, storage.StoreController{DefaultStore: imageStore}, log)
canBeSkipped, err = sig.canSkipOCIRefs(testImage, testImageManifestDigest.String(), refs)
So(err, ShouldBeNil)
@ -596,7 +600,7 @@ func TestSyncInternal(t *testing.T) {
testRootDir := path.Join(imageStore.RootDir(), testImage, SyncBlobUploadDir)
// testImagePath := path.Join(testRootDir, testImage)
err := pushSyncedLocalImage(testImage, testImageTag, testRootDir, imageStore, log)
err := pushSyncedLocalImage(testImage, testImageTag, testRootDir, nil, imageStore, log)
So(err, ShouldNotBeNil)
err = os.MkdirAll(testRootDir, 0o755)
@ -663,7 +667,7 @@ func TestSyncInternal(t *testing.T) {
_, err = testImageStore.PutImageManifest(repo, "latest", ispec.MediaTypeImageIndex, content)
So(err, ShouldBeNil)
err = pushSyncedLocalImage(repo, "latest", testRootDir, imageStore, log)
err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStore, log)
So(err, ShouldBeNil)
// trigger error on manifest pull
@ -671,7 +675,7 @@ func TestSyncInternal(t *testing.T) {
index.Manifests[0].Digest.Algorithm().String(), index.Manifests[0].Digest.Encoded()), 0o000)
So(err, ShouldBeNil)
err = pushSyncedLocalImage(repo, "latest", testRootDir, imageStore, log)
err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStore, log)
So(err, ShouldNotBeNil)
err = os.Chmod(path.Join(testRootDir, repo, "blobs",
@ -687,7 +691,7 @@ func TestSyncInternal(t *testing.T) {
}, nil,
)
err = pushSyncedLocalImage(repo, "latest", testRootDir, imageStoreWithLinter, log)
err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStoreWithLinter, log)
// linter error will be ignored by sync
So(err, ShouldBeNil)
@ -711,7 +715,7 @@ func TestSyncInternal(t *testing.T) {
manifest.Config.Digest.Algorithm().String(), manifest.Config.Digest.Encoded()))
So(err, ShouldBeNil)
err = pushSyncedLocalImage(repo, "latest", testRootDir, imageStore, log)
err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStore, log)
So(err, ShouldNotBeNil)
err = os.Chmod(configBlobPath, local.DefaultDirPerms)
@ -729,13 +733,45 @@ func TestSyncInternal(t *testing.T) {
err = os.MkdirAll(indexManifestPath, 0o000)
So(err, ShouldBeNil)
err = pushSyncedLocalImage(repo, "latest", testRootDir, imageStore, log)
err = pushSyncedLocalImage(repo, "latest", testRootDir, nil, imageStore, log)
So(err, ShouldNotBeNil)
err = os.Remove(indexManifestPath)
So(err, ShouldBeNil)
})
Convey("RepoDB Errors", func() {
multiArch, err := test.GetRandomMultiarchImage("bad-repodb-tag")
So(err, ShouldBeNil)
err = test.WriteMultiArchImageToFileSystem(multiArch, "repo", storage.StoreController{
DefaultStore: testImageStore,
})
So(err, ShouldBeNil)
// copyManifest Errors
err = pushSyncedLocalImage("repo", "bad-repodb-tag", testRootDir,
mocks.RepoDBMock{
SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error {
return ErrTestError
},
}, imageStore, log)
So(err, ShouldNotBeNil)
// SetMetadataFromInput
err = pushSyncedLocalImage("repo", "bad-repodb-tag", testRootDir,
mocks.RepoDBMock{
SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error {
if Reference == "bad-repodb-tag" {
return ErrTestError
}
return nil
},
}, imageStore, log)
So(err, ShouldNotBeNil)
})
Convey("manifest image errors", func() {
var manifest ispec.Manifest
@ -749,7 +785,7 @@ func TestSyncInternal(t *testing.T) {
if os.Geteuid() != 0 {
So(func() {
_ = pushSyncedLocalImage(testImage, testImageTag, testRootDir, imageStore, log)
_ = pushSyncedLocalImage(testImage, testImageTag, testRootDir, nil, imageStore, log)
}, ShouldPanic)
}
@ -762,7 +798,7 @@ func TestSyncInternal(t *testing.T) {
panic(err)
}
err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, imageStore, log)
err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, nil, imageStore, log)
So(err, ShouldNotBeNil)
if err := os.Chmod(path.Join(testRootDir, testImage, "blobs", "sha256",
@ -776,7 +812,7 @@ func TestSyncInternal(t *testing.T) {
panic(err)
}
err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, imageStore, log)
err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, nil, imageStore, log)
So(err, ShouldNotBeNil)
if err := os.Chmod(cachedManifestConfigPath, 0o755); err != nil {
@ -803,7 +839,7 @@ func TestSyncInternal(t *testing.T) {
panic(err)
}
err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, imageStore, log)
err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, nil, imageStore, log)
So(err, ShouldNotBeNil)
manifest.Config.Digest = configDigestBackup
@ -828,7 +864,7 @@ func TestSyncInternal(t *testing.T) {
panic(err)
}
err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, imageStore, log)
err = pushSyncedLocalImage(testImage, testImageTag, testRootDir, nil, imageStore, log)
So(err, ShouldNotBeNil)
})
})

221
pkg/extensions/sync/sync_test.go
View file

@ -46,6 +46,7 @@ import (
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
)
const (
@ -137,6 +138,12 @@ func startUpstreamServer(
srcConfig.Storage.RootDirectory = srcDir
defVal := true
srcConfig.Extensions = &extconf.ExtensionConfig{}
srcConfig.Extensions.Search = &extconf.SearchConfig{
BaseConfig: extconf.BaseConfig{Enable: &defVal},
}
sctlr := api.NewController(srcConfig)
scm := test.NewControllerManager(sctlr)
@ -193,7 +200,10 @@ func startDownstreamServer(
destConfig.Storage.GC = false
destConfig.Extensions = &extconf.ExtensionConfig{}
destConfig.Extensions.Search = nil
defVal := true
destConfig.Extensions.Search = &extconf.SearchConfig{
BaseConfig: extconf.BaseConfig{Enable: &defVal},
}
destConfig.Extensions.Sync = syncConfig
dctlr := api.NewController(destConfig)
@ -627,6 +637,186 @@ func TestOnDemand(t *testing.T) {
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 500)
})
Convey("Sync on Demand errors", t, func() {
Convey("Signature copier errors", func() {
// start upstream server
rootDir := t.TempDir()
port := test.GetFreePort()
srcBaseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
conf.Storage.GC = false
ctlr := api.NewController(conf)
ctlr.Config.Storage.RootDirectory = rootDir
cm := test.NewControllerManager(ctlr)
cm.StartAndWait(conf.HTTP.Port)
defer cm.StopServer()
imageConfig, layers, manifest, err := test.GetRandomImageComponents(10)
So(err, ShouldBeNil)
manifestBlob, err := json.Marshal(manifest)
So(err, ShouldBeNil)
manifestDigest := godigest.FromBytes(manifestBlob)
err = test.UploadImage(
test.Image{Config: imageConfig, Layers: layers, Manifest: manifest, Reference: "test"},
srcBaseURL,
"remote-repo",
)
So(err, ShouldBeNil)
// sign using cosign
err = test.SignImageUsingCosign(fmt.Sprintf("remote-repo@%s", manifestDigest.String()), port)
So(err, ShouldBeNil)
// add OCI Ref
OCIRefManifest := ispec.Artifact{
Subject: &ispec.Descriptor{
MediaType: ispec.MediaTypeImageManifest,
Digest: manifestDigest,
},
Blobs: []ispec.Descriptor{},
MediaType: ispec.MediaTypeArtifactManifest,
}
OCIRefManifestBlob, err := json.Marshal(OCIRefManifest)
So(err, ShouldBeNil)
resp, err := resty.R().
SetHeader("Content-type", ispec.MediaTypeArtifactManifest).
SetBody(OCIRefManifestBlob).
Put(srcBaseURL + "/v2/remote-repo/manifests/oci.ref")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
// add ORAS Ref
ORASRefManifest := artifactspec.Manifest{
Subject: &artifactspec.Descriptor{
MediaType: ispec.MediaTypeImageManifest,
Digest: manifestDigest,
},
Blobs: []artifactspec.Descriptor{},
MediaType: artifactspec.MediaTypeArtifactManifest,
}
ORASRefManifestBlob, err := json.Marshal(ORASRefManifest)
So(err, ShouldBeNil)
resp, err = resty.R().
SetHeader("Content-type", artifactspec.MediaTypeArtifactManifest).
SetBody(ORASRefManifestBlob).
Put(srcBaseURL + "/v2/remote-repo/manifests/oras.ref")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusCreated)
//------- Start downstream server
var tlsVerify bool
regex := ".*"
semver := true
syncRegistryConfig := syncconf.RegistryConfig{
Content: []syncconf.Content{
{
Prefix: "remote-repo",
Tags: &syncconf.Tags{
Regex: &regex,
Semver: &semver,
},
},
},
URLs: []string{srcBaseURL},
TLSVerify: &tlsVerify,
CertDir: "",
OnDemand: true,
}
defaultVal := true
syncConfig := &syncconf.Config{
Enable: &defaultVal,
Registries: []syncconf.RegistryConfig{syncRegistryConfig},
}
destPort := test.GetFreePort()
destConfig := config.New()
destBaseURL := test.GetBaseURL(destPort)
destConfig.HTTP.Port = destPort
destDir := t.TempDir()
destConfig.Storage.RootDirectory = destDir
destConfig.Storage.Dedupe = false
destConfig.Storage.GC = false
destConfig.Extensions = &extconf.ExtensionConfig{}
defVal := true
destConfig.Extensions.Search = &extconf.SearchConfig{
BaseConfig: extconf.BaseConfig{Enable: &defVal},
}
destConfig.Extensions.Sync = syncConfig
dctlr := api.NewController(destConfig)
dcm := test.NewControllerManager(dctlr)
dcm.StartAndWait(destPort)
// repodb fails for syncOCIRefs
dctlr.RepoDB = mocks.RepoDBMock{
SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error {
if mediaType == ispec.MediaTypeArtifactManifest {
return sync.ErrTestError
}
return nil
},
}
resp, err = resty.R().Get(destBaseURL + "/v2/remote-repo/manifests/test")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
// repodb fails for syncCosignSignature"
dctlr.RepoDB = mocks.RepoDBMock{
SetRepoReferenceFn: func(repo, reference string, manifestDigest godigest.Digest, mediaType string) error {
if strings.HasPrefix(reference, "sha256") || strings.HasSuffix(reference, ".sig") {
return sync.ErrTestError
}
return nil
},
}
resp, err = resty.R().Get(destBaseURL + "/v2/remote-repo/manifests/test")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
// repodb fails for getORASRefs
dctlr.RepoDB = mocks.RepoDBMock{
SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error {
if mediaType == artifactspec.MediaTypeArtifactManifest {
return sync.ErrTestError
}
return nil
},
}
resp, err = resty.R().Get(destBaseURL + "/v2/remote-repo/manifests/test")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
})
})
}
func TestPeriodically(t *testing.T) {
@ -4510,11 +4700,24 @@ func TestSyncWithDestination(t *testing.T) {
},
}
sctlr, srcBaseURL, _, _, _ := startUpstreamServer(t, false, false)
srcPort := test.GetFreePort()
srcConfig := config.New()
srcBaseURL := test.GetBaseURL(srcPort)
defer func() {
sctlr.Shutdown()
}()
srcConfig.HTTP.Port = srcPort
srcDir := t.TempDir()
srcConfig.Storage.RootDirectory = srcDir
defVal := true
srcConfig.Extensions = &extconf.ExtensionConfig{}
srcConfig.Extensions.Search = &extconf.SearchConfig{
BaseConfig: extconf.BaseConfig{Enable: &defVal},
}
sctlr := api.NewController(srcConfig)
test.CopyTestFiles("../../../test/data", srcDir)
err := os.MkdirAll(path.Join(sctlr.Config.Storage.RootDirectory, "/zot-fold"), local.DefaultDirPerms)
So(err, ShouldBeNil)
@ -4524,9 +4727,15 @@ func TestSyncWithDestination(t *testing.T) {
path.Join(sctlr.Config.Storage.RootDirectory, "zot-test"),
path.Join(sctlr.Config.Storage.RootDirectory, "/zot-fold/zot-test"),
)
So(err, ShouldBeNil)
scm := test.NewControllerManager(sctlr)
scm.StartAndWait(srcPort)
defer func() {
sctlr.Shutdown()
}()
Convey("Test peridiocally sync", func() {
for _, testCase := range testCases {
updateDuration, _ := time.ParseDuration("30m")

42
pkg/extensions/sync/utils.go
View file

@ -31,6 +31,7 @@ import (
syncconf "zotregistry.io/zot/pkg/extensions/config/sync"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/test"
@ -264,7 +265,7 @@ func getFileCredentials(filepath string) (syncconf.CredentialsFile, error) {
}
func pushSyncedLocalImage(localRepo, reference, localCachePath string,
imageStore storage.ImageStore, log log.Logger,
repoDB repodb.RepoDB, imageStore storage.ImageStore, log log.Logger,
) error {
log.Info().Msgf("pushing synced local image %s/%s:%s to local registry", localCachePath, localRepo, reference)
@ -275,7 +276,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
cacheImageStore := local.NewImageStore(localCachePath, false,
storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
manifestContent, _, mediaType, err := cacheImageStore.GetImageManifest(localRepo, reference)
manifestBlob, manifestDigest, mediaType, err := cacheImageStore.GetImageManifest(localRepo, reference)
if err != nil {
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Str("dir", path.Join(cacheImageStore.RootDir(), localRepo)).
@ -287,7 +288,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
// is image manifest
switch mediaType {
case ispec.MediaTypeImageManifest:
if err := copyManifest(localRepo, manifestContent, reference, cacheImageStore, imageStore, log); err != nil {
if err := copyManifest(localRepo, manifestBlob, reference, repoDB, cacheImageStore, imageStore, log); err != nil {
if errors.Is(err, zerr.ErrImageLintAnnotations) {
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't upload manifest because of missing annotations")
@ -301,7 +302,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
// is image index
var indexManifest ispec.Index
if err := json.Unmarshal(manifestContent, &indexManifest); err != nil {
if err := json.Unmarshal(manifestBlob, &indexManifest); err != nil {
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Str("dir", path.Join(cacheImageStore.RootDir(), localRepo)).
Msg("invalid JSON")
@ -322,7 +323,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
return err
}
if err := copyManifest(localRepo, manifestBuf, manifest.Digest.String(),
if err := copyManifest(localRepo, manifestBuf, manifest.Digest.String(), repoDB,
cacheImageStore, imageStore, log); err != nil {
if errors.Is(err, zerr.ErrImageLintAnnotations) {
log.Error().Str("errorType", common.TypeOf(err)).
@ -335,19 +336,29 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
}
}
_, err = imageStore.PutImageManifest(localRepo, reference, mediaType, manifestContent)
_, err = imageStore.PutImageManifest(localRepo, reference, mediaType, manifestBlob)
if err != nil {
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't upload manifest")
return err
}
if repoDB != nil {
err = repodb.SetMetadataFromInput(localRepo, reference, mediaType,
manifestDigest, manifestBlob, imageStore, repoDB, log)
if err != nil {
return fmt.Errorf("failed to set metadata for image '%s %s': %w", localRepo, reference, err)
}
log.Debug().Msgf("successfully set metadata for %s:%s", localRepo, reference)
}
}
return nil
}
func copyManifest(localRepo string, manifestContent []byte, reference string,
func copyManifest(localRepo string, manifestContent []byte, reference string, repoDB repodb.RepoDB,
cacheImageStore, imageStore storage.ImageStore, log log.Logger,
) error {
var manifest ispec.Manifest
@ -376,7 +387,7 @@ func copyManifest(localRepo string, manifestContent []byte, reference string,
return err
}
_, err = imageStore.PutImageManifest(localRepo, reference,
digest, err := imageStore.PutImageManifest(localRepo, reference,
ispec.MediaTypeImageManifest, manifestContent)
if err != nil {
log.Error().Str("errorType", common.TypeOf(err)).
@ -385,6 +396,19 @@ func copyManifest(localRepo string, manifestContent []byte, reference string,
return err
}
if repoDB != nil {
err = repodb.SetMetadataFromInput(localRepo, reference, ispec.MediaTypeImageManifest,
digest, manifestContent, imageStore, repoDB, log)
if err != nil {
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't set metadata from input")
return err
}
log.Debug().Msgf("successfully set metadata for %s:%s", localRepo, reference)
}
return nil
}
@ -720,7 +744,7 @@ func syncImageWithRefs(ctx context.Context, localRepo, upstreamRepo, reference s
}
// push from cache to repo
err = pushSyncedLocalImage(localRepo, reference, localCachePath, imageStore, log)
err = pushSyncedLocalImage(localRepo, reference, localCachePath, sig.repoDB, imageStore, log)
if err != nil {
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("error while pushing synced cached image %s",

77
pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go
View file

@ -79,7 +79,7 @@ func NewBoltDBWrapper(params DBParameters) (*DBWrapper, error) {
}, nil
}
func (bdw DBWrapper) SetManifestData(manifestDigest godigest.Digest, manifestData repodb.ManifestData) error {
func (bdw *DBWrapper) SetManifestData(manifestDigest godigest.Digest, manifestData repodb.ManifestData) error {
err := bdw.DB.Update(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.ManifestDataBucket))
@ -99,7 +99,7 @@ func (bdw DBWrapper) SetManifestData(manifestDigest godigest.Digest, manifestDat
return err
}
func (bdw DBWrapper) GetManifestData(manifestDigest godigest.Digest) (repodb.ManifestData, error) {
func (bdw *DBWrapper) GetManifestData(manifestDigest godigest.Digest) (repodb.ManifestData, error) {
var manifestData repodb.ManifestData
err := bdw.DB.View(func(tx *bolt.Tx) error {
@ -122,7 +122,7 @@ func (bdw DBWrapper) GetManifestData(manifestDigest godigest.Digest) (repodb.Man
return manifestData, err
}
func (bdw DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Digest, manifestMeta repodb.ManifestMetadata,
func (bdw *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Digest, manifestMeta repodb.ManifestMetadata,
) error {
err := bdw.DB.Update(func(tx *bolt.Tx) error {
dataBuck := tx.Bucket([]byte(repodb.ManifestDataBucket))
@ -169,7 +169,7 @@ func (bdw DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Digest
return err
}
func (bdw DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Digest) (repodb.ManifestMetadata, error) {
func (bdw *DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Digest) (repodb.ManifestMetadata, error) {
var manifestMetadata repodb.ManifestMetadata
err := bdw.DB.View(func(tx *bolt.Tx) error {
@ -214,7 +214,7 @@ func (bdw DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Digest
return manifestMetadata, err
}
func (bdw DBWrapper) SetIndexData(indexDigest godigest.Digest, indexMetadata repodb.IndexData) error {
func (bdw *DBWrapper) SetIndexData(indexDigest godigest.Digest, indexMetadata repodb.IndexData) error {
// we make the assumption that the oci layout is consistent and all manifests refferenced inside the
// index are present
err := bdw.DB.Update(func(tx *bolt.Tx) error {
@ -236,7 +236,7 @@ func (bdw DBWrapper) SetIndexData(indexDigest godigest.Digest, indexMetadata rep
return err
}
func (bdw DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData, error) {
func (bdw *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData, error) {
var indexMetadata repodb.IndexData
err := bdw.DB.View(func(tx *bolt.Tx) error {
@ -259,10 +259,10 @@ func (bdw DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData
return indexMetadata, err
}
func (bdw DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest.Digest,
func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest,
mediaType string,
) error {
if err := common.ValidateRepoTagInput(repo, tag, manifestDigest); err != nil {
if err := common.ValidateRepoReferenceInput(repo, reference, manifestDigest); err != nil {
return err
}
@ -273,29 +273,17 @@ func (bdw DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest
// object not found
if len(repoMetaBlob) == 0 {
// create a new object
repoMeta := repodb.RepoMetadata{
Name: repo,
Tags: map[string]repodb.Descriptor{
tag: {
Digest: manifestDigest.String(),
MediaType: mediaType,
},
},
Statistics: map[string]repodb.DescriptorStatistics{
manifestDigest.String(): {DownloadCount: 0},
},
Signatures: map[string]repodb.ManifestSignatures{
manifestDigest.String(): {},
},
}
var err error
repoMetaBlob, err := json.Marshal(repoMeta)
repoMetaBlob, err = json.Marshal(repodb.RepoMetadata{
Name: repo,
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{},
Signatures: map[string]repodb.ManifestSignatures{},
})
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
}
// object found
@ -306,11 +294,16 @@ func (bdw DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest
return err
}
repoMeta.Tags[tag] = repodb.Descriptor{
Digest: manifestDigest.String(),
MediaType: mediaType,
if !common.ReferenceIsDigest(reference) {
repoMeta.Tags[reference] = repodb.Descriptor{
Digest: manifestDigest.String(),
MediaType: mediaType,
}
}
repoMeta.Statistics[manifestDigest.String()] = repodb.DescriptorStatistics{DownloadCount: 0}
repoMeta.Signatures[manifestDigest.String()] = repodb.ManifestSignatures{}
repoMetaBlob, err = json.Marshal(repoMeta)
if err != nil {
return err
@ -322,7 +315,7 @@ func (bdw DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest
return err
}
func (bdw DBWrapper) GetRepoMeta(repo string) (repodb.RepoMetadata, error) {
func (bdw *DBWrapper) GetRepoMeta(repo string) (repodb.RepoMetadata, error) {
var repoMeta repodb.RepoMetadata
err := bdw.DB.Update(func(tx *bolt.Tx) error {
@ -347,7 +340,7 @@ func (bdw DBWrapper) GetRepoMeta(repo string) (repodb.RepoMetadata, error) {
return repoMeta, err
}
func (bdw DBWrapper) DeleteRepoTag(repo string, tag string) error {
func (bdw *DBWrapper) DeleteRepoTag(repo string, tag string) error {
err := bdw.DB.Update(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
@ -383,7 +376,7 @@ func (bdw DBWrapper) DeleteRepoTag(repo string, tag string) error {
return err
}
func (bdw DBWrapper) IncrementRepoStars(repo string) error {
func (bdw *DBWrapper) IncrementRepoStars(repo string) error {
err := bdw.DB.Update(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
@ -412,7 +405,7 @@ func (bdw DBWrapper) IncrementRepoStars(repo string) error {
return err
}
func (bdw DBWrapper) DecrementRepoStars(repo string) error {
func (bdw *DBWrapper) DecrementRepoStars(repo string) error {
err := bdw.DB.Update(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
@ -443,7 +436,7 @@ func (bdw DBWrapper) DecrementRepoStars(repo string) error {
return err
}
func (bdw DBWrapper) GetRepoStars(repo string) (int, error) {
func (bdw *DBWrapper) GetRepoStars(repo string) (int, error) {
stars := 0
err := bdw.DB.View(func(tx *bolt.Tx) error {
@ -470,7 +463,7 @@ func (bdw DBWrapper) GetRepoStars(repo string) (int, error) {
return stars, err
}
func (bdw DBWrapper) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta repodb.RepoMetadata) bool,
func (bdw *DBWrapper) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta repodb.RepoMetadata) bool,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, error) {
var (
@ -515,7 +508,7 @@ func (bdw DBWrapper) GetMultipleRepoMeta(ctx context.Context, filter func(repoMe
return foundRepos, err
}
func (bdw DBWrapper) IncrementImageDownloads(repo string, reference string) error {
func (bdw *DBWrapper) IncrementImageDownloads(repo string, reference string) error {
err := bdw.DB.Update(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
@ -559,7 +552,7 @@ func (bdw DBWrapper) IncrementImageDownloads(repo string, reference string) erro
return err
}
func (bdw DBWrapper) AddManifestSignature(repo string, signedManifestDigest godigest.Digest,
func (bdw *DBWrapper) AddManifestSignature(repo string, signedManifestDigest godigest.Digest,
sygMeta repodb.SignatureMetadata,
) error {
err := bdw.DB.Update(func(tx *bolt.Tx) error {
@ -616,7 +609,7 @@ func (bdw DBWrapper) AddManifestSignature(repo string, signedManifestDigest godi
return err
}
func (bdw DBWrapper) DeleteSignature(repo string, signedManifestDigest godigest.Digest,
func (bdw *DBWrapper) DeleteSignature(repo string, signedManifestDigest godigest.Digest,
sigMeta repodb.SignatureMetadata,
) error {
err := bdw.DB.Update(func(tx *bolt.Tx) error {
@ -670,7 +663,7 @@ func (bdw DBWrapper) DeleteSignature(repo string, signedManifestDigest godigest.
return err
}
func (bdw DBWrapper) SearchRepos(ctx context.Context, searchText string, filter repodb.Filter,
func (bdw *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo,
error,
@ -1020,7 +1013,7 @@ func NewManifestMetadata(manifestDigest string, repoMeta repodb.RepoMetadata,
return manifestMeta
}
func (bdw DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
func (bdw *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData,
repodb.PageInfo, error,
@ -1179,7 +1172,7 @@ func (bdw DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter,
func (bdw *DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
var (

18
pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go
View file

@ -70,7 +70,7 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("SetRepoTag", func() {
Convey("SetRepoReference", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
@ -78,7 +78,7 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetRepoTag("repo1", "tag", "digest", ispec.MediaTypeImageManifest)
err = boltdbWrapper.SetRepoReference("repo1", "tag", "digest", ispec.MediaTypeImageManifest)
So(err, ShouldNotBeNil)
})
@ -388,7 +388,7 @@ func TestWrapperErrors(t *testing.T) {
Convey("Bad index data", func() {
indexDigest := digest.FromString("indexDigest")
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadIndexData(boltdbWrapper.DB, indexDigest.String())
@ -404,7 +404,7 @@ func TestWrapperErrors(t *testing.T) {
Convey("Bad indexBlob in IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = boltdbWrapper.SetIndexData(indexDigest, repodb.IndexData{
@ -426,7 +426,7 @@ func TestWrapperErrors(t *testing.T) {
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
@ -557,7 +557,7 @@ func TestWrapperErrors(t *testing.T) {
Convey("FilterTags bad IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadIndexData(boltdbWrapper.DB, indexDigest.String())
@ -573,7 +573,7 @@ func TestWrapperErrors(t *testing.T) {
Convey("FilterTags bad indexBlob in IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = boltdbWrapper.SetIndexData(indexDigest, repodb.IndexData{
@ -595,7 +595,7 @@ func TestWrapperErrors(t *testing.T) {
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := boltdbWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
@ -631,7 +631,7 @@ func TestWrapperErrors(t *testing.T) {
Convey("Unsuported type", func() {
digest := digest.FromString("digest")
err := boltdbWrapper.SetRepoTag("repo", "tag1", digest, "invalid type") //nolint:contextcheck
err := boltdbWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})

4
pkg/meta/repodb/common/common.go
View file

@ -45,12 +45,12 @@ func ReferenceIsDigest(reference string) bool {
return err == nil
}
func ValidateRepoTagInput(repo, tag string, manifestDigest godigest.Digest) error {
func ValidateRepoReferenceInput(repo, reference string, manifestDigest godigest.Digest) error {
if repo == "" {
return zerr.ErrEmptyRepoName
}
if tag == "" {
if reference == "" {
return zerr.ErrEmptyTag
}

49
pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go
View file

@ -55,13 +55,13 @@ func TestIterator(t *testing.T) {
So(dynamoWrapper.ResetManifestDataTable(), ShouldBeNil)
So(dynamoWrapper.ResetRepoMetaTable(), ShouldBeNil)
err = dynamoWrapper.SetRepoTag("repo1", "tag1", "manifestType", "manifestDigest1")
err = dynamoWrapper.SetRepoReference("repo1", "tag1", "manifestType", "manifestDigest1")
So(err, ShouldBeNil)
err = dynamoWrapper.SetRepoTag("repo2", "tag2", "manifestType", "manifestDigest2")
err = dynamoWrapper.SetRepoReference("repo2", "tag2", "manifestType", "manifestDigest2")
So(err, ShouldBeNil)
err = dynamoWrapper.SetRepoTag("repo3", "tag3", "manifestType", "manifestDigest3")
err = dynamoWrapper.SetRepoReference("repo3", "tag3", "manifestType", "manifestDigest3")
So(err, ShouldBeNil)
repoMetaAttributeIterator := iterator.NewBaseDynamoAttributesIterator(
@ -196,7 +196,7 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("GetManifestMeta GetManifestData not found error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag", "dig", "")
err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "")
So(err, ShouldBeNil)
_, err = dynamoWrapper.GetManifestMeta("repo", "dig")
@ -270,7 +270,7 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("IncrementImageDownloads tag not found error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag", "dig", "")
err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "")
So(err, ShouldBeNil)
err = dynamoWrapper.IncrementImageDownloads("repo", "notFoundTag")
@ -278,7 +278,7 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("AddManifestSignature GetRepoMeta error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag", "dig", "")
err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "")
So(err, ShouldBeNil)
err = dynamoWrapper.AddManifestSignature("repoNotFound", "tag", repodb.SignatureMetadata{})
@ -286,7 +286,7 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("AddManifestSignature ManifestSignatures signedManifestDigest not found error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag", "dig", "")
err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "")
So(err, ShouldBeNil)
err = dynamoWrapper.AddManifestSignature("repo", "tagNotFound", repodb.SignatureMetadata{})
@ -294,7 +294,7 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("AddManifestSignature SignatureType repodb.NotationType", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag", "dig", "")
err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "")
So(err, ShouldBeNil)
err = dynamoWrapper.AddManifestSignature("repo", "tagNotFound", repodb.SignatureMetadata{
@ -349,7 +349,8 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("SearchRepos GetManifestMeta error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "notFoundDigest", ispec.MediaTypeImageManifest) //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", "notFoundDigest", //nolint:contextcheck
ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
@ -358,7 +359,7 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("SearchRepos config unmarshal error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData("dig1", repodb.ManifestData{ //nolint:contextcheck
@ -375,7 +376,7 @@ func TestWrapperErrors(t *testing.T) {
Convey("Unsuported type", func() {
digest := digest.FromString("digest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", digest, "invalid type") //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", repodb.Filter{}, repodb.PageInput{})
@ -395,7 +396,7 @@ func TestWrapperErrors(t *testing.T) {
Convey("SearchRepos bad index data", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck
@ -408,7 +409,7 @@ func TestWrapperErrors(t *testing.T) {
Convey("SearchRepos bad indexBlob in IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck
@ -427,7 +428,7 @@ func TestWrapperErrors(t *testing.T) {
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
@ -466,7 +467,7 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("SearchTags GetManifestMeta error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "manifestNotFound", //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", "manifestNotFound", //nolint:contextcheck
ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
@ -476,7 +477,7 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("SearchTags config unmarshal error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData( //nolint:contextcheck
@ -496,7 +497,7 @@ func TestWrapperErrors(t *testing.T) {
Convey("SearchTags bad index data", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck
@ -509,7 +510,7 @@ func TestWrapperErrors(t *testing.T) {
Convey("SearchTags bad indexBlob in IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck
@ -528,7 +529,7 @@ func TestWrapperErrors(t *testing.T) {
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
@ -573,7 +574,7 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("FilterTags manifestMeta not found", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "manifestNotFound", //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", "manifestNotFound", //nolint:contextcheck
ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
@ -589,7 +590,7 @@ func TestWrapperErrors(t *testing.T) {
})
Convey("FilterTags manifestMeta unmarshal error", func() {
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig", ispec.MediaTypeImageManifest) //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", "dig", ispec.MediaTypeImageManifest) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadManifestData(dynamoWrapper.Client, manifestDataTablename, "dig") //nolint:contextcheck
@ -609,7 +610,7 @@ func TestWrapperErrors(t *testing.T) {
Convey("FilterTags bad IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck
@ -625,7 +626,7 @@ func TestWrapperErrors(t *testing.T) {
Convey("FilterTags bad indexBlob in IndexData", func() {
indexDigest := digest.FromString("indexDigest")
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, repodb.IndexData{ //nolint:contextcheck
@ -647,7 +648,7 @@ func TestWrapperErrors(t *testing.T) {
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := dynamoWrapper.SetRepoTag("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{

79
pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go
View file

@ -91,7 +91,7 @@ func NewDynamoDBWrapper(params dynamoParams.DBDriverParameters) (*DBWrapper, err
return &dynamoWrapper, nil
}
func (dwr DBWrapper) SetManifestData(manifestDigest godigest.Digest, manifestData repodb.ManifestData) error {
func (dwr *DBWrapper) SetManifestData(manifestDigest godigest.Digest, manifestData repodb.ManifestData) error {
mdAttributeValue, err := attributevalue.Marshal(manifestData)
if err != nil {
return err
@ -116,7 +116,7 @@ func (dwr DBWrapper) SetManifestData(manifestDigest godigest.Digest, manifestDat
return err
}
func (dwr DBWrapper) GetManifestData(manifestDigest godigest.Digest) (repodb.ManifestData, error) {
func (dwr *DBWrapper) GetManifestData(manifestDigest godigest.Digest) (repodb.ManifestData, error) {
resp, err := dwr.Client.GetItem(context.Background(), &dynamodb.GetItemInput{
TableName: aws.String(dwr.ManifestDataTablename),
Key: map[string]types.AttributeValue{
@ -141,7 +141,7 @@ func (dwr DBWrapper) GetManifestData(manifestDigest godigest.Digest) (repodb.Man
return manifestData, nil
}
func (dwr DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Digest, manifestMeta repodb.ManifestMetadata,
func (dwr *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Digest, manifestMeta repodb.ManifestMetadata,
) error {
if manifestMeta.Signatures == nil {
manifestMeta.Signatures = repodb.ManifestSignatures{}
@ -179,7 +179,7 @@ func (dwr DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Digest
return err
}
func (dwr DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Digest,
func (dwr *DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Digest,
) (repodb.ManifestMetadata, error) { //nolint:contextcheck
manifestData, err := dwr.GetManifestData(manifestDigest)
if err != nil {
@ -218,7 +218,7 @@ func (dwr DBWrapper) GetManifestMeta(repo string, manifestDigest godigest.Digest
return manifestMetadata, nil
}
func (dwr DBWrapper) IncrementRepoStars(repo string) error {
func (dwr *DBWrapper) IncrementRepoStars(repo string) error {
repoMeta, err := dwr.GetRepoMeta(repo)
if err != nil {
return err
@ -231,7 +231,7 @@ func (dwr DBWrapper) IncrementRepoStars(repo string) error {
return err
}
func (dwr DBWrapper) DecrementRepoStars(repo string) error {
func (dwr *DBWrapper) DecrementRepoStars(repo string) error {
repoMeta, err := dwr.GetRepoMeta(repo)
if err != nil {
return err
@ -246,7 +246,7 @@ func (dwr DBWrapper) DecrementRepoStars(repo string) error {
return err
}
func (dwr DBWrapper) GetRepoStars(repo string) (int, error) {
func (dwr *DBWrapper) GetRepoStars(repo string) (int, error) {
repoMeta, err := dwr.GetRepoMeta(repo)
if err != nil {
return 0, err
@ -255,7 +255,7 @@ func (dwr DBWrapper) GetRepoStars(repo string) (int, error) {
return repoMeta.Stars, nil
}
func (dwr DBWrapper) SetIndexData(indexDigest godigest.Digest, indexData repodb.IndexData) error {
func (dwr *DBWrapper) SetIndexData(indexDigest godigest.Digest, indexData repodb.IndexData) error {
indexAttributeValue, err := attributevalue.Marshal(indexData)
if err != nil {
return err
@ -280,7 +280,7 @@ func (dwr DBWrapper) SetIndexData(indexDigest godigest.Digest, indexData repodb.
return err
}
func (dwr DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData, error) {
func (dwr *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData, error) {
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
TableName: aws.String(dwr.IndexDataTablename),
Key: map[string]types.AttributeValue{
@ -307,8 +307,10 @@ func (dwr DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexData
return indexData, nil
}
func (dwr DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest.Digest, mediaType string) error {
if err := common.ValidateRepoTagInput(repo, tag, manifestDigest); err != nil {
func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest,
mediaType string,
) error {
if err := common.ValidateRepoReferenceInput(repo, reference, manifestDigest); err != nil {
return err
}
@ -336,17 +338,22 @@ func (dwr DBWrapper) SetRepoTag(repo string, tag string, manifestDigest godigest
}
}
repoMeta.Tags[tag] = repodb.Descriptor{
Digest: manifestDigest.String(),
MediaType: mediaType,
if !common.ReferenceIsDigest(reference) {
repoMeta.Tags[reference] = repodb.Descriptor{
Digest: manifestDigest.String(),
MediaType: mediaType,
}
}
repoMeta.Statistics[manifestDigest.String()] = repodb.DescriptorStatistics{DownloadCount: 0}
repoMeta.Signatures[manifestDigest.String()] = repodb.ManifestSignatures{}
err = dwr.setRepoMeta(repo, repoMeta)
return err
}
func (dwr DBWrapper) DeleteRepoTag(repo string, tag string) error {
func (dwr *DBWrapper) DeleteRepoTag(repo string, tag string) error {
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
TableName: aws.String(dwr.RepoMetaTablename),
Key: map[string]types.AttributeValue{
@ -405,7 +412,7 @@ func (dwr DBWrapper) DeleteRepoTag(repo string, tag string) error {
return err
}
func (dwr DBWrapper) GetRepoMeta(repo string) (repodb.RepoMetadata, error) {
func (dwr *DBWrapper) GetRepoMeta(repo string) (repodb.RepoMetadata, error) {
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
TableName: aws.String(dwr.RepoMetaTablename),
Key: map[string]types.AttributeValue{
@ -430,7 +437,7 @@ func (dwr DBWrapper) GetRepoMeta(repo string) (repodb.RepoMetadata, error) {
return repoMeta, nil
}
func (dwr DBWrapper) IncrementImageDownloads(repo string, reference string) error {
func (dwr *DBWrapper) IncrementImageDownloads(repo string, reference string) error {
repoMeta, err := dwr.GetRepoMeta(repo)
if err != nil {
return err
@ -456,7 +463,7 @@ func (dwr DBWrapper) IncrementImageDownloads(repo string, reference string) erro
return dwr.setRepoMeta(repo, repoMeta)
}
func (dwr DBWrapper) AddManifestSignature(repo string, signedManifestDigest godigest.Digest,
func (dwr *DBWrapper) AddManifestSignature(repo string, signedManifestDigest godigest.Digest,
sygMeta repodb.SignatureMetadata,
) error {
repoMeta, err := dwr.GetRepoMeta(repo)
@ -497,7 +504,7 @@ func (dwr DBWrapper) AddManifestSignature(repo string, signedManifestDigest godi
return err
}
func (dwr DBWrapper) DeleteSignature(repo string, signedManifestDigest godigest.Digest,
func (dwr *DBWrapper) DeleteSignature(repo string, signedManifestDigest godigest.Digest,
sigMeta repodb.SignatureMetadata,
) error {
repoMeta, err := dwr.GetRepoMeta(repo)
@ -535,7 +542,7 @@ func (dwr DBWrapper) DeleteSignature(repo string, signedManifestDigest godigest.
return err
}
func (dwr DBWrapper) GetMultipleRepoMeta(ctx context.Context,
func (dwr *DBWrapper) GetMultipleRepoMeta(ctx context.Context,
filter func(repoMeta repodb.RepoMetadata) bool, requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, error) {
var (
@ -583,7 +590,7 @@ func (dwr DBWrapper) GetMultipleRepoMeta(ctx context.Context,
return foundRepos, err
}
func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter repodb.Filter,
func (dwr *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
var (
@ -743,7 +750,7 @@ func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func (dwr DBWrapper) fetchManifestMetaWithCheck(repoName string, manifestDigest string,
func (dwr *DBWrapper) fetchManifestMetaWithCheck(repoName string, manifestDigest string,
manifestMetadataMap map[string]repodb.ManifestMetadata,
) (repodb.ManifestMetadata, error) {
var (
@ -796,7 +803,7 @@ func collectImageManifestFilterData(digest string, repoMeta repodb.RepoMetadata,
}, nil
}
func (dwr DBWrapper) fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]repodb.IndexData,
func (dwr *DBWrapper) fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]repodb.IndexData,
) (repodb.IndexData, error) {
var (
indexData repodb.IndexData
@ -816,7 +823,7 @@ func (dwr DBWrapper) fetchIndexDataWithCheck(indexDigest string, indexDataMap ma
return indexData, err
}
func (dwr DBWrapper) collectImageIndexFilterInfo(indexDigest string, repoMeta repodb.RepoMetadata,
func (dwr *DBWrapper) collectImageIndexFilterInfo(indexDigest string, repoMeta repodb.RepoMetadata,
indexData repodb.IndexData, manifestMetadataMap map[string]repodb.ManifestMetadata,
) (repodb.FilterData, error) {
var indexContent ispec.Index
@ -871,7 +878,7 @@ func (dwr DBWrapper) collectImageIndexFilterInfo(indexDigest string, repoMeta re
}, nil
}
func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
func (dwr *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
var (
@ -1009,7 +1016,7 @@ func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter,
func (dwr *DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
var (
@ -1244,7 +1251,7 @@ func (dwr *DBWrapper) PatchDB() error {
return nil
}
func (dwr DBWrapper) setRepoMeta(repo string, repoMeta repodb.RepoMetadata) error {
func (dwr *DBWrapper) setRepoMeta(repo string, repoMeta repodb.RepoMetadata) error {
repoAttributeValue, err := attributevalue.Marshal(repoMeta)
if err != nil {
return err
@ -1269,7 +1276,7 @@ func (dwr DBWrapper) setRepoMeta(repo string, repoMeta repodb.RepoMetadata) erro
return err
}
func (dwr DBWrapper) createRepoMetaTable() error {
func (dwr *DBWrapper) createRepoMetaTable() error {
_, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{
TableName: aws.String(dwr.RepoMetaTablename),
AttributeDefinitions: []types.AttributeDefinition{
@ -1294,7 +1301,7 @@ func (dwr DBWrapper) createRepoMetaTable() error {
return dwr.waitTableToBeCreated(dwr.RepoMetaTablename)
}
func (dwr DBWrapper) deleteRepoMetaTable() error {
func (dwr *DBWrapper) deleteRepoMetaTable() error {
_, err := dwr.Client.DeleteTable(context.Background(), &dynamodb.DeleteTableInput{
TableName: aws.String(dwr.RepoMetaTablename),
})
@ -1306,7 +1313,7 @@ func (dwr DBWrapper) deleteRepoMetaTable() error {
return dwr.waitTableToBeDeleted(dwr.RepoMetaTablename)
}
func (dwr DBWrapper) ResetRepoMetaTable() error {
func (dwr *DBWrapper) ResetRepoMetaTable() error {
err := dwr.deleteRepoMetaTable()
if err != nil {
return err
@ -1315,7 +1322,7 @@ func (dwr DBWrapper) ResetRepoMetaTable() error {
return dwr.createRepoMetaTable()
}
func (dwr DBWrapper) waitTableToBeCreated(tableName string) error {
func (dwr *DBWrapper) waitTableToBeCreated(tableName string) error {
const maxWaitTime = 20 * time.Second
waiter := dynamodb.NewTableExistsWaiter(dwr.Client)
@ -1325,7 +1332,7 @@ func (dwr DBWrapper) waitTableToBeCreated(tableName string) error {
}, maxWaitTime)
}
func (dwr DBWrapper) waitTableToBeDeleted(tableName string) error {
func (dwr *DBWrapper) waitTableToBeDeleted(tableName string) error {
const maxWaitTime = 20 * time.Second
waiter := dynamodb.NewTableNotExistsWaiter(dwr.Client)
@ -1335,7 +1342,7 @@ func (dwr DBWrapper) waitTableToBeDeleted(tableName string) error {
}, maxWaitTime)
}
func (dwr DBWrapper) createManifestDataTable() error {
func (dwr *DBWrapper) createManifestDataTable() error {
_, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{
TableName: aws.String(dwr.ManifestDataTablename),
AttributeDefinitions: []types.AttributeDefinition{
@ -1360,7 +1367,7 @@ func (dwr DBWrapper) createManifestDataTable() error {
return dwr.waitTableToBeCreated(dwr.ManifestDataTablename)
}
func (dwr DBWrapper) createIndexDataTable() error {
func (dwr *DBWrapper) createIndexDataTable() error {
_, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{
TableName: aws.String(dwr.IndexDataTablename),
AttributeDefinitions: []types.AttributeDefinition{
@ -1470,7 +1477,7 @@ func (dwr *DBWrapper) getDBVersion() (string, error) {
return version, nil
}
func (dwr DBWrapper) deleteManifestDataTable() error {
func (dwr *DBWrapper) deleteManifestDataTable() error {
_, err := dwr.Client.DeleteTable(context.Background(), &dynamodb.DeleteTableInput{
TableName: aws.String(dwr.ManifestDataTablename),
})
@ -1482,7 +1489,7 @@ func (dwr DBWrapper) deleteManifestDataTable() error {
return dwr.waitTableToBeDeleted(dwr.ManifestDataTablename)
}
func (dwr DBWrapper) ResetManifestDataTable() error {
func (dwr *DBWrapper) ResetManifestDataTable() error {
err := dwr.deleteManifestDataTable()
if err != nil {
return err

4
pkg/meta/repodb/repodb.go
View file

@ -35,8 +35,8 @@ type RepoDB interface { //nolint:interfacebloat
// GetRepoStars returns the total number of stars a repo has
GetRepoStars(repo string) (int, error)
// SetRepoTag sets the tag of a manifest in the tag list of a repo
SetRepoTag(repo string, tag string, manifestDigest godigest.Digest, mediaType string) error
// SetRepoReference sets the reference of a manifest in the tag list of a repo
SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, mediaType string) error
// DeleteRepoTag delets the tag from the tag list of a repo
DeleteRepoTag(repo string, tag string) error

177
pkg/meta/repodb/repodb_test.go
View file

@ -176,7 +176,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
})
})
Convey("Test SetRepoTag", func() {
Convey("Test SetRepoReference", func() {
// test behaviours
var (
repo1 = "repo1"
@ -189,18 +189,33 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
)
Convey("Setting a good repo", func() {
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
repoMeta, err := repoDB.GetRepoMeta(repo1)
So(err, ShouldBeNil)
So(repoMeta.Name, ShouldResemble, repo1)
So(repoMeta.Tags[tag1].Digest, ShouldEqual, manifestDigest1)
})
Convey("Set multiple tags for repo", func() {
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
Convey("Setting a good repo using a digest", func() {
_, err := repoDB.GetRepoMeta(repo1)
So(err, ShouldNotBeNil)
digest := godigest.FromString("digest")
err = repoDB.SetRepoReference(repo1, digest.String(), digest,
ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
repoMeta, err := repoDB.GetRepoMeta(repo1)
So(err, ShouldBeNil)
So(repoMeta.Name, ShouldResemble, repo1)
})
Convey("Set multiple tags for repo", func() {
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
repoMeta, err := repoDB.GetRepoMeta(repo1)
@ -210,9 +225,9 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
})
Convey("Set multiple repos", func() {
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
repoMeta1, err := repoDB.GetRepoMeta(repo1)
@ -226,17 +241,17 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
Convey("Setting a repo with invalid fields", func() {
Convey("Repo name is not valid", func() {
err := repoDB.SetRepoTag("", tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference("", tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldNotBeNil)
})
Convey("Tag is not valid", func() {
err := repoDB.SetRepoTag(repo1, "", manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, "", manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldNotBeNil)
})
Convey("Manifest Digest is not valid", func() {
err := repoDB.SetRepoTag(repo1, tag1, "", ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, "", ispec.MediaTypeImageManifest)
So(err, ShouldNotBeNil)
})
})
@ -255,10 +270,10 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
InexistentRepo = "InexistentRepo"
)
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
Convey("Get a existent repo", func() {
@ -287,10 +302,10 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
manifestDigest2 = godigest.FromString("fake-manifest2")
)
err := repoDB.SetRepoTag(repo, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
Convey("Delete from repo a tag", func() {
@ -352,13 +367,13 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
manifestDigest2 = godigest.FromString("fake-manifest2")
)
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
Convey("Get all Repometa", func() {
@ -407,7 +422,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
manifestDigest1 = godigest.FromString("fake-manifest1")
)
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.IncrementRepoStars(repo1)
@ -439,7 +454,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
manifestDigest1 = godigest.FromString("fake-manifest1")
)
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.IncrementRepoStars(repo1)
@ -474,7 +489,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
manifestDigest1 = godigest.FromString("fake-manifest1")
)
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.IncrementRepoStars(repo1)
@ -508,7 +523,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
manifestDigest := godigest.FromBytes(manifestBlob)
err = repoDB.SetRepoTag(repo1, tag1, manifestDigest, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, tag1, manifestDigest, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetManifestMeta(repo1, manifestDigest, repodb.ManifestMetadata{
@ -544,7 +559,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
manifestDigest1 = godigest.FromString("fake-manifest1")
)
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetManifestMeta(repo1, manifestDigest1, repodb.ManifestMetadata{})
@ -572,7 +587,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
manifestDigest1 = godigest.FromString("fake-manifest1")
)
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetManifestMeta(repo1, manifestDigest1, repodb.ManifestMetadata{})
@ -633,11 +648,11 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
}
Convey("Search all repos", func() {
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo2, tag3, manifestDigest3, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo2, tag3, manifestDigest3, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta)
@ -657,7 +672,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
})
Convey("Search a repo by name", func() {
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta)
@ -671,10 +686,10 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
})
Convey("Search non-existing repo by name", func() {
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
repos, manifestMetaMap, _, _, err := repoDB.SearchRepos(ctx, "RepoThatDoesntExist", repodb.Filter{},
@ -685,11 +700,11 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
})
Convey("Search with partial match", func() {
err := repoDB.SetRepoTag("alpine", tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference("alpine", tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("pine", tag2, manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("pine", tag2, manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("golang", tag3, manifestDigest3, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("golang", tag3, manifestDigest3, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetManifestMeta("alpine", manifestDigest1, emptyRepoMeta)
@ -708,11 +723,11 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
})
Convey("Search multiple repos that share manifests", func() {
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta)
@ -729,11 +744,11 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
})
Convey("Search repos with access control", func() {
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta)
@ -789,7 +804,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
}
repoName := "repo" + strconv.Itoa(i)
err = repoDB.SetRepoTag(repoName, tag1, manifestDigest, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repoName, tag1, manifestDigest, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetManifestMeta(repoName, manifestDigest, manifestMeta)
@ -972,10 +987,10 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
})
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo", tag4, indexDigest, ispec.MediaTypeImageIndex)
err = repoDB.SetRepoReference("repo", tag4, indexDigest, ispec.MediaTypeImageIndex)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo", tag5, manifestDigest3, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo", tag5, manifestDigest3, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
repos, manifestMetaMap, indexDataMap, _, err := repoDB.SearchRepos(ctx, "repo", repodb.Filter{}, repodb.PageInput{})
@ -1015,17 +1030,17 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
ConfigBlob: emptyConfigBlob,
}
err = repoDB.SetRepoTag(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta)
@ -1093,11 +1108,11 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
tag3 = "0.0.3"
)
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
config := ispec.Image{}
@ -1199,13 +1214,13 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
})
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo", tag4, indexDigest, ispec.MediaTypeImageIndex)
err = repoDB.SetRepoReference("repo", tag4, indexDigest, ispec.MediaTypeImageIndex)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo", tag5, manifestDigest3, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo", tag5, manifestDigest3, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag("repo", tag6, manifestDigest4, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference("repo", tag6, manifestDigest4, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
repos, manifestMetaMap, indexDataMap, _, err := repoDB.SearchTags(ctx, "repo:0.0", repodb.Filter{},
@ -1236,15 +1251,15 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
tag5 = "0.0.5"
)
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, tag2, manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, tag2, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, tag3, manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, tag3, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, tag4, manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, tag4, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, tag5, manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, tag5, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
config := ispec.Image{}
@ -1309,15 +1324,15 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
manifestDigest3 = godigest.FromString("fake-manifest3")
)
err := repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo3, tag1, manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo3, tag1, manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
config1 := ispec.Image{
@ -1433,7 +1448,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
manifestFromIndexDigest2 = godigest.FromString("fake-manifestFromIndexDigest2")
)
err := repoDB.SetRepoTag(repo1, tag3, indexDigest, ispec.MediaTypeImageIndex)
err := repoDB.SetRepoReference(repo1, tag3, indexDigest, ispec.MediaTypeImageIndex)
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests(
@ -1449,15 +1464,15 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
})
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo3, tag1, manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo3, tag1, manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
config1 := ispec.Image{
@ -1561,7 +1576,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
ConfigBlob: emptyConfigBlob,
}
err = repoDB.SetRepoTag(repo1, "2.0.0", indexDigest, ispec.MediaTypeImageIndex)
err = repoDB.SetRepoReference(repo1, "2.0.0", indexDigest, ispec.MediaTypeImageIndex)
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]godigest.Digest{
@ -1575,17 +1590,17 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
})
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyManifestMeta)
@ -1797,15 +1812,15 @@ func TestRelevanceSorting(t *testing.T) {
ConfigBlob: emptyConfigBlob,
}
err = repoDB.SetRepoTag(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo2, tag3, manifestDigest3, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo2, tag3, manifestDigest3, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo3, tag3, manifestDigest3, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo3, tag3, manifestDigest3, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoTag(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest)
err = repoDB.SetRepoReference(repo4, tag1, manifestDigest3, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta)

28
pkg/meta/repodb/sync_repodb.go
View file

@ -82,7 +82,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
}
if manifestMetaIsPresent && hasTag {
err = repoDB.SetRepoTag(repo, tag, manifest.Digest, manifest.MediaType)
err = repoDB.SetRepoReference(repo, tag, manifest.Digest, manifest.MediaType)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to set repo tag for %s:%s", repo, tag)
@ -132,7 +132,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
}
err = SetMetadataFromInput(repo, reference, manifest.MediaType, manifest.Digest, manifestBlob,
storeController, repoDB, log)
imageStore, repoDB, log)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to set metadata for %s:%s", repo, tag)
@ -220,7 +220,7 @@ func isManifestMetaPresent(repo string, manifest ispec.Descriptor, repoDB RepoDB
}
// NewManifestMeta takes raw data about an image and createa a new ManifestMetadate object.
func NewManifestData(repoName string, manifestBlob []byte, storeController storage.StoreController,
func NewManifestData(repoName string, manifestBlob []byte, imgStore storage.ImageStore,
) (ManifestData, error) {
var (
manifestContent ispec.Manifest
@ -228,8 +228,6 @@ func NewManifestData(repoName string, manifestBlob []byte, storeController stora
manifestData ManifestData
)
imgStore := storeController.GetImageStore(repoName)
err := json.Unmarshal(manifestBlob, &manifestContent)
if err != nil {
return ManifestData{}, err
@ -251,7 +249,7 @@ func NewManifestData(repoName string, manifestBlob []byte, storeController stora
return manifestData, nil
}
func NewIndexData(repoName string, indexBlob []byte, storeController storage.StoreController,
func NewIndexData(repoName string, indexBlob []byte,
) IndexData {
indexData := IndexData{}
@ -263,11 +261,11 @@ func NewIndexData(repoName string, indexBlob []byte, storeController storage.Sto
// SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag
// (in case the reference is a tag). The function expects image manifests and indexes (multi arch images).
func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte,
storeController storage.StoreController, repoDB RepoDB, log log.Logger,
imageStore storage.ImageStore, repoDB RepoDB, log log.Logger,
) error {
switch mediaType {
case ispec.MediaTypeImageManifest:
imageData, err := NewManifestData(repo, descriptorBlob, storeController)
imageData, err := NewManifestData(repo, descriptorBlob, imageStore)
if err != nil {
return err
}
@ -279,7 +277,7 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig
return err
}
case ispec.MediaTypeImageIndex:
indexData := NewIndexData(repo, descriptorBlob, storeController)
indexData := NewIndexData(repo, descriptorBlob)
err := repoDB.SetIndexData(digest, indexData)
if err != nil {
@ -289,11 +287,7 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig
}
}
if refferenceIsDigest(reference) {
return nil
}
err := repoDB.SetRepoTag(repo, reference, digest, mediaType)
err := repoDB.SetRepoReference(repo, reference, digest, mediaType)
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting repo meta")
@ -302,9 +296,3 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig
return nil
}
func refferenceIsDigest(reference string) bool {
_, err := godigest.Parse(reference)
return err == nil
}

4
pkg/meta/repodb/sync_repodb_test.go
View file

@ -178,8 +178,8 @@ func TestSyncRepoDBErrors(t *testing.T) {
return indexBlob, nil
}
Convey("repoDB.SetRepoTag", func() {
repoDB.SetRepoTagFn = func(repo, tag string, manifestDigest godigest.Digest, mediaType string) error {
Convey("repoDB.SetRepoReference", func() {
repoDB.SetRepoReferenceFn = func(repo, tag string, manifestDigest godigest.Digest, mediaType string) error {
return ErrTestError
}

4
pkg/meta/repodb/update/update.go
View file

@ -52,7 +52,7 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest,
}
} else {
err := repodb.SetMetadataFromInput(name, reference, mediaType, digest, body,
storeController, repoDB, log)
imgStore, repoDB, log)
if err != nil {
metadataSuccessfullySet = false
}
@ -152,7 +152,7 @@ func OnGetManifest(name, reference string, digest godigest.Digest, body []byte,
if !isSignature {
err := repoDB.IncrementImageDownloads(name, reference)
if err != nil {
log.Error().Err(err).Msg("unexpected error")
log.Error().Err(err).Msgf("unexpected error for '%s:%s'", name, reference)
return err
}

5
pkg/meta/repodb/update/update_test.go
View file

@ -181,12 +181,11 @@ func TestUpdateErrors(t *testing.T) {
Convey("SetMetadataFromInput", func() {
imageStore := mocks.MockedImageStore{}
storeController := storage.StoreController{DefaultStore: &imageStore}
repoDB := mocks.RepoDBMock{}
log := log.NewLogger("debug", "")
err := repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest",
[]byte("BadManifestBlob"), storeController, repoDB, log)
[]byte("BadManifestBlob"), imageStore, repoDB, log)
So(err, ShouldNotBeNil)
// reference is digest
@ -203,7 +202,7 @@ func TestUpdateErrors(t *testing.T) {
}
err = repodb.SetMetadataFromInput("repo", string(godigest.FromString("reference")), "", "digest",
manifestBlob, storeController, repoDB, log)
manifestBlob, imageStore, repoDB, log)
So(err, ShouldBeNil)
})
})

3
pkg/test/common_test.go
View file

@ -6,6 +6,7 @@ package test_test
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path"
@ -23,6 +24,8 @@ import (
"zotregistry.io/zot/pkg/test"
)
var ErrTestError = errors.New("test error")
func TestCopyFiles(t *testing.T) {
Convey("sourceDir does not exist", t, func() {
err := test.CopyFiles("/path/to/some/unexisting/directory", os.TempDir())

10
pkg/test/mocks/repo_db_mock.go
View file

@ -19,7 +19,7 @@ type RepoDBMock struct {
SetRepoLogoFn func(repo string, logoPath string) error
SetRepoTagFn func(repo string, tag string, manifestDigest godigest.Digest, mediaType string) error
SetRepoReferenceFn func(repo string, Reference string, manifestDigest godigest.Digest, mediaType string) error
DeleteRepoTagFn func(repo string, tag string) error
@ -111,9 +111,11 @@ func (sdm RepoDBMock) SetRepoLogo(repo string, logoPath string) error {
return nil
}
func (sdm RepoDBMock) SetRepoTag(repo string, tag string, manifestDigest godigest.Digest, mediaType string) error {
if sdm.SetRepoTagFn != nil {
return sdm.SetRepoTagFn(repo, tag, manifestDigest, mediaType)
func (sdm RepoDBMock) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest,
mediaType string,
) error {
if sdm.SetRepoReferenceFn != nil {
return sdm.SetRepoReferenceFn(repo, reference, manifestDigest, mediaType)
}
return nil