0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00

fix: issues with nested index processing in CVE and metaDB code (#2732)

Also fix an issue with searching tags, which should work with case insensitive searches.

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
This commit is contained in:
Andrei Aaron 2024-10-21 19:57:43 +03:00 committed by GitHub
parent edb5491428
commit da6bd56a21
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 456 additions and 105 deletions

View file

@ -1350,7 +1350,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo
// Tag is not found
cveList, cveSummary, pageInfo, err = cveInfo.GetCVEListForImage(ctx, "repo-with-bad-tag-digest", "tag", "", "", "", pageInput)
So(err, ShouldEqual, zerr.ErrImageMetaNotFound)
So(err, ShouldWrap, zerr.ErrImageMetaNotFound)
So(len(cveList), ShouldEqual, 0)
So(pageInfo.ItemCount, ShouldEqual, 0)
So(pageInfo.TotalCount, ShouldEqual, 0)

View file

@ -960,7 +960,9 @@ func globalSearch(ctx context.Context, query string, metaDB mTypes.MetaDB, filte
pageInput := getPageInput(requestedPage)
expectedTag := strings.TrimPrefix(query, `:`)
matchTagName := func(repoName, actualTag string) bool { return strings.Contains(actualTag, expectedTag) }
matchTagName := func(repoName, actualTag string) bool {
return strings.Contains(strings.ToLower(actualTag), expectedTag)
}
fullImageMetaList, err := metaDB.FilterTags(ctx, matchTagName, mTypes.AcceptAllImageMeta)
if err != nil {

View file

@ -3005,7 +3005,7 @@ func TestGlobalSearchImageAuthor(t *testing.T) {
})
}
func TestGlobalSearch(t *testing.T) {
func TestGlobalSearch(t *testing.T) { //nolint: gocyclo
Convey("Test searching for repos with vulnerabitity scanning disabled", t, func() {
subpath := "/a"
@ -3904,6 +3904,328 @@ func TestGlobalSearch(t *testing.T) {
So(len(results.Images), ShouldEqual, 0)
So(len(results.Repos), ShouldEqual, 0)
})
Convey("test nested indexes", t, func() {
log := log.NewLogger("debug", "")
rootDir := t.TempDir()
port := GetFreePort()
baseURL := GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
conf.Storage.RootDirectory = rootDir
defaultVal := true
conf.Extensions = &extconf.ExtensionConfig{
Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}},
}
conf.Extensions.Search.CVE = nil
ctlr := api.NewController(conf)
ctlrManager := NewControllerManager(ctlr)
storeCtlr := ociutils.GetDefaultStoreController(rootDir, log)
// nested manifest/indexes:
// image111 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop
// image112 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop
// image121 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop
// image122 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop
// image211 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop
// image212 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop
// image31 -> multiArchMiddle3 -> multiArchTop
// image32 -> multiArchMiddle3 -> multiArchTop
repoName := "nested"
image111 := CreateRandomImage()
image112 := CreateRandomImage()
multiArchBottom11 := CreateMultiarchWith().Images([]Image{image111, image112}).Build()
err := WriteMultiArchImageToFileSystem(multiArchBottom11, repoName, multiArchBottom11.Digest().String(), storeCtlr)
So(err, ShouldBeNil)
image121 := CreateRandomImage()
image122 := CreateRandomImage()
multiArchBottom12 := CreateMultiarchWith().Images([]Image{image121, image122}).Build()
err = WriteMultiArchImageToFileSystem(multiArchBottom12, repoName, multiArchBottom12.Digest().String(), storeCtlr)
So(err, ShouldBeNil)
indexMultiArchMiddle1 := ispec.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: ispec.MediaTypeImageIndex,
Manifests: []ispec.Descriptor{
{
Digest: multiArchBottom11.IndexDescriptor.Digest,
Size: multiArchBottom11.IndexDescriptor.Size,
MediaType: ispec.MediaTypeImageIndex,
},
{
Digest: multiArchBottom12.IndexDescriptor.Digest,
Size: multiArchBottom12.IndexDescriptor.Size,
MediaType: ispec.MediaTypeImageIndex,
},
},
}
indexMultiArchMiddle1Blob, err := json.Marshal(indexMultiArchMiddle1)
So(err, ShouldBeNil)
indexMultiArchMiddle1Digest, _, err := storeCtlr.GetDefaultImageStore().PutImageManifest(repoName,
"multiArchMiddle1", ispec.MediaTypeImageIndex, indexMultiArchMiddle1Blob)
So(err, ShouldBeNil)
image211 := CreateRandomImage()
image212 := CreateRandomImage()
multiArchBottom21 := CreateMultiarchWith().Images([]Image{image211, image212}).Build()
err = WriteMultiArchImageToFileSystem(multiArchBottom21, repoName, multiArchBottom21.Digest().String(), storeCtlr)
So(err, ShouldBeNil)
indexMultiArchMiddle2 := ispec.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: ispec.MediaTypeImageIndex,
Manifests: []ispec.Descriptor{
{
Digest: multiArchBottom21.IndexDescriptor.Digest,
Size: multiArchBottom21.IndexDescriptor.Size,
MediaType: ispec.MediaTypeImageIndex,
},
},
}
indexMultiArchMiddle2Blob, err := json.Marshal(indexMultiArchMiddle2)
So(err, ShouldBeNil)
indexMultiArchMiddle2Digest, _, err := storeCtlr.GetDefaultImageStore().PutImageManifest(repoName,
"multiArchMiddle2", ispec.MediaTypeImageIndex, indexMultiArchMiddle2Blob)
So(err, ShouldBeNil)
image31 := CreateRandomImage()
image32 := CreateRandomImage()
multiArchBottom3 := CreateMultiarchWith().Images([]Image{image31, image32}).Build()
err = WriteMultiArchImageToFileSystem(multiArchBottom3, repoName, multiArchBottom3.Digest().String(), storeCtlr)
So(err, ShouldBeNil)
indexMultiArchTop := ispec.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: ispec.MediaTypeImageIndex,
Manifests: []ispec.Descriptor{
{
Digest: indexMultiArchMiddle1Digest,
Size: int64(len(indexMultiArchMiddle1Blob)),
MediaType: ispec.MediaTypeImageIndex,
},
{
Digest: indexMultiArchMiddle2Digest,
Size: int64(len(indexMultiArchMiddle2Blob)),
MediaType: ispec.MediaTypeImageIndex,
},
{
Digest: multiArchBottom3.IndexDescriptor.Digest,
Size: multiArchBottom3.IndexDescriptor.Size,
MediaType: ispec.MediaTypeImageIndex,
},
},
}
indexMultiArchTopBlob, err := json.Marshal(indexMultiArchTop)
So(err, ShouldBeNil)
_, _, err = storeCtlr.GetDefaultImageStore().PutImageManifest(repoName, "multiArchTop", ispec.MediaTypeImageIndex,
indexMultiArchTopBlob)
So(err, ShouldBeNil)
ctlrManager.StartAndWait(port)
defer ctlrManager.StopServer()
// Search for a specific tag cross-repo and return single arch images
results := GlobalSearchGQL(":multiArch", baseURL).GlobalSearch
So(len(results.Images), ShouldEqual, 3)
So(len(results.Repos), ShouldEqual, 0)
for _, image := range results.Images {
So(image.RepoName, ShouldEqual, repoName)
switch image.Tag {
case "multiArchMiddle1":
So(len(image.Manifests), ShouldEqual, 4)
case "multiArchMiddle2":
So(len(image.Manifests), ShouldEqual, 2)
case "multiArchTop":
So(len(image.Manifests), ShouldEqual, 8)
}
}
})
Convey("test nested indexes", t, func() {
log := log.NewLogger("debug", "")
rootDir := t.TempDir()
port := GetFreePort()
baseURL := GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
conf.Storage.RootDirectory = rootDir
defaultVal := true
updateDuration, _ := time.ParseDuration("1h")
trivyConfig := &extconf.TrivyConfig{
DBRepository: "ghcr.io/project-zot/trivy-db",
}
cveConfig := &extconf.CVEConfig{
UpdateInterval: updateDuration,
Trivy: trivyConfig,
}
searchConfig := &extconf.SearchConfig{
BaseConfig: extconf.BaseConfig{Enable: &defaultVal},
CVE: cveConfig,
}
conf.Extensions = &extconf.ExtensionConfig{
Search: searchConfig,
}
storeCtlr := ociutils.GetDefaultStoreController(rootDir, log)
// nested manifest/indexes:
// image111 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop
// image112 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop
// image121 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop
// image122 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop
// image211 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop
// image212 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop
// image31 -> multiArchMiddle3 -> multiArchTop
// image32 -> multiArchMiddle3 -> multiArchTop
repoName := "nested"
image111 := CreateRandomImage()
image112 := CreateRandomImage()
multiArchBottom11 := CreateMultiarchWith().Images([]Image{image111, image112}).Build()
err := WriteMultiArchImageToFileSystem(multiArchBottom11, repoName, multiArchBottom11.Digest().String(), storeCtlr)
So(err, ShouldBeNil)
image121 := CreateRandomImage()
image122 := CreateRandomImage()
multiArchBottom12 := CreateMultiarchWith().Images([]Image{image121, image122}).Build()
err = WriteMultiArchImageToFileSystem(multiArchBottom12, repoName, multiArchBottom12.Digest().String(), storeCtlr)
So(err, ShouldBeNil)
indexMultiArchMiddle1 := ispec.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: ispec.MediaTypeImageIndex,
Manifests: []ispec.Descriptor{
{
Digest: multiArchBottom11.IndexDescriptor.Digest,
Size: multiArchBottom11.IndexDescriptor.Size,
MediaType: ispec.MediaTypeImageIndex,
},
{
Digest: multiArchBottom12.IndexDescriptor.Digest,
Size: multiArchBottom12.IndexDescriptor.Size,
MediaType: ispec.MediaTypeImageIndex,
},
},
}
indexMultiArchMiddle1Blob, err := json.Marshal(indexMultiArchMiddle1)
So(err, ShouldBeNil)
indexMultiArchMiddle1Digest, _, err := storeCtlr.GetDefaultImageStore().PutImageManifest(repoName,
"multiArchMiddle1", ispec.MediaTypeImageIndex, indexMultiArchMiddle1Blob)
So(err, ShouldBeNil)
image211 := CreateRandomImage()
image212 := CreateRandomImage()
multiArchBottom21 := CreateMultiarchWith().Images([]Image{image211, image212}).Build()
err = WriteMultiArchImageToFileSystem(multiArchBottom21, repoName, multiArchBottom21.Digest().String(), storeCtlr)
So(err, ShouldBeNil)
indexMultiArchMiddle2 := ispec.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: ispec.MediaTypeImageIndex,
Manifests: []ispec.Descriptor{
{
Digest: multiArchBottom21.IndexDescriptor.Digest,
Size: multiArchBottom21.IndexDescriptor.Size,
MediaType: ispec.MediaTypeImageIndex,
},
},
}
indexMultiArchMiddle2Blob, err := json.Marshal(indexMultiArchMiddle2)
So(err, ShouldBeNil)
indexMultiArchMiddle2Digest, _, err := storeCtlr.GetDefaultImageStore().PutImageManifest(repoName,
"multiArchMiddle2", ispec.MediaTypeImageIndex, indexMultiArchMiddle2Blob)
So(err, ShouldBeNil)
image31 := CreateRandomImage()
image32 := CreateRandomImage()
multiArchBottom3 := CreateMultiarchWith().Images([]Image{image31, image32}).Build()
err = WriteMultiArchImageToFileSystem(multiArchBottom3, repoName, multiArchBottom3.Digest().String(), storeCtlr)
So(err, ShouldBeNil)
indexMultiArchTop := ispec.Index{
Versioned: specs.Versioned{SchemaVersion: 2},
MediaType: ispec.MediaTypeImageIndex,
Manifests: []ispec.Descriptor{
{
Digest: indexMultiArchMiddle1Digest,
Size: int64(len(indexMultiArchMiddle1Blob)),
MediaType: ispec.MediaTypeImageIndex,
},
{
Digest: indexMultiArchMiddle2Digest,
Size: int64(len(indexMultiArchMiddle2Blob)),
MediaType: ispec.MediaTypeImageIndex,
},
{
Digest: multiArchBottom3.IndexDescriptor.Digest,
Size: multiArchBottom3.IndexDescriptor.Size,
MediaType: ispec.MediaTypeImageIndex,
},
},
}
indexMultiArchTopBlob, err := json.Marshal(indexMultiArchTop)
So(err, ShouldBeNil)
_, _, err = storeCtlr.GetDefaultImageStore().PutImageManifest(repoName, "multiArchTop", ispec.MediaTypeImageIndex,
indexMultiArchTopBlob)
So(err, ShouldBeNil)
ctlr := api.NewController(conf)
if err := ctlr.Init(); err != nil {
panic(err)
}
ctlr.CveScanner = getMockCveScanner(ctlr.MetaDB)
go func() {
if err := ctlr.Run(); !errors.Is(err, http.ErrServerClosed) {
panic(err)
}
}()
defer ctlr.Shutdown()
WaitTillServerReady(baseURL)
// Search for a specific tag cross-repo and return single arch images
results := GlobalSearchGQL(":multiArch", baseURL).GlobalSearch
So(len(results.Images), ShouldEqual, 3)
So(len(results.Repos), ShouldEqual, 0)
for _, image := range results.Images {
So(image.RepoName, ShouldEqual, repoName)
switch image.Tag {
case "multiArchMiddle1":
So(len(image.Manifests), ShouldEqual, 4)
case "multiArchMiddle2":
So(len(image.Manifests), ShouldEqual, 2)
case "multiArchTop":
So(len(image.Manifests), ShouldEqual, 8)
}
}
})
}
func TestCleaningFilteringParamsGlobalSearch(t *testing.T) {

View file

@ -400,17 +400,11 @@ func (bdw *BoltDB) FilterImageMeta(ctx context.Context, digests []string,
}
if protoImageMeta.MediaType == ispec.MediaTypeImageIndex {
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
for _, manifest := range protoImageMeta.Index.Index.Manifests {
imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
_, manifestDataList, err := getAllContainedMeta(imageBuck, protoImageMeta)
if err != nil {
return err
}
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
}
protoImageMeta.Manifests = manifestDataList
}
@ -474,7 +468,7 @@ func getProtoImageMeta(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageM
imageMetaBlob := imageBuck.Get([]byte(digest))
if len(imageMetaBlob) == 0 {
return nil, zerr.ErrImageMetaNotFound
return nil, fmt.Errorf("%w for digest %s", zerr.ErrImageMetaNotFound, digest)
}
imageMeta := proto_go.ImageMeta{}
@ -487,6 +481,35 @@ func getProtoImageMeta(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageM
return &imageMeta, nil
}
func getAllContainedMeta(imageBuck *bbolt.Bucket, imageIndexData *proto_go.ImageMeta,
) ([]*proto_go.ImageMeta, []*proto_go.ManifestMeta, error) {
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests))
imageMetaList := make([]*proto_go.ImageMeta, 0, len(imageIndexData.Index.Index.Manifests))
for _, manifest := range imageIndexData.Index.Index.Manifests {
imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
if err != nil {
return imageMetaList, manifestDataList, err
}
switch imageManifestData.MediaType {
case ispec.MediaTypeImageManifest:
imageMetaList = append(imageMetaList, imageManifestData)
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
case ispec.MediaTypeImageIndex:
partialImageDataList, partialManifestDataList, err := getAllContainedMeta(imageBuck, imageManifestData)
if err != nil {
return imageMetaList, manifestDataList, err
}
imageMetaList = append(imageMetaList, partialImageDataList...)
manifestDataList = append(manifestDataList, partialManifestDataList...)
}
}
return imageMetaList, manifestDataList, nil
}
func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string,
) ([]mTypes.FullImageMeta, error) {
images := []mTypes.FullImageMeta{}
@ -552,17 +575,11 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string,
indexDigest, err)
}
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests))
for _, manifest := range imageIndexData.Index.Index.Manifests {
imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
_, manifestDataList, err := getAllContainedMeta(imageBuck, imageIndexData)
if err != nil {
return err
}
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
}
imageIndexData.Manifests = manifestDataList
protoImageMeta = imageIndexData
@ -649,16 +666,14 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRe
imageIndexMeta := mConvert.GetImageMeta(protoImageIndexMeta)
matchedManifests := []*proto_go.ManifestMeta{}
for _, manifest := range protoImageIndexMeta.Index.Index.Manifests {
manifestDigest := manifest.Digest
imageManifestData, err := getProtoImageMeta(imageMetaBuck, manifestDigest)
imageManifestDataList, _, err := getAllContainedMeta(imageMetaBuck, protoImageIndexMeta)
if err != nil {
viewError = errors.Join(viewError, err)
continue
}
for _, imageManifestData := range imageManifestDataList {
imageMeta := mConvert.GetImageMeta(imageManifestData)
partialImageMeta := common.GetPartialImageMeta(imageIndexMeta, imageMeta)
@ -798,17 +813,11 @@ func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string
}
if protoImageMeta.MediaType == ispec.MediaTypeImageIndex {
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
for _, manifest := range protoImageMeta.Index.Index.Manifests {
imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
_, manifestDataList, err := getAllContainedMeta(imageBuck, protoImageMeta)
if err != nil {
return err
}
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
}
protoImageMeta.Manifests = manifestDataList
}
@ -830,17 +839,11 @@ func (bdw *BoltDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, error
}
if protoImageMeta.MediaType == ispec.MediaTypeImageIndex {
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
for _, manifest := range protoImageMeta.Index.Index.Manifests {
imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
_, manifestDataList, err := getAllContainedMeta(imageBuck, protoImageMeta)
if err != nil {
return err
}
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
}
protoImageMeta.Manifests = manifestDataList
}

View file

@ -224,6 +224,45 @@ func (dwr *DynamoDB) GetProtoImageMeta(ctx context.Context, digest godigest.Dige
return imageMeta, nil
}
func (dwr *DynamoDB) getAllContainedMeta(ctx context.Context, imageIndexData *proto_go.ImageMeta,
) ([]*proto_go.ImageMeta, []*proto_go.ManifestMeta, error) {
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests))
imageMetaList := make([]*proto_go.ImageMeta, 0, len(imageIndexData.Index.Index.Manifests))
manifestDigests := make([]string, 0, len(imageIndexData.Index.Index.Manifests))
for i := range imageIndexData.Index.Index.Manifests {
manifestDigests = append(manifestDigests, imageIndexData.Index.Index.Manifests[i].Digest)
}
manifestsAttributes, err := dwr.fetchImageMetaAttributesByDigest(ctx, manifestDigests)
if err != nil {
return imageMetaList, manifestDataList, err
}
for _, manifestAttribute := range manifestsAttributes {
imageManifestData, err := getProtoImageMetaFromAttribute(manifestAttribute["ImageMeta"])
if err != nil {
return imageMetaList, manifestDataList, err
}
switch imageManifestData.MediaType {
case ispec.MediaTypeImageManifest:
imageMetaList = append(imageMetaList, imageManifestData)
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
case ispec.MediaTypeImageIndex:
partialImageDataList, partialManifestDataList, err := dwr.getAllContainedMeta(ctx, imageManifestData)
if err != nil {
return imageMetaList, manifestDataList, err
}
imageMetaList = append(imageMetaList, partialImageDataList...)
manifestDataList = append(manifestDataList, partialManifestDataList...)
}
}
return imageMetaList, manifestDataList, nil
}
func (dwr *DynamoDB) setProtoRepoMeta(repo string, repoMeta *proto_go.RepoMeta) error {
repoMeta.Name = repo
@ -640,19 +679,11 @@ func (dwr *DynamoDB) SearchTags(ctx context.Context, searchText string) ([]mType
fmt.Errorf("error fetching manifest meta for manifest with digest %s %w", indexDigest, err)
}
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests))
for _, manifest := range imageIndexData.Index.Index.Manifests {
manifestDigest := godigest.Digest(manifest.Digest)
imageManifestData, err := dwr.GetProtoImageMeta(ctx, manifestDigest)
_, manifestDataList, err := dwr.getAllContainedMeta(ctx, imageIndexData)
if err != nil {
return []mTypes.FullImageMeta{}, err
}
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
}
imageIndexData.Manifests = manifestDataList
protoImageMeta = imageIndexData
@ -739,16 +770,14 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterRepoTag mTypes.Filter
imageIndexMeta := mConvert.GetImageMeta(protoImageIndexMeta)
matchedManifests := []*proto_go.ManifestMeta{}
for _, manifest := range protoImageIndexMeta.Index.Index.Manifests {
manifestDigest := manifest.Digest
imageManifestData, err := dwr.GetProtoImageMeta(ctx, godigest.Digest(manifestDigest))
imageManifestDataList, _, err := dwr.getAllContainedMeta(context.Background(), protoImageIndexMeta)
if err != nil {
viewError = errors.Join(viewError, err)
continue
}
for _, imageManifestData := range imageManifestDataList {
imageMeta := mConvert.GetImageMeta(imageManifestData)
partialImageMeta := common.GetPartialImageMeta(imageIndexMeta, imageMeta)
@ -868,17 +897,11 @@ func (dwr *DynamoDB) GetFullImageMeta(ctx context.Context, repo string, tag stri
}
if protoImageMeta.MediaType == ispec.MediaTypeImageIndex {
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
for _, manifest := range protoImageMeta.Index.Index.Manifests {
imageManifestData, err := dwr.GetProtoImageMeta(ctx, godigest.Digest(manifest.Digest))
_, manifestDataList, err := dwr.getAllContainedMeta(ctx, protoImageMeta)
if err != nil {
return mTypes.FullImageMeta{}, err
}
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
}
protoImageMeta.Manifests = manifestDataList
}
@ -901,19 +924,11 @@ func (dwr *DynamoDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, err
}
if protoImageMeta.MediaType == ispec.MediaTypeImageIndex {
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
for _, manifest := range protoImageMeta.Index.Index.Manifests {
manifestDigest := godigest.Digest(manifest.Digest)
imageManifestData, err := dwr.GetProtoImageMeta(context.Background(), manifestDigest)
_, manifestDataList, err := dwr.getAllContainedMeta(context.Background(), protoImageMeta)
if err != nil {
return mTypes.ImageMeta{}, err
}
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
}
protoImageMeta.Manifests = manifestDataList
}
@ -1330,27 +1345,11 @@ func (dwr *DynamoDB) FilterImageMeta(ctx context.Context, digests []string,
}
if protoImageMeta.MediaType == ispec.MediaTypeImageIndex {
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
indexDigests := make([]string, 0, len(protoImageMeta.Index.Index.Manifests))
for i := range protoImageMeta.Index.Index.Manifests {
indexDigests = append(indexDigests, protoImageMeta.Index.Index.Manifests[i].Digest)
}
manifestsAttributes, err := dwr.fetchImageMetaAttributesByDigest(ctx, indexDigests)
_, manifestDataList, err := dwr.getAllContainedMeta(context.Background(), protoImageMeta)
if err != nil {
return nil, err
}
for _, manifestAttribute := range manifestsAttributes {
imageManifestData, err := getProtoImageMetaFromAttribute(manifestAttribute["ImageMeta"])
if err != nil {
return nil, err
}
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
}
protoImageMeta.Manifests = manifestDataList
}
@ -2001,7 +2000,32 @@ func (dwr *DynamoDB) fetchImageMetaAttributesByDigest(ctx context.Context, diges
start = end
}
return batchedResp, nil
// Order the responses based on initial digest order
// as BatchGetItem does not guarantee the key order is respected
orderedResp := []map[string]types.AttributeValue{}
respMap := map[string]map[string]types.AttributeValue{}
for _, item := range batchedResp {
var digest string
err := attributevalue.Unmarshal(item["TableKey"], &digest)
if err != nil {
return nil, err
}
respMap[digest] = item
}
for _, digest := range digests {
imageMeta, ok := respMap[digest]
if !ok {
return nil, fmt.Errorf("%w for digest %s", zerr.ErrImageMetaNotFound, digest)
}
orderedResp = append(orderedResp, imageMeta)
}
return orderedResp, nil
}
func getBatchImageKeys(digests []string) []map[string]types.AttributeValue {