From da6bd56a21e1c5d20d83ea7392914cb6947b0bb0 Mon Sep 17 00:00:00 2001 From: Andrei Aaron Date: Mon, 21 Oct 2024 19:57:43 +0300 Subject: [PATCH] fix: issues with nested index processing in CVE and metaDB code (#2732) Also fix an issue with searching tags, which should work with case insensitive searches. Signed-off-by: Andrei Aaron --- pkg/extensions/search/cve/cve_test.go | 2 +- pkg/extensions/search/resolver.go | 4 +- pkg/extensions/search/search_test.go | 324 +++++++++++++++++++++++++- pkg/meta/boltdb/boltdb.go | 93 ++++---- pkg/meta/dynamodb/dynamodb.go | 138 ++++++----- 5 files changed, 456 insertions(+), 105 deletions(-) diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index cd5f9096..9022ec79 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -1350,7 +1350,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo // Tag is not found cveList, cveSummary, pageInfo, err = cveInfo.GetCVEListForImage(ctx, "repo-with-bad-tag-digest", "tag", "", "", "", pageInput) - So(err, ShouldEqual, zerr.ErrImageMetaNotFound) + So(err, ShouldWrap, zerr.ErrImageMetaNotFound) So(len(cveList), ShouldEqual, 0) So(pageInfo.ItemCount, ShouldEqual, 0) So(pageInfo.TotalCount, ShouldEqual, 0) diff --git a/pkg/extensions/search/resolver.go b/pkg/extensions/search/resolver.go index 3cd3f9ec..c3366cda 100644 --- a/pkg/extensions/search/resolver.go +++ b/pkg/extensions/search/resolver.go @@ -960,7 +960,9 @@ func globalSearch(ctx context.Context, query string, metaDB mTypes.MetaDB, filte pageInput := getPageInput(requestedPage) expectedTag := strings.TrimPrefix(query, `:`) - matchTagName := func(repoName, actualTag string) bool { return strings.Contains(actualTag, expectedTag) } + matchTagName := func(repoName, actualTag string) bool { + return strings.Contains(strings.ToLower(actualTag), expectedTag) + } fullImageMetaList, err := metaDB.FilterTags(ctx, matchTagName, mTypes.AcceptAllImageMeta) if err != nil { diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index 9f2b2f13..b7f835cf 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -3005,7 +3005,7 @@ func TestGlobalSearchImageAuthor(t *testing.T) { }) } -func TestGlobalSearch(t *testing.T) { +func TestGlobalSearch(t *testing.T) { //nolint: gocyclo Convey("Test searching for repos with vulnerabitity scanning disabled", t, func() { subpath := "/a" @@ -3904,6 +3904,328 @@ func TestGlobalSearch(t *testing.T) { So(len(results.Images), ShouldEqual, 0) So(len(results.Repos), ShouldEqual, 0) }) + + Convey("test nested indexes", t, func() { + log := log.NewLogger("debug", "") + rootDir := t.TempDir() + port := GetFreePort() + baseURL := GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + conf.Storage.RootDirectory = rootDir + defaultVal := true + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, + } + conf.Extensions.Search.CVE = nil + + ctlr := api.NewController(conf) + ctlrManager := NewControllerManager(ctlr) + + storeCtlr := ociutils.GetDefaultStoreController(rootDir, log) + + // nested manifest/indexes: + // image111 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop + // image112 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop + // image121 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop + // image122 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop + // image211 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop + // image212 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop + // image31 -> multiArchMiddle3 -> multiArchTop + // image32 -> multiArchMiddle3 -> multiArchTop + + repoName := "nested" + + image111 := CreateRandomImage() + image112 := CreateRandomImage() + multiArchBottom11 := CreateMultiarchWith().Images([]Image{image111, image112}).Build() + err := WriteMultiArchImageToFileSystem(multiArchBottom11, repoName, multiArchBottom11.Digest().String(), storeCtlr) + So(err, ShouldBeNil) + + image121 := CreateRandomImage() + image122 := CreateRandomImage() + multiArchBottom12 := CreateMultiarchWith().Images([]Image{image121, image122}).Build() + err = WriteMultiArchImageToFileSystem(multiArchBottom12, repoName, multiArchBottom12.Digest().String(), storeCtlr) + So(err, ShouldBeNil) + + indexMultiArchMiddle1 := ispec.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ispec.MediaTypeImageIndex, + Manifests: []ispec.Descriptor{ + { + Digest: multiArchBottom11.IndexDescriptor.Digest, + Size: multiArchBottom11.IndexDescriptor.Size, + MediaType: ispec.MediaTypeImageIndex, + }, + { + Digest: multiArchBottom12.IndexDescriptor.Digest, + Size: multiArchBottom12.IndexDescriptor.Size, + MediaType: ispec.MediaTypeImageIndex, + }, + }, + } + + indexMultiArchMiddle1Blob, err := json.Marshal(indexMultiArchMiddle1) + So(err, ShouldBeNil) + + indexMultiArchMiddle1Digest, _, err := storeCtlr.GetDefaultImageStore().PutImageManifest(repoName, + "multiArchMiddle1", ispec.MediaTypeImageIndex, indexMultiArchMiddle1Blob) + So(err, ShouldBeNil) + + image211 := CreateRandomImage() + image212 := CreateRandomImage() + multiArchBottom21 := CreateMultiarchWith().Images([]Image{image211, image212}).Build() + err = WriteMultiArchImageToFileSystem(multiArchBottom21, repoName, multiArchBottom21.Digest().String(), storeCtlr) + So(err, ShouldBeNil) + + indexMultiArchMiddle2 := ispec.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ispec.MediaTypeImageIndex, + Manifests: []ispec.Descriptor{ + { + Digest: multiArchBottom21.IndexDescriptor.Digest, + Size: multiArchBottom21.IndexDescriptor.Size, + MediaType: ispec.MediaTypeImageIndex, + }, + }, + } + + indexMultiArchMiddle2Blob, err := json.Marshal(indexMultiArchMiddle2) + So(err, ShouldBeNil) + + indexMultiArchMiddle2Digest, _, err := storeCtlr.GetDefaultImageStore().PutImageManifest(repoName, + "multiArchMiddle2", ispec.MediaTypeImageIndex, indexMultiArchMiddle2Blob) + So(err, ShouldBeNil) + + image31 := CreateRandomImage() + image32 := CreateRandomImage() + multiArchBottom3 := CreateMultiarchWith().Images([]Image{image31, image32}).Build() + err = WriteMultiArchImageToFileSystem(multiArchBottom3, repoName, multiArchBottom3.Digest().String(), storeCtlr) + So(err, ShouldBeNil) + + indexMultiArchTop := ispec.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ispec.MediaTypeImageIndex, + Manifests: []ispec.Descriptor{ + { + Digest: indexMultiArchMiddle1Digest, + Size: int64(len(indexMultiArchMiddle1Blob)), + MediaType: ispec.MediaTypeImageIndex, + }, + { + Digest: indexMultiArchMiddle2Digest, + Size: int64(len(indexMultiArchMiddle2Blob)), + MediaType: ispec.MediaTypeImageIndex, + }, + { + Digest: multiArchBottom3.IndexDescriptor.Digest, + Size: multiArchBottom3.IndexDescriptor.Size, + MediaType: ispec.MediaTypeImageIndex, + }, + }, + } + + indexMultiArchTopBlob, err := json.Marshal(indexMultiArchTop) + So(err, ShouldBeNil) + + _, _, err = storeCtlr.GetDefaultImageStore().PutImageManifest(repoName, "multiArchTop", ispec.MediaTypeImageIndex, + indexMultiArchTopBlob) + So(err, ShouldBeNil) + + ctlrManager.StartAndWait(port) + defer ctlrManager.StopServer() + + // Search for a specific tag cross-repo and return single arch images + results := GlobalSearchGQL(":multiArch", baseURL).GlobalSearch + So(len(results.Images), ShouldEqual, 3) + So(len(results.Repos), ShouldEqual, 0) + + for _, image := range results.Images { + So(image.RepoName, ShouldEqual, repoName) + + switch image.Tag { + case "multiArchMiddle1": + So(len(image.Manifests), ShouldEqual, 4) + case "multiArchMiddle2": + So(len(image.Manifests), ShouldEqual, 2) + case "multiArchTop": + So(len(image.Manifests), ShouldEqual, 8) + } + } + }) + + Convey("test nested indexes", t, func() { + log := log.NewLogger("debug", "") + rootDir := t.TempDir() + port := GetFreePort() + baseURL := GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + conf.Storage.RootDirectory = rootDir + defaultVal := true + + updateDuration, _ := time.ParseDuration("1h") + trivyConfig := &extconf.TrivyConfig{ + DBRepository: "ghcr.io/project-zot/trivy-db", + } + cveConfig := &extconf.CVEConfig{ + UpdateInterval: updateDuration, + Trivy: trivyConfig, + } + searchConfig := &extconf.SearchConfig{ + BaseConfig: extconf.BaseConfig{Enable: &defaultVal}, + CVE: cveConfig, + } + conf.Extensions = &extconf.ExtensionConfig{ + Search: searchConfig, + } + + storeCtlr := ociutils.GetDefaultStoreController(rootDir, log) + + // nested manifest/indexes: + // image111 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop + // image112 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop + // image121 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop + // image122 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop + // image211 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop + // image212 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop + // image31 -> multiArchMiddle3 -> multiArchTop + // image32 -> multiArchMiddle3 -> multiArchTop + + repoName := "nested" + + image111 := CreateRandomImage() + image112 := CreateRandomImage() + multiArchBottom11 := CreateMultiarchWith().Images([]Image{image111, image112}).Build() + err := WriteMultiArchImageToFileSystem(multiArchBottom11, repoName, multiArchBottom11.Digest().String(), storeCtlr) + So(err, ShouldBeNil) + + image121 := CreateRandomImage() + image122 := CreateRandomImage() + multiArchBottom12 := CreateMultiarchWith().Images([]Image{image121, image122}).Build() + err = WriteMultiArchImageToFileSystem(multiArchBottom12, repoName, multiArchBottom12.Digest().String(), storeCtlr) + So(err, ShouldBeNil) + + indexMultiArchMiddle1 := ispec.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ispec.MediaTypeImageIndex, + Manifests: []ispec.Descriptor{ + { + Digest: multiArchBottom11.IndexDescriptor.Digest, + Size: multiArchBottom11.IndexDescriptor.Size, + MediaType: ispec.MediaTypeImageIndex, + }, + { + Digest: multiArchBottom12.IndexDescriptor.Digest, + Size: multiArchBottom12.IndexDescriptor.Size, + MediaType: ispec.MediaTypeImageIndex, + }, + }, + } + + indexMultiArchMiddle1Blob, err := json.Marshal(indexMultiArchMiddle1) + So(err, ShouldBeNil) + + indexMultiArchMiddle1Digest, _, err := storeCtlr.GetDefaultImageStore().PutImageManifest(repoName, + "multiArchMiddle1", ispec.MediaTypeImageIndex, indexMultiArchMiddle1Blob) + So(err, ShouldBeNil) + + image211 := CreateRandomImage() + image212 := CreateRandomImage() + multiArchBottom21 := CreateMultiarchWith().Images([]Image{image211, image212}).Build() + err = WriteMultiArchImageToFileSystem(multiArchBottom21, repoName, multiArchBottom21.Digest().String(), storeCtlr) + So(err, ShouldBeNil) + + indexMultiArchMiddle2 := ispec.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ispec.MediaTypeImageIndex, + Manifests: []ispec.Descriptor{ + { + Digest: multiArchBottom21.IndexDescriptor.Digest, + Size: multiArchBottom21.IndexDescriptor.Size, + MediaType: ispec.MediaTypeImageIndex, + }, + }, + } + + indexMultiArchMiddle2Blob, err := json.Marshal(indexMultiArchMiddle2) + So(err, ShouldBeNil) + + indexMultiArchMiddle2Digest, _, err := storeCtlr.GetDefaultImageStore().PutImageManifest(repoName, + "multiArchMiddle2", ispec.MediaTypeImageIndex, indexMultiArchMiddle2Blob) + So(err, ShouldBeNil) + + image31 := CreateRandomImage() + image32 := CreateRandomImage() + multiArchBottom3 := CreateMultiarchWith().Images([]Image{image31, image32}).Build() + err = WriteMultiArchImageToFileSystem(multiArchBottom3, repoName, multiArchBottom3.Digest().String(), storeCtlr) + So(err, ShouldBeNil) + + indexMultiArchTop := ispec.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ispec.MediaTypeImageIndex, + Manifests: []ispec.Descriptor{ + { + Digest: indexMultiArchMiddle1Digest, + Size: int64(len(indexMultiArchMiddle1Blob)), + MediaType: ispec.MediaTypeImageIndex, + }, + { + Digest: indexMultiArchMiddle2Digest, + Size: int64(len(indexMultiArchMiddle2Blob)), + MediaType: ispec.MediaTypeImageIndex, + }, + { + Digest: multiArchBottom3.IndexDescriptor.Digest, + Size: multiArchBottom3.IndexDescriptor.Size, + MediaType: ispec.MediaTypeImageIndex, + }, + }, + } + + indexMultiArchTopBlob, err := json.Marshal(indexMultiArchTop) + So(err, ShouldBeNil) + + _, _, err = storeCtlr.GetDefaultImageStore().PutImageManifest(repoName, "multiArchTop", ispec.MediaTypeImageIndex, + indexMultiArchTopBlob) + So(err, ShouldBeNil) + + ctlr := api.NewController(conf) + + if err := ctlr.Init(); err != nil { + panic(err) + } + + ctlr.CveScanner = getMockCveScanner(ctlr.MetaDB) + + go func() { + if err := ctlr.Run(); !errors.Is(err, http.ErrServerClosed) { + panic(err) + } + }() + + defer ctlr.Shutdown() + + WaitTillServerReady(baseURL) + + // Search for a specific tag cross-repo and return single arch images + results := GlobalSearchGQL(":multiArch", baseURL).GlobalSearch + So(len(results.Images), ShouldEqual, 3) + So(len(results.Repos), ShouldEqual, 0) + + for _, image := range results.Images { + So(image.RepoName, ShouldEqual, repoName) + + switch image.Tag { + case "multiArchMiddle1": + So(len(image.Manifests), ShouldEqual, 4) + case "multiArchMiddle2": + So(len(image.Manifests), ShouldEqual, 2) + case "multiArchTop": + So(len(image.Manifests), ShouldEqual, 8) + } + } + }) } func TestCleaningFilteringParamsGlobalSearch(t *testing.T) { diff --git a/pkg/meta/boltdb/boltdb.go b/pkg/meta/boltdb/boltdb.go index 22edebd9..851ee6d7 100644 --- a/pkg/meta/boltdb/boltdb.go +++ b/pkg/meta/boltdb/boltdb.go @@ -400,15 +400,9 @@ func (bdw *BoltDB) FilterImageMeta(ctx context.Context, digests []string, } if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { - manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) - - for _, manifest := range protoImageMeta.Index.Index.Manifests { - imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest) - if err != nil { - return err - } - - manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + _, manifestDataList, err := getAllContainedMeta(imageBuck, protoImageMeta) + if err != nil { + return err } protoImageMeta.Manifests = manifestDataList @@ -474,7 +468,7 @@ func getProtoImageMeta(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageM imageMetaBlob := imageBuck.Get([]byte(digest)) if len(imageMetaBlob) == 0 { - return nil, zerr.ErrImageMetaNotFound + return nil, fmt.Errorf("%w for digest %s", zerr.ErrImageMetaNotFound, digest) } imageMeta := proto_go.ImageMeta{} @@ -487,6 +481,35 @@ func getProtoImageMeta(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageM return &imageMeta, nil } +func getAllContainedMeta(imageBuck *bbolt.Bucket, imageIndexData *proto_go.ImageMeta, +) ([]*proto_go.ImageMeta, []*proto_go.ManifestMeta, error) { + manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests)) + imageMetaList := make([]*proto_go.ImageMeta, 0, len(imageIndexData.Index.Index.Manifests)) + + for _, manifest := range imageIndexData.Index.Index.Manifests { + imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest) + if err != nil { + return imageMetaList, manifestDataList, err + } + + switch imageManifestData.MediaType { + case ispec.MediaTypeImageManifest: + imageMetaList = append(imageMetaList, imageManifestData) + manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + case ispec.MediaTypeImageIndex: + partialImageDataList, partialManifestDataList, err := getAllContainedMeta(imageBuck, imageManifestData) + if err != nil { + return imageMetaList, manifestDataList, err + } + + imageMetaList = append(imageMetaList, partialImageDataList...) + manifestDataList = append(manifestDataList, partialManifestDataList...) + } + } + + return imageMetaList, manifestDataList, nil +} + func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, ) ([]mTypes.FullImageMeta, error) { images := []mTypes.FullImageMeta{} @@ -552,15 +575,9 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, indexDigest, err) } - manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests)) - - for _, manifest := range imageIndexData.Index.Index.Manifests { - imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest) - if err != nil { - return err - } - - manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + _, manifestDataList, err := getAllContainedMeta(imageBuck, imageIndexData) + if err != nil { + return err } imageIndexData.Manifests = manifestDataList @@ -649,16 +666,14 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRe imageIndexMeta := mConvert.GetImageMeta(protoImageIndexMeta) matchedManifests := []*proto_go.ManifestMeta{} - for _, manifest := range protoImageIndexMeta.Index.Index.Manifests { - manifestDigest := manifest.Digest + imageManifestDataList, _, err := getAllContainedMeta(imageMetaBuck, protoImageIndexMeta) + if err != nil { + viewError = errors.Join(viewError, err) - imageManifestData, err := getProtoImageMeta(imageMetaBuck, manifestDigest) - if err != nil { - viewError = errors.Join(viewError, err) - - continue - } + continue + } + for _, imageManifestData := range imageManifestDataList { imageMeta := mConvert.GetImageMeta(imageManifestData) partialImageMeta := common.GetPartialImageMeta(imageIndexMeta, imageMeta) @@ -798,15 +813,9 @@ func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string } if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { - manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) - - for _, manifest := range protoImageMeta.Index.Index.Manifests { - imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest) - if err != nil { - return err - } - - manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + _, manifestDataList, err := getAllContainedMeta(imageBuck, protoImageMeta) + if err != nil { + return err } protoImageMeta.Manifests = manifestDataList @@ -830,15 +839,9 @@ func (bdw *BoltDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, error } if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { - manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) - - for _, manifest := range protoImageMeta.Index.Index.Manifests { - imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest) - if err != nil { - return err - } - - manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + _, manifestDataList, err := getAllContainedMeta(imageBuck, protoImageMeta) + if err != nil { + return err } protoImageMeta.Manifests = manifestDataList diff --git a/pkg/meta/dynamodb/dynamodb.go b/pkg/meta/dynamodb/dynamodb.go index c96aa085..f375e768 100644 --- a/pkg/meta/dynamodb/dynamodb.go +++ b/pkg/meta/dynamodb/dynamodb.go @@ -224,6 +224,45 @@ func (dwr *DynamoDB) GetProtoImageMeta(ctx context.Context, digest godigest.Dige return imageMeta, nil } +func (dwr *DynamoDB) getAllContainedMeta(ctx context.Context, imageIndexData *proto_go.ImageMeta, +) ([]*proto_go.ImageMeta, []*proto_go.ManifestMeta, error) { + manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests)) + imageMetaList := make([]*proto_go.ImageMeta, 0, len(imageIndexData.Index.Index.Manifests)) + + manifestDigests := make([]string, 0, len(imageIndexData.Index.Index.Manifests)) + for i := range imageIndexData.Index.Index.Manifests { + manifestDigests = append(manifestDigests, imageIndexData.Index.Index.Manifests[i].Digest) + } + + manifestsAttributes, err := dwr.fetchImageMetaAttributesByDigest(ctx, manifestDigests) + if err != nil { + return imageMetaList, manifestDataList, err + } + + for _, manifestAttribute := range manifestsAttributes { + imageManifestData, err := getProtoImageMetaFromAttribute(manifestAttribute["ImageMeta"]) + if err != nil { + return imageMetaList, manifestDataList, err + } + + switch imageManifestData.MediaType { + case ispec.MediaTypeImageManifest: + imageMetaList = append(imageMetaList, imageManifestData) + manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + case ispec.MediaTypeImageIndex: + partialImageDataList, partialManifestDataList, err := dwr.getAllContainedMeta(ctx, imageManifestData) + if err != nil { + return imageMetaList, manifestDataList, err + } + + imageMetaList = append(imageMetaList, partialImageDataList...) + manifestDataList = append(manifestDataList, partialManifestDataList...) + } + } + + return imageMetaList, manifestDataList, nil +} + func (dwr *DynamoDB) setProtoRepoMeta(repo string, repoMeta *proto_go.RepoMeta) error { repoMeta.Name = repo @@ -640,17 +679,9 @@ func (dwr *DynamoDB) SearchTags(ctx context.Context, searchText string) ([]mType fmt.Errorf("error fetching manifest meta for manifest with digest %s %w", indexDigest, err) } - manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests)) - - for _, manifest := range imageIndexData.Index.Index.Manifests { - manifestDigest := godigest.Digest(manifest.Digest) - - imageManifestData, err := dwr.GetProtoImageMeta(ctx, manifestDigest) - if err != nil { - return []mTypes.FullImageMeta{}, err - } - - manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + _, manifestDataList, err := dwr.getAllContainedMeta(ctx, imageIndexData) + if err != nil { + return []mTypes.FullImageMeta{}, err } imageIndexData.Manifests = manifestDataList @@ -739,16 +770,14 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterRepoTag mTypes.Filter imageIndexMeta := mConvert.GetImageMeta(protoImageIndexMeta) matchedManifests := []*proto_go.ManifestMeta{} - for _, manifest := range protoImageIndexMeta.Index.Index.Manifests { - manifestDigest := manifest.Digest + imageManifestDataList, _, err := dwr.getAllContainedMeta(context.Background(), protoImageIndexMeta) + if err != nil { + viewError = errors.Join(viewError, err) - imageManifestData, err := dwr.GetProtoImageMeta(ctx, godigest.Digest(manifestDigest)) - if err != nil { - viewError = errors.Join(viewError, err) - - continue - } + continue + } + for _, imageManifestData := range imageManifestDataList { imageMeta := mConvert.GetImageMeta(imageManifestData) partialImageMeta := common.GetPartialImageMeta(imageIndexMeta, imageMeta) @@ -868,15 +897,9 @@ func (dwr *DynamoDB) GetFullImageMeta(ctx context.Context, repo string, tag stri } if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { - manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) - - for _, manifest := range protoImageMeta.Index.Index.Manifests { - imageManifestData, err := dwr.GetProtoImageMeta(ctx, godigest.Digest(manifest.Digest)) - if err != nil { - return mTypes.FullImageMeta{}, err - } - - manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + _, manifestDataList, err := dwr.getAllContainedMeta(ctx, protoImageMeta) + if err != nil { + return mTypes.FullImageMeta{}, err } protoImageMeta.Manifests = manifestDataList @@ -901,17 +924,9 @@ func (dwr *DynamoDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, err } if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { - manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) - - for _, manifest := range protoImageMeta.Index.Index.Manifests { - manifestDigest := godigest.Digest(manifest.Digest) - - imageManifestData, err := dwr.GetProtoImageMeta(context.Background(), manifestDigest) - if err != nil { - return mTypes.ImageMeta{}, err - } - - manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + _, manifestDataList, err := dwr.getAllContainedMeta(context.Background(), protoImageMeta) + if err != nil { + return mTypes.ImageMeta{}, err } protoImageMeta.Manifests = manifestDataList @@ -1330,27 +1345,11 @@ func (dwr *DynamoDB) FilterImageMeta(ctx context.Context, digests []string, } if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { - manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) - - indexDigests := make([]string, 0, len(protoImageMeta.Index.Index.Manifests)) - for i := range protoImageMeta.Index.Index.Manifests { - indexDigests = append(indexDigests, protoImageMeta.Index.Index.Manifests[i].Digest) - } - - manifestsAttributes, err := dwr.fetchImageMetaAttributesByDigest(ctx, indexDigests) + _, manifestDataList, err := dwr.getAllContainedMeta(context.Background(), protoImageMeta) if err != nil { return nil, err } - for _, manifestAttribute := range manifestsAttributes { - imageManifestData, err := getProtoImageMetaFromAttribute(manifestAttribute["ImageMeta"]) - if err != nil { - return nil, err - } - - manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) - } - protoImageMeta.Manifests = manifestDataList } @@ -2001,7 +2000,32 @@ func (dwr *DynamoDB) fetchImageMetaAttributesByDigest(ctx context.Context, diges start = end } - return batchedResp, nil + // Order the responses based on initial digest order + // as BatchGetItem does not guarantee the key order is respected + orderedResp := []map[string]types.AttributeValue{} + respMap := map[string]map[string]types.AttributeValue{} + + for _, item := range batchedResp { + var digest string + + err := attributevalue.Unmarshal(item["TableKey"], &digest) + if err != nil { + return nil, err + } + + respMap[digest] = item + } + + for _, digest := range digests { + imageMeta, ok := respMap[digest] + if !ok { + return nil, fmt.Errorf("%w for digest %s", zerr.ErrImageMetaNotFound, digest) + } + + orderedResp = append(orderedResp, imageMeta) + } + + return orderedResp, nil } func getBatchImageKeys(digests []string) []map[string]types.AttributeValue {