mirror of
https://github.com/project-zot/zot.git
synced 2024-12-16 21:56:37 -05:00
fix: issues with nested index processing in CVE and metaDB code (#2732)
Also fix an issue with searching tags, which should work with case insensitive searches. Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
This commit is contained in:
Andrei Aaron
GitHub
parent
edb5491428
commit
da6bd56a21
5 changed files with 456 additions and 105 deletions
|
|
@ -1350,7 +1350,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo
|
|||
|
||||
// Tag is not found
|
||||
cveList, cveSummary, pageInfo, err = cveInfo.GetCVEListForImage(ctx, "repo-with-bad-tag-digest", "tag", "", "", "", pageInput)
|
||||
So(err, ShouldEqual, zerr.ErrImageMetaNotFound)
|
||||
So(err, ShouldWrap, zerr.ErrImageMetaNotFound)
|
||||
So(len(cveList), ShouldEqual, 0)
|
||||
So(pageInfo.ItemCount, ShouldEqual, 0)
|
||||
So(pageInfo.TotalCount, ShouldEqual, 0)
|
||||
|
|
|
|||
|
|
@ -960,7 +960,9 @@ func globalSearch(ctx context.Context, query string, metaDB mTypes.MetaDB, filte
|
|||
pageInput := getPageInput(requestedPage)
|
||||
|
||||
expectedTag := strings.TrimPrefix(query, `:`)
|
||||
matchTagName := func(repoName, actualTag string) bool { return strings.Contains(actualTag, expectedTag) }
|
||||
matchTagName := func(repoName, actualTag string) bool {
|
||||
return strings.Contains(strings.ToLower(actualTag), expectedTag)
|
||||
}
|
||||
|
||||
fullImageMetaList, err := metaDB.FilterTags(ctx, matchTagName, mTypes.AcceptAllImageMeta)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -3005,7 +3005,7 @@ func TestGlobalSearchImageAuthor(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestGlobalSearch(t *testing.T) {
|
||||
func TestGlobalSearch(t *testing.T) { //nolint: gocyclo
|
||||
Convey("Test searching for repos with vulnerabitity scanning disabled", t, func() {
|
||||
subpath := "/a"
|
||||
|
||||
|
|
@ -3904,6 +3904,328 @@ func TestGlobalSearch(t *testing.T) {
|
|||
So(len(results.Images), ShouldEqual, 0)
|
||||
So(len(results.Repos), ShouldEqual, 0)
|
||||
})
|
||||
|
||||
Convey("test nested indexes", t, func() {
|
||||
log := log.NewLogger("debug", "")
|
||||
rootDir := t.TempDir()
|
||||
port := GetFreePort()
|
||||
baseURL := GetBaseURL(port)
|
||||
conf := config.New()
|
||||
conf.HTTP.Port = port
|
||||
conf.Storage.RootDirectory = rootDir
|
||||
defaultVal := true
|
||||
conf.Extensions = &extconf.ExtensionConfig{
|
||||
Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}},
|
||||
}
|
||||
conf.Extensions.Search.CVE = nil
|
||||
|
||||
ctlr := api.NewController(conf)
|
||||
ctlrManager := NewControllerManager(ctlr)
|
||||
|
||||
storeCtlr := ociutils.GetDefaultStoreController(rootDir, log)
|
||||
|
||||
// nested manifest/indexes:
|
||||
// image111 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop
|
||||
// image112 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop
|
||||
// image121 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop
|
||||
// image122 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop
|
||||
// image211 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop
|
||||
// image212 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop
|
||||
// image31 -> multiArchMiddle3 -> multiArchTop
|
||||
// image32 -> multiArchMiddle3 -> multiArchTop
|
||||
|
||||
repoName := "nested"
|
||||
|
||||
image111 := CreateRandomImage()
|
||||
image112 := CreateRandomImage()
|
||||
multiArchBottom11 := CreateMultiarchWith().Images([]Image{image111, image112}).Build()
|
||||
err := WriteMultiArchImageToFileSystem(multiArchBottom11, repoName, multiArchBottom11.Digest().String(), storeCtlr)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
image121 := CreateRandomImage()
|
||||
image122 := CreateRandomImage()
|
||||
multiArchBottom12 := CreateMultiarchWith().Images([]Image{image121, image122}).Build()
|
||||
err = WriteMultiArchImageToFileSystem(multiArchBottom12, repoName, multiArchBottom12.Digest().String(), storeCtlr)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
indexMultiArchMiddle1 := ispec.Index{
|
||||
Versioned: specs.Versioned{SchemaVersion: 2},
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
Manifests: []ispec.Descriptor{
|
||||
{
|
||||
Digest: multiArchBottom11.IndexDescriptor.Digest,
|
||||
Size: multiArchBottom11.IndexDescriptor.Size,
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
},
|
||||
{
|
||||
Digest: multiArchBottom12.IndexDescriptor.Digest,
|
||||
Size: multiArchBottom12.IndexDescriptor.Size,
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
indexMultiArchMiddle1Blob, err := json.Marshal(indexMultiArchMiddle1)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
indexMultiArchMiddle1Digest, _, err := storeCtlr.GetDefaultImageStore().PutImageManifest(repoName,
|
||||
"multiArchMiddle1", ispec.MediaTypeImageIndex, indexMultiArchMiddle1Blob)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
image211 := CreateRandomImage()
|
||||
image212 := CreateRandomImage()
|
||||
multiArchBottom21 := CreateMultiarchWith().Images([]Image{image211, image212}).Build()
|
||||
err = WriteMultiArchImageToFileSystem(multiArchBottom21, repoName, multiArchBottom21.Digest().String(), storeCtlr)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
indexMultiArchMiddle2 := ispec.Index{
|
||||
Versioned: specs.Versioned{SchemaVersion: 2},
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
Manifests: []ispec.Descriptor{
|
||||
{
|
||||
Digest: multiArchBottom21.IndexDescriptor.Digest,
|
||||
Size: multiArchBottom21.IndexDescriptor.Size,
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
indexMultiArchMiddle2Blob, err := json.Marshal(indexMultiArchMiddle2)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
indexMultiArchMiddle2Digest, _, err := storeCtlr.GetDefaultImageStore().PutImageManifest(repoName,
|
||||
"multiArchMiddle2", ispec.MediaTypeImageIndex, indexMultiArchMiddle2Blob)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
image31 := CreateRandomImage()
|
||||
image32 := CreateRandomImage()
|
||||
multiArchBottom3 := CreateMultiarchWith().Images([]Image{image31, image32}).Build()
|
||||
err = WriteMultiArchImageToFileSystem(multiArchBottom3, repoName, multiArchBottom3.Digest().String(), storeCtlr)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
indexMultiArchTop := ispec.Index{
|
||||
Versioned: specs.Versioned{SchemaVersion: 2},
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
Manifests: []ispec.Descriptor{
|
||||
{
|
||||
Digest: indexMultiArchMiddle1Digest,
|
||||
Size: int64(len(indexMultiArchMiddle1Blob)),
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
},
|
||||
{
|
||||
Digest: indexMultiArchMiddle2Digest,
|
||||
Size: int64(len(indexMultiArchMiddle2Blob)),
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
},
|
||||
{
|
||||
Digest: multiArchBottom3.IndexDescriptor.Digest,
|
||||
Size: multiArchBottom3.IndexDescriptor.Size,
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
indexMultiArchTopBlob, err := json.Marshal(indexMultiArchTop)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, _, err = storeCtlr.GetDefaultImageStore().PutImageManifest(repoName, "multiArchTop", ispec.MediaTypeImageIndex,
|
||||
indexMultiArchTopBlob)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
ctlrManager.StartAndWait(port)
|
||||
defer ctlrManager.StopServer()
|
||||
|
||||
// Search for a specific tag cross-repo and return single arch images
|
||||
results := GlobalSearchGQL(":multiArch", baseURL).GlobalSearch
|
||||
So(len(results.Images), ShouldEqual, 3)
|
||||
So(len(results.Repos), ShouldEqual, 0)
|
||||
|
||||
for _, image := range results.Images {
|
||||
So(image.RepoName, ShouldEqual, repoName)
|
||||
|
||||
switch image.Tag {
|
||||
case "multiArchMiddle1":
|
||||
So(len(image.Manifests), ShouldEqual, 4)
|
||||
case "multiArchMiddle2":
|
||||
So(len(image.Manifests), ShouldEqual, 2)
|
||||
case "multiArchTop":
|
||||
So(len(image.Manifests), ShouldEqual, 8)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
Convey("test nested indexes", t, func() {
|
||||
log := log.NewLogger("debug", "")
|
||||
rootDir := t.TempDir()
|
||||
port := GetFreePort()
|
||||
baseURL := GetBaseURL(port)
|
||||
conf := config.New()
|
||||
conf.HTTP.Port = port
|
||||
conf.Storage.RootDirectory = rootDir
|
||||
defaultVal := true
|
||||
|
||||
updateDuration, _ := time.ParseDuration("1h")
|
||||
trivyConfig := &extconf.TrivyConfig{
|
||||
DBRepository: "ghcr.io/project-zot/trivy-db",
|
||||
}
|
||||
cveConfig := &extconf.CVEConfig{
|
||||
UpdateInterval: updateDuration,
|
||||
Trivy: trivyConfig,
|
||||
}
|
||||
searchConfig := &extconf.SearchConfig{
|
||||
BaseConfig: extconf.BaseConfig{Enable: &defaultVal},
|
||||
CVE: cveConfig,
|
||||
}
|
||||
conf.Extensions = &extconf.ExtensionConfig{
|
||||
Search: searchConfig,
|
||||
}
|
||||
|
||||
storeCtlr := ociutils.GetDefaultStoreController(rootDir, log)
|
||||
|
||||
// nested manifest/indexes:
|
||||
// image111 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop
|
||||
// image112 -> multiArchBottom11 -> multiArchMiddle1 -> multiArchTop
|
||||
// image121 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop
|
||||
// image122 -> multiArchBottom12 -> multiArchMiddle1 -> multiArchTop
|
||||
// image211 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop
|
||||
// image212 -> multiArchBottom21 -> multiArchMiddle2 -> multiArchTop
|
||||
// image31 -> multiArchMiddle3 -> multiArchTop
|
||||
// image32 -> multiArchMiddle3 -> multiArchTop
|
||||
|
||||
repoName := "nested"
|
||||
|
||||
image111 := CreateRandomImage()
|
||||
image112 := CreateRandomImage()
|
||||
multiArchBottom11 := CreateMultiarchWith().Images([]Image{image111, image112}).Build()
|
||||
err := WriteMultiArchImageToFileSystem(multiArchBottom11, repoName, multiArchBottom11.Digest().String(), storeCtlr)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
image121 := CreateRandomImage()
|
||||
image122 := CreateRandomImage()
|
||||
multiArchBottom12 := CreateMultiarchWith().Images([]Image{image121, image122}).Build()
|
||||
err = WriteMultiArchImageToFileSystem(multiArchBottom12, repoName, multiArchBottom12.Digest().String(), storeCtlr)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
indexMultiArchMiddle1 := ispec.Index{
|
||||
Versioned: specs.Versioned{SchemaVersion: 2},
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
Manifests: []ispec.Descriptor{
|
||||
{
|
||||
Digest: multiArchBottom11.IndexDescriptor.Digest,
|
||||
Size: multiArchBottom11.IndexDescriptor.Size,
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
},
|
||||
{
|
||||
Digest: multiArchBottom12.IndexDescriptor.Digest,
|
||||
Size: multiArchBottom12.IndexDescriptor.Size,
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
indexMultiArchMiddle1Blob, err := json.Marshal(indexMultiArchMiddle1)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
indexMultiArchMiddle1Digest, _, err := storeCtlr.GetDefaultImageStore().PutImageManifest(repoName,
|
||||
"multiArchMiddle1", ispec.MediaTypeImageIndex, indexMultiArchMiddle1Blob)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
image211 := CreateRandomImage()
|
||||
image212 := CreateRandomImage()
|
||||
multiArchBottom21 := CreateMultiarchWith().Images([]Image{image211, image212}).Build()
|
||||
err = WriteMultiArchImageToFileSystem(multiArchBottom21, repoName, multiArchBottom21.Digest().String(), storeCtlr)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
indexMultiArchMiddle2 := ispec.Index{
|
||||
Versioned: specs.Versioned{SchemaVersion: 2},
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
Manifests: []ispec.Descriptor{
|
||||
{
|
||||
Digest: multiArchBottom21.IndexDescriptor.Digest,
|
||||
Size: multiArchBottom21.IndexDescriptor.Size,
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
indexMultiArchMiddle2Blob, err := json.Marshal(indexMultiArchMiddle2)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
indexMultiArchMiddle2Digest, _, err := storeCtlr.GetDefaultImageStore().PutImageManifest(repoName,
|
||||
"multiArchMiddle2", ispec.MediaTypeImageIndex, indexMultiArchMiddle2Blob)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
image31 := CreateRandomImage()
|
||||
image32 := CreateRandomImage()
|
||||
multiArchBottom3 := CreateMultiarchWith().Images([]Image{image31, image32}).Build()
|
||||
err = WriteMultiArchImageToFileSystem(multiArchBottom3, repoName, multiArchBottom3.Digest().String(), storeCtlr)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
indexMultiArchTop := ispec.Index{
|
||||
Versioned: specs.Versioned{SchemaVersion: 2},
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
Manifests: []ispec.Descriptor{
|
||||
{
|
||||
Digest: indexMultiArchMiddle1Digest,
|
||||
Size: int64(len(indexMultiArchMiddle1Blob)),
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
},
|
||||
{
|
||||
Digest: indexMultiArchMiddle2Digest,
|
||||
Size: int64(len(indexMultiArchMiddle2Blob)),
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
},
|
||||
{
|
||||
Digest: multiArchBottom3.IndexDescriptor.Digest,
|
||||
Size: multiArchBottom3.IndexDescriptor.Size,
|
||||
MediaType: ispec.MediaTypeImageIndex,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
indexMultiArchTopBlob, err := json.Marshal(indexMultiArchTop)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, _, err = storeCtlr.GetDefaultImageStore().PutImageManifest(repoName, "multiArchTop", ispec.MediaTypeImageIndex,
|
||||
indexMultiArchTopBlob)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
ctlr := api.NewController(conf)
|
||||
|
||||
if err := ctlr.Init(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
ctlr.CveScanner = getMockCveScanner(ctlr.MetaDB)
|
||||
|
||||
go func() {
|
||||
if err := ctlr.Run(); !errors.Is(err, http.ErrServerClosed) {
|
||||
panic(err)
|
||||
}
|
||||
}()
|
||||
|
||||
defer ctlr.Shutdown()
|
||||
|
||||
WaitTillServerReady(baseURL)
|
||||
|
||||
// Search for a specific tag cross-repo and return single arch images
|
||||
results := GlobalSearchGQL(":multiArch", baseURL).GlobalSearch
|
||||
So(len(results.Images), ShouldEqual, 3)
|
||||
So(len(results.Repos), ShouldEqual, 0)
|
||||
|
||||
for _, image := range results.Images {
|
||||
So(image.RepoName, ShouldEqual, repoName)
|
||||
|
||||
switch image.Tag {
|
||||
case "multiArchMiddle1":
|
||||
So(len(image.Manifests), ShouldEqual, 4)
|
||||
case "multiArchMiddle2":
|
||||
So(len(image.Manifests), ShouldEqual, 2)
|
||||
case "multiArchTop":
|
||||
So(len(image.Manifests), ShouldEqual, 8)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCleaningFilteringParamsGlobalSearch(t *testing.T) {
|
||||
|
|
|
|||
|
|
@ -400,15 +400,9 @@ func (bdw *BoltDB) FilterImageMeta(ctx context.Context, digests []string,
|
|||
}
|
||||
|
||||
if protoImageMeta.MediaType == ispec.MediaTypeImageIndex {
|
||||
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
|
||||
|
||||
for _, manifest := range protoImageMeta.Index.Index.Manifests {
|
||||
imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
|
||||
_, manifestDataList, err := getAllContainedMeta(imageBuck, protoImageMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
protoImageMeta.Manifests = manifestDataList
|
||||
|
|
@ -474,7 +468,7 @@ func getProtoImageMeta(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageM
|
|||
imageMetaBlob := imageBuck.Get([]byte(digest))
|
||||
|
||||
if len(imageMetaBlob) == 0 {
|
||||
return nil, zerr.ErrImageMetaNotFound
|
||||
return nil, fmt.Errorf("%w for digest %s", zerr.ErrImageMetaNotFound, digest)
|
||||
}
|
||||
|
||||
imageMeta := proto_go.ImageMeta{}
|
||||
|
|
@ -487,6 +481,35 @@ func getProtoImageMeta(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageM
|
|||
return &imageMeta, nil
|
||||
}
|
||||
|
||||
func getAllContainedMeta(imageBuck *bbolt.Bucket, imageIndexData *proto_go.ImageMeta,
|
||||
) ([]*proto_go.ImageMeta, []*proto_go.ManifestMeta, error) {
|
||||
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests))
|
||||
imageMetaList := make([]*proto_go.ImageMeta, 0, len(imageIndexData.Index.Index.Manifests))
|
||||
|
||||
for _, manifest := range imageIndexData.Index.Index.Manifests {
|
||||
imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
|
||||
if err != nil {
|
||||
return imageMetaList, manifestDataList, err
|
||||
}
|
||||
|
||||
switch imageManifestData.MediaType {
|
||||
case ispec.MediaTypeImageManifest:
|
||||
imageMetaList = append(imageMetaList, imageManifestData)
|
||||
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
|
||||
case ispec.MediaTypeImageIndex:
|
||||
partialImageDataList, partialManifestDataList, err := getAllContainedMeta(imageBuck, imageManifestData)
|
||||
if err != nil {
|
||||
return imageMetaList, manifestDataList, err
|
||||
}
|
||||
|
||||
imageMetaList = append(imageMetaList, partialImageDataList...)
|
||||
manifestDataList = append(manifestDataList, partialManifestDataList...)
|
||||
}
|
||||
}
|
||||
|
||||
return imageMetaList, manifestDataList, nil
|
||||
}
|
||||
|
||||
func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string,
|
||||
) ([]mTypes.FullImageMeta, error) {
|
||||
images := []mTypes.FullImageMeta{}
|
||||
|
|
@ -552,15 +575,9 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string,
|
|||
indexDigest, err)
|
||||
}
|
||||
|
||||
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests))
|
||||
|
||||
for _, manifest := range imageIndexData.Index.Index.Manifests {
|
||||
imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
|
||||
_, manifestDataList, err := getAllContainedMeta(imageBuck, imageIndexData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
imageIndexData.Manifests = manifestDataList
|
||||
|
|
@ -649,16 +666,14 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRe
|
|||
imageIndexMeta := mConvert.GetImageMeta(protoImageIndexMeta)
|
||||
matchedManifests := []*proto_go.ManifestMeta{}
|
||||
|
||||
for _, manifest := range protoImageIndexMeta.Index.Index.Manifests {
|
||||
manifestDigest := manifest.Digest
|
||||
imageManifestDataList, _, err := getAllContainedMeta(imageMetaBuck, protoImageIndexMeta)
|
||||
if err != nil {
|
||||
viewError = errors.Join(viewError, err)
|
||||
|
||||
imageManifestData, err := getProtoImageMeta(imageMetaBuck, manifestDigest)
|
||||
if err != nil {
|
||||
viewError = errors.Join(viewError, err)
|
||||
|
||||
continue
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
for _, imageManifestData := range imageManifestDataList {
|
||||
imageMeta := mConvert.GetImageMeta(imageManifestData)
|
||||
partialImageMeta := common.GetPartialImageMeta(imageIndexMeta, imageMeta)
|
||||
|
||||
|
|
@ -798,15 +813,9 @@ func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string
|
|||
}
|
||||
|
||||
if protoImageMeta.MediaType == ispec.MediaTypeImageIndex {
|
||||
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
|
||||
|
||||
for _, manifest := range protoImageMeta.Index.Index.Manifests {
|
||||
imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
|
||||
_, manifestDataList, err := getAllContainedMeta(imageBuck, protoImageMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
protoImageMeta.Manifests = manifestDataList
|
||||
|
|
@ -830,15 +839,9 @@ func (bdw *BoltDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, error
|
|||
}
|
||||
|
||||
if protoImageMeta.MediaType == ispec.MediaTypeImageIndex {
|
||||
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
|
||||
|
||||
for _, manifest := range protoImageMeta.Index.Index.Manifests {
|
||||
imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
|
||||
_, manifestDataList, err := getAllContainedMeta(imageBuck, protoImageMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
protoImageMeta.Manifests = manifestDataList
|
||||
|
|
|
|||
|
|
@ -224,6 +224,45 @@ func (dwr *DynamoDB) GetProtoImageMeta(ctx context.Context, digest godigest.Dige
|
|||
return imageMeta, nil
|
||||
}
|
||||
|
||||
func (dwr *DynamoDB) getAllContainedMeta(ctx context.Context, imageIndexData *proto_go.ImageMeta,
|
||||
) ([]*proto_go.ImageMeta, []*proto_go.ManifestMeta, error) {
|
||||
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests))
|
||||
imageMetaList := make([]*proto_go.ImageMeta, 0, len(imageIndexData.Index.Index.Manifests))
|
||||
|
||||
manifestDigests := make([]string, 0, len(imageIndexData.Index.Index.Manifests))
|
||||
for i := range imageIndexData.Index.Index.Manifests {
|
||||
manifestDigests = append(manifestDigests, imageIndexData.Index.Index.Manifests[i].Digest)
|
||||
}
|
||||
|
||||
manifestsAttributes, err := dwr.fetchImageMetaAttributesByDigest(ctx, manifestDigests)
|
||||
if err != nil {
|
||||
return imageMetaList, manifestDataList, err
|
||||
}
|
||||
|
||||
for _, manifestAttribute := range manifestsAttributes {
|
||||
imageManifestData, err := getProtoImageMetaFromAttribute(manifestAttribute["ImageMeta"])
|
||||
if err != nil {
|
||||
return imageMetaList, manifestDataList, err
|
||||
}
|
||||
|
||||
switch imageManifestData.MediaType {
|
||||
case ispec.MediaTypeImageManifest:
|
||||
imageMetaList = append(imageMetaList, imageManifestData)
|
||||
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
|
||||
case ispec.MediaTypeImageIndex:
|
||||
partialImageDataList, partialManifestDataList, err := dwr.getAllContainedMeta(ctx, imageManifestData)
|
||||
if err != nil {
|
||||
return imageMetaList, manifestDataList, err
|
||||
}
|
||||
|
||||
imageMetaList = append(imageMetaList, partialImageDataList...)
|
||||
manifestDataList = append(manifestDataList, partialManifestDataList...)
|
||||
}
|
||||
}
|
||||
|
||||
return imageMetaList, manifestDataList, nil
|
||||
}
|
||||
|
||||
func (dwr *DynamoDB) setProtoRepoMeta(repo string, repoMeta *proto_go.RepoMeta) error {
|
||||
repoMeta.Name = repo
|
||||
|
||||
|
|
@ -640,17 +679,9 @@ func (dwr *DynamoDB) SearchTags(ctx context.Context, searchText string) ([]mType
|
|||
fmt.Errorf("error fetching manifest meta for manifest with digest %s %w", indexDigest, err)
|
||||
}
|
||||
|
||||
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests))
|
||||
|
||||
for _, manifest := range imageIndexData.Index.Index.Manifests {
|
||||
manifestDigest := godigest.Digest(manifest.Digest)
|
||||
|
||||
imageManifestData, err := dwr.GetProtoImageMeta(ctx, manifestDigest)
|
||||
if err != nil {
|
||||
return []mTypes.FullImageMeta{}, err
|
||||
}
|
||||
|
||||
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
|
||||
_, manifestDataList, err := dwr.getAllContainedMeta(ctx, imageIndexData)
|
||||
if err != nil {
|
||||
return []mTypes.FullImageMeta{}, err
|
||||
}
|
||||
|
||||
imageIndexData.Manifests = manifestDataList
|
||||
|
|
@ -739,16 +770,14 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterRepoTag mTypes.Filter
|
|||
imageIndexMeta := mConvert.GetImageMeta(protoImageIndexMeta)
|
||||
matchedManifests := []*proto_go.ManifestMeta{}
|
||||
|
||||
for _, manifest := range protoImageIndexMeta.Index.Index.Manifests {
|
||||
manifestDigest := manifest.Digest
|
||||
imageManifestDataList, _, err := dwr.getAllContainedMeta(context.Background(), protoImageIndexMeta)
|
||||
if err != nil {
|
||||
viewError = errors.Join(viewError, err)
|
||||
|
||||
imageManifestData, err := dwr.GetProtoImageMeta(ctx, godigest.Digest(manifestDigest))
|
||||
if err != nil {
|
||||
viewError = errors.Join(viewError, err)
|
||||
|
||||
continue
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
for _, imageManifestData := range imageManifestDataList {
|
||||
imageMeta := mConvert.GetImageMeta(imageManifestData)
|
||||
partialImageMeta := common.GetPartialImageMeta(imageIndexMeta, imageMeta)
|
||||
|
||||
|
|
@ -868,15 +897,9 @@ func (dwr *DynamoDB) GetFullImageMeta(ctx context.Context, repo string, tag stri
|
|||
}
|
||||
|
||||
if protoImageMeta.MediaType == ispec.MediaTypeImageIndex {
|
||||
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
|
||||
|
||||
for _, manifest := range protoImageMeta.Index.Index.Manifests {
|
||||
imageManifestData, err := dwr.GetProtoImageMeta(ctx, godigest.Digest(manifest.Digest))
|
||||
if err != nil {
|
||||
return mTypes.FullImageMeta{}, err
|
||||
}
|
||||
|
||||
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
|
||||
_, manifestDataList, err := dwr.getAllContainedMeta(ctx, protoImageMeta)
|
||||
if err != nil {
|
||||
return mTypes.FullImageMeta{}, err
|
||||
}
|
||||
|
||||
protoImageMeta.Manifests = manifestDataList
|
||||
|
|
@ -901,17 +924,9 @@ func (dwr *DynamoDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, err
|
|||
}
|
||||
|
||||
if protoImageMeta.MediaType == ispec.MediaTypeImageIndex {
|
||||
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
|
||||
|
||||
for _, manifest := range protoImageMeta.Index.Index.Manifests {
|
||||
manifestDigest := godigest.Digest(manifest.Digest)
|
||||
|
||||
imageManifestData, err := dwr.GetProtoImageMeta(context.Background(), manifestDigest)
|
||||
if err != nil {
|
||||
return mTypes.ImageMeta{}, err
|
||||
}
|
||||
|
||||
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
|
||||
_, manifestDataList, err := dwr.getAllContainedMeta(context.Background(), protoImageMeta)
|
||||
if err != nil {
|
||||
return mTypes.ImageMeta{}, err
|
||||
}
|
||||
|
||||
protoImageMeta.Manifests = manifestDataList
|
||||
|
|
@ -1330,27 +1345,11 @@ func (dwr *DynamoDB) FilterImageMeta(ctx context.Context, digests []string,
|
|||
}
|
||||
|
||||
if protoImageMeta.MediaType == ispec.MediaTypeImageIndex {
|
||||
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
|
||||
|
||||
indexDigests := make([]string, 0, len(protoImageMeta.Index.Index.Manifests))
|
||||
for i := range protoImageMeta.Index.Index.Manifests {
|
||||
indexDigests = append(indexDigests, protoImageMeta.Index.Index.Manifests[i].Digest)
|
||||
}
|
||||
|
||||
manifestsAttributes, err := dwr.fetchImageMetaAttributesByDigest(ctx, indexDigests)
|
||||
_, manifestDataList, err := dwr.getAllContainedMeta(context.Background(), protoImageMeta)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, manifestAttribute := range manifestsAttributes {
|
||||
imageManifestData, err := getProtoImageMetaFromAttribute(manifestAttribute["ImageMeta"])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
manifestDataList = append(manifestDataList, imageManifestData.Manifests[0])
|
||||
}
|
||||
|
||||
protoImageMeta.Manifests = manifestDataList
|
||||
}
|
||||
|
||||
|
|
@ -2001,7 +2000,32 @@ func (dwr *DynamoDB) fetchImageMetaAttributesByDigest(ctx context.Context, diges
|
|||
start = end
|
||||
}
|
||||
|
||||
return batchedResp, nil
|
||||
// Order the responses based on initial digest order
|
||||
// as BatchGetItem does not guarantee the key order is respected
|
||||
orderedResp := []map[string]types.AttributeValue{}
|
||||
respMap := map[string]map[string]types.AttributeValue{}
|
||||
|
||||
for _, item := range batchedResp {
|
||||
var digest string
|
||||
|
||||
err := attributevalue.Unmarshal(item["TableKey"], &digest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
respMap[digest] = item
|
||||
}
|
||||
|
||||
for _, digest := range digests {
|
||||
imageMeta, ok := respMap[digest]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("%w for digest %s", zerr.ErrImageMetaNotFound, digest)
|
||||
}
|
||||
|
||||
orderedResp = append(orderedResp, imageMeta)
|
||||
}
|
||||
|
||||
return orderedResp, nil
|
||||
}
|
||||
|
||||
func getBatchImageKeys(digests []string) []map[string]types.AttributeValue {
|
||||
|
|
|
|||
Loading…
Reference in a new issue