0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2025-03-11 02:17:43 -05:00

feat(pagination): move pagination and sorting image summary results after conversion (#1637)

fix(config): check for config media type when pushing to repodb

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae 2023-07-31 22:16:09 +03:00 committed by GitHub
parent 20391a21c0
commit 9e38ca51e3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
39 changed files with 2361 additions and 3182 deletions

View file

@ -21,9 +21,9 @@ import (
)
func ref[T any](input T) *T {
obj := input
ref := input
return &obj
return &ref
}
const (
@ -46,7 +46,7 @@ func TestReferrersSearchers(t *testing.T) {
So(ok, ShouldBeFalse)
})
Convey("GetRepoRefference fails", func() {
Convey("GetRepoReference fails", func() {
conf := searchConfig{
params: map[string]*string{
"subject": ref("bad-subject"),
@ -119,7 +119,7 @@ func TestReferrersSearchers(t *testing.T) {
So(ok, ShouldBeFalse)
})
Convey("GetRepoRefference fails", func() {
Convey("GetRepoReference fails", func() {
conf := searchConfig{
params: map[string]*string{
"subject": ref("bad-subject"),

View file

@ -636,7 +636,7 @@ func (search referrerSearcherGQL) search(config searchConfig) (bool, error) {
username, password := getUsernameAndPassword(*config.user)
repo, ref, refIsTag, err := zcommon.GetRepoRefference(*config.params["subject"])
repo, ref, refIsTag, err := zcommon.GetRepoReference(*config.params["subject"])
if err != nil {
return true, err
}
@ -692,7 +692,7 @@ func (search referrerSearcher) search(config searchConfig) (bool, error) {
username, password := getUsernameAndPassword(*config.user)
repo, ref, refIsTag, err := zcommon.GetRepoRefference(*config.params["subject"])
repo, ref, refIsTag, err := zcommon.GetRepoReference(*config.params["subject"])
if err != nil {
return true, err
}

View file

@ -6,6 +6,7 @@ import (
"fmt"
"io/fs"
"os"
"strings"
"syscall"
"time"
"unicode/utf8"
@ -101,3 +102,13 @@ func MarshalThroughStruct(obj interface{}, throughStruct interface{}) ([]byte, e
return toJSON, nil
}
func ContainsStringIgnoreCase(strSlice []string, str string) bool {
for _, val := range strSlice {
if strings.EqualFold(val, str) {
return true
}
}
return false
}

View file

@ -84,7 +84,7 @@ func GetImageLastUpdated(imageInfo ispec.Image) time.Time {
return *timeStamp
}
// GetRepoRefference returns the components of a repoName:tag or repoName@digest string. If the format is wrong
// GetRepoReference returns the components of a repoName:tag or repoName@digest string. If the format is wrong
// an error is returned.
// The returned values have the following meaning:
//
@ -95,7 +95,7 @@ func GetImageLastUpdated(imageInfo ispec.Image) time.Time {
// - bool: value for the statement: "the reference is a tag"
//
// - error: error value.
func GetRepoRefference(repo string) (string, string, bool, error) {
func GetRepoReference(repo string) (string, string, bool, error) {
repoName, digest, found := strings.Cut(repo, "@")
if !found {

View file

@ -250,7 +250,7 @@ func EnablePeriodicSignaturesVerification(config *config.Config, taskScheduler *
repos, err := metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool {
return true
}, mTypes.PageInput{})
})
if err != nil {
return
}
@ -297,9 +297,7 @@ func (gen *taskGeneratorSigValidity) Reset() {
gen.repoIndex = -1
ctx := context.Background()
repos, err := gen.metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool {
return true
}, mTypes.PageInput{})
repos, err := gen.metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true })
if err != nil {
return
}

View file

@ -15,9 +15,11 @@ import (
"zotregistry.io/zot/pkg/extensions/search/convert"
cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model"
"zotregistry.io/zot/pkg/extensions/search/gql_generated"
"zotregistry.io/zot/pkg/extensions/search/pagination"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/boltdb"
mTypes "zotregistry.io/zot/pkg/meta/types"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
)
@ -59,8 +61,7 @@ func TestConvertErrors(t *testing.T) {
err = metaDB.SetRepoReference("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
repoMetas, manifestMetaMap, _, _, err := metaDB.SearchRepos(context.Background(), "", mTypes.Filter{},
mTypes.PageInput{})
repoMetas, manifestMetaMap, _, err := metaDB.SearchRepos(context.Background(), "")
So(err, ShouldBeNil)
ctx := graphql.WithResponseContext(context.Background(),
@ -163,7 +164,7 @@ func TestConvertErrors(t *testing.T) {
ctx := graphql.WithResponseContext(context.Background(),
graphql.DefaultErrorPresenter, graphql.DefaultRecover)
// with bad config json, error while unmarshaling
// with bad config json, shouldn't error when unmarshaling
_, _, err := convert.ImageManifest2ManifestSummary(
ctx,
"repo",
@ -186,7 +187,7 @@ func TestConvertErrors(t *testing.T) {
nil,
mocks.CveInfoMock{},
)
So(err, ShouldNotBeNil)
So(err, ShouldBeNil)
// CVE scan using platform
configBlob, err := json.Marshal(ispec.Image{
@ -255,7 +256,7 @@ func TestConvertErrors(t *testing.T) {
},
}, log.NewLogger("debug", ""),
)
So(len(imageSummaries), ShouldEqual, 0)
So(len(imageSummaries), ShouldEqual, 1)
// cveInfo present no error
_, imageSummaries = convert.RepoMeta2ExpandedRepoInfo(
@ -414,3 +415,358 @@ func TestGetSignaturesInfo(t *testing.T) {
So(*signaturesSummary[0].Tool, ShouldEqual, "notation")
})
}
func TestAcceptedByFilter(t *testing.T) {
Convey("Images", t, func() {
Convey("Os not found", func() {
found := convert.ImgSumAcceptedByFilter(
&gql_generated.ImageSummary{
Manifests: []*gql_generated.ManifestSummary{
{Platform: &gql_generated.Platform{Os: ref("os1")}},
{Platform: &gql_generated.Platform{Os: ref("os2")}},
},
},
mTypes.Filter{Os: []*string{ref("os3")}},
)
So(found, ShouldBeFalse)
})
Convey("Has to be signed ", func() {
found := convert.ImgSumAcceptedByFilter(
&gql_generated.ImageSummary{
Manifests: []*gql_generated.ManifestSummary{
{IsSigned: ref(false)},
},
IsSigned: ref(false),
},
mTypes.Filter{HasToBeSigned: ref(true)},
)
So(found, ShouldBeFalse)
})
})
Convey("Repos", t, func() {
Convey("Os not found", func() {
found := convert.RepoSumAcceptedByFilter(
&gql_generated.RepoSummary{
Platforms: []*gql_generated.Platform{
{Os: ref("os1")},
{Os: ref("os2")},
},
},
mTypes.Filter{Os: []*string{ref("os3")}},
)
So(found, ShouldBeFalse)
})
Convey("Arch not found", func() {
found := convert.RepoSumAcceptedByFilter(
&gql_generated.RepoSummary{
Platforms: []*gql_generated.Platform{
{Arch: ref("Arch")},
},
},
mTypes.Filter{Arch: []*string{ref("arch_not_found")}},
)
So(found, ShouldBeFalse)
})
Convey("Has to be signed ", func() {
found := convert.ImgSumAcceptedByFilter(
&gql_generated.ImageSummary{
Manifests: []*gql_generated.ManifestSummary{
{IsSigned: ref(false)},
},
IsSigned: ref(false),
},
mTypes.Filter{HasToBeSigned: ref(true)},
)
So(found, ShouldBeFalse)
})
})
}
func ref[T any](val T) *T {
ref := val
return &ref
}
func TestPaginatedConvert(t *testing.T) {
ctx := context.Background()
var (
badBothImage = test.CreateImageWith().DefaultLayers().ImageConfig(
ispec.Image{Platform: ispec.Platform{OS: "bad-os", Architecture: "bad-arch"}}).Build()
badOsImage = test.CreateImageWith().DefaultLayers().ImageConfig(
ispec.Image{Platform: ispec.Platform{OS: "bad-os", Architecture: "good-arch"}}).Build()
badArchImage = test.CreateImageWith().DefaultLayers().ImageConfig(
ispec.Image{Platform: ispec.Platform{OS: "good-os", Architecture: "bad-arch"}}).Build()
goodImage = test.CreateImageWith().DefaultLayers().ImageConfig(
ispec.Image{Platform: ispec.Platform{OS: "good-os", Architecture: "good-arch"}}).Build()
randomImage1 = test.CreateRandomImage()
randomImage2 = test.CreateRandomImage()
badMultiArch = test.CreateMultiarchWith().Images(
[]test.Image{badBothImage, badOsImage, badArchImage, randomImage1}).Build()
goodMultiArch = test.CreateMultiarchWith().Images(
[]test.Image{badOsImage, badArchImage, randomImage2, goodImage}).Build()
)
reposMeta, manifestMetaMap, indexDataMap := test.GetMetadataForRepos(
test.Repo{
Name: "repo1-only-images",
Images: []test.RepoImage{
{Image: goodImage, Tag: "goodImage"},
{Image: badOsImage, Tag: "badOsImage"},
{Image: badArchImage, Tag: "badArchImage"},
{Image: badBothImage, Tag: "badBothImage"},
},
IsBookmarked: true,
IsStarred: true,
},
test.Repo{
Name: "repo2-only-bad-images",
Images: []test.RepoImage{
{Image: randomImage1, Tag: "randomImage1"},
{Image: randomImage2, Tag: "randomImage2"},
{Image: badBothImage, Tag: "badBothImage"},
},
IsBookmarked: true,
IsStarred: true,
},
test.Repo{
Name: "repo3-only-multiarch",
MultiArchImages: []test.RepoMultiArchImage{
{MultiarchImage: badMultiArch, Tag: "badMultiArch"},
{MultiarchImage: goodMultiArch, Tag: "goodMultiArch"},
},
IsBookmarked: true,
IsStarred: true,
},
test.Repo{
Name: "repo4-not-bookmarked-or-starred",
Images: []test.RepoImage{
{Image: goodImage, Tag: "goodImage"},
},
MultiArchImages: []test.RepoMultiArchImage{
{MultiarchImage: goodMultiArch, Tag: "goodMultiArch"},
},
},
test.Repo{
Name: "repo5-signed",
Images: []test.RepoImage{
{Image: goodImage, Tag: "goodImage"}, // is fake signed by the image below
{Image: test.CreateFakeTestSignature(goodImage.DescriptorRef())},
},
},
)
skipCVE := convert.SkipQGLField{Vulnerabilities: true}
Convey("PaginatedRepoMeta2RepoSummaries filtering and sorting", t, func() {
// Test different combinations of the filter
reposSum, pageInfo, err := convert.PaginatedRepoMeta2RepoSummaries(
ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{},
mTypes.Filter{
Os: []*string{ref("good-os")},
Arch: []*string{ref("good-arch")},
IsBookmarked: ref(true),
IsStarred: ref(true),
},
pagination.PageInput{SortBy: pagination.AlphabeticAsc},
)
So(err, ShouldBeNil)
So(len(reposSum), ShouldEqual, 2)
So(*reposSum[0].Name, ShouldResemble, "repo1-only-images")
So(*reposSum[1].Name, ShouldResemble, "repo3-only-multiarch")
So(pageInfo.ItemCount, ShouldEqual, 2)
reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries(
ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{},
mTypes.Filter{
Os: []*string{ref("good-os")},
Arch: []*string{ref("good-arch")},
IsBookmarked: ref(true),
IsStarred: ref(true),
HasToBeSigned: ref(true),
},
pagination.PageInput{SortBy: pagination.AlphabeticAsc},
)
So(err, ShouldBeNil)
So(len(reposSum), ShouldEqual, 0)
So(pageInfo.ItemCount, ShouldEqual, 0)
reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries(
ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{},
mTypes.Filter{
HasToBeSigned: ref(true),
},
pagination.PageInput{SortBy: pagination.AlphabeticAsc},
)
So(err, ShouldBeNil)
So(len(reposSum), ShouldEqual, 1)
So(*reposSum[0].Name, ShouldResemble, "repo5-signed")
So(pageInfo.ItemCount, ShouldEqual, 1)
// no filter
reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries(
ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{},
mTypes.Filter{}, pagination.PageInput{SortBy: pagination.AlphabeticAsc},
)
So(err, ShouldBeNil)
So(len(reposSum), ShouldEqual, 5)
So(*reposSum[0].Name, ShouldResemble, "repo1-only-images")
So(*reposSum[1].Name, ShouldResemble, "repo2-only-bad-images")
So(*reposSum[2].Name, ShouldResemble, "repo3-only-multiarch")
So(*reposSum[3].Name, ShouldResemble, "repo4-not-bookmarked-or-starred")
So(*reposSum[4].Name, ShouldResemble, "repo5-signed")
So(pageInfo.ItemCount, ShouldEqual, 5)
// no filter opposite sorting
reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries(
ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{},
mTypes.Filter{}, pagination.PageInput{SortBy: pagination.AlphabeticDsc},
)
So(err, ShouldBeNil)
So(len(reposSum), ShouldEqual, 5)
So(*reposSum[0].Name, ShouldResemble, "repo5-signed")
So(*reposSum[1].Name, ShouldResemble, "repo4-not-bookmarked-or-starred")
So(*reposSum[2].Name, ShouldResemble, "repo3-only-multiarch")
So(*reposSum[3].Name, ShouldResemble, "repo2-only-bad-images")
So(*reposSum[4].Name, ShouldResemble, "repo1-only-images")
So(pageInfo.ItemCount, ShouldEqual, 5)
// add pagination
reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries(
ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{},
mTypes.Filter{
Os: []*string{ref("good-os")},
Arch: []*string{ref("good-arch")},
IsBookmarked: ref(true),
IsStarred: ref(true),
},
pagination.PageInput{Limit: 1, Offset: 0, SortBy: pagination.AlphabeticAsc},
)
So(err, ShouldBeNil)
So(len(reposSum), ShouldEqual, 1)
So(*reposSum[0].Name, ShouldResemble, "repo1-only-images")
So(pageInfo.ItemCount, ShouldEqual, 1)
So(pageInfo.TotalCount, ShouldEqual, 2)
reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries(
ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{},
mTypes.Filter{
Os: []*string{ref("good-os")},
Arch: []*string{ref("good-arch")},
IsBookmarked: ref(true),
IsStarred: ref(true),
},
pagination.PageInput{Limit: 1, Offset: 1, SortBy: pagination.AlphabeticAsc},
)
So(err, ShouldBeNil)
So(len(reposSum), ShouldEqual, 1)
So(*reposSum[0].Name, ShouldResemble, "repo3-only-multiarch")
So(pageInfo.ItemCount, ShouldEqual, 1)
So(pageInfo.TotalCount, ShouldEqual, 2)
})
Convey("PaginatedRepoMeta2ImageSummaries filtering and sorting", t, func() {
imgSum, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(
ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{},
mTypes.Filter{
Os: []*string{ref("good-os")},
Arch: []*string{ref("good-arch")},
},
pagination.PageInput{SortBy: pagination.AlphabeticAsc},
)
So(err, ShouldBeNil)
So(len(imgSum), ShouldEqual, 5)
So(*imgSum[0].RepoName, ShouldResemble, "repo1-only-images")
So(*imgSum[0].Tag, ShouldResemble, "goodImage")
So(*imgSum[1].RepoName, ShouldResemble, "repo3-only-multiarch")
So(*imgSum[1].Tag, ShouldResemble, "goodMultiArch")
So(*imgSum[2].RepoName, ShouldResemble, "repo4-not-bookmarked-or-starred")
So(*imgSum[2].Tag, ShouldResemble, "goodImage")
So(*imgSum[3].RepoName, ShouldResemble, "repo4-not-bookmarked-or-starred")
So(*imgSum[3].Tag, ShouldResemble, "goodMultiArch")
So(*imgSum[4].RepoName, ShouldResemble, "repo5-signed")
So(*imgSum[4].Tag, ShouldResemble, "goodImage")
So(pageInfo.ItemCount, ShouldEqual, 5)
// add page of size 2
imgSum, pageInfo, err = convert.PaginatedRepoMeta2ImageSummaries(
ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{},
mTypes.Filter{
Os: []*string{ref("good-os")},
Arch: []*string{ref("good-arch")},
},
pagination.PageInput{Limit: 2, Offset: 0, SortBy: pagination.AlphabeticAsc},
)
So(err, ShouldBeNil)
So(len(imgSum), ShouldEqual, 2)
So(*imgSum[0].RepoName, ShouldResemble, "repo1-only-images")
So(*imgSum[0].Tag, ShouldResemble, "goodImage")
So(*imgSum[1].RepoName, ShouldResemble, "repo3-only-multiarch")
So(*imgSum[1].Tag, ShouldResemble, "goodMultiArch")
So(pageInfo.ItemCount, ShouldEqual, 2)
So(pageInfo.TotalCount, ShouldEqual, 5)
// next page
imgSum, pageInfo, err = convert.PaginatedRepoMeta2ImageSummaries(
ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{},
mTypes.Filter{
Os: []*string{ref("good-os")},
Arch: []*string{ref("good-arch")},
},
pagination.PageInput{Limit: 2, Offset: 2, SortBy: pagination.AlphabeticAsc},
)
So(err, ShouldBeNil)
So(len(imgSum), ShouldEqual, 2)
So(*imgSum[0].RepoName, ShouldResemble, "repo4-not-bookmarked-or-starred")
So(*imgSum[0].Tag, ShouldResemble, "goodImage")
So(*imgSum[1].RepoName, ShouldResemble, "repo4-not-bookmarked-or-starred")
So(*imgSum[1].Tag, ShouldResemble, "goodMultiArch")
So(pageInfo.ItemCount, ShouldEqual, 2)
So(pageInfo.TotalCount, ShouldEqual, 5)
// last page
imgSum, pageInfo, err = convert.PaginatedRepoMeta2ImageSummaries(
ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{},
mTypes.Filter{
Os: []*string{ref("good-os")},
Arch: []*string{ref("good-arch")},
},
pagination.PageInput{Limit: 2, Offset: 4, SortBy: pagination.AlphabeticAsc},
)
So(err, ShouldBeNil)
So(len(imgSum), ShouldEqual, 1)
So(*imgSum[0].RepoName, ShouldResemble, "repo5-signed")
So(*imgSum[0].Tag, ShouldResemble, "goodImage")
So(pageInfo.ItemCount, ShouldEqual, 1)
So(pageInfo.TotalCount, ShouldEqual, 5)
// has to be signed
imgSum, pageInfo, err = convert.PaginatedRepoMeta2ImageSummaries(
ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{},
mTypes.Filter{
Os: []*string{ref("good-os")},
Arch: []*string{ref("good-arch")},
HasToBeSigned: ref(true),
},
pagination.PageInput{SortBy: pagination.AlphabeticAsc},
)
So(err, ShouldBeNil)
So(len(imgSum), ShouldEqual, 1)
So(*imgSum[0].RepoName, ShouldResemble, "repo5-signed")
So(*imgSum[0].Tag, ShouldResemble, "goodImage")
So(pageInfo.ItemCount, ShouldEqual, 1)
})
}

View file

@ -15,11 +15,13 @@ import (
"github.com/vektah/gqlparser/v2/gqlerror"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/common"
zcommon "zotregistry.io/zot/pkg/common"
cveinfo "zotregistry.io/zot/pkg/extensions/search/cve"
cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model"
"zotregistry.io/zot/pkg/extensions/search/gql_generated"
"zotregistry.io/zot/pkg/extensions/search/pagination"
"zotregistry.io/zot/pkg/log"
mcommon "zotregistry.io/zot/pkg/meta/common"
mTypes "zotregistry.io/zot/pkg/meta/types"
)
@ -134,9 +136,32 @@ func RepoMeta2RepoSummary(ctx context.Context, repoMeta mTypes.RepoMetadata,
StarCount: &repoStarCount,
IsBookmarked: &repoIsUserBookMarked,
IsStarred: &repoIsUserStarred,
Rank: &repoMeta.Rank,
}
}
func PaginatedRepoMeta2RepoSummaries(ctx context.Context, repoMetas []mTypes.RepoMetadata,
manifestMetaMap map[string]mTypes.ManifestMetadata, indexDataMap map[string]mTypes.IndexData,
skip SkipQGLField, cveInfo cveinfo.CveInfo, filter mTypes.Filter, pageInput pagination.PageInput,
) ([]*gql_generated.RepoSummary, zcommon.PageInfo, error) {
reposPageFinder, err := pagination.NewRepoSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy)
if err != nil {
return []*gql_generated.RepoSummary{}, zcommon.PageInfo{}, err
}
for _, repoMeta := range repoMetas {
repoSummary := RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo)
if RepoSumAcceptedByFilter(repoSummary, filter) {
reposPageFinder.Add(repoSummary)
}
}
page, pageInfo := reposPageFinder.Page()
return page, pageInfo, nil
}
func UpdateLastUpdatedTimestamp(repoLastUpdatedTimestamp *time.Time,
lastUpdatedImageSummary *gql_generated.ImageSummary, imageSummary *gql_generated.ImageSummary,
) *gql_generated.ImageSummary {
@ -265,6 +290,7 @@ func ImageIndex2ImageSummary(ctx context.Context, repo, tag string, indexDigest
Labels: &annotations.Labels,
Source: &annotations.Source,
Vendor: &annotations.Vendor,
Authors: &annotations.Authors,
Vulnerabilities: &gql_generated.ImageVulnerabilitySummary{
MaxSeverity: &imageCveSummary.MaxSeverity,
Count: &imageCveSummary.Count,
@ -292,22 +318,14 @@ func ImageManifest2ImageSummary(ctx context.Context, repo, tag string, digest go
return &gql_generated.ImageSummary{}, map[string]int64{}, err
}
var configContent ispec.Image
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal config blob for image: %s:%s, manifest digest: %s, error: %s",
repo, tag, manifestDigest, err.Error()))
return &gql_generated.ImageSummary{}, map[string]int64{}, err
}
configContent := mcommon.InitializeImageConfig(manifestMeta.ConfigBlob)
var (
repoName = repo
configDigest = manifestContent.Config.Digest.String()
configSize = manifestContent.Config.Size
artifactType = common.GetManifestArtifactType(manifestContent)
imageLastUpdated = common.GetImageLastUpdated(configContent)
artifactType = zcommon.GetManifestArtifactType(manifestContent)
imageLastUpdated = zcommon.GetImageLastUpdated(configContent)
downloadCount = repoMeta.Statistics[digest.String()].DownloadCount
isSigned = false
)
@ -460,22 +478,14 @@ func ImageManifest2ManifestSummary(ctx context.Context, repo, tag string, descri
return &gql_generated.ManifestSummary{}, map[string]int64{}, err
}
var configContent ispec.Image
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal config blob for image: %s:%s, manifest digest: %s, error: %s",
repo, tag, digest, err.Error()))
return &gql_generated.ManifestSummary{}, map[string]int64{}, err
}
configContent := mcommon.InitializeImageConfig(manifestMeta.ConfigBlob)
var (
manifestDigestStr = digest.String()
configDigest = manifestContent.Config.Digest.String()
configSize = manifestContent.Config.Size
artifactType = common.GetManifestArtifactType(manifestContent)
imageLastUpdated = common.GetImageLastUpdated(configContent)
artifactType = zcommon.GetManifestArtifactType(manifestContent)
imageLastUpdated = zcommon.GetImageLastUpdated(configContent)
downloadCount = manifestMeta.DownloadCount
isSigned = false
)
@ -599,6 +609,36 @@ func RepoMeta2ImageSummaries(ctx context.Context, repoMeta mTypes.RepoMetadata,
return imageSummaries
}
func PaginatedRepoMeta2ImageSummaries(ctx context.Context, reposMeta []mTypes.RepoMetadata,
manifestMetaMap map[string]mTypes.ManifestMetadata, indexDataMap map[string]mTypes.IndexData,
skip SkipQGLField, cveInfo cveinfo.CveInfo, filter mTypes.Filter, pageInput pagination.PageInput,
) ([]*gql_generated.ImageSummary, zcommon.PageInfo, error) {
imagePageFinder, err := pagination.NewImgSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy)
if err != nil {
return []*gql_generated.ImageSummary{}, zcommon.PageInfo{}, err
}
for _, repoMeta := range reposMeta {
for tag := range repoMeta.Tags {
descriptor := repoMeta.Tags[tag]
imageSummary, _, err := Descriptor2ImageSummary(ctx, descriptor, repoMeta.Name, tag, skip.Vulnerabilities,
repoMeta, manifestMetaMap, indexDataMap, cveInfo)
if err != nil {
continue
}
if ImgSumAcceptedByFilter(imageSummary, filter) {
imagePageFinder.Add(imageSummary)
}
}
}
page, pageInfo := imagePageFinder.Page()
return page, pageInfo, nil
}
func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta mTypes.RepoMetadata,
manifestMetaMap map[string]mTypes.ManifestMetadata, indexDataMap map[string]mTypes.IndexData,
skip SkipQGLField, cveInfo cveinfo.CveInfo, log log.Logger,

View file

@ -0,0 +1,114 @@
package convert
import (
zcommon "zotregistry.io/zot/pkg/common"
gql_gen "zotregistry.io/zot/pkg/extensions/search/gql_generated"
mTypes "zotregistry.io/zot/pkg/meta/types"
)
func ImgSumAcceptedByFilter(imageSummary *gql_gen.ImageSummary, filter mTypes.Filter) bool {
osFilters := strSliceFromRef(filter.Os)
archFilters := strSliceFromRef(filter.Arch)
platforms := getImagePlatforms(imageSummary)
platformMatchFound := len(platforms) == 0 && filter.Os == nil && filter.Arch == nil
for _, platform := range platforms {
osCheck := true
if len(osFilters) > 0 {
osCheck = platform.Os != nil && zcommon.ContainsStringIgnoreCase(osFilters, *platform.Os)
}
archCheck := true
if len(archFilters) > 0 {
archCheck = platform.Arch != nil && zcommon.ContainsStringIgnoreCase(archFilters, *platform.Arch)
}
if osCheck && archCheck {
platformMatchFound = true
break
}
}
if !platformMatchFound {
return false
}
if filter.HasToBeSigned != nil && *filter.HasToBeSigned && !*imageSummary.IsSigned {
return false
}
return true
}
func getImagePlatforms(imageSummary *gql_gen.ImageSummary) []*gql_gen.Platform {
platforms := []*gql_gen.Platform{}
for _, manifest := range imageSummary.Manifests {
if manifest.Platform != nil {
platforms = append(platforms, manifest.Platform)
}
}
return platforms
}
func RepoSumAcceptedByFilter(repoSummary *gql_gen.RepoSummary, filter mTypes.Filter) bool {
osFilters := strSliceFromRef(filter.Os)
archFilters := strSliceFromRef(filter.Arch)
platformMatchFound := len(repoSummary.Platforms) == 0 && filter.Os == nil && filter.Arch == nil
for _, platform := range repoSummary.Platforms {
osCheck := true
if len(osFilters) > 0 {
osCheck = platform.Os != nil && zcommon.ContainsStringIgnoreCase(osFilters, *platform.Os)
}
archCheck := true
if len(archFilters) > 0 {
archCheck = platform.Arch != nil && zcommon.ContainsStringIgnoreCase(archFilters, *platform.Arch)
}
if osCheck && archCheck {
platformMatchFound = true
break
}
}
if !platformMatchFound {
return false
}
if filter.HasToBeSigned != nil && *filter.HasToBeSigned && !*repoSummary.NewestImage.IsSigned {
return false
}
if filter.IsBookmarked != nil && *filter.IsBookmarked != *repoSummary.IsBookmarked {
return false
}
if filter.IsStarred != nil && *filter.IsStarred != *repoSummary.IsStarred {
return false
}
return true
}
func strSliceFromRef(slice []*string) []string {
resultSlice := make([]string, len(slice))
for i := range slice {
if slice[i] != nil {
resultSlice[i] = *slice[i]
}
}
return resultSlice
}

View file

@ -313,12 +313,10 @@ func getConfigAndDigest(metaDB mTypes.MetaDB, manifestDigestStr string) (ispec.I
var configContent ispec.Image
// we'll fail the execution if the config is not compatibe with ispec.Image because we can't scan this type of images.
err = json.Unmarshal(manifestData.ConfigBlob, &configContent)
if err != nil {
return ispec.Image{}, "", err
}
return configContent, manifestDigest, nil
return configContent, manifestDigest, err
}
func filterCVEList(cveMap map[string]cvemodel.CVE, searchedCVE string, pageFinder *CvePageFinder) {

View file

@ -195,6 +195,7 @@ type ComplexityRoot struct {
Name func(childComplexity int) int
NewestImage func(childComplexity int) int
Platforms func(childComplexity int) int
Rank func(childComplexity int) int
Size func(childComplexity int) int
StarCount func(childComplexity int) int
Vendors func(childComplexity int) int
@ -988,6 +989,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
return e.complexity.RepoSummary.Platforms(childComplexity), true
case "RepoSummary.Rank":
if e.complexity.RepoSummary.Rank == nil {
break
}
return e.complexity.RepoSummary.Rank(childComplexity), true
case "RepoSummary.Size":
if e.complexity.RepoSummary.Size == nil {
break
@ -1143,7 +1151,7 @@ type CVEResultForImage {
"""
Tag: String
"""
List of CVE objects which afect this specific image:tag
List of CVE objects which affect this specific image:tag
"""
CVEList: [CVE]
"""
@ -1354,7 +1362,7 @@ type ManifestSummary {
"""
Platform: Platform
"""
Total numer of image manifest downloads from this repository
Total number of image manifest downloads from this repository
"""
DownloadCount: Int
"""
@ -1424,7 +1432,7 @@ type RepoSummary {
"""
NewestImage: ImageSummary
"""
Total numer of image manifest downloads from this repository
Total number of image manifest downloads from this repository
"""
DownloadCount: Int
"""
@ -1436,9 +1444,13 @@ type RepoSummary {
"""
IsBookmarked: Boolean
"""
True if the repository is stared by the current user, fale otherwise
True if the repository is starred by the current user, false otherwise
"""
IsStarred: Boolean
"""
Rank represents how good the match was between the queried repo name and this repo summary.
"""
Rank: Int
}
"""
@ -2909,6 +2921,8 @@ func (ec *executionContext) fieldContext_GlobalSearchResult_Repos(ctx context.Co
return ec.fieldContext_RepoSummary_IsBookmarked(ctx, field)
case "IsStarred":
return ec.fieldContext_RepoSummary_IsStarred(ctx, field)
case "Rank":
return ec.fieldContext_RepoSummary_Rank(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type RepoSummary", field.Name)
},
@ -5333,6 +5347,8 @@ func (ec *executionContext) fieldContext_PaginatedReposResult_Results(ctx contex
return ec.fieldContext_RepoSummary_IsBookmarked(ctx, field)
case "IsStarred":
return ec.fieldContext_RepoSummary_IsStarred(ctx, field)
case "Rank":
return ec.fieldContext_RepoSummary_Rank(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type RepoSummary", field.Name)
},
@ -6806,6 +6822,8 @@ func (ec *executionContext) fieldContext_RepoInfo_Summary(ctx context.Context, f
return ec.fieldContext_RepoSummary_IsBookmarked(ctx, field)
case "IsStarred":
return ec.fieldContext_RepoSummary_IsStarred(ctx, field)
case "Rank":
return ec.fieldContext_RepoSummary_Rank(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type RepoSummary", field.Name)
},
@ -7271,6 +7289,47 @@ func (ec *executionContext) fieldContext_RepoSummary_IsStarred(ctx context.Conte
return fc, nil
}
func (ec *executionContext) _RepoSummary_Rank(ctx context.Context, field graphql.CollectedField, obj *RepoSummary) (ret graphql.Marshaler) {
fc, err := ec.fieldContext_RepoSummary_Rank(ctx, field)
if err != nil {
return graphql.Null
}
ctx = graphql.WithFieldContext(ctx, fc)
defer func() {
if r := recover(); r != nil {
ec.Error(ctx, ec.Recover(ctx, r))
ret = graphql.Null
}
}()
resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
ctx = rctx // use context from middleware stack in children
return obj.Rank, nil
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
return graphql.Null
}
res := resTmp.(*int)
fc.Result = res
return ec.marshalOInt2ᚖint(ctx, field.Selections, res)
}
func (ec *executionContext) fieldContext_RepoSummary_Rank(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) {
fc = &graphql.FieldContext{
Object: "RepoSummary",
Field: field,
IsMethod: false,
IsResolver: false,
Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) {
return nil, errors.New("field of type Int does not have child fields")
},
}
return fc, nil
}
func (ec *executionContext) _SignatureSummary_Tool(ctx context.Context, field graphql.CollectedField, obj *SignatureSummary) (ret graphql.Marshaler) {
fc, err := ec.fieldContext_SignatureSummary_Tool(ctx, field)
if err != nil {
@ -10421,6 +10480,8 @@ func (ec *executionContext) _RepoSummary(ctx context.Context, sel ast.SelectionS
out.Values[i] = ec._RepoSummary_IsBookmarked(ctx, field, obj)
case "IsStarred":
out.Values[i] = ec._RepoSummary_IsStarred(ctx, field, obj)
case "Rank":
out.Values[i] = ec._RepoSummary_Rank(ctx, field, obj)
default:
panic("unknown field " + strconv.Quote(field.Name))
}

View file

@ -37,7 +37,7 @@ type Cve struct {
type CVEResultForImage struct {
// Tag affected by the CVEs
Tag *string `json:"Tag,omitempty"`
// List of CVE objects which afect this specific image:tag
// List of CVE objects which affect this specific image:tag
CVEList []*Cve `json:"CVEList,omitempty"`
// The CVE pagination information, see PageInfo object for more details
Page *PageInfo `json:"Page,omitempty"`
@ -174,7 +174,7 @@ type ManifestSummary struct {
SignatureInfo []*SignatureSummary `json:"SignatureInfo,omitempty"`
// OS and architecture supported by this image
Platform *Platform `json:"Platform,omitempty"`
// Total numer of image manifest downloads from this repository
// Total number of image manifest downloads from this repository
DownloadCount *int `json:"DownloadCount,omitempty"`
// List of layers matching the search criteria
// NOTE: the actual search logic for layers is not implemented at the moment
@ -285,14 +285,16 @@ type RepoSummary struct {
// Details of the newest image inside the repository
// NOTE: not the image with the `latest` tag, the one with the most recent created timestamp
NewestImage *ImageSummary `json:"NewestImage,omitempty"`
// Total numer of image manifest downloads from this repository
// Total number of image manifest downloads from this repository
DownloadCount *int `json:"DownloadCount,omitempty"`
// Number of stars attributed to this repository by users
StarCount *int `json:"StarCount,omitempty"`
// True if the repository is bookmarked by the current user, false otherwise
IsBookmarked *bool `json:"IsBookmarked,omitempty"`
// True if the repository is stared by the current user, fale otherwise
// True if the repository is starred by the current user, false otherwise
IsStarred *bool `json:"IsStarred,omitempty"`
// Rank represents how good the match was between the queried repo name and this repo summary.
Rank *int `json:"Rank,omitempty"`
}
// Contains details about the signature

View file

@ -0,0 +1,164 @@
package pagination
import (
"fmt"
"sort"
"time"
zerr "zotregistry.io/zot/errors"
zcommon "zotregistry.io/zot/pkg/common"
gql_gen "zotregistry.io/zot/pkg/extensions/search/gql_generated"
)
type ImageSummariesPageFinder struct {
limit int
offset int
sortBy SortCriteria
pageBuffer []*gql_gen.ImageSummary
}
func NewImgSumPageFinder(limit, offset int, sortBy SortCriteria) (*ImageSummariesPageFinder, error) {
if sortBy == "" {
sortBy = AlphabeticAsc
}
if limit < 0 {
return nil, zerr.ErrLimitIsNegative
}
if offset < 0 {
return nil, zerr.ErrOffsetIsNegative
}
if _, found := ImgSumSortFuncs()[sortBy]; !found {
return nil, fmt.Errorf("sorting repos by '%s' is not supported %w",
sortBy, zerr.ErrSortCriteriaNotSupported)
}
return &ImageSummariesPageFinder{
limit: limit,
offset: offset,
sortBy: sortBy,
pageBuffer: []*gql_gen.ImageSummary{},
}, nil
}
func (pf *ImageSummariesPageFinder) Add(imgSum *gql_gen.ImageSummary) {
pf.pageBuffer = append(pf.pageBuffer, imgSum)
}
func (pf *ImageSummariesPageFinder) Page() ([]*gql_gen.ImageSummary, zcommon.PageInfo) {
if len(pf.pageBuffer) == 0 {
return []*gql_gen.ImageSummary{}, zcommon.PageInfo{}
}
pageInfo := zcommon.PageInfo{}
sort.Slice(pf.pageBuffer, ImgSumSortFuncs()[pf.sortBy](pf.pageBuffer))
// the offset and limit are calculated in terms of repos counted
start := pf.offset
end := pf.offset + pf.limit
// we'll return an empty array when the offset is greater than the number of elements
if start >= len(pf.pageBuffer) {
start = len(pf.pageBuffer)
end = start
}
if end >= len(pf.pageBuffer) {
end = len(pf.pageBuffer)
}
page := pf.pageBuffer[start:end]
pageInfo.ItemCount = len(page)
if start == 0 && end == 0 {
page = pf.pageBuffer
pageInfo.ItemCount = len(page)
}
pageInfo.TotalCount = len(pf.pageBuffer)
return page, pageInfo
}
func ImgSumSortFuncs() map[SortCriteria]func(pageBuffer []*gql_gen.ImageSummary) func(i, j int) bool {
return map[SortCriteria]func(pageBuffer []*gql_gen.ImageSummary) func(i, j int) bool{
AlphabeticAsc: ImgSortByAlphabeticAsc,
AlphabeticDsc: ImgSortByAlphabeticDsc,
UpdateTime: ImgSortByUpdateTime,
Relevance: ImgSortByRelevance,
Downloads: ImgSortByDownloads,
}
}
func ImgSortByAlphabeticAsc(pageBuffer []*gql_gen.ImageSummary) func(i, j int) bool {
return func(i, j int) bool { //nolint: varnamelen
if *pageBuffer[i].RepoName < *pageBuffer[j].RepoName {
return true
}
if *pageBuffer[i].RepoName == *pageBuffer[j].RepoName {
return *pageBuffer[i].Tag < *pageBuffer[j].Tag
}
return false
}
}
func ImgSortByAlphabeticDsc(pageBuffer []*gql_gen.ImageSummary) func(i, j int) bool {
return func(i, j int) bool { //nolint: varnamelen
if *pageBuffer[i].RepoName > *pageBuffer[j].RepoName {
return true
}
if *pageBuffer[i].RepoName == *pageBuffer[j].RepoName {
return *pageBuffer[i].Tag > *pageBuffer[j].Tag
}
return false
}
}
func ImgSortByRelevance(pageBuffer []*gql_gen.ImageSummary) func(i, j int) bool {
return func(i, j int) bool { //nolint: varnamelen
if *pageBuffer[i].RepoName < *pageBuffer[j].RepoName {
return true
}
if *pageBuffer[i].RepoName == *pageBuffer[j].RepoName {
return *pageBuffer[i].Tag < *pageBuffer[j].Tag
}
return false
}
}
// SortByUpdateTime sorting descending by time.
func ImgSortByUpdateTime(pageBuffer []*gql_gen.ImageSummary) func(i, j int) bool {
repos2LastUpdated := map[string]time.Time{}
for _, img := range pageBuffer {
lastUpdated, ok := repos2LastUpdated[*img.RepoName]
if !ok || lastUpdated.Before(*img.LastUpdated) {
repos2LastUpdated[*img.RepoName] = *img.LastUpdated
}
}
return func(i, j int) bool {
iRepoTime, jRepoTime := repos2LastUpdated[*pageBuffer[i].RepoName], repos2LastUpdated[*pageBuffer[j].RepoName]
return (iRepoTime.After(jRepoTime) || iRepoTime.Equal(jRepoTime)) &&
pageBuffer[i].LastUpdated.After(*pageBuffer[j].LastUpdated)
}
}
// SortByDownloads returns a comparison function for descendant sorting by downloads.
func ImgSortByDownloads(pageBuffer []*gql_gen.ImageSummary) func(i, j int) bool {
return func(i, j int) bool {
return *pageBuffer[i].DownloadCount > *pageBuffer[j].DownloadCount
}
}

View file

@ -0,0 +1,18 @@
package pagination
type SortCriteria string
type PageInput struct {
Limit int
Offset int
SortBy SortCriteria
}
const (
Relevance = SortCriteria("RELEVANCE")
UpdateTime = SortCriteria("UPDATE_TIME")
AlphabeticAsc = SortCriteria("ALPHABETIC_ASC")
AlphabeticDsc = SortCriteria("ALPHABETIC_DSC")
Stars = SortCriteria("STARS")
Downloads = SortCriteria("DOWNLOADS")
)

View file

@ -0,0 +1,294 @@
package pagination_test
import (
"testing"
"time"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/pkg/extensions/search/gql_generated"
"zotregistry.io/zot/pkg/extensions/search/pagination"
)
func TestImgSumPagination(t *testing.T) {
Convey("NewImgSumPageFinder errors", t, func() {
_, err := pagination.NewImgSumPageFinder(-1, 0, pagination.AlphabeticAsc)
So(err, ShouldNotBeNil)
_, err = pagination.NewImgSumPageFinder(0, -1, pagination.AlphabeticAsc)
So(err, ShouldNotBeNil)
_, err = pagination.NewImgSumPageFinder(0, 0, "unknown")
So(err, ShouldNotBeNil)
})
Convey("Sort Functions", t, func() {
Convey("ImgSortByAlphabeticAsc", func() {
// Case: repo1 is < repo2
pageBuff := []*gql_generated.ImageSummary{
{RepoName: ref("repo1:1")},
{RepoName: ref("repo2:2")},
}
sortFunc := pagination.ImgSortByAlphabeticAsc(pageBuff)
So(sortFunc(0, 1), ShouldBeTrue)
})
Convey("ImgSortByAlphabeticDsc", func() {
// Case: repo1 is < repo2
pageBuff := []*gql_generated.ImageSummary{
{RepoName: ref("repo1:1")},
{RepoName: ref("repo2:2")},
}
sortFunc := pagination.ImgSortByAlphabeticDsc(pageBuff)
So(sortFunc(0, 1), ShouldBeFalse)
})
Convey("ImgSortByRelevance", func() {
// Case: repo1 is < repo2
pageBuff := []*gql_generated.ImageSummary{
{RepoName: ref("repo1:1")},
{RepoName: ref("repo2:2")},
}
sortFunc := pagination.ImgSortByRelevance(pageBuff)
So(sortFunc(0, 1), ShouldBeTrue)
})
})
}
func TestRepoSumPagination(t *testing.T) {
Convey("NewRepoSumPageFinder errors", t, func() {
_, err := pagination.NewRepoSumPageFinder(-1, 0, pagination.AlphabeticAsc)
So(err, ShouldNotBeNil)
_, err = pagination.NewRepoSumPageFinder(0, -1, pagination.AlphabeticAsc)
So(err, ShouldNotBeNil)
_, err = pagination.NewRepoSumPageFinder(0, 0, "unknown")
So(err, ShouldNotBeNil)
})
}
func ref[T any](input T) *T {
obj := input
return &obj
}
func TestPagination(t *testing.T) {
Convey("Image Pagination", t, func() {
Convey("Sort functions", func() {
imgSum1 := gql_generated.ImageSummary{
RepoName: ref("1"),
Tag: ref("1"),
LastUpdated: ref(time.Date(2010, 1, 1, 1, 1, 1, 1, time.UTC)),
DownloadCount: ref(33),
}
imgSum2 := gql_generated.ImageSummary{
RepoName: ref("1"),
Tag: ref("latest"),
LastUpdated: ref(time.Date(2020, 1, 1, 1, 1, 1, 1, time.UTC)),
DownloadCount: ref(11),
}
imgSum3 := gql_generated.ImageSummary{
RepoName: ref("3"),
Tag: ref("1"),
LastUpdated: ref(time.Date(2011, 1, 1, 1, 1, 1, 1, time.UTC)),
DownloadCount: ref(22),
}
imgSum4 := gql_generated.ImageSummary{
RepoName: ref("4"),
Tag: ref("latest"),
LastUpdated: ref(time.Date(2012, 1, 1, 1, 1, 1, 1, time.UTC)),
DownloadCount: ref(44),
}
// ImgSortByAlphabeticAsc
imagePageFinder, err := pagination.NewImgSumPageFinder(4, 0, pagination.AlphabeticAsc)
So(err, ShouldBeNil)
imagePageFinder.Add(&imgSum1)
imagePageFinder.Add(&imgSum2)
imagePageFinder.Add(&imgSum3)
imagePageFinder.Add(&imgSum4)
page, _ := imagePageFinder.Page()
So(page, ShouldEqual, []*gql_generated.ImageSummary{
&imgSum1, &imgSum2, &imgSum3, &imgSum4,
})
// ImgSortByAlphabeticDsc
imagePageFinder, err = pagination.NewImgSumPageFinder(4, 0, pagination.AlphabeticDsc)
So(err, ShouldBeNil)
imagePageFinder.Add(&imgSum1)
imagePageFinder.Add(&imgSum2)
imagePageFinder.Add(&imgSum3)
imagePageFinder.Add(&imgSum4)
page, _ = imagePageFinder.Page()
So(page, ShouldEqual, []*gql_generated.ImageSummary{
&imgSum4, &imgSum3, &imgSum2, &imgSum1,
})
// ImgSortByRelevance
imagePageFinder, err = pagination.NewImgSumPageFinder(4, 0, pagination.Relevance)
So(err, ShouldBeNil)
imagePageFinder.Add(&imgSum1)
imagePageFinder.Add(&imgSum2)
imagePageFinder.Add(&imgSum3)
imagePageFinder.Add(&imgSum4)
page, _ = imagePageFinder.Page()
So(page, ShouldEqual, []*gql_generated.ImageSummary{
&imgSum1, &imgSum2, &imgSum3, &imgSum4,
})
// ImgSortByUpdateTime
imagePageFinder, err = pagination.NewImgSumPageFinder(4, 0, pagination.UpdateTime)
So(err, ShouldBeNil)
imagePageFinder.Add(&imgSum1)
imagePageFinder.Add(&imgSum2)
imagePageFinder.Add(&imgSum3)
imagePageFinder.Add(&imgSum4)
page, _ = imagePageFinder.Page()
So(page, ShouldEqual, []*gql_generated.ImageSummary{
&imgSum2, &imgSum1, &imgSum4, &imgSum3,
})
// ImgSortByDownloads
imagePageFinder, err = pagination.NewImgSumPageFinder(4, 0, pagination.Downloads)
So(err, ShouldBeNil)
imagePageFinder.Add(&imgSum1)
imagePageFinder.Add(&imgSum2)
imagePageFinder.Add(&imgSum3)
imagePageFinder.Add(&imgSum4)
page, _ = imagePageFinder.Page()
So(page, ShouldEqual, []*gql_generated.ImageSummary{
&imgSum4, &imgSum1, &imgSum3, &imgSum2,
})
})
Convey("Errors", func() {
imagePageFinder, err := pagination.NewImgSumPageFinder(2, 0, "")
So(err, ShouldBeNil)
So(imagePageFinder, ShouldNotBeNil)
_, err = pagination.NewImgSumPageFinder(-1, 0, "")
So(err, ShouldNotBeNil)
_, err = pagination.NewImgSumPageFinder(1, -1, "")
So(err, ShouldNotBeNil)
_, err = pagination.NewImgSumPageFinder(1, -1, "bad sort func")
So(err, ShouldNotBeNil)
})
})
Convey("Repos Pagination", t, func() {
Convey("Sort functions", func() {
repoSum1 := gql_generated.RepoSummary{
Name: ref("1"),
LastUpdated: ref(time.Date(2010, 1, 1, 1, 1, 1, 1, time.UTC)),
DownloadCount: ref(33),
Rank: ref(1),
}
repoSum2 := gql_generated.RepoSummary{
Name: ref("2"),
LastUpdated: ref(time.Date(2020, 1, 1, 1, 1, 1, 1, time.UTC)),
DownloadCount: ref(11),
Rank: ref(2),
}
repoSum3 := gql_generated.RepoSummary{
Name: ref("3"),
LastUpdated: ref(time.Date(2011, 1, 1, 1, 1, 1, 1, time.UTC)),
DownloadCount: ref(22),
Rank: ref(3),
}
repoSum4 := gql_generated.RepoSummary{
Name: ref("4"),
LastUpdated: ref(time.Date(2012, 1, 1, 1, 1, 1, 1, time.UTC)),
DownloadCount: ref(44),
Rank: ref(4),
}
// ImgSortByAlphabeticAsc
imagePageFinder, err := pagination.NewRepoSumPageFinder(4, 0, pagination.AlphabeticAsc)
So(err, ShouldBeNil)
imagePageFinder.Add(&repoSum1)
imagePageFinder.Add(&repoSum2)
imagePageFinder.Add(&repoSum3)
imagePageFinder.Add(&repoSum4)
page, _ := imagePageFinder.Page()
So(page, ShouldEqual, []*gql_generated.RepoSummary{
&repoSum1, &repoSum2, &repoSum3, &repoSum4,
})
// ImgSortByAlphabeticDsc
imagePageFinder, err = pagination.NewRepoSumPageFinder(4, 0, pagination.AlphabeticDsc)
So(err, ShouldBeNil)
imagePageFinder.Add(&repoSum1)
imagePageFinder.Add(&repoSum2)
imagePageFinder.Add(&repoSum3)
imagePageFinder.Add(&repoSum4)
page, _ = imagePageFinder.Page()
So(page, ShouldEqual, []*gql_generated.RepoSummary{
&repoSum4, &repoSum3, &repoSum2, &repoSum1,
})
// ImgSortByRelevance
imagePageFinder, err = pagination.NewRepoSumPageFinder(4, 0, pagination.Relevance)
So(err, ShouldBeNil)
imagePageFinder.Add(&repoSum1)
imagePageFinder.Add(&repoSum2)
imagePageFinder.Add(&repoSum3)
imagePageFinder.Add(&repoSum4)
page, _ = imagePageFinder.Page()
So(page, ShouldEqual, []*gql_generated.RepoSummary{
&repoSum1, &repoSum2, &repoSum3, &repoSum4,
})
// ImgSortByUpdateTime
imagePageFinder, err = pagination.NewRepoSumPageFinder(4, 0, pagination.UpdateTime)
So(err, ShouldBeNil)
imagePageFinder.Add(&repoSum1)
imagePageFinder.Add(&repoSum2)
imagePageFinder.Add(&repoSum3)
imagePageFinder.Add(&repoSum4)
page, _ = imagePageFinder.Page()
So(page, ShouldEqual, []*gql_generated.RepoSummary{
&repoSum2, &repoSum4, &repoSum3, &repoSum1,
})
// ImgSortByDownloads
imagePageFinder, err = pagination.NewRepoSumPageFinder(4, 0, pagination.Downloads)
So(err, ShouldBeNil)
imagePageFinder.Add(&repoSum1)
imagePageFinder.Add(&repoSum2)
imagePageFinder.Add(&repoSum3)
imagePageFinder.Add(&repoSum4)
page, _ = imagePageFinder.Page()
So(page, ShouldEqual, []*gql_generated.RepoSummary{
&repoSum4, &repoSum1, &repoSum3, &repoSum2,
})
})
Convey("Errors", func() {
repoPageFinder, err := pagination.NewRepoSumPageFinder(2, 0, "")
So(err, ShouldBeNil)
So(repoPageFinder, ShouldNotBeNil)
_, err = pagination.NewRepoSumPageFinder(-1, 0, "")
So(err, ShouldNotBeNil)
_, err = pagination.NewRepoSumPageFinder(1, -1, "")
So(err, ShouldNotBeNil)
_, err = pagination.NewRepoSumPageFinder(1, -1, "bad sort func")
So(err, ShouldNotBeNil)
})
})
}

View file

@ -0,0 +1,126 @@
package pagination
import (
"fmt"
"sort"
zerr "zotregistry.io/zot/errors"
zcommon "zotregistry.io/zot/pkg/common"
gql_gen "zotregistry.io/zot/pkg/extensions/search/gql_generated"
)
type RepoSummariesPageFinder struct {
limit int
offset int
sortBy SortCriteria
pageBuffer []*gql_gen.RepoSummary
}
func NewRepoSumPageFinder(limit, offset int, sortBy SortCriteria) (*RepoSummariesPageFinder, error) {
if sortBy == "" {
sortBy = AlphabeticAsc
}
if limit < 0 {
return nil, zerr.ErrLimitIsNegative
}
if offset < 0 {
return nil, zerr.ErrOffsetIsNegative
}
if _, found := RepoSumSortFuncs()[sortBy]; !found {
return nil, fmt.Errorf("sorting repos by '%s' is not supported %w",
sortBy, zerr.ErrSortCriteriaNotSupported)
}
return &RepoSummariesPageFinder{
limit: limit,
offset: offset,
sortBy: sortBy,
pageBuffer: []*gql_gen.RepoSummary{},
}, nil
}
func (pf *RepoSummariesPageFinder) Add(imgSum *gql_gen.RepoSummary) {
pf.pageBuffer = append(pf.pageBuffer, imgSum)
}
func (pf *RepoSummariesPageFinder) Page() ([]*gql_gen.RepoSummary, zcommon.PageInfo) {
if len(pf.pageBuffer) == 0 {
return []*gql_gen.RepoSummary{}, zcommon.PageInfo{}
}
pageInfo := zcommon.PageInfo{}
sort.Slice(pf.pageBuffer, RepoSumSortFuncs()[pf.sortBy](pf.pageBuffer))
// the offset and limit are calculated in terms of repos counted
start := pf.offset
end := pf.offset + pf.limit
// we'll return an empty array when the offset is greater than the number of elements
if start >= len(pf.pageBuffer) {
start = len(pf.pageBuffer)
end = start
}
if end >= len(pf.pageBuffer) {
end = len(pf.pageBuffer)
}
page := pf.pageBuffer[start:end]
pageInfo.ItemCount = len(page)
if start == 0 && end == 0 {
page = pf.pageBuffer
pageInfo.ItemCount = len(page)
}
pageInfo.TotalCount = len(pf.pageBuffer)
return page, pageInfo
}
func RepoSumSortFuncs() map[SortCriteria]func(pageBuffer []*gql_gen.RepoSummary) func(i, j int) bool {
return map[SortCriteria]func(pageBuffer []*gql_gen.RepoSummary) func(i, j int) bool{
AlphabeticAsc: RepoSortByAlphabeticAsc,
AlphabeticDsc: RepoSortByAlphabeticDsc,
Relevance: RepoSortByRelevance,
UpdateTime: RepoSortByUpdateTime,
Downloads: RepoSortByDownloads,
}
}
func RepoSortByAlphabeticAsc(pageBuffer []*gql_gen.RepoSummary) func(i, j int) bool {
return func(i, j int) bool {
return *pageBuffer[i].Name < *pageBuffer[j].Name
}
}
func RepoSortByAlphabeticDsc(pageBuffer []*gql_gen.RepoSummary) func(i, j int) bool {
return func(i, j int) bool {
return *pageBuffer[i].Name > *pageBuffer[j].Name
}
}
func RepoSortByRelevance(pageBuffer []*gql_gen.RepoSummary) func(i, j int) bool {
return func(i, j int) bool {
return *pageBuffer[i].Rank < *pageBuffer[j].Rank
}
}
// SortByUpdateTime sorting descending by time.
func RepoSortByUpdateTime(pageBuffer []*gql_gen.RepoSummary) func(i, j int) bool {
return func(i, j int) bool {
return pageBuffer[i].LastUpdated.After(*pageBuffer[j].LastUpdated)
}
}
// SortByDownloads returns a comparison function for descendant sorting by downloads.
func RepoSortByDownloads(pageBuffer []*gql_gen.RepoSummary) func(i, j int) bool {
return func(i, j int) bool {
return *pageBuffer[i].DownloadCount > *pageBuffer[j].DownloadCount
}
}

View file

@ -23,6 +23,7 @@ import (
cveinfo "zotregistry.io/zot/pkg/extensions/search/cve"
cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model"
"zotregistry.io/zot/pkg/extensions/search/gql_generated"
"zotregistry.io/zot/pkg/extensions/search/pagination"
"zotregistry.io/zot/pkg/log"
mTypes "zotregistry.io/zot/pkg/meta/types"
localCtx "zotregistry.io/zot/pkg/requestcontext"
@ -114,8 +115,6 @@ func FilterByDigest(digest string) mTypes.FilterFunc {
func getImageListForDigest(ctx context.Context, digest string, metaDB mTypes.MetaDB, cveInfo cveinfo.CveInfo,
requestedPage *gql_generated.PageInput,
) (*gql_generated.PaginatedImagesResult, error) {
imageList := make([]*gql_generated.ImageSummary, 0)
if requestedPage == nil {
requestedPage = &gql_generated.PageInput{}
}
@ -124,30 +123,28 @@ func getImageListForDigest(ctx context.Context, digest string, metaDB mTypes.Met
Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Images.Vulnerabilities"),
}
pageInput := mTypes.PageInput{
pageInput := pagination.PageInput{
Limit: safeDereferencing(requestedPage.Limit, 0),
Offset: safeDereferencing(requestedPage.Offset, 0),
SortBy: mTypes.SortCriteria(
SortBy: pagination.SortCriteria(
safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance),
),
}
// get all repos
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterTags(ctx,
FilterByDigest(digest), mTypes.Filter{}, pageInput)
reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, FilterByDigest(digest))
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
for _, repoMeta := range reposMeta {
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap,
skip, cveInfo)
imageList = append(imageList, imageSummaries...)
imageSummaries, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap,
indexDataMap, skip, cveInfo, mTypes.Filter{}, pageInput)
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
return &gql_generated.PaginatedImagesResult{
Results: imageList,
Results: imageSummaries,
Page: &gql_generated.PageInfo{
TotalCount: pageInfo.TotalCount,
ItemCount: pageInfo.ItemCount,
@ -380,6 +377,33 @@ func FilterByTagInfo(tagsInfo []cvemodel.TagInfo) mTypes.FilterFunc {
}
}
func FilterByRepoAndTagInfo(repo string, tagsInfo []cvemodel.TagInfo) mTypes.FilterFunc {
return func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool {
if repoMeta.Name != repo {
return false
}
manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String()
for _, tagInfo := range tagsInfo {
switch tagInfo.Descriptor.MediaType {
case ispec.MediaTypeImageManifest:
if tagInfo.Descriptor.Digest.String() == manifestDigest {
return true
}
case ispec.MediaTypeImageIndex:
for _, manifestDesc := range tagInfo.Manifests {
if manifestDesc.Digest.String() == manifestDigest {
return true
}
}
}
}
return false
}
}
func getImageListForCVE(
ctx context.Context,
cveID string,
@ -393,10 +417,7 @@ func getImageListForCVE(
// Infinite page to make sure we scan all repos in advance, before filtering results
// The CVE scan logic is called from here, not in the actual filter,
// this is because we shouldn't keep the DB locked while we wait on scan results
reposMeta, err := metaDB.GetMultipleRepoMeta(ctx,
func(repoMeta mTypes.RepoMetadata) bool { return true },
mTypes.PageInput{Limit: 0, Offset: 0, SortBy: mTypes.SortCriteria(gql_generated.SortCriteriaUpdateTime)},
)
reposMeta, err := metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true })
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
@ -419,8 +440,6 @@ func getImageListForCVE(
affectedImages = append(affectedImages, tagsInfo...)
}
imageList := make([]*gql_generated.ImageSummary, 0)
// We're not interested in other vulnerabilities
skip := convert.SkipQGLField{Vulnerabilities: true}
@ -440,29 +459,28 @@ func getImageListForCVE(
}
// Actual page requested by user
pageInput := mTypes.PageInput{
pageInput := pagination.PageInput{
Limit: safeDereferencing(requestedPage.Limit, 0),
Offset: safeDereferencing(requestedPage.Offset, 0),
SortBy: mTypes.SortCriteria(
SortBy: pagination.SortCriteria(
safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime),
),
}
// get all repos
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterTags(ctx,
FilterByTagInfo(affectedImages), localFilter, pageInput)
reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, FilterByTagInfo(affectedImages))
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
for _, repoMeta := range reposMeta {
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo)
imageList = append(imageList, imageSummaries...)
imageSummaries, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap,
indexDataMap, skip, cveInfo, localFilter, pageInput)
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
return &gql_generated.PaginatedImagesResult{
Results: imageList,
Results: imageSummaries,
Page: &gql_generated.PageInfo{
TotalCount: pageInfo.TotalCount,
ItemCount: pageInfo.ItemCount,
@ -514,33 +532,28 @@ func getImageListWithCVEFixed(
}
// Actual page requested by user
pageInput := mTypes.PageInput{
pageInput := pagination.PageInput{
Limit: safeDereferencing(requestedPage.Limit, 0),
Offset: safeDereferencing(requestedPage.Offset, 0),
SortBy: mTypes.SortCriteria(
SortBy: pagination.SortCriteria(
safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime),
),
}
// get all repos
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterTags(ctx,
FilterByTagInfo(tagsInfo), localFilter, pageInput,
)
reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, FilterByRepoAndTagInfo(repo, tagsInfo))
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
for _, repoMeta := range reposMeta {
if repoMeta.Name != repo {
continue
}
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo)
imageList = append(imageList, imageSummaries...)
imageSummaries, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap,
indexDataMap, skip, cveInfo, localFilter, pageInput)
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
return &gql_generated.PaginatedImagesResult{
Results: imageList,
Results: imageSummaries,
Page: &gql_generated.PageInfo{
TotalCount: pageInfo.TotalCount,
ItemCount: pageInfo.ItemCount,
@ -555,7 +568,6 @@ func repoListWithNewestImage(
requestedPage *gql_generated.PageInput,
metaDB mTypes.MetaDB,
) (*gql_generated.PaginatedReposResult, error) {
repos := []*gql_generated.RepoSummary{}
paginatedRepos := &gql_generated.PaginatedReposResult{}
if requestedPage == nil {
@ -566,29 +578,30 @@ func repoListWithNewestImage(
Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Results.NewestImage.Vulnerabilities"),
}
pageInput := mTypes.PageInput{
pageInput := pagination.PageInput{
Limit: safeDereferencing(requestedPage.Limit, 0),
Offset: safeDereferencing(requestedPage.Offset, 0),
SortBy: mTypes.SortCriteria(
SortBy: pagination.SortCriteria(
safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime),
),
}
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.SearchRepos(ctx, "", mTypes.Filter{}, pageInput)
reposMeta, manifestMetaMap, indexDataMap, err := metaDB.SearchRepos(ctx, "")
if err != nil {
return &gql_generated.PaginatedReposResult{}, err
}
for _, repoMeta := range reposMeta {
repoSummary := convert.RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, indexDataMap,
skip, cveInfo)
repos = append(repos, repoSummary)
repos, pageInfo, err := convert.PaginatedRepoMeta2RepoSummaries(ctx, reposMeta, manifestMetaMap, indexDataMap,
skip, cveInfo, mTypes.Filter{}, pageInput)
if err != nil {
return &gql_generated.PaginatedReposResult{}, err
}
paginatedRepos.Page = &gql_generated.PageInfo{
TotalCount: pageInfo.TotalCount,
ItemCount: pageInfo.ItemCount,
}
paginatedRepos.Results = repos
return paginatedRepos, nil
@ -640,9 +653,6 @@ func getFilteredPaginatedRepos(
requestedPage *gql_generated.PageInput,
metaDB mTypes.MetaDB,
) (*gql_generated.PaginatedReposResult, error) {
repos := []*gql_generated.RepoSummary{}
paginatedRepos := &gql_generated.PaginatedReposResult{}
if requestedPage == nil {
requestedPage = &gql_generated.PageInput{}
}
@ -651,32 +661,32 @@ func getFilteredPaginatedRepos(
Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Results.NewestImage.Vulnerabilities"),
}
pageInput := mTypes.PageInput{
pageInput := pagination.PageInput{
Limit: safeDereferencing(requestedPage.Limit, 0),
Offset: safeDereferencing(requestedPage.Offset, 0),
SortBy: mTypes.SortCriteria(
SortBy: pagination.SortCriteria(
safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime),
),
}
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterRepos(ctx, filterFn, pageInput)
reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterRepos(ctx, filterFn)
if err != nil {
return paginatedRepos, err
return &gql_generated.PaginatedReposResult{}, err
}
for _, repoMeta := range reposMeta {
repoSummary := convert.RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, indexDataMap,
skip, cveInfo)
repos = append(repos, repoSummary)
repos, pageInfo, err := convert.PaginatedRepoMeta2RepoSummaries(ctx, reposMeta, manifestMetaMap, indexDataMap,
skip, cveInfo, mTypes.Filter{}, pageInput)
if err != nil {
return &gql_generated.PaginatedReposResult{}, err
}
paginatedRepos.Page = &gql_generated.PageInfo{
TotalCount: pageInfo.TotalCount,
ItemCount: pageInfo.ItemCount,
}
paginatedRepos.Results = repos
return paginatedRepos, nil
return &gql_generated.PaginatedReposResult{
Results: repos,
Page: &gql_generated.PageInfo{
TotalCount: pageInfo.TotalCount,
ItemCount: pageInfo.ItemCount,
},
}, nil
}
func globalSearch(ctx context.Context, query string, metaDB mTypes.MetaDB, filter *gql_generated.Filter,
@ -684,7 +694,6 @@ func globalSearch(ctx context.Context, query string, metaDB mTypes.MetaDB, filte
) (*gql_generated.PaginatedReposResult, []*gql_generated.ImageSummary, []*gql_generated.LayerSummary, error,
) {
preloads := convert.GetPreloads(ctx)
repos := []*gql_generated.RepoSummary{}
paginatedRepos := gql_generated.PaginatedReposResult{}
images := []*gql_generated.ImageSummary{}
layers := []*gql_generated.LayerSummary{}
@ -709,24 +718,23 @@ func globalSearch(ctx context.Context, query string, metaDB mTypes.MetaDB, filte
Vulnerabilities: canSkipField(preloads, "Repos.NewestImage.Vulnerabilities"),
}
pageInput := mTypes.PageInput{
pageInput := pagination.PageInput{
Limit: safeDereferencing(requestedPage.Limit, 0),
Offset: safeDereferencing(requestedPage.Offset, 0),
SortBy: mTypes.SortCriteria(
SortBy: pagination.SortCriteria(
safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance),
),
}
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.SearchRepos(ctx, query, localFilter, pageInput)
reposMeta, manifestMetaMap, indexDataMap, err := metaDB.SearchRepos(ctx, query)
if err != nil {
return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err
}
for _, repoMeta := range reposMeta {
repoSummary := convert.RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, indexDataMap,
skip, cveInfo)
repos = append(repos, repoSummary)
repos, pageInfo, err := convert.PaginatedRepoMeta2RepoSummaries(ctx, reposMeta, manifestMetaMap, indexDataMap,
skip, cveInfo, localFilter, pageInput)
if err != nil {
return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err
}
paginatedRepos.Page = &gql_generated.PageInfo{
@ -740,25 +748,27 @@ func globalSearch(ctx context.Context, query string, metaDB mTypes.MetaDB, filte
Vulnerabilities: canSkipField(preloads, "Images.Vulnerabilities"),
}
pageInput := mTypes.PageInput{
pageInput := pagination.PageInput{
Limit: safeDereferencing(requestedPage.Limit, 0),
Offset: safeDereferencing(requestedPage.Offset, 0),
SortBy: mTypes.SortCriteria(
SortBy: pagination.SortCriteria(
safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance),
),
}
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.SearchTags(ctx, query, localFilter, pageInput)
reposMeta, manifestMetaMap, indexDataMap, err := metaDB.SearchTags(ctx, query)
if err != nil {
return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err
}
for _, repoMeta := range reposMeta {
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo)
images = append(images, imageSummaries...)
imageSummaries, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap,
indexDataMap, skip, cveInfo, localFilter, pageInput)
if err != nil {
return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err
}
images = imageSummaries
paginatedRepos.Page = &gql_generated.PageInfo{
TotalCount: pageInfo.TotalCount,
ItemCount: pageInfo.ItemCount,
@ -778,16 +788,14 @@ func derivedImageList(ctx context.Context, image string, digest *string, metaDB
requestedPage *gql_generated.PageInput,
cveInfo cveinfo.CveInfo, log log.Logger,
) (*gql_generated.PaginatedImagesResult, error) {
derivedList := make([]*gql_generated.ImageSummary, 0)
if requestedPage == nil {
requestedPage = &gql_generated.PageInput{}
}
pageInput := mTypes.PageInput{
pageInput := pagination.PageInput{
Limit: safeDereferencing(requestedPage.Limit, 0),
Offset: safeDereferencing(requestedPage.Offset, 0),
SortBy: mTypes.SortCriteria(
SortBy: pagination.SortCriteria(
safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime),
),
}
@ -811,26 +819,15 @@ func derivedImageList(ctx context.Context, image string, digest *string, metaDB
}
// we need all available tags
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterTags(ctx,
filterDerivedImages(searchedImage),
mTypes.Filter{},
pageInput)
reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, filterDerivedImages(searchedImage))
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
for _, repoMeta := range reposMeta {
summary := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo)
derivedList = append(derivedList, summary...)
}
if len(derivedList) == 0 {
log.Info().Msg("no images found")
return &gql_generated.PaginatedImagesResult{
Page: &gql_generated.PageInfo{},
Results: derivedList,
}, nil
derivedList, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, indexDataMap,
skip, cveInfo, mTypes.Filter{}, pageInput)
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
return &gql_generated.PaginatedImagesResult{
@ -892,16 +889,14 @@ func baseImageList(ctx context.Context, image string, digest *string, metaDB mTy
requestedPage *gql_generated.PageInput,
cveInfo cveinfo.CveInfo, log log.Logger,
) (*gql_generated.PaginatedImagesResult, error) {
imageSummaries := make([]*gql_generated.ImageSummary, 0)
if requestedPage == nil {
requestedPage = &gql_generated.PageInput{}
}
pageInput := mTypes.PageInput{
pageInput := pagination.PageInput{
Limit: safeDereferencing(requestedPage.Limit, 0),
Offset: safeDereferencing(requestedPage.Offset, 0),
SortBy: mTypes.SortCriteria(
SortBy: pagination.SortCriteria(
safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime),
),
}
@ -926,26 +921,15 @@ func baseImageList(ctx context.Context, image string, digest *string, metaDB mTy
}
// we need all available tags
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterTags(ctx,
filterBaseImages(searchedImage),
mTypes.Filter{},
pageInput)
reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, filterBaseImages(searchedImage))
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
for _, repoMeta := range reposMeta {
summary := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo)
imageSummaries = append(imageSummaries, summary...)
}
if len(imageSummaries) == 0 {
log.Info().Msg("no images found")
return &gql_generated.PaginatedImagesResult{
Results: imageSummaries,
Page: &gql_generated.PageInfo{},
}, nil
baseList, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, indexDataMap,
skip, cveInfo, mTypes.Filter{}, pageInput)
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
return &gql_generated.PaginatedImagesResult{
@ -953,7 +937,7 @@ func baseImageList(ctx context.Context, image string, digest *string, metaDB mTy
TotalCount: pageInfo.TotalCount,
ItemCount: pageInfo.ItemCount,
},
Results: imageSummaries,
Results: baseList,
}, nil
}
@ -1265,8 +1249,6 @@ func searchingForRepos(query string) bool {
func getImageList(ctx context.Context, repo string, metaDB mTypes.MetaDB, cveInfo cveinfo.CveInfo,
requestedPage *gql_generated.PageInput, log log.Logger, //nolint:unparam
) (*gql_generated.PaginatedImagesResult, error) {
imageList := make([]*gql_generated.ImageSummary, 0)
if requestedPage == nil {
requestedPage = &gql_generated.PageInput{}
}
@ -1275,32 +1257,26 @@ func getImageList(ctx context.Context, repo string, metaDB mTypes.MetaDB, cveInf
Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Images.Vulnerabilities"),
}
pageInput := mTypes.PageInput{
pageInput := pagination.PageInput{
Limit: safeDereferencing(requestedPage.Limit, 0),
Offset: safeDereferencing(requestedPage.Offset, 0),
SortBy: mTypes.SortCriteria(
SortBy: pagination.SortCriteria(
safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance),
),
}
// reposMeta, manifestMetaMap, err := metaDB.SearchRepos(ctx, repo, mTypes.Filter{}, pageInput)
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := metaDB.FilterTags(ctx,
reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool {
return true
},
mTypes.Filter{},
pageInput)
return repoMeta.Name == repo || repo == ""
})
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
for _, repoMeta := range reposMeta {
if repoMeta.Name != repo && repo != "" {
continue
}
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skip, cveInfo)
imageList = append(imageList, imageSummaries...)
imageList, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap,
indexDataMap, skip, cveInfo, mTypes.Filter{}, pageInput)
if err != nil {
return &gql_generated.PaginatedImagesResult{}, err
}
return &gql_generated.PaginatedImagesResult{

View file

@ -21,7 +21,6 @@ import (
"zotregistry.io/zot/pkg/extensions/search/gql_generated"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/boltdb"
"zotregistry.io/zot/pkg/meta/pagination"
mTypes "zotregistry.io/zot/pkg/meta/types"
localCtx "zotregistry.io/zot/pkg/requestcontext"
"zotregistry.io/zot/pkg/storage"
@ -35,13 +34,12 @@ func TestGlobalSearch(t *testing.T) {
const query = "repo1"
Convey("MetaDB SearchRepos error", func() {
mockMetaDB := mocks.MetaDBMock{
SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
SearchReposFn: func(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
return make([]mTypes.RepoMetadata, 0), make(map[string]mTypes.ManifestMetadata),
map[string]mTypes.IndexData{}, common.PageInfo{}, ErrTestError
map[string]mTypes.IndexData{}, ErrTestError
},
}
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
@ -55,12 +53,28 @@ func TestGlobalSearch(t *testing.T) {
So(repos.Results, ShouldBeEmpty)
})
Convey("paginated fail", func() {
pageInput := &gql_generated.PageInput{
Limit: ref(-1),
}
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, _, _, err := globalSearch(responseContext, "repo", mocks.MetaDBMock{}, &gql_generated.Filter{},
pageInput, mocks.CveInfoMock{}, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
_, _, _, err = globalSearch(responseContext, "repo:tag", mocks.MetaDBMock{}, &gql_generated.Filter{},
pageInput, mocks.CveInfoMock{}, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
})
Convey("MetaDB SearchRepo is successful", func() {
mockMetaDB := mocks.MetaDBMock{
SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
SearchReposFn: func(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
repos := []mTypes.RepoMetadata{
{
@ -120,7 +134,7 @@ func TestGlobalSearch(t *testing.T) {
},
}
return repos, manifestMetas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil
return repos, manifestMetas, map[string]mTypes.IndexData{}, nil
},
}
@ -148,10 +162,9 @@ func TestGlobalSearch(t *testing.T) {
Convey("MetaDB SearchRepo Bad manifest referenced", func() {
mockMetaDB := mocks.MetaDBMock{
SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
SearchReposFn: func(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
repos := []mTypes.RepoMetadata{
{
@ -183,7 +196,7 @@ func TestGlobalSearch(t *testing.T) {
},
}
return repos, manifestMetas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil
return repos, manifestMetas, map[string]mTypes.IndexData{}, nil
},
}
@ -222,10 +235,9 @@ func TestGlobalSearch(t *testing.T) {
Convey("MetaDB SearchRepo good manifest referenced and bad config blob", func() {
mockMetaDB := mocks.MetaDBMock{
SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
SearchReposFn: func(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
repos := []mTypes.RepoMetadata{
{
@ -257,7 +269,7 @@ func TestGlobalSearch(t *testing.T) {
},
}
return repos, manifestMetas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil
return repos, manifestMetas, map[string]mTypes.IndexData{}, nil
},
}
@ -295,13 +307,12 @@ func TestGlobalSearch(t *testing.T) {
Convey("MetaDB SearchTags gives error", func() {
mockMetaDB := mocks.MetaDBMock{
SearchTagsFn: func(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo,
SearchTagsFn: func(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
error,
) {
return make([]mTypes.RepoMetadata, 0), make(map[string]mTypes.ManifestMetadata),
map[string]mTypes.IndexData{}, common.PageInfo{}, ErrTestError
map[string]mTypes.IndexData{}, ErrTestError
},
}
const query = "repo1:1.0.1"
@ -319,10 +330,9 @@ func TestGlobalSearch(t *testing.T) {
Convey("MetaDB SearchTags is successful", func() {
mockMetaDB := mocks.MetaDBMock{
SearchTagsFn: func(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
SearchTagsFn: func(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
repos := []mTypes.RepoMetadata{
{
@ -376,7 +386,7 @@ func TestGlobalSearch(t *testing.T) {
},
}
return repos, manifestMetas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil
return repos, manifestMetas, map[string]mTypes.IndexData{}, nil
},
}
@ -408,13 +418,11 @@ func TestRepoListWithNewestImage(t *testing.T) {
Convey("RepoListWithNewestImage", t, func() {
Convey("MetaDB SearchRepos error", func() {
mockMetaDB := mocks.MetaDBMock{
SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo,
error,
SearchReposFn: func(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error,
) {
return make([]mTypes.RepoMetadata, 0), make(map[string]mTypes.ManifestMetadata),
map[string]mTypes.IndexData{}, common.PageInfo{}, ErrTestError
map[string]mTypes.IndexData{}, ErrTestError
},
}
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
@ -434,12 +442,24 @@ func TestRepoListWithNewestImage(t *testing.T) {
So(repos.Results, ShouldBeEmpty)
})
Convey("paginated fail", func() {
pageInput := &gql_generated.PageInput{
Limit: ref(-1),
}
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, err := repoListWithNewestImage(responseContext, mocks.CveInfoMock{}, log.NewLogger("debug", ""),
pageInput, mocks.MetaDBMock{})
So(err, ShouldNotBeNil)
})
Convey("MetaDB SearchRepo bad manifest referenced", func() {
mockMetaDB := mocks.MetaDBMock{
SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
SearchReposFn: func(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
repos := []mTypes.RepoMetadata{
{
@ -496,7 +516,7 @@ func TestRepoListWithNewestImage(t *testing.T) {
},
}
return repos, manifestMetas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil
return repos, manifestMetas, map[string]mTypes.IndexData{}, nil
},
}
@ -521,15 +541,10 @@ func TestRepoListWithNewestImage(t *testing.T) {
createTime := time.Now()
createTime2 := createTime.Add(time.Second)
mockMetaDB := mocks.MetaDBMock{
SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
SearchReposFn: func(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset,
requestedPage.SortBy)
So(err, ShouldBeNil)
repos := []mTypes.RepoMetadata{
{
Name: "repo1",
@ -567,16 +582,6 @@ func TestRepoListWithNewestImage(t *testing.T) {
},
}
for _, repoMeta := range repos {
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
UpdateTime: createTime,
})
createTime = createTime.Add(time.Second)
}
repos, _ = pageFinder.Page()
configBlob1, err := json.Marshal(ispec.Image{
Config: ispec.ImageConfig{
Labels: map[string]string{},
@ -607,7 +612,7 @@ func TestRepoListWithNewestImage(t *testing.T) {
},
}
return repos, manifestMetas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil
return repos, manifestMetas, map[string]mTypes.IndexData{}, nil
},
}
Convey("MetaDB missing requestedPage", func() {
@ -694,30 +699,42 @@ func TestGetFilteredPaginatedRepos(t *testing.T) {
log.NewLogger("debug", ""),
nil,
mocks.MetaDBMock{
FilterReposFn: func(ctx context.Context, filter mTypes.FilterRepoFunc, requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo,
FilterReposFn: func(ctx context.Context, filter mTypes.FilterRepoFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
error,
) {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
common.PageInfo{}, ErrTestError
ErrTestError
},
},
)
So(err, ShouldNotBeNil)
})
Convey("Paginated convert fails", t, func() {
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, err := getFilteredPaginatedRepos(responseContext,
mocks.CveInfoMock{},
func(repoMeta mTypes.RepoMetadata) bool { return true },
log.NewLogger("debug", ""),
&gql_generated.PageInput{Limit: ref(-1)},
mocks.MetaDBMock{},
)
So(err, ShouldNotBeNil)
})
}
func TestImageListForDigest(t *testing.T) {
Convey("getImageList", t, func() {
Convey("no page requested, FilterTagsFn returns error", func() {
mockSearchDB := mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo,
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
error,
) {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
common.PageInfo{}, ErrTestError
ErrTestError
},
}
@ -727,12 +744,19 @@ func TestImageListForDigest(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("Paginated convert fails", func() {
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, err := getImageListForDigest(responseContext, "invalid", mocks.MetaDBMock{}, mocks.CveInfoMock{},
&gql_generated.PageInput{Limit: ref(-1)})
So(err, ShouldNotBeNil)
})
Convey("invalid manifest blob", func() {
mockSearchDB := mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter,
requestedPage mTypes.PageInput,
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
repos := []mTypes.RepoMetadata{
{
@ -760,7 +784,7 @@ func TestImageListForDigest(t *testing.T) {
},
}
return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil
return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, nil
},
}
@ -779,10 +803,9 @@ func TestImageListForDigest(t *testing.T) {
manifestDigest := godigest.FromBytes(manifestBlob).String()
mockSearchDB := mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter,
requestedPage mTypes.PageInput,
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
repos := []mTypes.RepoMetadata{
{
@ -816,7 +839,7 @@ func TestImageListForDigest(t *testing.T) {
repos[0].Tags = matchedTags
return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil
return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, nil
},
}
@ -854,10 +877,9 @@ func TestImageListForDigest(t *testing.T) {
configDigest := godigest.FromBytes(configBlob)
mockSearchDB := mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter,
requestedPage mTypes.PageInput,
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
repos := []mTypes.RepoMetadata{
{
@ -896,7 +918,7 @@ func TestImageListForDigest(t *testing.T) {
repos[0].Tags = matchedTags
return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil
return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, nil
},
}
@ -930,10 +952,9 @@ func TestImageListForDigest(t *testing.T) {
layerDigest := godigest.Digest("validDigest")
mockSearchDB := mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter,
requestedPage mTypes.PageInput,
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
repos := []mTypes.RepoMetadata{
{
@ -974,7 +995,7 @@ func TestImageListForDigest(t *testing.T) {
repos[0].Tags = matchedTags
return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil
return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, nil
},
}
@ -1006,10 +1027,9 @@ func TestImageListForDigest(t *testing.T) {
So(err, ShouldBeNil)
mockSearchDB := mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter,
requestedPage mTypes.PageInput,
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
repos := []mTypes.RepoMetadata{
{
@ -1045,7 +1065,7 @@ func TestImageListForDigest(t *testing.T) {
repos[i].Tags = matchedTags
}
return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil
return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, nil
},
}
@ -1064,7 +1084,7 @@ func TestImageListForDigest(t *testing.T) {
imageSummaries, err := getImageListForDigest(responseContext, manifestDigest,
mockSearchDB, mocks.CveInfoMock{}, &pageInput)
So(err, ShouldBeNil)
So(len(imageSummaries.Results), ShouldEqual, 2)
So(len(imageSummaries.Results), ShouldEqual, 1)
})
Convey("valid imageListForDigest, multiple matching tags limited by pageInput", func() {
@ -1077,18 +1097,9 @@ func TestImageListForDigest(t *testing.T) {
So(err, ShouldBeNil)
mockSearchDB := mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo,
error,
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error,
) {
pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset,
requestedPage.SortBy)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
common.PageInfo{}, err
}
repos := []mTypes.RepoMetadata{
{
Name: "test",
@ -1122,14 +1133,10 @@ func TestImageListForDigest(t *testing.T) {
repos[i].Tags = matchedTags
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repo,
})
repos = append(repos, repo)
}
repos, _ = pageFinder.Page()
return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil
return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, nil
},
}
@ -1360,13 +1367,12 @@ func TestImageList(t *testing.T) {
testLogger := log.NewLogger("debug", "")
Convey("no page requested, SearchRepoFn returns error", func() {
mockSearchDB := mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo,
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
error,
) {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{},
map[string]mTypes.IndexData{}, common.PageInfo{}, ErrTestError
map[string]mTypes.IndexData{}, ErrTestError
},
}
@ -1377,12 +1383,21 @@ func TestImageList(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("Paginated convert fails", func() {
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, err := getImageList(responseContext, "test", mocks.MetaDBMock{}, mocks.CveInfoMock{},
&gql_generated.PageInput{Limit: ref(-1)}, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
})
Convey("valid repoList returned", func() {
mockSearchDB := mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter,
requestedPage mTypes.PageInput,
FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
repos := []mTypes.RepoMetadata{
{
@ -1427,7 +1442,12 @@ func TestImageList(t *testing.T) {
},
}
return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil
if !filterFunc(repos[0], manifestMetaDatas["digestTag1.0.1"]) {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{},
map[string]mTypes.IndexData{}, nil
}
return repos, manifestMetaDatas, map[string]mTypes.IndexData{}, nil
},
}
@ -1567,7 +1587,6 @@ func TestQueryResolverErrors(t *testing.T) {
},
mocks.MetaDBMock{
GetMultipleRepoMetaFn: func(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, error) {
return []mTypes.RepoMetadata{}, ErrTestError
},
@ -1591,12 +1610,12 @@ func TestQueryResolverErrors(t *testing.T) {
},
mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context,
filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput,
filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
common.PageInfo{}, ErrTestError
ErrTestError
},
},
mocks.CveInfoMock{},
@ -1618,12 +1637,12 @@ func TestQueryResolverErrors(t *testing.T) {
},
mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context,
filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput,
filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
common.PageInfo{}, ErrTestError
ErrTestError
},
},
mocks.CveInfoMock{},
@ -1644,12 +1663,11 @@ func TestQueryResolverErrors(t *testing.T) {
DefaultStore: mocks.MockedImageStore{},
},
mocks.MetaDBMock{
SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
SearchReposFn: func(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
return nil, nil, nil, common.PageInfo{}, ErrTestError
return nil, nil, nil, ErrTestError
},
},
mocks.CveInfoMock{},
@ -1668,12 +1686,11 @@ func TestQueryResolverErrors(t *testing.T) {
log,
storage.StoreController{},
mocks.MetaDBMock{
SearchReposFn: func(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
SearchReposFn: func(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
return nil, nil, nil, common.PageInfo{}, ErrTestError
return nil, nil, nil, ErrTestError
},
},
mocks.CveInfoMock{},
@ -1693,12 +1710,12 @@ func TestQueryResolverErrors(t *testing.T) {
storage.StoreController{},
mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context,
filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput,
filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
common.PageInfo{}, ErrTestError
ErrTestError
},
},
mocks.CveInfoMock{},
@ -1790,12 +1807,12 @@ func TestQueryResolverErrors(t *testing.T) {
storage.StoreController{},
mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context,
filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput,
filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
common.PageInfo{}, ErrTestError
ErrTestError
},
GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) {
return mTypes.RepoMetadata{
@ -2211,6 +2228,18 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo
So(expectedCves, ShouldContain, *cve.ID)
}
})
Convey("paginated fail", func() {
pageInput := &gql_generated.PageInput{
Limit: ref(-1),
}
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, err = getCVEListForImage(responseContext, "repo1:1.1.0", cveInfo, pageInput, "", log)
So(err, ShouldNotBeNil)
})
})
Convey("Get a list of images affected by a particular CVE ", t, func() {
@ -2260,6 +2289,19 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo
}
})
Convey("paginated fail", func() {
pageInput := &gql_generated.PageInput{
Limit: ref(-1),
}
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, err = getImageListForCVE(responseContext, "repo1:1.1.0", cveInfo, &gql_generated.Filter{},
pageInput, mocks.MetaDBMock{}, log)
So(err, ShouldNotBeNil)
})
Convey("Paginated requests", func() {
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover,
@ -2503,6 +2545,30 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo
So(len(images.Results), ShouldEqual, 0)
})
Convey("paginated fail", func() {
pageInput := &gql_generated.PageInput{
Limit: ref(-1),
}
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, err = getImageListWithCVEFixed(responseContext, "cve", "repo1:1.1.0", cveInfo, &gql_generated.Filter{},
pageInput, mocks.MetaDBMock{
GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) {
return mTypes.RepoMetadata{
Tags: map[string]mTypes.Descriptor{
"1.1.0": {
Digest: godigest.FromString("str").String(),
MediaType: ispec.MediaTypeImageManifest,
},
},
}, nil
},
}, log)
So(err, ShouldNotBeNil)
})
Convey("Paginated requests", func() {
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover,
@ -2690,7 +2756,6 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo
nil,
mocks.MetaDBMock{
GetMultipleRepoMetaFn: func(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, error) {
return []mTypes.RepoMetadata{{}}, nil
},
@ -2715,12 +2780,12 @@ func TestDerivedImageList(t *testing.T) {
Convey("MetaDB FilterTags error", t, func() {
mockSearchDB := mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context,
filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput,
filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
return make([]mTypes.RepoMetadata, 0), make(map[string]mTypes.ManifestMetadata),
make(map[string]mTypes.IndexData), common.PageInfo{}, ErrTestError
make(map[string]mTypes.IndexData), ErrTestError
},
GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) {
return mTypes.RepoMetadata{}, ErrTestError
@ -2739,6 +2804,32 @@ func TestDerivedImageList(t *testing.T) {
So(images.Results, ShouldBeEmpty)
})
Convey("paginated fail", t, func() {
pageInput := &gql_generated.PageInput{
Limit: ref(-1),
}
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, err := derivedImageList(responseContext, "repo1:1.0.1", nil,
mocks.MetaDBMock{
GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) {
return mTypes.RepoMetadata{
Tags: map[string]mTypes.Descriptor{
"1.0.1": {
Digest: godigest.FromString("str").String(),
MediaType: ispec.MediaTypeImageManifest,
},
},
}, nil
},
},
pageInput,
mocks.CveInfoMock{}, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
})
//nolint: dupl
Convey("MetaDB FilterTags no repo available", t, func() {
configBlob, err := json.Marshal(ispec.Image{
@ -2757,12 +2848,12 @@ func TestDerivedImageList(t *testing.T) {
mockSearchDB := mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context,
filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput,
filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
common.PageInfo{}, nil
nil
},
GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) {
return mTypes.RepoMetadata{
@ -2908,14 +2999,10 @@ func TestDerivedImageList(t *testing.T) {
}, nil
},
FilterTagsFn: func(ctx context.Context,
filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput,
filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset,
requestedPage.SortBy)
So(err, ShouldBeNil)
repos := []mTypes.RepoMetadata{
{
Name: "repo1",
@ -2941,14 +3028,9 @@ func TestDerivedImageList(t *testing.T) {
}
repos[i].Tags = matchedTags
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repo,
})
}
repos, pageInfo := pageFinder.Page()
return repos, manifestMetas, map[string]mTypes.IndexData{}, pageInfo, nil
return repos, manifestMetas, map[string]mTypes.IndexData{}, nil
},
}
@ -2988,12 +3070,12 @@ func TestBaseImageList(t *testing.T) {
Convey("MetaDB FilterTags error", t, func() {
mockSearchDB := mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context,
filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput,
filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
common.PageInfo{}, ErrTestError
ErrTestError
},
GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) {
return mTypes.RepoMetadata{}, ErrTestError
@ -3012,6 +3094,30 @@ func TestBaseImageList(t *testing.T) {
So(images.Results, ShouldBeEmpty)
})
Convey("paginated fail", t, func() {
pageInput := &gql_generated.PageInput{
Limit: ref(-1),
}
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, err := baseImageList(responseContext, "repo1:1.0.2", nil,
mocks.MetaDBMock{
GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) {
return mTypes.RepoMetadata{
Tags: map[string]mTypes.Descriptor{
"1.0.2": {
Digest: godigest.FromString("str").String(),
MediaType: ispec.MediaTypeImageManifest,
},
},
}, nil
},
},
pageInput, mocks.CveInfoMock{}, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
})
//nolint: dupl
Convey("MetaDB FilterTags no repo available", t, func() {
configBlob, err := json.Marshal(ispec.Image{
@ -3030,12 +3136,12 @@ func TestBaseImageList(t *testing.T) {
mockSearchDB := mocks.MetaDBMock{
FilterTagsFn: func(ctx context.Context,
filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput,
filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
common.PageInfo{}, nil
nil
},
GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) {
return mTypes.RepoMetadata{
@ -3175,14 +3281,10 @@ func TestBaseImageList(t *testing.T) {
}, nil
},
FilterTagsFn: func(ctx context.Context,
filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput,
filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset,
requestedPage.SortBy)
So(err, ShouldBeNil)
repos := []mTypes.RepoMetadata{
{
Name: "repo1",
@ -3208,15 +3310,9 @@ func TestBaseImageList(t *testing.T) {
}
repos[i].Tags = matchedTags
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repo,
})
}
repos, pageInfo := pageFinder.Page()
return repos, manifestMetas, map[string]mTypes.IndexData{}, pageInfo, nil
return repos, manifestMetas, map[string]mTypes.IndexData{}, nil
},
}
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
@ -3352,14 +3448,10 @@ func TestBaseImageList(t *testing.T) {
}, nil
},
FilterTagsFn: func(ctx context.Context,
filterFunc mTypes.FilterFunc, filter mTypes.Filter, requestedPage mTypes.PageInput,
filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
common.PageInfo, error,
error,
) {
pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset,
requestedPage.SortBy)
So(err, ShouldBeNil)
repos := []mTypes.RepoMetadata{
{
Name: "repo1",
@ -3384,13 +3476,9 @@ func TestBaseImageList(t *testing.T) {
}
repos[i].Tags = matchedTags
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repo,
})
}
return repos, manifestMetas, map[string]mTypes.IndexData{}, common.PageInfo{}, nil
return repos, manifestMetas, map[string]mTypes.IndexData{}, nil
},
}
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
@ -3464,7 +3552,7 @@ func TestExpandedRepoInfo(t *testing.T) {
})
So(err, ShouldBeNil)
goodIndexGoodManfestBlob, err := json.Marshal(ispec.Index{
goodIndexGoodManifestBlob, err := json.Marshal(ispec.Index{
Manifests: []ispec.Descriptor{
{
Digest: "goodManifest",
@ -3483,7 +3571,7 @@ func TestExpandedRepoInfo(t *testing.T) {
}, nil
case "goodIndexGoodManfest":
return mTypes.IndexData{
IndexBlob: goodIndexGoodManfestBlob,
IndexBlob: goodIndexGoodManifestBlob,
}, nil
default:
return mTypes.IndexData{}, nil
@ -3537,5 +3625,31 @@ func TestFilterFunctions(t *testing.T) {
)
So(ok, ShouldBeFalse)
})
Convey("FilterByTagInfo", func() {
fFunc := FilterByTagInfo([]cvemodel.TagInfo{
{
Descriptor: cvemodel.Descriptor{
MediaType: ispec.MediaTypeImageIndex,
},
Manifests: []cvemodel.DescriptorInfo{
{
Descriptor: cvemodel.Descriptor{
Digest: godigest.FromString("{}"),
},
},
},
},
})
ok := fFunc(mTypes.RepoMetadata{}, mTypes.ManifestMetadata{ManifestBlob: []byte("{}")})
So(ok, ShouldBeTrue)
})
})
}
func ref[T any](val T) *T {
ref := val
return &ref
}

View file

@ -19,7 +19,7 @@ type CVEResultForImage {
"""
Tag: String
"""
List of CVE objects which afect this specific image:tag
List of CVE objects which affect this specific image:tag
"""
CVEList: [CVE]
"""
@ -230,7 +230,7 @@ type ManifestSummary {
"""
Platform: Platform
"""
Total numer of image manifest downloads from this repository
Total number of image manifest downloads from this repository
"""
DownloadCount: Int
"""
@ -300,7 +300,7 @@ type RepoSummary {
"""
NewestImage: ImageSummary
"""
Total numer of image manifest downloads from this repository
Total number of image manifest downloads from this repository
"""
DownloadCount: Int
"""
@ -312,9 +312,13 @@ type RepoSummary {
"""
IsBookmarked: Boolean
"""
True if the repository is stared by the current user, fale otherwise
True if the repository is starred by the current user, false otherwise
"""
IsStarred: Boolean
"""
Rank represents how good the match was between the queried repo name and this repo summary.
"""
Rank: Int
}
"""

View file

@ -4583,7 +4583,7 @@ func TestMetaDBWhenPushingImages(t *testing.T) {
Layers: layers1,
}, baseURL, "repo1", "1.0.1",
)
So(err, ShouldBeNil)
So(err, ShouldNotBeNil)
})
})
}
@ -6426,3 +6426,81 @@ func TestImageSummary(t *testing.T) {
}
})
}
func TestUplodingArtifactsWithDifferentMediaType(t *testing.T) {
Convey("", t, func() {
port := GetFreePort()
baseURL := GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
conf.Storage.RootDirectory = t.TempDir()
conf.Storage.GC = false
defaultVal := true
conf.Extensions = &extconf.ExtensionConfig{
Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}, CVE: nil},
}
conf.Log = &config.LogConfig{Level: "debug", Output: "/dev/null"}
ctlr := api.NewController(conf)
ctlrManager := NewControllerManager(ctlr)
ctlrManager.StartAndWait(port)
defer ctlrManager.StopServer()
const customMediaType = "application/custom.media.type+json"
imageWithIncompatibleConfig := CreateImageWith().DefaultLayers().
CustomConfigBlob([]byte(`{"author": {"key": "val"}}`), customMediaType).Build()
defaultImage := CreateDefaultImage()
var configContent ispec.Image
err := json.Unmarshal(imageWithIncompatibleConfig.ConfigDescriptor.Data, &configContent)
So(err, ShouldNotBeNil)
err = UploadImage(imageWithIncompatibleConfig, baseURL, "repo", "incompatible-image")
So(err, ShouldBeNil)
err = UploadImage(defaultImage, baseURL, "repo", "default-image")
So(err, ShouldBeNil)
manifestData, err := ctlr.MetaDB.GetManifestData(imageWithIncompatibleConfig.ManifestDescriptor.Digest)
So(err, ShouldBeNil)
So(manifestData.ConfigBlob, ShouldEqual, imageWithIncompatibleConfig.ConfigDescriptor.Data)
So(manifestData.ManifestBlob, ShouldEqual, imageWithIncompatibleConfig.ManifestDescriptor.Data)
manifestData, err = ctlr.MetaDB.GetManifestData(defaultImage.ManifestDescriptor.Digest)
So(err, ShouldBeNil)
So(manifestData.ConfigBlob, ShouldEqual, defaultImage.ConfigDescriptor.Data)
So(manifestData.ManifestBlob, ShouldEqual, defaultImage.ManifestDescriptor.Data)
query := `
{
GlobalSearch(query:"repo:incompatible-image"){
Images {
RepoName Tag
Manifests {
Digest ConfigDigest
}
}
}
}`
resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query))
So(resp, ShouldNotBeNil)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct := &zcommon.GlobalSearchResultResp{}
err = json.Unmarshal(resp.Body(), responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Images), ShouldEqual, 1)
So(responseStruct.Images[0].Manifests[0].Digest, ShouldResemble,
imageWithIncompatibleConfig.ManifestDescriptor.Digest.String())
So(responseStruct.Images[0].Manifests[0].ConfigDigest, ShouldResemble,
imageWithIncompatibleConfig.ConfigDescriptor.Digest.String())
})
}

View file

@ -16,7 +16,6 @@ import (
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/common"
"zotregistry.io/zot/pkg/meta/pagination"
"zotregistry.io/zot/pkg/meta/signatures"
mTypes "zotregistry.io/zot/pkg/meta/types"
"zotregistry.io/zot/pkg/meta/version"
@ -297,17 +296,17 @@ func (bdw BoltDB) SetReferrer(repo string, referredDigest godigest.Digest, refer
return err
}
refferers := repoMeta.Referrers[referredDigest.String()]
referrers := repoMeta.Referrers[referredDigest.String()]
for i := range refferers {
if refferers[i].Digest == referrer.Digest {
for i := range referrers {
if referrers[i].Digest == referrer.Digest {
return nil
}
}
refferers = append(refferers, referrer)
referrers = append(referrers, referrer)
repoMeta.Referrers[referredDigest.String()] = refferers
repoMeta.Referrers[referredDigest.String()] = referrers
repoMetaBlob, err = json.Marshal(repoMeta)
if err != nil {
@ -645,19 +644,10 @@ func (bdw *BoltDB) GetRepoStars(repo string) (int, error) {
}
func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, error) {
var (
foundRepos = make([]mTypes.RepoMetadata, 0)
pageFinder pagination.PageFinder
)
foundRepos := []mTypes.RepoMetadata{}
pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return nil, err
}
err = bdw.DB.View(func(tx *bbolt.Tx) error {
err := bdw.DB.View(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetadataBucket))
cursor := buck.Cursor()
@ -675,14 +665,10 @@ func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta
}
if filter(repoMeta) {
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
foundRepos = append(foundRepos, repoMeta)
}
}
foundRepos, _ = pageFinder.Page()
return nil
})
@ -965,34 +951,21 @@ func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Di
return err
}
func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, zcommon.PageInfo,
error,
) {
func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) {
var (
foundRepos = make([]mTypes.RepoMetadata, 0)
foundManifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
foundindexDataMap = make(map[string]mTypes.IndexData)
pageFinder pagination.PageFinder
pageInfo zcommon.PageInfo
foundRepos = make([]mTypes.RepoMetadata, 0)
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
)
pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
zcommon.PageInfo{}, err
}
err = bdw.DB.View(func(transaction *bbolt.Tx) error {
err := bdw.DB.View(func(transaction *bbolt.Tx) error {
var (
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
repoBuck = transaction.Bucket([]byte(RepoMetadataBucket))
indexBuck = transaction.Bucket([]byte(IndexDataBucket))
manifestBuck = transaction.Bucket([]byte(ManifestDataBucket))
userBookmarks = getUserBookmarks(ctx, transaction)
userStars = getUserStars(ctx, transaction)
repoBuck = transaction.Bucket([]byte(RepoMetadataBucket))
indexBuck = transaction.Bucket([]byte(IndexDataBucket))
manifestBuck = transaction.Bucket([]byte(ManifestDataBucket))
userBookmarks = getUserBookmarks(ctx, transaction)
userStars = getUserStars(ctx, transaction)
)
cursor := repoBuck.Cursor()
@ -1009,22 +982,14 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, filter mT
return err
}
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
rank := common.RankRepoName(searchText, repoMeta.Name)
if rank == -1 {
continue
}
var (
repoDownloads = 0
repoLastUpdated = time.Time{}
osSet = map[string]bool{}
archSet = map[string]bool{}
noImageChecked = true
isSigned = false
)
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
repoMeta.Rank = rank
for tag, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
@ -1038,24 +1003,6 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, filter mT
manifestDigest, err)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w",
manifestDigest, err)
}
repoDownloads += manifestFilterData.DownloadCount
for _, os := range manifestFilterData.OsList {
osSet[os] = true
}
for _, arch := range manifestFilterData.ArchList {
archSet[arch] = true
}
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, manifestFilterData)
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
@ -1074,27 +1021,18 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, filter mT
repoName, tag, err)
}
// this also updates manifestMetadataMap
indexFilterData, err := collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, manifestMetadataMap,
manifestBuck)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for index with digest %s %w",
indexDigest, err)
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest.String(),
manifestMetadataMap, manifestBuck)
if err != nil {
return err
}
manifestMetadataMap[manifest.Digest.String()] = manifestMeta
}
for _, arch := range indexFilterData.ArchList {
archSet[arch] = true
}
for _, os := range indexFilterData.OsList {
osSet[os] = true
}
repoDownloads += indexFilterData.DownloadCount
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, indexFilterData)
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type")
@ -1103,37 +1041,13 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, filter mT
}
}
repoFilterData := mTypes.FilterData{
OsList: common.GetMapKeys(osSet),
ArchList: common.GetMapKeys(archSet),
LastUpdated: repoLastUpdated,
DownloadCount: repoDownloads,
IsSigned: isSigned,
IsBookmarked: repoMeta.IsBookmarked,
IsStarred: repoMeta.IsStarred,
}
if !common.AcceptedByFilter(filter, repoFilterData) {
continue
}
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
Rank: rank,
Downloads: repoDownloads,
UpdateTime: repoLastUpdated,
})
foundRepos = append(foundRepos, repoMeta)
}
foundRepos, pageInfo = pageFinder.Page()
foundManifestMetadataMap, foundindexDataMap, err = common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
return err
return nil
})
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
return foundRepos, manifestMetadataMap, indexDataMap, err
}
func fetchManifestMetaWithCheck(repoMeta mTypes.RepoMetadata, manifestDigest string,
@ -1187,95 +1101,6 @@ func fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]mTypes.
return indexData, err
}
func collectImageManifestFilterData(digest string, repoMeta mTypes.RepoMetadata,
manifestMeta mTypes.ManifestMetadata,
) (mTypes.FilterData, error) {
// get fields related to filtering
var (
configContent ispec.Image
osList []string
archList []string
)
err := json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return mTypes.FilterData{},
fmt.Errorf("metadb: error while unmarshaling config content %w", err)
}
if configContent.OS != "" {
osList = append(osList, configContent.OS)
}
if configContent.Architecture != "" {
archList = append(archList, configContent.Architecture)
}
return mTypes.FilterData{
DownloadCount: repoMeta.Statistics[digest].DownloadCount,
OsList: osList,
ArchList: archList,
LastUpdated: common.GetImageLastUpdatedTimestamp(configContent),
IsSigned: common.CheckIsSigned(repoMeta.Signatures[digest]),
}, nil
}
func collectImageIndexFilterInfo(indexDigest string, repoMeta mTypes.RepoMetadata,
indexData mTypes.IndexData, manifestMetadataMap map[string]mTypes.ManifestMetadata,
manifestBuck *bbolt.Bucket,
) (mTypes.FilterData, error) {
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return mTypes.FilterData{},
fmt.Errorf("metadb: error while unmarshaling index content for digest %s %w", indexDigest, err)
}
var (
indexLastUpdated time.Time
firstManifestChecked = false
indexOsList = []string{}
indexArchList = []string{}
)
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest.String(),
manifestMetadataMap, manifestBuck)
if err != nil {
return mTypes.FilterData{},
fmt.Errorf("%w", err)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest.String(), repoMeta,
manifestMeta)
if err != nil {
return mTypes.FilterData{},
fmt.Errorf("%w", err)
}
indexOsList = append(indexOsList, manifestFilterData.OsList...)
indexArchList = append(indexArchList, manifestFilterData.ArchList...)
if !firstManifestChecked || indexLastUpdated.Before(manifestFilterData.LastUpdated) {
indexLastUpdated = manifestFilterData.LastUpdated
firstManifestChecked = true
}
manifestMetadataMap[manifest.Digest.String()] = manifestMeta
}
return mTypes.FilterData{
DownloadCount: repoMeta.Statistics[indexDigest].DownloadCount,
LastUpdated: indexLastUpdated,
OsList: indexOsList,
ArchList: indexArchList,
IsSigned: common.CheckIsSigned(repoMeta.Signatures[indexDigest]),
}, nil
}
func NewManifestMetadata(manifestDigest string, repoMeta mTypes.RepoMetadata,
manifestData mTypes.ManifestData,
) mTypes.ManifestMetadata {
@ -1294,28 +1119,16 @@ func NewManifestMetadata(manifestDigest string, repoMeta mTypes.RepoMetadata,
return manifestMeta
}
func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
zcommon.PageInfo, error,
func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error,
) {
var (
foundRepos = make([]mTypes.RepoMetadata, 0)
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
foundManifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
foundindexDataMap = make(map[string]mTypes.IndexData)
pageFinder pagination.PageFinder
pageInfo zcommon.PageInfo
foundRepos = make([]mTypes.RepoMetadata, 0)
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
)
pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
zcommon.PageInfo{}, err
}
err = bdw.DB.View(func(transaction *bbolt.Tx) error {
err := bdw.DB.View(func(transaction *bbolt.Tx) error {
var (
repoBuck = transaction.Bucket([]byte(RepoMetadataBucket))
indexBuck = transaction.Bucket([]byte(IndexDataBucket))
@ -1354,16 +1167,6 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
return fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", manifestDigest, err)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w",
manifestDigest, err)
}
if !common.AcceptedByFilter(filter, imageFilterData) {
continue
}
if filterFunc(repoMeta, manifestMeta) {
matchedTags[tag] = descriptor
manifestMetadataMap[manifestDigest] = manifestMeta
@ -1393,16 +1196,6 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
return fmt.Errorf("metadb: error while getting manifest data for digest %s %w", manifestDigest, err)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w",
manifestDigest, err)
}
if !common.AcceptedByFilter(filter, manifestFilterData) {
continue
}
if filterFunc(repoMeta, manifestMeta) {
matchedManifests = append(matchedManifests, manifest)
manifestMetadataMap[manifestDigest] = manifestMeta
@ -1435,44 +1228,21 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
repoMeta.Tags = matchedTags
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
foundRepos = append(foundRepos, repoMeta)
}
foundRepos, pageInfo = pageFinder.Page()
foundManifestMetadataMap, foundindexDataMap, err = common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
return err
return nil
})
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
return foundRepos, manifestMetadataMap, indexDataMap, err
}
func (bdw *BoltDB) FilterRepos(ctx context.Context,
filter mTypes.FilterRepoFunc,
requestedPage mTypes.PageInput,
) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, zcommon.PageInfo, error,
func (bdw *BoltDB) FilterRepos(ctx context.Context, filter mTypes.FilterRepoFunc) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error,
) {
var (
foundRepos = make([]mTypes.RepoMetadata, 0)
pageFinder pagination.PageFinder
pageInfo zcommon.PageInfo
)
foundRepos := make([]mTypes.RepoMetadata, 0)
pageFinder, err := pagination.NewBaseRepoPageFinder(
requestedPage.Limit,
requestedPage.Offset,
requestedPage.SortBy,
)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, pageInfo, err
}
err = bdw.DB.View(func(tx *bbolt.Tx) error {
err := bdw.DB.View(func(tx *bbolt.Tx) error {
var (
buck = tx.Bucket([]byte(RepoMetadataBucket))
cursor = buck.Cursor()
@ -1496,49 +1266,32 @@ func (bdw *BoltDB) FilterRepos(ctx context.Context,
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
if filter(repoMeta) {
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
foundRepos = append(foundRepos, repoMeta)
}
}
foundRepos, pageInfo = pageFinder.Page()
return nil
})
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, pageInfo, err
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, err
}
foundManifestMetadataMap, foundIndexDataMap, err := common.FetchDataForRepos(bdw, foundRepos)
return foundRepos, foundManifestMetadataMap, foundIndexDataMap, pageInfo, err
return foundRepos, foundManifestMetadataMap, foundIndexDataMap, err
}
func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, zcommon.PageInfo, error) {
func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) {
var (
foundRepos = make([]mTypes.RepoMetadata, 0)
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
foundManifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
foundindexDataMap = make(map[string]mTypes.IndexData)
pageInfo zcommon.PageInfo
pageFinder pagination.PageFinder
foundRepos = make([]mTypes.RepoMetadata, 0)
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
)
pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
zcommon.PageInfo{}, err
}
searchedRepo, searchedTag, err := common.GetRepoTag(searchText)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
zcommon.PageInfo{},
fmt.Errorf("metadb: error while parsing search text, invalid format %w", err)
}
@ -1547,143 +1300,99 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, filter mTy
repoBuck = transaction.Bucket([]byte(RepoMetadataBucket))
indexBuck = transaction.Bucket([]byte(IndexDataBucket))
manifestBuck = transaction.Bucket([]byte(ManifestDataBucket))
cursor = repoBuck.Cursor()
userBookmarks = getUserBookmarks(ctx, transaction)
userStars = getUserStars(ctx, transaction)
)
repoName, repoMetaBlob := cursor.Seek([]byte(searchedRepo))
repoName, repoMetaBlob := repoBuck.Cursor().Seek([]byte(searchedRepo))
for ; repoName != nil; repoName, repoMetaBlob = cursor.Next() {
if ok, err := localCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil {
if string(repoName) != searchedRepo {
return nil
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil {
return err
}
repoMeta := mTypes.RepoMetadata{}
err := json.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil {
return err
}
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
matchedTags := make(map[string]mTypes.Descriptor)
for tag, descriptor := range repoMeta.Tags {
if !strings.HasPrefix(tag, searchedTag) {
continue
}
repoMeta := mTypes.RepoMetadata{}
matchedTags[tag] = descriptor
err := json.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil {
return err
}
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
if string(repoName) != searchedRepo {
continue
}
matchedTags := make(map[string]mTypes.Descriptor)
for tag, descriptor := range repoMeta.Tags {
if !strings.HasPrefix(tag, searchedTag) {
continue
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w",
manifestDigest, err)
}
matchedTags[tag] = descriptor
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck)
if err != nil {
return fmt.Errorf("metadb: error fetching index data for index with digest %s %w",
indexDigest, err)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for index with digest %s %w",
indexDigest, err)
}
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w",
return fmt.Errorf("metadb: error fetching from db manifest meta for manifest with digest %s %w",
manifestDigest, err)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w",
manifestDigest, err)
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
continue
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck)
if err != nil {
return fmt.Errorf("metadb: error fetching index data for index with digest %s %w",
indexDigest, err)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for index with digest %s %w",
indexDigest, err)
}
manifestHasBeenMatched := false
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return fmt.Errorf("metadb: error fetching from db manifest meta for manifest with digest %s %w",
manifestDigest, err)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w",
manifestDigest, err)
}
manifestMetadataMap[manifestDigest] = manifestMeta
if common.AcceptedByFilter(filter, manifestFilterData) {
manifestHasBeenMatched = true
}
}
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
delete(manifestMetadataMap, manifest.Digest.String())
}
continue
}
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type")
continue
manifestMetadataMap[manifestDigest] = manifestMeta
}
}
if len(matchedTags) == 0 {
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type")
continue
}
repoMeta.Tags = matchedTags
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
}
foundRepos, pageInfo = pageFinder.Page()
if len(matchedTags) == 0 {
return nil
}
foundManifestMetadataMap, foundindexDataMap, err = common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
repoMeta.Tags = matchedTags
foundRepos = append(foundRepos, repoMeta)
return nil
})
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
return foundRepos, manifestMetadataMap, indexDataMap, err
}
func (bdw *BoltDB) ToggleStarRepo(ctx context.Context, repo string) (mTypes.ToggleState, error) {

View file

@ -314,8 +314,8 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterRepos(context.Background(),
func(repoMeta mTypes.RepoMetadata) bool { return true }, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.FilterRepos(context.Background(),
func(repoMeta mTypes.RepoMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
@ -453,7 +453,7 @@ func TestWrapperErrors(t *testing.T) {
_, err = boltdbWrapper.GetMultipleRepoMeta(context.TODO(), func(repoMeta mTypes.RepoMetadata) bool {
return true
}, mTypes.PageInput{})
})
So(err, ShouldNotBeNil)
})
@ -619,7 +619,7 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "")
So(err, ShouldNotBeNil)
err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
@ -660,47 +660,10 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1")
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo2", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket))
dataBuck := tx.Bucket([]byte(boltdb.ManifestDataBucket))
manifestMeta := mTypes.ManifestMetadata{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("wrong json"),
Signatures: mTypes.ManifestSignatures{},
}
manifestMetaBlob, err := json.Marshal(manifestMeta)
if err != nil {
return err
}
err = dataBuck.Put([]byte("dig1"), manifestMetaBlob)
if err != nil {
return err
}
repoMeta = mTypes.RepoMetadata{
Name: "repo1",
Tags: map[string]mTypes.Descriptor{
"tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
},
Signatures: map[string]mTypes.ManifestSignatures{},
}
repoMetaBlob, err = json.Marshal(repoMeta)
So(err, ShouldBeNil)
return repoBuck.Put([]byte("repo1"), repoMetaBlob)
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo2")
So(err, ShouldNotBeNil)
})
@ -714,10 +677,10 @@ func TestWrapperErrors(t *testing.T) {
err = setBadIndexData(boltdbWrapper.DB, indexDigest.String())
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchRepos(ctx, "")
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:")
So(err, ShouldNotBeNil)
})
@ -732,49 +695,10 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchRepos(ctx, "")
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("Good index data, bad manifest inside index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetIndexData(indexDigest, mTypes.IndexData{
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:")
So(err, ShouldNotBeNil)
})
})
@ -789,10 +713,10 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchTags(ctx, "")
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:")
So(err, ShouldNotBeNil)
err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
@ -866,13 +790,10 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:")
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo2:", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo3:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo2:")
So(err, ShouldNotBeNil)
})
@ -886,11 +807,8 @@ func TestWrapperErrors(t *testing.T) {
err = setBadIndexData(boltdbWrapper.DB, indexDigest.String())
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
@ -905,11 +823,8 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
@ -945,77 +860,10 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return false },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return false })
So(err, ShouldBeNil)
})
Convey("FilterTags bad config blob in image with a single manifest", func() {
manifestDigest := digest.FromString("manifestDigestBadConfig")
err := boltdbWrapper.SetRepoReference("repo", "tag1", //nolint:contextcheck
manifestDigest, ispec.MediaTypeImageManifest,
)
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigest, mTypes.ManifestData{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad blob"),
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
So(err, ShouldNotBeNil)
})
Convey("FilterTags bad config blob in index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndexGoodConfig")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndexBadConfig")
)
err := boltdbWrapper.SetRepoReference("repo", "tag1", //nolint:contextcheck
indexDigest, ispec.MediaTypeImageIndex,
)
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetIndexData(indexDigest, mTypes.IndexData{
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad blob"),
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
So(err, ShouldNotBeNil)
})
})
Convey("ToggleStarRepo bad context errors", func() {
@ -1161,18 +1009,14 @@ func TestWrapperErrors(t *testing.T) {
err := boltdbWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchRepos(ctx, "")
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:")
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterTags(
ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = boltdbWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true })
So(err, ShouldBeNil)
})

View file

@ -142,34 +142,6 @@ func RankRepoName(searchText string, repoName string) int {
return -1
}
func GetImageLastUpdatedTimestamp(configContent ispec.Image) time.Time {
var timeStamp *time.Time
if configContent.Created != nil && !configContent.Created.IsZero() {
return *configContent.Created
}
if len(configContent.History) != 0 {
timeStamp = configContent.History[len(configContent.History)-1].Created
}
if timeStamp == nil {
timeStamp = &time.Time{}
}
return *timeStamp
}
func CheckIsSigned(signatures mTypes.ManifestSignatures) bool {
for _, signatures := range signatures {
if len(signatures) > 0 {
return true
}
}
return false
}
func GetRepoTag(searchText string) (string, string, error) {
const repoTagCount = 2
@ -185,66 +157,6 @@ func GetRepoTag(searchText string) (string, string, error) {
return repo, tag, nil
}
func GetMapKeys[K comparable, V any](genericMap map[K]V) []K {
keys := make([]K, 0, len(genericMap))
for k := range genericMap {
keys = append(keys, k)
}
return keys
}
// acceptedByFilter checks that data contains at least 1 element of each filter
// criteria(os, arch) present in filter.
func AcceptedByFilter(filter mTypes.Filter, data mTypes.FilterData) bool {
if filter.Arch != nil {
foundArch := false
for _, arch := range filter.Arch {
foundArch = foundArch || containsString(data.ArchList, *arch)
}
if !foundArch {
return false
}
}
if filter.Os != nil {
foundOs := false
for _, os := range filter.Os {
foundOs = foundOs || containsString(data.OsList, *os)
}
if !foundOs {
return false
}
}
if filter.HasToBeSigned != nil && *filter.HasToBeSigned != data.IsSigned {
return false
}
if filter.IsBookmarked != nil && *filter.IsBookmarked != data.IsBookmarked {
return false
}
if filter.IsStarred != nil && *filter.IsStarred != data.IsStarred {
return false
}
return true
}
func containsString(strSlice []string, str string) bool {
for _, val := range strSlice {
if strings.EqualFold(val, str) {
return true
}
}
return false
}
func GetReferredSubject(descriptorBlob []byte) (godigest.Digest, bool) {
var manifest ispec.Manifest
@ -421,3 +333,14 @@ func GetImageDescriptor(metaDB mTypes.MetaDB, repo, tag string) (mTypes.Descript
return imageDescriptor, nil
}
func InitializeImageConfig(blob []byte) ispec.Image {
var configContent ispec.Image
err := json.Unmarshal(blob, &configContent)
if err != nil {
return ispec.Image{}
}
return configContent
}

View file

@ -118,6 +118,39 @@ func TestUtils(t *testing.T) {
})
Convey("FilterDataByRepo", t, func() {
Convey("Functionality", func() {
_, _, err := common.FilterDataByRepo(
[]mTypes.RepoMetadata{{
Tags: map[string]mTypes.Descriptor{
"manifest": {
Digest: "manifestDigest",
MediaType: ispec.MediaTypeImageManifest,
},
"index": {
Digest: "indexDigest",
MediaType: ispec.MediaTypeImageIndex,
},
"rand": {
Digest: "randDigest",
MediaType: "rand",
},
},
}},
map[string]mTypes.ManifestMetadata{},
map[string]mTypes.IndexData{
"indexDigest": {
IndexBlob: []byte(`{
"manifests": [
{
"digest": "manifestDigest"
}
]
}`),
},
},
)
So(err, ShouldBeNil)
})
Convey("Errors", func() {
// Unmarshal index data error
_, _, err := common.FilterDataByRepo(

View file

@ -19,7 +19,6 @@ import (
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/common"
"zotregistry.io/zot/pkg/meta/pagination"
"zotregistry.io/zot/pkg/meta/signatures"
mTypes "zotregistry.io/zot/pkg/meta/types"
"zotregistry.io/zot/pkg/meta/version"
@ -328,17 +327,17 @@ func (dwr DynamoDB) SetReferrer(repo string, referredDigest godigest.Digest, ref
}
}
refferers := repoMeta.Referrers[referredDigest.String()]
referrers := repoMeta.Referrers[referredDigest.String()]
for i := range refferers {
if refferers[i].Digest == referrer.Digest {
for i := range referrers {
if referrers[i].Digest == referrer.Digest {
return nil
}
}
refferers = append(refferers, referrer)
referrers = append(referrers, referrer)
repoMeta.Referrers[referredDigest.String()] = refferers
repoMeta.Referrers[referredDigest.String()] = referrers
return dwr.SetRepoMeta(repo, repoMeta)
}
@ -787,27 +786,21 @@ func (dwr *DynamoDB) DeleteSignature(repo string, signedManifestDigest godigest.
}
func (dwr *DynamoDB) GetMultipleRepoMeta(ctx context.Context,
filter func(repoMeta mTypes.RepoMetadata) bool, requestedPage mTypes.PageInput,
filter func(repoMeta mTypes.RepoMetadata) bool,
) ([]mTypes.RepoMetadata, error) {
var (
foundRepos = []mTypes.RepoMetadata{}
repoMetaAttributeIterator AttributesIterator
pageFinder pagination.PageFinder
)
repoMetaAttributeIterator = NewBaseDynamoAttributesIterator(
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
)
pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return nil, err
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
// log
return []mTypes.RepoMetadata{}, err
}
@ -823,26 +816,20 @@ func (dwr *DynamoDB) GetMultipleRepoMeta(ctx context.Context,
}
if filter(repoMeta) {
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
foundRepos = append(foundRepos, repoMeta)
}
}
foundRepos, _ := pageFinder.Page()
return foundRepos, err
}
func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, zcommon.PageInfo, error) {
func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) {
var (
repos = []mTypes.RepoMetadata{}
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
repoMetaAttributeIterator AttributesIterator
pageFinder pagination.PageFinder
pageInfo zcommon.PageInfo
userBookmarks = getUserBookmarks(ctx, dwr)
userStars = getUserStars(ctx, dwr)
@ -852,18 +839,12 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
)
pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
err
}
var repoMeta mTypes.RepoMetadata
@ -871,7 +852,7 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
@ -885,15 +866,7 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
var (
repoDownloads = 0
repoLastUpdated = time.Time{}
osSet = map[string]bool{}
archSet = map[string]bool{}
noImageChecked = true
isSigned = false
)
repoMeta.Rank = rank
for _, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
@ -904,64 +877,37 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
err
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
repoDownloads += manifestFilterData.DownloadCount
for _, os := range manifestFilterData.OsList {
osSet[os] = true
}
for _, arch := range manifestFilterData.ArchList {
archSet[arch] = true
}
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, manifestFilterData)
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
indexData, err := dwr.fetchIndexDataWithCheck(descriptor.Digest, indexDataMap) //nolint:contextcheck
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
err
}
// this also updates manifestMetadataMap
indexFilterData, err := dwr.collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, //nolint:contextcheck
manifestMetadataMap)
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
fmt.Errorf("metadb: error while unmarshaling index content for digest %s %w", descriptor.Digest, err)
}
for _, arch := range indexFilterData.ArchList {
archSet[arch] = true
for _, manifest := range indexContent.Manifests {
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifest.Digest.String(), //nolint: contextcheck
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
err
}
manifestMetadataMap[manifest.Digest.String()] = manifestMeta
}
for _, os := range indexFilterData.OsList {
osSet[os] = true
}
repoDownloads += indexFilterData.DownloadCount
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, indexFilterData)
indexDataMap[indexDigest] = indexData
indexDataMap[descriptor.Digest] = indexData
default:
dwr.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type")
@ -969,32 +915,10 @@ func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, filter
}
}
repoFilterData := mTypes.FilterData{
OsList: common.GetMapKeys(osSet),
ArchList: common.GetMapKeys(archSet),
LastUpdated: repoLastUpdated,
DownloadCount: repoDownloads,
IsSigned: isSigned,
}
if !common.AcceptedByFilter(filter, repoFilterData) {
continue
}
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
Rank: rank,
Downloads: repoDownloads,
UpdateTime: repoLastUpdated,
})
repos = append(repos, repoMeta)
}
foundRepos, pageInfo := pageFinder.Page()
foundManifestMetadataMap, foundindexDataMap, err := common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
return repos, manifestMetadataMap, indexDataMap, nil
}
func getUserStars(ctx context.Context, dwr *DynamoDB) []string {
@ -1035,38 +959,6 @@ func (dwr *DynamoDB) fetchManifestMetaWithCheck(repoName string, manifestDigest
return manifestMeta, nil
}
func collectImageManifestFilterData(digest string, repoMeta mTypes.RepoMetadata,
manifestMeta mTypes.ManifestMetadata,
) (mTypes.FilterData, error) {
// get fields related to filtering
var (
configContent ispec.Image
osList []string
archList []string
)
err := json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
if err != nil {
return mTypes.FilterData{}, fmt.Errorf("metadb: error while unmarshaling config content %w", err)
}
if configContent.OS != "" {
osList = append(osList, configContent.OS)
}
if configContent.Architecture != "" {
archList = append(archList, configContent.Architecture)
}
return mTypes.FilterData{
DownloadCount: repoMeta.Statistics[digest].DownloadCount,
OsList: osList,
ArchList: archList,
LastUpdated: common.GetImageLastUpdatedTimestamp(configContent),
IsSigned: common.CheckIsSigned(repoMeta.Signatures[digest]),
}, nil
}
func (dwr *DynamoDB) fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]mTypes.IndexData,
) (mTypes.IndexData, error) {
var (
@ -1087,72 +979,14 @@ func (dwr *DynamoDB) fetchIndexDataWithCheck(indexDigest string, indexDataMap ma
return indexData, err
}
func (dwr *DynamoDB) collectImageIndexFilterInfo(indexDigest string, repoMeta mTypes.RepoMetadata,
indexData mTypes.IndexData, manifestMetadataMap map[string]mTypes.ManifestMetadata,
) (mTypes.FilterData, error) {
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return mTypes.FilterData{},
fmt.Errorf("metadb: error while unmarshaling index content for digest %s %w", indexDigest, err)
}
var (
indexLastUpdated time.Time
firstManifestChecked = false
indexOsList = []string{}
indexArchList = []string{}
)
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest.String(),
manifestMetadataMap)
if err != nil {
return mTypes.FilterData{},
fmt.Errorf("%w", err)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest.String(), repoMeta,
manifestMeta)
if err != nil {
return mTypes.FilterData{},
fmt.Errorf("%w", err)
}
indexOsList = append(indexOsList, manifestFilterData.OsList...)
indexArchList = append(indexArchList, manifestFilterData.ArchList...)
if !firstManifestChecked || indexLastUpdated.Before(manifestFilterData.LastUpdated) {
indexLastUpdated = manifestFilterData.LastUpdated
firstManifestChecked = true
}
manifestMetadataMap[manifest.Digest.String()] = manifestMeta
}
return mTypes.FilterData{
DownloadCount: repoMeta.Statistics[indexDigest].DownloadCount,
LastUpdated: indexLastUpdated,
OsList: indexOsList,
ArchList: indexArchList,
IsSigned: common.CheckIsSigned(repoMeta.Signatures[indexDigest]),
}, nil
}
func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
zcommon.PageInfo, error,
func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error,
) {
var (
foundRepos = make([]mTypes.RepoMetadata, 0)
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
repoMetaAttributeIterator AttributesIterator
pageFinder pagination.PageFinder
pageInfo zcommon.PageInfo
userBookmarks = getUserBookmarks(ctx, dwr)
userStars = getUserStars(ctx, dwr)
)
@ -1161,18 +995,12 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
)
pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
err
}
var repoMeta mTypes.RepoMetadata
@ -1180,7 +1008,7 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
@ -1201,23 +1029,9 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s \n%w", manifestDigest, err)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w", manifestDigest, err)
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
continue
}
if filterFunc(repoMeta, manifestMeta) {
matchedTags[tag] = descriptor
manifestMetadataMap[manifestDigest] = manifestMeta
@ -1228,7 +1042,6 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error while getting index data for digest %s %w", indexDigest, err)
}
@ -1237,7 +1050,6 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err)
}
@ -1250,21 +1062,9 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w metadb: error while getting manifest data for digest %s", err, manifestDigest)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error collecting filter data for manifest with digest %s %w", manifestDigest, err)
}
if !common.AcceptedByFilter(filter, manifestFilterData) {
continue
}
if filterFunc(repoMeta, manifestMeta) {
matchedManifests = append(matchedManifests, manifest)
manifestMetadataMap[manifestDigest] = manifestMeta
@ -1277,7 +1077,7 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
indexBlob, err := json.Marshal(indexContent)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
err
}
indexData.IndexBlob = indexBlob
@ -1298,29 +1098,17 @@ func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFun
repoMeta.Tags = matchedTags
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
foundRepos = append(foundRepos, repoMeta)
}
foundRepos, pageInfo := pageFinder.Page()
foundManifestMetadataMap, foundindexDataMap, err := common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
return foundRepos, manifestMetadataMap, indexDataMap, err
}
func (dwr *DynamoDB) FilterRepos(ctx context.Context,
filter mTypes.FilterRepoFunc,
requestedPage mTypes.PageInput,
) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
zcommon.PageInfo, error,
) {
func (dwr *DynamoDB) FilterRepos(ctx context.Context, filter mTypes.FilterRepoFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) {
var (
foundRepos = []mTypes.RepoMetadata{}
repoMetaAttributeIterator AttributesIterator
pageInfo zcommon.PageInfo
userBookmarks = getUserBookmarks(ctx, dwr)
userStars = getUserStars(ctx, dwr)
)
@ -1329,22 +1117,12 @@ func (dwr *DynamoDB) FilterRepos(ctx context.Context,
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
)
pageFinder, err := pagination.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
}
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
err
}
var repoMeta mTypes.RepoMetadata
@ -1352,7 +1130,7 @@ func (dwr *DynamoDB) FilterRepos(ctx context.Context,
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
@ -1363,40 +1141,28 @@ func (dwr *DynamoDB) FilterRepos(ctx context.Context,
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
if filter(repoMeta) {
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
foundRepos = append(foundRepos, repoMeta)
}
}
foundRepos, pageInfo := pageFinder.Page()
foundManifestMetadataMap, foundIndexDataMap, err := common.FetchDataForRepos(dwr, foundRepos)
return foundRepos, foundManifestMetadataMap, foundIndexDataMap, pageInfo, err
return foundRepos, foundManifestMetadataMap, foundIndexDataMap, err
}
func (dwr *DynamoDB) SearchTags(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
func (dwr *DynamoDB) SearchTags(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
zcommon.PageInfo, error,
error,
) {
var (
foundRepos = make([]mTypes.RepoMetadata, 0, 1)
manifestMetadataMap = make(map[string]mTypes.ManifestMetadata)
indexDataMap = make(map[string]mTypes.IndexData)
repoMetaAttributeIterator AttributesIterator
pageFinder pagination.PageFinder
pageInfo zcommon.PageInfo
userBookmarks = getUserBookmarks(ctx, dwr)
userStars = getUserStars(ctx, dwr)
)
pageFinder, err := pagination.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
}
repoMetaAttributeIterator = NewBaseDynamoAttributesIterator(
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
)
@ -1404,154 +1170,105 @@ func (dwr *DynamoDB) SearchTags(ctx context.Context, searchText string, filter m
searchedRepo, searchedTag, err := common.GetRepoTag(searchText)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error while parsing search text, invalid format %w", err)
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
err
}
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
// log
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
}
var repoMeta mTypes.RepoMetadata
var repoMeta mTypes.RepoMetadata
err = attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
err
}
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo, err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
if repoMeta.Name != searchedRepo {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
err
}
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
matchedTags := make(map[string]mTypes.Descriptor)
for tag, descriptor := range repoMeta.Tags {
if !strings.HasPrefix(tag, searchedTag) {
continue
}
if repoMeta.Name != searchedRepo {
continue
}
matchedTags[tag] = descriptor
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
matchedTags := make(map[string]mTypes.Descriptor)
for tag, descriptor := range repoMeta.Tags {
if !strings.HasPrefix(tag, searchedTag) {
continue
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", descriptor.Digest, err)
}
matchedTags[tag] = descriptor
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
fmt.Errorf("%w", err)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err)
}
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", descriptor.Digest, err)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
continue
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err)
}
manifestHasBeenMatched := false
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
manifestMetadataMap[manifestDigest] = manifestMeta
if common.AcceptedByFilter(filter, manifestFilterData) {
manifestHasBeenMatched = true
}
}
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
delete(manifestMetadataMap, manifest.Digest.String())
}
continue
}
indexDataMap[indexDigest] = indexData
default:
dwr.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type")
continue
manifestMetadataMap[manifestDigest] = manifestMeta
}
}
if len(matchedTags) == 0 {
indexDataMap[indexDigest] = indexData
default:
dwr.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type")
continue
}
repoMeta.Tags = matchedTags
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
}
foundRepos, pageInfo := pageFinder.Page()
if len(matchedTags) == 0 {
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{},
err
}
foundManifestMetadataMap, foundindexDataMap, err := common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
repoMeta.Tags = matchedTags
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
foundRepos = append(foundRepos, repoMeta)
return foundRepos, manifestMetadataMap, indexDataMap, err
}
func (dwr *DynamoDB) PatchDB() error {

View file

@ -823,8 +823,7 @@ func TestWrapperErrors(t *testing.T) {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, err = dynamoWrapper.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true },
mTypes.PageInput{})
_, err = dynamoWrapper.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
@ -833,7 +832,50 @@ func TestWrapperErrors(t *testing.T) {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "")
So(err, ShouldNotBeNil)
})
Convey("SearchRepos bad tablename", func() {
dynamoWrapper.RepoMetaTablename = badTablename
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "")
So(err, ShouldNotBeNil)
})
Convey("GetMultipleRepoMeta bad tablename", func() {
dynamoWrapper.RepoMetaTablename = badTablename
_, err = dynamoWrapper.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
Convey("FilterTags bad tablename", func() {
dynamoWrapper.RepoMetaTablename = badTablename
_, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool {
return true
})
So(err, ShouldNotBeNil)
})
Convey("FilterRepos bad tablename", func() {
dynamoWrapper.RepoMetaTablename = badTablename
_, _, _, err = dynamoWrapper.FilterRepos(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
Convey("SearchTags bad tablename", func() {
dynamoWrapper.RepoMetaTablename = badTablename
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:tag")
So(err, ShouldNotBeNil)
})
@ -843,22 +885,7 @@ func TestWrapperErrors(t *testing.T) {
ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchRepos config unmarshal error", func() {
err := dynamoWrapper.SetRepoReference("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData("dig1", mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "")
So(err, ShouldNotBeNil)
})
@ -869,18 +896,14 @@ func TestWrapperErrors(t *testing.T) {
err := dynamoWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "")
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:")
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true })
So(err, ShouldBeNil)
})
@ -893,7 +916,7 @@ func TestWrapperErrors(t *testing.T) {
err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "")
So(err, ShouldNotBeNil)
})
@ -908,43 +931,7 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchRepos good index data, bad manifest inside index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchRepos(ctx, "", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchRepos(ctx, "")
So(err, ShouldNotBeNil)
})
@ -952,7 +939,7 @@ func TestWrapperErrors(t *testing.T) {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:")
So(err, ShouldNotBeNil)
})
@ -962,25 +949,7 @@ func TestWrapperErrors(t *testing.T) {
ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("SearchTags config unmarshal error", func() {
err := dynamoWrapper.SetRepoReference("repo", "tag1", "dig1", ispec.MediaTypeImageManifest) //nolint:contextcheck
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData( //nolint:contextcheck
"dig1",
mTypes.ManifestData{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad json"),
},
)
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:")
So(err, ShouldNotBeNil)
})
@ -994,7 +963,7 @@ func TestWrapperErrors(t *testing.T) {
err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:")
So(err, ShouldNotBeNil)
})
@ -1009,57 +978,31 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:")
So(err, ShouldNotBeNil)
})
Convey("SearchTags good index data, bad manifest inside index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2")
)
err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck
Convey("SearchRepos attr", func() {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("Bad Manifest"),
ConfigBlob: []byte("Bad Manifest"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:", mTypes.Filter{}, mTypes.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("FilterRepos NewBaseRepoPageFinder errors", func() {
_, _, _, _, err := dynamoWrapper.SearchRepos(ctx, "text", mTypes.Filter{},
mTypes.PageInput{Offset: -2, Limit: -2})
_, _, _, err := dynamoWrapper.SearchRepos(ctx, "repo")
So(err, ShouldNotBeNil)
})
Convey("FilterRepos attributevalue.Unmarshal(repoMetaAttribute) errors", func() {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
dynamoWrapper.RepoMetaTablename = "bad-table-FilterRepos"
_, _, _, _, err := dynamoWrapper.SearchRepos(ctx, "repo", mTypes.Filter{}, mTypes.PageInput{})
_, _, _, err := dynamoWrapper.FilterRepos(ctx, func(repoMeta mTypes.RepoMetadata) bool {
return true
})
So(err, ShouldNotBeNil)
})
Convey("SearchRepos bad RepoMeta table name", func() {
dynamoWrapper.RepoMetaTablename = "SearchRepos-bad-table"
_, _, _, err := dynamoWrapper.SearchRepos(ctx, "repo")
So(err, ShouldNotBeNil)
})
@ -1067,14 +1010,21 @@ func TestWrapperErrors(t *testing.T) {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(
ctx,
_, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool {
return true
},
mTypes.Filter{},
mTypes.PageInput{},
)
})
So(err, ShouldNotBeNil)
})
Convey("FilterTags bad RepoMeta table name", func() {
dynamoWrapper.RepoMetaTablename = "bad-table"
_, _, _, err := dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool {
return true
})
So(err, ShouldNotBeNil)
})
@ -1084,14 +1034,10 @@ func TestWrapperErrors(t *testing.T) {
ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(
ctx,
_, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool {
return true
},
mTypes.Filter{},
mTypes.PageInput{},
)
})
So(err, ShouldNotBeNil)
})
@ -1103,14 +1049,11 @@ func TestWrapperErrors(t *testing.T) {
err = setBadManifestData(dynamoWrapper.Client, manifestDataTablename, "dig") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(
_, _, _, err = dynamoWrapper.FilterTags(
ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool {
return true
},
mTypes.Filter{},
mTypes.PageInput{},
)
})
So(err, ShouldNotBeNil)
})
@ -1124,11 +1067,8 @@ func TestWrapperErrors(t *testing.T) {
err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
@ -1143,11 +1083,8 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true })
So(err, ShouldNotBeNil)
})
@ -1183,78 +1120,11 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return false },
mTypes.Filter{},
mTypes.PageInput{},
)
_, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return false })
So(err, ShouldBeNil)
})
Convey("FilterTags bad config blob in image with a single manifest", func() {
manifestDigest := digest.FromString("manifestDigestBadConfig")
err := dynamoWrapper.SetRepoReference( //nolint:contextcheck
"repo", "tag1", manifestDigest, ispec.MediaTypeImageManifest,
)
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigest, mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad blob"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
So(err, ShouldNotBeNil)
})
Convey("FilterTags bad config blob in index", func() {
var (
indexDigest = digest.FromString("indexDigest")
manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndexGoodConfig")
manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndexBadConfig")
)
err := dynamoWrapper.SetRepoReference( //nolint:contextcheck
"repo", "tag1", indexDigest, ispec.MediaTypeImageIndex,
)
So(err, ShouldBeNil)
indexBlob, err := test.GetIndexBlobWithManifests([]digest.Digest{
manifestDigestFromIndex1, manifestDigestFromIndex2,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck
IndexBlob: indexBlob,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{ //nolint:contextcheck
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad blob"),
})
So(err, ShouldBeNil)
_, _, _, _, err = dynamoWrapper.FilterTags(ctx,
func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true },
mTypes.Filter{},
mTypes.PageInput{},
)
So(err, ShouldNotBeNil)
})
Convey("PatchDB dwr.getDBVersion errors", func() {
dynamoWrapper.VersionTablename = badTablename

View file

@ -54,7 +54,7 @@ func (dii *BaseAttributesIterator) First(ctx context.Context) (types.AttributeVa
Limit: dii.readLimit,
})
if err != nil {
return nil, err
return &types.AttributeValueMemberBOOL{}, err
}
if len(scanOutput.Items) == 0 {

View file

@ -58,7 +58,7 @@ func OnUpdateManifest(repo, reference, mediaType string, digest godigest.Digest,
}
}
} else {
err := SetImageMetaFromInput(repo, reference, mediaType, digest, body,
err = SetImageMetaFromInput(repo, reference, mediaType, digest, body,
imgStore, metaDB, log)
if err != nil {
metadataSuccessfullySet = false
@ -66,7 +66,7 @@ func OnUpdateManifest(repo, reference, mediaType string, digest godigest.Digest,
}
if !metadataSuccessfullySet {
log.Info().Str("tag", reference).Str("repository", repo).Msg("uploding image meta was unsuccessful for tag in repo")
log.Info().Str("tag", reference).Str("repository", repo).Msg("uploading image meta was unsuccessful for tag in repo")
if err := imgStore.DeleteImageManifest(repo, reference, false); err != nil {
log.Error().Err(err).Str("reference", reference).Str("repository", repo).
@ -122,8 +122,8 @@ func OnDeleteManifest(repo, reference, mediaType string, digest godigest.Digest,
manageRepoMetaSuccessfully = false
}
if refferredDigest, hasSubject := common.GetReferredSubject(manifestBlob); hasSubject {
err := metaDB.DeleteReferrer(repo, refferredDigest, digest)
if referredDigest, hasSubject := common.GetReferredSubject(manifestBlob); hasSubject {
err := metaDB.DeleteReferrer(repo, referredDigest, digest)
if err != nil {
log.Error().Err(err).Msg("metadb: error while deleting referrer")

File diff suppressed because it is too large Load diff

View file

@ -1,273 +0,0 @@
package pagination
import (
"fmt"
"sort"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/common"
mTypes "zotregistry.io/zot/pkg/meta/types"
)
// PageFinder permits keeping a pool of objects using Add
// and returning a specific page.
type PageFinder interface {
Add(detailedRepoMeta mTypes.DetailedRepoMeta)
Page() ([]mTypes.RepoMetadata, common.PageInfo)
Reset()
}
// RepoPageFinder implements PageFinder. It manages RepoMeta objects and calculates the page
// using the given limit, offset and sortBy option.
type RepoPageFinder struct {
limit int
offset int
sortBy mTypes.SortCriteria
pageBuffer []mTypes.DetailedRepoMeta
}
func NewBaseRepoPageFinder(limit, offset int, sortBy mTypes.SortCriteria) (*RepoPageFinder, error) {
if sortBy == "" {
sortBy = mTypes.AlphabeticAsc
}
if limit < 0 {
return nil, zerr.ErrLimitIsNegative
}
if offset < 0 {
return nil, zerr.ErrOffsetIsNegative
}
if _, found := mTypes.SortFunctions()[sortBy]; !found {
return nil, fmt.Errorf("sorting repos by '%s' is not supported %w",
sortBy, zerr.ErrSortCriteriaNotSupported)
}
return &RepoPageFinder{
limit: limit,
offset: offset,
sortBy: sortBy,
pageBuffer: make([]mTypes.DetailedRepoMeta, 0, limit),
}, nil
}
func (bpt *RepoPageFinder) Reset() {
bpt.pageBuffer = []mTypes.DetailedRepoMeta{}
}
func (bpt *RepoPageFinder) Add(namedRepoMeta mTypes.DetailedRepoMeta) {
bpt.pageBuffer = append(bpt.pageBuffer, namedRepoMeta)
}
func (bpt *RepoPageFinder) Page() ([]mTypes.RepoMetadata, common.PageInfo) {
if len(bpt.pageBuffer) == 0 {
return []mTypes.RepoMetadata{}, common.PageInfo{}
}
pageInfo := &common.PageInfo{}
sort.Slice(bpt.pageBuffer, mTypes.SortFunctions()[bpt.sortBy](bpt.pageBuffer))
// the offset and limit are calculatd in terms of repos counted
start := bpt.offset
end := bpt.offset + bpt.limit
// we'll return an empty array when the offset is greater than the number of elements
if start >= len(bpt.pageBuffer) {
start = len(bpt.pageBuffer)
end = start
}
if end >= len(bpt.pageBuffer) {
end = len(bpt.pageBuffer)
}
detailedReposPage := bpt.pageBuffer[start:end]
pageInfo.ItemCount = len(detailedReposPage)
if start == 0 && end == 0 {
detailedReposPage = bpt.pageBuffer
pageInfo.ItemCount = len(detailedReposPage)
}
repos := make([]mTypes.RepoMetadata, 0, len(detailedReposPage))
for _, drm := range detailedReposPage {
repos = append(repos, drm.RepoMetadata)
}
pageInfo.TotalCount = len(bpt.pageBuffer)
return repos, *pageInfo
}
type ImagePageFinder struct {
limit int
offset int
sortBy mTypes.SortCriteria
pageBuffer []mTypes.DetailedRepoMeta
}
func NewBaseImagePageFinder(limit, offset int, sortBy mTypes.SortCriteria) (*ImagePageFinder, error) {
if sortBy == "" {
sortBy = mTypes.AlphabeticAsc
}
if limit < 0 {
return nil, zerr.ErrLimitIsNegative
}
if offset < 0 {
return nil, zerr.ErrOffsetIsNegative
}
if _, found := mTypes.SortFunctions()[sortBy]; !found {
return nil, fmt.Errorf("sorting repos by '%s' is not supported %w",
sortBy, zerr.ErrSortCriteriaNotSupported)
}
return &ImagePageFinder{
limit: limit,
offset: offset,
sortBy: sortBy,
pageBuffer: make([]mTypes.DetailedRepoMeta, 0, limit),
}, nil
}
func (bpt *ImagePageFinder) Reset() {
bpt.pageBuffer = []mTypes.DetailedRepoMeta{}
}
func (bpt *ImagePageFinder) Add(namedRepoMeta mTypes.DetailedRepoMeta) {
bpt.pageBuffer = append(bpt.pageBuffer, namedRepoMeta)
}
func (bpt *ImagePageFinder) Page() ([]mTypes.RepoMetadata, common.PageInfo) {
if len(bpt.pageBuffer) == 0 {
return []mTypes.RepoMetadata{}, common.PageInfo{}
}
pageInfo := common.PageInfo{}
for _, drm := range bpt.pageBuffer {
repo := drm.RepoMetadata
pageInfo.TotalCount += len(repo.Tags)
}
sort.Slice(bpt.pageBuffer, mTypes.SortFunctions()[bpt.sortBy](bpt.pageBuffer))
repoStartIndex := 0
tagStartIndex := 0
// the offset and limit are calculatd in terms of tags counted
remainingOffset := bpt.offset
remainingLimit := bpt.limit
repos := make([]mTypes.RepoMetadata, 0)
if remainingOffset == 0 && remainingLimit == 0 {
for _, drm := range bpt.pageBuffer {
repo := drm.RepoMetadata
repos = append(repos, repo)
pageInfo.ItemCount += len(repo.Tags)
}
return repos, pageInfo
}
// bring cursor to position in RepoMeta array
for _, drm := range bpt.pageBuffer {
if remainingOffset < len(drm.Tags) {
tagStartIndex = remainingOffset
break
}
remainingOffset -= len(drm.Tags)
repoStartIndex++
}
// offset is larger than the number of tags
if repoStartIndex >= len(bpt.pageBuffer) {
return []mTypes.RepoMetadata{}, common.PageInfo{}
}
// finish counting remaining tags inside the first repo meta
partialTags := map[string]mTypes.Descriptor{}
firstRepoMeta := bpt.pageBuffer[repoStartIndex].RepoMetadata
tags := make([]string, 0, len(firstRepoMeta.Tags))
for k := range firstRepoMeta.Tags {
tags = append(tags, k)
}
sort.Strings(tags)
for i := tagStartIndex; i < len(tags); i++ {
tag := tags[i]
partialTags[tag] = firstRepoMeta.Tags[tag]
remainingLimit--
if remainingLimit == 0 {
firstRepoMeta.Tags = partialTags
repos = append(repos, firstRepoMeta)
pageInfo.ItemCount = len(partialTags)
return repos, pageInfo
}
}
firstRepoMeta.Tags = partialTags
pageInfo.ItemCount += len(firstRepoMeta.Tags)
repos = append(repos, firstRepoMeta)
repoStartIndex++
// continue with the remaining repos
for i := repoStartIndex; i < len(bpt.pageBuffer); i++ {
repoMeta := bpt.pageBuffer[i].RepoMetadata
if len(repoMeta.Tags) > remainingLimit {
partialTags := map[string]mTypes.Descriptor{}
tags := make([]string, 0, len(repoMeta.Tags))
for k := range repoMeta.Tags {
tags = append(tags, k)
}
sort.Strings(tags)
for _, tag := range tags {
partialTags[tag] = repoMeta.Tags[tag]
remainingLimit--
if remainingLimit == 0 {
repoMeta.Tags = partialTags
repos = append(repos, repoMeta)
pageInfo.ItemCount += len(partialTags)
break
}
}
return repos, pageInfo
}
// add the whole repo
repos = append(repos, repoMeta)
pageInfo.ItemCount += len(repoMeta.Tags)
remainingLimit -= len(repoMeta.Tags)
if remainingLimit == 0 {
return repos, pageInfo
}
}
// we arrive here when the limit is bigger than the number of tags
return repos, pageInfo
}

View file

@ -1,241 +0,0 @@
package pagination_test
import (
"testing"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/pkg/meta/pagination"
mTypes "zotregistry.io/zot/pkg/meta/types"
)
func TestPagination(t *testing.T) {
Convey("Repo Pagination", t, func() {
Convey("reset", func() {
pageFinder, err := pagination.NewBaseRepoPageFinder(1, 0, mTypes.AlphabeticAsc)
So(err, ShouldBeNil)
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(mTypes.DetailedRepoMeta{})
pageFinder.Add(mTypes.DetailedRepoMeta{})
pageFinder.Add(mTypes.DetailedRepoMeta{})
pageFinder.Reset()
result, _ := pageFinder.Page()
So(result, ShouldBeEmpty)
})
})
Convey("Image Pagination", t, func() {
Convey("create new pageFinder errors", func() {
pageFinder, err := pagination.NewBaseImagePageFinder(-1, 10, mTypes.AlphabeticAsc)
So(pageFinder, ShouldBeNil)
So(err, ShouldNotBeNil)
pageFinder, err = pagination.NewBaseImagePageFinder(2, -1, mTypes.AlphabeticAsc)
So(pageFinder, ShouldBeNil)
So(err, ShouldNotBeNil)
pageFinder, err = pagination.NewBaseImagePageFinder(2, 1, "wrong sorting criteria")
So(pageFinder, ShouldBeNil)
So(err, ShouldNotBeNil)
})
Convey("Reset", func() {
pageFinder, err := pagination.NewBaseImagePageFinder(1, 0, mTypes.AlphabeticAsc)
So(err, ShouldBeNil)
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(mTypes.DetailedRepoMeta{})
pageFinder.Add(mTypes.DetailedRepoMeta{})
pageFinder.Add(mTypes.DetailedRepoMeta{})
pageFinder.Reset()
result, _ := pageFinder.Page()
So(result, ShouldBeEmpty)
})
Convey("Page", func() {
Convey("no limit or offset", func() {
pageFinder, err := pagination.NewBaseImagePageFinder(0, 0, mTypes.AlphabeticAsc)
So(err, ShouldBeNil)
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo1",
Tags: map[string]mTypes.Descriptor{
"tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
},
},
})
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo2",
Tags: map[string]mTypes.Descriptor{
"Tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
"Tag2": {Digest: "dig2", MediaType: ispec.MediaTypeImageManifest},
"Tag3": {Digest: "dig3", MediaType: ispec.MediaTypeImageManifest},
"Tag4": {Digest: "dig4", MediaType: ispec.MediaTypeImageManifest},
},
},
})
_, pageInfo := pageFinder.Page()
So(pageInfo.ItemCount, ShouldEqual, 5)
So(pageInfo.TotalCount, ShouldEqual, 5)
})
Convey("Test 1 limit < len(tags)", func() {
pageFinder, err := pagination.NewBaseImagePageFinder(5, 2, mTypes.AlphabeticAsc)
So(err, ShouldBeNil)
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo1",
Tags: map[string]mTypes.Descriptor{
"tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
},
},
})
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo2",
Tags: map[string]mTypes.Descriptor{
"Tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
"Tag2": {Digest: "dig2", MediaType: ispec.MediaTypeImageManifest},
"Tag3": {Digest: "dig3", MediaType: ispec.MediaTypeImageManifest},
"Tag4": {Digest: "dig4", MediaType: ispec.MediaTypeImageManifest},
},
},
})
_, pageInfo := pageFinder.Page()
So(pageInfo.ItemCount, ShouldEqual, 3)
So(pageInfo.TotalCount, ShouldEqual, 5)
})
Convey("Test 2 limit < len(tags)", func() {
pageFinder, err := pagination.NewBaseImagePageFinder(5, 2, mTypes.AlphabeticAsc)
So(err, ShouldBeNil)
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo1",
Tags: map[string]mTypes.Descriptor{
"tag1": {
Digest: "dig1",
MediaType: ispec.MediaTypeImageManifest,
},
},
},
})
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo2",
Tags: map[string]mTypes.Descriptor{
"Tag1": {
Digest: "dig1",
MediaType: ispec.MediaTypeImageManifest,
},
"Tag2": {
Digest: "dig2",
MediaType: ispec.MediaTypeImageManifest,
},
"Tag3": {
Digest: "dig3",
MediaType: ispec.MediaTypeImageManifest,
},
"Tag4": {
Digest: "dig4",
MediaType: ispec.MediaTypeImageManifest,
},
},
},
})
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo3",
Tags: map[string]mTypes.Descriptor{
"Tag11": {
Digest: "dig11",
MediaType: ispec.MediaTypeImageManifest,
},
"Tag12": {
Digest: "dig12",
MediaType: ispec.MediaTypeImageManifest,
},
"Tag13": {
Digest: "dig13",
MediaType: ispec.MediaTypeImageManifest,
},
"Tag14": {
Digest: "dig14",
MediaType: ispec.MediaTypeImageManifest,
},
},
},
})
result, pageInfo := pageFinder.Page()
So(result[0].Tags, ShouldContainKey, "Tag2")
So(result[0].Tags, ShouldContainKey, "Tag3")
So(result[0].Tags, ShouldContainKey, "Tag4")
So(result[1].Tags, ShouldContainKey, "Tag11")
So(result[1].Tags, ShouldContainKey, "Tag12")
So(pageInfo.ItemCount, ShouldEqual, 5)
So(pageInfo.TotalCount, ShouldEqual, 9)
})
Convey("Test 2 limit > len(tags)", func() {
pageFinder, err := pagination.NewBaseImagePageFinder(3, 0, mTypes.AlphabeticAsc)
So(err, ShouldBeNil)
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo1",
Tags: map[string]mTypes.Descriptor{
"tag1": {
Digest: "dig1",
MediaType: ispec.MediaTypeImageManifest,
},
},
},
})
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo2",
Tags: map[string]mTypes.Descriptor{
"Tag1": {
Digest: "dig1",
MediaType: ispec.MediaTypeImageManifest,
},
},
},
})
pageFinder.Add(mTypes.DetailedRepoMeta{
RepoMetadata: mTypes.RepoMetadata{
Name: "repo3",
Tags: map[string]mTypes.Descriptor{
"Tag11": {
Digest: "dig11",
MediaType: ispec.MediaTypeImageManifest,
},
},
},
})
result, _ := pageFinder.Page()
So(result[0].Tags, ShouldContainKey, "tag1")
So(result[1].Tags, ShouldContainKey, "Tag1")
So(result[2].Tags, ShouldContainKey, "Tag11")
})
})
})
}

View file

@ -332,9 +332,11 @@ func NewManifestData(repoName string, manifestBlob []byte, imageStore storageTyp
return mTypes.ManifestData{}, err
}
err = json.Unmarshal(configBlob, &configContent)
if err != nil {
return mTypes.ManifestData{}, err
if manifestContent.Config.MediaType == ispec.MediaTypeImageConfig {
err = json.Unmarshal(configBlob, &configContent)
if err != nil {
return mTypes.ManifestData{}, err
}
}
manifestData.ManifestBlob = manifestBlob
@ -381,9 +383,9 @@ func SetImageMetaFromInput(repo, reference, mediaType string, digest godigest.Di
}
}
refferredDigest, referrerInfo, hasSubject, err := GetReferredInfo(descriptorBlob, digest.String(), mediaType)
referredDigest, referrerInfo, hasSubject, err := GetReferredInfo(descriptorBlob, digest.String(), mediaType)
if hasSubject && err == nil {
err := metaDB.SetReferrer(repo, refferredDigest, referrerInfo)
err := metaDB.SetReferrer(repo, referredDigest, referrerInfo)
if err != nil {
log.Error().Err(err).Msg("metadb: error while settingg referrer")

View file

@ -223,15 +223,6 @@ func TestParseStorageErrors(t *testing.T) {
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
Convey("json.Unmarshal(configBlob errors", func() {
imageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte("invalid JSON"), nil
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
Convey("CheckIsImageSignature -> is signature", func() {
@ -474,11 +465,8 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) {
err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
repos, err := metaDB.GetMultipleRepoMeta(
context.Background(),
func(repoMeta mTypes.RepoMetadata) bool { return true },
mTypes.PageInput{},
)
repos, err := metaDB.GetMultipleRepoMeta(context.Background(),
func(repoMeta mTypes.RepoMetadata) bool { return true })
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
@ -540,7 +528,6 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) {
repos, err := metaDB.GetMultipleRepoMeta(
context.Background(),
func(repoMeta mTypes.RepoMetadata) bool { return true },
mTypes.PageInput{},
)
So(err, ShouldBeNil)

View file

@ -1,54 +0,0 @@
package types
type SortCriteria string
const (
Relevance = SortCriteria("RELEVANCE")
UpdateTime = SortCriteria("UPDATE_TIME")
AlphabeticAsc = SortCriteria("ALPHABETIC_ASC")
AlphabeticDsc = SortCriteria("ALPHABETIC_DSC")
Stars = SortCriteria("STARS")
Downloads = SortCriteria("DOWNLOADS")
)
func SortFunctions() map[SortCriteria]func(pageBuffer []DetailedRepoMeta) func(i, j int) bool {
return map[SortCriteria]func(pageBuffer []DetailedRepoMeta) func(i, j int) bool{
AlphabeticAsc: SortByAlphabeticAsc,
AlphabeticDsc: SortByAlphabeticDsc,
Relevance: SortByRelevance,
UpdateTime: SortByUpdateTime,
Downloads: SortByDownloads,
}
}
func SortByAlphabeticAsc(pageBuffer []DetailedRepoMeta) func(i, j int) bool {
return func(i, j int) bool {
return pageBuffer[i].Name < pageBuffer[j].Name
}
}
func SortByAlphabeticDsc(pageBuffer []DetailedRepoMeta) func(i, j int) bool {
return func(i, j int) bool {
return pageBuffer[i].Name > pageBuffer[j].Name
}
}
func SortByRelevance(pageBuffer []DetailedRepoMeta) func(i, j int) bool {
return func(i, j int) bool {
return pageBuffer[i].Rank < pageBuffer[j].Rank
}
}
// SortByUpdateTime sorting descending by time.
func SortByUpdateTime(pageBuffer []DetailedRepoMeta) func(i, j int) bool {
return func(i, j int) bool {
return pageBuffer[i].UpdateTime.After(pageBuffer[j].UpdateTime)
}
}
// SortByDownloads returns a comparison function for descendant sorting by downloads.
func SortByDownloads(pageBuffer []DetailedRepoMeta) func(i, j int) bool {
return func(i, j int) bool {
return pageBuffer[i].Downloads > pageBuffer[j].Downloads
}
}

View file

@ -5,8 +5,6 @@ import (
"time"
godigest "github.com/opencontainers/go-digest"
"zotregistry.io/zot/pkg/common"
)
// DetailedRepoMeta is a auxiliary structure used for sorting RepoMeta arrays by information
@ -62,7 +60,7 @@ type MetaDB interface { //nolint:interfacebloat
// GetMultipleRepoMeta returns information about all repositories as map[string]RepoMetadata filtered by the filter
// function
GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta RepoMetadata) bool, requestedPage PageInput) (
GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta RepoMetadata) bool) (
[]RepoMetadata, error)
// SetManifestData sets ManifestData for a given manifest in the database
@ -107,20 +105,20 @@ type MetaDB interface { //nolint:interfacebloat
UpdateSignaturesValidity(repo string, manifestDigest godigest.Digest) error
// SearchRepos searches for repos given a search string
SearchRepos(ctx context.Context, searchText string, filter Filter, requestedPage PageInput) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, common.PageInfo, error)
SearchRepos(ctx context.Context, searchText string) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error)
// SearchTags searches for images(repo:tag) given a search string
SearchTags(ctx context.Context, searchText string, filter Filter, requestedPage PageInput) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, common.PageInfo, error)
SearchTags(ctx context.Context, searchText string) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error)
// FilterRepos filters for repos given a filter function
FilterRepos(ctx context.Context, filter FilterRepoFunc, requestedPage PageInput) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, common.PageInfo, error)
FilterRepos(ctx context.Context, filter FilterRepoFunc) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error)
// FilterTags filters for images given a filter function
FilterTags(ctx context.Context, filterFunc FilterFunc, filter Filter,
requestedPage PageInput) ([]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, common.PageInfo, error)
FilterTags(ctx context.Context, filterFunc FilterFunc) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error)
PatchDB() error
}
@ -204,6 +202,7 @@ type RepoMetadata struct {
IsStarred bool
IsBookmarked bool
Rank int
Stars int
}
@ -234,12 +233,6 @@ type UserData struct {
APIKeys map[string]APIKeyDetails
}
type PageInput struct {
Limit int
Offset int
SortBy SortCriteria
}
type Filter struct {
Os []*string
Arch []*string

View file

@ -14,6 +14,10 @@ import (
storageConstants "zotregistry.io/zot/pkg/storage/constants"
)
const (
TestFakeSignatureArtType = "application/test.fake.signature"
)
// LayerBuilder abstracts the first step in creating an OCI image, specifying the layers of the image.
type LayerBuilder interface {
// LayerBlobs sets the image layers from the gives blobs array, adding a default zipped layer media type.
@ -46,6 +50,8 @@ type ConfigBuilder interface {
ArtifactConfig(artifactType string) ManifestBuilder
// DefaultConfig sets the default config, platform linux/amd64.
DefaultConfig() ManifestBuilder
// CustomConfigBlob will set a custom blob as the image config without other checks.
CustomConfigBlob(configBlob []byte, mediaType string) ManifestBuilder
// RandomConfig sets a randomly generated config.
RandomConfig() ManifestBuilder
}
@ -163,6 +169,13 @@ func CreateRandomVulnerableImageWith() ManifestBuilder {
return CreateImageWith().VulnerableLayers().RandomVulnConfig()
}
// CreateFakeTestSignature returns a test signature that is used to mark a image as signed
// when creating a test Repo. It won't be recognized as a signature if uploaded to the repository directly.
func CreateFakeTestSignature(subject *ispec.Descriptor) Image {
return CreateImageWith().RandomLayers(1, 10).DefaultConfig().
ArtifactType(TestFakeSignatureArtType).Subject(subject).Build()
}
type BaseImageBuilder struct {
layers []Layer
@ -283,6 +296,19 @@ func (ib *BaseImageBuilder) ArtifactConfig(artifactType string) ManifestBuilder
return ib
}
func (ib *BaseImageBuilder) CustomConfigBlob(configBlob []byte, mediaType string) ManifestBuilder {
ib.config = ispec.Image{}
ib.configDescriptor = ispec.Descriptor{
MediaType: mediaType,
Size: int64(len(configBlob)),
Data: configBlob,
Digest: godigest.FromBytes(configBlob),
}
return ib
}
func (ib *BaseImageBuilder) RandomConfig() ManifestBuilder {
config := GetDefaultConfig()
config.Author = getRandomAuthor()

View file

@ -5,7 +5,6 @@ import (
godigest "github.com/opencontainers/go-digest"
"zotregistry.io/zot/pkg/common"
mTypes "zotregistry.io/zot/pkg/meta/types"
)
@ -30,8 +29,8 @@ type MetaDBMock struct {
SetRepoMetaFn func(repo string, repoMeta mTypes.RepoMetadata) error
GetMultipleRepoMetaFn func(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool,
requestedPage mTypes.PageInput) ([]mTypes.RepoMetadata, error)
GetMultipleRepoMetaFn func(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool) (
[]mTypes.RepoMetadata, error)
GetManifestDataFn func(manifestDigest godigest.Digest) (mTypes.ManifestData, error)
@ -62,34 +61,19 @@ type MetaDBMock struct {
DeleteSignatureFn func(repo string, signedManifestDigest godigest.Digest, sm mTypes.SignatureMetadata) error
SearchReposFn func(ctx context.Context, txt string, filter mTypes.Filter, requestedPage mTypes.PageInput) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo,
SearchReposFn func(ctx context.Context, txt string) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
error)
SearchTagsFn func(ctx context.Context, txt string, filter mTypes.Filter, requestedPage mTypes.PageInput) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo,
SearchTagsFn func(ctx context.Context, txt string) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData,
error)
FilterReposFn func(ctx context.Context, filter mTypes.FilterRepoFunc, requestedPage mTypes.PageInput) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo,
error)
FilterReposFn func(ctx context.Context, filter mTypes.FilterRepoFunc) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error)
FilterTagsFn func(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo,
error)
SearchDigestsFn func(ctx context.Context, searchText string, requestedPage mTypes.PageInput) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error)
SearchLayersFn func(ctx context.Context, searchText string, requestedPage mTypes.PageInput) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error)
SearchForAscendantImagesFn func(ctx context.Context, searchText string, requestedPage mTypes.PageInput) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error)
SearchForDescendantImagesFn func(ctx context.Context, searchText string, requestedPage mTypes.PageInput) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error)
FilterTagsFn func(ctx context.Context, filterFunc mTypes.FilterFunc) (
[]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error)
GetStarredReposFn func(ctx context.Context) ([]string, error)
@ -195,10 +179,9 @@ func (sdm MetaDBMock) SetRepoMeta(repo string, repoMeta mTypes.RepoMetadata) err
}
func (sdm MetaDBMock) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, error) {
if sdm.GetMultipleRepoMetaFn != nil {
return sdm.GetMultipleRepoMetaFn(ctx, filter, requestedPage)
return sdm.GetMultipleRepoMetaFn(ctx, filter)
}
return []mTypes.RepoMetadata{}, nil
@ -272,86 +255,44 @@ func (sdm MetaDBMock) DeleteSignature(repo string, signedManifestDigest godigest
return nil
}
func (sdm MetaDBMock) SearchRepos(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, error) {
func (sdm MetaDBMock) SearchRepos(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) {
if sdm.SearchReposFn != nil {
return sdm.SearchReposFn(ctx, searchText, filter, requestedPage)
return sdm.SearchReposFn(ctx, searchText)
}
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{},
map[string]mTypes.IndexData{}, common.PageInfo{}, nil
map[string]mTypes.IndexData{}, nil
}
func (sdm MetaDBMock) SearchTags(ctx context.Context, searchText string, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, error) {
func (sdm MetaDBMock) SearchTags(ctx context.Context, searchText string,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) {
if sdm.SearchTagsFn != nil {
return sdm.SearchTagsFn(ctx, searchText, filter, requestedPage)
return sdm.SearchTagsFn(ctx, searchText)
}
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{},
map[string]mTypes.IndexData{}, common.PageInfo{}, nil
map[string]mTypes.IndexData{}, nil
}
func (sdm MetaDBMock) FilterRepos(ctx context.Context, filter mTypes.FilterRepoFunc,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, error) {
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) {
if sdm.FilterReposFn != nil {
return sdm.FilterReposFn(ctx, filter, requestedPage)
return sdm.FilterReposFn(ctx, filter)
}
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{},
map[string]mTypes.IndexData{}, common.PageInfo{}, nil
map[string]mTypes.IndexData{}, nil
}
func (sdm MetaDBMock) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, filter mTypes.Filter,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, common.PageInfo, error) {
func (sdm MetaDBMock) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) {
if sdm.FilterTagsFn != nil {
return sdm.FilterTagsFn(ctx, filterFunc, filter, requestedPage)
return sdm.FilterTagsFn(ctx, filterFunc)
}
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{},
map[string]mTypes.IndexData{}, common.PageInfo{}, nil
}
func (sdm MetaDBMock) SearchDigests(ctx context.Context, searchText string, requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error) {
if sdm.SearchDigestsFn != nil {
return sdm.SearchDigestsFn(ctx, searchText, requestedPage)
}
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, nil
}
func (sdm MetaDBMock) SearchLayers(ctx context.Context, searchText string, requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error) {
if sdm.SearchLayersFn != nil {
return sdm.SearchLayersFn(ctx, searchText, requestedPage)
}
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, nil
}
func (sdm MetaDBMock) SearchForAscendantImages(ctx context.Context, searchText string,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error) {
if sdm.SearchForAscendantImagesFn != nil {
return sdm.SearchForAscendantImagesFn(ctx, searchText, requestedPage)
}
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, nil
}
func (sdm MetaDBMock) SearchForDescendantImages(ctx context.Context, searchText string,
requestedPage mTypes.PageInput,
) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, error) {
if sdm.SearchForDescendantImagesFn != nil {
return sdm.SearchForDescendantImagesFn(ctx, searchText, requestedPage)
}
return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, nil
map[string]mTypes.IndexData{}, nil
}
func (sdm MetaDBMock) SetIndexData(digest godigest.Digest, indexData mTypes.IndexData) error {

91
pkg/test/repo.go Normal file
View file

@ -0,0 +1,91 @@
package test
import (
ispec "github.com/opencontainers/image-spec/specs-go/v1"
mTypes "zotregistry.io/zot/pkg/meta/types"
)
type RepoImage struct {
Image
Tag string
}
type RepoMultiArchImage struct {
MultiarchImage
Tag string
}
type Repo struct {
Name string
Images []RepoImage
MultiArchImages []RepoMultiArchImage
IsBookmarked bool
IsStarred bool
}
func GetMetadataForRepos(repos ...Repo) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata,
map[string]mTypes.IndexData,
) {
var (
reposMetadata = []mTypes.RepoMetadata{}
manifestMetadataMap = map[string]mTypes.ManifestMetadata{}
indexDataMap = map[string]mTypes.IndexData{}
)
for _, repo := range repos {
repoMeta := mTypes.RepoMetadata{
Name: repo.Name,
Tags: map[string]mTypes.Descriptor{},
Signatures: map[string]mTypes.ManifestSignatures{},
IsStarred: repo.IsStarred,
IsBookmarked: repo.IsBookmarked,
}
for _, image := range repo.Images {
if image.Tag != "" {
repoMeta.Tags[image.Tag] = mTypes.Descriptor{
MediaType: ispec.MediaTypeImageManifest,
Digest: image.DigestStr(),
}
}
// here we can do many more checks about the images like check for referrers, signatures but it's not needed yet
// I need just the tags for now and the fake signature.
// This is done just to mark a manifest as signed in the resulted RepoMeta
if image.Manifest.ArtifactType == TestFakeSignatureArtType && image.Manifest.Subject != nil {
signedManifestDig := image.Manifest.Subject.Digest.String()
repoMeta.Signatures[signedManifestDig] = mTypes.ManifestSignatures{
"fakeSignature": []mTypes.SignatureInfo{{SignatureManifestDigest: image.ManifestDescriptor.Digest.String()}},
}
}
manifestMetadataMap[image.ManifestDescriptor.Digest.String()] = mTypes.ManifestMetadata{
ManifestBlob: image.ManifestDescriptor.Data,
ConfigBlob: image.ConfigDescriptor.Data,
}
}
for _, multiArch := range repo.MultiArchImages {
repoMeta.Tags[multiArch.Tag] = mTypes.Descriptor{
MediaType: ispec.MediaTypeImageIndex,
Digest: multiArch.DigestStr(),
}
for _, image := range multiArch.Images {
manifestMetadataMap[image.ManifestDescriptor.Digest.String()] = mTypes.ManifestMetadata{
ManifestBlob: image.ManifestDescriptor.Data,
ConfigBlob: image.ConfigDescriptor.Data,
}
}
indexDataMap[multiArch.indexDescriptor.Digest.String()] = mTypes.IndexData{
IndexBlob: multiArch.indexDescriptor.Data,
}
}
reposMetadata = append(reposMetadata, repoMeta)
}
return reposMetadata, manifestMetadataMap, indexDataMap
}