mirror of
https://github.com/project-zot/zot.git
synced 2025-01-06 22:40:28 -05:00
feat(repodb): add pagination for ImageListForDigest and implement FilterTags (#1102)
* feat(repodb): add pagination for ImageListForDigest and implement FilterTags ImageListForDigest can now return paginated results, directly from DB. It uses FilterTags, a new method to filter tags (obviously) based on the criteria provided in the filter function. Pagination of tags is now slightly different, it shows all results if no limit and offset are provided. Signed-off-by: Alex Stan <alexandrustan96@yahoo.ro> bug(tests): cli tests for digests expecting wrong size Signed-off-by: Andrei Aaron <aaaron@luxoft.com> (cherry picked from commit 369216df931a4053c18278a8d89f86d2e1e6a436) fix(repodb): do not include repo metadata in search results if no matching tags are identified Signed-off-by: Andrei Aaron <aaaron@luxoft.com> * fix(repodb): Fix an issue in FilterTags where repometa was not proceesed correctly The filter function was called only once per manifest digest. The function is supposed to also take into consideration repometa, but only the first repometa-manifestmeta pair was processed. Also increase code coverage. Signed-off-by: Andrei Aaron <aaaron@luxoft.com> Signed-off-by: Andrei Aaron <aaaron@luxoft.com> Co-authored-by: Alex Stan <alexandrustan96@yahoo.ro>
This commit is contained in:
parent
9f8bc60b20
commit
e8e7c343ad
13 changed files with 994 additions and 206 deletions
pkg
cli
extensions/search
meta/repodb
test/mocks
|
@ -1046,8 +1046,8 @@ func TestServerResponseGQL(t *testing.T) {
|
|||
// repo7 test:2.0 a0ca253b 15B
|
||||
// repo7 test:1.0 a0ca253b 15B
|
||||
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
|
||||
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 15B")
|
||||
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 15B")
|
||||
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B")
|
||||
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B")
|
||||
|
||||
Convey("with shorthand", func() {
|
||||
args := []string{"imagetest", "-d", "883fc0c5"}
|
||||
|
@ -1064,8 +1064,8 @@ func TestServerResponseGQL(t *testing.T) {
|
|||
str := space.ReplaceAllString(buff.String(), " ")
|
||||
actual := strings.TrimSpace(str)
|
||||
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
|
||||
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 15B")
|
||||
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 15B")
|
||||
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B")
|
||||
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B")
|
||||
})
|
||||
|
||||
Convey("nonexistent digest", func() {
|
||||
|
|
|
@ -143,7 +143,7 @@ type ComplexityRoot struct {
|
|||
Image func(childComplexity int, image string) int
|
||||
ImageList func(childComplexity int, repo string) int
|
||||
ImageListForCve func(childComplexity int, id string) int
|
||||
ImageListForDigest func(childComplexity int, id string) int
|
||||
ImageListForDigest func(childComplexity int, id string, requestedPage *PageInput) int
|
||||
ImageListWithCVEFixed func(childComplexity int, id string, image string) int
|
||||
Referrers func(childComplexity int, repo string, digest string, typeArg string) int
|
||||
RepoListWithNewestImage func(childComplexity int, requestedPage *PageInput) int
|
||||
|
@ -181,7 +181,7 @@ type QueryResolver interface {
|
|||
CVEListForImage(ctx context.Context, image string) (*CVEResultForImage, error)
|
||||
ImageListForCve(ctx context.Context, id string) ([]*ImageSummary, error)
|
||||
ImageListWithCVEFixed(ctx context.Context, id string, image string) ([]*ImageSummary, error)
|
||||
ImageListForDigest(ctx context.Context, id string) ([]*ImageSummary, error)
|
||||
ImageListForDigest(ctx context.Context, id string, requestedPage *PageInput) ([]*ImageSummary, error)
|
||||
RepoListWithNewestImage(ctx context.Context, requestedPage *PageInput) ([]*RepoSummary, error)
|
||||
ImageList(ctx context.Context, repo string) ([]*ImageSummary, error)
|
||||
ExpandedRepoInfo(ctx context.Context, repo string) (*RepoInfo, error)
|
||||
|
@ -698,7 +698,7 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
|
|||
return 0, false
|
||||
}
|
||||
|
||||
return e.complexity.Query.ImageListForDigest(childComplexity, args["id"].(string)), true
|
||||
return e.complexity.Query.ImageListForDigest(childComplexity, args["id"].(string), args["requestedPage"].(*PageInput)), true
|
||||
|
||||
case "Query.ImageListWithCVEFixed":
|
||||
if e.complexity.Query.ImageListWithCVEFixed == nil {
|
||||
|
@ -1124,7 +1124,7 @@ type Query {
|
|||
"""
|
||||
Returns a list of images which contain the specified digest
|
||||
"""
|
||||
ImageListForDigest(id: String!): [ImageSummary!]
|
||||
ImageListForDigest(id: String!, requestedPage: PageInput): [ImageSummary!]
|
||||
|
||||
"""
|
||||
Returns a list of repos with the newest tag within
|
||||
|
@ -1295,6 +1295,15 @@ func (ec *executionContext) field_Query_ImageListForDigest_args(ctx context.Cont
|
|||
}
|
||||
}
|
||||
args["id"] = arg0
|
||||
var arg1 *PageInput
|
||||
if tmp, ok := rawArgs["requestedPage"]; ok {
|
||||
ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("requestedPage"))
|
||||
arg1, err = ec.unmarshalOPageInput2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐPageInput(ctx, tmp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
args["requestedPage"] = arg1
|
||||
return args, nil
|
||||
}
|
||||
|
||||
|
@ -4130,7 +4139,7 @@ func (ec *executionContext) _Query_ImageListForDigest(ctx context.Context, field
|
|||
}()
|
||||
resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
|
||||
ctx = rctx // use context from middleware stack in children
|
||||
return ec.resolvers.Query().ImageListForDigest(rctx, fc.Args["id"].(string))
|
||||
return ec.resolvers.Query().ImageListForDigest(rctx, fc.Args["id"].(string), fc.Args["requestedPage"].(*PageInput))
|
||||
})
|
||||
if err != nil {
|
||||
ec.Error(ctx, err)
|
||||
|
|
|
@ -6,6 +6,7 @@ package search
|
|||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
|
||||
"github.com/99designs/gqlgen/graphql"
|
||||
|
@ -75,37 +76,78 @@ func NewResolver(log log.Logger, storeController storage.StoreController,
|
|||
return resolver
|
||||
}
|
||||
|
||||
func (r *queryResolver) getImageListForDigest(repoList []string, digest string) ([]*gql_generated.ImageSummary, error) {
|
||||
imgResultForDigest := []*gql_generated.ImageSummary{}
|
||||
olu := common.NewBaseOciLayoutUtils(r.storeController, r.log)
|
||||
func FilterByDigest(digest string) repodb.FilterFunc {
|
||||
return func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
|
||||
lookupDigest := digest
|
||||
contains := false
|
||||
|
||||
var errResult error
|
||||
var manifest ispec.Manifest
|
||||
|
||||
for _, repo := range repoList {
|
||||
r.log.Info().Str("repo", repo).Msg("filtering list of tags in image repo by digest")
|
||||
|
||||
imgTags, err := r.digestInfo.GetImageTagsByDigest(repo, digest)
|
||||
err := json.Unmarshal(manifestMeta.ManifestBlob, &manifest)
|
||||
if err != nil {
|
||||
r.log.Error().Err(err).Msg("unable to get filtered list of image tags")
|
||||
|
||||
return []*gql_generated.ImageSummary{}, err
|
||||
return false
|
||||
}
|
||||
|
||||
for _, imageInfo := range imgTags {
|
||||
imageConfig, err := olu.GetImageConfigInfo(repo, imageInfo.Digest)
|
||||
if err != nil {
|
||||
return []*gql_generated.ImageSummary{}, err
|
||||
manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String()
|
||||
|
||||
// Check the image manifest in index.json matches the search digest
|
||||
// This is a blob with mediaType application/vnd.oci.image.manifest.v1+json
|
||||
if strings.Contains(manifestDigest, lookupDigest) {
|
||||
contains = true
|
||||
}
|
||||
|
||||
// Check the image config matches the search digest
|
||||
// This is a blob with mediaType application/vnd.oci.image.config.v1+json
|
||||
if strings.Contains(manifest.Config.Digest.String(), lookupDigest) {
|
||||
contains = true
|
||||
}
|
||||
|
||||
// Check to see if the individual layers in the oci image manifest match the digest
|
||||
// These are blobs with mediaType application/vnd.oci.image.layer.v1.tar+gzip
|
||||
for _, layer := range manifest.Layers {
|
||||
if strings.Contains(layer.Digest.String(), lookupDigest) {
|
||||
contains = true
|
||||
}
|
||||
|
||||
isSigned := olu.CheckManifestSignature(repo, imageInfo.Digest)
|
||||
imageInfo := convert.BuildImageInfo(repo, imageInfo.Tag, imageInfo.Digest,
|
||||
imageInfo.Manifest, imageConfig, isSigned)
|
||||
|
||||
imgResultForDigest = append(imgResultForDigest, imageInfo)
|
||||
}
|
||||
|
||||
return contains
|
||||
}
|
||||
}
|
||||
|
||||
func getImageListForDigest(ctx context.Context, digest string, repoDB repodb.RepoDB, cveInfo cveinfo.CveInfo,
|
||||
requestedPage *gql_generated.PageInput,
|
||||
) ([]*gql_generated.ImageSummary, error) {
|
||||
imageList := make([]*gql_generated.ImageSummary, 0)
|
||||
|
||||
if requestedPage == nil {
|
||||
requestedPage = &gql_generated.PageInput{}
|
||||
}
|
||||
|
||||
return imgResultForDigest, errResult
|
||||
skip := convert.SkipQGLField{
|
||||
Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Images.Vulnerabilities"),
|
||||
}
|
||||
|
||||
pageInput := repodb.PageInput{
|
||||
Limit: safeDerefferencing(requestedPage.Limit, 0),
|
||||
Offset: safeDerefferencing(requestedPage.Offset, 0),
|
||||
SortBy: repodb.SortCriteria(
|
||||
safeDerefferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance),
|
||||
),
|
||||
}
|
||||
|
||||
// get all repos
|
||||
reposMeta, manifestMetaMap, err := repoDB.FilterTags(ctx, FilterByDigest(digest), pageInput)
|
||||
if err != nil {
|
||||
return []*gql_generated.ImageSummary{}, err
|
||||
}
|
||||
|
||||
for _, repoMeta := range reposMeta {
|
||||
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, skip, cveInfo)
|
||||
|
||||
imageList = append(imageList, imageSummaries...)
|
||||
}
|
||||
|
||||
return imageList, nil
|
||||
}
|
||||
|
||||
func getImageSummary(ctx context.Context, repo, tag string, repoDB repodb.RepoDB,
|
||||
|
|
|
@ -107,11 +107,11 @@ func TestGlobalSearch(t *testing.T) {
|
|||
|
||||
const query = "repo1"
|
||||
limit := 1
|
||||
ofset := 0
|
||||
offset := 0
|
||||
sortCriteria := gql_generated.SortCriteriaAlphabeticAsc
|
||||
pageInput := gql_generated.PageInput{
|
||||
Limit: &limit,
|
||||
Offset: &ofset,
|
||||
Offset: &offset,
|
||||
SortBy: &sortCriteria,
|
||||
}
|
||||
|
||||
|
@ -160,11 +160,11 @@ func TestGlobalSearch(t *testing.T) {
|
|||
|
||||
query := "repo1"
|
||||
limit := 1
|
||||
ofset := 0
|
||||
offset := 0
|
||||
sortCriteria := gql_generated.SortCriteriaAlphabeticAsc
|
||||
pageInput := gql_generated.PageInput{
|
||||
Limit: &limit,
|
||||
Offset: &ofset,
|
||||
Offset: &offset,
|
||||
SortBy: &sortCriteria,
|
||||
}
|
||||
|
||||
|
@ -224,11 +224,11 @@ func TestGlobalSearch(t *testing.T) {
|
|||
|
||||
query := "repo1"
|
||||
limit := 1
|
||||
ofset := 0
|
||||
offset := 0
|
||||
sortCriteria := gql_generated.SortCriteriaAlphabeticAsc
|
||||
pageInput := gql_generated.PageInput{
|
||||
Limit: &limit,
|
||||
Offset: &ofset,
|
||||
Offset: &offset,
|
||||
SortBy: &sortCriteria,
|
||||
}
|
||||
|
||||
|
@ -329,11 +329,11 @@ func TestGlobalSearch(t *testing.T) {
|
|||
|
||||
const query = "repo1:1.0.1"
|
||||
limit := 1
|
||||
ofset := 0
|
||||
offset := 0
|
||||
sortCriteria := gql_generated.SortCriteriaAlphabeticAsc
|
||||
pageInput := gql_generated.PageInput{
|
||||
Limit: &limit,
|
||||
Offset: &ofset,
|
||||
Offset: &offset,
|
||||
SortBy: &sortCriteria,
|
||||
}
|
||||
|
||||
|
@ -365,11 +365,11 @@ func TestRepoListWithNewestImage(t *testing.T) {
|
|||
mockCve := mocks.CveInfoMock{}
|
||||
|
||||
limit := 1
|
||||
ofset := 0
|
||||
offset := 0
|
||||
sortCriteria := gql_generated.SortCriteriaUpdateTime
|
||||
pageInput := gql_generated.PageInput{
|
||||
Limit: &limit,
|
||||
Offset: &ofset,
|
||||
Offset: &offset,
|
||||
SortBy: &sortCriteria,
|
||||
}
|
||||
repos, err := repoListWithNewestImage(responseContext, mockCve, log.NewLogger("debug", ""), &pageInput, mockRepoDB)
|
||||
|
@ -431,11 +431,11 @@ func TestRepoListWithNewestImage(t *testing.T) {
|
|||
mockCve := mocks.CveInfoMock{}
|
||||
|
||||
limit := 1
|
||||
ofset := 0
|
||||
offset := 0
|
||||
sortCriteria := gql_generated.SortCriteriaUpdateTime
|
||||
pageInput := gql_generated.PageInput{
|
||||
Limit: &limit,
|
||||
Offset: &ofset,
|
||||
Offset: &offset,
|
||||
SortBy: &sortCriteria,
|
||||
}
|
||||
repos, err := repoListWithNewestImage(responseContext, mockCve, log.NewLogger("debug", ""), &pageInput, mockRepoDB)
|
||||
|
@ -529,11 +529,11 @@ func TestRepoListWithNewestImage(t *testing.T) {
|
|||
|
||||
Convey("RepoDB SearchRepo is successful", func() {
|
||||
limit := 2
|
||||
ofset := 0
|
||||
offset := 0
|
||||
sortCriteria := gql_generated.SortCriteriaUpdateTime
|
||||
pageInput := gql_generated.PageInput{
|
||||
Limit: &limit,
|
||||
Offset: &ofset,
|
||||
Offset: &offset,
|
||||
SortBy: &sortCriteria,
|
||||
}
|
||||
|
||||
|
@ -553,6 +553,439 @@ func TestRepoListWithNewestImage(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestImageListForDigest(t *testing.T) {
|
||||
Convey("getImageList", t, func() {
|
||||
Convey("no page requested, FilterTagsFn returns error", func() {
|
||||
mockSearchDB := mocks.RepoDBMock{
|
||||
FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc,
|
||||
requestedPage repodb.PageInput,
|
||||
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
|
||||
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, ErrTestError
|
||||
},
|
||||
}
|
||||
|
||||
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
|
||||
graphql.DefaultRecover)
|
||||
|
||||
_, err := getImageListForDigest(responseContext, "invalid", mockSearchDB, mocks.CveInfoMock{}, nil)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("invalid manifest blob", func() {
|
||||
mockSearchDB := mocks.RepoDBMock{
|
||||
FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc,
|
||||
requestedPage repodb.PageInput,
|
||||
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
|
||||
repos := []repodb.RepoMetadata{
|
||||
{
|
||||
Name: "test",
|
||||
Tags: map[string]repodb.Descriptor{
|
||||
"1.0.1": {Digest: "digestTag1.0.1", MediaType: ispec.MediaTypeImageManifest},
|
||||
},
|
||||
Stars: 100,
|
||||
},
|
||||
}
|
||||
|
||||
configBlob, err := json.Marshal(ispec.Image{
|
||||
Config: ispec.ImageConfig{
|
||||
Labels: map[string]string{},
|
||||
},
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestBlob := []byte("invalid")
|
||||
|
||||
manifestMetaDatas := map[string]repodb.ManifestMetadata{
|
||||
"digestTag1.0.1": {
|
||||
ManifestBlob: manifestBlob,
|
||||
ConfigBlob: configBlob,
|
||||
DownloadCount: 0,
|
||||
},
|
||||
}
|
||||
|
||||
return repos, manifestMetaDatas, nil
|
||||
},
|
||||
}
|
||||
|
||||
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
|
||||
graphql.DefaultRecover)
|
||||
|
||||
imageList, err := getImageListForDigest(responseContext, "test", mockSearchDB, mocks.CveInfoMock{}, nil)
|
||||
So(err, ShouldBeNil)
|
||||
So(imageList, ShouldBeEmpty)
|
||||
})
|
||||
|
||||
Convey("valid imageListForDigest returned for matching manifest digest", func() {
|
||||
manifestBlob, err := json.Marshal(ispec.Manifest{})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestDigest := godigest.FromBytes(manifestBlob).String()
|
||||
|
||||
mockSearchDB := mocks.RepoDBMock{
|
||||
FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc,
|
||||
requestedPage repodb.PageInput,
|
||||
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
|
||||
repos := []repodb.RepoMetadata{
|
||||
{
|
||||
Name: "test",
|
||||
Tags: map[string]repodb.Descriptor{
|
||||
"1.0.1": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest},
|
||||
},
|
||||
Stars: 100,
|
||||
},
|
||||
}
|
||||
|
||||
configBlob, err := json.Marshal(ispec.ImageConfig{})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestMetaDatas := map[string]repodb.ManifestMetadata{
|
||||
manifestDigest: {
|
||||
ManifestBlob: manifestBlob,
|
||||
ConfigBlob: configBlob,
|
||||
DownloadCount: 0,
|
||||
},
|
||||
}
|
||||
|
||||
matchedTags := repos[0].Tags
|
||||
for tag, descriptor := range repos[0].Tags {
|
||||
if !filter(repos[0], manifestMetaDatas[descriptor.Digest]) {
|
||||
delete(matchedTags, tag)
|
||||
delete(manifestMetaDatas, descriptor.Digest)
|
||||
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
repos[0].Tags = matchedTags
|
||||
|
||||
return repos, manifestMetaDatas, nil
|
||||
},
|
||||
}
|
||||
|
||||
limit := 1
|
||||
offset := 0
|
||||
sortCriteria := gql_generated.SortCriteriaAlphabeticAsc
|
||||
pageInput := gql_generated.PageInput{
|
||||
Limit: &limit,
|
||||
Offset: &offset,
|
||||
SortBy: &sortCriteria,
|
||||
}
|
||||
|
||||
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
|
||||
graphql.DefaultRecover)
|
||||
|
||||
imageSummaries, err := getImageListForDigest(responseContext, manifestDigest,
|
||||
mockSearchDB, mocks.CveInfoMock{}, &pageInput)
|
||||
So(err, ShouldBeNil)
|
||||
So(len(imageSummaries), ShouldEqual, 1)
|
||||
|
||||
imageSummaries, err = getImageListForDigest(responseContext, "invalid",
|
||||
mockSearchDB, mocks.CveInfoMock{}, &pageInput)
|
||||
So(err, ShouldBeNil)
|
||||
So(len(imageSummaries), ShouldEqual, 0)
|
||||
})
|
||||
|
||||
Convey("valid imageListForDigest returned for matching config digest", func() {
|
||||
manifestBlob, err := json.Marshal(ispec.Manifest{})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestDigest := godigest.FromBytes(manifestBlob).String()
|
||||
|
||||
configBlob, err := json.Marshal(ispec.Image{})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
configDigest := godigest.FromBytes(configBlob)
|
||||
|
||||
mockSearchDB := mocks.RepoDBMock{
|
||||
FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc,
|
||||
requestedPage repodb.PageInput,
|
||||
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
|
||||
repos := []repodb.RepoMetadata{
|
||||
{
|
||||
Name: "test",
|
||||
Tags: map[string]repodb.Descriptor{
|
||||
"1.0.1": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest},
|
||||
},
|
||||
Stars: 100,
|
||||
},
|
||||
}
|
||||
|
||||
manifestBlob, err := json.Marshal(ispec.Manifest{
|
||||
Config: ispec.Descriptor{
|
||||
Digest: configDigest,
|
||||
},
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestMetaDatas := map[string]repodb.ManifestMetadata{
|
||||
manifestDigest: {
|
||||
ManifestBlob: manifestBlob,
|
||||
ConfigBlob: configBlob,
|
||||
DownloadCount: 0,
|
||||
},
|
||||
}
|
||||
|
||||
matchedTags := repos[0].Tags
|
||||
for tag, descriptor := range repos[0].Tags {
|
||||
if !filter(repos[0], manifestMetaDatas[descriptor.Digest]) {
|
||||
delete(matchedTags, tag)
|
||||
delete(manifestMetaDatas, descriptor.Digest)
|
||||
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
repos[0].Tags = matchedTags
|
||||
|
||||
return repos, manifestMetaDatas, nil
|
||||
},
|
||||
}
|
||||
|
||||
limit := 1
|
||||
offset := 0
|
||||
sortCriteria := gql_generated.SortCriteriaAlphabeticAsc
|
||||
pageInput := gql_generated.PageInput{
|
||||
Limit: &limit,
|
||||
Offset: &offset,
|
||||
SortBy: &sortCriteria,
|
||||
}
|
||||
|
||||
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
|
||||
graphql.DefaultRecover)
|
||||
|
||||
imageSummaries, err := getImageListForDigest(responseContext, configDigest.String(),
|
||||
mockSearchDB, mocks.CveInfoMock{}, &pageInput)
|
||||
So(err, ShouldBeNil)
|
||||
So(len(imageSummaries), ShouldEqual, 1)
|
||||
})
|
||||
|
||||
Convey("valid imageListForDigest returned for matching layer digest", func() {
|
||||
manifestBlob, err := json.Marshal(ispec.Manifest{})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestDigest := godigest.FromBytes(manifestBlob).String()
|
||||
|
||||
configBlob, err := json.Marshal(ispec.Image{})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
layerDigest := godigest.Digest("validDigest")
|
||||
|
||||
mockSearchDB := mocks.RepoDBMock{
|
||||
FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc,
|
||||
requestedPage repodb.PageInput,
|
||||
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
|
||||
repos := []repodb.RepoMetadata{
|
||||
{
|
||||
Name: "test",
|
||||
Tags: map[string]repodb.Descriptor{
|
||||
"1.0.1": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest},
|
||||
},
|
||||
Stars: 100,
|
||||
},
|
||||
}
|
||||
|
||||
manifestBlob, err := json.Marshal(ispec.Manifest{
|
||||
Layers: []ispec.Descriptor{
|
||||
{
|
||||
Digest: layerDigest,
|
||||
},
|
||||
},
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestMetaDatas := map[string]repodb.ManifestMetadata{
|
||||
manifestDigest: {
|
||||
ManifestBlob: manifestBlob,
|
||||
ConfigBlob: configBlob,
|
||||
DownloadCount: 0,
|
||||
},
|
||||
}
|
||||
|
||||
matchedTags := repos[0].Tags
|
||||
for tag, descriptor := range repos[0].Tags {
|
||||
if !filter(repos[0], manifestMetaDatas[descriptor.Digest]) {
|
||||
delete(matchedTags, tag)
|
||||
delete(manifestMetaDatas, descriptor.Digest)
|
||||
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
repos[0].Tags = matchedTags
|
||||
|
||||
return repos, manifestMetaDatas, nil
|
||||
},
|
||||
}
|
||||
|
||||
limit := 1
|
||||
offset := 0
|
||||
sortCriteria := gql_generated.SortCriteriaAlphabeticAsc
|
||||
pageInput := gql_generated.PageInput{
|
||||
Limit: &limit,
|
||||
Offset: &offset,
|
||||
SortBy: &sortCriteria,
|
||||
}
|
||||
|
||||
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
|
||||
graphql.DefaultRecover)
|
||||
|
||||
imageSummaries, err := getImageListForDigest(responseContext, layerDigest.String(),
|
||||
mockSearchDB, mocks.CveInfoMock{}, &pageInput)
|
||||
So(err, ShouldBeNil)
|
||||
So(len(imageSummaries), ShouldEqual, 1)
|
||||
})
|
||||
|
||||
Convey("valid imageListForDigest, multiple matching tags", func() {
|
||||
manifestBlob, err := json.Marshal(ispec.Manifest{})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestDigest := godigest.FromBytes(manifestBlob).String()
|
||||
|
||||
configBlob, err := json.Marshal(ispec.Image{})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
mockSearchDB := mocks.RepoDBMock{
|
||||
FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc,
|
||||
requestedPage repodb.PageInput,
|
||||
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
|
||||
repos := []repodb.RepoMetadata{
|
||||
{
|
||||
Name: "test",
|
||||
Tags: map[string]repodb.Descriptor{
|
||||
"1.0.1": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest},
|
||||
"1.0.2": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest},
|
||||
},
|
||||
Stars: 100,
|
||||
},
|
||||
}
|
||||
|
||||
manifestMetaDatas := map[string]repodb.ManifestMetadata{
|
||||
manifestDigest: {
|
||||
ManifestBlob: manifestBlob,
|
||||
ConfigBlob: configBlob,
|
||||
DownloadCount: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for i, repo := range repos {
|
||||
matchedTags := repo.Tags
|
||||
|
||||
for tag, descriptor := range repo.Tags {
|
||||
if !filter(repo, manifestMetaDatas[descriptor.Digest]) {
|
||||
delete(matchedTags, tag)
|
||||
delete(manifestMetaDatas, descriptor.Digest)
|
||||
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
repos[i].Tags = matchedTags
|
||||
}
|
||||
|
||||
return repos, manifestMetaDatas, nil
|
||||
},
|
||||
}
|
||||
|
||||
limit := 1
|
||||
offset := 0
|
||||
sortCriteria := gql_generated.SortCriteriaAlphabeticAsc
|
||||
pageInput := gql_generated.PageInput{
|
||||
Limit: &limit,
|
||||
Offset: &offset,
|
||||
SortBy: &sortCriteria,
|
||||
}
|
||||
|
||||
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
|
||||
graphql.DefaultRecover)
|
||||
|
||||
imageSummaries, err := getImageListForDigest(responseContext, manifestDigest,
|
||||
mockSearchDB, mocks.CveInfoMock{}, &pageInput)
|
||||
So(err, ShouldBeNil)
|
||||
So(len(imageSummaries), ShouldEqual, 2)
|
||||
})
|
||||
|
||||
Convey("valid imageListForDigest, multiple matching tags limited by pageInput", func() {
|
||||
manifestBlob, err := json.Marshal(ispec.Manifest{})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestDigest := godigest.FromBytes(manifestBlob).String()
|
||||
|
||||
configBlob, err := json.Marshal(ispec.Image{})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
mockSearchDB := mocks.RepoDBMock{
|
||||
FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc,
|
||||
requestedPage repodb.PageInput,
|
||||
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
|
||||
pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
|
||||
if err != nil {
|
||||
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, err
|
||||
}
|
||||
|
||||
repos := []repodb.RepoMetadata{
|
||||
{
|
||||
Name: "test",
|
||||
Tags: map[string]repodb.Descriptor{
|
||||
"1.0.1": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest},
|
||||
"1.0.2": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest},
|
||||
},
|
||||
Stars: 100,
|
||||
},
|
||||
}
|
||||
|
||||
manifestMetaDatas := map[string]repodb.ManifestMetadata{
|
||||
manifestDigest: {
|
||||
ManifestBlob: manifestBlob,
|
||||
ConfigBlob: configBlob,
|
||||
DownloadCount: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for i, repo := range repos {
|
||||
matchedTags := repo.Tags
|
||||
|
||||
for tag, descriptor := range repo.Tags {
|
||||
if !filter(repo, manifestMetaDatas[descriptor.Digest]) {
|
||||
delete(matchedTags, tag)
|
||||
delete(manifestMetaDatas, descriptor.Digest)
|
||||
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
repos[i].Tags = matchedTags
|
||||
|
||||
pageFinder.Add(repodb.DetailedRepoMeta{
|
||||
RepoMeta: repo,
|
||||
})
|
||||
}
|
||||
|
||||
repos = pageFinder.Page()
|
||||
|
||||
return repos, manifestMetaDatas, nil
|
||||
},
|
||||
}
|
||||
|
||||
limit := 1
|
||||
offset := 0
|
||||
sortCriteria := gql_generated.SortCriteriaAlphabeticAsc
|
||||
pageInput := gql_generated.PageInput{
|
||||
Limit: &limit,
|
||||
Offset: &offset,
|
||||
SortBy: &sortCriteria,
|
||||
}
|
||||
|
||||
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
|
||||
graphql.DefaultRecover)
|
||||
|
||||
imageSummaries, err := getImageListForDigest(responseContext, manifestDigest,
|
||||
mockSearchDB, mocks.CveInfoMock{}, &pageInput)
|
||||
So(err, ShouldBeNil)
|
||||
So(len(imageSummaries), ShouldEqual, 1)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetReferrers(t *testing.T) {
|
||||
Convey("getReferrers", t, func() {
|
||||
Convey("GetReferrers returns error", func() {
|
||||
|
@ -907,120 +1340,6 @@ func TestQueryResolverErrors(t *testing.T) {
|
|||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("ImageListForDigest defaultStore.GetRepositories() errors", func() {
|
||||
resolverConfig := NewResolver(
|
||||
log,
|
||||
storage.StoreController{
|
||||
DefaultStore: mocks.MockedImageStore{
|
||||
GetRepositoriesFn: func() ([]string, error) {
|
||||
return nil, ErrTestError
|
||||
},
|
||||
},
|
||||
},
|
||||
mocks.RepoDBMock{},
|
||||
mocks.CveInfoMock{},
|
||||
)
|
||||
|
||||
qr := queryResolver{
|
||||
resolverConfig,
|
||||
}
|
||||
|
||||
_, err := qr.ImageListForDigest(ctx, "")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("ImageListForDigest getImageListForDigest() errors", func() {
|
||||
resolverConfig := NewResolver(
|
||||
log,
|
||||
storage.StoreController{
|
||||
DefaultStore: mocks.MockedImageStore{
|
||||
GetRepositoriesFn: func() ([]string, error) {
|
||||
return []string{"repo"}, nil
|
||||
},
|
||||
GetIndexContentFn: func(repo string) ([]byte, error) {
|
||||
return nil, ErrTestError
|
||||
},
|
||||
},
|
||||
},
|
||||
mocks.RepoDBMock{},
|
||||
mocks.CveInfoMock{},
|
||||
)
|
||||
|
||||
qr := queryResolver{
|
||||
resolverConfig,
|
||||
}
|
||||
|
||||
_, err := qr.ImageListForDigest(ctx, "")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("ImageListForDigest substores store.GetRepositories() errors", func() {
|
||||
resolverConfig := NewResolver(
|
||||
log,
|
||||
storage.StoreController{
|
||||
DefaultStore: mocks.MockedImageStore{
|
||||
GetIndexContentFn: func(repo string) ([]byte, error) {
|
||||
return []byte("{}"), nil
|
||||
},
|
||||
GetRepositoriesFn: func() ([]string, error) {
|
||||
return []string{"repo"}, nil
|
||||
},
|
||||
},
|
||||
SubStore: map[string]storage.ImageStore{
|
||||
"sub1": mocks.MockedImageStore{
|
||||
GetRepositoriesFn: func() ([]string, error) {
|
||||
return []string{"repo"}, ErrTestError
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
mocks.RepoDBMock{},
|
||||
mocks.CveInfoMock{},
|
||||
)
|
||||
|
||||
qr := queryResolver{
|
||||
resolverConfig,
|
||||
}
|
||||
|
||||
_, err := qr.ImageListForDigest(ctx, "")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("ImageListForDigest substores getImageListForDigest() errors", func() {
|
||||
resolverConfig := NewResolver(
|
||||
log,
|
||||
storage.StoreController{
|
||||
DefaultStore: mocks.MockedImageStore{
|
||||
GetIndexContentFn: func(repo string) ([]byte, error) {
|
||||
return []byte("{}"), nil
|
||||
},
|
||||
GetRepositoriesFn: func() ([]string, error) {
|
||||
return []string{"repo"}, nil
|
||||
},
|
||||
},
|
||||
SubStore: map[string]storage.ImageStore{
|
||||
"/sub1": mocks.MockedImageStore{
|
||||
GetRepositoriesFn: func() ([]string, error) {
|
||||
return []string{"sub1/repo"}, nil
|
||||
},
|
||||
GetIndexContentFn: func(repo string) ([]byte, error) {
|
||||
return nil, ErrTestError
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
mocks.RepoDBMock{},
|
||||
mocks.CveInfoMock{},
|
||||
)
|
||||
|
||||
qr := queryResolver{
|
||||
resolverConfig,
|
||||
}
|
||||
|
||||
_, err := qr.ImageListForDigest(ctx, "")
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("RepoListWithNewestImage repoListWithNewestImage() errors", func() {
|
||||
resolverConfig := NewResolver(
|
||||
log,
|
||||
|
|
|
@ -205,7 +205,7 @@ type Query {
|
|||
"""
|
||||
Returns a list of images which contain the specified digest
|
||||
"""
|
||||
ImageListForDigest(id: String!): [ImageSummary!]
|
||||
ImageListForDigest(id: String!, requestedPage: PageInput): [ImageSummary!]
|
||||
|
||||
"""
|
||||
Returns a list of repos with the newest tag within
|
||||
|
|
|
@ -163,51 +163,12 @@ func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, im
|
|||
}
|
||||
|
||||
// ImageListForDigest is the resolver for the ImageListForDigest field.
|
||||
func (r *queryResolver) ImageListForDigest(ctx context.Context, id string) ([]*gql_generated.ImageSummary, error) {
|
||||
imgResultForDigest := []*gql_generated.ImageSummary{}
|
||||
|
||||
func (r *queryResolver) ImageListForDigest(ctx context.Context, id string, requestedPage *gql_generated.PageInput) ([]*gql_generated.ImageSummary, error) {
|
||||
r.log.Info().Msg("extracting repositories")
|
||||
|
||||
defaultStore := r.storeController.DefaultStore
|
||||
imgResultForDigest, err := getImageListForDigest(ctx, id, r.repoDB, r.cveInfo, requestedPage)
|
||||
|
||||
repoList, err := defaultStore.GetRepositories()
|
||||
if err != nil {
|
||||
r.log.Error().Err(err).Msg("unable to search repositories")
|
||||
|
||||
return imgResultForDigest, err
|
||||
}
|
||||
|
||||
r.log.Info().Msg("scanning each global repository")
|
||||
|
||||
partialImgResultForDigest, err := r.getImageListForDigest(repoList, id)
|
||||
if err != nil {
|
||||
r.log.Error().Err(err).Msg("unable to get image and tag list for global repositories")
|
||||
|
||||
return imgResultForDigest, err
|
||||
}
|
||||
|
||||
imgResultForDigest = append(imgResultForDigest, partialImgResultForDigest...)
|
||||
|
||||
subStore := r.storeController.SubStore
|
||||
for _, store := range subStore {
|
||||
subRepoList, err := store.GetRepositories()
|
||||
if err != nil {
|
||||
r.log.Error().Err(err).Msg("unable to search sub-repositories")
|
||||
|
||||
return imgResultForDigest, err
|
||||
}
|
||||
|
||||
partialImgResultForDigest, err = r.getImageListForDigest(subRepoList, id)
|
||||
if err != nil {
|
||||
r.log.Error().Err(err).Msg("unable to get image and tag list for sub-repositories")
|
||||
|
||||
return imgResultForDigest, err
|
||||
}
|
||||
|
||||
imgResultForDigest = append(imgResultForDigest, partialImgResultForDigest...)
|
||||
}
|
||||
|
||||
return imgResultForDigest, nil
|
||||
return imgResultForDigest, err
|
||||
}
|
||||
|
||||
// RepoListWithNewestImage is the resolver for the RepoListWithNewestImage field.
|
||||
|
|
|
@ -743,6 +743,113 @@ func (bdw DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
|
|||
return foundRepos, foundManifestMetadataMap, err
|
||||
}
|
||||
|
||||
func (bdw DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
|
||||
requestedPage repodb.PageInput,
|
||||
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
|
||||
var (
|
||||
foundRepos = make([]repodb.RepoMetadata, 0)
|
||||
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
|
||||
pageFinder repodb.PageFinder
|
||||
)
|
||||
|
||||
pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
|
||||
if err != nil {
|
||||
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, err
|
||||
}
|
||||
|
||||
err = bdw.DB.View(func(tx *bolt.Tx) error {
|
||||
var (
|
||||
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
|
||||
repoBuck = tx.Bucket([]byte(repodb.RepoMetadataBucket))
|
||||
dataBuck = tx.Bucket([]byte(repodb.ManifestDataBucket))
|
||||
cursor = repoBuck.Cursor()
|
||||
)
|
||||
|
||||
repoName, repoMetaBlob := cursor.First()
|
||||
|
||||
for ; repoName != nil; repoName, repoMetaBlob = cursor.Next() {
|
||||
if ok, err := localCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
repoMeta := repodb.RepoMetadata{}
|
||||
|
||||
err := json.Unmarshal(repoMetaBlob, &repoMeta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
matchedTags := make(map[string]repodb.Descriptor)
|
||||
// take all manifestMetas
|
||||
for tag, descriptor := range repoMeta.Tags {
|
||||
manifestDigest := descriptor.Digest
|
||||
|
||||
matchedTags[tag] = descriptor
|
||||
|
||||
// in case tags reference the same manifest we don't download from DB multiple times
|
||||
manifestMeta, manifestExists := manifestMetadataMap[manifestDigest]
|
||||
|
||||
if !manifestExists {
|
||||
manifestDataBlob := dataBuck.Get([]byte(manifestDigest))
|
||||
if manifestDataBlob == nil {
|
||||
return zerr.ErrManifestMetaNotFound
|
||||
}
|
||||
|
||||
var manifestData repodb.ManifestData
|
||||
|
||||
err := json.Unmarshal(manifestDataBlob, &manifestData)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", manifestDigest)
|
||||
}
|
||||
|
||||
var configContent ispec.Image
|
||||
|
||||
err = json.Unmarshal(manifestData.ConfigBlob, &configContent)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", manifestDigest)
|
||||
}
|
||||
|
||||
manifestMeta = repodb.ManifestMetadata{
|
||||
ConfigBlob: manifestData.ConfigBlob,
|
||||
ManifestBlob: manifestData.ManifestBlob,
|
||||
}
|
||||
}
|
||||
|
||||
if !filter(repoMeta, manifestMeta) {
|
||||
delete(matchedTags, tag)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
manifestMetadataMap[manifestDigest] = manifestMeta
|
||||
}
|
||||
|
||||
if len(matchedTags) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
repoMeta.Tags = matchedTags
|
||||
|
||||
pageFinder.Add(repodb.DetailedRepoMeta{
|
||||
RepoMeta: repoMeta,
|
||||
})
|
||||
}
|
||||
|
||||
foundRepos = pageFinder.Page()
|
||||
|
||||
// keep just the manifestMeta we need
|
||||
for _, repoMeta := range foundRepos {
|
||||
for _, descriptor := range repoMeta.Tags {
|
||||
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
|
||||
return foundRepos, foundManifestMetadataMap, err
|
||||
}
|
||||
|
||||
func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter,
|
||||
requestedPage repodb.PageInput,
|
||||
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
|
||||
|
@ -819,7 +926,7 @@ func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
|
|||
|
||||
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
|
||||
if err != nil {
|
||||
return errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", descriptor.Digest)
|
||||
return errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", descriptor.Digest)
|
||||
}
|
||||
|
||||
imageFilterData := repodb.FilterData{
|
||||
|
@ -838,6 +945,10 @@ func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
|
|||
manifestMetadataMap[descriptor.Digest] = manifestMeta
|
||||
}
|
||||
|
||||
if len(matchedTags) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
repoMeta.Tags = matchedTags
|
||||
|
||||
pageFinder.Add(repodb.DetailedRepoMeta{
|
||||
|
|
|
@ -393,6 +393,75 @@ func TestWrapperErrors(t *testing.T) {
|
|||
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("FilterTags repoMeta unmarshal error", func() {
|
||||
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, _, err = dynamoWrapper.FilterTags(
|
||||
ctx,
|
||||
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
|
||||
return true
|
||||
},
|
||||
repodb.PageInput{},
|
||||
)
|
||||
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("FilterTags manifestMeta not found", func() {
|
||||
err := dynamoWrapper.SetRepoTag("repo", "tag1", "manifestNotFound", "") //nolint:contextcheck
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, _, err = dynamoWrapper.FilterTags(
|
||||
ctx,
|
||||
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
|
||||
return true
|
||||
},
|
||||
repodb.PageInput{},
|
||||
)
|
||||
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("FilterTags manifestMeta unmarshal error", func() {
|
||||
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig", "") //nolint:contextcheck
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = setBadManifestData(dynamoWrapper.Client, manifestDataTablename, "dig") //nolint:contextcheck
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, _, err = dynamoWrapper.FilterTags(
|
||||
ctx,
|
||||
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
|
||||
return true
|
||||
},
|
||||
repodb.PageInput{},
|
||||
)
|
||||
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
||||
Convey("FilterTags config unmarshal error", func() {
|
||||
err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", "") //nolint:contextcheck
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = dynamoWrapper.SetManifestData("dig1", repodb.ManifestData{ //nolint:contextcheck
|
||||
ManifestBlob: []byte("{}"),
|
||||
ConfigBlob: []byte("bad json"),
|
||||
})
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, _, err = dynamoWrapper.FilterTags(
|
||||
ctx,
|
||||
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
|
||||
return true
|
||||
},
|
||||
repodb.PageInput{},
|
||||
)
|
||||
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
@ -650,6 +650,101 @@ func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
|
|||
return foundRepos, foundManifestMetadataMap, err
|
||||
}
|
||||
|
||||
func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
|
||||
requestedPage repodb.PageInput,
|
||||
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
|
||||
var (
|
||||
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
|
||||
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
|
||||
pageFinder repodb.PageFinder
|
||||
repoMetaAttributeIterator iterator.AttributesIterator
|
||||
)
|
||||
|
||||
repoMetaAttributeIterator = iterator.NewBaseDynamoAttributesIterator(
|
||||
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
|
||||
)
|
||||
|
||||
pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
|
||||
if err != nil {
|
||||
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, err
|
||||
}
|
||||
|
||||
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
|
||||
|
||||
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
|
||||
if err != nil {
|
||||
// log
|
||||
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, err
|
||||
}
|
||||
|
||||
var repoMeta repodb.RepoMetadata
|
||||
|
||||
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
|
||||
if err != nil {
|
||||
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, err
|
||||
}
|
||||
|
||||
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
|
||||
continue
|
||||
}
|
||||
matchedTags := make(map[string]repodb.Descriptor)
|
||||
// take all manifestMetas
|
||||
for tag, descriptor := range repoMeta.Tags {
|
||||
manifestDigest := descriptor.Digest
|
||||
|
||||
matchedTags[tag] = descriptor
|
||||
|
||||
// in case tags reference the same manifest we don't download from DB multiple times
|
||||
manifestMeta, manifestExists := manifestMetadataMap[manifestDigest]
|
||||
|
||||
if !manifestExists {
|
||||
manifestMeta, err := dwr.GetManifestMeta(repoMeta.Name, godigest.Digest(manifestDigest)) //nolint:contextcheck
|
||||
if err != nil {
|
||||
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{},
|
||||
errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", manifestDigest)
|
||||
}
|
||||
|
||||
var configContent ispec.Image
|
||||
|
||||
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
|
||||
if err != nil {
|
||||
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{},
|
||||
errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", manifestDigest)
|
||||
}
|
||||
}
|
||||
|
||||
if !filter(repoMeta, manifestMeta) {
|
||||
delete(matchedTags, tag)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
manifestMetadataMap[manifestDigest] = manifestMeta
|
||||
}
|
||||
|
||||
if len(matchedTags) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
repoMeta.Tags = matchedTags
|
||||
|
||||
pageFinder.Add(repodb.DetailedRepoMeta{
|
||||
RepoMeta: repoMeta,
|
||||
})
|
||||
}
|
||||
|
||||
foundRepos := pageFinder.Page()
|
||||
|
||||
// keep just the manifestMeta we need
|
||||
for _, repoMeta := range foundRepos {
|
||||
for _, descriptor := range repoMeta.Tags {
|
||||
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
|
||||
}
|
||||
}
|
||||
|
||||
return foundRepos, foundManifestMetadataMap, err
|
||||
}
|
||||
|
||||
func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter,
|
||||
requestedPage repodb.PageInput,
|
||||
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
|
||||
|
@ -721,7 +816,7 @@ func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
|
|||
err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent)
|
||||
if err != nil {
|
||||
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{},
|
||||
errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", descriptor.Digest)
|
||||
errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", descriptor.Digest)
|
||||
}
|
||||
|
||||
imageFilterData := repodb.FilterData{
|
||||
|
@ -740,6 +835,10 @@ func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter r
|
|||
manifestMetadataMap[descriptor.Digest] = manifestMeta
|
||||
}
|
||||
|
||||
if len(matchedTags) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
repoMeta.Tags = matchedTags
|
||||
|
||||
pageFinder.Add(repodb.DetailedRepoMeta{
|
||||
|
|
|
@ -149,6 +149,17 @@ func (bpt *ImagePageFinder) Page() []RepoMetadata {
|
|||
remainingOffset := bpt.offset
|
||||
remainingLimit := bpt.limit
|
||||
|
||||
repos := make([]RepoMetadata, 0)
|
||||
|
||||
if remainingOffset == 0 && remainingLimit == 0 {
|
||||
for _, drm := range bpt.pageBuffer {
|
||||
repo := drm.RepoMeta
|
||||
repos = append(repos, repo)
|
||||
}
|
||||
|
||||
return repos
|
||||
}
|
||||
|
||||
// bring cursor to position in RepoMeta array
|
||||
for _, drm := range bpt.pageBuffer {
|
||||
if remainingOffset < len(drm.RepoMeta.Tags) {
|
||||
|
@ -166,8 +177,6 @@ func (bpt *ImagePageFinder) Page() []RepoMetadata {
|
|||
return []RepoMetadata{}
|
||||
}
|
||||
|
||||
repos := make([]RepoMetadata, 0)
|
||||
|
||||
// finish counting remaining tags inside the first repo meta
|
||||
partialTags := map[string]Descriptor{}
|
||||
firstRepoMeta := bpt.pageBuffer[repoStartIndex].RepoMeta
|
||||
|
|
|
@ -21,6 +21,8 @@ const (
|
|||
CosignType = "cosign"
|
||||
)
|
||||
|
||||
type FilterFunc func(repoMeta RepoMetadata, manifestMeta ManifestMetadata) bool
|
||||
|
||||
type RepoDB interface { //nolint:interfacebloat
|
||||
// IncrementRepoStars adds 1 to the star count of an image
|
||||
IncrementRepoStars(repo string) error
|
||||
|
@ -74,6 +76,10 @@ type RepoDB interface { //nolint:interfacebloat
|
|||
SearchTags(ctx context.Context, searchText string, filter Filter, requestedPage PageInput) (
|
||||
[]RepoMetadata, map[string]ManifestMetadata, error)
|
||||
|
||||
// FilterTags filters for images given a filter function
|
||||
FilterTags(ctx context.Context, filter FilterFunc,
|
||||
requestedPage PageInput) ([]RepoMetadata, map[string]ManifestMetadata, error)
|
||||
|
||||
PatchDB() error
|
||||
}
|
||||
|
||||
|
|
|
@ -1286,6 +1286,155 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
|
|||
So(err, ShouldBeNil)
|
||||
So(len(repos), ShouldEqual, 0)
|
||||
})
|
||||
|
||||
Convey("Test FilterTags", func() {
|
||||
var (
|
||||
repo1 = "repo1"
|
||||
repo2 = "repo2"
|
||||
manifestDigest1 = digest.FromString("fake-manifest1")
|
||||
manifestDigest2 = digest.FromString("fake-manifest2")
|
||||
manifestDigest3 = digest.FromString("fake-manifest3")
|
||||
ctx = context.Background()
|
||||
emptyManifest ispec.Manifest
|
||||
emptyConfig ispec.Image
|
||||
)
|
||||
|
||||
emptyManifestBlob, err := json.Marshal(emptyManifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
emptyConfigBlob, err := json.Marshal(emptyConfig)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
emptyRepoMeta := repodb.ManifestMetadata{
|
||||
ManifestBlob: emptyManifestBlob,
|
||||
ConfigBlob: emptyConfigBlob,
|
||||
}
|
||||
|
||||
err = repoDB.SetRepoTag(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest)
|
||||
So(err, ShouldBeNil)
|
||||
err = repoDB.SetRepoTag(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest)
|
||||
So(err, ShouldBeNil)
|
||||
err = repoDB.SetRepoTag(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest)
|
||||
So(err, ShouldBeNil)
|
||||
err = repoDB.SetRepoTag(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest)
|
||||
So(err, ShouldBeNil)
|
||||
err = repoDB.SetRepoTag(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest)
|
||||
So(err, ShouldBeNil)
|
||||
err = repoDB.SetRepoTag(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta)
|
||||
So(err, ShouldBeNil)
|
||||
err = repoDB.SetManifestMeta(repo1, manifestDigest2, emptyRepoMeta)
|
||||
So(err, ShouldBeNil)
|
||||
err = repoDB.SetManifestMeta(repo1, manifestDigest3, emptyRepoMeta)
|
||||
So(err, ShouldBeNil)
|
||||
err = repoDB.SetManifestMeta(repo2, manifestDigest3, emptyRepoMeta)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
Convey("Return all tags", func() {
|
||||
repos, manifesMetaMap, err := repoDB.FilterTags(
|
||||
ctx,
|
||||
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
|
||||
return true
|
||||
},
|
||||
repodb.PageInput{Limit: 10, Offset: 0, SortBy: repodb.AlphabeticAsc},
|
||||
)
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
So(len(repos), ShouldEqual, 2)
|
||||
So(repos[0].Name, ShouldEqual, "repo1")
|
||||
So(repos[1].Name, ShouldEqual, "repo2")
|
||||
So(len(repos[0].Tags), ShouldEqual, 5)
|
||||
So(len(repos[1].Tags), ShouldEqual, 1)
|
||||
So(repos[0].Tags, ShouldContainKey, "0.0.1")
|
||||
So(repos[0].Tags, ShouldContainKey, "0.0.2")
|
||||
So(repos[0].Tags, ShouldContainKey, "0.1.0")
|
||||
So(repos[0].Tags, ShouldContainKey, "1.0.0")
|
||||
So(repos[0].Tags, ShouldContainKey, "1.0.1")
|
||||
So(repos[1].Tags, ShouldContainKey, "0.0.1")
|
||||
So(manifesMetaMap, ShouldContainKey, manifestDigest1.String())
|
||||
So(manifesMetaMap, ShouldContainKey, manifestDigest2.String())
|
||||
So(manifesMetaMap, ShouldContainKey, manifestDigest3.String())
|
||||
})
|
||||
|
||||
Convey("Return all tags in a specific repo", func() {
|
||||
repos, manifesMetaMap, err := repoDB.FilterTags(
|
||||
ctx,
|
||||
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
|
||||
return repoMeta.Name == repo1
|
||||
},
|
||||
repodb.PageInput{Limit: 10, Offset: 0, SortBy: repodb.AlphabeticAsc},
|
||||
)
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
So(len(repos), ShouldEqual, 1)
|
||||
So(repos[0].Name, ShouldEqual, repo1)
|
||||
So(len(repos[0].Tags), ShouldEqual, 5)
|
||||
So(repos[0].Tags, ShouldContainKey, "0.0.1")
|
||||
So(repos[0].Tags, ShouldContainKey, "0.0.2")
|
||||
So(repos[0].Tags, ShouldContainKey, "0.1.0")
|
||||
So(repos[0].Tags, ShouldContainKey, "1.0.0")
|
||||
So(repos[0].Tags, ShouldContainKey, "1.0.1")
|
||||
So(manifesMetaMap, ShouldContainKey, manifestDigest1.String())
|
||||
So(manifesMetaMap, ShouldContainKey, manifestDigest2.String())
|
||||
So(manifesMetaMap, ShouldContainKey, manifestDigest3.String())
|
||||
})
|
||||
|
||||
Convey("Filter everything out", func() {
|
||||
repos, manifesMetaMap, err := repoDB.FilterTags(
|
||||
ctx,
|
||||
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
|
||||
return false
|
||||
},
|
||||
repodb.PageInput{Limit: 10, Offset: 0, SortBy: repodb.AlphabeticAsc},
|
||||
)
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
So(len(repos), ShouldEqual, 0)
|
||||
So(len(manifesMetaMap), ShouldEqual, 0)
|
||||
})
|
||||
|
||||
Convey("Search with access control", func() {
|
||||
acCtx := localCtx.AccessControlContext{
|
||||
ReadGlobPatterns: map[string]bool{
|
||||
repo1: false,
|
||||
repo2: true,
|
||||
},
|
||||
Username: "username",
|
||||
}
|
||||
|
||||
authzCtxKey := localCtx.GetContextKey()
|
||||
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
|
||||
|
||||
repos, manifesMetaMap, err := repoDB.FilterTags(
|
||||
ctx,
|
||||
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
|
||||
return true
|
||||
},
|
||||
repodb.PageInput{Limit: 10, Offset: 0, SortBy: repodb.AlphabeticAsc},
|
||||
)
|
||||
|
||||
So(err, ShouldBeNil)
|
||||
So(len(repos), ShouldEqual, 1)
|
||||
So(repos[0].Name, ShouldResemble, repo2)
|
||||
So(len(repos[0].Tags), ShouldEqual, 1)
|
||||
So(repos[0].Tags, ShouldContainKey, "0.0.1")
|
||||
So(manifesMetaMap, ShouldContainKey, manifestDigest3.String())
|
||||
})
|
||||
|
||||
Convey("With wrong pagination input", func() {
|
||||
repos, _, err := repoDB.FilterTags(
|
||||
ctx,
|
||||
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
|
||||
return true
|
||||
},
|
||||
repodb.PageInput{Limit: -1},
|
||||
)
|
||||
So(err, ShouldNotBeNil)
|
||||
So(repos, ShouldBeEmpty)
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
@ -48,6 +48,10 @@ type RepoDBMock struct {
|
|||
SearchTagsFn func(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput) (
|
||||
[]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error)
|
||||
|
||||
FilterTagsFn func(ctx context.Context, filter repodb.FilterFunc,
|
||||
requestedPage repodb.PageInput,
|
||||
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error)
|
||||
|
||||
SearchDigestsFn func(ctx context.Context, searchText string, requestedPage repodb.PageInput) (
|
||||
[]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error)
|
||||
|
||||
|
@ -217,6 +221,16 @@ func (sdm RepoDBMock) SearchTags(ctx context.Context, searchText string, filter
|
|||
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, nil
|
||||
}
|
||||
|
||||
func (sdm RepoDBMock) FilterTags(ctx context.Context, filter repodb.FilterFunc,
|
||||
requestedPage repodb.PageInput,
|
||||
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
|
||||
if sdm.FilterTagsFn != nil {
|
||||
return sdm.FilterTagsFn(ctx, filter, requestedPage)
|
||||
}
|
||||
|
||||
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, nil
|
||||
}
|
||||
|
||||
func (sdm RepoDBMock) SearchDigests(ctx context.Context, searchText string, requestedPage repodb.PageInput,
|
||||
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
|
||||
if sdm.SearchDigestsFn != nil {
|
||||
|
|
Loading…
Reference in a new issue