From e8e7c343ad9837e78f9416794764934c477d3b76 Mon Sep 17 00:00:00 2001 From: Andrei Aaron Date: Wed, 18 Jan 2023 00:31:54 +0200 Subject: [PATCH] feat(repodb): add pagination for ImageListForDigest and implement FilterTags (#1102) * feat(repodb): add pagination for ImageListForDigest and implement FilterTags ImageListForDigest can now return paginated results, directly from DB. It uses FilterTags, a new method to filter tags (obviously) based on the criteria provided in the filter function. Pagination of tags is now slightly different, it shows all results if no limit and offset are provided. Signed-off-by: Alex Stan bug(tests): cli tests for digests expecting wrong size Signed-off-by: Andrei Aaron (cherry picked from commit 369216df931a4053c18278a8d89f86d2e1e6a436) fix(repodb): do not include repo metadata in search results if no matching tags are identified Signed-off-by: Andrei Aaron * fix(repodb): Fix an issue in FilterTags where repometa was not proceesed correctly The filter function was called only once per manifest digest. The function is supposed to also take into consideration repometa, but only the first repometa-manifestmeta pair was processed. Also increase code coverage. Signed-off-by: Andrei Aaron Signed-off-by: Andrei Aaron Co-authored-by: Alex Stan --- pkg/cli/image_cmd_test.go | 8 +- .../search/gql_generated/generated.go | 19 +- pkg/extensions/search/resolver.go | 86 ++- pkg/extensions/search/resolver_test.go | 575 ++++++++++++++---- pkg/extensions/search/schema.graphql | 2 +- pkg/extensions/search/schema.resolvers.go | 45 +- .../repodb/boltdb-wrapper/boltdb_wrapper.go | 113 +++- .../repodb/dynamodb-wrapper/dynamo_test.go | 69 +++ .../repodb/dynamodb-wrapper/dynamo_wrapper.go | 101 ++- pkg/meta/repodb/pagination.go | 13 +- pkg/meta/repodb/repodb.go | 6 + pkg/meta/repodb/repodb_test.go | 149 +++++ pkg/test/mocks/repo_db_mock.go | 14 + 13 files changed, 994 insertions(+), 206 deletions(-) diff --git a/pkg/cli/image_cmd_test.go b/pkg/cli/image_cmd_test.go index 66538718..8a01981c 100644 --- a/pkg/cli/image_cmd_test.go +++ b/pkg/cli/image_cmd_test.go @@ -1046,8 +1046,8 @@ func TestServerResponseGQL(t *testing.T) { // repo7 test:2.0 a0ca253b 15B // repo7 test:1.0 a0ca253b 15B So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE") - So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 15B") - So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 15B") + So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B") Convey("with shorthand", func() { args := []string{"imagetest", "-d", "883fc0c5"} @@ -1064,8 +1064,8 @@ func TestServerResponseGQL(t *testing.T) { str := space.ReplaceAllString(buff.String(), " ") actual := strings.TrimSpace(str) So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE") - So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 15B") - So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 15B") + So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B") + So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B") }) Convey("nonexistent digest", func() { diff --git a/pkg/extensions/search/gql_generated/generated.go b/pkg/extensions/search/gql_generated/generated.go index f15c756e..c0fdde8b 100644 --- a/pkg/extensions/search/gql_generated/generated.go +++ b/pkg/extensions/search/gql_generated/generated.go @@ -143,7 +143,7 @@ type ComplexityRoot struct { Image func(childComplexity int, image string) int ImageList func(childComplexity int, repo string) int ImageListForCve func(childComplexity int, id string) int - ImageListForDigest func(childComplexity int, id string) int + ImageListForDigest func(childComplexity int, id string, requestedPage *PageInput) int ImageListWithCVEFixed func(childComplexity int, id string, image string) int Referrers func(childComplexity int, repo string, digest string, typeArg string) int RepoListWithNewestImage func(childComplexity int, requestedPage *PageInput) int @@ -181,7 +181,7 @@ type QueryResolver interface { CVEListForImage(ctx context.Context, image string) (*CVEResultForImage, error) ImageListForCve(ctx context.Context, id string) ([]*ImageSummary, error) ImageListWithCVEFixed(ctx context.Context, id string, image string) ([]*ImageSummary, error) - ImageListForDigest(ctx context.Context, id string) ([]*ImageSummary, error) + ImageListForDigest(ctx context.Context, id string, requestedPage *PageInput) ([]*ImageSummary, error) RepoListWithNewestImage(ctx context.Context, requestedPage *PageInput) ([]*RepoSummary, error) ImageList(ctx context.Context, repo string) ([]*ImageSummary, error) ExpandedRepoInfo(ctx context.Context, repo string) (*RepoInfo, error) @@ -698,7 +698,7 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return 0, false } - return e.complexity.Query.ImageListForDigest(childComplexity, args["id"].(string)), true + return e.complexity.Query.ImageListForDigest(childComplexity, args["id"].(string), args["requestedPage"].(*PageInput)), true case "Query.ImageListWithCVEFixed": if e.complexity.Query.ImageListWithCVEFixed == nil { @@ -1124,7 +1124,7 @@ type Query { """ Returns a list of images which contain the specified digest """ - ImageListForDigest(id: String!): [ImageSummary!] + ImageListForDigest(id: String!, requestedPage: PageInput): [ImageSummary!] """ Returns a list of repos with the newest tag within @@ -1295,6 +1295,15 @@ func (ec *executionContext) field_Query_ImageListForDigest_args(ctx context.Cont } } args["id"] = arg0 + var arg1 *PageInput + if tmp, ok := rawArgs["requestedPage"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("requestedPage")) + arg1, err = ec.unmarshalOPageInput2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐPageInput(ctx, tmp) + if err != nil { + return nil, err + } + } + args["requestedPage"] = arg1 return args, nil } @@ -4130,7 +4139,7 @@ func (ec *executionContext) _Query_ImageListForDigest(ctx context.Context, field }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return ec.resolvers.Query().ImageListForDigest(rctx, fc.Args["id"].(string)) + return ec.resolvers.Query().ImageListForDigest(rctx, fc.Args["id"].(string), fc.Args["requestedPage"].(*PageInput)) }) if err != nil { ec.Error(ctx, err) diff --git a/pkg/extensions/search/resolver.go b/pkg/extensions/search/resolver.go index 745e8e3c..17836ac2 100644 --- a/pkg/extensions/search/resolver.go +++ b/pkg/extensions/search/resolver.go @@ -6,6 +6,7 @@ package search import ( "context" + "encoding/json" "strings" "github.com/99designs/gqlgen/graphql" @@ -75,37 +76,78 @@ func NewResolver(log log.Logger, storeController storage.StoreController, return resolver } -func (r *queryResolver) getImageListForDigest(repoList []string, digest string) ([]*gql_generated.ImageSummary, error) { - imgResultForDigest := []*gql_generated.ImageSummary{} - olu := common.NewBaseOciLayoutUtils(r.storeController, r.log) +func FilterByDigest(digest string) repodb.FilterFunc { + return func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { + lookupDigest := digest + contains := false - var errResult error + var manifest ispec.Manifest - for _, repo := range repoList { - r.log.Info().Str("repo", repo).Msg("filtering list of tags in image repo by digest") - - imgTags, err := r.digestInfo.GetImageTagsByDigest(repo, digest) + err := json.Unmarshal(manifestMeta.ManifestBlob, &manifest) if err != nil { - r.log.Error().Err(err).Msg("unable to get filtered list of image tags") - - return []*gql_generated.ImageSummary{}, err + return false } - for _, imageInfo := range imgTags { - imageConfig, err := olu.GetImageConfigInfo(repo, imageInfo.Digest) - if err != nil { - return []*gql_generated.ImageSummary{}, err + manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String() + + // Check the image manifest in index.json matches the search digest + // This is a blob with mediaType application/vnd.oci.image.manifest.v1+json + if strings.Contains(manifestDigest, lookupDigest) { + contains = true + } + + // Check the image config matches the search digest + // This is a blob with mediaType application/vnd.oci.image.config.v1+json + if strings.Contains(manifest.Config.Digest.String(), lookupDigest) { + contains = true + } + + // Check to see if the individual layers in the oci image manifest match the digest + // These are blobs with mediaType application/vnd.oci.image.layer.v1.tar+gzip + for _, layer := range manifest.Layers { + if strings.Contains(layer.Digest.String(), lookupDigest) { + contains = true } - - isSigned := olu.CheckManifestSignature(repo, imageInfo.Digest) - imageInfo := convert.BuildImageInfo(repo, imageInfo.Tag, imageInfo.Digest, - imageInfo.Manifest, imageConfig, isSigned) - - imgResultForDigest = append(imgResultForDigest, imageInfo) } + + return contains + } +} + +func getImageListForDigest(ctx context.Context, digest string, repoDB repodb.RepoDB, cveInfo cveinfo.CveInfo, + requestedPage *gql_generated.PageInput, +) ([]*gql_generated.ImageSummary, error) { + imageList := make([]*gql_generated.ImageSummary, 0) + + if requestedPage == nil { + requestedPage = &gql_generated.PageInput{} } - return imgResultForDigest, errResult + skip := convert.SkipQGLField{ + Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Images.Vulnerabilities"), + } + + pageInput := repodb.PageInput{ + Limit: safeDerefferencing(requestedPage.Limit, 0), + Offset: safeDerefferencing(requestedPage.Offset, 0), + SortBy: repodb.SortCriteria( + safeDerefferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance), + ), + } + + // get all repos + reposMeta, manifestMetaMap, err := repoDB.FilterTags(ctx, FilterByDigest(digest), pageInput) + if err != nil { + return []*gql_generated.ImageSummary{}, err + } + + for _, repoMeta := range reposMeta { + imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, skip, cveInfo) + + imageList = append(imageList, imageSummaries...) + } + + return imageList, nil } func getImageSummary(ctx context.Context, repo, tag string, repoDB repodb.RepoDB, diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index 3d6fee03..1db6fe85 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -107,11 +107,11 @@ func TestGlobalSearch(t *testing.T) { const query = "repo1" limit := 1 - ofset := 0 + offset := 0 sortCriteria := gql_generated.SortCriteriaAlphabeticAsc pageInput := gql_generated.PageInput{ Limit: &limit, - Offset: &ofset, + Offset: &offset, SortBy: &sortCriteria, } @@ -160,11 +160,11 @@ func TestGlobalSearch(t *testing.T) { query := "repo1" limit := 1 - ofset := 0 + offset := 0 sortCriteria := gql_generated.SortCriteriaAlphabeticAsc pageInput := gql_generated.PageInput{ Limit: &limit, - Offset: &ofset, + Offset: &offset, SortBy: &sortCriteria, } @@ -224,11 +224,11 @@ func TestGlobalSearch(t *testing.T) { query := "repo1" limit := 1 - ofset := 0 + offset := 0 sortCriteria := gql_generated.SortCriteriaAlphabeticAsc pageInput := gql_generated.PageInput{ Limit: &limit, - Offset: &ofset, + Offset: &offset, SortBy: &sortCriteria, } @@ -329,11 +329,11 @@ func TestGlobalSearch(t *testing.T) { const query = "repo1:1.0.1" limit := 1 - ofset := 0 + offset := 0 sortCriteria := gql_generated.SortCriteriaAlphabeticAsc pageInput := gql_generated.PageInput{ Limit: &limit, - Offset: &ofset, + Offset: &offset, SortBy: &sortCriteria, } @@ -365,11 +365,11 @@ func TestRepoListWithNewestImage(t *testing.T) { mockCve := mocks.CveInfoMock{} limit := 1 - ofset := 0 + offset := 0 sortCriteria := gql_generated.SortCriteriaUpdateTime pageInput := gql_generated.PageInput{ Limit: &limit, - Offset: &ofset, + Offset: &offset, SortBy: &sortCriteria, } repos, err := repoListWithNewestImage(responseContext, mockCve, log.NewLogger("debug", ""), &pageInput, mockRepoDB) @@ -431,11 +431,11 @@ func TestRepoListWithNewestImage(t *testing.T) { mockCve := mocks.CveInfoMock{} limit := 1 - ofset := 0 + offset := 0 sortCriteria := gql_generated.SortCriteriaUpdateTime pageInput := gql_generated.PageInput{ Limit: &limit, - Offset: &ofset, + Offset: &offset, SortBy: &sortCriteria, } repos, err := repoListWithNewestImage(responseContext, mockCve, log.NewLogger("debug", ""), &pageInput, mockRepoDB) @@ -529,11 +529,11 @@ func TestRepoListWithNewestImage(t *testing.T) { Convey("RepoDB SearchRepo is successful", func() { limit := 2 - ofset := 0 + offset := 0 sortCriteria := gql_generated.SortCriteriaUpdateTime pageInput := gql_generated.PageInput{ Limit: &limit, - Offset: &ofset, + Offset: &offset, SortBy: &sortCriteria, } @@ -553,6 +553,439 @@ func TestRepoListWithNewestImage(t *testing.T) { }) } +func TestImageListForDigest(t *testing.T) { + Convey("getImageList", t, func() { + Convey("no page requested, FilterTagsFn returns error", func() { + mockSearchDB := mocks.RepoDBMock{ + FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc, + requestedPage repodb.PageInput, + ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, ErrTestError + }, + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + _, err := getImageListForDigest(responseContext, "invalid", mockSearchDB, mocks.CveInfoMock{}, nil) + So(err, ShouldNotBeNil) + }) + + Convey("invalid manifest blob", func() { + mockSearchDB := mocks.RepoDBMock{ + FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc, + requestedPage repodb.PageInput, + ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { + repos := []repodb.RepoMetadata{ + { + Name: "test", + Tags: map[string]repodb.Descriptor{ + "1.0.1": {Digest: "digestTag1.0.1", MediaType: ispec.MediaTypeImageManifest}, + }, + Stars: 100, + }, + } + + configBlob, err := json.Marshal(ispec.Image{ + Config: ispec.ImageConfig{ + Labels: map[string]string{}, + }, + }) + So(err, ShouldBeNil) + + manifestBlob := []byte("invalid") + + manifestMetaDatas := map[string]repodb.ManifestMetadata{ + "digestTag1.0.1": { + ManifestBlob: manifestBlob, + ConfigBlob: configBlob, + DownloadCount: 0, + }, + } + + return repos, manifestMetaDatas, nil + }, + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + imageList, err := getImageListForDigest(responseContext, "test", mockSearchDB, mocks.CveInfoMock{}, nil) + So(err, ShouldBeNil) + So(imageList, ShouldBeEmpty) + }) + + Convey("valid imageListForDigest returned for matching manifest digest", func() { + manifestBlob, err := json.Marshal(ispec.Manifest{}) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(manifestBlob).String() + + mockSearchDB := mocks.RepoDBMock{ + FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc, + requestedPage repodb.PageInput, + ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { + repos := []repodb.RepoMetadata{ + { + Name: "test", + Tags: map[string]repodb.Descriptor{ + "1.0.1": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest}, + }, + Stars: 100, + }, + } + + configBlob, err := json.Marshal(ispec.ImageConfig{}) + So(err, ShouldBeNil) + + manifestMetaDatas := map[string]repodb.ManifestMetadata{ + manifestDigest: { + ManifestBlob: manifestBlob, + ConfigBlob: configBlob, + DownloadCount: 0, + }, + } + + matchedTags := repos[0].Tags + for tag, descriptor := range repos[0].Tags { + if !filter(repos[0], manifestMetaDatas[descriptor.Digest]) { + delete(matchedTags, tag) + delete(manifestMetaDatas, descriptor.Digest) + + continue + } + } + + repos[0].Tags = matchedTags + + return repos, manifestMetaDatas, nil + }, + } + + limit := 1 + offset := 0 + sortCriteria := gql_generated.SortCriteriaAlphabeticAsc + pageInput := gql_generated.PageInput{ + Limit: &limit, + Offset: &offset, + SortBy: &sortCriteria, + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + imageSummaries, err := getImageListForDigest(responseContext, manifestDigest, + mockSearchDB, mocks.CveInfoMock{}, &pageInput) + So(err, ShouldBeNil) + So(len(imageSummaries), ShouldEqual, 1) + + imageSummaries, err = getImageListForDigest(responseContext, "invalid", + mockSearchDB, mocks.CveInfoMock{}, &pageInput) + So(err, ShouldBeNil) + So(len(imageSummaries), ShouldEqual, 0) + }) + + Convey("valid imageListForDigest returned for matching config digest", func() { + manifestBlob, err := json.Marshal(ispec.Manifest{}) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(manifestBlob).String() + + configBlob, err := json.Marshal(ispec.Image{}) + So(err, ShouldBeNil) + + configDigest := godigest.FromBytes(configBlob) + + mockSearchDB := mocks.RepoDBMock{ + FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc, + requestedPage repodb.PageInput, + ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { + repos := []repodb.RepoMetadata{ + { + Name: "test", + Tags: map[string]repodb.Descriptor{ + "1.0.1": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest}, + }, + Stars: 100, + }, + } + + manifestBlob, err := json.Marshal(ispec.Manifest{ + Config: ispec.Descriptor{ + Digest: configDigest, + }, + }) + So(err, ShouldBeNil) + + manifestMetaDatas := map[string]repodb.ManifestMetadata{ + manifestDigest: { + ManifestBlob: manifestBlob, + ConfigBlob: configBlob, + DownloadCount: 0, + }, + } + + matchedTags := repos[0].Tags + for tag, descriptor := range repos[0].Tags { + if !filter(repos[0], manifestMetaDatas[descriptor.Digest]) { + delete(matchedTags, tag) + delete(manifestMetaDatas, descriptor.Digest) + + continue + } + } + + repos[0].Tags = matchedTags + + return repos, manifestMetaDatas, nil + }, + } + + limit := 1 + offset := 0 + sortCriteria := gql_generated.SortCriteriaAlphabeticAsc + pageInput := gql_generated.PageInput{ + Limit: &limit, + Offset: &offset, + SortBy: &sortCriteria, + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + imageSummaries, err := getImageListForDigest(responseContext, configDigest.String(), + mockSearchDB, mocks.CveInfoMock{}, &pageInput) + So(err, ShouldBeNil) + So(len(imageSummaries), ShouldEqual, 1) + }) + + Convey("valid imageListForDigest returned for matching layer digest", func() { + manifestBlob, err := json.Marshal(ispec.Manifest{}) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(manifestBlob).String() + + configBlob, err := json.Marshal(ispec.Image{}) + So(err, ShouldBeNil) + + layerDigest := godigest.Digest("validDigest") + + mockSearchDB := mocks.RepoDBMock{ + FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc, + requestedPage repodb.PageInput, + ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { + repos := []repodb.RepoMetadata{ + { + Name: "test", + Tags: map[string]repodb.Descriptor{ + "1.0.1": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest}, + }, + Stars: 100, + }, + } + + manifestBlob, err := json.Marshal(ispec.Manifest{ + Layers: []ispec.Descriptor{ + { + Digest: layerDigest, + }, + }, + }) + So(err, ShouldBeNil) + + manifestMetaDatas := map[string]repodb.ManifestMetadata{ + manifestDigest: { + ManifestBlob: manifestBlob, + ConfigBlob: configBlob, + DownloadCount: 0, + }, + } + + matchedTags := repos[0].Tags + for tag, descriptor := range repos[0].Tags { + if !filter(repos[0], manifestMetaDatas[descriptor.Digest]) { + delete(matchedTags, tag) + delete(manifestMetaDatas, descriptor.Digest) + + continue + } + } + + repos[0].Tags = matchedTags + + return repos, manifestMetaDatas, nil + }, + } + + limit := 1 + offset := 0 + sortCriteria := gql_generated.SortCriteriaAlphabeticAsc + pageInput := gql_generated.PageInput{ + Limit: &limit, + Offset: &offset, + SortBy: &sortCriteria, + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + imageSummaries, err := getImageListForDigest(responseContext, layerDigest.String(), + mockSearchDB, mocks.CveInfoMock{}, &pageInput) + So(err, ShouldBeNil) + So(len(imageSummaries), ShouldEqual, 1) + }) + + Convey("valid imageListForDigest, multiple matching tags", func() { + manifestBlob, err := json.Marshal(ispec.Manifest{}) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(manifestBlob).String() + + configBlob, err := json.Marshal(ispec.Image{}) + So(err, ShouldBeNil) + + mockSearchDB := mocks.RepoDBMock{ + FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc, + requestedPage repodb.PageInput, + ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { + repos := []repodb.RepoMetadata{ + { + Name: "test", + Tags: map[string]repodb.Descriptor{ + "1.0.1": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest}, + "1.0.2": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest}, + }, + Stars: 100, + }, + } + + manifestMetaDatas := map[string]repodb.ManifestMetadata{ + manifestDigest: { + ManifestBlob: manifestBlob, + ConfigBlob: configBlob, + DownloadCount: 0, + }, + } + + for i, repo := range repos { + matchedTags := repo.Tags + + for tag, descriptor := range repo.Tags { + if !filter(repo, manifestMetaDatas[descriptor.Digest]) { + delete(matchedTags, tag) + delete(manifestMetaDatas, descriptor.Digest) + + continue + } + } + + repos[i].Tags = matchedTags + } + + return repos, manifestMetaDatas, nil + }, + } + + limit := 1 + offset := 0 + sortCriteria := gql_generated.SortCriteriaAlphabeticAsc + pageInput := gql_generated.PageInput{ + Limit: &limit, + Offset: &offset, + SortBy: &sortCriteria, + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + imageSummaries, err := getImageListForDigest(responseContext, manifestDigest, + mockSearchDB, mocks.CveInfoMock{}, &pageInput) + So(err, ShouldBeNil) + So(len(imageSummaries), ShouldEqual, 2) + }) + + Convey("valid imageListForDigest, multiple matching tags limited by pageInput", func() { + manifestBlob, err := json.Marshal(ispec.Manifest{}) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(manifestBlob).String() + + configBlob, err := json.Marshal(ispec.Image{}) + So(err, ShouldBeNil) + + mockSearchDB := mocks.RepoDBMock{ + FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc, + requestedPage repodb.PageInput, + ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { + pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, err + } + + repos := []repodb.RepoMetadata{ + { + Name: "test", + Tags: map[string]repodb.Descriptor{ + "1.0.1": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest}, + "1.0.2": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest}, + }, + Stars: 100, + }, + } + + manifestMetaDatas := map[string]repodb.ManifestMetadata{ + manifestDigest: { + ManifestBlob: manifestBlob, + ConfigBlob: configBlob, + DownloadCount: 0, + }, + } + + for i, repo := range repos { + matchedTags := repo.Tags + + for tag, descriptor := range repo.Tags { + if !filter(repo, manifestMetaDatas[descriptor.Digest]) { + delete(matchedTags, tag) + delete(manifestMetaDatas, descriptor.Digest) + + continue + } + } + + repos[i].Tags = matchedTags + + pageFinder.Add(repodb.DetailedRepoMeta{ + RepoMeta: repo, + }) + } + + repos = pageFinder.Page() + + return repos, manifestMetaDatas, nil + }, + } + + limit := 1 + offset := 0 + sortCriteria := gql_generated.SortCriteriaAlphabeticAsc + pageInput := gql_generated.PageInput{ + Limit: &limit, + Offset: &offset, + SortBy: &sortCriteria, + } + + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + imageSummaries, err := getImageListForDigest(responseContext, manifestDigest, + mockSearchDB, mocks.CveInfoMock{}, &pageInput) + So(err, ShouldBeNil) + So(len(imageSummaries), ShouldEqual, 1) + }) + }) +} + func TestGetReferrers(t *testing.T) { Convey("getReferrers", t, func() { Convey("GetReferrers returns error", func() { @@ -907,120 +1340,6 @@ func TestQueryResolverErrors(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("ImageListForDigest defaultStore.GetRepositories() errors", func() { - resolverConfig := NewResolver( - log, - storage.StoreController{ - DefaultStore: mocks.MockedImageStore{ - GetRepositoriesFn: func() ([]string, error) { - return nil, ErrTestError - }, - }, - }, - mocks.RepoDBMock{}, - mocks.CveInfoMock{}, - ) - - qr := queryResolver{ - resolverConfig, - } - - _, err := qr.ImageListForDigest(ctx, "") - So(err, ShouldNotBeNil) - }) - - Convey("ImageListForDigest getImageListForDigest() errors", func() { - resolverConfig := NewResolver( - log, - storage.StoreController{ - DefaultStore: mocks.MockedImageStore{ - GetRepositoriesFn: func() ([]string, error) { - return []string{"repo"}, nil - }, - GetIndexContentFn: func(repo string) ([]byte, error) { - return nil, ErrTestError - }, - }, - }, - mocks.RepoDBMock{}, - mocks.CveInfoMock{}, - ) - - qr := queryResolver{ - resolverConfig, - } - - _, err := qr.ImageListForDigest(ctx, "") - So(err, ShouldNotBeNil) - }) - - Convey("ImageListForDigest substores store.GetRepositories() errors", func() { - resolverConfig := NewResolver( - log, - storage.StoreController{ - DefaultStore: mocks.MockedImageStore{ - GetIndexContentFn: func(repo string) ([]byte, error) { - return []byte("{}"), nil - }, - GetRepositoriesFn: func() ([]string, error) { - return []string{"repo"}, nil - }, - }, - SubStore: map[string]storage.ImageStore{ - "sub1": mocks.MockedImageStore{ - GetRepositoriesFn: func() ([]string, error) { - return []string{"repo"}, ErrTestError - }, - }, - }, - }, - mocks.RepoDBMock{}, - mocks.CveInfoMock{}, - ) - - qr := queryResolver{ - resolverConfig, - } - - _, err := qr.ImageListForDigest(ctx, "") - So(err, ShouldNotBeNil) - }) - - Convey("ImageListForDigest substores getImageListForDigest() errors", func() { - resolverConfig := NewResolver( - log, - storage.StoreController{ - DefaultStore: mocks.MockedImageStore{ - GetIndexContentFn: func(repo string) ([]byte, error) { - return []byte("{}"), nil - }, - GetRepositoriesFn: func() ([]string, error) { - return []string{"repo"}, nil - }, - }, - SubStore: map[string]storage.ImageStore{ - "/sub1": mocks.MockedImageStore{ - GetRepositoriesFn: func() ([]string, error) { - return []string{"sub1/repo"}, nil - }, - GetIndexContentFn: func(repo string) ([]byte, error) { - return nil, ErrTestError - }, - }, - }, - }, - mocks.RepoDBMock{}, - mocks.CveInfoMock{}, - ) - - qr := queryResolver{ - resolverConfig, - } - - _, err := qr.ImageListForDigest(ctx, "") - So(err, ShouldNotBeNil) - }) - Convey("RepoListWithNewestImage repoListWithNewestImage() errors", func() { resolverConfig := NewResolver( log, diff --git a/pkg/extensions/search/schema.graphql b/pkg/extensions/search/schema.graphql index eac9bf59..77aa0e6a 100644 --- a/pkg/extensions/search/schema.graphql +++ b/pkg/extensions/search/schema.graphql @@ -205,7 +205,7 @@ type Query { """ Returns a list of images which contain the specified digest """ - ImageListForDigest(id: String!): [ImageSummary!] + ImageListForDigest(id: String!, requestedPage: PageInput): [ImageSummary!] """ Returns a list of repos with the newest tag within diff --git a/pkg/extensions/search/schema.resolvers.go b/pkg/extensions/search/schema.resolvers.go index 26f911b1..5310a2a0 100644 --- a/pkg/extensions/search/schema.resolvers.go +++ b/pkg/extensions/search/schema.resolvers.go @@ -163,51 +163,12 @@ func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, im } // ImageListForDigest is the resolver for the ImageListForDigest field. -func (r *queryResolver) ImageListForDigest(ctx context.Context, id string) ([]*gql_generated.ImageSummary, error) { - imgResultForDigest := []*gql_generated.ImageSummary{} - +func (r *queryResolver) ImageListForDigest(ctx context.Context, id string, requestedPage *gql_generated.PageInput) ([]*gql_generated.ImageSummary, error) { r.log.Info().Msg("extracting repositories") - defaultStore := r.storeController.DefaultStore + imgResultForDigest, err := getImageListForDigest(ctx, id, r.repoDB, r.cveInfo, requestedPage) - repoList, err := defaultStore.GetRepositories() - if err != nil { - r.log.Error().Err(err).Msg("unable to search repositories") - - return imgResultForDigest, err - } - - r.log.Info().Msg("scanning each global repository") - - partialImgResultForDigest, err := r.getImageListForDigest(repoList, id) - if err != nil { - r.log.Error().Err(err).Msg("unable to get image and tag list for global repositories") - - return imgResultForDigest, err - } - - imgResultForDigest = append(imgResultForDigest, partialImgResultForDigest...) - - subStore := r.storeController.SubStore - for _, store := range subStore { - subRepoList, err := store.GetRepositories() - if err != nil { - r.log.Error().Err(err).Msg("unable to search sub-repositories") - - return imgResultForDigest, err - } - - partialImgResultForDigest, err = r.getImageListForDigest(subRepoList, id) - if err != nil { - r.log.Error().Err(err).Msg("unable to get image and tag list for sub-repositories") - - return imgResultForDigest, err - } - - imgResultForDigest = append(imgResultForDigest, partialImgResultForDigest...) - } - - return imgResultForDigest, nil + return imgResultForDigest, err } // RepoListWithNewestImage is the resolver for the RepoListWithNewestImage field. diff --git a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go index 00997119..02a1824f 100644 --- a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go +++ b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go @@ -743,6 +743,113 @@ func (bdw DBWrapper) SearchRepos(ctx context.Context, searchText string, filter return foundRepos, foundManifestMetadataMap, err } +func (bdw DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, + requestedPage repodb.PageInput, +) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { + var ( + foundRepos = make([]repodb.RepoMetadata, 0) + foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata) + pageFinder repodb.PageFinder + ) + + pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, err + } + + err = bdw.DB.View(func(tx *bolt.Tx) error { + var ( + manifestMetadataMap = make(map[string]repodb.ManifestMetadata) + repoBuck = tx.Bucket([]byte(repodb.RepoMetadataBucket)) + dataBuck = tx.Bucket([]byte(repodb.ManifestDataBucket)) + cursor = repoBuck.Cursor() + ) + + repoName, repoMetaBlob := cursor.First() + + for ; repoName != nil; repoName, repoMetaBlob = cursor.Next() { + if ok, err := localCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { + continue + } + + repoMeta := repodb.RepoMetadata{} + + err := json.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + matchedTags := make(map[string]repodb.Descriptor) + // take all manifestMetas + for tag, descriptor := range repoMeta.Tags { + manifestDigest := descriptor.Digest + + matchedTags[tag] = descriptor + + // in case tags reference the same manifest we don't download from DB multiple times + manifestMeta, manifestExists := manifestMetadataMap[manifestDigest] + + if !manifestExists { + manifestDataBlob := dataBuck.Get([]byte(manifestDigest)) + if manifestDataBlob == nil { + return zerr.ErrManifestMetaNotFound + } + + var manifestData repodb.ManifestData + + err := json.Unmarshal(manifestDataBlob, &manifestData) + if err != nil { + return errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", manifestDigest) + } + + var configContent ispec.Image + + err = json.Unmarshal(manifestData.ConfigBlob, &configContent) + if err != nil { + return errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", manifestDigest) + } + + manifestMeta = repodb.ManifestMetadata{ + ConfigBlob: manifestData.ConfigBlob, + ManifestBlob: manifestData.ManifestBlob, + } + } + + if !filter(repoMeta, manifestMeta) { + delete(matchedTags, tag) + + continue + } + + manifestMetadataMap[manifestDigest] = manifestMeta + } + + if len(matchedTags) == 0 { + continue + } + + repoMeta.Tags = matchedTags + + pageFinder.Add(repodb.DetailedRepoMeta{ + RepoMeta: repoMeta, + }) + } + + foundRepos = pageFinder.Page() + + // keep just the manifestMeta we need + for _, repoMeta := range foundRepos { + for _, descriptor := range repoMeta.Tags { + foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest] + } + } + + return nil + }) + + return foundRepos, foundManifestMetadataMap, err +} + func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { @@ -819,7 +926,7 @@ func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter r err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent) if err != nil { - return errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", descriptor.Digest) + return errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", descriptor.Digest) } imageFilterData := repodb.FilterData{ @@ -838,6 +945,10 @@ func (bdw DBWrapper) SearchTags(ctx context.Context, searchText string, filter r manifestMetadataMap[descriptor.Digest] = manifestMeta } + if len(matchedTags) == 0 { + continue + } + repoMeta.Tags = matchedTags pageFinder.Add(repodb.DetailedRepoMeta{ diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go index 451f441a..6a679ccd 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go @@ -393,6 +393,75 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) + + Convey("FilterTags repoMeta unmarshal error", func() { + err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck + So(err, ShouldBeNil) + + _, _, err = dynamoWrapper.FilterTags( + ctx, + func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { + return true + }, + repodb.PageInput{}, + ) + + So(err, ShouldNotBeNil) + }) + + Convey("FilterTags manifestMeta not found", func() { + err := dynamoWrapper.SetRepoTag("repo", "tag1", "manifestNotFound", "") //nolint:contextcheck + So(err, ShouldBeNil) + + _, _, err = dynamoWrapper.FilterTags( + ctx, + func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { + return true + }, + repodb.PageInput{}, + ) + + So(err, ShouldNotBeNil) + }) + + Convey("FilterTags manifestMeta unmarshal error", func() { + err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig", "") //nolint:contextcheck + So(err, ShouldBeNil) + + err = setBadManifestData(dynamoWrapper.Client, manifestDataTablename, "dig") //nolint:contextcheck + So(err, ShouldBeNil) + + _, _, err = dynamoWrapper.FilterTags( + ctx, + func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { + return true + }, + repodb.PageInput{}, + ) + + So(err, ShouldNotBeNil) + }) + + Convey("FilterTags config unmarshal error", func() { + err := dynamoWrapper.SetRepoTag("repo", "tag1", "dig1", "") //nolint:contextcheck + So(err, ShouldBeNil) + + err = dynamoWrapper.SetManifestData("dig1", repodb.ManifestData{ //nolint:contextcheck + ManifestBlob: []byte("{}"), + ConfigBlob: []byte("bad json"), + }) + So(err, ShouldBeNil) + + _, _, err = dynamoWrapper.FilterTags( + ctx, + func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { + return true + }, + repodb.PageInput{}, + ) + + So(err, ShouldNotBeNil) + }) }) } diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go index 2f8fa0a3..a65ba567 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go @@ -650,6 +650,101 @@ func (dwr DBWrapper) SearchRepos(ctx context.Context, searchText string, filter return foundRepos, foundManifestMetadataMap, err } +func (dwr DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, + requestedPage repodb.PageInput, +) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { + var ( + foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata) + manifestMetadataMap = make(map[string]repodb.ManifestMetadata) + pageFinder repodb.PageFinder + repoMetaAttributeIterator iterator.AttributesIterator + ) + + repoMetaAttributeIterator = iterator.NewBaseDynamoAttributesIterator( + dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, + ) + + pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, err + } + + repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) + + for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { + if err != nil { + // log + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, err + } + + var repoMeta repodb.RepoMetadata + + err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, err + } + + if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil { + continue + } + matchedTags := make(map[string]repodb.Descriptor) + // take all manifestMetas + for tag, descriptor := range repoMeta.Tags { + manifestDigest := descriptor.Digest + + matchedTags[tag] = descriptor + + // in case tags reference the same manifest we don't download from DB multiple times + manifestMeta, manifestExists := manifestMetadataMap[manifestDigest] + + if !manifestExists { + manifestMeta, err := dwr.GetManifestMeta(repoMeta.Name, godigest.Digest(manifestDigest)) //nolint:contextcheck + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, + errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", manifestDigest) + } + + var configContent ispec.Image + + err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, + errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", manifestDigest) + } + } + + if !filter(repoMeta, manifestMeta) { + delete(matchedTags, tag) + + continue + } + + manifestMetadataMap[manifestDigest] = manifestMeta + } + + if len(matchedTags) == 0 { + continue + } + + repoMeta.Tags = matchedTags + + pageFinder.Add(repodb.DetailedRepoMeta{ + RepoMeta: repoMeta, + }) + } + + foundRepos := pageFinder.Page() + + // keep just the manifestMeta we need + for _, repoMeta := range foundRepos { + for _, descriptor := range repoMeta.Tags { + foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest] + } + } + + return foundRepos, foundManifestMetadataMap, err +} + func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { @@ -721,7 +816,7 @@ func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter r err = json.Unmarshal(manifestMeta.ConfigBlob, &configContent) if err != nil { return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, - errors.Wrapf(err, "repodb: error while unmashaling manifest metadata for digest %s", descriptor.Digest) + errors.Wrapf(err, "repodb: error while unmashaling config for manifest with digest %s", descriptor.Digest) } imageFilterData := repodb.FilterData{ @@ -740,6 +835,10 @@ func (dwr DBWrapper) SearchTags(ctx context.Context, searchText string, filter r manifestMetadataMap[descriptor.Digest] = manifestMeta } + if len(matchedTags) == 0 { + continue + } + repoMeta.Tags = matchedTags pageFinder.Add(repodb.DetailedRepoMeta{ diff --git a/pkg/meta/repodb/pagination.go b/pkg/meta/repodb/pagination.go index 90a1da2b..5f756c54 100644 --- a/pkg/meta/repodb/pagination.go +++ b/pkg/meta/repodb/pagination.go @@ -149,6 +149,17 @@ func (bpt *ImagePageFinder) Page() []RepoMetadata { remainingOffset := bpt.offset remainingLimit := bpt.limit + repos := make([]RepoMetadata, 0) + + if remainingOffset == 0 && remainingLimit == 0 { + for _, drm := range bpt.pageBuffer { + repo := drm.RepoMeta + repos = append(repos, repo) + } + + return repos + } + // bring cursor to position in RepoMeta array for _, drm := range bpt.pageBuffer { if remainingOffset < len(drm.RepoMeta.Tags) { @@ -166,8 +177,6 @@ func (bpt *ImagePageFinder) Page() []RepoMetadata { return []RepoMetadata{} } - repos := make([]RepoMetadata, 0) - // finish counting remaining tags inside the first repo meta partialTags := map[string]Descriptor{} firstRepoMeta := bpt.pageBuffer[repoStartIndex].RepoMeta diff --git a/pkg/meta/repodb/repodb.go b/pkg/meta/repodb/repodb.go index 9f4341cb..91cdea97 100644 --- a/pkg/meta/repodb/repodb.go +++ b/pkg/meta/repodb/repodb.go @@ -21,6 +21,8 @@ const ( CosignType = "cosign" ) +type FilterFunc func(repoMeta RepoMetadata, manifestMeta ManifestMetadata) bool + type RepoDB interface { //nolint:interfacebloat // IncrementRepoStars adds 1 to the star count of an image IncrementRepoStars(repo string) error @@ -74,6 +76,10 @@ type RepoDB interface { //nolint:interfacebloat SearchTags(ctx context.Context, searchText string, filter Filter, requestedPage PageInput) ( []RepoMetadata, map[string]ManifestMetadata, error) + // FilterTags filters for images given a filter function + FilterTags(ctx context.Context, filter FilterFunc, + requestedPage PageInput) ([]RepoMetadata, map[string]ManifestMetadata, error) + PatchDB() error } diff --git a/pkg/meta/repodb/repodb_test.go b/pkg/meta/repodb/repodb_test.go index 0c3e337d..13e77f13 100644 --- a/pkg/meta/repodb/repodb_test.go +++ b/pkg/meta/repodb/repodb_test.go @@ -1286,6 +1286,155 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { So(err, ShouldBeNil) So(len(repos), ShouldEqual, 0) }) + + Convey("Test FilterTags", func() { + var ( + repo1 = "repo1" + repo2 = "repo2" + manifestDigest1 = digest.FromString("fake-manifest1") + manifestDigest2 = digest.FromString("fake-manifest2") + manifestDigest3 = digest.FromString("fake-manifest3") + ctx = context.Background() + emptyManifest ispec.Manifest + emptyConfig ispec.Image + ) + + emptyManifestBlob, err := json.Marshal(emptyManifest) + So(err, ShouldBeNil) + + emptyConfigBlob, err := json.Marshal(emptyConfig) + So(err, ShouldBeNil) + + emptyRepoMeta := repodb.ManifestMetadata{ + ManifestBlob: emptyManifestBlob, + ConfigBlob: emptyConfigBlob, + } + + err = repoDB.SetRepoTag(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + err = repoDB.SetRepoTag(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + err = repoDB.SetRepoTag(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + err = repoDB.SetRepoTag(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + err = repoDB.SetRepoTag(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + err = repoDB.SetRepoTag(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + err = repoDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) + So(err, ShouldBeNil) + err = repoDB.SetManifestMeta(repo1, manifestDigest2, emptyRepoMeta) + So(err, ShouldBeNil) + err = repoDB.SetManifestMeta(repo1, manifestDigest3, emptyRepoMeta) + So(err, ShouldBeNil) + err = repoDB.SetManifestMeta(repo2, manifestDigest3, emptyRepoMeta) + So(err, ShouldBeNil) + + Convey("Return all tags", func() { + repos, manifesMetaMap, err := repoDB.FilterTags( + ctx, + func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { + return true + }, + repodb.PageInput{Limit: 10, Offset: 0, SortBy: repodb.AlphabeticAsc}, + ) + + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 2) + So(repos[0].Name, ShouldEqual, "repo1") + So(repos[1].Name, ShouldEqual, "repo2") + So(len(repos[0].Tags), ShouldEqual, 5) + So(len(repos[1].Tags), ShouldEqual, 1) + So(repos[0].Tags, ShouldContainKey, "0.0.1") + So(repos[0].Tags, ShouldContainKey, "0.0.2") + So(repos[0].Tags, ShouldContainKey, "0.1.0") + So(repos[0].Tags, ShouldContainKey, "1.0.0") + So(repos[0].Tags, ShouldContainKey, "1.0.1") + So(repos[1].Tags, ShouldContainKey, "0.0.1") + So(manifesMetaMap, ShouldContainKey, manifestDigest1.String()) + So(manifesMetaMap, ShouldContainKey, manifestDigest2.String()) + So(manifesMetaMap, ShouldContainKey, manifestDigest3.String()) + }) + + Convey("Return all tags in a specific repo", func() { + repos, manifesMetaMap, err := repoDB.FilterTags( + ctx, + func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { + return repoMeta.Name == repo1 + }, + repodb.PageInput{Limit: 10, Offset: 0, SortBy: repodb.AlphabeticAsc}, + ) + + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos[0].Name, ShouldEqual, repo1) + So(len(repos[0].Tags), ShouldEqual, 5) + So(repos[0].Tags, ShouldContainKey, "0.0.1") + So(repos[0].Tags, ShouldContainKey, "0.0.2") + So(repos[0].Tags, ShouldContainKey, "0.1.0") + So(repos[0].Tags, ShouldContainKey, "1.0.0") + So(repos[0].Tags, ShouldContainKey, "1.0.1") + So(manifesMetaMap, ShouldContainKey, manifestDigest1.String()) + So(manifesMetaMap, ShouldContainKey, manifestDigest2.String()) + So(manifesMetaMap, ShouldContainKey, manifestDigest3.String()) + }) + + Convey("Filter everything out", func() { + repos, manifesMetaMap, err := repoDB.FilterTags( + ctx, + func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { + return false + }, + repodb.PageInput{Limit: 10, Offset: 0, SortBy: repodb.AlphabeticAsc}, + ) + + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + So(len(manifesMetaMap), ShouldEqual, 0) + }) + + Convey("Search with access control", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + repo1: false, + repo2: true, + }, + Username: "username", + } + + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + repos, manifesMetaMap, err := repoDB.FilterTags( + ctx, + func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { + return true + }, + repodb.PageInput{Limit: 10, Offset: 0, SortBy: repodb.AlphabeticAsc}, + ) + + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos[0].Name, ShouldResemble, repo2) + So(len(repos[0].Tags), ShouldEqual, 1) + So(repos[0].Tags, ShouldContainKey, "0.0.1") + So(manifesMetaMap, ShouldContainKey, manifestDigest3.String()) + }) + + Convey("With wrong pagination input", func() { + repos, _, err := repoDB.FilterTags( + ctx, + func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { + return true + }, + repodb.PageInput{Limit: -1}, + ) + So(err, ShouldNotBeNil) + So(repos, ShouldBeEmpty) + }) + }) }) } diff --git a/pkg/test/mocks/repo_db_mock.go b/pkg/test/mocks/repo_db_mock.go index de5f2886..544161ce 100644 --- a/pkg/test/mocks/repo_db_mock.go +++ b/pkg/test/mocks/repo_db_mock.go @@ -48,6 +48,10 @@ type RepoDBMock struct { SearchTagsFn func(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput) ( []repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) + FilterTagsFn func(ctx context.Context, filter repodb.FilterFunc, + requestedPage repodb.PageInput, + ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) + SearchDigestsFn func(ctx context.Context, searchText string, requestedPage repodb.PageInput) ( []repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) @@ -217,6 +221,16 @@ func (sdm RepoDBMock) SearchTags(ctx context.Context, searchText string, filter return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, nil } +func (sdm RepoDBMock) FilterTags(ctx context.Context, filter repodb.FilterFunc, + requestedPage repodb.PageInput, +) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { + if sdm.FilterTagsFn != nil { + return sdm.FilterTagsFn(ctx, filter, requestedPage) + } + + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, nil +} + func (sdm RepoDBMock) SearchDigests(ctx context.Context, searchText string, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) { if sdm.SearchDigestsFn != nil {