diff --git a/pkg/extensions/search/common/common_test.go b/pkg/extensions/search/common/common_test.go index a91b2dbe..e7aab5a7 100644 --- a/pkg/extensions/search/common/common_test.go +++ b/pkg/extensions/search/common/common_test.go @@ -1024,10 +1024,7 @@ func TestGetReferrersGQL(t *testing.T) { gqlQuery := ` { Referrers( - repo: "%s", - digest: "%s", - type: "" - ){ + repo: "%s", digest: "%s", type: ""){ ArtifactType, Digest, MediaType, @@ -1756,6 +1753,13 @@ func TestUtilsMethod(t *testing.T) { fixedTags := common.GetFixedTags(allTags, vulnerableTags) So(len(fixedTags), ShouldEqual, 2) + fixedTags = common.GetFixedTags(allTags, append(vulnerableTags, common.TagInfo{ + Name: "taginfo", + Descriptor: common.Descriptor{}, + Timestamp: time.Date(2000, time.July, 20, 10, 10, 10, 10, time.UTC), + })) + So(len(fixedTags), ShouldEqual, 3) + log := log.NewLogger("debug", "") rootDir := t.TempDir() @@ -6395,6 +6399,7 @@ func TestImageSummary(t *testing.T) { conf := config.New() conf.HTTP.Port = port conf.Storage.RootDirectory = t.TempDir() + conf.Storage.GC = false defaultVal := true conf.Extensions = &extconf.ExtensionConfig{ @@ -6426,6 +6431,7 @@ func TestImageSummary(t *testing.T) { LastUpdated Size Vulnerabilities { Count MaxSeverity } + Referrers {MediaType ArtifactType Digest Annotations {Key Value}} } }` @@ -6452,38 +6458,51 @@ func TestImageSummary(t *testing.T) { }` gqlEndpoint := fmt.Sprintf("%s%s?query=", baseURL, graphqlQueryPrefix) - config, layers, manifest, err := GetImageComponents(100) - So(err, ShouldBeNil) - createdTime := time.Date(2010, 1, 1, 12, 0, 0, 0, time.UTC) - config.History = append(config.History, ispec.History{Created: &createdTime}) - manifest, err = updateManifestConfig(manifest, config) - So(err, ShouldBeNil) - configBlob, errConfig := json.Marshal(config) - configDigest := godigest.FromBytes(configBlob) - So(errConfig, ShouldBeNil) // marshall success, config is valid JSON ctlrManager := NewControllerManager(ctlr) ctlrManager.StartAndWait(port) defer ctlrManager.StopServer() - manifestBlob, errMarsal := json.Marshal(manifest) - So(errMarsal, ShouldBeNil) - So(manifestBlob, ShouldNotBeNil) - manifestDigest := godigest.FromBytes(manifestBlob) repoName := "test-repo" //nolint:goconst - tagTarget := "latest" - err = UploadImage( - Image{ - Manifest: manifest, - Config: config, - Layers: layers, - Reference: tagTarget, + + createdTime := time.Date(2010, 1, 1, 12, 0, 0, 0, time.UTC) + + image, err := GetImageWithConfig( + ispec.Image{ + History: []ispec.History{{Created: &createdTime}}, + Platform: ispec.Platform{ + Architecture: "amd64", + OS: "linux", + }, }, - baseURL, - repoName, ) So(err, ShouldBeNil) + image.Reference = tagTarget + + manifestDigest, err := image.Digest() + So(err, ShouldBeNil) + + err = UploadImage(image, baseURL, repoName) + So(err, ShouldBeNil) + + // ------ Add a referrer + referrerImage, err := GetImageWithConfig(ispec.Image{}) + So(err, ShouldBeNil) + + referrerImage.Manifest.Subject = &ispec.Descriptor{ + Digest: manifestDigest, + MediaType: ispec.MediaTypeImageManifest, + } + referrerImage.Manifest.Config.MediaType = "test.artifact.type" + referrerImage.Manifest.Annotations = map[string]string{"testAnnotationKey": "testAnnotationValue"} + referrerManifestDigest, err := referrerImage.Digest() + So(err, ShouldBeNil) + referrerImage.Reference = referrerManifestDigest.String() + + err = UploadImage(referrerImage, baseURL, repoName) + So(err, ShouldBeNil) + var ( imgSummaryResponse ImageSummaryResult strQuery string @@ -6531,11 +6550,11 @@ func TestImageSummary(t *testing.T) { imgSummary := imgSummaryResponse.SingleImageSummary.ImageSummary So(imgSummary.RepoName, ShouldContainSubstring, repoName) So(imgSummary.Tag, ShouldContainSubstring, tagTarget) - So(imgSummary.Manifests[0].ConfigDigest, ShouldContainSubstring, configDigest.Encoded()) + So(imgSummary.Manifests[0].ConfigDigest, ShouldContainSubstring, image.Manifest.Config.Digest.Encoded()) So(imgSummary.Manifests[0].Digest, ShouldContainSubstring, manifestDigest.Encoded()) So(len(imgSummary.Manifests[0].Layers), ShouldEqual, 1) So(imgSummary.Manifests[0].Layers[0].Digest, ShouldContainSubstring, - godigest.FromBytes(layers[0]).Encoded()) + image.Manifest.Layers[0].Digest.Encoded()) So(imgSummary.LastUpdated, ShouldEqual, createdTime) So(imgSummary.IsSigned, ShouldEqual, false) So(imgSummary.Manifests[0].Platform.Os, ShouldEqual, "linux") @@ -6545,6 +6564,13 @@ func TestImageSummary(t *testing.T) { // No vulnerabilities should be detected since trivy is disabled So(imgSummary.Vulnerabilities.Count, ShouldEqual, 0) So(imgSummary.Vulnerabilities.MaxSeverity, ShouldEqual, "") + So(len(imgSummary.Referrers), ShouldEqual, 1) + So(imgSummary.Referrers[0], ShouldResemble, common.Referrer{ + MediaType: ispec.MediaTypeImageManifest, + ArtifactType: "test.artifact.type", + Digest: referrerManifestDigest.String(), + Annotations: []common.Annotation{{Key: "testAnnotationKey", Value: "testAnnotationValue"}}, + }) t.Log("starting Test retrieve duplicated image same layers based on image identifier") // gqlEndpoint diff --git a/pkg/extensions/search/common/model.go b/pkg/extensions/search/common/model.go index 399d99ce..4a9d19e7 100644 --- a/pkg/extensions/search/common/model.go +++ b/pkg/extensions/search/common/model.go @@ -37,6 +37,7 @@ type ImageSummary struct { Authors string `json:"authors"` Vendor string `json:"vendor"` Vulnerabilities ImageVulnerabilitySummary `json:"vulnerabilities"` + Referrers []Referrer `json:"referrers"` } type ManifestSummary struct { diff --git a/pkg/extensions/search/convert/convert_test.go b/pkg/extensions/search/convert/convert_test.go index 55a219d7..94efa72c 100644 --- a/pkg/extensions/search/convert/convert_test.go +++ b/pkg/extensions/search/convert/convert_test.go @@ -172,6 +172,7 @@ func TestConvertErrors(t *testing.T) { ManifestBlob: []byte("{}"), ConfigBlob: []byte("bad json"), }, + nil, mocks.CveInfoMock{ GetCVESummaryForImageFn: func(repo, reference string, ) (cveinfo.ImageCVESummary, error) { @@ -203,6 +204,7 @@ func TestConvertErrors(t *testing.T) { ManifestBlob: []byte("{}"), ConfigBlob: configBlob, }, + nil, mocks.CveInfoMock{ GetCVESummaryForImageFn: func(repo, reference string, ) (cveinfo.ImageCVESummary, error) { diff --git a/pkg/extensions/search/convert/repodb.go b/pkg/extensions/search/convert/repodb.go index dea957fa..26857e56 100644 --- a/pkg/extensions/search/convert/repodb.go +++ b/pkg/extensions/search/convert/repodb.go @@ -193,7 +193,7 @@ func ImageIndex2ImageSummary(ctx context.Context, repo, tag string, indexDigest for _, descriptor := range indexContent.Manifests { manifestSummary, manifestBlobs, err := ImageManifest2ManifestSummary(ctx, repo, tag, descriptor, false, - manifestMetaMap[descriptor.Digest.String()], cveInfo) + manifestMetaMap[descriptor.Digest.String()], repoMeta.Referrers[descriptor.Digest.String()], cveInfo) if err != nil { return &gql_generated.ImageSummary{}, map[string]int64{}, err } @@ -260,6 +260,7 @@ func ImageIndex2ImageSummary(ctx context.Context, repo, tag string, indexDigest MaxSeverity: &imageCveSummary.MaxSeverity, Count: &imageCveSummary.Count, }, + Referrers: getReferrers(repoMeta.Referrers[indexDigest.String()]), } return &indexSummary, indexBlobs, nil @@ -382,13 +383,48 @@ func ImageManifest2ImageSummary(ctx context.Context, repo, tag string, digest go MaxSeverity: &imageCveSummary.MaxSeverity, Count: &imageCveSummary.Count, }, + Referrers: getReferrers(repoMeta.Referrers[manifestDigest]), } return &imageSummary, imageBlobsMap, nil } +func getReferrers(referrersInfo []repodb.ReferrerInfo) []*gql_generated.Referrer { + referrers := make([]*gql_generated.Referrer, 0, len(referrersInfo)) + + for _, referrerInfo := range referrersInfo { + referrerInfo := referrerInfo + + referrers = append(referrers, &gql_generated.Referrer{ + MediaType: &referrerInfo.MediaType, + ArtifactType: &referrerInfo.ArtifactType, + Size: &referrerInfo.Size, + Digest: &referrerInfo.Digest, + Annotations: getAnnotationsFromMap(referrerInfo.Annotations), + }) + } + + return referrers +} + +func getAnnotationsFromMap(annotationsMap map[string]string) []*gql_generated.Annotation { + annotations := make([]*gql_generated.Annotation, 0, len(annotationsMap)) + + for key, value := range annotationsMap { + key := key + value := value + + annotations = append(annotations, &gql_generated.Annotation{ + Key: &key, + Value: &value, + }) + } + + return annotations +} + func ImageManifest2ManifestSummary(ctx context.Context, repo, tag string, descriptor ispec.Descriptor, - skipCVE bool, manifestMeta repodb.ManifestMetadata, cveInfo cveinfo.CveInfo, + skipCVE bool, manifestMeta repodb.ManifestMetadata, referrersInfo []repodb.ReferrerInfo, cveInfo cveinfo.CveInfo, ) (*gql_generated.ManifestSummary, map[string]int64, error) { var ( manifestContent ispec.Manifest @@ -469,6 +505,7 @@ func ImageManifest2ManifestSummary(ctx context.Context, repo, tag string, descri MaxSeverity: &imageCveSummary.MaxSeverity, Count: &imageCveSummary.Count, }, + Referrers: getReferrers(referrersInfo), } return &manifestSummary, imageBlobsMap, nil diff --git a/pkg/extensions/search/gql_generated/generated.go b/pkg/extensions/search/gql_generated/generated.go index 1a7e7456..90f28483 100644 --- a/pkg/extensions/search/gql_generated/generated.go +++ b/pkg/extensions/search/gql_generated/generated.go @@ -86,6 +86,7 @@ type ComplexityRoot struct { LastUpdated func(childComplexity int) int Licenses func(childComplexity int) int Manifests func(childComplexity int) int + Referrers func(childComplexity int) int RepoName func(childComplexity int) int Score func(childComplexity int) int Size func(childComplexity int) int @@ -120,6 +121,7 @@ type ComplexityRoot struct { LastUpdated func(childComplexity int) int Layers func(childComplexity int) int Platform func(childComplexity int) int + Referrers func(childComplexity int) int Size func(childComplexity int) int Vulnerabilities func(childComplexity int) int } @@ -419,6 +421,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.ImageSummary.Manifests(childComplexity), true + case "ImageSummary.Referrers": + if e.complexity.ImageSummary.Referrers == nil { + break + } + + return e.complexity.ImageSummary.Referrers(childComplexity), true + case "ImageSummary.RepoName": if e.complexity.ImageSummary.RepoName == nil { break @@ -573,6 +582,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.ManifestSummary.Platform(childComplexity), true + case "ManifestSummary.Referrers": + if e.complexity.ManifestSummary.Referrers == nil { + break + } + + return e.complexity.ManifestSummary.Referrers(childComplexity), true + case "ManifestSummary.Size": if e.complexity.ManifestSummary.Size == nil { break @@ -1175,6 +1191,10 @@ type ImageSummary { Short summary of the identified CVEs """ Vulnerabilities: ImageVulnerabilitySummary + """ + Information about objects that reference this image + """ + Referrers: [Referrer] } """ Details about a specific version of an image for a certain operating system and architecture. @@ -1217,6 +1237,10 @@ type ManifestSummary { Short summary of the identified CVEs """ Vulnerabilities: ImageVulnerabilitySummary + """ + Information about objects that reference this image + """ + Referrers: [Referrer] } """ @@ -2593,6 +2617,8 @@ func (ec *executionContext) fieldContext_GlobalSearchResult_Images(ctx context.C return ec.fieldContext_ImageSummary_Authors(ctx, field) case "Vulnerabilities": return ec.fieldContext_ImageSummary_Vulnerabilities(ctx, field) + case "Referrers": + return ec.fieldContext_ImageSummary_Referrers(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type ImageSummary", field.Name) }, @@ -3055,6 +3081,8 @@ func (ec *executionContext) fieldContext_ImageSummary_Manifests(ctx context.Cont return ec.fieldContext_ManifestSummary_History(ctx, field) case "Vulnerabilities": return ec.fieldContext_ManifestSummary_Vulnerabilities(ctx, field) + case "Referrers": + return ec.fieldContext_ManifestSummary_Referrers(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type ManifestSummary", field.Name) }, @@ -3642,6 +3670,59 @@ func (ec *executionContext) fieldContext_ImageSummary_Vulnerabilities(ctx contex return fc, nil } +func (ec *executionContext) _ImageSummary_Referrers(ctx context.Context, field graphql.CollectedField, obj *ImageSummary) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ImageSummary_Referrers(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Referrers, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.([]*Referrer) + fc.Result = res + return ec.marshalOReferrer2ᚕᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐReferrer(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ImageSummary_Referrers(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ImageSummary", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "MediaType": + return ec.fieldContext_Referrer_MediaType(ctx, field) + case "ArtifactType": + return ec.fieldContext_Referrer_ArtifactType(ctx, field) + case "Size": + return ec.fieldContext_Referrer_Size(ctx, field) + case "Digest": + return ec.fieldContext_Referrer_Digest(ctx, field) + case "Annotations": + return ec.fieldContext_Referrer_Annotations(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type Referrer", field.Name) + }, + } + return fc, nil +} + func (ec *executionContext) _ImageVulnerabilitySummary_MaxSeverity(ctx context.Context, field graphql.CollectedField, obj *ImageVulnerabilitySummary) (ret graphql.Marshaler) { fc, err := ec.fieldContext_ImageVulnerabilitySummary_MaxSeverity(ctx, field) if err != nil { @@ -4344,6 +4425,59 @@ func (ec *executionContext) fieldContext_ManifestSummary_Vulnerabilities(ctx con return fc, nil } +func (ec *executionContext) _ManifestSummary_Referrers(ctx context.Context, field graphql.CollectedField, obj *ManifestSummary) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_ManifestSummary_Referrers(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return obj.Referrers, nil + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.([]*Referrer) + fc.Result = res + return ec.marshalOReferrer2ᚕᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐReferrer(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_ManifestSummary_Referrers(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "ManifestSummary", + Field: field, + IsMethod: false, + IsResolver: false, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "MediaType": + return ec.fieldContext_Referrer_MediaType(ctx, field) + case "ArtifactType": + return ec.fieldContext_Referrer_ArtifactType(ctx, field) + case "Size": + return ec.fieldContext_Referrer_Size(ctx, field) + case "Digest": + return ec.fieldContext_Referrer_Digest(ctx, field) + case "Annotations": + return ec.fieldContext_Referrer_Annotations(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type Referrer", field.Name) + }, + } + return fc, nil +} + func (ec *executionContext) _PackageInfo_Name(ctx context.Context, field graphql.CollectedField, obj *PackageInfo) (ret graphql.Marshaler) { fc, err := ec.fieldContext_PackageInfo_Name(ctx, field) if err != nil { @@ -4675,6 +4809,8 @@ func (ec *executionContext) fieldContext_PaginatedImagesResult_Results(ctx conte return ec.fieldContext_ImageSummary_Authors(ctx, field) case "Vulnerabilities": return ec.fieldContext_ImageSummary_Vulnerabilities(ctx, field) + case "Referrers": + return ec.fieldContext_ImageSummary_Referrers(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type ImageSummary", field.Name) }, @@ -5557,6 +5693,8 @@ func (ec *executionContext) fieldContext_Query_Image(ctx context.Context, field return ec.fieldContext_ImageSummary_Authors(ctx, field) case "Vulnerabilities": return ec.fieldContext_ImageSummary_Vulnerabilities(ctx, field) + case "Referrers": + return ec.fieldContext_ImageSummary_Referrers(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type ImageSummary", field.Name) }, @@ -6052,6 +6190,8 @@ func (ec *executionContext) fieldContext_RepoInfo_Images(ctx context.Context, fi return ec.fieldContext_ImageSummary_Authors(ctx, field) case "Vulnerabilities": return ec.fieldContext_ImageSummary_Vulnerabilities(ctx, field) + case "Referrers": + return ec.fieldContext_ImageSummary_Referrers(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type ImageSummary", field.Name) }, @@ -6446,6 +6586,8 @@ func (ec *executionContext) fieldContext_RepoSummary_NewestImage(ctx context.Con return ec.fieldContext_ImageSummary_Authors(ctx, field) case "Vulnerabilities": return ec.fieldContext_ImageSummary_Vulnerabilities(ctx, field) + case "Referrers": + return ec.fieldContext_ImageSummary_Referrers(ctx, field) } return nil, fmt.Errorf("no field named %q was found under type ImageSummary", field.Name) }, @@ -8745,6 +8887,10 @@ func (ec *executionContext) _ImageSummary(ctx context.Context, sel ast.Selection out.Values[i] = ec._ImageSummary_Vulnerabilities(ctx, field, obj) + case "Referrers": + + out.Values[i] = ec._ImageSummary_Referrers(ctx, field, obj) + default: panic("unknown field " + strconv.Quote(field.Name)) } @@ -8893,6 +9039,10 @@ func (ec *executionContext) _ManifestSummary(ctx context.Context, sel ast.Select out.Values[i] = ec._ManifestSummary_Vulnerabilities(ctx, field, obj) + case "Referrers": + + out.Values[i] = ec._ManifestSummary_Referrers(ctx, field, obj) + default: panic("unknown field " + strconv.Quote(field.Name)) } @@ -10777,6 +10927,47 @@ func (ec *executionContext) marshalOPlatform2ᚖzotregistryᚗioᚋzotᚋpkgᚋe return ec._Platform(ctx, sel, v) } +func (ec *executionContext) marshalOReferrer2ᚕᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐReferrer(ctx context.Context, sel ast.SelectionSet, v []*Referrer) graphql.Marshaler { + if v == nil { + return graphql.Null + } + ret := make(graphql.Array, len(v)) + var wg sync.WaitGroup + isLen1 := len(v) == 1 + if !isLen1 { + wg.Add(len(v)) + } + for i := range v { + i := i + fc := &graphql.FieldContext{ + Index: &i, + Result: &v[i], + } + ctx := graphql.WithFieldContext(ctx, fc) + f := func(i int) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = nil + } + }() + if !isLen1 { + defer wg.Done() + } + ret[i] = ec.marshalOReferrer2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐReferrer(ctx, sel, v[i]) + } + if isLen1 { + f(i) + } else { + go f(i) + } + + } + wg.Wait() + + return ret +} + func (ec *executionContext) marshalOReferrer2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐReferrer(ctx context.Context, sel ast.SelectionSet, v *Referrer) graphql.Marshaler { if v == nil { return graphql.Null diff --git a/pkg/extensions/search/gql_generated/models_gen.go b/pkg/extensions/search/gql_generated/models_gen.go index 8b1f536b..47a66efc 100644 --- a/pkg/extensions/search/gql_generated/models_gen.go +++ b/pkg/extensions/search/gql_generated/models_gen.go @@ -122,6 +122,8 @@ type ImageSummary struct { Authors *string `json:"Authors"` // Short summary of the identified CVEs Vulnerabilities *ImageVulnerabilitySummary `json:"Vulnerabilities"` + // Information about objects that reference this image + Referrers []*Referrer `json:"Referrers"` } // Contains summary of vulnerabilities found in a specific image @@ -171,6 +173,8 @@ type ManifestSummary struct { History []*LayerHistory `json:"History"` // Short summary of the identified CVEs Vulnerabilities *ImageVulnerabilitySummary `json:"Vulnerabilities"` + // Information about objects that reference this image + Referrers []*Referrer `json:"Referrers"` } // Contains the name of the package, the current installed version and the version where the CVE was fixed diff --git a/pkg/extensions/search/resolver.go b/pkg/extensions/search/resolver.go index f32c390c..d6b73e1e 100644 --- a/pkg/extensions/search/resolver.go +++ b/pkg/extensions/search/resolver.go @@ -1200,7 +1200,7 @@ func getReferrers(repoDB repodb.RepoDB, repo string, referredDigest string, arti referredDigest, err) } - referrers, err := repoDB.GetFilteredReferrersInfo(repo, refDigest, artifactTypes) + referrers, err := repoDB.GetReferrersInfo(repo, refDigest, artifactTypes) if err != nil { return nil, err } diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index 5fce2676..3e999cfe 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -1374,7 +1374,7 @@ func TestGetReferrers(t *testing.T) { Convey("GetReferrers returns error", func() { testLogger := log.NewLogger("debug", "") mockedStore := mocks.RepoDBMock{ - GetFilteredReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string, + GetReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string, ) ([]repodb.ReferrerInfo, error) { return nil, ErrTestError }, @@ -1396,7 +1396,7 @@ func TestGetReferrers(t *testing.T) { }, } mockedStore := mocks.RepoDBMock{ - GetFilteredReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string, + GetReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string, ) ([]repodb.ReferrerInfo, error) { return []repodb.ReferrerInfo{ { diff --git a/pkg/extensions/search/schema.graphql b/pkg/extensions/search/schema.graphql index 60ad01a7..df541180 100644 --- a/pkg/extensions/search/schema.graphql +++ b/pkg/extensions/search/schema.graphql @@ -184,6 +184,10 @@ type ImageSummary { Short summary of the identified CVEs """ Vulnerabilities: ImageVulnerabilitySummary + """ + Information about objects that reference this image + """ + Referrers: [Referrer] } """ Details about a specific version of an image for a certain operating system and architecture. @@ -226,6 +230,10 @@ type ManifestSummary { Short summary of the identified CVEs """ Vulnerabilities: ImageVulnerabilitySummary + """ + Information about objects that reference this image + """ + Referrers: [Referrer] } """ diff --git a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go index 9ce057cf..fdc007f3 100644 --- a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go +++ b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go @@ -138,7 +138,7 @@ func (bdw *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Diges Tags: map[string]repodb.Descriptor{}, Statistics: map[string]repodb.DescriptorStatistics{}, Signatures: map[string]repodb.ManifestSignatures{}, - Referrers: map[string][]repodb.Descriptor{}, + Referrers: map[string][]repodb.ReferrerInfo{}, } repoMetaBlob := repoBuck.Get([]byte(repo)) @@ -308,7 +308,7 @@ func (bdw DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.Art return artifactData, err } -func (bdw DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error { +func (bdw DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error { err := bdw.DB.Update(func(tx *bolt.Tx) error { buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) @@ -328,12 +328,9 @@ func (bdw DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, re Signatures: map[string]repodb.ManifestSignatures{ referredDigest.String(): {}, }, - Referrers: map[string][]repodb.Descriptor{ + Referrers: map[string][]repodb.ReferrerInfo{ referredDigest.String(): { - { - Digest: referrer.Digest, - MediaType: referrer.MediaType, - }, + referrer, }, }, } @@ -360,10 +357,7 @@ func (bdw DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, re } } - refferers = append(refferers, repodb.Descriptor{ - Digest: referrer.Digest, - MediaType: referrer.MediaType, - }) + refferers = append(refferers, referrer) repoMeta.Referrers[referredDigest.String()] = refferers @@ -418,8 +412,9 @@ func (bdw DBWrapper) DeleteReferrer(repo string, referredDigest godigest.Digest, }) } -func (bdw DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) { - var referrers []repodb.Descriptor +func (bdw DBWrapper) GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string, +) ([]repodb.ReferrerInfo, error) { + referrersInfoResult := []repodb.ReferrerInfo{} err := bdw.DB.View(func(tx *bolt.Tx) error { buck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) @@ -436,117 +431,20 @@ func (bdw DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) ( return err } - referrers = repoMeta.Referrers[referredDigest.String()] + referrersInfo := repoMeta.Referrers[referredDigest.String()] - return nil - }) - - return referrers, err -} - -func (bdw DBWrapper) GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest, - artifactTypes []string, -) ([]repodb.ReferrerInfo, error) { - referrersDescriptors, err := bdw.GetReferrers(repo, referredDigest) - if err != nil { - bdw.Log.Error().Msgf("repodb: failed to get referrers for '%s@%s'", repo, referredDigest.String()) - - return nil, err - } - - referrersInfo := []repodb.ReferrerInfo{} - - err = bdw.DB.View(func(tx *bolt.Tx) error { - artifactBuck := tx.Bucket([]byte(repodb.ArtifactDataBucket)) - manifestBuck := tx.Bucket([]byte(repodb.ManifestDataBucket)) - - for _, descriptor := range referrersDescriptors { - referrerInfo := repodb.ReferrerInfo{} - - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDataBlob := manifestBuck.Get([]byte(descriptor.Digest)) - - if len(manifestDataBlob) == 0 { - bdw.Log.Error().Msgf("repodb: manifest data not found for digest %s", descriptor.Digest) - - continue - } - - var manifestData repodb.ManifestData - - err = json.Unmarshal(manifestDataBlob, &manifestData) - if err != nil { - bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal manifest data for digest %s", - descriptor.Digest) - - continue - } - - var manifestContent ispec.Manifest - - err := json.Unmarshal(manifestData.ManifestBlob, &manifestContent) - if err != nil { - bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal manifest for digest %s", - descriptor.Digest) - - continue - } - - referrerInfo = repodb.ReferrerInfo{ - Digest: descriptor.Digest, - MediaType: ispec.MediaTypeImageManifest, - ArtifactType: manifestContent.Config.MediaType, - Size: len(manifestData.ManifestBlob), - Annotations: manifestContent.Annotations, - } - case ispec.MediaTypeArtifactManifest: - artifactDataBlob := artifactBuck.Get([]byte(descriptor.Digest)) - - if len(artifactDataBlob) == 0 { - bdw.Log.Error().Msgf("repodb: artifact data not found for digest %s", descriptor.Digest) - - continue - } - - var artifactData repodb.ArtifactData - - err = json.Unmarshal(artifactDataBlob, &artifactData) - if err != nil { - bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal artifact data for digest %s", descriptor.Digest) - - continue - } - - manifestContent := ispec.Artifact{} - - err := json.Unmarshal(artifactData.ManifestBlob, &manifestContent) - if err != nil { - bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal artifact manifest for digest %s", descriptor.Digest) - - continue - } - - referrerInfo = repodb.ReferrerInfo{ - Size: len(artifactData.ManifestBlob), - Digest: descriptor.Digest, - MediaType: manifestContent.MediaType, - Annotations: manifestContent.Annotations, - ArtifactType: manifestContent.ArtifactType, - } - } - - if !common.MatchesArtifactTypes(referrerInfo.ArtifactType, artifactTypes) { + for i := range referrersInfo { + if !common.MatchesArtifactTypes(referrersInfo[i].ArtifactType, artifactTypes) { continue } - referrersInfo = append(referrersInfo, referrerInfo) + referrersInfoResult = append(referrersInfoResult, referrersInfo[i]) } return nil }) - return referrersInfo, err + return referrersInfoResult, err } func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, @@ -570,7 +468,7 @@ func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDi Tags: map[string]repodb.Descriptor{}, Statistics: map[string]repodb.DescriptorStatistics{}, Signatures: map[string]repodb.ManifestSignatures{}, - Referrers: map[string][]repodb.Descriptor{}, + Referrers: map[string][]repodb.ReferrerInfo{}, } repoMetaBlob, err = json.Marshal(repoMeta) @@ -596,7 +494,7 @@ func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDi repoMeta.Statistics[manifestDigest.String()] = repodb.DescriptorStatistics{DownloadCount: 0} repoMeta.Signatures[manifestDigest.String()] = repodb.ManifestSignatures{} - repoMeta.Referrers[manifestDigest.String()] = []repodb.Descriptor{} + repoMeta.Referrers[manifestDigest.String()] = []repodb.ReferrerInfo{} repoMetaBlob, err = json.Marshal(repoMeta) if err != nil { diff --git a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go index 20904ab3..83cf9995 100644 --- a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go +++ b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go @@ -90,7 +90,7 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - err = boltdbWrapper.SetReferrer("repo", "ref", repodb.Descriptor{}) + err = boltdbWrapper.SetReferrer("repo", "ref", repodb.ReferrerInfo{}) So(err, ShouldNotBeNil) }) @@ -113,99 +113,6 @@ func TestWrapperErrors(t *testing.T) { }) }) - Convey("GetReferrers", func() { - Convey("RepoMeta not found", func() { - _, err := boltdbWrapper.GetReferrers("repo", "dig") - So(err, ShouldNotBeNil) - }) - - Convey("bad repo meta blob", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - _, err = boltdbWrapper.GetReferrers("repo", "dig") - So(err, ShouldNotBeNil) - }) - }) - - Convey("GetFilteredReferrersInfo", func() { - Convey("getReferrers fails", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - _, err = boltdbWrapper.GetFilteredReferrersInfo("repo", "", nil) - So(err, ShouldNotBeNil) - }) - - Convey("unmarshal erorrs", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - manifestBuck := tx.Bucket([]byte(repodb.ManifestDataBucket)) - artifactBuck := tx.Bucket([]byte(repodb.ArtifactDataBucket)) - - err = manifestBuck.Put([]byte("manifestDataRef"), []byte("bad json")) - So(err, ShouldBeNil) - - err = artifactBuck.Put([]byte("artifactDataRef"), []byte("bad json")) - So(err, ShouldBeNil) - - badBlob, err := json.Marshal(repodb.ArtifactData{ - ManifestBlob: []byte("bad json"), - }) - So(err, ShouldBeNil) - - err = artifactBuck.Put([]byte("artifactManifestRef"), badBlob) - So(err, ShouldBeNil) - - badBlob, err = json.Marshal(repodb.ManifestData{ - ManifestBlob: []byte("bad json"), - }) - So(err, ShouldBeNil) - - err = manifestBuck.Put([]byte("badManifest"), badBlob) - So(err, ShouldBeNil) - - return nil - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{ - Digest: "manifestDataRef", - MediaType: ispec.MediaTypeImageManifest, - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{ - Digest: "artifactDataRef", - MediaType: ispec.MediaTypeArtifactManifest, - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{ - Digest: "badManifest", - MediaType: ispec.MediaTypeImageManifest, - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{ - Digest: "artifactManifestRef", - MediaType: ispec.MediaTypeArtifactManifest, - }) - So(err, ShouldBeNil) - - refInfo, err := boltdbWrapper.GetFilteredReferrersInfo("repo", "refDigest", nil) - So(err, ShouldBeNil) - So(len(refInfo), ShouldEqual, 0) - }) - }) - Convey("SetRepoReference", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) @@ -218,6 +125,18 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("GetRepoMeta", func() { + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + + return repoBuck.Put([]byte("repo1"), []byte("wrong json")) + }) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.GetRepoMeta("repo1") + So(err, ShouldNotBeNil) + }) + Convey("DeleteRepoTag", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) @@ -230,6 +149,21 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("GetReferrersInfo", func() { + _, err = boltdbWrapper.GetReferrersInfo("repo1", "tag", nil) + So(err, ShouldNotBeNil) + + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) + + return repoBuck.Put([]byte("repo1"), []byte("wrong json")) + }) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.GetReferrersInfo("repo1", "tag", nil) + So(err, ShouldNotBeNil) + }) + Convey("IncrementRepoStars", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket)) diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go index 861561ce..b7e7f2c4 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go @@ -244,7 +244,7 @@ func TestWrapperErrors(t *testing.T) { Convey("SetReferrer client error", func() { dynamoWrapper.RepoMetaTablename = badTablename - err := dynamoWrapper.SetReferrer("repo", "", repodb.Descriptor{}) + err := dynamoWrapper.SetReferrer("repo", "", repodb.ReferrerInfo{}) So(err, ShouldNotBeNil) }) @@ -252,7 +252,7 @@ func TestWrapperErrors(t *testing.T) { err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") So(err, ShouldBeNil) - err = dynamoWrapper.SetReferrer("repo", "", repodb.Descriptor{}) + err = dynamoWrapper.SetReferrer("repo", "", repodb.ReferrerInfo{}) So(err, ShouldNotBeNil) }) @@ -284,32 +284,32 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("GetFilteredReferrersInfo GetReferrers errors", func() { + Convey("GetReferrersInfo GetReferrers errors", func() { dynamoWrapper.RepoMetaTablename = badTablename - _, err := dynamoWrapper.GetFilteredReferrersInfo("repo", "", nil) + _, err := dynamoWrapper.GetReferrersInfo("repo", "", nil) So(err, ShouldNotBeNil) }) - Convey("GetFilteredReferrersInfo getData fails", func() { + Convey("GetReferrersInfo getData fails", func() { dynamoWrapper.ManifestDataTablename = badTablename dynamoWrapper.ArtifactDataTablename = badTablename - err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{ + err = dynamoWrapper.SetReferrer("repo", "rf", repodb.ReferrerInfo{ Digest: "dig1", MediaType: ispec.MediaTypeImageManifest, }) So(err, ShouldBeNil) - err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{ + err = dynamoWrapper.SetReferrer("repo", "rf", repodb.ReferrerInfo{ Digest: "dig2", MediaType: ispec.MediaTypeArtifactManifest, }) So(err, ShouldBeNil) - _, err := dynamoWrapper.GetFilteredReferrersInfo("repo", "rf", nil) + _, err := dynamoWrapper.GetReferrersInfo("repo", "rf", nil) So(err, ShouldBeNil) }) - Convey("GetFilteredReferrersInfo bad descriptor blob", func() { + Convey("GetReferrersInfo bad descriptor blob", func() { err = dynamoWrapper.SetArtifactData("dig2", repodb.ArtifactData{ ManifestBlob: []byte("bad json"), }) @@ -320,19 +320,19 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{ + err = dynamoWrapper.SetReferrer("repo", "rf", repodb.ReferrerInfo{ Digest: "dig2", MediaType: ispec.MediaTypeArtifactManifest, }) So(err, ShouldBeNil) - err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{ + err = dynamoWrapper.SetReferrer("repo", "rf", repodb.ReferrerInfo{ Digest: "dig3", MediaType: ispec.MediaTypeImageManifest, }) So(err, ShouldBeNil) - _, err := dynamoWrapper.GetFilteredReferrersInfo("repo", "rf", nil) + _, err := dynamoWrapper.GetReferrersInfo("repo", "rf", nil) So(err, ShouldBeNil) }) diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go index f6f55eaa..2feaed64 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go @@ -168,7 +168,7 @@ func (dwr *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Diges Tags: map[string]repodb.Descriptor{}, Statistics: map[string]repodb.DescriptorStatistics{}, Signatures: map[string]repodb.ManifestSignatures{}, - Referrers: map[string][]repodb.Descriptor{}, + Referrers: map[string][]repodb.ReferrerInfo{}, } } @@ -368,7 +368,7 @@ func (dwr DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.Art return artifactData, nil } -func (dwr DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error { +func (dwr DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error { resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ TableName: aws.String(dwr.RepoMetaTablename), Key: map[string]types.AttributeValue{ @@ -384,7 +384,7 @@ func (dwr DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, re Tags: map[string]repodb.Descriptor{}, Statistics: map[string]repodb.DescriptorStatistics{}, Signatures: map[string]repodb.ManifestSignatures{}, - Referrers: map[string][]repodb.Descriptor{}, + Referrers: map[string][]repodb.ReferrerInfo{}, } if resp.Item != nil { @@ -409,7 +409,7 @@ func (dwr DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, re return dwr.setRepoMeta(repo, repoMeta) } -func (dwr DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) { +func (dwr DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.ReferrerInfo, error) { resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ TableName: aws.String(dwr.RepoMetaTablename), Key: map[string]types.AttributeValue{ @@ -417,7 +417,7 @@ func (dwr DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) ( }, }) if err != nil { - return []repodb.Descriptor{}, err + return []repodb.ReferrerInfo{}, err } repoMeta := repodb.RepoMetadata{ @@ -425,13 +425,13 @@ func (dwr DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) ( Tags: map[string]repodb.Descriptor{}, Statistics: map[string]repodb.DescriptorStatistics{}, Signatures: map[string]repodb.ManifestSignatures{}, - Referrers: map[string][]repodb.Descriptor{}, + Referrers: map[string][]repodb.ReferrerInfo{}, } if resp.Item != nil { err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta) if err != nil { - return []repodb.Descriptor{}, err + return []repodb.ReferrerInfo{}, err } } @@ -456,7 +456,7 @@ func (dwr DBWrapper) DeleteReferrer(repo string, referredDigest godigest.Digest, Tags: map[string]repodb.Descriptor{}, Statistics: map[string]repodb.DescriptorStatistics{}, Signatures: map[string]repodb.ManifestSignatures{}, - Referrers: map[string][]repodb.Descriptor{}, + Referrers: map[string][]repodb.ReferrerInfo{}, } if resp.Item != nil { @@ -481,79 +481,25 @@ func (dwr DBWrapper) DeleteReferrer(repo string, referredDigest godigest.Digest, return dwr.setRepoMeta(repo, repoMeta) } -func (dwr DBWrapper) GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest, +func (dwr DBWrapper) GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string, ) ([]repodb.ReferrerInfo, error) { - referrersDescriptors, err := dwr.GetReferrers(repo, referredDigest) + referrersInfo, err := dwr.GetReferrers(repo, referredDigest) if err != nil { return nil, err } - referrersInfo := []repodb.ReferrerInfo{} - - for _, descriptor := range referrersDescriptors { - referrerInfo := repodb.ReferrerInfo{} - - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestData, err := dwr.GetManifestData(godigest.Digest(descriptor.Digest)) - if err != nil { - dwr.Log.Error().Msgf("repodb: manifest data not found for digest %s", descriptor.Digest) - - continue - } - - var manifestContent ispec.Manifest - - err = json.Unmarshal(manifestData.ManifestBlob, &manifestContent) - if err != nil { - dwr.Log.Error().Err(err).Msgf("repodb: can't unmarhsal manifest for digest %s", - descriptor.Digest) - - continue - } - - referrerInfo = repodb.ReferrerInfo{ - Digest: descriptor.Digest, - MediaType: ispec.MediaTypeImageManifest, - ArtifactType: manifestContent.Config.MediaType, - Size: len(manifestData.ManifestBlob), - Annotations: manifestContent.Annotations, - } - case ispec.MediaTypeArtifactManifest: - artifactData, err := dwr.GetArtifactData(godigest.Digest(descriptor.Digest)) - if err != nil { - dwr.Log.Error().Msgf("repodb: artifact data not found for digest %s", descriptor.Digest) - - continue - } - - manifestContent := ispec.Artifact{} - - err = json.Unmarshal(artifactData.ManifestBlob, &manifestContent) - if err != nil { - dwr.Log.Error().Err(err).Msgf("repodb: can't unmarhsal artifact manifest for digest %s", descriptor.Digest) - - continue - } - - referrerInfo = repodb.ReferrerInfo{ - Digest: descriptor.Digest, - MediaType: manifestContent.MediaType, - ArtifactType: manifestContent.ArtifactType, - Size: len(artifactData.ManifestBlob), - Annotations: manifestContent.Annotations, - } - } + filteredResults := make([]repodb.ReferrerInfo, 0, len(referrersInfo)) + for _, referrerInfo := range referrersInfo { if !common.MatchesArtifactTypes(referrerInfo.ArtifactType, artifactTypes) { continue } - referrersInfo = append(referrersInfo, referrerInfo) + filteredResults = append(filteredResults, referrerInfo) } - return referrersInfo, nil + return filteredResults, nil } func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, @@ -578,7 +524,7 @@ func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDi Tags: map[string]repodb.Descriptor{}, Statistics: map[string]repodb.DescriptorStatistics{}, Signatures: map[string]repodb.ManifestSignatures{}, - Referrers: map[string][]repodb.Descriptor{}, + Referrers: map[string][]repodb.ReferrerInfo{}, } if resp.Item != nil { @@ -597,7 +543,7 @@ func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDi repoMeta.Statistics[manifestDigest.String()] = repodb.DescriptorStatistics{DownloadCount: 0} repoMeta.Signatures[manifestDigest.String()] = repodb.ManifestSignatures{} - repoMeta.Referrers[manifestDigest.String()] = []repodb.Descriptor{} + repoMeta.Referrers[manifestDigest.String()] = []repodb.ReferrerInfo{} err = dwr.setRepoMeta(repo, repoMeta) diff --git a/pkg/meta/repodb/load_repodb.go b/pkg/meta/repodb/load_repodb.go index bf5b8cd9..0c90c6e6 100644 --- a/pkg/meta/repodb/load_repodb.go +++ b/pkg/meta/repodb/load_repodb.go @@ -3,6 +3,7 @@ package repodb import ( "encoding/json" "errors" + "fmt" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -305,11 +306,9 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig } } - if refferredDigest, hasSubject := GetReferredSubject(descriptorBlob); hasSubject { - err := repoDB.SetReferrer(repo, refferredDigest, Descriptor{ - Digest: digest.String(), - MediaType: mediaType, - }) + refferredDigest, referrerInfo, hasSubject, err := GetReferredSubject(descriptorBlob, digest.String(), mediaType) + if hasSubject && err == nil { + err := repoDB.SetReferrer(repo, refferredDigest, referrerInfo) if err != nil { log.Error().Err(err).Msg("repodb: error while settingg referrer") @@ -317,7 +316,7 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig } } - err := repoDB.SetRepoReference(repo, reference, digest, mediaType) + err = repoDB.SetRepoReference(repo, reference, digest, mediaType) if err != nil { log.Error().Err(err).Msg("repodb: error while putting repo meta") @@ -327,17 +326,55 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig return nil } -func GetReferredSubject(descriptorBlob []byte) (godigest.Digest, bool) { - var manifest ispec.Manifest +func GetReferredSubject(descriptorBlob []byte, referrerDigest, mediaType string, +) (godigest.Digest, ReferrerInfo, bool, error) { + var ( + referrerInfo ReferrerInfo + referrerSubject *ispec.Descriptor + ) - err := json.Unmarshal(descriptorBlob, &manifest) - if err != nil { - return "", false + switch mediaType { + case ispec.MediaTypeImageManifest: + var manifestContent ispec.Manifest + + err := json.Unmarshal(descriptorBlob, &manifestContent) + if err != nil { + return "", referrerInfo, false, + fmt.Errorf("repodb: can't unmarhsal manifest for digest %s: %w", referrerDigest, err) + } + + referrerSubject = manifestContent.Subject + + referrerInfo = ReferrerInfo{ + Digest: referrerDigest, + MediaType: mediaType, + ArtifactType: manifestContent.Config.MediaType, + Size: len(descriptorBlob), + Annotations: manifestContent.Annotations, + } + case ispec.MediaTypeArtifactManifest: + manifestContent := ispec.Artifact{} + + err := json.Unmarshal(descriptorBlob, &manifestContent) + if err != nil { + return "", referrerInfo, false, + fmt.Errorf("repodb: can't unmarhsal artifact manifest for digest %s: %w", referrerDigest, err) + } + + referrerSubject = manifestContent.Subject + + referrerInfo = ReferrerInfo{ + Digest: referrerDigest, + MediaType: manifestContent.MediaType, + ArtifactType: manifestContent.ArtifactType, + Size: len(descriptorBlob), + Annotations: manifestContent.Annotations, + } } - if manifest.Subject == nil || manifest.Subject.Digest.String() == "" { - return "", false + if referrerSubject == nil || referrerSubject.Digest.String() == "" { + return "", ReferrerInfo{}, false, nil } - return manifest.Subject.Digest, true + return referrerSubject.Digest, referrerInfo, true, nil } diff --git a/pkg/meta/repodb/load_repodb_test.go b/pkg/meta/repodb/load_repodb_test.go index c82376ec..5331f4e4 100644 --- a/pkg/meta/repodb/load_repodb_test.go +++ b/pkg/meta/repodb/load_repodb_test.go @@ -645,7 +645,10 @@ func TestLoadOCILayoutDynamoWrapper(t *testing.T) { func TestGetReferredSubject(t *testing.T) { Convey("GetReferredSubject error", t, func() { - _, err := repodb.GetReferredSubject([]byte("bad json")) + _, _, _, err := repodb.GetReferredSubject([]byte("bad json"), "digest", ispec.MediaTypeArtifactManifest) + So(err, ShouldNotBeNil) + + _, _, _, err = repodb.GetReferredSubject([]byte("bad json"), "digest", ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) }) } diff --git a/pkg/meta/repodb/repodb.go b/pkg/meta/repodb/repodb.go index 6acb014e..be8df499 100644 --- a/pkg/meta/repodb/repodb.go +++ b/pkg/meta/repodb/repodb.go @@ -75,17 +75,14 @@ type RepoDB interface { //nolint:interfacebloat GetArtifactData(artifactDigest godigest.Digest) (ArtifactData, error) // SetReferrer adds a referrer to the referrers list of a manifest inside a repo - SetReferrer(repo string, referredDigest godigest.Digest, referrer Descriptor) error + SetReferrer(repo string, referredDigest godigest.Digest, referrer ReferrerInfo) error // SetReferrer delets a referrer to the referrers list of a manifest inside a repo DeleteReferrer(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest) error - // GetReferrers returns the list of referrers for a referred manifest - GetReferrers(repo string, referredDigest godigest.Digest) ([]Descriptor, error) - - // GetFilteredReferrersInfo returnes a list of for all referrers of the given digest that match one of the + // GetReferrersInfo returnes a list of for all referrers of the given digest that match one of the // artifact types. - GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string) ( + GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string) ( []ReferrerInfo, error) // IncrementManifestDownloads adds 1 to the download count of a manifest @@ -158,7 +155,7 @@ type RepoMetadata struct { Statistics map[string]DescriptorStatistics Signatures map[string]ManifestSignatures - Referrers map[string][]Descriptor + Referrers map[string][]ReferrerInfo Stars int } diff --git a/pkg/meta/repodb/repodb_test.go b/pkg/meta/repodb/repodb_test.go index 53b8990b..0bf4381b 100644 --- a/pkg/meta/repodb/repodb_test.go +++ b/pkg/meta/repodb/repodb_test.go @@ -1816,7 +1816,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { artifactDigest1, err := artifact1.Digest() So(err, ShouldBeNil) - err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{ Digest: artifactDigest1.String(), MediaType: ispec.MediaTypeImageManifest, }) @@ -1833,7 +1833,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { artifactDigest2, err := artifact2.Digest() So(err, ShouldBeNil) - err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{ Digest: artifactDigest2.String(), MediaType: ispec.MediaTypeArtifactManifest, }) @@ -1841,13 +1841,13 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { // ------ GetReferrers - referrers, err := repoDB.GetReferrers("repo", referredDigest) + referrers, err := repoDB.GetReferrersInfo("repo", referredDigest, nil) So(len(referrers), ShouldEqual, 2) - So(referrers, ShouldContain, repodb.Descriptor{ + So(referrers, ShouldContain, repodb.ReferrerInfo{ Digest: artifactDigest1.String(), MediaType: ispec.MediaTypeImageManifest, }) - So(referrers, ShouldContain, repodb.Descriptor{ + So(referrers, ShouldContain, repodb.ReferrerInfo{ Digest: artifactDigest2.String(), MediaType: ispec.MediaTypeArtifactManifest, }) @@ -1861,7 +1861,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { err = repoDB.DeleteReferrer("repo", referredDigest, artifactDigest2) So(err, ShouldBeNil) - referrers, err = repoDB.GetReferrers("repo", referredDigest) + referrers, err = repoDB.GetReferrersInfo("repo", referredDigest, nil) So(err, ShouldBeNil) So(len(referrers), ShouldEqual, 0) }) @@ -1874,7 +1874,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { referredDigest := godigest.FromString("referredDigest") referrerDigest := godigest.FromString("referrerDigest") - err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{ Digest: referrerDigest.String(), MediaType: ispec.MediaTypeImageManifest, }) @@ -1893,13 +1893,13 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { referredDigest := godigest.FromString("referredDigest") referrerDigest := godigest.FromString("referrerDigest") - err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{ Digest: referrerDigest.String(), MediaType: ispec.MediaTypeImageManifest, }) So(err, ShouldBeNil) - err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{ Digest: referrerDigest.String(), MediaType: ispec.MediaTypeImageManifest, }) @@ -1910,16 +1910,16 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { So(len(repoMeta.Referrers[referredDigest.String()]), ShouldEqual, 1) }) - Convey("GetFilteredReferrersInfo", func() { + Convey("GetReferrersInfo", func() { referredDigest := godigest.FromString("referredDigest") - err := repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + err := repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{ Digest: "inexistendManifestDigest", MediaType: ispec.MediaTypeImageManifest, }) So(err, ShouldBeNil) - err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ + err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{ Digest: "inexistendArtifactManifestDigest", MediaType: ispec.MediaTypeArtifactManifest, }) @@ -1932,9 +1932,10 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { }) So(err, ShouldBeNil) - err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ - Digest: "goodManifest", - MediaType: ispec.MediaTypeImageManifest, + err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{ + Digest: "goodManifest", + MediaType: ispec.MediaTypeImageManifest, + ArtifactType: "unwantedType", }) So(err, ShouldBeNil) @@ -1943,13 +1944,14 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { }) So(err, ShouldBeNil) - err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{ - Digest: "goodArtifact", - MediaType: ispec.MediaTypeArtifactManifest, + err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{ + Digest: "goodArtifact", + MediaType: ispec.MediaTypeArtifactManifest, + ArtifactType: "wantedType", }) So(err, ShouldBeNil) - referrerInfo, err := repoDB.GetFilteredReferrersInfo("repo", referredDigest, []string{"wantedType"}) + referrerInfo, err := repoDB.GetReferrersInfo("repo", referredDigest, []string{"wantedType"}) So(err, ShouldBeNil) So(len(referrerInfo), ShouldEqual, 1) So(referrerInfo[0].ArtifactType, ShouldResemble, "wantedType") diff --git a/pkg/meta/repodb/update/update_test.go b/pkg/meta/repodb/update/update_test.go index 769a5b11..cdbe4913 100644 --- a/pkg/meta/repodb/update/update_test.go +++ b/pkg/meta/repodb/update/update_test.go @@ -269,7 +269,7 @@ func TestUpdateErrors(t *testing.T) { log := log.NewLogger("debug", "") repoDB := mocks.RepoDBMock{ - SetReferrerFn: func(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error { + SetReferrerFn: func(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error { return ErrTestError }, } diff --git a/pkg/test/mocks/repo_db_mock.go b/pkg/test/mocks/repo_db_mock.go index f741b51a..2c2a256f 100644 --- a/pkg/test/mocks/repo_db_mock.go +++ b/pkg/test/mocks/repo_db_mock.go @@ -44,13 +44,13 @@ type RepoDBMock struct { GetArtifactDataFn func(artifactDigest godigest.Digest) (repodb.ArtifactData, error) - SetReferrerFn func(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error + SetReferrerFn func(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error DeleteReferrerFn func(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest) error GetReferrersFn func(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) - GetFilteredReferrersInfoFn func(repo string, referredDigest godigest.Digest, artifactTypes []string) ( + GetReferrersInfoFn func(repo string, referredDigest godigest.Digest, artifactTypes []string) ( []repodb.ReferrerInfo, error) IncrementImageDownloadsFn func(repo string, reference string) error @@ -322,7 +322,7 @@ func (sdm RepoDBMock) GetArtifactData(artifactDigest godigest.Digest) (repodb.Ar return repodb.ArtifactData{}, nil } -func (sdm RepoDBMock) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error { +func (sdm RepoDBMock) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error { if sdm.SetReferrerFn != nil { return sdm.SetReferrerFn(repo, referredDigest, referrer) } @@ -340,19 +340,11 @@ func (sdm RepoDBMock) DeleteReferrer(repo string, referredDigest godigest.Digest return nil } -func (sdm RepoDBMock) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) { - if sdm.GetReferrersFn != nil { - return sdm.GetReferrersFn(repo, referredDigest) - } - - return []repodb.Descriptor{}, nil -} - -func (sdm RepoDBMock) GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest, +func (sdm RepoDBMock) GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string, ) ([]repodb.ReferrerInfo, error) { - if sdm.GetFilteredReferrersInfoFn != nil { - return sdm.GetFilteredReferrersInfoFn(repo, referredDigest, artifactTypes) + if sdm.GetReferrersInfoFn != nil { + return sdm.GetReferrersInfoFn(repo, referredDigest, artifactTypes) } return []repodb.ReferrerInfo{}, nil diff --git a/test/blackbox/referrers.bats b/test/blackbox/referrers.bats index 6373f5f5..a26c630d 100644 --- a/test/blackbox/referrers.bats +++ b/test/blackbox/referrers.bats @@ -7,7 +7,7 @@ function setup() { fi # Download test data to folder common for the entire suite, not just this file - skopeo --insecure-policy copy --format=oci docker://alpine:latest oci:${TEST_DATA_DIR}alpine:latest + skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20 # Setup zot server ZOT_ROOT_DIR=${BATS_FILE_TMPDIR}/zot @@ -60,23 +60,23 @@ EOF wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog" run skopeo --insecure-policy copy --dest-tls-verify=false \ - oci:${TEST_DATA_DIR}/alpine:latest \ - docker://127.0.0.1:8080/alpine:latest + oci:${TEST_DATA_DIR}/golang:1.20 \ + docker://127.0.0.1:8080/golang:1.20 [ "$status" -eq 0 ] run curl http://127.0.0.1:8080/v2/_catalog [ "$status" -eq 0 ] - [ $(echo "${lines[-1]}" | jq '.repositories[]') = '"alpine"' ] + [ $(echo "${lines[-1]}" | jq '.repositories[]') = '"golang"' ] - run oras attach --plain-http --image-spec v1.1-image --artifact-type image.type 127.0.0.1:8080/alpine:latest ${IMAGE_MANIFEST_REFERRER} + run oras attach --plain-http --image-spec v1.1-image --artifact-type image.type 127.0.0.1:8080/golang:1.20 ${IMAGE_MANIFEST_REFERRER} [ "$status" -eq 0 ] - run oras attach --plain-http --image-spec v1.1-artifact --artifact-type artifact.type 127.0.0.1:8080/alpine:latest ${ARTIFACT_MANIFEST_REFERRER} + run oras attach --plain-http --image-spec v1.1-artifact --artifact-type artifact.type 127.0.0.1:8080/golang:1.20 ${ARTIFACT_MANIFEST_REFERRER} [ "$status" -eq 0 ] - MANIFEST_DIGEST=$(skopeo inspect --tls-verify=false docker://localhost:8080/alpine:latest | jq -r '.Digest') + MANIFEST_DIGEST=$(skopeo inspect --tls-verify=false docker://localhost:8080/golang:1.20 | jq -r '.Digest') echo ${MANIFEST_DIGEST} - curl -X GET http://127.0.0.1:8080/v2/alpine/referrers/${MANIFEST_DIGEST}?artifactType=image.type + curl -X GET http://127.0.0.1:8080/v2/golang/referrers/${MANIFEST_DIGEST}?artifactType=image.type } function teardown() { @@ -88,21 +88,21 @@ function teardown() { @test "add referrers, one artifact and one image" { # Check referrers API using the normal REST endpoint - run curl -X GET http://127.0.0.1:8080/v2/alpine/referrers/${MANIFEST_DIGEST}?artifactType=image.type + run curl -X GET http://127.0.0.1:8080/v2/golang/referrers/${MANIFEST_DIGEST}?artifactType=image.type [ "$status" -eq 0 ] [ $(echo "${lines[-1]}" | jq '.manifests[].artifactType') = '"image.type"' ] - run curl -X GET http://127.0.0.1:8080/v2/alpine/referrers/${MANIFEST_DIGEST}?artifactType=artifact.type + run curl -X GET http://127.0.0.1:8080/v2/golang/referrers/${MANIFEST_DIGEST}?artifactType=artifact.type [ "$status" -eq 0 ] [ $(echo "${lines[-1]}" | jq '.manifests[].artifactType') = '"artifact.type"' ] # Check referrers API using the GQL endpoint - REFERRER_QUERY_DATA="{ \"query\": \"{ Referrers(repo:\\\"alpine\\\", digest:\\\"${MANIFEST_DIGEST}\\\", type:[\\\"image.type\\\"]) { MediaType ArtifactType Digest Size} }\"}" + REFERRER_QUERY_DATA="{ \"query\": \"{ Referrers(repo:\\\"golang\\\", digest:\\\"${MANIFEST_DIGEST}\\\", type:[\\\"image.type\\\"]) { MediaType ArtifactType Digest Size} }\"}" run curl -X POST -H "Content-Type: application/json" --data "${REFERRER_QUERY_DATA}" http://localhost:8080/v2/_zot/ext/search [ "$status" -eq 0 ] [ $(echo "${lines[-1]}" | jq '.data.Referrers[].ArtifactType') = '"image.type"' ] - REFERRER_QUERY_DATA="{ \"query\": \"{ Referrers(repo:\\\"alpine\\\", digest:\\\"${MANIFEST_DIGEST}\\\", type:[\\\"artifact.type\\\"]) { MediaType ArtifactType Digest Size} }\"}" + REFERRER_QUERY_DATA="{ \"query\": \"{ Referrers(repo:\\\"golang\\\", digest:\\\"${MANIFEST_DIGEST}\\\", type:[\\\"artifact.type\\\"]) { MediaType ArtifactType Digest Size} }\"}" run curl -X POST -H "Content-Type: application/json" --data "${REFERRER_QUERY_DATA}" http://localhost:8080/v2/_zot/ext/search [ "$status" -eq 0 ] [ $(echo "${lines[-1]}" | jq '.data.Referrers[].ArtifactType') = '"artifact.type"' ]