Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00
Code Issues Activity

feat(search): add referrers field to ImageSummary (#1261)

Browse source 
Changed repodb to store more information about the referrer needed for the referrers query

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae 2023-03-20 18:14:17 +02:00 committed by GitHub
parent 17a554b504
commit ed01292ad2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 472 additions and 394 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

82
pkg/extensions/search/common/common_test.go
View file

@ -1024,10 +1024,7 @@ func TestGetReferrersGQL(t *testing.T) {
gqlQuery := `
{
Referrers(
repo: "%s",
digest: "%s",
type: ""
){
repo: "%s", digest: "%s", type: ""){
ArtifactType,
Digest,
MediaType,
@ -1756,6 +1753,13 @@ func TestUtilsMethod(t *testing.T) {
fixedTags := common.GetFixedTags(allTags, vulnerableTags)
So(len(fixedTags), ShouldEqual, 2)
fixedTags = common.GetFixedTags(allTags, append(vulnerableTags, common.TagInfo{
Name: "taginfo",
Descriptor: common.Descriptor{},
Timestamp: time.Date(2000, time.July, 20, 10, 10, 10, 10, time.UTC),
}))
So(len(fixedTags), ShouldEqual, 3)
log := log.NewLogger("debug", "")
rootDir := t.TempDir()
@ -6395,6 +6399,7 @@ func TestImageSummary(t *testing.T) {
conf := config.New()
conf.HTTP.Port = port
conf.Storage.RootDirectory = t.TempDir()
conf.Storage.GC = false
defaultVal := true
conf.Extensions = &extconf.ExtensionConfig{
@ -6426,6 +6431,7 @@ func TestImageSummary(t *testing.T) {
LastUpdated
Size
Vulnerabilities { Count MaxSeverity }
Referrers {MediaType ArtifactType Digest Annotations {Key Value}}
}
}`
@ -6452,38 +6458,51 @@ func TestImageSummary(t *testing.T) {
}`
gqlEndpoint := fmt.Sprintf("%s%s?query=", baseURL, graphqlQueryPrefix)
config, layers, manifest, err := GetImageComponents(100)
So(err, ShouldBeNil)
createdTime := time.Date(2010, 1, 1, 12, 0, 0, 0, time.UTC)
config.History = append(config.History, ispec.History{Created: &createdTime})
manifest, err = updateManifestConfig(manifest, config)
So(err, ShouldBeNil)
configBlob, errConfig := json.Marshal(config)
configDigest := godigest.FromBytes(configBlob)
So(errConfig, ShouldBeNil) // marshall success, config is valid JSON
ctlrManager := NewControllerManager(ctlr)
ctlrManager.StartAndWait(port)
defer ctlrManager.StopServer()
manifestBlob, errMarsal := json.Marshal(manifest)
So(errMarsal, ShouldBeNil)
So(manifestBlob, ShouldNotBeNil)
manifestDigest := godigest.FromBytes(manifestBlob)
repoName := "test-repo" //nolint:goconst
tagTarget := "latest"
err = UploadImage(
Image{
Manifest: manifest,
Config: config,
Layers: layers,
Reference: tagTarget,
createdTime := time.Date(2010, 1, 1, 12, 0, 0, 0, time.UTC)
image, err := GetImageWithConfig(
ispec.Image{
History: []ispec.History{{Created: &createdTime}},
Platform: ispec.Platform{
Architecture: "amd64",
OS: "linux",
},
},
baseURL,
repoName,
)
So(err, ShouldBeNil)
image.Reference = tagTarget
manifestDigest, err := image.Digest()
So(err, ShouldBeNil)
err = UploadImage(image, baseURL, repoName)
So(err, ShouldBeNil)
// ------ Add a referrer
referrerImage, err := GetImageWithConfig(ispec.Image{})
So(err, ShouldBeNil)
referrerImage.Manifest.Subject = &ispec.Descriptor{
Digest: manifestDigest,
MediaType: ispec.MediaTypeImageManifest,
}
referrerImage.Manifest.Config.MediaType = "test.artifact.type"
referrerImage.Manifest.Annotations = map[string]string{"testAnnotationKey": "testAnnotationValue"}
referrerManifestDigest, err := referrerImage.Digest()
So(err, ShouldBeNil)
referrerImage.Reference = referrerManifestDigest.String()
err = UploadImage(referrerImage, baseURL, repoName)
So(err, ShouldBeNil)
var (
imgSummaryResponse ImageSummaryResult
strQuery string
@ -6531,11 +6550,11 @@ func TestImageSummary(t *testing.T) {
imgSummary := imgSummaryResponse.SingleImageSummary.ImageSummary
So(imgSummary.RepoName, ShouldContainSubstring, repoName)
So(imgSummary.Tag, ShouldContainSubstring, tagTarget)
So(imgSummary.Manifests[0].ConfigDigest, ShouldContainSubstring, configDigest.Encoded())
So(imgSummary.Manifests[0].ConfigDigest, ShouldContainSubstring, image.Manifest.Config.Digest.Encoded())
So(imgSummary.Manifests[0].Digest, ShouldContainSubstring, manifestDigest.Encoded())
So(len(imgSummary.Manifests[0].Layers), ShouldEqual, 1)
So(imgSummary.Manifests[0].Layers[0].Digest, ShouldContainSubstring,
godigest.FromBytes(layers[0]).Encoded())
image.Manifest.Layers[0].Digest.Encoded())
So(imgSummary.LastUpdated, ShouldEqual, createdTime)
So(imgSummary.IsSigned, ShouldEqual, false)
So(imgSummary.Manifests[0].Platform.Os, ShouldEqual, "linux")
@ -6545,6 +6564,13 @@ func TestImageSummary(t *testing.T) {
// No vulnerabilities should be detected since trivy is disabled
So(imgSummary.Vulnerabilities.Count, ShouldEqual, 0)
So(imgSummary.Vulnerabilities.MaxSeverity, ShouldEqual, "")
So(len(imgSummary.Referrers), ShouldEqual, 1)
So(imgSummary.Referrers[0], ShouldResemble, common.Referrer{
MediaType: ispec.MediaTypeImageManifest,
ArtifactType: "test.artifact.type",
Digest: referrerManifestDigest.String(),
Annotations: []common.Annotation{{Key: "testAnnotationKey", Value: "testAnnotationValue"}},
})
t.Log("starting Test retrieve duplicated image same layers based on image identifier")
// gqlEndpoint

1
pkg/extensions/search/common/model.go
View file

@ -37,6 +37,7 @@ type ImageSummary struct {
Authors string `json:"authors"`
Vendor string `json:"vendor"`
Vulnerabilities ImageVulnerabilitySummary `json:"vulnerabilities"`
Referrers []Referrer `json:"referrers"`
}
type ManifestSummary struct {

2
pkg/extensions/search/convert/convert_test.go
View file

@ -172,6 +172,7 @@ func TestConvertErrors(t *testing.T) {
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("bad json"),
},
nil,
mocks.CveInfoMock{
GetCVESummaryForImageFn: func(repo, reference string,
) (cveinfo.ImageCVESummary, error) {
@ -203,6 +204,7 @@ func TestConvertErrors(t *testing.T) {
ManifestBlob: []byte("{}"),
ConfigBlob: configBlob,
},
nil,
mocks.CveInfoMock{
GetCVESummaryForImageFn: func(repo, reference string,
) (cveinfo.ImageCVESummary, error) {

41
pkg/extensions/search/convert/repodb.go
View file

@ -193,7 +193,7 @@ func ImageIndex2ImageSummary(ctx context.Context, repo, tag string, indexDigest
for _, descriptor := range indexContent.Manifests {
manifestSummary, manifestBlobs, err := ImageManifest2ManifestSummary(ctx, repo, tag, descriptor, false,
manifestMetaMap[descriptor.Digest.String()], cveInfo)
manifestMetaMap[descriptor.Digest.String()], repoMeta.Referrers[descriptor.Digest.String()], cveInfo)
if err != nil {
return &gql_generated.ImageSummary{}, map[string]int64{}, err
}
@ -260,6 +260,7 @@ func ImageIndex2ImageSummary(ctx context.Context, repo, tag string, indexDigest
MaxSeverity: &imageCveSummary.MaxSeverity,
Count: &imageCveSummary.Count,
},
Referrers: getReferrers(repoMeta.Referrers[indexDigest.String()]),
}
return &indexSummary, indexBlobs, nil
@ -382,13 +383,48 @@ func ImageManifest2ImageSummary(ctx context.Context, repo, tag string, digest go
MaxSeverity: &imageCveSummary.MaxSeverity,
Count: &imageCveSummary.Count,
},
Referrers: getReferrers(repoMeta.Referrers[manifestDigest]),
}
return &imageSummary, imageBlobsMap, nil
}
func getReferrers(referrersInfo []repodb.ReferrerInfo) []*gql_generated.Referrer {
referrers := make([]*gql_generated.Referrer, 0, len(referrersInfo))
for _, referrerInfo := range referrersInfo {
referrerInfo := referrerInfo
referrers = append(referrers, &gql_generated.Referrer{
MediaType: &referrerInfo.MediaType,
ArtifactType: &referrerInfo.ArtifactType,
Size: &referrerInfo.Size,
Digest: &referrerInfo.Digest,
Annotations: getAnnotationsFromMap(referrerInfo.Annotations),
})
}
return referrers
}
func getAnnotationsFromMap(annotationsMap map[string]string) []*gql_generated.Annotation {
annotations := make([]*gql_generated.Annotation, 0, len(annotationsMap))
for key, value := range annotationsMap {
key := key
value := value
annotations = append(annotations, &gql_generated.Annotation{
Key: &key,
Value: &value,
})
}
return annotations
}
func ImageManifest2ManifestSummary(ctx context.Context, repo, tag string, descriptor ispec.Descriptor,
skipCVE bool, manifestMeta repodb.ManifestMetadata, cveInfo cveinfo.CveInfo,
skipCVE bool, manifestMeta repodb.ManifestMetadata, referrersInfo []repodb.ReferrerInfo, cveInfo cveinfo.CveInfo,
) (*gql_generated.ManifestSummary, map[string]int64, error) {
var (
manifestContent ispec.Manifest
@ -469,6 +505,7 @@ func ImageManifest2ManifestSummary(ctx context.Context, repo, tag string, descri
MaxSeverity: &imageCveSummary.MaxSeverity,
Count: &imageCveSummary.Count,
},
Referrers: getReferrers(referrersInfo),
}
return &manifestSummary, imageBlobsMap, nil

191
pkg/extensions/search/gql_generated/generated.go
View file

@ -86,6 +86,7 @@ type ComplexityRoot struct {
LastUpdated func(childComplexity int) int
Licenses func(childComplexity int) int
Manifests func(childComplexity int) int
Referrers func(childComplexity int) int
RepoName func(childComplexity int) int
Score func(childComplexity int) int
Size func(childComplexity int) int
@ -120,6 +121,7 @@ type ComplexityRoot struct {
LastUpdated func(childComplexity int) int
Layers func(childComplexity int) int
Platform func(childComplexity int) int
Referrers func(childComplexity int) int
Size func(childComplexity int) int
Vulnerabilities func(childComplexity int) int
}
@ -419,6 +421,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
return e.complexity.ImageSummary.Manifests(childComplexity), true
case "ImageSummary.Referrers":
if e.complexity.ImageSummary.Referrers == nil {
break
}
return e.complexity.ImageSummary.Referrers(childComplexity), true
case "ImageSummary.RepoName":
if e.complexity.ImageSummary.RepoName == nil {
break
@ -573,6 +582,13 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
return e.complexity.ManifestSummary.Platform(childComplexity), true
case "ManifestSummary.Referrers":
if e.complexity.ManifestSummary.Referrers == nil {
break
}
return e.complexity.ManifestSummary.Referrers(childComplexity), true
case "ManifestSummary.Size":
if e.complexity.ManifestSummary.Size == nil {
break
@ -1175,6 +1191,10 @@ type ImageSummary {
Short summary of the identified CVEs
"""
Vulnerabilities: ImageVulnerabilitySummary
"""
Information about objects that reference this image
"""
Referrers: [Referrer]
}
"""
Details about a specific version of an image for a certain operating system and architecture.
@ -1217,6 +1237,10 @@ type ManifestSummary {
Short summary of the identified CVEs
"""
Vulnerabilities: ImageVulnerabilitySummary
"""
Information about objects that reference this image
"""
Referrers: [Referrer]
}
"""
@ -2593,6 +2617,8 @@ func (ec *executionContext) fieldContext_GlobalSearchResult_Images(ctx context.C
return ec.fieldContext_ImageSummary_Authors(ctx, field)
case "Vulnerabilities":
return ec.fieldContext_ImageSummary_Vulnerabilities(ctx, field)
case "Referrers":
return ec.fieldContext_ImageSummary_Referrers(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type ImageSummary", field.Name)
},
@ -3055,6 +3081,8 @@ func (ec *executionContext) fieldContext_ImageSummary_Manifests(ctx context.Cont
return ec.fieldContext_ManifestSummary_History(ctx, field)
case "Vulnerabilities":
return ec.fieldContext_ManifestSummary_Vulnerabilities(ctx, field)
case "Referrers":
return ec.fieldContext_ManifestSummary_Referrers(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type ManifestSummary", field.Name)
},
@ -3642,6 +3670,59 @@ func (ec *executionContext) fieldContext_ImageSummary_Vulnerabilities(ctx contex
return fc, nil
}
func (ec *executionContext) _ImageSummary_Referrers(ctx context.Context, field graphql.CollectedField, obj *ImageSummary) (ret graphql.Marshaler) {
fc, err := ec.fieldContext_ImageSummary_Referrers(ctx, field)
if err != nil {
return graphql.Null
}
ctx = graphql.WithFieldContext(ctx, fc)
defer func() {
if r := recover(); r != nil {
ec.Error(ctx, ec.Recover(ctx, r))
ret = graphql.Null
}
}()
resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
ctx = rctx // use context from middleware stack in children
return obj.Referrers, nil
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
return graphql.Null
}
res := resTmp.([]*Referrer)
fc.Result = res
return ec.marshalOReferrer2ᚕᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐReferrer(ctx, field.Selections, res)
}
func (ec *executionContext) fieldContext_ImageSummary_Referrers(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) {
fc = &graphql.FieldContext{
Object: "ImageSummary",
Field: field,
IsMethod: false,
IsResolver: false,
Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) {
switch field.Name {
case "MediaType":
return ec.fieldContext_Referrer_MediaType(ctx, field)
case "ArtifactType":
return ec.fieldContext_Referrer_ArtifactType(ctx, field)
case "Size":
return ec.fieldContext_Referrer_Size(ctx, field)
case "Digest":
return ec.fieldContext_Referrer_Digest(ctx, field)
case "Annotations":
return ec.fieldContext_Referrer_Annotations(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type Referrer", field.Name)
},
}
return fc, nil
}
func (ec *executionContext) _ImageVulnerabilitySummary_MaxSeverity(ctx context.Context, field graphql.CollectedField, obj *ImageVulnerabilitySummary) (ret graphql.Marshaler) {
fc, err := ec.fieldContext_ImageVulnerabilitySummary_MaxSeverity(ctx, field)
if err != nil {
@ -4344,6 +4425,59 @@ func (ec *executionContext) fieldContext_ManifestSummary_Vulnerabilities(ctx con
return fc, nil
}
func (ec *executionContext) _ManifestSummary_Referrers(ctx context.Context, field graphql.CollectedField, obj *ManifestSummary) (ret graphql.Marshaler) {
fc, err := ec.fieldContext_ManifestSummary_Referrers(ctx, field)
if err != nil {
return graphql.Null
}
ctx = graphql.WithFieldContext(ctx, fc)
defer func() {
if r := recover(); r != nil {
ec.Error(ctx, ec.Recover(ctx, r))
ret = graphql.Null
}
}()
resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
ctx = rctx // use context from middleware stack in children
return obj.Referrers, nil
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
return graphql.Null
}
res := resTmp.([]*Referrer)
fc.Result = res
return ec.marshalOReferrer2ᚕᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐReferrer(ctx, field.Selections, res)
}
func (ec *executionContext) fieldContext_ManifestSummary_Referrers(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) {
fc = &graphql.FieldContext{
Object: "ManifestSummary",
Field: field,
IsMethod: false,
IsResolver: false,
Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) {
switch field.Name {
case "MediaType":
return ec.fieldContext_Referrer_MediaType(ctx, field)
case "ArtifactType":
return ec.fieldContext_Referrer_ArtifactType(ctx, field)
case "Size":
return ec.fieldContext_Referrer_Size(ctx, field)
case "Digest":
return ec.fieldContext_Referrer_Digest(ctx, field)
case "Annotations":
return ec.fieldContext_Referrer_Annotations(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type Referrer", field.Name)
},
}
return fc, nil
}
func (ec *executionContext) _PackageInfo_Name(ctx context.Context, field graphql.CollectedField, obj *PackageInfo) (ret graphql.Marshaler) {
fc, err := ec.fieldContext_PackageInfo_Name(ctx, field)
if err != nil {
@ -4675,6 +4809,8 @@ func (ec *executionContext) fieldContext_PaginatedImagesResult_Results(ctx conte
return ec.fieldContext_ImageSummary_Authors(ctx, field)
case "Vulnerabilities":
return ec.fieldContext_ImageSummary_Vulnerabilities(ctx, field)
case "Referrers":
return ec.fieldContext_ImageSummary_Referrers(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type ImageSummary", field.Name)
},
@ -5557,6 +5693,8 @@ func (ec *executionContext) fieldContext_Query_Image(ctx context.Context, field
return ec.fieldContext_ImageSummary_Authors(ctx, field)
case "Vulnerabilities":
return ec.fieldContext_ImageSummary_Vulnerabilities(ctx, field)
case "Referrers":
return ec.fieldContext_ImageSummary_Referrers(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type ImageSummary", field.Name)
},
@ -6052,6 +6190,8 @@ func (ec *executionContext) fieldContext_RepoInfo_Images(ctx context.Context, fi
return ec.fieldContext_ImageSummary_Authors(ctx, field)
case "Vulnerabilities":
return ec.fieldContext_ImageSummary_Vulnerabilities(ctx, field)
case "Referrers":
return ec.fieldContext_ImageSummary_Referrers(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type ImageSummary", field.Name)
},
@ -6446,6 +6586,8 @@ func (ec *executionContext) fieldContext_RepoSummary_NewestImage(ctx context.Con
return ec.fieldContext_ImageSummary_Authors(ctx, field)
case "Vulnerabilities":
return ec.fieldContext_ImageSummary_Vulnerabilities(ctx, field)
case "Referrers":
return ec.fieldContext_ImageSummary_Referrers(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type ImageSummary", field.Name)
},
@ -8745,6 +8887,10 @@ func (ec *executionContext) _ImageSummary(ctx context.Context, sel ast.Selection
out.Values[i] = ec._ImageSummary_Vulnerabilities(ctx, field, obj)
case "Referrers":
out.Values[i] = ec._ImageSummary_Referrers(ctx, field, obj)
default:
panic("unknown field " + strconv.Quote(field.Name))
}
@ -8893,6 +9039,10 @@ func (ec *executionContext) _ManifestSummary(ctx context.Context, sel ast.Select
out.Values[i] = ec._ManifestSummary_Vulnerabilities(ctx, field, obj)
case "Referrers":
out.Values[i] = ec._ManifestSummary_Referrers(ctx, field, obj)
default:
panic("unknown field " + strconv.Quote(field.Name))
}
@ -10777,6 +10927,47 @@ func (ec *executionContext) marshalOPlatform2ᚖzotregistryᚗioᚋzotᚋpkgᚋe
return ec._Platform(ctx, sel, v)
}
func (ec *executionContext) marshalOReferrer2ᚕᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐReferrer(ctx context.Context, sel ast.SelectionSet, v []*Referrer) graphql.Marshaler {
if v == nil {
return graphql.Null
}
ret := make(graphql.Array, len(v))
var wg sync.WaitGroup
isLen1 := len(v) == 1
if !isLen1 {
wg.Add(len(v))
}
for i := range v {
i := i
fc := &graphql.FieldContext{
Index: &i,
Result: &v[i],
}
ctx := graphql.WithFieldContext(ctx, fc)
f := func(i int) {
defer func() {
if r := recover(); r != nil {
ec.Error(ctx, ec.Recover(ctx, r))
ret = nil
}
}()
if !isLen1 {
defer wg.Done()
}
ret[i] = ec.marshalOReferrer2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐReferrer(ctx, sel, v[i])
}
if isLen1 {
f(i)
} else {
go f(i)
}
}
wg.Wait()
return ret
}
func (ec *executionContext) marshalOReferrer2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐReferrer(ctx context.Context, sel ast.SelectionSet, v *Referrer) graphql.Marshaler {
if v == nil {
return graphql.Null

4
pkg/extensions/search/gql_generated/models_gen.go
View file

@ -122,6 +122,8 @@ type ImageSummary struct {
Authors *string `json:"Authors"`
// Short summary of the identified CVEs
Vulnerabilities *ImageVulnerabilitySummary `json:"Vulnerabilities"`
// Information about objects that reference this image
Referrers []*Referrer `json:"Referrers"`
}
// Contains summary of vulnerabilities found in a specific image
@ -171,6 +173,8 @@ type ManifestSummary struct {
History []*LayerHistory `json:"History"`
// Short summary of the identified CVEs
Vulnerabilities *ImageVulnerabilitySummary `json:"Vulnerabilities"`
// Information about objects that reference this image
Referrers []*Referrer `json:"Referrers"`
}
// Contains the name of the package, the current installed version and the version where the CVE was fixed

2
pkg/extensions/search/resolver.go
View file

@ -1200,7 +1200,7 @@ func getReferrers(repoDB repodb.RepoDB, repo string, referredDigest string, arti
referredDigest, err)
}
referrers, err := repoDB.GetFilteredReferrersInfo(repo, refDigest, artifactTypes)
referrers, err := repoDB.GetReferrersInfo(repo, refDigest, artifactTypes)
if err != nil {
return nil, err
}

4
pkg/extensions/search/resolver_test.go
View file

@ -1374,7 +1374,7 @@ func TestGetReferrers(t *testing.T) {
Convey("GetReferrers returns error", func() {
testLogger := log.NewLogger("debug", "")
mockedStore := mocks.RepoDBMock{
GetFilteredReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string,
GetReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string,
) ([]repodb.ReferrerInfo, error) {
return nil, ErrTestError
},
@ -1396,7 +1396,7 @@ func TestGetReferrers(t *testing.T) {
},
}
mockedStore := mocks.RepoDBMock{
GetFilteredReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string,
GetReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string,
) ([]repodb.ReferrerInfo, error) {
return []repodb.ReferrerInfo{
{

8
pkg/extensions/search/schema.graphql
View file

@ -184,6 +184,10 @@ type ImageSummary {
Short summary of the identified CVEs
"""
Vulnerabilities: ImageVulnerabilitySummary
"""
Information about objects that reference this image
"""
Referrers: [Referrer]
}
"""
Details about a specific version of an image for a certain operating system and architecture.
@ -226,6 +230,10 @@ type ManifestSummary {
Short summary of the identified CVEs
"""
Vulnerabilities: ImageVulnerabilitySummary
"""
Information about objects that reference this image
"""
Referrers: [Referrer]
}
"""

132
pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go
View file

@ -138,7 +138,7 @@ func (bdw *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Diges
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{},
Signatures: map[string]repodb.ManifestSignatures{},
Referrers: map[string][]repodb.Descriptor{},
Referrers: map[string][]repodb.ReferrerInfo{},
}
repoMetaBlob := repoBuck.Get([]byte(repo))
@ -308,7 +308,7 @@ func (bdw DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.Art
return artifactData, err
}
func (bdw DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error {
func (bdw DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error {
err := bdw.DB.Update(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
@ -328,12 +328,9 @@ func (bdw DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, re
Signatures: map[string]repodb.ManifestSignatures{
referredDigest.String(): {},
},
Referrers: map[string][]repodb.Descriptor{
Referrers: map[string][]repodb.ReferrerInfo{
referredDigest.String(): {
{
Digest: referrer.Digest,
MediaType: referrer.MediaType,
},
referrer,
},
},
}
@ -360,10 +357,7 @@ func (bdw DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, re
}
}
refferers = append(refferers, repodb.Descriptor{
Digest: referrer.Digest,
MediaType: referrer.MediaType,
})
refferers = append(refferers, referrer)
repoMeta.Referrers[referredDigest.String()] = refferers
@ -418,8 +412,9 @@ func (bdw DBWrapper) DeleteReferrer(repo string, referredDigest godigest.Digest,
})
}
func (bdw DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) {
var referrers []repodb.Descriptor
func (bdw DBWrapper) GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string,
) ([]repodb.ReferrerInfo, error) {
referrersInfoResult := []repodb.ReferrerInfo{}
err := bdw.DB.View(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
@ -436,117 +431,20 @@ func (bdw DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) (
return err
}
referrers = repoMeta.Referrers[referredDigest.String()]
referrersInfo := repoMeta.Referrers[referredDigest.String()]
return nil
})
return referrers, err
}
func (bdw DBWrapper) GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest,
artifactTypes []string,
) ([]repodb.ReferrerInfo, error) {
referrersDescriptors, err := bdw.GetReferrers(repo, referredDigest)
if err != nil {
bdw.Log.Error().Msgf("repodb: failed to get referrers for '%s@%s'", repo, referredDigest.String())
return nil, err
}
referrersInfo := []repodb.ReferrerInfo{}
err = bdw.DB.View(func(tx *bolt.Tx) error {
artifactBuck := tx.Bucket([]byte(repodb.ArtifactDataBucket))
manifestBuck := tx.Bucket([]byte(repodb.ManifestDataBucket))
for _, descriptor := range referrersDescriptors {
referrerInfo := repodb.ReferrerInfo{}
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDataBlob := manifestBuck.Get([]byte(descriptor.Digest))
if len(manifestDataBlob) == 0 {
bdw.Log.Error().Msgf("repodb: manifest data not found for digest %s", descriptor.Digest)
continue
}
var manifestData repodb.ManifestData
err = json.Unmarshal(manifestDataBlob, &manifestData)
if err != nil {
bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal manifest data for digest %s",
descriptor.Digest)
continue
}
var manifestContent ispec.Manifest
err := json.Unmarshal(manifestData.ManifestBlob, &manifestContent)
if err != nil {
bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal manifest for digest %s",
descriptor.Digest)
continue
}
referrerInfo = repodb.ReferrerInfo{
Digest: descriptor.Digest,
MediaType: ispec.MediaTypeImageManifest,
ArtifactType: manifestContent.Config.MediaType,
Size: len(manifestData.ManifestBlob),
Annotations: manifestContent.Annotations,
}
case ispec.MediaTypeArtifactManifest:
artifactDataBlob := artifactBuck.Get([]byte(descriptor.Digest))
if len(artifactDataBlob) == 0 {
bdw.Log.Error().Msgf("repodb: artifact data not found for digest %s", descriptor.Digest)
continue
}
var artifactData repodb.ArtifactData
err = json.Unmarshal(artifactDataBlob, &artifactData)
if err != nil {
bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal artifact data for digest %s", descriptor.Digest)
continue
}
manifestContent := ispec.Artifact{}
err := json.Unmarshal(artifactData.ManifestBlob, &manifestContent)
if err != nil {
bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal artifact manifest for digest %s", descriptor.Digest)
continue
}
referrerInfo = repodb.ReferrerInfo{
Size: len(artifactData.ManifestBlob),
Digest: descriptor.Digest,
MediaType: manifestContent.MediaType,
Annotations: manifestContent.Annotations,
ArtifactType: manifestContent.ArtifactType,
}
}
if !common.MatchesArtifactTypes(referrerInfo.ArtifactType, artifactTypes) {
for i := range referrersInfo {
if !common.MatchesArtifactTypes(referrersInfo[i].ArtifactType, artifactTypes) {
continue
}
referrersInfo = append(referrersInfo, referrerInfo)
referrersInfoResult = append(referrersInfoResult, referrersInfo[i])
}
return nil
})
return referrersInfo, err
return referrersInfoResult, err
}
func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest,
@ -570,7 +468,7 @@ func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDi
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{},
Signatures: map[string]repodb.ManifestSignatures{},
Referrers: map[string][]repodb.Descriptor{},
Referrers: map[string][]repodb.ReferrerInfo{},
}
repoMetaBlob, err = json.Marshal(repoMeta)
@ -596,7 +494,7 @@ func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDi
repoMeta.Statistics[manifestDigest.String()] = repodb.DescriptorStatistics{DownloadCount: 0}
repoMeta.Signatures[manifestDigest.String()] = repodb.ManifestSignatures{}
repoMeta.Referrers[manifestDigest.String()] = []repodb.Descriptor{}
repoMeta.Referrers[manifestDigest.String()] = []repodb.ReferrerInfo{}
repoMetaBlob, err = json.Marshal(repoMeta)
if err != nil {

122
pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go
View file

@ -90,7 +90,7 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetReferrer("repo", "ref", repodb.Descriptor{})
err = boltdbWrapper.SetReferrer("repo", "ref", repodb.ReferrerInfo{})
So(err, ShouldNotBeNil)
})
@ -113,99 +113,6 @@ func TestWrapperErrors(t *testing.T) {
})
})
Convey("GetReferrers", func() {
Convey("RepoMeta not found", func() {
_, err := boltdbWrapper.GetReferrers("repo", "dig")
So(err, ShouldNotBeNil)
})
Convey("bad repo meta blob", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
return repoBuck.Put([]byte("repo"), []byte("wrong json"))
})
So(err, ShouldBeNil)
_, err = boltdbWrapper.GetReferrers("repo", "dig")
So(err, ShouldNotBeNil)
})
})
Convey("GetFilteredReferrersInfo", func() {
Convey("getReferrers fails", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
return repoBuck.Put([]byte("repo"), []byte("wrong json"))
})
So(err, ShouldBeNil)
_, err = boltdbWrapper.GetFilteredReferrersInfo("repo", "", nil)
So(err, ShouldNotBeNil)
})
Convey("unmarshal erorrs", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
manifestBuck := tx.Bucket([]byte(repodb.ManifestDataBucket))
artifactBuck := tx.Bucket([]byte(repodb.ArtifactDataBucket))
err = manifestBuck.Put([]byte("manifestDataRef"), []byte("bad json"))
So(err, ShouldBeNil)
err = artifactBuck.Put([]byte("artifactDataRef"), []byte("bad json"))
So(err, ShouldBeNil)
badBlob, err := json.Marshal(repodb.ArtifactData{
ManifestBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
err = artifactBuck.Put([]byte("artifactManifestRef"), badBlob)
So(err, ShouldBeNil)
badBlob, err = json.Marshal(repodb.ManifestData{
ManifestBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
err = manifestBuck.Put([]byte("badManifest"), badBlob)
So(err, ShouldBeNil)
return nil
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{
Digest: "manifestDataRef",
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{
Digest: "artifactDataRef",
MediaType: ispec.MediaTypeArtifactManifest,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{
Digest: "badManifest",
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{
Digest: "artifactManifestRef",
MediaType: ispec.MediaTypeArtifactManifest,
})
So(err, ShouldBeNil)
refInfo, err := boltdbWrapper.GetFilteredReferrersInfo("repo", "refDigest", nil)
So(err, ShouldBeNil)
So(len(refInfo), ShouldEqual, 0)
})
})
Convey("SetRepoReference", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
@ -218,6 +125,18 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("GetRepoMeta", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
return repoBuck.Put([]byte("repo1"), []byte("wrong json"))
})
So(err, ShouldBeNil)
_, err = boltdbWrapper.GetRepoMeta("repo1")
So(err, ShouldNotBeNil)
})
Convey("DeleteRepoTag", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
@ -230,6 +149,21 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("GetReferrersInfo", func() {
_, err = boltdbWrapper.GetReferrersInfo("repo1", "tag", nil)
So(err, ShouldNotBeNil)
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
return repoBuck.Put([]byte("repo1"), []byte("wrong json"))
})
So(err, ShouldBeNil)
_, err = boltdbWrapper.GetReferrersInfo("repo1", "tag", nil)
So(err, ShouldNotBeNil)
})
Convey("IncrementRepoStars", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))

24
pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go
View file

@ -244,7 +244,7 @@ func TestWrapperErrors(t *testing.T) {
Convey("SetReferrer client error", func() {
dynamoWrapper.RepoMetaTablename = badTablename
err := dynamoWrapper.SetReferrer("repo", "", repodb.Descriptor{})
err := dynamoWrapper.SetReferrer("repo", "", repodb.ReferrerInfo{})
So(err, ShouldNotBeNil)
})
@ -252,7 +252,7 @@ func TestWrapperErrors(t *testing.T) {
err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo")
So(err, ShouldBeNil)
err = dynamoWrapper.SetReferrer("repo", "", repodb.Descriptor{})
err = dynamoWrapper.SetReferrer("repo", "", repodb.ReferrerInfo{})
So(err, ShouldNotBeNil)
})
@ -284,32 +284,32 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("GetFilteredReferrersInfo GetReferrers errors", func() {
Convey("GetReferrersInfo GetReferrers errors", func() {
dynamoWrapper.RepoMetaTablename = badTablename
_, err := dynamoWrapper.GetFilteredReferrersInfo("repo", "", nil)
_, err := dynamoWrapper.GetReferrersInfo("repo", "", nil)
So(err, ShouldNotBeNil)
})
Convey("GetFilteredReferrersInfo getData fails", func() {
Convey("GetReferrersInfo getData fails", func() {
dynamoWrapper.ManifestDataTablename = badTablename
dynamoWrapper.ArtifactDataTablename = badTablename
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.ReferrerInfo{
Digest: "dig1",
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.ReferrerInfo{
Digest: "dig2",
MediaType: ispec.MediaTypeArtifactManifest,
})
So(err, ShouldBeNil)
_, err := dynamoWrapper.GetFilteredReferrersInfo("repo", "rf", nil)
_, err := dynamoWrapper.GetReferrersInfo("repo", "rf", nil)
So(err, ShouldBeNil)
})
Convey("GetFilteredReferrersInfo bad descriptor blob", func() {
Convey("GetReferrersInfo bad descriptor blob", func() {
err = dynamoWrapper.SetArtifactData("dig2", repodb.ArtifactData{
ManifestBlob: []byte("bad json"),
})
@ -320,19 +320,19 @@ func TestWrapperErrors(t *testing.T) {
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.ReferrerInfo{
Digest: "dig2",
MediaType: ispec.MediaTypeArtifactManifest,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.ReferrerInfo{
Digest: "dig3",
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
_, err := dynamoWrapper.GetFilteredReferrersInfo("repo", "rf", nil)
_, err := dynamoWrapper.GetReferrersInfo("repo", "rf", nil)
So(err, ShouldBeNil)
})

86
pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go
View file

@ -168,7 +168,7 @@ func (dwr *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Diges
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{},
Signatures: map[string]repodb.ManifestSignatures{},
Referrers: map[string][]repodb.Descriptor{},
Referrers: map[string][]repodb.ReferrerInfo{},
}
}
@ -368,7 +368,7 @@ func (dwr DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.Art
return artifactData, nil
}
func (dwr DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error {
func (dwr DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error {
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
TableName: aws.String(dwr.RepoMetaTablename),
Key: map[string]types.AttributeValue{
@ -384,7 +384,7 @@ func (dwr DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, re
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{},
Signatures: map[string]repodb.ManifestSignatures{},
Referrers: map[string][]repodb.Descriptor{},
Referrers: map[string][]repodb.ReferrerInfo{},
}
if resp.Item != nil {
@ -409,7 +409,7 @@ func (dwr DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, re
return dwr.setRepoMeta(repo, repoMeta)
}
func (dwr DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) {
func (dwr DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.ReferrerInfo, error) {
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
TableName: aws.String(dwr.RepoMetaTablename),
Key: map[string]types.AttributeValue{
@ -417,7 +417,7 @@ func (dwr DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) (
},
})
if err != nil {
return []repodb.Descriptor{}, err
return []repodb.ReferrerInfo{}, err
}
repoMeta := repodb.RepoMetadata{
@ -425,13 +425,13 @@ func (dwr DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) (
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{},
Signatures: map[string]repodb.ManifestSignatures{},
Referrers: map[string][]repodb.Descriptor{},
Referrers: map[string][]repodb.ReferrerInfo{},
}
if resp.Item != nil {
err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta)
if err != nil {
return []repodb.Descriptor{}, err
return []repodb.ReferrerInfo{}, err
}
}
@ -456,7 +456,7 @@ func (dwr DBWrapper) DeleteReferrer(repo string, referredDigest godigest.Digest,
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{},
Signatures: map[string]repodb.ManifestSignatures{},
Referrers: map[string][]repodb.Descriptor{},
Referrers: map[string][]repodb.ReferrerInfo{},
}
if resp.Item != nil {
@ -481,79 +481,25 @@ func (dwr DBWrapper) DeleteReferrer(repo string, referredDigest godigest.Digest,
return dwr.setRepoMeta(repo, repoMeta)
}
func (dwr DBWrapper) GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest,
func (dwr DBWrapper) GetReferrersInfo(repo string, referredDigest godigest.Digest,
artifactTypes []string,
) ([]repodb.ReferrerInfo, error) {
referrersDescriptors, err := dwr.GetReferrers(repo, referredDigest)
referrersInfo, err := dwr.GetReferrers(repo, referredDigest)
if err != nil {
return nil, err
}
referrersInfo := []repodb.ReferrerInfo{}
for _, descriptor := range referrersDescriptors {
referrerInfo := repodb.ReferrerInfo{}
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestData, err := dwr.GetManifestData(godigest.Digest(descriptor.Digest))
if err != nil {
dwr.Log.Error().Msgf("repodb: manifest data not found for digest %s", descriptor.Digest)
continue
}
var manifestContent ispec.Manifest
err = json.Unmarshal(manifestData.ManifestBlob, &manifestContent)
if err != nil {
dwr.Log.Error().Err(err).Msgf("repodb: can't unmarhsal manifest for digest %s",
descriptor.Digest)
continue
}
referrerInfo = repodb.ReferrerInfo{
Digest: descriptor.Digest,
MediaType: ispec.MediaTypeImageManifest,
ArtifactType: manifestContent.Config.MediaType,
Size: len(manifestData.ManifestBlob),
Annotations: manifestContent.Annotations,
}
case ispec.MediaTypeArtifactManifest:
artifactData, err := dwr.GetArtifactData(godigest.Digest(descriptor.Digest))
if err != nil {
dwr.Log.Error().Msgf("repodb: artifact data not found for digest %s", descriptor.Digest)
continue
}
manifestContent := ispec.Artifact{}
err = json.Unmarshal(artifactData.ManifestBlob, &manifestContent)
if err != nil {
dwr.Log.Error().Err(err).Msgf("repodb: can't unmarhsal artifact manifest for digest %s", descriptor.Digest)
continue
}
referrerInfo = repodb.ReferrerInfo{
Digest: descriptor.Digest,
MediaType: manifestContent.MediaType,
ArtifactType: manifestContent.ArtifactType,
Size: len(artifactData.ManifestBlob),
Annotations: manifestContent.Annotations,
}
}
filteredResults := make([]repodb.ReferrerInfo, 0, len(referrersInfo))
for _, referrerInfo := range referrersInfo {
if !common.MatchesArtifactTypes(referrerInfo.ArtifactType, artifactTypes) {
continue
}
referrersInfo = append(referrersInfo, referrerInfo)
filteredResults = append(filteredResults, referrerInfo)
}
return referrersInfo, nil
return filteredResults, nil
}
func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest,
@ -578,7 +524,7 @@ func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDi
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{},
Signatures: map[string]repodb.ManifestSignatures{},
Referrers: map[string][]repodb.Descriptor{},
Referrers: map[string][]repodb.ReferrerInfo{},
}
if resp.Item != nil {
@ -597,7 +543,7 @@ func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDi
repoMeta.Statistics[manifestDigest.String()] = repodb.DescriptorStatistics{DownloadCount: 0}
repoMeta.Signatures[manifestDigest.String()] = repodb.ManifestSignatures{}
repoMeta.Referrers[manifestDigest.String()] = []repodb.Descriptor{}
repoMeta.Referrers[manifestDigest.String()] = []repodb.ReferrerInfo{}
err = dwr.setRepoMeta(repo, repoMeta)

65
pkg/meta/repodb/load_repodb.go
View file

@ -3,6 +3,7 @@ package repodb
import (
"encoding/json"
"errors"
"fmt"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
@ -305,11 +306,9 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig
}
}
if refferredDigest, hasSubject := GetReferredSubject(descriptorBlob); hasSubject {
err := repoDB.SetReferrer(repo, refferredDigest, Descriptor{
Digest: digest.String(),
MediaType: mediaType,
})
refferredDigest, referrerInfo, hasSubject, err := GetReferredSubject(descriptorBlob, digest.String(), mediaType)
if hasSubject && err == nil {
err := repoDB.SetReferrer(repo, refferredDigest, referrerInfo)
if err != nil {
log.Error().Err(err).Msg("repodb: error while settingg referrer")
@ -317,7 +316,7 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig
}
}
err := repoDB.SetRepoReference(repo, reference, digest, mediaType)
err = repoDB.SetRepoReference(repo, reference, digest, mediaType)
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting repo meta")
@ -327,17 +326,55 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig
return nil
}
func GetReferredSubject(descriptorBlob []byte) (godigest.Digest, bool) {
var manifest ispec.Manifest
func GetReferredSubject(descriptorBlob []byte, referrerDigest, mediaType string,
) (godigest.Digest, ReferrerInfo, bool, error) {
var (
referrerInfo ReferrerInfo
referrerSubject *ispec.Descriptor
)
err := json.Unmarshal(descriptorBlob, &manifest)
if err != nil {
return "", false
switch mediaType {
case ispec.MediaTypeImageManifest:
var manifestContent ispec.Manifest
err := json.Unmarshal(descriptorBlob, &manifestContent)
if err != nil {
return "", referrerInfo, false,
fmt.Errorf("repodb: can't unmarhsal manifest for digest %s: %w", referrerDigest, err)
}
referrerSubject = manifestContent.Subject
referrerInfo = ReferrerInfo{
Digest: referrerDigest,
MediaType: mediaType,
ArtifactType: manifestContent.Config.MediaType,
Size: len(descriptorBlob),
Annotations: manifestContent.Annotations,
}
case ispec.MediaTypeArtifactManifest:
manifestContent := ispec.Artifact{}
err := json.Unmarshal(descriptorBlob, &manifestContent)
if err != nil {
return "", referrerInfo, false,
fmt.Errorf("repodb: can't unmarhsal artifact manifest for digest %s: %w", referrerDigest, err)
}
referrerSubject = manifestContent.Subject
referrerInfo = ReferrerInfo{
Digest: referrerDigest,
MediaType: manifestContent.MediaType,
ArtifactType: manifestContent.ArtifactType,
Size: len(descriptorBlob),
Annotations: manifestContent.Annotations,
}
}
if manifest.Subject == nil || manifest.Subject.Digest.String() == "" {
return "", false
if referrerSubject == nil || referrerSubject.Digest.String() == "" {
return "", ReferrerInfo{}, false, nil
}
return manifest.Subject.Digest, true
return referrerSubject.Digest, referrerInfo, true, nil
}

5
pkg/meta/repodb/load_repodb_test.go
View file

@ -645,7 +645,10 @@ func TestLoadOCILayoutDynamoWrapper(t *testing.T) {
func TestGetReferredSubject(t *testing.T) {
Convey("GetReferredSubject error", t, func() {
_, err := repodb.GetReferredSubject([]byte("bad json"))
_, _, _, err := repodb.GetReferredSubject([]byte("bad json"), "digest", ispec.MediaTypeArtifactManifest)
So(err, ShouldNotBeNil)
_, _, _, err = repodb.GetReferredSubject([]byte("bad json"), "digest", ispec.MediaTypeImageManifest)
So(err, ShouldNotBeNil)
})
}

11
pkg/meta/repodb/repodb.go
View file

@ -75,17 +75,14 @@ type RepoDB interface { //nolint:interfacebloat
GetArtifactData(artifactDigest godigest.Digest) (ArtifactData, error)
// SetReferrer adds a referrer to the referrers list of a manifest inside a repo
SetReferrer(repo string, referredDigest godigest.Digest, referrer Descriptor) error
SetReferrer(repo string, referredDigest godigest.Digest, referrer ReferrerInfo) error
// SetReferrer delets a referrer to the referrers list of a manifest inside a repo
DeleteReferrer(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest) error
// GetReferrers returns the list of referrers for a referred manifest
GetReferrers(repo string, referredDigest godigest.Digest) ([]Descriptor, error)
// GetFilteredReferrersInfo returnes a list of for all referrers of the given digest that match one of the
// GetReferrersInfo returnes a list of for all referrers of the given digest that match one of the
// artifact types.
GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string) (
GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string) (
[]ReferrerInfo, error)
// IncrementManifestDownloads adds 1 to the download count of a manifest
@ -158,7 +155,7 @@ type RepoMetadata struct {
Statistics map[string]DescriptorStatistics
Signatures map[string]ManifestSignatures
Referrers map[string][]Descriptor
Referrers map[string][]ReferrerInfo
Stars int
}

40
pkg/meta/repodb/repodb_test.go
View file

@ -1816,7 +1816,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
artifactDigest1, err := artifact1.Digest()
So(err, ShouldBeNil)
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{
Digest: artifactDigest1.String(),
MediaType: ispec.MediaTypeImageManifest,
})
@ -1833,7 +1833,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
artifactDigest2, err := artifact2.Digest()
So(err, ShouldBeNil)
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{
Digest: artifactDigest2.String(),
MediaType: ispec.MediaTypeArtifactManifest,
})
@ -1841,13 +1841,13 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
// ------ GetReferrers
referrers, err := repoDB.GetReferrers("repo", referredDigest)
referrers, err := repoDB.GetReferrersInfo("repo", referredDigest, nil)
So(len(referrers), ShouldEqual, 2)
So(referrers, ShouldContain, repodb.Descriptor{
So(referrers, ShouldContain, repodb.ReferrerInfo{
Digest: artifactDigest1.String(),
MediaType: ispec.MediaTypeImageManifest,
})
So(referrers, ShouldContain, repodb.Descriptor{
So(referrers, ShouldContain, repodb.ReferrerInfo{
Digest: artifactDigest2.String(),
MediaType: ispec.MediaTypeArtifactManifest,
})
@ -1861,7 +1861,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
err = repoDB.DeleteReferrer("repo", referredDigest, artifactDigest2)
So(err, ShouldBeNil)
referrers, err = repoDB.GetReferrers("repo", referredDigest)
referrers, err = repoDB.GetReferrersInfo("repo", referredDigest, nil)
So(err, ShouldBeNil)
So(len(referrers), ShouldEqual, 0)
})
@ -1874,7 +1874,7 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
referredDigest := godigest.FromString("referredDigest")
referrerDigest := godigest.FromString("referrerDigest")
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{
Digest: referrerDigest.String(),
MediaType: ispec.MediaTypeImageManifest,
})
@ -1893,13 +1893,13 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
referredDigest := godigest.FromString("referredDigest")
referrerDigest := godigest.FromString("referrerDigest")
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{
Digest: referrerDigest.String(),
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{
Digest: referrerDigest.String(),
MediaType: ispec.MediaTypeImageManifest,
})
@ -1910,16 +1910,16 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
So(len(repoMeta.Referrers[referredDigest.String()]), ShouldEqual, 1)
})
Convey("GetFilteredReferrersInfo", func() {
Convey("GetReferrersInfo", func() {
referredDigest := godigest.FromString("referredDigest")
err := repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
err := repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{
Digest: "inexistendManifestDigest",
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{
Digest: "inexistendArtifactManifestDigest",
MediaType: ispec.MediaTypeArtifactManifest,
})
@ -1932,9 +1932,10 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
})
So(err, ShouldBeNil)
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
Digest: "goodManifest",
MediaType: ispec.MediaTypeImageManifest,
err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{
Digest: "goodManifest",
MediaType: ispec.MediaTypeImageManifest,
ArtifactType: "unwantedType",
})
So(err, ShouldBeNil)
@ -1943,13 +1944,14 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
})
So(err, ShouldBeNil)
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
Digest: "goodArtifact",
MediaType: ispec.MediaTypeArtifactManifest,
err = repoDB.SetReferrer("repo", referredDigest, repodb.ReferrerInfo{
Digest: "goodArtifact",
MediaType: ispec.MediaTypeArtifactManifest,
ArtifactType: "wantedType",
})
So(err, ShouldBeNil)
referrerInfo, err := repoDB.GetFilteredReferrersInfo("repo", referredDigest, []string{"wantedType"})
referrerInfo, err := repoDB.GetReferrersInfo("repo", referredDigest, []string{"wantedType"})
So(err, ShouldBeNil)
So(len(referrerInfo), ShouldEqual, 1)
So(referrerInfo[0].ArtifactType, ShouldResemble, "wantedType")

2
pkg/meta/repodb/update/update_test.go
View file

@ -269,7 +269,7 @@ func TestUpdateErrors(t *testing.T) {
log := log.NewLogger("debug", "")
repoDB := mocks.RepoDBMock{
SetReferrerFn: func(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error {
SetReferrerFn: func(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error {
return ErrTestError
},
}

20
pkg/test/mocks/repo_db_mock.go
View file

@ -44,13 +44,13 @@ type RepoDBMock struct {
GetArtifactDataFn func(artifactDigest godigest.Digest) (repodb.ArtifactData, error)
SetReferrerFn func(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error
SetReferrerFn func(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error
DeleteReferrerFn func(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest) error
GetReferrersFn func(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error)
GetFilteredReferrersInfoFn func(repo string, referredDigest godigest.Digest, artifactTypes []string) (
GetReferrersInfoFn func(repo string, referredDigest godigest.Digest, artifactTypes []string) (
[]repodb.ReferrerInfo, error)
IncrementImageDownloadsFn func(repo string, reference string) error
@ -322,7 +322,7 @@ func (sdm RepoDBMock) GetArtifactData(artifactDigest godigest.Digest) (repodb.Ar
return repodb.ArtifactData{}, nil
}
func (sdm RepoDBMock) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error {
func (sdm RepoDBMock) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.ReferrerInfo) error {
if sdm.SetReferrerFn != nil {
return sdm.SetReferrerFn(repo, referredDigest, referrer)
}
@ -340,19 +340,11 @@ func (sdm RepoDBMock) DeleteReferrer(repo string, referredDigest godigest.Digest
return nil
}
func (sdm RepoDBMock) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) {
if sdm.GetReferrersFn != nil {
return sdm.GetReferrersFn(repo, referredDigest)
}
return []repodb.Descriptor{}, nil
}
func (sdm RepoDBMock) GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest,
func (sdm RepoDBMock) GetReferrersInfo(repo string, referredDigest godigest.Digest,
artifactTypes []string,
) ([]repodb.ReferrerInfo, error) {
if sdm.GetFilteredReferrersInfoFn != nil {
return sdm.GetFilteredReferrersInfoFn(repo, referredDigest, artifactTypes)
if sdm.GetReferrersInfoFn != nil {
return sdm.GetReferrersInfoFn(repo, referredDigest, artifactTypes)
}
return []repodb.ReferrerInfo{}, nil

24
test/blackbox/referrers.bats
View file

@ -7,7 +7,7 @@ function setup() {
fi
# Download test data to folder common for the entire suite, not just this file
skopeo --insecure-policy copy --format=oci docker://alpine:latest oci:${TEST_DATA_DIR}alpine:latest
skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20
# Setup zot server
ZOT_ROOT_DIR=${BATS_FILE_TMPDIR}/zot
@ -60,23 +60,23 @@ EOF
wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
run skopeo --insecure-policy copy --dest-tls-verify=false \
oci:${TEST_DATA_DIR}/alpine:latest \
docker://127.0.0.1:8080/alpine:latest
oci:${TEST_DATA_DIR}/golang:1.20 \
docker://127.0.0.1:8080/golang:1.20
[ "$status" -eq 0 ]
run curl http://127.0.0.1:8080/v2/_catalog
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.repositories[]') = '"alpine"' ]
[ $(echo "${lines[-1]}" | jq '.repositories[]') = '"golang"' ]
run oras attach --plain-http --image-spec v1.1-image --artifact-type image.type 127.0.0.1:8080/alpine:latest ${IMAGE_MANIFEST_REFERRER}
run oras attach --plain-http --image-spec v1.1-image --artifact-type image.type 127.0.0.1:8080/golang:1.20 ${IMAGE_MANIFEST_REFERRER}
[ "$status" -eq 0 ]
run oras attach --plain-http --image-spec v1.1-artifact --artifact-type artifact.type 127.0.0.1:8080/alpine:latest ${ARTIFACT_MANIFEST_REFERRER}
run oras attach --plain-http --image-spec v1.1-artifact --artifact-type artifact.type 127.0.0.1:8080/golang:1.20 ${ARTIFACT_MANIFEST_REFERRER}
[ "$status" -eq 0 ]
MANIFEST_DIGEST=$(skopeo inspect --tls-verify=false docker://localhost:8080/alpine:latest | jq -r '.Digest')
MANIFEST_DIGEST=$(skopeo inspect --tls-verify=false docker://localhost:8080/golang:1.20 | jq -r '.Digest')
echo ${MANIFEST_DIGEST}
curl -X GET http://127.0.0.1:8080/v2/alpine/referrers/${MANIFEST_DIGEST}?artifactType=image.type
curl -X GET http://127.0.0.1:8080/v2/golang/referrers/${MANIFEST_DIGEST}?artifactType=image.type
}
function teardown() {
@ -88,21 +88,21 @@ function teardown() {
@test "add referrers, one artifact and one image" {
# Check referrers API using the normal REST endpoint
run curl -X GET http://127.0.0.1:8080/v2/alpine/referrers/${MANIFEST_DIGEST}?artifactType=image.type
run curl -X GET http://127.0.0.1:8080/v2/golang/referrers/${MANIFEST_DIGEST}?artifactType=image.type
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.manifests[].artifactType') = '"image.type"' ]
run curl -X GET http://127.0.0.1:8080/v2/alpine/referrers/${MANIFEST_DIGEST}?artifactType=artifact.type
run curl -X GET http://127.0.0.1:8080/v2/golang/referrers/${MANIFEST_DIGEST}?artifactType=artifact.type
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.manifests[].artifactType') = '"artifact.type"' ]
# Check referrers API using the GQL endpoint
REFERRER_QUERY_DATA="{ \"query\": \"{ Referrers(repo:\\\"alpine\\\", digest:\\\"${MANIFEST_DIGEST}\\\", type:[\\\"image.type\\\"]) { MediaType ArtifactType Digest Size} }\"}"
REFERRER_QUERY_DATA="{ \"query\": \"{ Referrers(repo:\\\"golang\\\", digest:\\\"${MANIFEST_DIGEST}\\\", type:[\\\"image.type\\\"]) { MediaType ArtifactType Digest Size} }\"}"
run curl -X POST -H "Content-Type: application/json" --data "${REFERRER_QUERY_DATA}" http://localhost:8080/v2/_zot/ext/search
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.data.Referrers[].ArtifactType') = '"image.type"' ]
REFERRER_QUERY_DATA="{ \"query\": \"{ Referrers(repo:\\\"alpine\\\", digest:\\\"${MANIFEST_DIGEST}\\\", type:[\\\"artifact.type\\\"]) { MediaType ArtifactType Digest Size} }\"}"
REFERRER_QUERY_DATA="{ \"query\": \"{ Referrers(repo:\\\"golang\\\", digest:\\\"${MANIFEST_DIGEST}\\\", type:[\\\"artifact.type\\\"]) { MediaType ArtifactType Digest Size} }\"}"
run curl -X POST -H "Content-Type: application/json" --data "${REFERRER_QUERY_DATA}" http://localhost:8080/v2/_zot/ext/search
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.data.Referrers[].ArtifactType') = '"artifact.type"' ]