Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2025-03-18 02:22:53 -05:00
Code Issues Activity

feat(referrers): added index support for referrers queries (#1560)

Browse source 
Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae 2023-07-05 19:42:16 +03:00 • committed by GitHub
parent 62889c3cb1
commit 96d9d318df
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 236 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
go.mod
View file

@ -29,7 +29,7 @@ require (
github.com/nmcclain/ldap v0.0.0-20210720162743-7f8d1e44eeba
github.com/olekukonko/tablewriter v0.0.5
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.0-rc3.0.20230608172643-f8b2ca8c6ba1
github.com/opencontainers/image-spec v1.1.0-rc4
github.com/opencontainers/umoci v0.4.8-0.20210922062158-e60a0cc726e6
github.com/oras-project/artifacts-spec v1.0.0-rc.2
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
@ -58,7 +58,7 @@ require (
github.com/sigstore/cosign/v2 v2.0.2
github.com/swaggo/http-swagger v1.3.4
modernc.org/sqlite v1.23.1
oras.land/oras-go/v2 v2.2.0
oras.land/oras-go/v2 v2.2.1
)
require (
@ -453,7 +453,7 @@ require (
golang.org/x/mod v0.10.0 // indirect
golang.org/x/net v0.11.0 // indirect
golang.org/x/oauth2 v0.9.0 // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.9.0 // indirect
golang.org/x/term v0.9.0 // indirect
golang.org/x/text v0.10.0 // indirect

12
go.sum
View file

@ -1333,8 +1333,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/image-spec v1.1.0-rc3.0.20230608172643-f8b2ca8c6ba1 h1:d11iHLE/a8/B4Cnn8E8C+3ozjx444ed7AsOgXAG9coU=
github.com/opencontainers/image-spec v1.1.0-rc3.0.20230608172643-f8b2ca8c6ba1/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0=
github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0=
github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs=
github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg=
@ -1918,8 +1918,8 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -2466,8 +2466,8 @@ modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY=
oras.land/oras-go v1.2.3 h1:v8PJl+gEAntI1pJ/LCrDgsuk+1PKVavVEPsYIHFE5uY=
oras.land/oras-go v1.2.3/go.mod h1:M/uaPdYklze0Vf3AakfarnpoEckvw0ESbRdN8Z1vdJg=
oras.land/oras-go/v2 v2.2.0 h1:E1fqITD56Eg5neZbxBtAdZVgDHD6wBabJo6xESTcQyo=
oras.land/oras-go/v2 v2.2.0/go.mod h1:pXjn0+KfarspMHHNR3A56j3tgvr+mxArHuI8qVn59v8=
oras.land/oras-go/v2 v2.2.1 h1:3VJTYqy5KfelEF9c2jo1MLSpr+TM3mX8K42wzZcd6qE=
oras.land/oras-go/v2 v2.2.1/go.mod h1:GeAwLuC4G/JpNwkd+bSZ6SkDMGaaYglt6YK2WvZP7uQ=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

4
pkg/common/oci.go
View file

@ -58,6 +58,10 @@ func GetManifestArtifactType(manifestContent ispec.Manifest) string {
return manifestContent.Config.MediaType
}
func GetIndexArtifactType(indexContent ispec.Index) string {
return indexContent.ArtifactType
}
// GetImageLastUpdated This method will return last updated timestamp.
// The Created timestamp is used, but if it is missing, look at the
// history field and, if provided, return the timestamp of last entry in history.

101
pkg/extensions/search/search_test.go
View file

@ -1064,6 +1064,107 @@ func TestGetReferrersGQL(t *testing.T) {
So(referrersResp.Referrers[0].Digest, ShouldEqual, artifactManifestDigest)
})
Convey("Get referrers with index as referrer", t, func() {
port := GetFreePort()
baseURL := GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
conf.Storage.RootDirectory = t.TempDir()
conf.Storage.GC = false
defaultVal := true
conf.Extensions = &extconf.ExtensionConfig{
Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}},
Lint: &extconf.LintConfig{
BaseConfig: extconf.BaseConfig{
Enable: &defaultVal,
},
},
}
conf.Extensions.Search.CVE = nil
ctlr := api.NewController(conf)
ctlrManager := NewControllerManager(ctlr)
ctlrManager.StartAndWait(port)
defer ctlrManager.StopServer()
// Upload the index referrer
targetImg, err := GetRandomImage("")
So(err, ShouldBeNil)
targetDigest, err := targetImg.Digest()
So(err, ShouldBeNil)
err = UploadImage(targetImg, baseURL, "repo")
So(err, ShouldBeNil)
indexReferrer, err := GetRandomMultiarchImage("ref")
So(err, ShouldBeNil)
artifactType := "art.type"
indexReferrer.Index.ArtifactType = artifactType
indexReferrer.Index.Subject = &ispec.Descriptor{
MediaType: ispec.MediaTypeImageManifest,
Digest: targetDigest,
}
indexReferrerDigest, err := indexReferrer.Digest()
So(err, ShouldBeNil)
err = UploadMultiarchImage(indexReferrer, baseURL, "repo")
So(err, ShouldBeNil)
// Call Referrers GQL
referrersQuery := `
{
Referrers( repo: "%s", digest: "%s"){
ArtifactType,
Digest,
MediaType,
Size,
Annotations{
Key
Value
}
}
}`
referrersQuery = fmt.Sprintf(referrersQuery, "repo", targetDigest.String())
resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(referrersQuery))
So(resp, ShouldNotBeNil)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
So(resp.Body(), ShouldNotBeNil)
So(err, ShouldBeNil)
referrersResp := &zcommon.ReferrersResp{}
err = json.Unmarshal(resp.Body(), referrersResp)
So(err, ShouldBeNil)
So(len(referrersResp.Referrers), ShouldEqual, 1)
So(referrersResp.Referrers[0].ArtifactType, ShouldResemble, artifactType)
So(referrersResp.Referrers[0].Digest, ShouldResemble, indexReferrerDigest.String())
So(referrersResp.Referrers[0].MediaType, ShouldResemble, ispec.MediaTypeImageIndex)
// Make REST call
resp, err = resty.R().Get(baseURL + "/v2/repo/referrers/" + targetDigest.String())
So(err, ShouldBeNil)
var index ispec.Index
err = json.Unmarshal(resp.Body(), &index)
So(err, ShouldBeNil)
So(len(index.Manifests), ShouldEqual, 1)
So(index.Manifests[0].ArtifactType, ShouldEqual, artifactType)
So(index.Manifests[0].Digest.String(), ShouldResemble, indexReferrerDigest.String())
So(index.Manifests[0].MediaType, ShouldResemble, ispec.MediaTypeImageIndex)
})
}
func TestExpandedRepoInfo(t *testing.T) {

47
pkg/meta/repodb/storage_parsing.go
View file

@ -407,22 +407,43 @@ func GetReferredSubject(descriptorBlob []byte, referrerDigest, mediaType string,
referrerSubject *ispec.Descriptor
)
var manifestContent ispec.Manifest
switch mediaType {
case ispec.MediaTypeImageManifest:
var manifestContent ispec.Manifest
err := json.Unmarshal(descriptorBlob, &manifestContent)
if err != nil {
return "", referrerInfo, false,
fmt.Errorf("repodb: can't unmarshal manifest for digest %s: %w", referrerDigest, err)
}
err := json.Unmarshal(descriptorBlob, &manifestContent)
if err != nil {
return "", referrerInfo, false,
fmt.Errorf("repodb: can't unmarshal manifest for digest %s: %w", referrerDigest, err)
}
referrerSubject = manifestContent.Subject
referrerSubject = manifestContent.Subject
referrerInfo = ReferrerInfo{
Digest: referrerDigest,
MediaType: mediaType,
ArtifactType: zcommon.GetManifestArtifactType(manifestContent),
Size: len(descriptorBlob),
Annotations: manifestContent.Annotations,
referrerInfo = ReferrerInfo{
Digest: referrerDigest,
MediaType: mediaType,
ArtifactType: zcommon.GetManifestArtifactType(manifestContent),
Size: len(descriptorBlob),
Annotations: manifestContent.Annotations,
}
case ispec.MediaTypeImageIndex:
var indexContent ispec.Index
err := json.Unmarshal(descriptorBlob, &indexContent)
if err != nil {
return "", referrerInfo, false,
fmt.Errorf("repodb: can't unmarshal manifest for digest %s: %w", referrerDigest, err)
}
referrerSubject = indexContent.Subject
referrerInfo = ReferrerInfo{
Digest: referrerDigest,
MediaType: mediaType,
ArtifactType: zcommon.GetIndexArtifactType(indexContent),
Size: len(descriptorBlob),
Annotations: indexContent.Annotations,
}
}
if referrerSubject == nil || referrerSubject.Digest.String() == "" {

3
pkg/meta/repodb/storage_parsing_test.go
View file

@ -621,6 +621,9 @@ func TestGetReferredSubject(t *testing.T) {
Convey("GetReferredSubject error", t, func() {
_, _, _, err := repodb.GetReferredSubject([]byte("bad json"), "digest", ispec.MediaTypeImageManifest)
So(err, ShouldNotBeNil)
_, _, _, err = repodb.GetReferredSubject([]byte("bad json"), "digest", ispec.MediaTypeImageIndex)
So(err, ShouldNotBeNil)
})
}

56
pkg/storage/common/common.go
View file

@ -581,14 +581,14 @@ func GetReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godiges
result := []ispec.Descriptor{}
for _, manifest := range index.Manifests {
if manifest.Digest == gdigest {
for _, descriptor := range index.Manifests {
if descriptor.Digest == gdigest {
continue
}
buf, err := imgStore.GetBlobContent(repo, manifest.Digest)
buf, err := imgStore.GetBlobContent(repo, descriptor.Digest)
if err != nil {
log.Error().Err(err).Str("blob", imgStore.BlobPath(repo, manifest.Digest)).Msg("failed to read manifest")
log.Error().Err(err).Str("blob", imgStore.BlobPath(repo, descriptor.Digest)).Msg("failed to read manifest")
if errors.Is(err, zerr.ErrBlobNotFound) {
return nilIndex, zerr.ErrManifestNotFound
@ -597,31 +597,59 @@ func GetReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godiges
return nilIndex, err
}
if manifest.MediaType == ispec.MediaTypeImageManifest {
var mfst ispec.Manifest
if err := json.Unmarshal(buf, &mfst); err != nil {
log.Error().Err(err).Str("manifest digest", manifest.Digest.String()).Msg("invalid JSON")
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
var manifestContent ispec.Manifest
if err := json.Unmarshal(buf, &manifestContent); err != nil {
log.Error().Err(err).Str("manifest digest", descriptor.Digest.String()).Msg("invalid JSON")
return nilIndex, err
}
if mfst.Subject == nil || mfst.Subject.Digest != gdigest {
if manifestContent.Subject == nil || manifestContent.Subject.Digest != gdigest {
continue
}
// filter by artifact type
manifestArtifactType := zcommon.GetManifestArtifactType(mfst)
manifestArtifactType := zcommon.GetManifestArtifactType(manifestContent)
if len(artifactTypes) > 0 && !zcommon.Contains(artifactTypes, manifestArtifactType) {
continue
}
result = append(result, ispec.Descriptor{
MediaType: manifest.MediaType,
MediaType: descriptor.MediaType,
ArtifactType: manifestArtifactType,
Size: manifest.Size,
Digest: manifest.Digest,
Annotations: mfst.Annotations,
Size: descriptor.Size,
Digest: descriptor.Digest,
Annotations: manifestContent.Annotations,
})
case ispec.MediaTypeImageIndex:
var indexContent ispec.Index
if err := json.Unmarshal(buf, &indexContent); err != nil {
log.Error().Err(err).Str("manifest digest", descriptor.Digest.String()).Msg("invalid JSON")
return nilIndex, err
}
if indexContent.Subject == nil || indexContent.Subject.Digest != gdigest {
continue
}
indexArtifactType := zcommon.GetIndexArtifactType(indexContent)
if len(artifactTypes) > 0 && !zcommon.Contains(artifactTypes, indexArtifactType) {
continue
}
result = append(result, ispec.Descriptor{
MediaType: descriptor.MediaType,
ArtifactType: indexArtifactType,
Size: descriptor.Size,
Digest: descriptor.Digest,
Annotations: indexContent.Annotations,
})
}
}

43
pkg/storage/common/common_test.go
View file

@ -334,6 +334,49 @@ func TestGetReferrersErrors(t *testing.T) {
[]string{artifactType}, log.With().Caller().Logger())
So(err, ShouldBeNil)
})
Convey("Index bad blob", func() {
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return []byte(`{
"manifests": [{
"digest": "digest",
"mediaType": "application/vnd.oci.image.index.v1+json"
}]
}`), nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte("bad blob"), nil
},
}
_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
[]string{}, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
Convey("Index bad artifac type", func() {
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return []byte(`{
"manifests": [{
"digest": "digest",
"mediaType": "application/vnd.oci.image.index.v1+json"
}]
}`), nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte(`{
"subject": {"digest": "` + validDigest.String() + `"}
}`), nil
},
}
ref, err := common.GetReferrers(imgStore, "zot-test", validDigest,
[]string{"art.type"}, log.With().Caller().Logger())
So(err, ShouldBeNil)
So(len(ref.Manifests), ShouldEqual, 0)
})
})
}