From 96d9d318df6bb4ed063ab3170873ee83461d448b Mon Sep 17 00:00:00 2001 From: LaurentiuNiculae Date: Wed, 5 Jul 2023 19:42:16 +0300 Subject: [PATCH] feat(referrers): added index support for referrers queries (#1560) Signed-off-by: Laurentiu Niculae --- go.mod | 6 +- go.sum | 12 +-- pkg/common/oci.go | 4 + pkg/extensions/search/search_test.go | 101 ++++++++++++++++++++++++ pkg/meta/repodb/storage_parsing.go | 47 ++++++++--- pkg/meta/repodb/storage_parsing_test.go | 3 + pkg/storage/common/common.go | 56 +++++++++---- pkg/storage/common/common_test.go | 43 ++++++++++ 8 files changed, 236 insertions(+), 36 deletions(-) diff --git a/go.mod b/go.mod index ed5a88b8..21f0c852 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/nmcclain/ldap v0.0.0-20210720162743-7f8d1e44eeba github.com/olekukonko/tablewriter v0.0.5 github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/image-spec v1.1.0-rc3.0.20230608172643-f8b2ca8c6ba1 + github.com/opencontainers/image-spec v1.1.0-rc4 github.com/opencontainers/umoci v0.4.8-0.20210922062158-e60a0cc726e6 github.com/oras-project/artifacts-spec v1.0.0-rc.2 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 @@ -58,7 +58,7 @@ require ( github.com/sigstore/cosign/v2 v2.0.2 github.com/swaggo/http-swagger v1.3.4 modernc.org/sqlite v1.23.1 - oras.land/oras-go/v2 v2.2.0 + oras.land/oras-go/v2 v2.2.1 ) require ( @@ -453,7 +453,7 @@ require ( golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.11.0 // indirect golang.org/x/oauth2 v0.9.0 // indirect - golang.org/x/sync v0.2.0 // indirect + golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.9.0 // indirect golang.org/x/term v0.9.0 // indirect golang.org/x/text v0.10.0 // indirect diff --git a/go.sum b/go.sum index 331c4eff..81584d30 100644 --- a/go.sum +++ b/go.sum @@ -1333,8 +1333,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.1.0-rc3.0.20230608172643-f8b2ca8c6ba1 h1:d11iHLE/a8/B4Cnn8E8C+3ozjx444ed7AsOgXAG9coU= -github.com/opencontainers/image-spec v1.1.0-rc3.0.20230608172643-f8b2ca8c6ba1/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= +github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runc v1.1.5 h1:L44KXEpKmfWDcS02aeGm8QNTFXTo2D+8MYGDIJ/GDEs= github.com/opencontainers/runc v1.1.5/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= @@ -1918,8 +1918,8 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2466,8 +2466,8 @@ modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY= oras.land/oras-go v1.2.3 h1:v8PJl+gEAntI1pJ/LCrDgsuk+1PKVavVEPsYIHFE5uY= oras.land/oras-go v1.2.3/go.mod h1:M/uaPdYklze0Vf3AakfarnpoEckvw0ESbRdN8Z1vdJg= -oras.land/oras-go/v2 v2.2.0 h1:E1fqITD56Eg5neZbxBtAdZVgDHD6wBabJo6xESTcQyo= -oras.land/oras-go/v2 v2.2.0/go.mod h1:pXjn0+KfarspMHHNR3A56j3tgvr+mxArHuI8qVn59v8= +oras.land/oras-go/v2 v2.2.1 h1:3VJTYqy5KfelEF9c2jo1MLSpr+TM3mX8K42wzZcd6qE= +oras.land/oras-go/v2 v2.2.1/go.mod h1:GeAwLuC4G/JpNwkd+bSZ6SkDMGaaYglt6YK2WvZP7uQ= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/pkg/common/oci.go b/pkg/common/oci.go index 620f05d3..08aadccd 100644 --- a/pkg/common/oci.go +++ b/pkg/common/oci.go @@ -58,6 +58,10 @@ func GetManifestArtifactType(manifestContent ispec.Manifest) string { return manifestContent.Config.MediaType } +func GetIndexArtifactType(indexContent ispec.Index) string { + return indexContent.ArtifactType +} + // GetImageLastUpdated This method will return last updated timestamp. // The Created timestamp is used, but if it is missing, look at the // history field and, if provided, return the timestamp of last entry in history. diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index d414b548..087a36c7 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -1064,6 +1064,107 @@ func TestGetReferrersGQL(t *testing.T) { So(referrersResp.Referrers[0].Digest, ShouldEqual, artifactManifestDigest) }) + + Convey("Get referrers with index as referrer", t, func() { + port := GetFreePort() + baseURL := GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + conf.Storage.RootDirectory = t.TempDir() + conf.Storage.GC = false + + defaultVal := true + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, + Lint: &extconf.LintConfig{ + BaseConfig: extconf.BaseConfig{ + Enable: &defaultVal, + }, + }, + } + + conf.Extensions.Search.CVE = nil + + ctlr := api.NewController(conf) + ctlrManager := NewControllerManager(ctlr) + ctlrManager.StartAndWait(port) + defer ctlrManager.StopServer() + + // Upload the index referrer + + targetImg, err := GetRandomImage("") + So(err, ShouldBeNil) + targetDigest, err := targetImg.Digest() + So(err, ShouldBeNil) + + err = UploadImage(targetImg, baseURL, "repo") + So(err, ShouldBeNil) + + indexReferrer, err := GetRandomMultiarchImage("ref") + So(err, ShouldBeNil) + + artifactType := "art.type" + indexReferrer.Index.ArtifactType = artifactType + indexReferrer.Index.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: targetDigest, + } + + indexReferrerDigest, err := indexReferrer.Digest() + So(err, ShouldBeNil) + + err = UploadMultiarchImage(indexReferrer, baseURL, "repo") + So(err, ShouldBeNil) + + // Call Referrers GQL + + referrersQuery := ` + { + Referrers( repo: "%s", digest: "%s"){ + ArtifactType, + Digest, + MediaType, + Size, + Annotations{ + Key + Value + } + } + }` + + referrersQuery = fmt.Sprintf(referrersQuery, "repo", targetDigest.String()) + + resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(referrersQuery)) + So(resp, ShouldNotBeNil) + So(err, ShouldBeNil) + + So(resp.StatusCode(), ShouldEqual, 200) + So(resp.Body(), ShouldNotBeNil) + So(err, ShouldBeNil) + + referrersResp := &zcommon.ReferrersResp{} + + err = json.Unmarshal(resp.Body(), referrersResp) + So(err, ShouldBeNil) + So(len(referrersResp.Referrers), ShouldEqual, 1) + So(referrersResp.Referrers[0].ArtifactType, ShouldResemble, artifactType) + So(referrersResp.Referrers[0].Digest, ShouldResemble, indexReferrerDigest.String()) + So(referrersResp.Referrers[0].MediaType, ShouldResemble, ispec.MediaTypeImageIndex) + + // Make REST call + + resp, err = resty.R().Get(baseURL + "/v2/repo/referrers/" + targetDigest.String()) + So(err, ShouldBeNil) + + var index ispec.Index + + err = json.Unmarshal(resp.Body(), &index) + So(err, ShouldBeNil) + So(len(index.Manifests), ShouldEqual, 1) + So(index.Manifests[0].ArtifactType, ShouldEqual, artifactType) + So(index.Manifests[0].Digest.String(), ShouldResemble, indexReferrerDigest.String()) + So(index.Manifests[0].MediaType, ShouldResemble, ispec.MediaTypeImageIndex) + }) } func TestExpandedRepoInfo(t *testing.T) { diff --git a/pkg/meta/repodb/storage_parsing.go b/pkg/meta/repodb/storage_parsing.go index ba22e006..e51dc48f 100644 --- a/pkg/meta/repodb/storage_parsing.go +++ b/pkg/meta/repodb/storage_parsing.go @@ -407,22 +407,43 @@ func GetReferredSubject(descriptorBlob []byte, referrerDigest, mediaType string, referrerSubject *ispec.Descriptor ) - var manifestContent ispec.Manifest + switch mediaType { + case ispec.MediaTypeImageManifest: + var manifestContent ispec.Manifest - err := json.Unmarshal(descriptorBlob, &manifestContent) - if err != nil { - return "", referrerInfo, false, - fmt.Errorf("repodb: can't unmarshal manifest for digest %s: %w", referrerDigest, err) - } + err := json.Unmarshal(descriptorBlob, &manifestContent) + if err != nil { + return "", referrerInfo, false, + fmt.Errorf("repodb: can't unmarshal manifest for digest %s: %w", referrerDigest, err) + } - referrerSubject = manifestContent.Subject + referrerSubject = manifestContent.Subject - referrerInfo = ReferrerInfo{ - Digest: referrerDigest, - MediaType: mediaType, - ArtifactType: zcommon.GetManifestArtifactType(manifestContent), - Size: len(descriptorBlob), - Annotations: manifestContent.Annotations, + referrerInfo = ReferrerInfo{ + Digest: referrerDigest, + MediaType: mediaType, + ArtifactType: zcommon.GetManifestArtifactType(manifestContent), + Size: len(descriptorBlob), + Annotations: manifestContent.Annotations, + } + case ispec.MediaTypeImageIndex: + var indexContent ispec.Index + + err := json.Unmarshal(descriptorBlob, &indexContent) + if err != nil { + return "", referrerInfo, false, + fmt.Errorf("repodb: can't unmarshal manifest for digest %s: %w", referrerDigest, err) + } + + referrerSubject = indexContent.Subject + + referrerInfo = ReferrerInfo{ + Digest: referrerDigest, + MediaType: mediaType, + ArtifactType: zcommon.GetIndexArtifactType(indexContent), + Size: len(descriptorBlob), + Annotations: indexContent.Annotations, + } } if referrerSubject == nil || referrerSubject.Digest.String() == "" { diff --git a/pkg/meta/repodb/storage_parsing_test.go b/pkg/meta/repodb/storage_parsing_test.go index 709a4c92..4b5e289d 100644 --- a/pkg/meta/repodb/storage_parsing_test.go +++ b/pkg/meta/repodb/storage_parsing_test.go @@ -621,6 +621,9 @@ func TestGetReferredSubject(t *testing.T) { Convey("GetReferredSubject error", t, func() { _, _, _, err := repodb.GetReferredSubject([]byte("bad json"), "digest", ispec.MediaTypeImageManifest) So(err, ShouldNotBeNil) + + _, _, _, err = repodb.GetReferredSubject([]byte("bad json"), "digest", ispec.MediaTypeImageIndex) + So(err, ShouldNotBeNil) }) } diff --git a/pkg/storage/common/common.go b/pkg/storage/common/common.go index cd8a1cab..fb0cf88f 100644 --- a/pkg/storage/common/common.go +++ b/pkg/storage/common/common.go @@ -581,14 +581,14 @@ func GetReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godiges result := []ispec.Descriptor{} - for _, manifest := range index.Manifests { - if manifest.Digest == gdigest { + for _, descriptor := range index.Manifests { + if descriptor.Digest == gdigest { continue } - buf, err := imgStore.GetBlobContent(repo, manifest.Digest) + buf, err := imgStore.GetBlobContent(repo, descriptor.Digest) if err != nil { - log.Error().Err(err).Str("blob", imgStore.BlobPath(repo, manifest.Digest)).Msg("failed to read manifest") + log.Error().Err(err).Str("blob", imgStore.BlobPath(repo, descriptor.Digest)).Msg("failed to read manifest") if errors.Is(err, zerr.ErrBlobNotFound) { return nilIndex, zerr.ErrManifestNotFound @@ -597,31 +597,59 @@ func GetReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godiges return nilIndex, err } - if manifest.MediaType == ispec.MediaTypeImageManifest { - var mfst ispec.Manifest - if err := json.Unmarshal(buf, &mfst); err != nil { - log.Error().Err(err).Str("manifest digest", manifest.Digest.String()).Msg("invalid JSON") + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + var manifestContent ispec.Manifest + + if err := json.Unmarshal(buf, &manifestContent); err != nil { + log.Error().Err(err).Str("manifest digest", descriptor.Digest.String()).Msg("invalid JSON") return nilIndex, err } - if mfst.Subject == nil || mfst.Subject.Digest != gdigest { + if manifestContent.Subject == nil || manifestContent.Subject.Digest != gdigest { continue } // filter by artifact type - manifestArtifactType := zcommon.GetManifestArtifactType(mfst) + manifestArtifactType := zcommon.GetManifestArtifactType(manifestContent) if len(artifactTypes) > 0 && !zcommon.Contains(artifactTypes, manifestArtifactType) { continue } result = append(result, ispec.Descriptor{ - MediaType: manifest.MediaType, + MediaType: descriptor.MediaType, ArtifactType: manifestArtifactType, - Size: manifest.Size, - Digest: manifest.Digest, - Annotations: mfst.Annotations, + Size: descriptor.Size, + Digest: descriptor.Digest, + Annotations: manifestContent.Annotations, + }) + case ispec.MediaTypeImageIndex: + var indexContent ispec.Index + + if err := json.Unmarshal(buf, &indexContent); err != nil { + log.Error().Err(err).Str("manifest digest", descriptor.Digest.String()).Msg("invalid JSON") + + return nilIndex, err + } + + if indexContent.Subject == nil || indexContent.Subject.Digest != gdigest { + continue + } + + indexArtifactType := zcommon.GetIndexArtifactType(indexContent) + + if len(artifactTypes) > 0 && !zcommon.Contains(artifactTypes, indexArtifactType) { + continue + } + + result = append(result, ispec.Descriptor{ + MediaType: descriptor.MediaType, + ArtifactType: indexArtifactType, + Size: descriptor.Size, + Digest: descriptor.Digest, + Annotations: indexContent.Annotations, }) } } diff --git a/pkg/storage/common/common_test.go b/pkg/storage/common/common_test.go index 43aefbcb..76ae36ec 100644 --- a/pkg/storage/common/common_test.go +++ b/pkg/storage/common/common_test.go @@ -334,6 +334,49 @@ func TestGetReferrersErrors(t *testing.T) { []string{artifactType}, log.With().Caller().Logger()) So(err, ShouldBeNil) }) + + Convey("Index bad blob", func() { + imgStore = &mocks.MockedImageStore{ + GetIndexContentFn: func(repo string) ([]byte, error) { + return []byte(`{ + "manifests": [{ + "digest": "digest", + "mediaType": "application/vnd.oci.image.index.v1+json" + }] + }`), nil + }, + GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) { + return []byte("bad blob"), nil + }, + } + + _, err = common.GetReferrers(imgStore, "zot-test", validDigest, + []string{}, log.With().Caller().Logger()) + So(err, ShouldNotBeNil) + }) + + Convey("Index bad artifac type", func() { + imgStore = &mocks.MockedImageStore{ + GetIndexContentFn: func(repo string) ([]byte, error) { + return []byte(`{ + "manifests": [{ + "digest": "digest", + "mediaType": "application/vnd.oci.image.index.v1+json" + }] + }`), nil + }, + GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) { + return []byte(`{ + "subject": {"digest": "` + validDigest.String() + `"} + }`), nil + }, + } + + ref, err := common.GetReferrers(imgStore, "zot-test", validDigest, + []string{"art.type"}, log.With().Caller().Logger()) + So(err, ShouldBeNil) + So(len(ref.Manifests), ShouldEqual, 0) + }) }) }