From 0586c6227ec3731dea093de924cb4d457e75eb68 Mon Sep 17 00:00:00 2001 From: Nicol Date: Tue, 18 Apr 2023 21:07:47 +0300 Subject: [PATCH] refactor: remove pkg/extensions/search/common and move the code to the appropriate packages (#1358) Signed-off-by: Nicol Draghici --- pkg/cli/discover.go | 3 +- pkg/cli/service.go | 26 +- pkg/common/common_test.go | 5 + pkg/{extensions/search => }/common/model.go | 5 + pkg/common/utils.go | 70 ++++ pkg/extensions/search/common/common.go | 317 ------------------ pkg/extensions/search/convert/annotations.go | 135 ++++++++ pkg/extensions/search/convert/convert_test.go | 52 +++ pkg/extensions/search/convert/repodb.go | 6 +- pkg/extensions/search/cve/cve.go | 81 ++++- pkg/extensions/search/cve/cve_test.go | 60 ++++ pkg/extensions/search/cve/model/models.go | 17 + pkg/extensions/search/cve/trivy/scanner.go | 3 +- .../search/cve/trivy/scanner_internal_test.go | 2 +- pkg/extensions/search/digest_test.go | 9 +- pkg/extensions/search/resolver.go | 7 +- pkg/extensions/search/resolver_test.go | 2 +- pkg/extensions/search/schema.resolvers.go | 2 +- .../{common/common_test.go => search_test.go} | 191 +---------- pkg/storage/storage_controller.go | 4 +- pkg/storage/storage_test.go | 13 + pkg/test/mocks/cve_mock.go | 13 +- pkg/test/mocks/oci_mock.go | 15 +- pkg/test/{ => oci-layout}/oci_layout.go | 37 +- pkg/test/{ => oci-layout}/oci_layout_test.go | 81 ++++- 25 files changed, 561 insertions(+), 595 deletions(-) rename pkg/{extensions/search => }/common/model.go (97%) create mode 100644 pkg/common/utils.go delete mode 100644 pkg/extensions/search/common/common.go create mode 100644 pkg/extensions/search/convert/annotations.go rename pkg/extensions/search/{common/common_test.go => search_test.go} (97%) rename pkg/test/{ => oci-layout}/oci_layout.go (94%) rename pkg/test/{ => oci-layout}/oci_layout_test.go (80%) diff --git a/pkg/cli/discover.go b/pkg/cli/discover.go index 0ac6d3fa..baf05d82 100644 --- a/pkg/cli/discover.go +++ b/pkg/cli/discover.go @@ -10,6 +10,7 @@ import ( distext "github.com/opencontainers/distribution-spec/specs-go/v1/extensions" "zotregistry.io/zot/pkg/api/constants" + "zotregistry.io/zot/pkg/common" ) type field struct { @@ -24,7 +25,7 @@ type schemaList struct { } `json:"queryType"` //nolint:tagliatelle // graphQL schema } `json:"__schema"` //nolint:tagliatelle // graphQL schema } `json:"data"` - Errors []errorGraphQL `json:"errors"` + Errors []common.ErrorGraphQL `json:"errors"` } func containsGQLQuery(queryList []field, query string) bool { diff --git a/pkg/cli/service.go b/pkg/cli/service.go index 402420e0..3036b67b 100644 --- a/pkg/cli/service.go +++ b/pkg/cli/service.go @@ -22,6 +22,7 @@ import ( zotErrors "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/api/constants" + "zotregistry.io/zot/pkg/common" ) type SearchService interface { //nolint:interfacebloat @@ -843,7 +844,7 @@ func (service searchService) makeGraphQLQuery(ctx context.Context, return nil } -func checkResultGraphQLQuery(ctx context.Context, err error, resultErrors []errorGraphQL, +func checkResultGraphQLQuery(ctx context.Context, err error, resultErrors []common.ErrorGraphQL, ) error { if err != nil { if isContextDone(ctx) { @@ -899,13 +900,8 @@ func addManifestCallToPool(ctx context.Context, config searchConfig, pool *reque } type cveResult struct { - Errors []errorGraphQL `json:"errors"` - Data cveData `json:"data"` -} - -type errorGraphQL struct { - Message string `json:"message"` - Path []string `json:"path"` + Errors []common.ErrorGraphQL `json:"errors"` + Data cveData `json:"data"` } type tagListResp struct { @@ -996,14 +992,14 @@ func (cve cveResult) stringYAML() (string, error) { } type fixedTags struct { - Errors []errorGraphQL `json:"errors"` + Errors []common.ErrorGraphQL `json:"errors"` Data struct { PaginatedImagesResult `json:"ImageListWithCVEFixed"` //nolint:tagliatelle // graphQL schema } `json:"data"` } type imagesForCve struct { - Errors []errorGraphQL `json:"errors"` + Errors []common.ErrorGraphQL `json:"errors"` Data struct { PaginatedImagesResult `json:"ImageListForCVE"` //nolint:tagliatelle // graphQL schema } `json:"data"` @@ -1047,35 +1043,35 @@ type BaseImageList struct { } type imageListStructGQL struct { - Errors []errorGraphQL `json:"errors"` + Errors []common.ErrorGraphQL `json:"errors"` Data struct { PaginatedImagesResult `json:"ImageList"` //nolint:tagliatelle } `json:"data"` } type imageListStructForDigestGQL struct { - Errors []errorGraphQL `json:"errors"` + Errors []common.ErrorGraphQL `json:"errors"` Data struct { PaginatedImagesResult `json:"ImageListForDigest"` //nolint:tagliatelle } `json:"data"` } type imageListStructForDerivedImagesGQL struct { - Errors []errorGraphQL `json:"errors"` + Errors []common.ErrorGraphQL `json:"errors"` Data struct { PaginatedImagesResult `json:"DerivedImageList"` //nolint:tagliatelle } `json:"data"` } type imageListStructForBaseImagesGQL struct { - Errors []errorGraphQL `json:"errors"` + Errors []common.ErrorGraphQL `json:"errors"` Data struct { PaginatedImagesResult `json:"BaseImageList"` //nolint:tagliatelle } `json:"data"` } type imagesForDigest struct { - Errors []errorGraphQL `json:"errors"` + Errors []common.ErrorGraphQL `json:"errors"` Data struct { PaginatedImagesResult `json:"ImageListForDigest"` //nolint:tagliatelle // graphQL schema } `json:"data"` diff --git a/pkg/common/common_test.go b/pkg/common/common_test.go index 8c456357..ea8248ab 100644 --- a/pkg/common/common_test.go +++ b/pkg/common/common_test.go @@ -119,4 +119,9 @@ func TestCommon(t *testing.T) { resultPtr, baseURL+"/v2/", ispec.MediaTypeImageManifest, log.NewLogger("", "")) So(err, ShouldNotBeNil) }) + Convey("Test image dir and digest", t, func() { + repo, digest := common.GetImageDirAndDigest("image") + So(repo, ShouldResemble, "image") + So(digest, ShouldResemble, "") + }) } diff --git a/pkg/extensions/search/common/model.go b/pkg/common/model.go similarity index 97% rename from pkg/extensions/search/common/model.go rename to pkg/common/model.go index 320dd100..a83efaed 100644 --- a/pkg/extensions/search/common/model.go +++ b/pkg/common/model.go @@ -57,6 +57,11 @@ type Platform struct { Arch string `json:"arch"` } +type ErrorGraphQL struct { + Message string `json:"message"` + Path []string `json:"path"` +} + type ImageVulnerabilitySummary struct { MaxSeverity string `json:"maxSeverity"` Count int `json:"count"` diff --git a/pkg/common/utils.go b/pkg/common/utils.go new file mode 100644 index 00000000..02840acb --- /dev/null +++ b/pkg/common/utils.go @@ -0,0 +1,70 @@ +package common + +import ( + "strings" + "time" + + ispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +func GetImageDirAndTag(imageName string) (string, string) { + var imageDir string + + var imageTag string + + if strings.Contains(imageName, ":") { + imageDir, imageTag, _ = strings.Cut(imageName, ":") + } else { + imageDir = imageName + } + + return imageDir, imageTag +} + +func GetImageDirAndDigest(imageName string) (string, string) { + var imageDir string + + var imageDigest string + + if strings.Contains(imageName, "@") { + imageDir, imageDigest, _ = strings.Cut(imageName, "@") + } else { + imageDir = imageName + } + + return imageDir, imageDigest +} + +// GetImageDirAndReference returns the repo, digest and isTag. +func GetImageDirAndReference(imageName string) (string, string, bool) { + if strings.Contains(imageName, "@") { + repo, digest := GetImageDirAndDigest(imageName) + + return repo, digest, false + } + + repo, tag := GetImageDirAndTag(imageName) + + return repo, tag, true +} + +// GetImageLastUpdated This method will return last updated timestamp. +// The Created timestamp is used, but if it is missing, look at the +// history field and, if provided, return the timestamp of last entry in history. +func GetImageLastUpdated(imageInfo ispec.Image) time.Time { + timeStamp := imageInfo.Created + + if timeStamp != nil && !timeStamp.IsZero() { + return *timeStamp + } + + if len(imageInfo.History) > 0 { + timeStamp = imageInfo.History[len(imageInfo.History)-1].Created + } + + if timeStamp == nil { + timeStamp = &time.Time{} + } + + return *timeStamp +} diff --git a/pkg/extensions/search/common/common.go b/pkg/extensions/search/common/common.go deleted file mode 100644 index a695a68c..00000000 --- a/pkg/extensions/search/common/common.go +++ /dev/null @@ -1,317 +0,0 @@ -package common - -import ( - "fmt" - "sort" - "strings" - "time" - - godigest "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - - "zotregistry.io/zot/pkg/storage" -) - -const ( - // See https://github.com/opencontainers/image-spec/blob/main/annotations.md#back-compatibility-with-label-schema - AnnotationLabels = "org.label-schema.labels" - LabelAnnotationCreated = "org.label-schema.build-date" - LabelAnnotationVendor = "org.label-schema.vendor" - LabelAnnotationDescription = "org.label-schema.description" - LabelAnnotationLicenses = "org.label-schema.license" - LabelAnnotationTitle = "org.label-schema.name" - LabelAnnotationDocumentation = "org.label-schema.usage" - LabelAnnotationSource = "org.label-schema.vcs-url" -) - -type Descriptor struct { - Digest godigest.Digest - MediaType string -} - -type TagInfo struct { - Name string - Descriptor Descriptor - Timestamp time.Time -} - -func GetRootDir(image string, storeController storage.StoreController) string { - var rootDir string - - prefixName := GetRoutePrefix(image) - - subStore := storeController.SubStore - - if subStore != nil { - imgStore, ok := storeController.SubStore[prefixName] - if ok { - rootDir = imgStore.RootDir() - } else { - rootDir = storeController.DefaultStore.RootDir() - } - } else { - rootDir = storeController.DefaultStore.RootDir() - } - - return rootDir -} - -func GetRepo(image string) string { - if strings.Contains(image, ":") { - splitString := strings.SplitN(image, ":", 2) //nolint:gomnd - if len(splitString) != 2 { //nolint:gomnd - return image - } - - return splitString[0] - } - - return image -} - -func GetImageDirAndTag(imageName string) (string, string) { - var imageDir string - - var imageTag string - - if strings.Contains(imageName, ":") { - imageDir, imageTag, _ = strings.Cut(imageName, ":") - } else { - imageDir = imageName - } - - return imageDir, imageTag -} - -func GetImageDirAndDigest(imageName string) (string, string) { - var imageDir string - - var imageDigest string - - if strings.Contains(imageName, "@") { - imageDir, imageDigest, _ = strings.Cut(imageName, "@") - } else { - imageDir = imageName - } - - return imageDir, imageDigest -} - -// GetImageDirAndReference returns the repo, digest and isTag. -func GetImageDirAndReference(imageName string) (string, string, bool) { - if strings.Contains(imageName, "@") { - repo, digest := GetImageDirAndDigest(imageName) - - return repo, digest, false - } - - repo, tag := GetImageDirAndTag(imageName) - - return repo, tag, true -} - -// GetImageLastUpdated This method will return last updated timestamp. -// The Created timestamp is used, but if it is missing, look at the -// history field and, if provided, return the timestamp of last entry in history. -func GetImageLastUpdated(imageInfo ispec.Image) time.Time { - timeStamp := imageInfo.Created - - if timeStamp != nil && !timeStamp.IsZero() { - return *timeStamp - } - - if len(imageInfo.History) > 0 { - timeStamp = imageInfo.History[len(imageInfo.History)-1].Created - } - - if timeStamp == nil { - timeStamp = &time.Time{} - } - - return *timeStamp -} - -func GetFixedTags(allTags, vulnerableTags []TagInfo) []TagInfo { - sort.Slice(allTags, func(i, j int) bool { - return allTags[i].Timestamp.Before(allTags[j].Timestamp) - }) - - earliestVulnerable := vulnerableTags[0] - vulnerableTagMap := make(map[string]TagInfo, len(vulnerableTags)) - - for _, tag := range vulnerableTags { - vulnerableTagMap[tag.Name] = tag - - if tag.Timestamp.Before(earliestVulnerable.Timestamp) { - earliestVulnerable = tag - } - } - - var fixedTags []TagInfo - - // There are some downsides to this logic - // We assume there can't be multiple "branches" of the same - // image built at different times containing different fixes - // There may be older images which have a fix or - // newer images which don't - for _, tag := range allTags { - if tag.Timestamp.Before(earliestVulnerable.Timestamp) { - // The vulnerability did not exist at the time this - // image was built - continue - } - // If the image is old enough for the vulnerability to - // exist, but it was not detected, it means it contains - // the fix - if _, ok := vulnerableTagMap[tag.Name]; !ok { - fixedTags = append(fixedTags, tag) - } - } - - return fixedTags -} - -func GetLatestTag(allTags []TagInfo) TagInfo { - sort.Slice(allTags, func(i, j int) bool { - return allTags[i].Timestamp.Before(allTags[j].Timestamp) - }) - - return allTags[len(allTags)-1] -} - -func GetRoutePrefix(name string) string { - names := strings.SplitN(name, "/", 2) //nolint:gomnd - - if len(names) != 2 { //nolint:gomnd - // it means route is of global storage e.g "centos:latest" - if len(names) == 1 { - return "/" - } - } - - return fmt.Sprintf("/%s", names[0]) -} - -type ImageAnnotations struct { - Description string - Licenses string - Title string - Documentation string - Source string - Labels string - Vendor string - Authors string -} - -/* - OCI annotation/label with backwards compatibility - -arg can be either lables or annotations -https://github.com/opencontainers/image-spec/blob/main/annotations.md. -*/ -func GetAnnotationValue(annotations map[string]string, annotationKey, labelKey string) string { - value, ok := annotations[annotationKey] - if !ok || value == "" { - value, ok = annotations[labelKey] - if !ok { - value = "" - } - } - - return value -} - -func GetDescription(annotations map[string]string) string { - return GetAnnotationValue(annotations, ispec.AnnotationDescription, LabelAnnotationDescription) -} - -func GetLicenses(annotations map[string]string) string { - return GetAnnotationValue(annotations, ispec.AnnotationLicenses, LabelAnnotationLicenses) -} - -func GetVendor(annotations map[string]string) string { - return GetAnnotationValue(annotations, ispec.AnnotationVendor, LabelAnnotationVendor) -} - -func GetAuthors(annotations map[string]string) string { - authors := annotations[ispec.AnnotationAuthors] - - return authors -} - -func GetTitle(annotations map[string]string) string { - return GetAnnotationValue(annotations, ispec.AnnotationTitle, LabelAnnotationTitle) -} - -func GetDocumentation(annotations map[string]string) string { - return GetAnnotationValue(annotations, ispec.AnnotationDocumentation, LabelAnnotationDocumentation) -} - -func GetSource(annotations map[string]string) string { - return GetAnnotationValue(annotations, ispec.AnnotationSource, LabelAnnotationSource) -} - -func GetCategories(labels map[string]string) string { - categories := labels[AnnotationLabels] - - return categories -} - -func GetAnnotations(annotations, labels map[string]string) ImageAnnotations { - description := GetDescription(annotations) - if description == "" { - description = GetDescription(labels) - } - - title := GetTitle(annotations) - if title == "" { - title = GetTitle(labels) - } - - documentation := GetDocumentation(annotations) - if documentation == "" { - documentation = GetDocumentation(labels) - } - - source := GetSource(annotations) - if source == "" { - source = GetSource(labels) - } - - licenses := GetLicenses(annotations) - if licenses == "" { - licenses = GetLicenses(labels) - } - - categories := GetCategories(annotations) - if categories == "" { - categories = GetCategories(labels) - } - - vendor := GetVendor(annotations) - if vendor == "" { - vendor = GetVendor(labels) - } - - authors := GetAuthors(annotations) - if authors == "" { - authors = GetAuthors(labels) - } - - return ImageAnnotations{ - Description: description, - Title: title, - Documentation: documentation, - Source: source, - Licenses: licenses, - Labels: categories, - Vendor: vendor, - Authors: authors, - } -} - -func ReferenceIsDigest(reference string) bool { - _, err := godigest.Parse(reference) - - return err == nil -} diff --git a/pkg/extensions/search/convert/annotations.go b/pkg/extensions/search/convert/annotations.go new file mode 100644 index 00000000..ad530fa4 --- /dev/null +++ b/pkg/extensions/search/convert/annotations.go @@ -0,0 +1,135 @@ +package convert + +import ( + ispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +const ( + // See https://github.com/opencontainers/image-spec/blob/main/annotations.md#back-compatibility-with-label-schema + AnnotationLabels = "org.label-schema.labels" + LabelAnnotationCreated = "org.label-schema.build-date" + LabelAnnotationVendor = "org.label-schema.vendor" + LabelAnnotationDescription = "org.label-schema.description" + LabelAnnotationLicenses = "org.label-schema.license" + LabelAnnotationTitle = "org.label-schema.name" + LabelAnnotationDocumentation = "org.label-schema.usage" + LabelAnnotationSource = "org.label-schema.vcs-url" +) + +type ImageAnnotations struct { + Description string + Licenses string + Title string + Documentation string + Source string + Labels string + Vendor string + Authors string +} + +/* + OCI annotation/label with backwards compatibility + +arg can be either lables or annotations +https://github.com/opencontainers/image-spec/blob/main/annotations.md. +*/ +func GetAnnotationValue(annotations map[string]string, annotationKey, labelKey string) string { + value, ok := annotations[annotationKey] + if !ok || value == "" { + value, ok = annotations[labelKey] + if !ok { + value = "" + } + } + + return value +} + +func GetDescription(annotations map[string]string) string { + return GetAnnotationValue(annotations, ispec.AnnotationDescription, LabelAnnotationDescription) +} + +func GetLicenses(annotations map[string]string) string { + return GetAnnotationValue(annotations, ispec.AnnotationLicenses, LabelAnnotationLicenses) +} + +func GetVendor(annotations map[string]string) string { + return GetAnnotationValue(annotations, ispec.AnnotationVendor, LabelAnnotationVendor) +} + +func GetAuthors(annotations map[string]string) string { + authors := annotations[ispec.AnnotationAuthors] + + return authors +} + +func GetTitle(annotations map[string]string) string { + return GetAnnotationValue(annotations, ispec.AnnotationTitle, LabelAnnotationTitle) +} + +func GetDocumentation(annotations map[string]string) string { + return GetAnnotationValue(annotations, ispec.AnnotationDocumentation, LabelAnnotationDocumentation) +} + +func GetSource(annotations map[string]string) string { + return GetAnnotationValue(annotations, ispec.AnnotationSource, LabelAnnotationSource) +} + +func GetCategories(labels map[string]string) string { + categories := labels[AnnotationLabels] + + return categories +} + +func GetAnnotations(annotations, labels map[string]string) ImageAnnotations { + description := GetDescription(annotations) + if description == "" { + description = GetDescription(labels) + } + + title := GetTitle(annotations) + if title == "" { + title = GetTitle(labels) + } + + documentation := GetDocumentation(annotations) + if documentation == "" { + documentation = GetDocumentation(labels) + } + + source := GetSource(annotations) + if source == "" { + source = GetSource(labels) + } + + licenses := GetLicenses(annotations) + if licenses == "" { + licenses = GetLicenses(labels) + } + + categories := GetCategories(annotations) + if categories == "" { + categories = GetCategories(labels) + } + + vendor := GetVendor(annotations) + if vendor == "" { + vendor = GetVendor(labels) + } + + authors := GetAuthors(annotations) + if authors == "" { + authors = GetAuthors(labels) + } + + return ImageAnnotations{ + Description: description, + Title: title, + Documentation: documentation, + Source: source, + Licenses: licenses, + Labels: categories, + Vendor: vendor, + Authors: authors, + } +} diff --git a/pkg/extensions/search/convert/convert_test.go b/pkg/extensions/search/convert/convert_test.go index ca5dc58a..d2b7a2ce 100644 --- a/pkg/extensions/search/convert/convert_test.go +++ b/pkg/extensions/search/convert/convert_test.go @@ -333,3 +333,55 @@ func TestUpdateLastUpdatedTimestam(t *testing.T) { So(*img.LastUpdated, ShouldResemble, after) }) } + +func TestLabels(t *testing.T) { + Convey("Test labels", t, func() { + // Test various labels + labels := make(map[string]string) + + desc := convert.GetDescription(labels) + So(desc, ShouldEqual, "") + + license := convert.GetLicenses(labels) + So(license, ShouldEqual, "") + + vendor := convert.GetVendor(labels) + So(vendor, ShouldEqual, "") + + categories := convert.GetCategories(labels) + So(categories, ShouldEqual, "") + + labels[ispec.AnnotationVendor] = "zot" + labels[ispec.AnnotationDescription] = "zot-desc" + labels[ispec.AnnotationLicenses] = "zot-license" + labels[convert.AnnotationLabels] = "zot-labels" + + desc = convert.GetDescription(labels) + So(desc, ShouldEqual, "zot-desc") + + license = convert.GetLicenses(labels) + So(license, ShouldEqual, "zot-license") + + vendor = convert.GetVendor(labels) + So(vendor, ShouldEqual, "zot") + + categories = convert.GetCategories(labels) + So(categories, ShouldEqual, "zot-labels") + + labels = make(map[string]string) + + // Use diff key + labels[convert.LabelAnnotationVendor] = "zot-vendor" + labels[convert.LabelAnnotationDescription] = "zot-label-desc" + labels[ispec.AnnotationLicenses] = "zot-label-license" + + desc = convert.GetDescription(labels) + So(desc, ShouldEqual, "zot-label-desc") + + license = convert.GetLicenses(labels) + So(license, ShouldEqual, "zot-label-license") + + vendor = convert.GetVendor(labels) + So(vendor, ShouldEqual, "zot-vendor") + }) +} diff --git a/pkg/extensions/search/convert/repodb.go b/pkg/extensions/search/convert/repodb.go index a78a27cb..3383e441 100644 --- a/pkg/extensions/search/convert/repodb.go +++ b/pkg/extensions/search/convert/repodb.go @@ -14,7 +14,7 @@ import ( "github.com/vektah/gqlparser/v2/gqlerror" zerr "zotregistry.io/zot/errors" - "zotregistry.io/zot/pkg/extensions/search/common" + "zotregistry.io/zot/pkg/common" cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/extensions/search/gql_generated" @@ -240,7 +240,7 @@ func ImageIndex2ImageSummary(ctx context.Context, repo, tag string, indexDigest indexSize = strconv.FormatInt(totalIndexSize, 10) - annotations := common.GetAnnotations(indexContent.Annotations, map[string]string{}) + annotations := GetAnnotations(indexContent.Annotations, map[string]string{}) indexSummary := gql_generated.ImageSummary{ RepoName: &repo, @@ -327,7 +327,7 @@ func ImageManifest2ImageSummary(ctx context.Context, repo, tag string, digest go manifestContent.Layers) imageSize := strconv.FormatInt(size, 10) - annotations := common.GetAnnotations(manifestContent.Annotations, configContent.Config.Labels) + annotations := GetAnnotations(manifestContent.Annotations, configContent.Config.Labels) authors := annotations.Authors if authors == "" { diff --git a/pkg/extensions/search/cve/cve.go b/pkg/extensions/search/cve/cve.go index 5ea3a811..a5bcc470 100644 --- a/pkg/extensions/search/cve/cve.go +++ b/pkg/extensions/search/cve/cve.go @@ -3,13 +3,14 @@ package cveinfo import ( "encoding/json" "fmt" + "sort" "strings" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" "zotregistry.io/zot/errors" - "zotregistry.io/zot/pkg/extensions/search/common" + "zotregistry.io/zot/pkg/common" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/extensions/search/cve/trivy" "zotregistry.io/zot/pkg/log" @@ -18,8 +19,8 @@ import ( ) type CveInfo interface { - GetImageListForCVE(repo, cveID string) ([]common.TagInfo, error) - GetImageListWithCVEFixed(repo, cveID string) ([]common.TagInfo, error) + GetImageListForCVE(repo, cveID string) ([]cvemodel.TagInfo, error) + GetImageListWithCVEFixed(repo, cveID string) ([]cvemodel.TagInfo, error) GetCVEListForImage(repo, tag string, searchedCVE string, pageinput PageInput) ([]cvemodel.CVE, PageInfo, error) GetCVESummaryForImage(repo, tag string) (ImageCVESummary, error) CompareSeverities(severity1, severity2 string) int @@ -56,8 +57,8 @@ func NewCVEInfo(storeController storage.StoreController, repoDB repodb.RepoDB, } } -func (cveinfo BaseCveInfo) GetImageListForCVE(repo, cveID string) ([]common.TagInfo, error) { - imgList := make([]common.TagInfo, 0) +func (cveinfo BaseCveInfo) GetImageListForCVE(repo, cveID string) ([]cvemodel.TagInfo, error) { + imgList := make([]cvemodel.TagInfo, 0) repoMeta, err := cveinfo.RepoDB.GetRepoMeta(repo) if err != nil { @@ -89,9 +90,9 @@ func (cveinfo BaseCveInfo) GetImageListForCVE(repo, cveID string) ([]common.TagI } if _, hasCVE := cveMap[cveID]; hasCVE { - imgList = append(imgList, common.TagInfo{ + imgList = append(imgList, cvemodel.TagInfo{ Name: tag, - Descriptor: common.Descriptor{ + Descriptor: cvemodel.Descriptor{ Digest: manifestDigest, MediaType: descriptor.MediaType, }, @@ -105,17 +106,17 @@ func (cveinfo BaseCveInfo) GetImageListForCVE(repo, cveID string) ([]common.TagI return imgList, nil } -func (cveinfo BaseCveInfo) GetImageListWithCVEFixed(repo, cveID string) ([]common.TagInfo, error) { +func (cveinfo BaseCveInfo) GetImageListWithCVEFixed(repo, cveID string) ([]cvemodel.TagInfo, error) { repoMeta, err := cveinfo.RepoDB.GetRepoMeta(repo) if err != nil { cveinfo.Log.Error().Err(err).Str("repo", repo).Str("cve-id", cveID). Msg("unable to get list of tags from repo") - return []common.TagInfo{}, err + return []cvemodel.TagInfo{}, err } - vulnerableTags := make([]common.TagInfo, 0) - allTags := make([]common.TagInfo, 0) + vulnerableTags := make([]cvemodel.TagInfo, 0) + allTags := make([]cvemodel.TagInfo, 0) var hasCVE bool @@ -150,10 +151,10 @@ func (cveinfo BaseCveInfo) GetImageListWithCVEFixed(repo, cveID string) ([]commo continue } - tagInfo := common.TagInfo{ + tagInfo := cvemodel.TagInfo{ Name: tag, Timestamp: common.GetImageLastUpdated(configContent), - Descriptor: common.Descriptor{Digest: manifestDigest, MediaType: descriptor.MediaType}, + Descriptor: cvemodel.Descriptor{Digest: manifestDigest, MediaType: descriptor.MediaType}, } allTags = append(allTags, tagInfo) @@ -196,16 +197,16 @@ func (cveinfo BaseCveInfo) GetImageListWithCVEFixed(repo, cveID string) ([]commo default: cveinfo.Log.Error().Msgf("media type not supported '%s'", descriptor.MediaType) - return []common.TagInfo{}, + return []cvemodel.TagInfo{}, fmt.Errorf("media type '%s' is not supported: %w", descriptor.MediaType, errors.ErrNotImplemented) } } - var fixedTags []common.TagInfo + var fixedTags []cvemodel.TagInfo if len(vulnerableTags) != 0 { cveinfo.Log.Info().Str("repo", repo).Str("cve-id", cveID).Msgf("Vulnerable tags: %v", vulnerableTags) - fixedTags = common.GetFixedTags(allTags, vulnerableTags) + fixedTags = GetFixedTags(allTags, vulnerableTags) cveinfo.Log.Info().Str("repo", repo).Str("cve-id", cveID).Msgf("Fixed tags: %v", fixedTags) } else { cveinfo.Log.Info().Str("repo", repo).Str("cve-id", cveID). @@ -297,10 +298,16 @@ func (cveinfo BaseCveInfo) GetCVESummaryForImage(repo, tag string, return imageCVESummary, nil } +func referenceIsDigest(reference string) bool { + _, err := godigest.Parse(reference) + + return err == nil +} + func getImageString(repo, reference string) string { image := repo + ":" + reference - if common.ReferenceIsDigest(reference) { + if referenceIsDigest(reference) { image = repo + "@" + reference } @@ -314,3 +321,43 @@ func (cveinfo BaseCveInfo) UpdateDB() error { func (cveinfo BaseCveInfo) CompareSeverities(severity1, severity2 string) int { return cveinfo.Scanner.CompareSeverities(severity1, severity2) } + +func GetFixedTags(allTags, vulnerableTags []cvemodel.TagInfo) []cvemodel.TagInfo { + sort.Slice(allTags, func(i, j int) bool { + return allTags[i].Timestamp.Before(allTags[j].Timestamp) + }) + + earliestVulnerable := vulnerableTags[0] + vulnerableTagMap := make(map[string]cvemodel.TagInfo, len(vulnerableTags)) + + for _, tag := range vulnerableTags { + vulnerableTagMap[tag.Name] = tag + + if tag.Timestamp.Before(earliestVulnerable.Timestamp) { + earliestVulnerable = tag + } + } + + var fixedTags []cvemodel.TagInfo + + // There are some downsides to this logic + // We assume there can't be multiple "branches" of the same + // image built at different times containing different fixes + // There may be older images which have a fix or + // newer images which don't + for _, tag := range allTags { + if tag.Timestamp.Before(earliestVulnerable.Timestamp) { + // The vulnerability did not exist at the time this + // image was built + continue + } + // If the image is old enough for the vulnerability to + // exist, but it was not detected, it means it contains + // the fix + if _, ok := vulnerableTagMap[tag.Name]; !ok { + fixedTags = append(fixedTags, tag) + } + } + + return fixedTags +} diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 73c28dac..8e4598f7 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -1440,3 +1440,63 @@ func TestCVEStruct(t *testing.T) { So(err, ShouldBeNil) }) } + +func getTags() ([]cvemodel.TagInfo, []cvemodel.TagInfo) { + tags := make([]cvemodel.TagInfo, 0) + + firstTag := cvemodel.TagInfo{ + Name: "1.0.0", + Descriptor: cvemodel.Descriptor{ + Digest: "sha256:eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb", + MediaType: ispec.MediaTypeImageManifest, + }, + Timestamp: time.Now(), + } + secondTag := cvemodel.TagInfo{ + Name: "1.0.1", + Descriptor: cvemodel.Descriptor{ + Digest: "sha256:eca04f027f414362596f2632746d8a179362170b9ac9af772011fedcc3877ebb", + MediaType: ispec.MediaTypeImageManifest, + }, + Timestamp: time.Now(), + } + thirdTag := cvemodel.TagInfo{ + Name: "1.0.2", + Descriptor: cvemodel.Descriptor{ + Digest: "sha256:eca04f027f414362596f2632746d8a170362170b9ac9af772011fedcc3877ebb", + MediaType: ispec.MediaTypeImageManifest, + }, + Timestamp: time.Now(), + } + fourthTag := cvemodel.TagInfo{ + Name: "1.0.3", + Descriptor: cvemodel.Descriptor{ + Digest: "sha256:eca04f027f414362596f2632746d8a171362170b9ac9af772011fedcc3877ebb", + MediaType: ispec.MediaTypeImageManifest, + }, + Timestamp: time.Now(), + } + + tags = append(tags, firstTag, secondTag, thirdTag, fourthTag) + + vulnerableTags := make([]cvemodel.TagInfo, 0) + vulnerableTags = append(vulnerableTags, secondTag) + + return tags, vulnerableTags +} + +func TestFixedTags(t *testing.T) { + Convey("Test fixed tags", t, func() { + allTags, vulnerableTags := getTags() + + fixedTags := cveinfo.GetFixedTags(allTags, vulnerableTags) + So(len(fixedTags), ShouldEqual, 2) + + fixedTags = cveinfo.GetFixedTags(allTags, append(vulnerableTags, cvemodel.TagInfo{ + Name: "taginfo", + Descriptor: cvemodel.Descriptor{}, + Timestamp: time.Date(2000, time.July, 20, 10, 10, 10, 10, time.UTC), + })) + So(len(fixedTags), ShouldEqual, 3) + }) +} diff --git a/pkg/extensions/search/cve/model/models.go b/pkg/extensions/search/cve/model/models.go index 47d8390c..ca69ae4a 100644 --- a/pkg/extensions/search/cve/model/models.go +++ b/pkg/extensions/search/cve/model/models.go @@ -1,5 +1,11 @@ package model +import ( + "time" + + godigest "github.com/opencontainers/go-digest" +) + //nolint:tagliatelle // graphQL schema type CVE struct { ID string `json:"Id"` @@ -35,3 +41,14 @@ func SeverityValue(severity string) int { return sevMap[severity] } + +type Descriptor struct { + Digest godigest.Digest + MediaType string +} + +type TagInfo struct { + Name string + Descriptor Descriptor + Timestamp time.Time +} diff --git a/pkg/extensions/search/cve/trivy/scanner.go b/pkg/extensions/search/cve/trivy/scanner.go index e812db4c..b568a10f 100644 --- a/pkg/extensions/search/cve/trivy/scanner.go +++ b/pkg/extensions/search/cve/trivy/scanner.go @@ -17,7 +17,6 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" zerr "zotregistry.io/zot/errors" - "zotregistry.io/zot/pkg/extensions/search/common" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/repodb" @@ -122,7 +121,7 @@ func NewScanner(storeController storage.StoreController, func (scanner Scanner) getTrivyOptions(image string) flag.Options { // Split image to get route prefix - prefixName := common.GetRoutePrefix(image) + prefixName := storage.GetRoutePrefix(image) var opts flag.Options diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index 66dfdd26..ebd76dc0 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -15,8 +15,8 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" + "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/monitoring" - "zotregistry.io/zot/pkg/extensions/search/common" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/bolt" "zotregistry.io/zot/pkg/meta/repodb" diff --git a/pkg/extensions/search/digest_test.go b/pkg/extensions/search/digest_test.go index 0dafd9d9..b5f04336 100644 --- a/pkg/extensions/search/digest_test.go +++ b/pkg/extensions/search/digest_test.go @@ -30,7 +30,7 @@ type ImgResponseForDigest struct { //nolint:tagliatelle // graphQL schema type ImgListForDigest struct { - PaginatedImagesResult `json:"ImageListForDigest"` + PaginatedImagesResultForDigest `json:"ImageListForDigest"` } //nolint:tagliatelle // graphQL schema @@ -42,12 +42,7 @@ type ImgInfo struct { Size string `json:"Size"` } -type ErrorGQL struct { - Message string `json:"message"` - Path []string `json:"path"` -} - -type PaginatedImagesResult struct { +type PaginatedImagesResultForDigest struct { Results []ImgInfo `json:"results"` Page repodb.PageInfo `json:"page"` } diff --git a/pkg/extensions/search/resolver.go b/pkg/extensions/search/resolver.go index b8e66128..581f4e35 100644 --- a/pkg/extensions/search/resolver.go +++ b/pkg/extensions/search/resolver.go @@ -18,9 +18,10 @@ import ( "github.com/vektah/gqlparser/v2/gqlerror" zerr "zotregistry.io/zot/errors" - "zotregistry.io/zot/pkg/extensions/search/common" + "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/search/convert" cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" + cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/extensions/search/gql_generated" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/repodb" @@ -357,7 +358,7 @@ func getCVEListForImage( }, nil } -func FilterByTagInfo(tagsInfo []common.TagInfo) repodb.FilterFunc { +func FilterByTagInfo(tagsInfo []cvemodel.TagInfo) repodb.FilterFunc { return func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String() @@ -391,7 +392,7 @@ func getImageListForCVE( return &gql_generated.PaginatedImagesResult{}, err } - affectedImages := []common.TagInfo{} + affectedImages := []cvemodel.TagInfo{} for _, repoMeta := range reposMeta { repo := repoMeta.Name diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index 69a4f8f4..5da554b2 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -14,7 +14,7 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" - "zotregistry.io/zot/pkg/extensions/search/common" + "zotregistry.io/zot/pkg/common" cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/extensions/search/gql_generated" diff --git a/pkg/extensions/search/schema.resolvers.go b/pkg/extensions/search/schema.resolvers.go index bbda5461..6cd84905 100644 --- a/pkg/extensions/search/schema.resolvers.go +++ b/pkg/extensions/search/schema.resolvers.go @@ -9,7 +9,7 @@ import ( "github.com/vektah/gqlparser/v2/gqlerror" zerr "zotregistry.io/zot/errors" - "zotregistry.io/zot/pkg/extensions/search/common" + "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/search/gql_generated" ) diff --git a/pkg/extensions/search/common/common_test.go b/pkg/extensions/search/search_test.go similarity index 97% rename from pkg/extensions/search/common/common_test.go rename to pkg/extensions/search/search_test.go index 466ec01a..b5c25d89 100644 --- a/pkg/extensions/search/common/common_test.go +++ b/pkg/extensions/search/search_test.go @@ -1,7 +1,7 @@ //go:build search // +build search -package common_test +package search_test import ( "context" @@ -31,9 +31,9 @@ import ( "zotregistry.io/zot/pkg/api" "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/api/constants" + "zotregistry.io/zot/pkg/common" extconf "zotregistry.io/zot/pkg/extensions/config" "zotregistry.io/zot/pkg/extensions/monitoring" - "zotregistry.io/zot/pkg/extensions/search/common" cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/log" @@ -42,6 +42,7 @@ import ( "zotregistry.io/zot/pkg/storage/local" . "zotregistry.io/zot/pkg/test" "zotregistry.io/zot/pkg/test/mocks" + ocilayout "zotregistry.io/zot/pkg/test/oci-layout" ) const ( @@ -146,50 +147,6 @@ type ImageSummaryResult struct { Errors []ErrorGQL `json:"errors"` } -func getTags() ([]common.TagInfo, []common.TagInfo) { - tags := make([]common.TagInfo, 0) - - firstTag := common.TagInfo{ - Name: "1.0.0", - Descriptor: common.Descriptor{ - Digest: "sha256:eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb", - MediaType: ispec.MediaTypeImageManifest, - }, - Timestamp: time.Now(), - } - secondTag := common.TagInfo{ - Name: "1.0.1", - Descriptor: common.Descriptor{ - Digest: "sha256:eca04f027f414362596f2632746d8a179362170b9ac9af772011fedcc3877ebb", - MediaType: ispec.MediaTypeImageManifest, - }, - Timestamp: time.Now(), - } - thirdTag := common.TagInfo{ - Name: "1.0.2", - Descriptor: common.Descriptor{ - Digest: "sha256:eca04f027f414362596f2632746d8a170362170b9ac9af772011fedcc3877ebb", - MediaType: ispec.MediaTypeImageManifest, - }, - Timestamp: time.Now(), - } - fourthTag := common.TagInfo{ - Name: "1.0.3", - Descriptor: common.Descriptor{ - Digest: "sha256:eca04f027f414362596f2632746d8a171362170b9ac9af772011fedcc3877ebb", - MediaType: ispec.MediaTypeImageManifest, - }, - Timestamp: time.Now(), - } - - tags = append(tags, firstTag, secondTag, thirdTag, fourthTag) - - vulnerableTags := make([]common.TagInfo, 0) - vulnerableTags = append(vulnerableTags, secondTag) - - return tags, vulnerableTags -} - func readFileAndSearchString(filePath string, stringToMatch string, timeout time.Duration) (bool, error) { ctx, cancelFunc := context.WithTimeout(context.Background(), timeout) defer cancelFunc() @@ -1796,136 +1753,6 @@ func TestExpandedRepoInfo(t *testing.T) { }) } -func TestUtilsMethod(t *testing.T) { - Convey("Test utils", t, func() { - // Test GetRepo method - repo := common.GetRepo("test") - So(repo, ShouldEqual, "test") - - repo = common.GetRepo(":") - So(repo, ShouldEqual, "") - - repo = common.GetRepo("") - So(repo, ShouldEqual, "") - - repo = common.GetRepo("test:123") - So(repo, ShouldEqual, "test") - - repo = common.GetRepo("a/test:123") - So(repo, ShouldEqual, "a/test") - - repo = common.GetRepo("a/test:123:456") - So(repo, ShouldEqual, "a/test") - - // Test various labels - labels := make(map[string]string) - - desc := common.GetDescription(labels) - So(desc, ShouldEqual, "") - - license := common.GetLicenses(labels) - So(license, ShouldEqual, "") - - vendor := common.GetVendor(labels) - So(vendor, ShouldEqual, "") - - categories := common.GetCategories(labels) - So(categories, ShouldEqual, "") - - labels[ispec.AnnotationVendor] = "zot" - labels[ispec.AnnotationDescription] = "zot-desc" - labels[ispec.AnnotationLicenses] = "zot-license" - labels[common.AnnotationLabels] = "zot-labels" - - desc = common.GetDescription(labels) - So(desc, ShouldEqual, "zot-desc") - - license = common.GetLicenses(labels) - So(license, ShouldEqual, "zot-license") - - vendor = common.GetVendor(labels) - So(vendor, ShouldEqual, "zot") - - categories = common.GetCategories(labels) - So(categories, ShouldEqual, "zot-labels") - - labels = make(map[string]string) - - // Use diff key - labels[common.LabelAnnotationVendor] = "zot-vendor" - labels[common.LabelAnnotationDescription] = "zot-label-desc" - labels[ispec.AnnotationLicenses] = "zot-label-license" - - desc = common.GetDescription(labels) - So(desc, ShouldEqual, "zot-label-desc") - - license = common.GetLicenses(labels) - So(license, ShouldEqual, "zot-label-license") - - vendor = common.GetVendor(labels) - So(vendor, ShouldEqual, "zot-vendor") - - routePrefix := common.GetRoutePrefix("test:latest") - So(routePrefix, ShouldEqual, "/") - - routePrefix = common.GetRoutePrefix("a/test:latest") - So(routePrefix, ShouldEqual, "/a") - - routePrefix = common.GetRoutePrefix("a/b/test:latest") - So(routePrefix, ShouldEqual, "/a") - - allTags, vulnerableTags := getTags() - - latestTag := common.GetLatestTag(allTags) - So(latestTag.Name, ShouldEqual, "1.0.3") - - fixedTags := common.GetFixedTags(allTags, vulnerableTags) - So(len(fixedTags), ShouldEqual, 2) - - fixedTags = common.GetFixedTags(allTags, append(vulnerableTags, common.TagInfo{ - Name: "taginfo", - Descriptor: common.Descriptor{}, - Timestamp: time.Date(2000, time.July, 20, 10, 10, 10, 10, time.UTC), - })) - So(len(fixedTags), ShouldEqual, 3) - - log := log.NewLogger("debug", "") - - rootDir := t.TempDir() - - subRootDir := t.TempDir() - - conf := config.New() - conf.Extensions = &extconf.ExtensionConfig{} - conf.Extensions.Lint = &extconf.LintConfig{} - - metrics := monitoring.NewMetricsServer(false, log) - defaultStore := local.NewImageStore(rootDir, false, - storage.DefaultGCDelay, false, false, log, metrics, nil, nil) - - subStore := local.NewImageStore(subRootDir, false, - storage.DefaultGCDelay, false, false, log, metrics, nil, nil) - - subStoreMap := make(map[string]storage.ImageStore) - - subStoreMap["/b"] = subStore - - storeController := storage.StoreController{DefaultStore: defaultStore, SubStore: subStoreMap} - - dir := common.GetRootDir("a/zot-cve-test", storeController) - - So(dir, ShouldEqual, rootDir) - - dir = common.GetRootDir("b/zot-cve-test", storeController) - - So(dir, ShouldEqual, subRootDir) - - repo, digest := common.GetImageDirAndDigest("image") - So(repo, ShouldResemble, "image") - So(digest, ShouldResemble, "") - }) -} - func TestDerivedImageList(t *testing.T) { rootDir := t.TempDir() @@ -2390,7 +2217,7 @@ func TestGetImageManifest(t *testing.T) { storeController := storage.StoreController{ DefaultStore: mockImageStore, } - olu := NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + olu := ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) _, _, err := olu.GetImageManifest("nonexistent-repo", "latest") So(err, ShouldNotBeNil) @@ -2406,7 +2233,7 @@ func TestGetImageManifest(t *testing.T) { storeController := storage.StoreController{ DefaultStore: mockImageStore, } - olu := NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + olu := ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) _, _, err := olu.GetImageManifest("test-repo", "latest") //nolint:goconst So(err, ShouldNotBeNil) @@ -3068,7 +2895,7 @@ func TestGetRepositories(t *testing.T) { DefaultStore: mockImageStore, SubStore: map[string]storage.ImageStore{"test": mockImageStore}, } - olu := NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + olu := ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) repoList, err := olu.GetRepositories() So(repoList, ShouldBeEmpty) @@ -3078,7 +2905,7 @@ func TestGetRepositories(t *testing.T) { DefaultStore: mocks.MockedImageStore{}, SubStore: map[string]storage.ImageStore{"test": mockImageStore}, } - olu = NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + olu = ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) repoList, err = olu.GetRepositories() So(repoList, ShouldBeEmpty) @@ -3388,7 +3215,7 @@ func TestGlobalSearch(t *testing.T) { ) So(err, ShouldBeNil) - olu := NewBaseOciLayoutUtils(ctlr.StoreController, log.NewLogger("debug", "")) + olu := ocilayout.NewBaseOciLayoutUtils(ctlr.StoreController, log.NewLogger("debug", "")) // Initialize the objects containing the expected data repos, err := olu.GetRepositories() @@ -3717,7 +3544,7 @@ func TestGlobalSearch(t *testing.T) { ) So(err, ShouldBeNil) - olu := NewBaseOciLayoutUtils(ctlr.StoreController, log.NewLogger("debug", "")) + olu := ocilayout.NewBaseOciLayoutUtils(ctlr.StoreController, log.NewLogger("debug", "")) // Initialize the objects containing the expected data repos, err := olu.GetRepositories() diff --git a/pkg/storage/storage_controller.go b/pkg/storage/storage_controller.go index 9cc91047..5668f847 100644 --- a/pkg/storage/storage_controller.go +++ b/pkg/storage/storage_controller.go @@ -16,7 +16,7 @@ type BlobUpload struct { ID string } -func getRoutePrefix(name string) string { +func GetRoutePrefix(name string) string { names := strings.SplitN(name, "/", 2) //nolint:gomnd if len(names) != 2 { //nolint:gomnd @@ -32,7 +32,7 @@ func getRoutePrefix(name string) string { func (sc StoreController) GetImageStore(name string) ImageStore { if sc.SubStore != nil { // SubStore is being provided, now we need to find equivalent image store and this will be found by splitting name - prefixName := getRoutePrefix(name) + prefixName := GetRoutePrefix(name) imgStore, ok := sc.SubStore[prefixName] if !ok { diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 09392952..b20b3edc 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -902,3 +902,16 @@ func TestStorageHandler(t *testing.T) { }) } } + +func TestRoutePrefix(t *testing.T) { + Convey("Test route prefix", t, func() { + routePrefix := storage.GetRoutePrefix("test:latest") + So(routePrefix, ShouldEqual, "/") + + routePrefix = storage.GetRoutePrefix("a/test:latest") + So(routePrefix, ShouldEqual, "/a") + + routePrefix = storage.GetRoutePrefix("a/b/test:latest") + So(routePrefix, ShouldEqual, "/a") + }) +} diff --git a/pkg/test/mocks/cve_mock.go b/pkg/test/mocks/cve_mock.go index d32fa57e..514699d3 100644 --- a/pkg/test/mocks/cve_mock.go +++ b/pkg/test/mocks/cve_mock.go @@ -1,14 +1,13 @@ package mocks import ( - "zotregistry.io/zot/pkg/extensions/search/common" cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" ) type CveInfoMock struct { - GetImageListForCVEFn func(repo, cveID string) ([]common.TagInfo, error) - GetImageListWithCVEFixedFn func(repo, cveID string) ([]common.TagInfo, error) + GetImageListForCVEFn func(repo, cveID string) ([]cvemodel.TagInfo, error) + GetImageListWithCVEFixedFn func(repo, cveID string) ([]cvemodel.TagInfo, error) GetCVEListForImageFn func(repo string, reference string, searchedCVE string, pageInput cveinfo.PageInput, ) ([]cvemodel.CVE, cveinfo.PageInfo, error) GetCVESummaryForImageFn func(repo string, reference string, @@ -17,20 +16,20 @@ type CveInfoMock struct { UpdateDBFn func() error } -func (cveInfo CveInfoMock) GetImageListForCVE(repo, cveID string) ([]common.TagInfo, error) { +func (cveInfo CveInfoMock) GetImageListForCVE(repo, cveID string) ([]cvemodel.TagInfo, error) { if cveInfo.GetImageListForCVEFn != nil { return cveInfo.GetImageListForCVEFn(repo, cveID) } - return []common.TagInfo{}, nil + return []cvemodel.TagInfo{}, nil } -func (cveInfo CveInfoMock) GetImageListWithCVEFixed(repo, cveID string) ([]common.TagInfo, error) { +func (cveInfo CveInfoMock) GetImageListWithCVEFixed(repo, cveID string) ([]cvemodel.TagInfo, error) { if cveInfo.GetImageListWithCVEFixedFn != nil { return cveInfo.GetImageListWithCVEFixedFn(repo, cveID) } - return []common.TagInfo{}, nil + return []cvemodel.TagInfo{}, nil } func (cveInfo CveInfoMock) GetCVEListForImage(repo string, reference string, diff --git a/pkg/test/mocks/oci_mock.go b/pkg/test/mocks/oci_mock.go index 5af34bae..507b1f04 100644 --- a/pkg/test/mocks/oci_mock.go +++ b/pkg/test/mocks/oci_mock.go @@ -4,7 +4,8 @@ import ( godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" - "zotregistry.io/zot/pkg/extensions/search/common" + "zotregistry.io/zot/pkg/common" + cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" ) type OciLayoutUtilsMock struct { @@ -12,11 +13,11 @@ type OciLayoutUtilsMock struct { GetImageManifestsFn func(repo string) ([]ispec.Descriptor, error) GetImageBlobManifestFn func(repo string, digest godigest.Digest) (ispec.Manifest, error) GetImageInfoFn func(repo string, digest godigest.Digest) (ispec.Image, error) - GetImageTagsWithTimestampFn func(repo string) ([]common.TagInfo, error) + GetImageTagsWithTimestampFn func(repo string) ([]cvemodel.TagInfo, error) GetImagePlatformFn func(imageInfo ispec.Image) (string, string) GetImageManifestSizeFn func(repo string, manifestDigest godigest.Digest) int64 GetImageConfigSizeFn func(repo string, manifestDigest godigest.Digest) int64 - GetRepoLastUpdatedFn func(repo string) (common.TagInfo, error) + GetRepoLastUpdatedFn func(repo string) (cvemodel.TagInfo, error) GetExpandedRepoInfoFn func(name string) (common.RepoInfo, error) GetImageConfigInfoFn func(repo string, manifestDigest godigest.Digest) (ispec.Image, error) CheckManifestSignatureFn func(name string, digest godigest.Digest) bool @@ -64,12 +65,12 @@ func (olum OciLayoutUtilsMock) GetImageInfo(repo string, digest godigest.Digest) return ispec.Image{}, nil } -func (olum OciLayoutUtilsMock) GetImageTagsWithTimestamp(repo string) ([]common.TagInfo, error) { +func (olum OciLayoutUtilsMock) GetImageTagsWithTimestamp(repo string) ([]cvemodel.TagInfo, error) { if olum.GetImageTagsWithTimestampFn != nil { return olum.GetImageTagsWithTimestampFn(repo) } - return []common.TagInfo{}, nil + return []cvemodel.TagInfo{}, nil } func (olum OciLayoutUtilsMock) GetImagePlatform(imageInfo ispec.Image) (string, string) { @@ -96,12 +97,12 @@ func (olum OciLayoutUtilsMock) GetImageConfigSize(repo string, manifestDigest go return 0 } -func (olum OciLayoutUtilsMock) GetRepoLastUpdated(repo string) (common.TagInfo, error) { +func (olum OciLayoutUtilsMock) GetRepoLastUpdated(repo string) (cvemodel.TagInfo, error) { if olum.GetRepoLastUpdatedFn != nil { return olum.GetRepoLastUpdatedFn(repo) } - return common.TagInfo{}, nil + return cvemodel.TagInfo{}, nil } func (olum OciLayoutUtilsMock) GetExpandedRepoInfo(name string) (common.RepoInfo, error) { diff --git a/pkg/test/oci_layout.go b/pkg/test/oci-layout/oci_layout.go similarity index 94% rename from pkg/test/oci_layout.go rename to pkg/test/oci-layout/oci_layout.go index 3184592d..f792e0dc 100644 --- a/pkg/test/oci_layout.go +++ b/pkg/test/oci-layout/oci_layout.go @@ -1,13 +1,14 @@ //go:build sync && scrub && metrics && search // +build sync,scrub,metrics,search -package test +package ocilayout import ( "encoding/json" goerrors "errors" "fmt" "path" + "sort" "strconv" "strings" "time" @@ -17,20 +18,22 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" zerr "zotregistry.io/zot/errors" - "zotregistry.io/zot/pkg/extensions/search/common" + "zotregistry.io/zot/pkg/common" + "zotregistry.io/zot/pkg/extensions/search/convert" + cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" ) -type OciLayoutUtils interface { //nolint: interfacebloat +type OciUtils interface { //nolint: interfacebloat GetImageManifest(repo string, reference string) (ispec.Manifest, godigest.Digest, error) GetImageManifests(repo string) ([]ispec.Descriptor, error) GetImageBlobManifest(repo string, digest godigest.Digest) (ispec.Manifest, error) GetImageInfo(repo string, configDigest godigest.Digest) (ispec.Image, error) - GetImageTagsWithTimestamp(repo string) ([]common.TagInfo, error) + GetImageTagsWithTimestamp(repo string) ([]cvemodel.TagInfo, error) GetImagePlatform(imageInfo ispec.Image) (string, string) GetImageManifestSize(repo string, manifestDigest godigest.Digest) int64 - GetRepoLastUpdated(repo string) (common.TagInfo, error) + GetRepoLastUpdated(repo string) (cvemodel.TagInfo, error) GetExpandedRepoInfo(name string) (common.RepoInfo, error) GetImageConfigInfo(repo string, manifestDigest godigest.Digest) (ispec.Image, error) CheckManifestSignature(name string, digest godigest.Digest) bool @@ -180,8 +183,8 @@ func (olu BaseOciLayoutUtils) GetImageInfo(repo string, configDigest godigest.Di } // GetImageTagsWithTimestamp returns a list of image tags with timestamp available in the specified repository. -func (olu BaseOciLayoutUtils) GetImageTagsWithTimestamp(repo string) ([]common.TagInfo, error) { - tagsInfo := make([]common.TagInfo, 0) +func (olu BaseOciLayoutUtils) GetImageTagsWithTimestamp(repo string) ([]cvemodel.TagInfo, error) { + tagsInfo := make([]cvemodel.TagInfo, 0) manifests, err := olu.GetImageManifests(repo) if err != nil { @@ -212,10 +215,10 @@ func (olu BaseOciLayoutUtils) GetImageTagsWithTimestamp(repo string) ([]common.T timeStamp := common.GetImageLastUpdated(imageInfo) tagsInfo = append(tagsInfo, - common.TagInfo{ + cvemodel.TagInfo{ Name: val, Timestamp: timeStamp, - Descriptor: common.Descriptor{ + Descriptor: cvemodel.Descriptor{ Digest: digest, MediaType: manifest.MediaType, }, @@ -330,13 +333,13 @@ func (olu BaseOciLayoutUtils) GetImageConfigSize(repo string, manifestDigest god return imageBlobManifest.Config.Size } -func (olu BaseOciLayoutUtils) GetRepoLastUpdated(repo string) (common.TagInfo, error) { +func (olu BaseOciLayoutUtils) GetRepoLastUpdated(repo string) (cvemodel.TagInfo, error) { tagsInfo, err := olu.GetImageTagsWithTimestamp(repo) if err != nil || len(tagsInfo) == 0 { - return common.TagInfo{}, err + return cvemodel.TagInfo{}, err } - latestTag := common.GetLatestTag(tagsInfo) + latestTag := GetLatestTag(tagsInfo) return latestTag, nil } @@ -433,7 +436,7 @@ func (olu BaseOciLayoutUtils) GetExpandedRepoInfo(repoName string) (common.RepoI imageSize := imageLayersSize + manifestSize + configSize // get image info from manifest annotation, if not found get from image config labels. - annotations := common.GetAnnotations(manifest.Annotations, imageConfigInfo.Config.Labels) + annotations := convert.GetAnnotations(manifest.Annotations, imageConfigInfo.Config.Labels) if annotations.Vendor != "" { repoVendorsSet[annotations.Vendor] = true @@ -579,3 +582,11 @@ func (olu BaseOciLayoutUtils) ExtractImageDetails( return digest, &manifest, &imageConfig, nil } + +func GetLatestTag(allTags []cvemodel.TagInfo) cvemodel.TagInfo { + sort.Slice(allTags, func(i, j int) bool { + return allTags[i].Timestamp.Before(allTags[j].Timestamp) + }) + + return allTags[len(allTags)-1] +} diff --git a/pkg/test/oci_layout_test.go b/pkg/test/oci-layout/oci_layout_test.go similarity index 80% rename from pkg/test/oci_layout_test.go rename to pkg/test/oci-layout/oci_layout_test.go index d585622f..c962adb2 100644 --- a/pkg/test/oci_layout_test.go +++ b/pkg/test/oci-layout/oci_layout_test.go @@ -1,7 +1,7 @@ //go:build sync && scrub && metrics && search // +build sync,scrub,metrics,search -package test_test +package ocilayout_test import ( "encoding/json" @@ -9,6 +9,7 @@ import ( "os" "path" "testing" + "time" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -19,13 +20,17 @@ import ( "zotregistry.io/zot/pkg/api/config" extconf "zotregistry.io/zot/pkg/extensions/config" "zotregistry.io/zot/pkg/extensions/monitoring" + cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/local" . "zotregistry.io/zot/pkg/test" "zotregistry.io/zot/pkg/test/mocks" + ocilayout "zotregistry.io/zot/pkg/test/oci-layout" ) +var ErrTestError = fmt.Errorf("testError") + func TestBaseOciLayoutUtils(t *testing.T) { manifestDigest := GetTestBlobDigest("zot-test", "config").String() @@ -37,7 +42,7 @@ func TestBaseOciLayoutUtils(t *testing.T) { } storeController := storage.StoreController{DefaultStore: mockStoreController} - olu := NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + olu := ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) size := olu.GetImageManifestSize("", "") So(size, ShouldBeZeroValue) @@ -51,7 +56,7 @@ func TestBaseOciLayoutUtils(t *testing.T) { } storeController := storage.StoreController{DefaultStore: mockStoreController} - olu := NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + olu := ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) size := olu.GetImageConfigSize("", "") So(size, ShouldBeZeroValue) @@ -86,7 +91,7 @@ func TestBaseOciLayoutUtils(t *testing.T) { } storeController := storage.StoreController{DefaultStore: mockStoreController} - olu := NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + olu := ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) size := olu.GetImageConfigSize("", "") So(size, ShouldBeZeroValue) @@ -100,7 +105,7 @@ func TestBaseOciLayoutUtils(t *testing.T) { } storeController := storage.StoreController{DefaultStore: mockStoreController} - olu := NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + olu := ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) _, err := olu.GetRepoLastUpdated("") So(err, ShouldNotBeNil) @@ -126,7 +131,7 @@ func TestBaseOciLayoutUtils(t *testing.T) { } storeController := storage.StoreController{DefaultStore: mockStoreController} - olu := NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + olu := ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) _, err = olu.GetImageTagsWithTimestamp("rep") So(err, ShouldNotBeNil) @@ -170,7 +175,7 @@ func TestBaseOciLayoutUtils(t *testing.T) { } storeController := storage.StoreController{DefaultStore: mockStoreController} - olu := NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + olu := ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) _, err = olu.GetImageTagsWithTimestamp("repo") So(err, ShouldNotBeNil) @@ -213,7 +218,7 @@ func TestBaseOciLayoutUtils(t *testing.T) { } storeController := storage.StoreController{DefaultStore: mockStoreController} - olu := NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + olu := ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) _, err = olu.GetExpandedRepoInfo("rep") So(err, ShouldNotBeNil) @@ -226,7 +231,7 @@ func TestBaseOciLayoutUtils(t *testing.T) { } storeController = storage.StoreController{DefaultStore: mockStoreController} - olu = NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + olu = ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) _, err = olu.GetExpandedRepoInfo("rep") So(err, ShouldNotBeNil) @@ -243,7 +248,7 @@ func TestBaseOciLayoutUtils(t *testing.T) { } storeController = storage.StoreController{DefaultStore: mockStoreController} - olu = NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + olu = ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) _, err = olu.GetExpandedRepoInfo("rep") So(err, ShouldBeNil) @@ -257,7 +262,7 @@ func TestBaseOciLayoutUtils(t *testing.T) { } storeController := storage.StoreController{DefaultStore: mockStoreController} - olu := NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + olu := ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) _, err := olu.GetImageInfo("", "") So(err, ShouldNotBeNil) @@ -275,7 +280,7 @@ func TestBaseOciLayoutUtils(t *testing.T) { } storeController := storage.StoreController{DefaultStore: mockStoreController} - olu := NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + olu := ocilayout.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) check := olu.CheckManifestSignature("rep", godigest.FromString("")) So(check, ShouldBeFalse) @@ -324,7 +329,7 @@ func TestBaseOciLayoutUtils(t *testing.T) { ) So(err, ShouldBeNil) - olu = NewBaseOciLayoutUtils(ctlr.StoreController, log.NewLogger("debug", "")) + olu = ocilayout.NewBaseOciLayoutUtils(ctlr.StoreController, log.NewLogger("debug", "")) manifestList, err := olu.GetImageManifests(repo) So(err, ShouldBeNil) So(len(manifestList), ShouldEqual, 1) @@ -369,7 +374,7 @@ func TestExtractImageDetails(t *testing.T) { So(err, ShouldBeNil) configDigest := godigest.FromBytes(configBlob) - olu := NewBaseOciLayoutUtils(storeController, testLogger) + olu := ocilayout.NewBaseOciLayoutUtils(storeController, testLogger) resDigest, resManifest, resIspecImage, resErr := olu.ExtractImageDetails("zot-test", "latest", testLogger) So(string(resDigest), ShouldContainSubstring, "sha256:c52f15d2d4") So(resManifest.Config.Digest.String(), ShouldContainSubstring, configDigest.Encoded()) @@ -388,7 +393,7 @@ func TestExtractImageDetails(t *testing.T) { DefaultStore: imageStore, } - olu := NewBaseOciLayoutUtils(storeController, testLogger) + olu := ocilayout.NewBaseOciLayoutUtils(storeController, testLogger) resDigest, resManifest, resIspecImage, resErr := olu.ExtractImageDetails("zot-test", "latest", testLogger) So(resErr, ShouldEqual, zerr.ErrRepoNotFound) @@ -431,7 +436,7 @@ func TestExtractImageDetails(t *testing.T) { panic(err) } - olu := NewBaseOciLayoutUtils(storeController, testLogger) + olu := ocilayout.NewBaseOciLayoutUtils(storeController, testLogger) resDigest, resManifest, resIspecImage, resErr := olu.ExtractImageDetails("zot-test", "latest", testLogger) So(resErr, ShouldEqual, zerr.ErrBlobNotFound) So(string(resDigest), ShouldEqual, "") @@ -439,3 +444,47 @@ func TestExtractImageDetails(t *testing.T) { So(resIspecImage, ShouldBeNil) }) } + +func TestTagsInfo(t *testing.T) { + Convey("Test tags info", t, func() { + allTags := make([]cvemodel.TagInfo, 0) + + firstTag := cvemodel.TagInfo{ + Name: "1.0.0", + Descriptor: cvemodel.Descriptor{ + Digest: "sha256:eca04f027f414362596f2632746d8a178362170b9ac9af772011fedcc3877ebb", + MediaType: ispec.MediaTypeImageManifest, + }, + Timestamp: time.Now(), + } + secondTag := cvemodel.TagInfo{ + Name: "1.0.1", + Descriptor: cvemodel.Descriptor{ + Digest: "sha256:eca04f027f414362596f2632746d8a179362170b9ac9af772011fedcc3877ebb", + MediaType: ispec.MediaTypeImageManifest, + }, + Timestamp: time.Now(), + } + thirdTag := cvemodel.TagInfo{ + Name: "1.0.2", + Descriptor: cvemodel.Descriptor{ + Digest: "sha256:eca04f027f414362596f2632746d8a170362170b9ac9af772011fedcc3877ebb", + MediaType: ispec.MediaTypeImageManifest, + }, + Timestamp: time.Now(), + } + fourthTag := cvemodel.TagInfo{ + Name: "1.0.3", + Descriptor: cvemodel.Descriptor{ + Digest: "sha256:eca04f027f414362596f2632746d8a171362170b9ac9af772011fedcc3877ebb", + MediaType: ispec.MediaTypeImageManifest, + }, + Timestamp: time.Now(), + } + + allTags = append(allTags, firstTag, secondTag, thirdTag, fourthTag) + + latestTag := ocilayout.GetLatestTag(allTags) + So(latestTag.Name, ShouldEqual, "1.0.3") + }) +}