0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2025-01-06 22:40:28 -05:00
zot/pkg/extensions/search/convert/metadb.go
Andrei Aaron d0eb043be5
feat: Get the image LastUpdated timestamp from annotations (#2240)
Fallback to Created field and the History entries in the image config
only if the annotation "org.opencontainers.image.created" is not available

closes #2210

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
2024-02-14 09:14:24 -08:00

644 lines
19 KiB
Go

package convert
import (
"context"
"sort"
"strconv"
"time"
"github.com/99designs/gqlgen/graphql"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/vektah/gqlparser/v2/gqlerror"
zerr "zotregistry.dev/zot/errors"
zcommon "zotregistry.dev/zot/pkg/common"
cveinfo "zotregistry.dev/zot/pkg/extensions/search/cve"
"zotregistry.dev/zot/pkg/extensions/search/gql_generated"
"zotregistry.dev/zot/pkg/extensions/search/pagination"
"zotregistry.dev/zot/pkg/log"
mTypes "zotregistry.dev/zot/pkg/meta/types"
reqCtx "zotregistry.dev/zot/pkg/requestcontext"
)
type SkipQGLField struct {
Vulnerabilities bool
}
func UpdateLastUpdatedTimestamp(repoLastUpdatedTimestamp *time.Time,
lastUpdatedImageSummary *gql_generated.ImageSummary, imageSummary *gql_generated.ImageSummary,
) *gql_generated.ImageSummary {
newLastUpdatedImageSummary := lastUpdatedImageSummary
if repoLastUpdatedTimestamp.Equal(time.Time{}) {
// initialize with first time value
*repoLastUpdatedTimestamp = *imageSummary.LastUpdated
newLastUpdatedImageSummary = imageSummary
} else if repoLastUpdatedTimestamp.Before(*imageSummary.LastUpdated) {
*repoLastUpdatedTimestamp = *imageSummary.LastUpdated
newLastUpdatedImageSummary = imageSummary
}
return newLastUpdatedImageSummary
}
func getReferrers(referrersInfo []mTypes.ReferrerInfo) []*gql_generated.Referrer {
referrers := make([]*gql_generated.Referrer, 0, len(referrersInfo))
for _, referrerInfo := range referrersInfo {
referrerInfo := referrerInfo
referrers = append(referrers, &gql_generated.Referrer{
MediaType: &referrerInfo.MediaType,
ArtifactType: &referrerInfo.ArtifactType,
Size: &referrerInfo.Size,
Digest: &referrerInfo.Digest,
Annotations: getAnnotationsFromMap(referrerInfo.Annotations),
})
}
return referrers
}
func getAnnotationsFromMap(annotationsMap map[string]string) []*gql_generated.Annotation {
annotations := make([]*gql_generated.Annotation, 0, len(annotationsMap))
for key, value := range annotationsMap {
key := key
value := value
annotations = append(annotations, &gql_generated.Annotation{
Key: &key,
Value: &value,
})
}
return annotations
}
func getImageBlobsInfo(manifestDigest string, manifestSize int64, configDigest string, configSize int64,
layers []ispec.Descriptor,
) (int64, map[string]int64) {
imageBlobsMap := map[string]int64{}
imageSize := int64(0)
// add config size
imageSize += configSize
imageBlobsMap[configDigest] = configSize
// add manifest size
imageSize += manifestSize
imageBlobsMap[manifestDigest] = manifestSize
// add layers size
for _, layer := range layers {
imageBlobsMap[layer.Digest.String()] = layer.Size
imageSize += layer.Size
}
return imageSize, imageBlobsMap
}
func RepoMeta2ImageSummaries(ctx context.Context, repoMeta mTypes.RepoMeta,
imageMeta map[string]mTypes.ImageMeta, skip SkipQGLField, cveInfo cveinfo.CveInfo,
) []*gql_generated.ImageSummary {
imageSummaries := make([]*gql_generated.ImageSummary, 0, len(repoMeta.Tags))
// Make sure the tags are sorted
// We need to implement a proper fix for this taking into account
// the sorting criteria used in the requested page
tags := make([]string, 0, len(repoMeta.Tags))
for tag := range repoMeta.Tags {
tags = append(tags, tag)
}
// Sorting ascending by tag name should do for now
sort.Strings(tags)
for _, tag := range tags {
descriptor := repoMeta.Tags[tag]
imageSummary, _, err := FullImageMeta2ImageSummary(ctx, GetFullImageMeta(tag, repoMeta, imageMeta[descriptor.Digest]))
if err != nil {
continue
}
// CVE scanning is expensive, only scan for final slice of results
updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo)
imageSummaries = append(imageSummaries, imageSummary)
}
return imageSummaries
}
func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta mTypes.RepoMeta,
imageMetaMap map[string]mTypes.ImageMeta, skip SkipQGLField, cveInfo cveinfo.CveInfo, log log.Logger,
) (*gql_generated.RepoSummary, []*gql_generated.ImageSummary) {
repoName := repoMeta.Name
imageSummaries := make([]*gql_generated.ImageSummary, 0, len(repoMeta.Tags))
userCanDeleteTag, _ := reqCtx.CanDelete(ctx, repoName)
for tag, descriptor := range repoMeta.Tags {
imageMeta := imageMetaMap[descriptor.Digest]
imageSummary, _, err := FullImageMeta2ImageSummary(ctx, GetFullImageMeta(tag, repoMeta, imageMeta))
if err != nil {
log.Error().Str("repository", repoName).Str("reference", tag).Str("component", "metadb").
Msg("error while converting descriptor for image")
continue
}
imageSummary.IsDeletable = &userCanDeleteTag
updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo)
imageSummaries = append(imageSummaries, imageSummary)
}
repoSummary := RepoMeta2RepoSummary(ctx, repoMeta, imageMetaMap)
updateRepoSummaryVulnerabilities(ctx, repoSummary, skip, cveInfo)
return repoSummary, imageSummaries
}
func GetFullImageMeta(tag string, repoMeta mTypes.RepoMeta, imageMeta mTypes.ImageMeta,
) mTypes.FullImageMeta {
return mTypes.FullImageMeta{
Repo: repoMeta.Name,
Tag: tag,
MediaType: imageMeta.MediaType,
Digest: imageMeta.Digest,
Size: imageMeta.Size,
Index: imageMeta.Index,
Manifests: GetFullManifestMeta(repoMeta, imageMeta.Manifests),
Referrers: repoMeta.Referrers[imageMeta.Digest.String()],
Statistics: repoMeta.Statistics[imageMeta.Digest.String()],
Signatures: repoMeta.Signatures[imageMeta.Digest.String()],
}
}
func GetFullManifestMeta(repoMeta mTypes.RepoMeta, manifests []mTypes.ManifestMeta) []mTypes.FullManifestMeta {
results := make([]mTypes.FullManifestMeta, 0, len(manifests))
for i := range manifests {
results = append(results, mTypes.FullManifestMeta{
ManifestMeta: manifests[i],
Referrers: repoMeta.Referrers[manifests[i].Digest.String()],
Statistics: repoMeta.Statistics[manifests[i].Digest.String()],
Signatures: repoMeta.Signatures[manifests[i].Digest.String()],
})
}
return results
}
func StringMap2Annotations(strMap map[string]string) []*gql_generated.Annotation {
annotations := make([]*gql_generated.Annotation, 0, len(strMap))
for key, value := range strMap {
key := key
value := value
annotations = append(annotations, &gql_generated.Annotation{
Key: &key,
Value: &value,
})
}
return annotations
}
func GetPreloads(ctx context.Context) map[string]bool {
if !graphql.HasOperationContext(ctx) {
return map[string]bool{}
}
nestedPreloads := GetNestedPreloads(
graphql.GetOperationContext(ctx),
graphql.CollectFieldsCtx(ctx, nil),
"",
)
preloads := map[string]bool{}
for _, str := range nestedPreloads {
preloads[str] = true
}
return preloads
}
func GetNestedPreloads(ctx *graphql.OperationContext, fields []graphql.CollectedField, prefix string,
) []string {
preloads := []string{}
for _, column := range fields {
prefixColumn := GetPreloadString(prefix, column.Name)
preloads = append(preloads, prefixColumn)
preloads = append(preloads,
GetNestedPreloads(ctx, graphql.CollectFields(ctx, column.Selections, nil), prefixColumn)...,
)
}
return preloads
}
func GetPreloadString(prefix, name string) string {
if len(prefix) > 0 {
return prefix + "." + name
}
return name
}
func GetSignaturesInfo(isSigned bool, signatures mTypes.ManifestSignatures) []*gql_generated.SignatureSummary {
signaturesInfo := []*gql_generated.SignatureSummary{}
if !isSigned {
return signaturesInfo
}
for sigType, signatures := range signatures {
for _, sig := range signatures {
for _, layer := range sig.LayersInfo {
var (
isTrusted bool
author string
tool string
)
if layer.Signer != "" {
author = layer.Signer
if !layer.Date.IsZero() && time.Now().After(layer.Date) {
isTrusted = false
} else {
isTrusted = true
}
} else {
isTrusted = false
author = ""
}
tool = sigType
signaturesInfo = append(signaturesInfo,
&gql_generated.SignatureSummary{Tool: &tool, IsTrusted: &isTrusted, Author: &author})
}
}
}
return signaturesInfo
}
func PaginatedRepoMeta2RepoSummaries(ctx context.Context, repoMetaList []mTypes.RepoMeta,
imageMetaMap map[string]mTypes.ImageMeta, filter mTypes.Filter, pageInput pagination.PageInput,
cveInfo cveinfo.CveInfo, skip SkipQGLField,
) ([]*gql_generated.RepoSummary, zcommon.PageInfo, error) {
reposPageFinder, err := pagination.NewRepoSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy)
if err != nil {
return []*gql_generated.RepoSummary{}, zcommon.PageInfo{}, err
}
for _, repoMeta := range repoMetaList {
repoSummary := RepoMeta2RepoSummary(ctx, repoMeta, imageMetaMap)
if RepoSumAcceptedByFilter(repoSummary, filter) {
reposPageFinder.Add(repoSummary)
}
}
page, pageInfo := reposPageFinder.Page()
// CVE scanning is expensive, only scan for the current page
for _, repoSummary := range page {
updateRepoSummaryVulnerabilities(ctx, repoSummary, skip, cveInfo)
}
return page, pageInfo, nil
}
func RepoMeta2RepoSummary(ctx context.Context, repoMeta mTypes.RepoMeta,
imageMetaMap map[string]mTypes.ImageMeta,
) *gql_generated.RepoSummary {
var (
repoName = repoMeta.Name
lastUpdatedImage = deref(repoMeta.LastUpdatedImage, mTypes.LastUpdatedImage{})
lastUpdatedImageMeta = imageMetaMap[lastUpdatedImage.Digest]
lastUpdatedTag = lastUpdatedImage.Tag
repoLastUpdatedTimestamp = lastUpdatedImage.LastUpdated
repoPlatforms = repoMeta.Platforms
repoVendors = repoMeta.Vendors
repoDownloadCount = repoMeta.DownloadCount
repoStarCount = repoMeta.StarCount
repoIsUserStarred = repoMeta.IsStarred // value specific to the current user
repoIsUserBookMarked = repoMeta.IsBookmarked // value specific to the current user
repoSize = repoMeta.Size
)
if repoLastUpdatedTimestamp == nil {
repoLastUpdatedTimestamp = &time.Time{}
}
imageSummary, _, err := FullImageMeta2ImageSummary(ctx, GetFullImageMeta(lastUpdatedTag, repoMeta,
lastUpdatedImageMeta))
_ = err
return &gql_generated.RepoSummary{
Name: &repoName,
LastUpdated: repoLastUpdatedTimestamp,
Size: ref(strconv.FormatInt(repoSize, 10)),
Platforms: getGqlPlatforms(repoPlatforms),
Vendors: getGqlVendors(repoVendors),
NewestImage: imageSummary,
DownloadCount: &repoDownloadCount,
StarCount: &repoStarCount,
IsBookmarked: &repoIsUserBookMarked,
IsStarred: &repoIsUserStarred,
Rank: ref(repoMeta.Rank),
}
}
func getGqlVendors(repoVendors []string) []*string {
result := make([]*string, 0, len(repoVendors))
for i := range repoVendors {
result = append(result, &repoVendors[i])
}
return result
}
func getGqlPlatforms(repoPlatforms []ispec.Platform) []*gql_generated.Platform {
result := make([]*gql_generated.Platform, 0, len(repoPlatforms))
for i := range repoPlatforms {
result = append(result, &gql_generated.Platform{
Os: ref(repoPlatforms[i].OS),
Arch: ref(getArch(repoPlatforms[i].Architecture, repoPlatforms[i].Variant)),
})
}
return result
}
type (
ManifestDigest = string
BlobDigest = string
)
func FullImageMeta2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullImageMeta,
) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) {
switch fullImageMeta.MediaType {
case ispec.MediaTypeImageManifest:
return ImageManifest2ImageSummary(ctx, fullImageMeta)
case ispec.MediaTypeImageIndex:
return ImageIndex2ImageSummary(ctx, fullImageMeta)
default:
return nil, nil, zerr.ErrMediaTypeNotSupported
}
}
func ImageIndex2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullImageMeta,
) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) {
var (
repo = fullImageMeta.Repo
tag = fullImageMeta.Tag
indexLastUpdated time.Time
isSigned = isImageSigned(fullImageMeta.Signatures)
indexSize = int64(0)
manifestAnnotations *ImageAnnotations
manifestSummaries = make([]*gql_generated.ManifestSummary, 0, len(fullImageMeta.Manifests))
indexBlobs = map[string]int64{}
indexDigestStr = fullImageMeta.Digest.String()
indexMediaType = ispec.MediaTypeImageIndex
)
for _, imageManifest := range fullImageMeta.Manifests {
imageManifestSummary, manifestBlobs, err := ImageManifest2ImageSummary(ctx, mTypes.FullImageMeta{
Repo: fullImageMeta.Repo,
Tag: fullImageMeta.Tag,
MediaType: ispec.MediaTypeImageManifest,
Digest: imageManifest.Digest,
Size: imageManifest.Size,
Manifests: []mTypes.FullManifestMeta{imageManifest},
Referrers: imageManifest.Referrers,
Statistics: imageManifest.Statistics,
Signatures: imageManifest.Signatures,
})
if err != nil {
return &gql_generated.ImageSummary{}, map[string]int64{}, err
}
manifestSize := int64(0)
for digest, size := range manifestBlobs {
indexBlobs[digest] = size
manifestSize += size
}
if indexLastUpdated.Before(*imageManifestSummary.LastUpdated) {
indexLastUpdated = *imageManifestSummary.LastUpdated
}
annotations := GetAnnotations(imageManifest.Manifest.Annotations, imageManifest.Config.Config.Labels)
if manifestAnnotations == nil {
manifestAnnotations = &annotations
}
indexSize += manifestSize
manifestSummaries = append(manifestSummaries, imageManifestSummary.Manifests[0])
}
signaturesInfo := GetSignaturesInfo(isSigned, fullImageMeta.Signatures)
if manifestAnnotations == nil {
manifestAnnotations = &ImageAnnotations{}
}
annotations := GetIndexAnnotations(fullImageMeta.Index.Annotations, manifestAnnotations)
imageLastUpdated := annotations.Created
if imageLastUpdated == nil {
imageLastUpdated = &indexLastUpdated
}
indexSummary := gql_generated.ImageSummary{
RepoName: &repo,
Tag: &tag,
Digest: &indexDigestStr,
MediaType: &indexMediaType,
Manifests: manifestSummaries,
LastUpdated: imageLastUpdated,
IsSigned: &isSigned,
SignatureInfo: signaturesInfo,
Size: ref(strconv.FormatInt(indexSize, 10)),
DownloadCount: ref(fullImageMeta.Statistics.DownloadCount),
Description: &annotations.Description,
Title: &annotations.Title,
Documentation: &annotations.Documentation,
Licenses: &annotations.Licenses,
Labels: &annotations.Labels,
Source: &annotations.Source,
Vendor: &annotations.Vendor,
Authors: &annotations.Authors,
Referrers: getReferrers(fullImageMeta.Referrers),
}
return &indexSummary, indexBlobs, nil
}
func ImageManifest2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullImageMeta,
) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) {
manifest := fullImageMeta.Manifests[0]
var (
repoName = fullImageMeta.Repo
tag = fullImageMeta.Tag
configDigest = manifest.Manifest.Config.Digest.String()
configSize = manifest.Manifest.Config.Size
manifestDigest = manifest.Digest.String()
manifestSize = manifest.Size
mediaType = manifest.Manifest.MediaType
artifactType = zcommon.GetManifestArtifactType(fullImageMeta.Manifests[0].Manifest)
platform = getPlatform(manifest.Config.Platform)
downloadCount = fullImageMeta.Statistics.DownloadCount
isSigned = isImageSigned(fullImageMeta.Signatures)
)
imageSize, imageBlobsMap := getImageBlobsInfo(manifestDigest, manifestSize, configDigest, configSize,
manifest.Manifest.Layers)
imageSizeStr := strconv.FormatInt(imageSize, 10)
annotations := GetAnnotations(manifest.Manifest.Annotations, manifest.Config.Config.Labels)
authors := annotations.Authors
if authors == "" {
authors = manifest.Config.Author
}
imageLastUpdated := annotations.Created
if imageLastUpdated == nil {
configCreated := zcommon.GetImageLastUpdated(manifest.Config)
imageLastUpdated = &configCreated
}
historyEntries, err := getAllHistory(manifest.Manifest, manifest.Config)
if err != nil {
graphql.AddError(ctx, gqlerror.Errorf("error generating history on tag %s in repo %s: "+
"manifest digest: %s, error: %s", tag, repoName, manifest.Digest, err.Error()))
}
signaturesInfo := GetSignaturesInfo(isSigned, fullImageMeta.Signatures)
manifestSummary := gql_generated.ManifestSummary{
Digest: &manifestDigest,
ConfigDigest: &configDigest,
LastUpdated: imageLastUpdated,
Size: &imageSizeStr,
IsSigned: &isSigned,
SignatureInfo: signaturesInfo,
Platform: &platform,
DownloadCount: &downloadCount,
Layers: getLayersSummaries(manifest.Manifest),
History: historyEntries,
Referrers: getReferrers(fullImageMeta.Referrers),
ArtifactType: &artifactType,
}
imageSummary := gql_generated.ImageSummary{
RepoName: &repoName,
Tag: &tag,
Digest: &manifestDigest,
MediaType: &mediaType,
Manifests: []*gql_generated.ManifestSummary{&manifestSummary},
LastUpdated: imageLastUpdated,
IsSigned: &isSigned,
SignatureInfo: signaturesInfo,
Size: &imageSizeStr,
DownloadCount: &downloadCount,
Description: &annotations.Description,
Title: &annotations.Title,
Documentation: &annotations.Documentation,
Licenses: &annotations.Licenses,
Labels: &annotations.Labels,
Source: &annotations.Source,
Vendor: &annotations.Vendor,
Authors: &authors,
Referrers: manifestSummary.Referrers,
}
return &imageSummary, imageBlobsMap, nil
}
func isImageSigned(manifestSignatures mTypes.ManifestSignatures) bool {
for _, signatures := range manifestSignatures {
if len(signatures) > 0 {
return true
}
}
return false
}
func getPlatform(platform ispec.Platform) gql_generated.Platform {
return gql_generated.Platform{
Os: ref(platform.OS),
Arch: ref(getArch(platform.Architecture, platform.Variant)),
}
}
func getArch(arch string, variant string) string {
if variant != "" {
arch = arch + "/" + variant
}
return arch
}
func ref[T any](val T) *T {
ref := val
return &ref
}
func deref[T any](pointer *T, defaultVal T) T {
if pointer != nil {
return *pointer
}
return defaultVal
}
func PaginatedFullImageMeta2ImageSummaries(ctx context.Context, imageMetaList []mTypes.FullImageMeta, skip SkipQGLField,
cveInfo cveinfo.CveInfo, filter mTypes.Filter, pageInput pagination.PageInput,
) ([]*gql_generated.ImageSummary, zcommon.PageInfo, error) {
imagePageFinder, err := pagination.NewImgSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy)
if err != nil {
return []*gql_generated.ImageSummary{}, zcommon.PageInfo{}, err
}
for _, imageMeta := range imageMetaList {
imageSummary, _, err := FullImageMeta2ImageSummary(ctx, imageMeta)
if err != nil {
continue
}
if ImgSumAcceptedByFilter(imageSummary, filter) {
imagePageFinder.Add(imageSummary)
}
}
page, pageInfo := imagePageFinder.Page()
for _, imageSummary := range page {
// CVE scanning is expensive, only scan for this page
updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo)
}
return page, pageInfo, nil
}