2023-07-18 12:27:26 -05:00
|
|
|
package meta
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
import (
|
2023-11-01 11:16:18 -05:00
|
|
|
"context"
|
2023-01-09 15:37:44 -05:00
|
|
|
"encoding/json"
|
|
|
|
"errors"
|
2023-08-23 11:29:23 -05:00
|
|
|
"time"
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
godigest "github.com/opencontainers/go-digest"
|
|
|
|
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
|
|
|
|
|
|
|
zerr "zotregistry.io/zot/errors"
|
2023-05-10 12:15:33 -05:00
|
|
|
zcommon "zotregistry.io/zot/pkg/common"
|
2023-01-09 15:37:44 -05:00
|
|
|
"zotregistry.io/zot/pkg/log"
|
2023-10-30 15:06:04 -05:00
|
|
|
"zotregistry.io/zot/pkg/meta/convert"
|
2023-07-18 12:27:26 -05:00
|
|
|
mTypes "zotregistry.io/zot/pkg/meta/types"
|
2023-01-09 15:37:44 -05:00
|
|
|
"zotregistry.io/zot/pkg/storage"
|
2023-05-26 13:08:19 -05:00
|
|
|
storageTypes "zotregistry.io/zot/pkg/storage/types"
|
2023-01-09 15:37:44 -05:00
|
|
|
)
|
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
const (
|
|
|
|
CosignType = "cosign"
|
|
|
|
NotationType = "notation"
|
|
|
|
)
|
|
|
|
|
2023-03-10 13:37:29 -05:00
|
|
|
// ParseStorage will sync all repos found in the rootdirectory of the oci layout that zot was deployed on with the
|
|
|
|
// ParseStorage database.
|
2023-07-18 12:27:26 -05:00
|
|
|
func ParseStorage(metaDB mTypes.MetaDB, storeController storage.StoreController, log log.Logger) error {
|
2023-09-17 17:12:20 -05:00
|
|
|
log.Info().Msg("Started parsing storage and updating MetaDB")
|
|
|
|
|
2023-01-09 15:37:44 -05:00
|
|
|
allRepos, err := getAllRepos(storeController)
|
|
|
|
if err != nil {
|
|
|
|
rootDir := storeController.DefaultStore.RootDir()
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Str("rootDir", rootDir).
|
|
|
|
Msg("load-local-layout: failed to get all repo names present under rootDir")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
for i, repo := range allRepos {
|
|
|
|
log.Info().Int("total", len(allRepos)).Int("progress", i).Str("current-repo", repo).
|
|
|
|
Msgf("parsing next repo '%s'", repo)
|
|
|
|
|
2023-07-18 12:27:26 -05:00
|
|
|
err := ParseRepo(repo, metaDB, storeController, log)
|
2023-01-09 15:37:44 -05:00
|
|
|
if err != nil {
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Str("repository", repo).Msg("load-local-layout: failed to sync repo")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-09-17 17:12:20 -05:00
|
|
|
log.Info().Msg("Done parsing storage and updating MetaDB")
|
|
|
|
|
2023-01-09 15:37:44 -05:00
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-03-10 13:37:29 -05:00
|
|
|
// ParseRepo reads the contents of a repo and syncs all images and signatures found.
|
2023-07-18 12:27:26 -05:00
|
|
|
func ParseRepo(repo string, metaDB mTypes.MetaDB, storeController storage.StoreController, log log.Logger) error {
|
2023-01-09 15:37:44 -05:00
|
|
|
imageStore := storeController.GetImageStore(repo)
|
|
|
|
|
2023-08-23 11:29:23 -05:00
|
|
|
var lockLatency time.Time
|
|
|
|
|
|
|
|
imageStore.RLock(&lockLatency)
|
|
|
|
defer imageStore.RUnlock(&lockLatency)
|
|
|
|
|
2023-01-09 15:37:44 -05:00
|
|
|
indexBlob, err := imageStore.GetIndexContent(repo)
|
|
|
|
if err != nil {
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Str("repository", repo).Msg("load-repo: failed to read index.json for repo")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
var indexContent ispec.Index
|
|
|
|
|
|
|
|
err = json.Unmarshal(indexBlob, &indexContent)
|
|
|
|
if err != nil {
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Str("repository", repo).Msg("load-repo: failed to unmarshal index.json for repo")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
err = metaDB.ResetRepoReferences(repo)
|
2023-01-09 15:37:44 -05:00
|
|
|
if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Str("repository", repo).Msg("load-repo: failed to reset tag field in RepoMetadata for repo")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
for _, manifest := range indexContent.Manifests {
|
|
|
|
tag := manifest.Annotations[ispec.AnnotationRefName]
|
2023-01-09 15:37:44 -05:00
|
|
|
|
2023-10-12 20:45:20 -05:00
|
|
|
if zcommon.IsReferrersTag(tag) {
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
manifestBlob, _, _, err := imageStore.GetImageManifest(repo, manifest.Digest.String())
|
2023-01-09 15:37:44 -05:00
|
|
|
if err != nil {
|
2023-10-30 15:06:04 -05:00
|
|
|
log.Error().Err(err).Str("repository", repo).Str("digest", manifest.Digest.String()).
|
|
|
|
Msg("load-repo: failed to get blob for image")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
2023-05-15 04:02:23 -05:00
|
|
|
return err
|
2023-01-09 15:37:44 -05:00
|
|
|
}
|
|
|
|
|
2023-02-27 14:23:18 -05:00
|
|
|
reference := tag
|
2023-01-09 15:37:44 -05:00
|
|
|
|
2023-02-27 14:23:18 -05:00
|
|
|
if tag == "" {
|
2023-10-30 15:06:04 -05:00
|
|
|
reference = manifest.Digest.String()
|
2023-01-09 15:37:44 -05:00
|
|
|
}
|
|
|
|
|
2023-11-01 11:16:18 -05:00
|
|
|
err = SetImageMetaFromInput(context.Background(), repo, reference, manifest.MediaType, manifest.Digest, manifestBlob,
|
2023-07-18 12:27:26 -05:00
|
|
|
imageStore, metaDB, log)
|
2023-01-09 15:37:44 -05:00
|
|
|
if err != nil {
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Str("repository", repo).Str("tag", tag).
|
|
|
|
Msg("load-repo: failed to set metadata for image")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func getAllRepos(storeController storage.StoreController) ([]string, error) {
|
|
|
|
allRepos, err := storeController.DefaultStore.GetRepositories()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
if storeController.SubStore != nil {
|
|
|
|
for _, store := range storeController.SubStore {
|
|
|
|
substoreRepos, err := store.GetRepositories()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
allRepos = append(allRepos, substoreRepos...)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return allRepos, nil
|
|
|
|
}
|
|
|
|
|
2023-05-26 13:08:19 -05:00
|
|
|
func GetSignatureLayersInfo(repo, tag, manifestDigest, signatureType string, manifestBlob []byte,
|
|
|
|
imageStore storageTypes.ImageStore, log log.Logger,
|
2023-07-18 12:27:26 -05:00
|
|
|
) ([]mTypes.LayerInfo, error) {
|
2023-05-24 11:46:16 -05:00
|
|
|
switch signatureType {
|
2023-08-19 00:52:03 -05:00
|
|
|
case zcommon.CosignSignature:
|
2023-05-24 11:46:16 -05:00
|
|
|
return getCosignSignatureLayersInfo(repo, tag, manifestDigest, manifestBlob, imageStore, log)
|
2023-08-19 00:52:03 -05:00
|
|
|
case zcommon.NotationSignature:
|
2023-05-24 11:46:16 -05:00
|
|
|
return getNotationSignatureLayersInfo(repo, manifestDigest, manifestBlob, imageStore, log)
|
|
|
|
default:
|
2023-07-18 12:27:26 -05:00
|
|
|
return []mTypes.LayerInfo{}, nil
|
2023-05-24 11:46:16 -05:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func getCosignSignatureLayersInfo(
|
2023-05-26 13:08:19 -05:00
|
|
|
repo, tag, manifestDigest string, manifestBlob []byte, imageStore storageTypes.ImageStore, log log.Logger,
|
2023-07-18 12:27:26 -05:00
|
|
|
) ([]mTypes.LayerInfo, error) {
|
|
|
|
layers := []mTypes.LayerInfo{}
|
2023-05-24 11:46:16 -05:00
|
|
|
|
|
|
|
var manifestContent ispec.Manifest
|
|
|
|
if err := json.Unmarshal(manifestBlob, &manifestContent); err != nil {
|
|
|
|
log.Error().Err(err).Str("repository", repo).Str("reference", tag).Str("digest", manifestDigest).Msg(
|
|
|
|
"load-repo: unable to marshal blob index")
|
|
|
|
|
|
|
|
return layers, err
|
|
|
|
}
|
|
|
|
|
2023-08-23 11:29:23 -05:00
|
|
|
var lockLatency time.Time
|
|
|
|
|
|
|
|
imageStore.RLock(&lockLatency)
|
|
|
|
defer imageStore.RUnlock(&lockLatency)
|
|
|
|
|
2023-05-24 11:46:16 -05:00
|
|
|
for _, layer := range manifestContent.Layers {
|
|
|
|
layerContent, err := imageStore.GetBlobContent(repo, layer.Digest)
|
|
|
|
if err != nil {
|
|
|
|
log.Error().Err(err).Str("repository", repo).Str("reference", tag).Str("layerDigest", layer.Digest.String()).Msg(
|
|
|
|
"load-repo: unable to get cosign signature layer content")
|
|
|
|
|
|
|
|
return layers, err
|
|
|
|
}
|
|
|
|
|
2023-08-19 00:52:03 -05:00
|
|
|
layerSigKey, ok := layer.Annotations[zcommon.CosignSigKey]
|
2023-05-24 11:46:16 -05:00
|
|
|
if !ok {
|
|
|
|
log.Error().Err(err).Str("repository", repo).Str("reference", tag).Str("layerDigest", layer.Digest.String()).Msg(
|
|
|
|
"load-repo: unable to get specific annotation of cosign signature")
|
|
|
|
}
|
|
|
|
|
2023-07-18 12:27:26 -05:00
|
|
|
layers = append(layers, mTypes.LayerInfo{
|
2023-05-24 11:46:16 -05:00
|
|
|
LayerDigest: layer.Digest.String(),
|
|
|
|
LayerContent: layerContent,
|
|
|
|
SignatureKey: layerSigKey,
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
return layers, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
func getNotationSignatureLayersInfo(
|
2023-05-26 13:08:19 -05:00
|
|
|
repo, manifestDigest string, manifestBlob []byte, imageStore storageTypes.ImageStore, log log.Logger,
|
2023-07-18 12:27:26 -05:00
|
|
|
) ([]mTypes.LayerInfo, error) {
|
|
|
|
layers := []mTypes.LayerInfo{}
|
2023-05-24 11:46:16 -05:00
|
|
|
|
|
|
|
var manifestContent ispec.Manifest
|
|
|
|
if err := json.Unmarshal(manifestBlob, &manifestContent); err != nil {
|
|
|
|
log.Error().Err(err).Str("repository", repo).Str("reference", manifestDigest).Msg(
|
|
|
|
"load-repo: unable to marshal blob index")
|
|
|
|
|
|
|
|
return layers, err
|
|
|
|
}
|
|
|
|
|
2023-10-12 20:45:20 -05:00
|
|
|
// skip if is a notation index
|
|
|
|
if manifestContent.MediaType == ispec.MediaTypeImageIndex {
|
|
|
|
return []mTypes.LayerInfo{}, nil
|
|
|
|
}
|
|
|
|
|
2023-05-24 11:46:16 -05:00
|
|
|
if len(manifestContent.Layers) != 1 {
|
|
|
|
log.Error().Err(zerr.ErrBadManifest).Str("repository", repo).Str("reference", manifestDigest).
|
|
|
|
Msg("load-repo: notation signature manifest requires exactly one layer but it does not")
|
|
|
|
|
|
|
|
return layers, zerr.ErrBadManifest
|
|
|
|
}
|
|
|
|
|
|
|
|
layer := manifestContent.Layers[0].Digest
|
|
|
|
|
2023-08-23 11:29:23 -05:00
|
|
|
var lockLatency time.Time
|
|
|
|
|
|
|
|
imageStore.RLock(&lockLatency)
|
|
|
|
defer imageStore.RUnlock(&lockLatency)
|
|
|
|
|
2023-05-24 11:46:16 -05:00
|
|
|
layerContent, err := imageStore.GetBlobContent(repo, layer)
|
|
|
|
if err != nil {
|
|
|
|
log.Error().Err(err).Str("repository", repo).Str("reference", manifestDigest).Str("layerDigest", layer.String()).Msg(
|
|
|
|
"load-repo: unable to get notation signature blob content")
|
|
|
|
|
|
|
|
return layers, err
|
|
|
|
}
|
|
|
|
|
|
|
|
layerSigKey := manifestContent.Layers[0].MediaType
|
|
|
|
|
2023-07-18 12:27:26 -05:00
|
|
|
layers = append(layers, mTypes.LayerInfo{
|
2023-05-24 11:46:16 -05:00
|
|
|
LayerDigest: layer.String(),
|
|
|
|
LayerContent: layerContent,
|
|
|
|
SignatureKey: layerSigKey,
|
|
|
|
})
|
|
|
|
|
|
|
|
return layers, nil
|
|
|
|
}
|
|
|
|
|
2023-02-27 14:23:18 -05:00
|
|
|
// SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag
|
|
|
|
// (in case the reference is a tag). The function expects image manifests and indexes (multi arch images).
|
2023-11-01 11:16:18 -05:00
|
|
|
func SetImageMetaFromInput(ctx context.Context, repo, reference, mediaType string, digest godigest.Digest, blob []byte,
|
2023-07-18 12:27:26 -05:00
|
|
|
imageStore storageTypes.ImageStore, metaDB mTypes.MetaDB, log log.Logger,
|
2023-02-27 14:23:18 -05:00
|
|
|
) error {
|
2023-10-30 15:06:04 -05:00
|
|
|
var imageMeta mTypes.ImageMeta
|
|
|
|
|
2023-02-27 14:23:18 -05:00
|
|
|
switch mediaType {
|
|
|
|
case ispec.MediaTypeImageManifest:
|
2023-10-30 15:06:04 -05:00
|
|
|
manifestContent := ispec.Manifest{}
|
|
|
|
configContent := ispec.Image{}
|
|
|
|
|
|
|
|
err := json.Unmarshal(blob, &manifestContent)
|
2023-02-27 14:23:18 -05:00
|
|
|
if err != nil {
|
2023-11-01 11:16:18 -05:00
|
|
|
log.Error().Err(err).Msg("metadb: error while getting image data")
|
|
|
|
|
2023-02-27 14:23:18 -05:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
if manifestContent.Config.MediaType == ispec.MediaTypeImageConfig {
|
|
|
|
configBlob, err := imageStore.GetBlobContent(repo, manifestContent.Config.Digest)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-02-27 14:23:18 -05:00
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
err = json.Unmarshal(configBlob, &configContent)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-02-27 14:23:18 -05:00
|
|
|
}
|
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
if isSig, sigType, signedManifestDigest := isSignature(reference, manifestContent); isSig {
|
|
|
|
layers, err := GetSignatureLayersInfo(repo, reference, digest.String(), sigType,
|
|
|
|
blob, imageStore, log)
|
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
2023-02-27 14:23:18 -05:00
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
err = metaDB.AddManifestSignature(repo, signedManifestDigest,
|
|
|
|
mTypes.SignatureMetadata{
|
|
|
|
SignatureType: sigType,
|
|
|
|
SignatureDigest: digest.String(),
|
2023-11-06 17:09:39 -05:00
|
|
|
SignatureTag: reference,
|
2023-10-30 15:06:04 -05:00
|
|
|
LayersInfo: layers,
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
log.Error().Err(err).Str("repository", repo).Str("tag", reference).
|
|
|
|
Str("manifestDigest", signedManifestDigest.String()).
|
|
|
|
Msg("load-repo: failed set signature meta for signed image")
|
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
err = metaDB.UpdateSignaturesValidity(repo, signedManifestDigest)
|
|
|
|
if err != nil {
|
|
|
|
log.Error().Err(err).Str("repository", repo).Str("reference", reference).Str("digest",
|
|
|
|
signedManifestDigest.String()).Msg("load-repo: failed verify signatures validity for signed image")
|
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
2023-03-10 13:37:29 -05:00
|
|
|
}
|
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
imageMeta = convert.GetImageManifestMeta(manifestContent, configContent, int64(len(blob)), digest)
|
|
|
|
case ispec.MediaTypeImageIndex:
|
|
|
|
indexContent := ispec.Index{}
|
2023-03-10 13:37:29 -05:00
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
err := json.Unmarshal(blob, &indexContent)
|
|
|
|
if err != nil {
|
2023-02-27 14:23:18 -05:00
|
|
|
return err
|
|
|
|
}
|
2023-10-30 15:06:04 -05:00
|
|
|
|
|
|
|
imageMeta = convert.GetImageIndexMeta(indexContent, int64(len(blob)), digest)
|
|
|
|
default:
|
|
|
|
return nil
|
2023-02-27 14:23:18 -05:00
|
|
|
}
|
|
|
|
|
2023-11-01 11:16:18 -05:00
|
|
|
err := metaDB.SetRepoReference(ctx, repo, reference, imageMeta)
|
2023-02-27 14:23:18 -05:00
|
|
|
if err != nil {
|
2023-11-01 11:16:18 -05:00
|
|
|
log.Error().Err(err).Msg("metadb: error while setting repo meta")
|
2023-02-27 14:23:18 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
2023-03-10 13:37:29 -05:00
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
func isSignature(reference string, manifestContent ispec.Manifest) (bool, string, godigest.Digest) {
|
|
|
|
manifestArtifactType := zcommon.GetManifestArtifactType(manifestContent)
|
2023-03-20 11:14:17 -05:00
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
// check notation signature
|
|
|
|
if manifestArtifactType == zcommon.ArtifactTypeNotation && manifestContent.Subject != nil {
|
|
|
|
return true, NotationType, manifestContent.Subject.Digest
|
|
|
|
}
|
2023-07-05 11:42:16 -05:00
|
|
|
|
2023-11-06 17:09:39 -05:00
|
|
|
// check cosign signature
|
|
|
|
if manifestArtifactType == zcommon.ArtifactTypeCosign && manifestContent.Subject != nil {
|
|
|
|
return true, CosignType, manifestContent.Subject.Digest
|
|
|
|
}
|
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
if tag := reference; zcommon.IsCosignTag(reference) {
|
|
|
|
prefixLen := len("sha256-")
|
|
|
|
digestLen := 64
|
|
|
|
signedImageManifestDigestEncoded := tag[prefixLen : prefixLen+digestLen]
|
2023-07-05 11:42:16 -05:00
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
signedImageManifestDigest := godigest.NewDigestFromEncoded(godigest.SHA256,
|
|
|
|
signedImageManifestDigestEncoded)
|
2023-03-10 13:37:29 -05:00
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
return true, CosignType, signedImageManifestDigest
|
2023-03-10 13:37:29 -05:00
|
|
|
}
|
|
|
|
|
2023-10-30 15:06:04 -05:00
|
|
|
return false, "", ""
|
2023-03-10 13:37:29 -05:00
|
|
|
}
|