2023-01-09 15:37:44 -05:00
|
|
|
package repodb
|
|
|
|
|
|
|
|
import (
|
|
|
|
"encoding/json"
|
|
|
|
"errors"
|
2023-03-20 11:14:17 -05:00
|
|
|
"fmt"
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
godigest "github.com/opencontainers/go-digest"
|
|
|
|
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
|
|
|
|
|
|
|
zerr "zotregistry.io/zot/errors"
|
2023-05-10 12:15:33 -05:00
|
|
|
zcommon "zotregistry.io/zot/pkg/common"
|
2023-01-09 15:37:44 -05:00
|
|
|
"zotregistry.io/zot/pkg/log"
|
|
|
|
"zotregistry.io/zot/pkg/storage"
|
|
|
|
)
|
|
|
|
|
2023-03-10 13:37:29 -05:00
|
|
|
// ParseStorage will sync all repos found in the rootdirectory of the oci layout that zot was deployed on with the
|
|
|
|
// ParseStorage database.
|
|
|
|
func ParseStorage(repoDB RepoDB, storeController storage.StoreController, log log.Logger) error {
|
2023-01-09 15:37:44 -05:00
|
|
|
allRepos, err := getAllRepos(storeController)
|
|
|
|
if err != nil {
|
|
|
|
rootDir := storeController.DefaultStore.RootDir()
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Str("rootDir", rootDir).
|
|
|
|
Msg("load-local-layout: failed to get all repo names present under rootDir")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, repo := range allRepos {
|
2023-03-10 13:37:29 -05:00
|
|
|
err := ParseRepo(repo, repoDB, storeController, log)
|
2023-01-09 15:37:44 -05:00
|
|
|
if err != nil {
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Str("repository", repo).Msg("load-local-layout: failed to sync repo")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-03-10 13:37:29 -05:00
|
|
|
// ParseRepo reads the contents of a repo and syncs all images and signatures found.
|
|
|
|
func ParseRepo(repo string, repoDB RepoDB, storeController storage.StoreController, log log.Logger) error {
|
2023-01-09 15:37:44 -05:00
|
|
|
imageStore := storeController.GetImageStore(repo)
|
|
|
|
|
|
|
|
indexBlob, err := imageStore.GetIndexContent(repo)
|
|
|
|
if err != nil {
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Str("repository", repo).Msg("load-repo: failed to read index.json for repo")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
var indexContent ispec.Index
|
|
|
|
|
|
|
|
err = json.Unmarshal(indexBlob, &indexContent)
|
|
|
|
if err != nil {
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Str("repository", repo).Msg("load-repo: failed to unmarshal index.json for repo")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
err = resetRepoMetaTags(repo, repoDB, log)
|
|
|
|
if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Str("repository", repo).Msg("load-repo: failed to reset tag field in RepoMetadata for repo")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
for _, manifest := range indexContent.Manifests {
|
|
|
|
tag, hasTag := manifest.Annotations[ispec.AnnotationRefName]
|
|
|
|
|
|
|
|
manifestMetaIsPresent, err := isManifestMetaPresent(repo, manifest, repoDB)
|
|
|
|
if err != nil {
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Msg("load-repo: error checking manifestMeta in RepoDB")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
2023-05-15 04:02:23 -05:00
|
|
|
// this check helps reduce unecesary reads from storage
|
2023-02-27 14:23:18 -05:00
|
|
|
if manifestMetaIsPresent && hasTag {
|
2023-03-09 13:41:48 -05:00
|
|
|
err = repoDB.SetRepoReference(repo, tag, manifest.Digest, manifest.MediaType)
|
2023-01-09 15:37:44 -05:00
|
|
|
if err != nil {
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Str("repository", repo).Str("tag", tag).Msg("load-repo: failed to set repo tag")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
|
|
|
manifestBlob, digest, _, err := imageStore.GetImageManifest(repo, manifest.Digest.String())
|
|
|
|
if err != nil {
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Str("repository", repo).Str("tag", tag).
|
|
|
|
Msg("load-repo: failed to set repo tag for image")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(repo,
|
2023-05-15 04:02:23 -05:00
|
|
|
manifestBlob, tag)
|
2023-01-09 15:37:44 -05:00
|
|
|
if err != nil {
|
2023-05-15 04:02:23 -05:00
|
|
|
log.Error().Err(err).Str("repository", repo).Str("tag", tag).
|
|
|
|
Msg("load-repo: failed checking if image is signature for specified image")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
2023-05-15 04:02:23 -05:00
|
|
|
return err
|
2023-01-09 15:37:44 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
if isSignature {
|
2023-05-15 04:02:23 -05:00
|
|
|
err := repoDB.AddManifestSignature(repo, signedManifestDigest,
|
|
|
|
SignatureMetadata{
|
|
|
|
SignatureType: signatureType,
|
|
|
|
SignatureDigest: digest.String(),
|
|
|
|
})
|
|
|
|
if err != nil {
|
|
|
|
log.Error().Err(err).Str("repository", repo).Str("tag", tag).
|
|
|
|
Str("manifestDigest", signedManifestDigest.String()).
|
|
|
|
Msg("load-repo: failed set signature meta for signed image manifest digest")
|
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
continue
|
|
|
|
}
|
|
|
|
|
2023-02-27 14:23:18 -05:00
|
|
|
reference := tag
|
2023-01-09 15:37:44 -05:00
|
|
|
|
2023-02-27 14:23:18 -05:00
|
|
|
if tag == "" {
|
|
|
|
reference = manifest.Digest.String()
|
2023-01-09 15:37:44 -05:00
|
|
|
}
|
|
|
|
|
2023-05-15 04:02:23 -05:00
|
|
|
err = SetImageMetaFromInput(repo, reference, manifest.MediaType, manifest.Digest, manifestBlob,
|
2023-03-09 13:41:48 -05:00
|
|
|
imageStore, repoDB, log)
|
2023-01-09 15:37:44 -05:00
|
|
|
if err != nil {
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Str("repository", repo).Str("tag", tag).
|
|
|
|
Msg("load-repo: failed to set metadata for image")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// resetRepoMetaTags will delete all tags from a repometadata.
|
|
|
|
func resetRepoMetaTags(repo string, repoDB RepoDB, log log.Logger) error {
|
|
|
|
repoMeta, err := repoDB.GetRepoMeta(repo)
|
|
|
|
if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Error().Err(err).Str("repository", repo).Msg("load-repo: failed to get RepoMeta for repo")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
if errors.Is(err, zerr.ErrRepoMetaNotFound) {
|
2023-04-27 21:44:22 -05:00
|
|
|
log.Info().Str("repository", repo).Msg("load-repo: RepoMeta not found for repo, new RepoMeta will be created")
|
2023-01-09 15:37:44 -05:00
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-03-23 13:08:11 -05:00
|
|
|
return repoDB.SetRepoMeta(repo, RepoMetadata{
|
|
|
|
Name: repoMeta.Name,
|
|
|
|
Tags: map[string]Descriptor{},
|
|
|
|
Statistics: repoMeta.Statistics,
|
|
|
|
Signatures: map[string]ManifestSignatures{},
|
|
|
|
Referrers: map[string][]ReferrerInfo{},
|
|
|
|
Stars: repoMeta.Stars,
|
|
|
|
})
|
2023-01-09 15:37:44 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
func getAllRepos(storeController storage.StoreController) ([]string, error) {
|
|
|
|
allRepos, err := storeController.DefaultStore.GetRepositories()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
if storeController.SubStore != nil {
|
|
|
|
for _, store := range storeController.SubStore {
|
|
|
|
substoreRepos, err := store.GetRepositories()
|
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
allRepos = append(allRepos, substoreRepos...)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return allRepos, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// isManifestMetaPresent checks if the manifest with a certain digest is present in a certain repo.
|
|
|
|
func isManifestMetaPresent(repo string, manifest ispec.Descriptor, repoDB RepoDB) (bool, error) {
|
|
|
|
_, err := repoDB.GetManifestMeta(repo, manifest.Digest)
|
|
|
|
if err != nil && !errors.Is(err, zerr.ErrManifestMetaNotFound) {
|
|
|
|
return false, err
|
|
|
|
}
|
|
|
|
|
|
|
|
if errors.Is(err, zerr.ErrManifestMetaNotFound) {
|
|
|
|
return false, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
return true, nil
|
|
|
|
}
|
|
|
|
|
|
|
|
// NewManifestMeta takes raw data about an image and createa a new ManifestMetadate object.
|
2023-03-10 13:37:29 -05:00
|
|
|
func NewManifestData(repoName string, manifestBlob []byte, imageStore storage.ImageStore,
|
2023-01-09 15:37:44 -05:00
|
|
|
) (ManifestData, error) {
|
|
|
|
var (
|
|
|
|
manifestContent ispec.Manifest
|
|
|
|
configContent ispec.Image
|
|
|
|
manifestData ManifestData
|
|
|
|
)
|
|
|
|
|
|
|
|
err := json.Unmarshal(manifestBlob, &manifestContent)
|
|
|
|
if err != nil {
|
|
|
|
return ManifestData{}, err
|
|
|
|
}
|
|
|
|
|
2023-03-10 13:37:29 -05:00
|
|
|
configBlob, err := imageStore.GetBlobContent(repoName, manifestContent.Config.Digest)
|
2023-01-09 15:37:44 -05:00
|
|
|
if err != nil {
|
|
|
|
return ManifestData{}, err
|
|
|
|
}
|
|
|
|
|
|
|
|
err = json.Unmarshal(configBlob, &configContent)
|
|
|
|
if err != nil {
|
|
|
|
return ManifestData{}, err
|
|
|
|
}
|
|
|
|
|
|
|
|
manifestData.ManifestBlob = manifestBlob
|
|
|
|
manifestData.ConfigBlob = configBlob
|
|
|
|
|
|
|
|
return manifestData, nil
|
|
|
|
}
|
2023-02-27 14:23:18 -05:00
|
|
|
|
2023-03-10 13:37:29 -05:00
|
|
|
func NewIndexData(repoName string, indexBlob []byte, imageStore storage.ImageStore,
|
2023-02-27 14:23:18 -05:00
|
|
|
) IndexData {
|
|
|
|
indexData := IndexData{}
|
|
|
|
|
|
|
|
indexData.IndexBlob = indexBlob
|
|
|
|
|
|
|
|
return indexData
|
|
|
|
}
|
|
|
|
|
|
|
|
// SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag
|
|
|
|
// (in case the reference is a tag). The function expects image manifests and indexes (multi arch images).
|
2023-05-15 04:02:23 -05:00
|
|
|
func SetImageMetaFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte,
|
2023-03-09 13:41:48 -05:00
|
|
|
imageStore storage.ImageStore, repoDB RepoDB, log log.Logger,
|
2023-02-27 14:23:18 -05:00
|
|
|
) error {
|
|
|
|
switch mediaType {
|
|
|
|
case ispec.MediaTypeImageManifest:
|
2023-03-09 13:41:48 -05:00
|
|
|
imageData, err := NewManifestData(repo, descriptorBlob, imageStore)
|
2023-02-27 14:23:18 -05:00
|
|
|
if err != nil {
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
err = repoDB.SetManifestData(digest, imageData)
|
|
|
|
if err != nil {
|
|
|
|
log.Error().Err(err).Msg("repodb: error while putting manifest meta")
|
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
case ispec.MediaTypeImageIndex:
|
2023-03-10 13:37:29 -05:00
|
|
|
indexData := NewIndexData(repo, descriptorBlob, imageStore)
|
2023-02-27 14:23:18 -05:00
|
|
|
|
|
|
|
err := repoDB.SetIndexData(digest, indexData)
|
|
|
|
if err != nil {
|
|
|
|
log.Error().Err(err).Msg("repodb: error while putting index data")
|
|
|
|
|
2023-03-10 13:37:29 -05:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-20 11:14:17 -05:00
|
|
|
refferredDigest, referrerInfo, hasSubject, err := GetReferredSubject(descriptorBlob, digest.String(), mediaType)
|
|
|
|
if hasSubject && err == nil {
|
|
|
|
err := repoDB.SetReferrer(repo, refferredDigest, referrerInfo)
|
2023-03-10 13:37:29 -05:00
|
|
|
if err != nil {
|
|
|
|
log.Error().Err(err).Msg("repodb: error while settingg referrer")
|
|
|
|
|
2023-02-27 14:23:18 -05:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-03-20 11:14:17 -05:00
|
|
|
err = repoDB.SetRepoReference(repo, reference, digest, mediaType)
|
2023-02-27 14:23:18 -05:00
|
|
|
if err != nil {
|
|
|
|
log.Error().Err(err).Msg("repodb: error while putting repo meta")
|
|
|
|
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
return nil
|
|
|
|
}
|
2023-03-10 13:37:29 -05:00
|
|
|
|
2023-03-20 11:14:17 -05:00
|
|
|
func GetReferredSubject(descriptorBlob []byte, referrerDigest, mediaType string,
|
|
|
|
) (godigest.Digest, ReferrerInfo, bool, error) {
|
|
|
|
var (
|
|
|
|
referrerInfo ReferrerInfo
|
|
|
|
referrerSubject *ispec.Descriptor
|
|
|
|
)
|
2023-03-10 13:37:29 -05:00
|
|
|
|
2023-05-10 12:15:33 -05:00
|
|
|
var manifestContent ispec.Manifest
|
2023-03-20 11:14:17 -05:00
|
|
|
|
2023-05-10 12:15:33 -05:00
|
|
|
err := json.Unmarshal(descriptorBlob, &manifestContent)
|
|
|
|
if err != nil {
|
|
|
|
return "", referrerInfo, false,
|
|
|
|
fmt.Errorf("repodb: can't unmarshal manifest for digest %s: %w", referrerDigest, err)
|
|
|
|
}
|
2023-03-20 11:14:17 -05:00
|
|
|
|
2023-05-10 12:15:33 -05:00
|
|
|
referrerSubject = manifestContent.Subject
|
2023-03-20 11:14:17 -05:00
|
|
|
|
2023-05-10 12:15:33 -05:00
|
|
|
referrerInfo = ReferrerInfo{
|
|
|
|
Digest: referrerDigest,
|
|
|
|
MediaType: mediaType,
|
|
|
|
ArtifactType: zcommon.GetManifestArtifactType(manifestContent),
|
|
|
|
Size: len(descriptorBlob),
|
|
|
|
Annotations: manifestContent.Annotations,
|
2023-03-10 13:37:29 -05:00
|
|
|
}
|
|
|
|
|
2023-03-20 11:14:17 -05:00
|
|
|
if referrerSubject == nil || referrerSubject.Digest.String() == "" {
|
|
|
|
return "", ReferrerInfo{}, false, nil
|
2023-03-10 13:37:29 -05:00
|
|
|
}
|
|
|
|
|
2023-03-20 11:14:17 -05:00
|
|
|
return referrerSubject.Digest, referrerInfo, true, nil
|
2023-03-10 13:37:29 -05:00
|
|
|
}
|