mirror of
https://github.com/project-zot/zot.git
synced 2024-12-16 21:56:37 -05:00
d62c09e2cc
* feat(repodb): index logic + tests Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com> * feat(cli): printing indexes support using the rest api Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com> --------- Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
310 lines
8.3 KiB
Go
310 lines
8.3 KiB
Go
package repodb
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
|
|
godigest "github.com/opencontainers/go-digest"
|
|
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
|
|
|
zerr "zotregistry.io/zot/errors"
|
|
"zotregistry.io/zot/pkg/log"
|
|
"zotregistry.io/zot/pkg/storage"
|
|
)
|
|
|
|
// SyncRepoDB will sync all repos found in the rootdirectory of the oci layout that zot was deployed on.
|
|
func SyncRepoDB(repoDB RepoDB, storeController storage.StoreController, log log.Logger) error {
|
|
allRepos, err := getAllRepos(storeController)
|
|
if err != nil {
|
|
rootDir := storeController.DefaultStore.RootDir()
|
|
log.Error().Err(err).Msgf("sync-repodb: failed to get all repo names present under %s", rootDir)
|
|
|
|
return err
|
|
}
|
|
|
|
for _, repo := range allRepos {
|
|
err := SyncRepo(repo, repoDB, storeController, log)
|
|
if err != nil {
|
|
log.Error().Err(err).Msgf("sync-repodb: failed to sync repo %s", repo)
|
|
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// SyncRepo reads the contents of a repo and syncs all images signatures found.
|
|
func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreController, log log.Logger) error {
|
|
imageStore := storeController.GetImageStore(repo)
|
|
|
|
indexBlob, err := imageStore.GetIndexContent(repo)
|
|
if err != nil {
|
|
log.Error().Err(err).Msgf("sync-repo: failed to read index.json for repo %s", repo)
|
|
|
|
return err
|
|
}
|
|
|
|
var indexContent ispec.Index
|
|
|
|
err = json.Unmarshal(indexBlob, &indexContent)
|
|
if err != nil {
|
|
log.Error().Err(err).Msgf("sync-repo: failed to unmarshal index.json for repo %s", repo)
|
|
|
|
return err
|
|
}
|
|
|
|
err = resetRepoMetaTags(repo, repoDB, log)
|
|
if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
|
|
log.Error().Err(err).Msgf("sync-repo: failed to reset tag field in RepoMetadata for repo %s", repo)
|
|
|
|
return err
|
|
}
|
|
|
|
type foundSignatureData struct {
|
|
repo string
|
|
tag string
|
|
signatureType string
|
|
signedManifestDigest string
|
|
signatureDigest string
|
|
}
|
|
|
|
var signaturesFound []foundSignatureData
|
|
|
|
for _, manifest := range indexContent.Manifests {
|
|
tag, hasTag := manifest.Annotations[ispec.AnnotationRefName]
|
|
|
|
manifestMetaIsPresent, err := isManifestMetaPresent(repo, manifest, repoDB)
|
|
if err != nil {
|
|
log.Error().Err(err).Msgf("sync-repo: error checking manifestMeta in RepoDB")
|
|
|
|
return err
|
|
}
|
|
|
|
if manifestMetaIsPresent && hasTag {
|
|
err = repoDB.SetRepoTag(repo, tag, manifest.Digest, manifest.MediaType)
|
|
if err != nil {
|
|
log.Error().Err(err).Msgf("sync-repo: failed to set repo tag for %s:%s", repo, tag)
|
|
|
|
return err
|
|
}
|
|
|
|
continue
|
|
}
|
|
|
|
manifestBlob, digest, _, err := imageStore.GetImageManifest(repo, manifest.Digest.String())
|
|
if err != nil {
|
|
log.Error().Err(err).Msgf("sync-repo: failed to set repo tag for %s:%s", repo, tag)
|
|
|
|
return err
|
|
}
|
|
|
|
isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(repo,
|
|
manifestBlob, tag, storeController)
|
|
if err != nil {
|
|
if errors.Is(err, zerr.ErrOrphanSignature) {
|
|
continue
|
|
} else {
|
|
log.Error().Err(err).Msgf("sync-repo: failed checking if image is signature for %s:%s", repo, tag)
|
|
|
|
return err
|
|
}
|
|
}
|
|
|
|
if isSignature {
|
|
// We'll ignore signatures now because the order in which the signed image and signature are added into
|
|
// the DB matters. First we add the normal images then the signatures
|
|
signaturesFound = append(signaturesFound, foundSignatureData{
|
|
repo: repo,
|
|
tag: tag,
|
|
signatureType: signatureType,
|
|
signedManifestDigest: signedManifestDigest.String(),
|
|
signatureDigest: digest.String(),
|
|
})
|
|
|
|
continue
|
|
}
|
|
|
|
reference := tag
|
|
|
|
if tag == "" {
|
|
reference = manifest.Digest.String()
|
|
}
|
|
|
|
err = SetMetadataFromInput(repo, reference, manifest.MediaType, manifest.Digest, manifestBlob,
|
|
storeController, repoDB, log)
|
|
if err != nil {
|
|
log.Error().Err(err).Msgf("sync-repo: failed to set metadata for %s:%s", repo, tag)
|
|
|
|
return err
|
|
}
|
|
}
|
|
|
|
// manage the signatures found
|
|
for _, sigData := range signaturesFound {
|
|
err := repoDB.AddManifestSignature(repo, godigest.Digest(sigData.signedManifestDigest), SignatureMetadata{
|
|
SignatureType: sigData.signatureType,
|
|
SignatureDigest: sigData.signatureDigest,
|
|
})
|
|
if err != nil {
|
|
log.Error().Err(err).Msgf("sync-repo: failed set signature meta for signed image %s:%s manifest digest %s ",
|
|
sigData.repo, sigData.tag, sigData.signedManifestDigest)
|
|
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// resetRepoMetaTags will delete all tags from a repometadata.
|
|
func resetRepoMetaTags(repo string, repoDB RepoDB, log log.Logger) error {
|
|
repoMeta, err := repoDB.GetRepoMeta(repo)
|
|
if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
|
|
log.Error().Err(err).Msgf("sync-repo: failed to get RepoMeta for repo %s", repo)
|
|
|
|
return err
|
|
}
|
|
|
|
if errors.Is(err, zerr.ErrRepoMetaNotFound) {
|
|
log.Info().Msgf("sync-repo: RepoMeta not found for repo %s, new RepoMeta will be created", repo)
|
|
|
|
return nil
|
|
}
|
|
|
|
for tag := range repoMeta.Tags {
|
|
// We should have a way to delete all tags at once
|
|
err := repoDB.DeleteRepoTag(repo, tag)
|
|
if err != nil {
|
|
log.Error().Err(err).Msgf("sync-repo: failed to delete tag %s from RepoMeta for repo %s", tag, repo)
|
|
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func getAllRepos(storeController storage.StoreController) ([]string, error) {
|
|
allRepos, err := storeController.DefaultStore.GetRepositories()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if storeController.SubStore != nil {
|
|
for _, store := range storeController.SubStore {
|
|
substoreRepos, err := store.GetRepositories()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
allRepos = append(allRepos, substoreRepos...)
|
|
}
|
|
}
|
|
|
|
return allRepos, nil
|
|
}
|
|
|
|
// isManifestMetaPresent checks if the manifest with a certain digest is present in a certain repo.
|
|
func isManifestMetaPresent(repo string, manifest ispec.Descriptor, repoDB RepoDB) (bool, error) {
|
|
_, err := repoDB.GetManifestMeta(repo, manifest.Digest)
|
|
if err != nil && !errors.Is(err, zerr.ErrManifestMetaNotFound) {
|
|
return false, err
|
|
}
|
|
|
|
if errors.Is(err, zerr.ErrManifestMetaNotFound) {
|
|
return false, nil
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
// NewManifestMeta takes raw data about an image and createa a new ManifestMetadate object.
|
|
func NewManifestData(repoName string, manifestBlob []byte, storeController storage.StoreController,
|
|
) (ManifestData, error) {
|
|
var (
|
|
manifestContent ispec.Manifest
|
|
configContent ispec.Image
|
|
manifestData ManifestData
|
|
)
|
|
|
|
imgStore := storeController.GetImageStore(repoName)
|
|
|
|
err := json.Unmarshal(manifestBlob, &manifestContent)
|
|
if err != nil {
|
|
return ManifestData{}, err
|
|
}
|
|
|
|
configBlob, err := imgStore.GetBlobContent(repoName, manifestContent.Config.Digest)
|
|
if err != nil {
|
|
return ManifestData{}, err
|
|
}
|
|
|
|
err = json.Unmarshal(configBlob, &configContent)
|
|
if err != nil {
|
|
return ManifestData{}, err
|
|
}
|
|
|
|
manifestData.ManifestBlob = manifestBlob
|
|
manifestData.ConfigBlob = configBlob
|
|
|
|
return manifestData, nil
|
|
}
|
|
|
|
func NewIndexData(repoName string, indexBlob []byte, storeController storage.StoreController,
|
|
) IndexData {
|
|
indexData := IndexData{}
|
|
|
|
indexData.IndexBlob = indexBlob
|
|
|
|
return indexData
|
|
}
|
|
|
|
// SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag
|
|
// (in case the reference is a tag). The function expects image manifests and indexes (multi arch images).
|
|
func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte,
|
|
storeController storage.StoreController, repoDB RepoDB, log log.Logger,
|
|
) error {
|
|
switch mediaType {
|
|
case ispec.MediaTypeImageManifest:
|
|
imageData, err := NewManifestData(repo, descriptorBlob, storeController)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = repoDB.SetManifestData(digest, imageData)
|
|
if err != nil {
|
|
log.Error().Err(err).Msg("repodb: error while putting manifest meta")
|
|
|
|
return err
|
|
}
|
|
case ispec.MediaTypeImageIndex:
|
|
indexData := NewIndexData(repo, descriptorBlob, storeController)
|
|
|
|
err := repoDB.SetIndexData(digest, indexData)
|
|
if err != nil {
|
|
log.Error().Err(err).Msg("repodb: error while putting index data")
|
|
|
|
return err
|
|
}
|
|
}
|
|
|
|
if refferenceIsDigest(reference) {
|
|
return nil
|
|
}
|
|
|
|
err := repoDB.SetRepoTag(repo, reference, digest, mediaType)
|
|
if err != nil {
|
|
log.Error().Err(err).Msg("repodb: error while putting repo meta")
|
|
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func refferenceIsDigest(reference string) bool {
|
|
_, err := godigest.Parse(reference)
|
|
|
|
return err == nil
|
|
}
|