zot/pkg/meta/parse.go

package meta

import (
	"context"
	"encoding/json"
	"errors"
	"time"

	godigest "github.com/opencontainers/go-digest"
	ispec "github.com/opencontainers/image-spec/specs-go/v1"

	zerr "zotregistry.io/zot/errors"
	zcommon "zotregistry.io/zot/pkg/common"
	"zotregistry.io/zot/pkg/log"
	"zotregistry.io/zot/pkg/meta/convert"
	mTypes "zotregistry.io/zot/pkg/meta/types"
	"zotregistry.io/zot/pkg/storage"
	storageTypes "zotregistry.io/zot/pkg/storage/types"
)

const (
	CosignType   = "cosign"
	NotationType = "notation"
)

// ParseStorage will sync all repos found in the rootdirectory of the oci layout that zot was deployed on with the
// ParseStorage database.
func ParseStorage(metaDB mTypes.MetaDB, storeController storage.StoreController, log log.Logger) error {
	log.Info().Msg("Started parsing storage and updating MetaDB")

	allRepos, err := getAllRepos(storeController)
	if err != nil {
		rootDir := storeController.DefaultStore.RootDir()
		log.Error().Err(err).Str("rootDir", rootDir).
			Msg("load-local-layout: failed to get all repo names present under rootDir")

		return err
	}

	for i, repo := range allRepos {
		log.Info().Int("total", len(allRepos)).Int("progress", i).Str("current-repo", repo).
			Msgf("parsing next repo '%s'", repo)

		err := ParseRepo(repo, metaDB, storeController, log)
		if err != nil {
			log.Error().Err(err).Str("repository", repo).Msg("load-local-layout: failed to sync repo")

			return err
		}
	}

	log.Info().Msg("Done parsing storage and updating MetaDB")

	return nil
}

// ParseRepo reads the contents of a repo and syncs all images and signatures found.
func ParseRepo(repo string, metaDB mTypes.MetaDB, storeController storage.StoreController, log log.Logger) error {
	imageStore := storeController.GetImageStore(repo)

	var lockLatency time.Time

	imageStore.RLock(&lockLatency)
	defer imageStore.RUnlock(&lockLatency)

	indexBlob, err := imageStore.GetIndexContent(repo)
	if err != nil {
		log.Error().Err(err).Str("repository", repo).Msg("load-repo: failed to read index.json for repo")

		return err
	}

	var indexContent ispec.Index

	err = json.Unmarshal(indexBlob, &indexContent)
	if err != nil {
		log.Error().Err(err).Str("repository", repo).Msg("load-repo: failed to unmarshal index.json for repo")

		return err
	}

	err = metaDB.ResetRepoReferences(repo)
	if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
		log.Error().Err(err).Str("repository", repo).Msg("load-repo: failed to reset tag field in RepoMetadata for repo")

		return err
	}

	for _, manifest := range indexContent.Manifests {
		tag := manifest.Annotations[ispec.AnnotationRefName]

		if zcommon.IsReferrersTag(tag) {
			continue
		}

		manifestBlob, _, _, err := imageStore.GetImageManifest(repo, manifest.Digest.String())
		if err != nil {
			log.Error().Err(err).Str("repository", repo).Str("digest", manifest.Digest.String()).
				Msg("load-repo: failed to get blob for image")

			return err
		}

		reference := tag

		if tag == "" {
			reference = manifest.Digest.String()
		}

		err = SetImageMetaFromInput(context.Background(), repo, reference, manifest.MediaType, manifest.Digest, manifestBlob,
			imageStore, metaDB, log)
		if err != nil {
			log.Error().Err(err).Str("repository", repo).Str("tag", tag).
				Msg("load-repo: failed to set metadata for image")

			return err
		}
	}

	return nil
}

func getAllRepos(storeController storage.StoreController) ([]string, error) {
	allRepos, err := storeController.DefaultStore.GetRepositories()
	if err != nil {
		return nil, err
	}

	if storeController.SubStore != nil {
		for _, store := range storeController.SubStore {
			substoreRepos, err := store.GetRepositories()
			if err != nil {
				return nil, err
			}

			allRepos = append(allRepos, substoreRepos...)
		}
	}

	return allRepos, nil
}

func GetSignatureLayersInfo(repo, tag, manifestDigest, signatureType string, manifestBlob []byte,
	imageStore storageTypes.ImageStore, log log.Logger,
) ([]mTypes.LayerInfo, error) {
	switch signatureType {
	case zcommon.CosignSignature:
		return getCosignSignatureLayersInfo(repo, tag, manifestDigest, manifestBlob, imageStore, log)
	case zcommon.NotationSignature:
		return getNotationSignatureLayersInfo(repo, manifestDigest, manifestBlob, imageStore, log)
	default:
		return []mTypes.LayerInfo{}, nil
	}
}

func getCosignSignatureLayersInfo(
	repo, tag, manifestDigest string, manifestBlob []byte, imageStore storageTypes.ImageStore, log log.Logger,
) ([]mTypes.LayerInfo, error) {
	layers := []mTypes.LayerInfo{}

	var manifestContent ispec.Manifest
	if err := json.Unmarshal(manifestBlob, &manifestContent); err != nil {
		log.Error().Err(err).Str("repository", repo).Str("reference", tag).Str("digest", manifestDigest).Msg(
			"load-repo: unable to marshal blob index")

		return layers, err
	}

	var lockLatency time.Time

	imageStore.RLock(&lockLatency)
	defer imageStore.RUnlock(&lockLatency)

	for _, layer := range manifestContent.Layers {
		layerContent, err := imageStore.GetBlobContent(repo, layer.Digest)
		if err != nil {
			log.Error().Err(err).Str("repository", repo).Str("reference", tag).Str("layerDigest", layer.Digest.String()).Msg(
				"load-repo: unable to get cosign signature layer content")

			return layers, err
		}

		layerSigKey, ok := layer.Annotations[zcommon.CosignSigKey]
		if !ok {
			log.Error().Err(err).Str("repository", repo).Str("reference", tag).Str("layerDigest", layer.Digest.String()).Msg(
				"load-repo: unable to get specific annotation of cosign signature")
		}

		layers = append(layers, mTypes.LayerInfo{
			LayerDigest:  layer.Digest.String(),
			LayerContent: layerContent,
			SignatureKey: layerSigKey,
		})
	}

	return layers, nil
}

func getNotationSignatureLayersInfo(
	repo, manifestDigest string, manifestBlob []byte, imageStore storageTypes.ImageStore, log log.Logger,
) ([]mTypes.LayerInfo, error) {
	layers := []mTypes.LayerInfo{}

	var manifestContent ispec.Manifest
	if err := json.Unmarshal(manifestBlob, &manifestContent); err != nil {
		log.Error().Err(err).Str("repository", repo).Str("reference", manifestDigest).Msg(
			"load-repo: unable to marshal blob index")

		return layers, err
	}

	// skip if is a notation index
	if manifestContent.MediaType == ispec.MediaTypeImageIndex {
		return []mTypes.LayerInfo{}, nil
	}

	if len(manifestContent.Layers) != 1 {
		log.Error().Err(zerr.ErrBadManifest).Str("repository", repo).Str("reference", manifestDigest).
			Msg("load-repo: notation signature manifest requires exactly one layer but it does not")

		return layers, zerr.ErrBadManifest
	}

	layer := manifestContent.Layers[0].Digest

	var lockLatency time.Time

	imageStore.RLock(&lockLatency)
	defer imageStore.RUnlock(&lockLatency)

	layerContent, err := imageStore.GetBlobContent(repo, layer)
	if err != nil {
		log.Error().Err(err).Str("repository", repo).Str("reference", manifestDigest).Str("layerDigest", layer.String()).Msg(
			"load-repo: unable to get notation signature blob content")

		return layers, err
	}

	layerSigKey := manifestContent.Layers[0].MediaType

	layers = append(layers, mTypes.LayerInfo{
		LayerDigest:  layer.String(),
		LayerContent: layerContent,
		SignatureKey: layerSigKey,
	})

	return layers, nil
}

// SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag
// (in case the reference is a tag). The function expects image manifests and indexes (multi arch images).
func SetImageMetaFromInput(ctx context.Context, repo, reference, mediaType string, digest godigest.Digest, blob []byte,
	imageStore storageTypes.ImageStore, metaDB mTypes.MetaDB, log log.Logger,
) error {
	var imageMeta mTypes.ImageMeta

	switch mediaType {
	case ispec.MediaTypeImageManifest:
		manifestContent := ispec.Manifest{}
		configContent := ispec.Image{}

		err := json.Unmarshal(blob, &manifestContent)
		if err != nil {
			log.Error().Err(err).Msg("metadb: error while getting image data")

			return err
		}

		if manifestContent.Config.MediaType == ispec.MediaTypeImageConfig {
			configBlob, err := imageStore.GetBlobContent(repo, manifestContent.Config.Digest)
			if err != nil {
				return err
			}

			err = json.Unmarshal(configBlob, &configContent)
			if err != nil {
				return err
			}
		}

		if isSig, sigType, signedManifestDigest := isSignature(reference, manifestContent); isSig {
			layers, err := GetSignatureLayersInfo(repo, reference, digest.String(), sigType,
				blob, imageStore, log)
			if err != nil {
				return err
			}

			err = metaDB.AddManifestSignature(repo, signedManifestDigest,
				mTypes.SignatureMetadata{
					SignatureType:   sigType,
					SignatureDigest: digest.String(),
					SignatureTag:    reference,
					LayersInfo:      layers,
				})
			if err != nil {
				log.Error().Err(err).Str("repository", repo).Str("tag", reference).
					Str("manifestDigest", signedManifestDigest.String()).
					Msg("load-repo: failed set signature meta for signed image")

				return err
			}

			err = metaDB.UpdateSignaturesValidity(repo, signedManifestDigest)
			if err != nil {
				log.Error().Err(err).Str("repository", repo).Str("reference", reference).Str("digest",
					signedManifestDigest.String()).Msg("load-repo: failed verify signatures validity for signed image")

				return err
			}

			return nil
		}

		imageMeta = convert.GetImageManifestMeta(manifestContent, configContent, int64(len(blob)), digest)
	case ispec.MediaTypeImageIndex:
		indexContent := ispec.Index{}

		err := json.Unmarshal(blob, &indexContent)
		if err != nil {
			return err
		}

		imageMeta = convert.GetImageIndexMeta(indexContent, int64(len(blob)), digest)
	default:
		return nil
	}

	err := metaDB.SetRepoReference(ctx, repo, reference, imageMeta)
	if err != nil {
		log.Error().Err(err).Msg("metadb: error while setting repo meta")

		return err
	}

	return nil
}

func isSignature(reference string, manifestContent ispec.Manifest) (bool, string, godigest.Digest) {
	manifestArtifactType := zcommon.GetManifestArtifactType(manifestContent)

	// check notation signature
	if manifestArtifactType == zcommon.ArtifactTypeNotation && manifestContent.Subject != nil {
		return true, NotationType, manifestContent.Subject.Digest
	}

	// check cosign signature
	if manifestArtifactType == zcommon.ArtifactTypeCosign && manifestContent.Subject != nil {
		return true, CosignType, manifestContent.Subject.Digest
	}

	if tag := reference; zcommon.IsCosignTag(reference) {
		prefixLen := len("sha256-")
		digestLen := 64
		signedImageManifestDigestEncoded := tag[prefixLen : prefixLen+digestLen]

		signedImageManifestDigest := godigest.NewDigestFromEncoded(godigest.SHA256,
			signedImageManifestDigestEncoded)

		return true, CosignType, signedImageManifestDigest
	}

	return false, "", ""
}