package meta import ( "context" "encoding/json" "errors" "time" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" zerr "zotregistry.dev/zot/errors" zcommon "zotregistry.dev/zot/pkg/common" "zotregistry.dev/zot/pkg/log" "zotregistry.dev/zot/pkg/meta/convert" mTypes "zotregistry.dev/zot/pkg/meta/types" stypes "zotregistry.dev/zot/pkg/storage/types" ) const ( CosignType = "cosign" NotationType = "notation" ) // ParseStorage will sync all repos found in the rootdirectory of the oci layout that zot was deployed on with the // ParseStorage database. func ParseStorage(metaDB mTypes.MetaDB, storeController stypes.StoreController, log log.Logger) error { log.Info().Str("component", "metadb").Msg("parsing storage and initializing") allStorageRepos, err := getAllRepos(storeController, log) if err != nil { return err } allMetaDBRepos, err := metaDB.GetAllRepoNames() if err != nil { rootDir := storeController.GetDefaultImageStore().RootDir() log.Error().Err(err).Str("component", "metadb").Str("rootDir", rootDir). Msg("failed to get all repo names present under rootDir") return err } for _, repo := range getReposToBeDeleted(allStorageRepos, allMetaDBRepos) { err := metaDB.DeleteRepoMeta(repo) if err != nil { log.Error().Err(err).Str("rootDir", storeController.GetImageStore(repo).RootDir()).Str("component", "metadb"). Str("repo", repo).Msg("failed to delete repo meta") return err } } for i, repo := range allStorageRepos { log.Info().Int("total", len(allStorageRepos)).Int("progress", i).Str("current-repo", repo). Msgf("parsing next repo '%s'", repo) imgStore := storeController.GetImageStore(repo) _, _, storageLastUpdated, err := imgStore.StatIndex(repo) if err != nil { log.Error().Err(err).Str("rootDir", imgStore.RootDir()). Str("repo", repo).Msg("failed to sync repo") continue } metaLastUpdated := metaDB.GetRepoLastUpdated(repo) if storageLastUpdated.Before(metaLastUpdated) { continue } err = ParseRepo(repo, metaDB, storeController, log) if err != nil { log.Error().Err(err).Str("repo", repo).Str("rootDir", imgStore.RootDir()).Msg("failed to sync repo") continue } } log.Info().Str("component", "metadb").Msg("successfully initialized") return nil } // getReposToBeDeleted will return all repos that are found in metaDB but not found in storage anymore. func getReposToBeDeleted(allStorageRepos []string, allMetaDBRepos []string) []string { toBeDeleted := []string{} storageRepoNameSet := map[string]struct{}{} for i := range allStorageRepos { storageRepoNameSet[allStorageRepos[i]] = struct{}{} } for _, metaDBRepo := range allMetaDBRepos { if _, found := storageRepoNameSet[metaDBRepo]; !found { toBeDeleted = append(toBeDeleted, metaDBRepo) } } return toBeDeleted } // ParseRepo reads the contents of a repo and syncs all images and signatures found. func ParseRepo(repo string, metaDB mTypes.MetaDB, storeController stypes.StoreController, log log.Logger) error { imageStore := storeController.GetImageStore(repo) var lockLatency time.Time imageStore.RLock(&lockLatency) defer imageStore.RUnlock(&lockLatency) indexBlob, err := imageStore.GetIndexContent(repo) if err != nil { log.Error().Err(err).Str("repository", repo).Msg("failed to read index.json for repo") return err } var indexContent ispec.Index err = json.Unmarshal(indexBlob, &indexContent) if err != nil { log.Error().Err(err).Str("repository", repo).Msg("failed to unmarshal index.json for repo") return err } err = metaDB.ResetRepoReferences(repo) if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { log.Error().Err(err).Str("repository", repo).Msg("failed to reset tag field in RepoMetadata for repo") return err } for _, manifest := range indexContent.Manifests { tag := manifest.Annotations[ispec.AnnotationRefName] if zcommon.IsReferrersTag(tag) { continue } manifestBlob, _, _, err := imageStore.GetImageManifest(repo, manifest.Digest.String()) if err != nil { log.Error().Err(err).Str("repository", repo).Str("digest", manifest.Digest.String()). Msg("failed to get blob for image") return err } reference := tag if tag == "" { reference = manifest.Digest.String() } err = SetImageMetaFromInput(context.Background(), repo, reference, manifest.MediaType, manifest.Digest, manifestBlob, imageStore, metaDB, log) if err != nil { log.Error().Err(err).Str("repository", repo).Str("tag", tag). Msg("failed to set metadata for image") return err } } return nil } func getAllRepos(storeController stypes.StoreController, log log.Logger) ([]string, error) { allRepos, err := storeController.GetDefaultImageStore().GetRepositories() if err != nil { log.Error().Err(err).Str("rootDir", storeController.GetDefaultImageStore().RootDir()). Msg("failed to get all repo names present under rootDir") return nil, err } if storeController.GetImageSubStores() != nil { for _, store := range storeController.GetImageSubStores() { substoreRepos, err := store.GetRepositories() if err != nil { log.Error().Err(err).Str("rootDir", store.RootDir()). Msg("failed to get all repo names present under rootDir") return nil, err } allRepos = append(allRepos, substoreRepos...) } } return allRepos, nil } func GetSignatureLayersInfo(repo, tag, manifestDigest, signatureType string, manifestBlob []byte, imageStore stypes.ImageStore, log log.Logger, ) ([]mTypes.LayerInfo, error) { switch signatureType { case zcommon.CosignSignature: return getCosignSignatureLayersInfo(repo, tag, manifestDigest, manifestBlob, imageStore, log) case zcommon.NotationSignature: return getNotationSignatureLayersInfo(repo, manifestDigest, manifestBlob, imageStore, log) default: return []mTypes.LayerInfo{}, nil } } func getCosignSignatureLayersInfo( repo, tag, manifestDigest string, manifestBlob []byte, imageStore stypes.ImageStore, log log.Logger, ) ([]mTypes.LayerInfo, error) { layers := []mTypes.LayerInfo{} var manifestContent ispec.Manifest if err := json.Unmarshal(manifestBlob, &manifestContent); err != nil { log.Error().Err(err).Str("repository", repo).Str("reference", tag).Str("digest", manifestDigest).Msg( "failed to marshal blob index") return layers, err } var lockLatency time.Time imageStore.RLock(&lockLatency) defer imageStore.RUnlock(&lockLatency) for _, layer := range manifestContent.Layers { layerContent, err := imageStore.GetBlobContent(repo, layer.Digest) if err != nil { log.Error().Err(err).Str("repository", repo).Str("reference", tag).Str("layerDigest", layer.Digest.String()).Msg( "failed to get cosign signature layer content") return layers, err } layerSigKey, ok := layer.Annotations[zcommon.CosignSigKey] if !ok { log.Error().Err(err).Str("repository", repo).Str("reference", tag).Str("layerDigest", layer.Digest.String()).Msg( "failed to get specific annotation of cosign signature") } layers = append(layers, mTypes.LayerInfo{ LayerDigest: layer.Digest.String(), LayerContent: layerContent, SignatureKey: layerSigKey, }) } return layers, nil } func getNotationSignatureLayersInfo( repo, manifestDigest string, manifestBlob []byte, imageStore stypes.ImageStore, log log.Logger, ) ([]mTypes.LayerInfo, error) { layers := []mTypes.LayerInfo{} var manifestContent ispec.Manifest if err := json.Unmarshal(manifestBlob, &manifestContent); err != nil { log.Error().Err(err).Str("repository", repo).Str("reference", manifestDigest).Msg( "failed to marshal blob index") return layers, err } // skip if is a notation index if manifestContent.MediaType == ispec.MediaTypeImageIndex { return []mTypes.LayerInfo{}, nil } if len(manifestContent.Layers) != 1 { log.Error().Err(zerr.ErrBadManifest).Str("repository", repo).Str("reference", manifestDigest). Msg("notation signature manifest requires exactly one layer but it does not") return layers, zerr.ErrBadManifest } layer := manifestContent.Layers[0].Digest var lockLatency time.Time imageStore.RLock(&lockLatency) defer imageStore.RUnlock(&lockLatency) layerContent, err := imageStore.GetBlobContent(repo, layer) if err != nil { log.Error().Err(err).Str("repository", repo).Str("reference", manifestDigest).Str("layerDigest", layer.String()).Msg( "failed to get notation signature blob content") return layers, err } layerSigKey := manifestContent.Layers[0].MediaType layers = append(layers, mTypes.LayerInfo{ LayerDigest: layer.String(), LayerContent: layerContent, SignatureKey: layerSigKey, }) return layers, nil } // SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag // (in case the reference is a tag). The function expects image manifests and indexes (multi arch images). func SetImageMetaFromInput(ctx context.Context, repo, reference, mediaType string, digest godigest.Digest, blob []byte, imageStore stypes.ImageStore, metaDB mTypes.MetaDB, log log.Logger, ) error { var imageMeta mTypes.ImageMeta switch mediaType { case ispec.MediaTypeImageManifest: manifestContent := ispec.Manifest{} configContent := ispec.Image{} err := json.Unmarshal(blob, &manifestContent) if err != nil { log.Error().Err(err).Str("component", "metadb").Msg("failed to unmarshal image manifest") return err } if manifestContent.Config.MediaType == ispec.MediaTypeImageConfig { configBlob, err := imageStore.GetBlobContent(repo, manifestContent.Config.Digest) if err != nil { return err } err = json.Unmarshal(configBlob, &configContent) if err != nil { return err } } if isSig, sigType, signedManifestDigest := isSignature(reference, manifestContent); isSig { layers, err := GetSignatureLayersInfo(repo, reference, digest.String(), sigType, blob, imageStore, log) if err != nil { return err } err = metaDB.AddManifestSignature(repo, signedManifestDigest, mTypes.SignatureMetadata{ SignatureType: sigType, SignatureDigest: digest.String(), SignatureTag: reference, LayersInfo: layers, }) if err != nil { log.Error().Err(err).Str("repository", repo).Str("tag", reference). Str("manifestDigest", signedManifestDigest.String()). Msg("failed set signature meta for signed image") return err } err = metaDB.UpdateSignaturesValidity(ctx, repo, signedManifestDigest) if err != nil { log.Error().Err(err).Str("repository", repo).Str("reference", reference).Str("digest", signedManifestDigest.String()).Msg("failed to verify signature validity for signed image") return err } return nil } imageMeta = convert.GetImageManifestMeta(manifestContent, configContent, int64(len(blob)), digest) case ispec.MediaTypeImageIndex: indexContent := ispec.Index{} err := json.Unmarshal(blob, &indexContent) if err != nil { return err } imageMeta = convert.GetImageIndexMeta(indexContent, int64(len(blob)), digest) default: return nil } err := metaDB.SetRepoReference(ctx, repo, reference, imageMeta) if err != nil { log.Error().Err(err).Str("component", "metadb").Msg("failed to set repo meta") return err } return nil } func isSignature(reference string, manifestContent ispec.Manifest) (bool, string, godigest.Digest) { manifestArtifactType := zcommon.GetManifestArtifactType(manifestContent) // check notation signature if manifestArtifactType == zcommon.ArtifactTypeNotation && manifestContent.Subject != nil { return true, NotationType, manifestContent.Subject.Digest } // check cosign signature if manifestArtifactType == zcommon.ArtifactTypeCosign && manifestContent.Subject != nil { return true, CosignType, manifestContent.Subject.Digest } if tag := reference; zcommon.IsCosignTag(reference) { prefixLen := len("sha256-") digestLen := 64 signedImageManifestDigestEncoded := tag[prefixLen : prefixLen+digestLen] signedImageManifestDigest := godigest.NewDigestFromEncoded(godigest.SHA256, signedImageManifestDigestEncoded) return true, CosignType, signedImageManifestDigest } return false, "", "" }