package sync import ( "context" "encoding/json" "errors" "fmt" "net/http" "net/url" "path" "strings" notreg "github.com/notaryproject/notation-go/registry" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" oras "github.com/oras-project/artifacts-spec/specs-go/v1" "github.com/sigstore/cosign/v2/pkg/oci/remote" zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/api/constants" "zotregistry.io/zot/pkg/common" syncconf "zotregistry.io/zot/pkg/extensions/config/sync" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/repodb" "zotregistry.io/zot/pkg/meta/signatures" "zotregistry.io/zot/pkg/storage" storageTypes "zotregistry.io/zot/pkg/storage/types" ) type signaturesCopier struct { client *http.Client upstreamURL url.URL credentials syncconf.Credentials repoDB repodb.RepoDB storeController storage.StoreController log log.Logger } func newSignaturesCopier(httpClient *http.Client, credentials syncconf.Credentials, upstreamURL url.URL, repoDB repodb.RepoDB, storeController storage.StoreController, log log.Logger, ) *signaturesCopier { return &signaturesCopier{ client: httpClient, credentials: credentials, upstreamURL: upstreamURL, repoDB: repoDB, storeController: storeController, log: log, } } func (sig *signaturesCopier) getCosignManifest(repo, digestStr string) (*ispec.Manifest, error) { var cosignManifest ispec.Manifest cosignTag := getCosignTagFromImageDigest(digestStr) getCosignManifestURL := sig.upstreamURL getCosignManifestURL.Path = path.Join(getCosignManifestURL.Path, "v2", repo, "manifests", cosignTag) getCosignManifestURL.RawQuery = getCosignManifestURL.Query().Encode() _, statusCode, err := common.MakeHTTPGetRequest(sig.client, sig.credentials.Username, sig.credentials.Password, &cosignManifest, getCosignManifestURL.String(), ispec.MediaTypeImageManifest, sig.log) if err != nil { if statusCode == http.StatusNotFound { sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err). Str("manifest", getCosignManifestURL.String()).Msg("couldn't find any cosign manifest") return nil, zerr.ErrSyncReferrerNotFound } sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err). Str("manifest", getCosignManifestURL.String()).Msg("couldn't get cosign manifest") return nil, err } return &cosignManifest, nil } func (sig *signaturesCopier) getORASRefs(repo, digestStr string) (ReferenceList, error) { var referrers ReferenceList getReferrersURL := sig.upstreamURL // based on manifest digest get referrers getReferrersURL.Path = path.Join(getReferrersURL.Path, constants.ArtifactSpecRoutePrefix, repo, "manifests", digestStr, "referrers") getReferrersURL.RawQuery = getReferrersURL.Query().Encode() _, statusCode, err := common.MakeHTTPGetRequest(sig.client, sig.credentials.Username, sig.credentials.Password, &referrers, getReferrersURL.String(), "application/json", sig.log) if err != nil { if statusCode == http.StatusNotFound { sig.log.Info().Err(err).Msg("couldn't find any ORAS artifact") return referrers, zerr.ErrSyncReferrerNotFound } sig.log.Error().Err(err).Msg("couldn't get ORAS artifacts") return referrers, err } return referrers, nil } func (sig *signaturesCopier) getOCIRefs(repo, digestStr string) (ispec.Index, error) { var index ispec.Index getReferrersURL := sig.upstreamURL // based on manifest digest get referrers getReferrersURL.Path = path.Join(getReferrersURL.Path, "v2", repo, "referrers", digestStr) getReferrersURL.RawQuery = getReferrersURL.Query().Encode() _, statusCode, err := common.MakeHTTPGetRequest(sig.client, sig.credentials.Username, sig.credentials.Password, &index, getReferrersURL.String(), "application/json", sig.log) if err != nil { if statusCode == http.StatusNotFound { sig.log.Info().Str("referrers", getReferrersURL.String()).Int("statusCode", statusCode). Msg("couldn't find any oci reference from referrers, skipping") return index, zerr.ErrSyncReferrerNotFound } sig.log.Error().Str("errorType", common.TypeOf(zerr.ErrSyncReferrer)).Err(zerr.ErrSyncReferrer). Str("referrers", getReferrersURL.String()).Int("statusCode", statusCode). Msg("couldn't get oci reference from referrers, skipping") return index, zerr.ErrSyncReferrer } return index, nil } func (sig *signaturesCopier) syncCosignSignature(localRepo, remoteRepo, digestStr string, cosignManifest *ispec.Manifest, ) error { cosignTag := getCosignTagFromImageDigest(digestStr) // if no manifest found if cosignManifest == nil { return nil } skipCosignSig, err := sig.canSkipCosignSignature(localRepo, digestStr, cosignManifest) if err != nil { sig.log.Error().Err(err).Str("repository", remoteRepo).Str("reference", digestStr). Msg("couldn't check if the upstream image cosign signature can be skipped") } if skipCosignSig { return nil } imageStore := sig.storeController.GetImageStore(localRepo) sig.log.Info().Msg("syncing cosign signatures") for _, blob := range cosignManifest.Layers { if err := syncBlob(sig, imageStore, localRepo, remoteRepo, blob.Digest); err != nil { return err } } // sync config blob if err := syncBlob(sig, imageStore, localRepo, remoteRepo, cosignManifest.Config.Digest); err != nil { return err } cosignManifestBuf, err := json.Marshal(cosignManifest) if err != nil { sig.log.Error().Str("errorType", common.TypeOf(err)). Err(err).Msg("couldn't marshal cosign manifest") } // push manifest signatureDigest, _, err := imageStore.PutImageManifest(localRepo, cosignTag, ispec.MediaTypeImageManifest, cosignManifestBuf) if err != nil { sig.log.Error().Str("errorType", common.TypeOf(err)). Err(err).Msg("couldn't upload cosign manifest") return err } if sig.repoDB != nil { sig.log.Debug().Str("repository", localRepo).Str("digest", digestStr). Msg("trying to sync cosign signature for repo digest") err := sig.repoDB.AddManifestSignature(localRepo, godigest.Digest(digestStr), repodb.SignatureMetadata{ SignatureType: signatures.CosignSignature, SignatureDigest: signatureDigest.String(), }) if err != nil { return fmt.Errorf("failed to set metadata for cosign signature '%s@%s': %w", localRepo, digestStr, err) } sig.log.Info().Str("repository", localRepo).Str("digest", digestStr). Msg("successfully added cosign signature to RepoDB for repo digest") } return nil } func (sig *signaturesCopier) syncORASRefs(localRepo, remoteRepo, digestStr string, referrers ReferenceList, ) error { if len(referrers.References) == 0 { return nil } skipORASRefs, err := sig.canSkipORASRefs(localRepo, digestStr, referrers) if err != nil { sig.log.Error().Err(err).Str("repository", remoteRepo).Str("reference", digestStr). Msg("couldn't check if the upstream image ORAS artifact can be skipped") } if skipORASRefs { return nil } imageStore := sig.storeController.GetImageStore(localRepo) sig.log.Info().Msg("syncing ORAS artifacts") for _, ref := range referrers.References { // get referrer manifest getRefManifestURL := sig.upstreamURL getRefManifestURL.Path = path.Join(getRefManifestURL.Path, "v2", remoteRepo, "manifests", ref.Digest.String()) getRefManifestURL.RawQuery = getRefManifestURL.Query().Encode() var artifactManifest oras.Manifest body, statusCode, err := common.MakeHTTPGetRequest(sig.client, sig.credentials.Username, sig.credentials.Password, &artifactManifest, getRefManifestURL.String(), ref.MediaType, sig.log) if err != nil { if statusCode == http.StatusNotFound { sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err). Str("manifest", getRefManifestURL.String()).Msg("couldn't find any ORAS manifest") return zerr.ErrSyncReferrerNotFound } sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err). Str("manifest", getRefManifestURL.String()).Msg("couldn't get ORAS manifest") return err } for _, blob := range artifactManifest.Blobs { if err := syncBlob(sig, imageStore, localRepo, remoteRepo, blob.Digest); err != nil { return err } } signatureDigest, _, err := imageStore.PutImageManifest(localRepo, ref.Digest.String(), oras.MediaTypeArtifactManifest, body) if err != nil { sig.log.Error().Str("errorType", common.TypeOf(err)). Err(err).Msg("couldn't upload ORAS manifest") return err } // this is for notation signatures if sig.repoDB != nil { sig.log.Debug().Str("repository", localRepo).Str("digest", digestStr). Msg("trying to sync oras artifact for digest") err := sig.repoDB.AddManifestSignature(localRepo, godigest.Digest(digestStr), repodb.SignatureMetadata{ SignatureType: signatures.NotationSignature, SignatureDigest: signatureDigest.String(), }) if err != nil { return fmt.Errorf("failed to set metadata for oras artifact '%s@%s': %w", localRepo, digestStr, err) } sig.log.Info().Str("repository", localRepo).Str("digest", digestStr). Msg("successfully added oras artifacts to RepoDB for digest") } } sig.log.Info().Str("repository", localRepo).Str("digest", digestStr). Msg("successfully synced ORAS artifacts for digest") return nil } func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, subjectStr string, index ispec.Index, ) error { if len(index.Manifests) == 0 { return nil } skipOCIRefs, err := sig.canSkipOCIRefs(localRepo, subjectStr, index) if err != nil { sig.log.Error().Err(err).Str("repository", remoteRepo).Str("reference", subjectStr). Msg("couldn't check if the upstream image oci references can be skipped") } if skipOCIRefs { return nil } imageStore := sig.storeController.GetImageStore(localRepo) sig.log.Info().Msg("syncing oci references") for _, ref := range index.Manifests { getRefManifestURL := sig.upstreamURL getRefManifestURL.Path = path.Join(getRefManifestURL.Path, "v2", remoteRepo, "manifests", ref.Digest.String()) getRefManifestURL.RawQuery = getRefManifestURL.Query().Encode() var artifactManifest oras.Manifest OCIRefBody, statusCode, err := common.MakeHTTPGetRequest(sig.client, sig.credentials.Username, sig.credentials.Password, &artifactManifest, getRefManifestURL.String(), ref.MediaType, sig.log) if err != nil { if statusCode == http.StatusNotFound { sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err). Str("manifest", getRefManifestURL.String()).Msg("couldn't find any oci reference manifest") return zerr.ErrSyncReferrerNotFound } sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err). Str("manifest", getRefManifestURL.String()).Msg("couldn't get oci reference manifest") return err } if ref.MediaType == ispec.MediaTypeImageManifest { // read manifest var manifest ispec.Manifest err = json.Unmarshal(OCIRefBody, &manifest) if err != nil { sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err). Str("manifest", getRefManifestURL.String()).Msg("couldn't unmarshal oci reference manifest") return err } for _, layer := range manifest.Layers { if err := syncBlob(sig, imageStore, localRepo, remoteRepo, layer.Digest); err != nil { return err } } // sync config blob if err := syncBlob(sig, imageStore, localRepo, remoteRepo, manifest.Config.Digest); err != nil { return err } } else { continue } refDigest, _, err := imageStore.PutImageManifest(localRepo, ref.Digest.String(), ref.MediaType, OCIRefBody) if err != nil { sig.log.Error().Str("errorType", common.TypeOf(err)). Err(err).Msg("couldn't upload oci reference manifest") return err } if sig.repoDB != nil { sig.log.Debug().Str("repository", localRepo).Str("digest", subjectStr).Msg("trying to add OCI refs for repo digest") isSig, sigType, signedManifestDig, err := storage.CheckIsImageSignature(localRepo, OCIRefBody, ref.Digest.String()) if err != nil { return fmt.Errorf("failed to set metadata for OCI ref in '%s@%s': %w", localRepo, subjectStr, err) } if isSig { err = sig.repoDB.AddManifestSignature(localRepo, signedManifestDig, repodb.SignatureMetadata{ SignatureType: sigType, SignatureDigest: refDigest.String(), }) } else { err = repodb.SetImageMetaFromInput(localRepo, refDigest.String(), ref.MediaType, refDigest, OCIRefBody, sig.storeController.GetImageStore(localRepo), sig.repoDB, sig.log) } if err != nil { return fmt.Errorf("failed to set metadata for OCI ref in '%s@%s': %w", localRepo, subjectStr, err) } sig.log.Info().Str("repository", localRepo).Str("digest", subjectStr). Msg("successfully added OCI refs to RepoDB for digest") } } sig.log.Info().Str("repository", localRepo).Str("digest", subjectStr). Msg("successfully synced OCI refs for digest") return nil } func (sig *signaturesCopier) canSkipORASRefs(localRepo, digestStr string, refs ReferenceList, ) (bool, error) { imageStore := sig.storeController.GetImageStore(localRepo) digest := godigest.Digest(digestStr) // check oras artifacts already synced if len(refs.References) > 0 { localRefs, err := imageStore.GetOrasReferrers(localRepo, digest, "") if err != nil { if errors.Is(err, zerr.ErrManifestNotFound) { return false, nil } sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err). Str("repository", localRepo).Str("reference", digestStr).Msg("couldn't get local ORAS artifact manifest") return false, err } if !artifactDescriptorsEqual(localRefs, refs.References) { sig.log.Info().Str("repository", localRepo).Str("reference", digestStr). Msg("upstream ORAS artifacts changed, syncing again") return false, nil } } sig.log.Info().Str("repository", localRepo).Str("reference", digestStr). Msg("skipping ORAS artifact, already synced") return true, nil } func (sig *signaturesCopier) canSkipCosignSignature(localRepo, digestStr string, cosignManifest *ispec.Manifest, ) (bool, error) { imageStore := sig.storeController.GetImageStore(localRepo) // check cosign signature already synced if cosignManifest != nil { var localCosignManifest ispec.Manifest /* we need to use tag (cosign format: sha256-$IMAGE_TAG.sig) instead of digest to get local cosign manifest because of an issue where cosign digests differs between upstream and downstream */ cosignManifestTag := getCosignTagFromImageDigest(digestStr) localCosignManifestBuf, _, _, err := imageStore.GetImageManifest(localRepo, cosignManifestTag) if err != nil { if errors.Is(err, zerr.ErrManifestNotFound) { return false, nil } sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err). Str("repository", localRepo).Str("reference", digestStr). Msg("couldn't get local cosign manifest") return false, err } err = json.Unmarshal(localCosignManifestBuf, &localCosignManifest) if err != nil { sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err). Str("repository", localRepo).Str("reference", digestStr). Msg("couldn't unmarshal local cosign signature manifest") return false, err } if !manifestsEqual(localCosignManifest, *cosignManifest) { sig.log.Info().Str("repository", localRepo).Str("reference", digestStr). Msg("upstream cosign signatures changed, syncing again") return false, nil } } sig.log.Info().Str("repository", localRepo).Str("reference", digestStr). Msg("skipping cosign signature, already synced") return true, nil } func (sig *signaturesCopier) canSkipOCIRefs(localRepo, digestStr string, index ispec.Index, ) (bool, error) { imageStore := sig.storeController.GetImageStore(localRepo) digest := godigest.Digest(digestStr) // check oci references already synced if len(index.Manifests) > 0 { localRefs, err := imageStore.GetReferrers(localRepo, digest, nil) if err != nil { if errors.Is(err, zerr.ErrManifestNotFound) { return false, nil } sig.log.Error().Str("errorType", common.TypeOf(err)).Err(err). Str("repository", localRepo).Str("reference", digestStr). Msg("couldn't get local ocireferences for manifest") return false, err } if !descriptorsEqual(localRefs.Manifests, index.Manifests) { sig.log.Info().Str("repository", localRepo).Str("reference", digestStr). Msg("upstream oci references for manifest changed, syncing again") return false, nil } } sig.log.Info().Str("repository", localRepo).Str("reference", digestStr). Msg("skipping oci references, already synced") return true, nil } func syncBlob(sig *signaturesCopier, imageStore storageTypes.ImageStore, localRepo, remoteRepo string, digest godigest.Digest, ) error { getBlobURL := sig.upstreamURL getBlobURL.Path = path.Join(getBlobURL.Path, "v2", remoteRepo, "blobs", digest.String()) getBlobURL.RawQuery = getBlobURL.Query().Encode() req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, getBlobURL.String(), nil) if err != nil { return err } resp, err := sig.client.Do(req) if err != nil { sig.log.Error().Str("errorType", common.TypeOf(err)).Str("blob url", getBlobURL.String()). Err(err).Msg("couldn't get blob from url") return err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { sig.log.Info().Str("url", getBlobURL.String()).Str("blob url", getBlobURL.String()). Int("statusCode", resp.StatusCode).Msg("couldn't find blob from url, status code") return zerr.ErrSyncReferrer } _, _, err = imageStore.FullBlobUpload(localRepo, resp.Body, digest) if err != nil { sig.log.Error().Str("errorType", common.TypeOf(err)).Str("digest", digest.String()). Err(err).Msg("couldn't upload blob") return err } return nil } // sync feature will try to pull cosign signature because for sync cosign signature is just an image // this function will check if tag is a cosign tag. func isCosignTag(tag string) bool { if strings.HasPrefix(tag, "sha256-") && strings.HasSuffix(tag, remote.SignatureTagSuffix) { return true } return false } func getCosignTagFromImageDigest(digestStr string) string { if !isCosignTag(digestStr) { return strings.Replace(digestStr, ":", "-", 1) + "." + remote.SignatureTagSuffix } return digestStr } func getNotationManifestsFromOCIRefs(ociRefs ispec.Index) []ispec.Descriptor { notaryManifests := []ispec.Descriptor{} for _, ref := range ociRefs.Manifests { if ref.ArtifactType == notreg.ArtifactTypeNotation { notaryManifests = append(notaryManifests, ref) } } return notaryManifests }