Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00
Code Issues Activity

feat(sync,s3): added s3 logic for ORAS and OCI artifacts (#985)

Browse source 
added sync logic for OCI artifacts

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
peusebiu 2022-11-15 08:21:49 +02:00 committed by GitHub
parent fb3e4ec2ef
commit e96c80c344
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 1243 additions and 423 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
errors/errors.go
View file

@ -51,8 +51,8 @@ var (
ErrInjected = errors.New("test: injected failure")
ErrSyncInvalidUpstreamURL = errors.New("sync: upstream url not found in sync config")
ErrRegistryNoContent = errors.New("sync: could not find a Content that matches localRepo")
ErrSyncSignatureNotFound = errors.New("sync: couldn't find any upstream notary/cosign signatures")
ErrSyncSignature = errors.New("sync: couldn't get upstream notary/cosign signatures")
ErrSyncReferrerNotFound = errors.New("sync: couldn't find upstream referrer")
ErrSyncReferrer = errors.New("sync: failed to get upstream referrer")
ErrImageLintAnnotations = errors.New("routes: lint checks failed")
ErrParsingAuthHeader = errors.New("auth: failed parsing authorization header")
ErrBadType = errors.New("core: invalid type")

19
pkg/api/routes.go
View file

@ -34,6 +34,7 @@ import (
gqlPlayground "zotregistry.io/zot/pkg/debug/gqlplayground"
debug "zotregistry.io/zot/pkg/debug/swagger"
ext "zotregistry.io/zot/pkg/extensions"
"zotregistry.io/zot/pkg/extensions/sync"
"zotregistry.io/zot/pkg/log"
localCtx "zotregistry.io/zot/pkg/requestcontext"
"zotregistry.io/zot/pkg/storage"
@ -413,7 +414,6 @@ func getReferrers(ctx context.Context, routeHandler *RouteHandler,
imgStore storage.ImageStore, name string, digest godigest.Digest,
artifactType string,
) (ispec.Index, error) {
// first get the subject and then all its referrers
references, err := imgStore.GetReferrers(name, digest, artifactType)
if err != nil {
if routeHandler.c.Config.Extensions != nil &&
@ -423,24 +423,13 @@ func getReferrers(ctx context.Context, routeHandler *RouteHandler,
name, digest)
errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.StoreController,
name, digest.String(), false, routeHandler.c.Log)
name, digest.String(), sync.OCIReference, routeHandler.c.Log)
if errSync != nil {
routeHandler.c.Log.Error().Err(err).Str("name", name).Str("digest", digest.String()).Msg("unable to get references")
return ispec.Index{}, err
}
for _, ref := range references.Manifests {
errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.StoreController,
name, ref.Digest.String(), false, routeHandler.c.Log)
if errSync != nil {
routeHandler.c.Log.Error().Err(err).Str("name", name).
Str("digest", ref.Digest.String()).Msg("unable to get references")
return ispec.Index{}, err
}
}
references, err = imgStore.GetReferrers(name, digest, artifactType)
}
}
@ -1589,7 +1578,7 @@ func getImageManifest(ctx context.Context, routeHandler *RouteHandler, imgStore
name, reference)
errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.StoreController,
name, reference, false, routeHandler.c.Log)
name, reference, "", routeHandler.c.Log)
if errSync != nil {
routeHandler.c.Log.Err(errSync).Msgf("error encounter while syncing image %s:%s",
name, reference)
@ -1619,7 +1608,7 @@ func getOrasReferrers(ctx context.Context, routeHandler *RouteHandler,
name, digest.String())
errSync := ext.SyncOneImage(ctx, routeHandler.c.Config, routeHandler.c.StoreController,
name, digest.String(), true, routeHandler.c.Log)
name, digest.String(), sync.OrasArtifact, routeHandler.c.Log)
if errSync != nil {
routeHandler.c.Log.Error().Err(err).Str("name", name).Str("digest", digest.String()).Msg("unable to get references")

4
pkg/extensions/extension_sync.go
View file

@ -26,11 +26,11 @@ func EnableSyncExtension(ctx context.Context, config *config.Config, wg *goSync.
}
func SyncOneImage(ctx context.Context, config *config.Config, storeController storage.StoreController,
repoName, reference string, isArtifact bool, log log.Logger,
repoName, reference string, artifactType string, log log.Logger,
) error {
log.Info().Msgf("syncing image %s:%s", repoName, reference)
err := sync.OneImage(ctx, *config.Extensions.Sync, storeController, repoName, reference, isArtifact, log)
err := sync.OneImage(ctx, *config.Extensions.Sync, storeController, repoName, reference, artifactType, log)
return err
}

2
pkg/extensions/extension_sync_disabled.go
View file

@ -23,7 +23,7 @@ func EnableSyncExtension(ctx context.Context,
// SyncOneImage ...
func SyncOneImage(ctx context.Context, config *config.Config, storeController storage.StoreController,
repoName, reference string, isArtifact bool, log log.Logger,
repoName, reference string, artifactType string, log log.Logger,
) error {
log.Warn().Msg("skipping syncing on demand because given zot binary doesn't include this feature," +
"please build a binary that does so")

124
pkg/extensions/sync/on_demand.go
View file

@ -17,6 +17,11 @@ import (
"zotregistry.io/zot/pkg/storage"
)
const (
OrasArtifact = "orasArtifact"
OCIReference = "ociReference"
)
type syncContextUtils struct {
policyCtx *signature.PolicyContext
localCtx *types.SystemContext
@ -51,7 +56,7 @@ func (di *demandedImages) delete(key string) {
}
func OneImage(ctx context.Context, cfg Config, storeController storage.StoreController,
repo, reference string, isArtifact bool, log log.Logger,
repo, reference string, artifactType string, log log.Logger,
) error {
// guard against multiple parallel requests
demandedImage := fmt.Sprintf("%s:%s", repo, reference)
@ -73,7 +78,7 @@ func OneImage(ctx context.Context, cfg Config, storeController storage.StoreCont
defer demandedImgs.delete(demandedImage)
defer close(imageChannel)
go syncOneImage(ctx, imageChannel, cfg, storeController, repo, reference, isArtifact, log)
go syncOneImage(ctx, imageChannel, cfg, storeController, repo, reference, artifactType, log)
err, ok := <-imageChannel
if !ok {
@ -84,7 +89,7 @@ func OneImage(ctx context.Context, cfg Config, storeController storage.StoreCont
}
func syncOneImage(ctx context.Context, imageChannel chan error, cfg Config, storeController storage.StoreController,
localRepo, reference string, isArtifact bool, log log.Logger,
localRepo, reference string, artifactType string, log log.Logger,
) {
var credentialsFile CredentialsFile
@ -160,44 +165,11 @@ func syncOneImage(ctx context.Context, imageChannel chan error, cfg Config, stor
upstreamCtx := getUpstreamContext(&regCfg, credentialsFile[upstreamAddr])
options := getCopyOptions(upstreamCtx, localCtx)
// demanded 'image' is a signature
if isCosignTag(reference) {
// at tis point we should already have images synced, but not their signatures.
// is cosign signature
cosignManifest, err := sig.getCosignManifest(upstreamRepo, reference)
/* demanded object is a signature or artifact
at tis point we already have images synced, but not their signatures. */
if isCosignTag(reference) || artifactType != "" {
err = syncSignaturesArtifacts(sig, localRepo, upstreamRepo, reference, artifactType)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get upstream image %s:%s:%s cosign manifest", upstreamURL, upstreamRepo, reference)
continue
}
err = sig.syncCosignSignature(localRepo, upstreamRepo, reference, cosignManifest)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't copy upstream image cosign signature %s/%s:%s", upstreamURL, upstreamRepo, reference)
continue
}
imageChannel <- nil
return
} else if isArtifact {
// is notary signature
refs, err := sig.getNotaryRefs(upstreamRepo, reference)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get upstream image %s/%s:%s notary references", upstreamURL, upstreamRepo, reference)
continue
}
err = sig.syncNotarySignature(localRepo, upstreamRepo, reference, refs)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't copy image signature %s/%s:%s", upstreamURL, upstreamRepo, reference)
continue
}
@ -300,7 +272,7 @@ func syncRun(regCfg RegistryConfig,
Err(err).Msgf("couldn't get upstream image %s cosign manifest", upstreamImageRef.DockerReference())
}
refs, err := sig.getNotaryRefs(upstreamRepo, upstreamImageDigest.String())
refs, err := sig.getNotarySignatures(upstreamRepo, upstreamImageDigest.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get upstream image %s notary references", upstreamImageRef.DockerReference())
@ -355,6 +327,17 @@ func syncRun(regCfg RegistryConfig,
return false, err
}
index, err := sig.getOCIRefs(upstreamRepo, upstreamImageDigest.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get upstream image %s oci references", upstreamImageRef.DockerReference())
}
err = sig.syncOCIRefs(localRepo, upstreamRepo, upstreamImageDigest.String(), index)
if err != nil {
return false, err
}
err = sig.syncCosignSignature(localRepo, upstreamRepo, upstreamImageDigest.String(), cosignManifest)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
@ -375,3 +358,62 @@ func syncRun(regCfg RegistryConfig,
return false, nil
}
func syncSignaturesArtifacts(sig *signaturesCopier, localRepo, upstreamRepo, reference, artifactType string) error {
upstreamURL := sig.upstreamURL.String()
switch {
case isCosignTag(reference):
// is cosign signature
cosignManifest, err := sig.getCosignManifest(upstreamRepo, reference)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get upstream image %s:%s:%s cosign manifest", upstreamURL, upstreamRepo, reference)
return err
}
err = sig.syncCosignSignature(localRepo, upstreamRepo, reference, cosignManifest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't copy upstream image cosign signature %s/%s:%s", upstreamURL, upstreamRepo, reference)
return err
}
case artifactType == OrasArtifact:
// is notary signature
refs, err := sig.getNotarySignatures(upstreamRepo, reference)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get upstream image %s/%s:%s notary references", upstreamURL, upstreamRepo, reference)
return err
}
err = sig.syncNotarySignature(localRepo, upstreamRepo, reference, refs)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't copy image signature %s/%s:%s", upstreamURL, upstreamRepo, reference)
return err
}
case artifactType == OCIReference:
index, err := sig.getOCIRefs(upstreamRepo, reference)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get oci references %s/%s:%s", upstreamURL, upstreamRepo, reference)
return err
}
err = sig.syncOCIRefs(localRepo, upstreamRepo, reference, index)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't copy oci references %s/%s:%s", upstreamURL, upstreamRepo, reference)
return err
}
}
return nil
}

315
pkg/extensions/sync/signatures.go
View file

@ -11,7 +11,7 @@ import (
notreg "github.com/notaryproject/notation-go/registry"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
oras "github.com/oras-project/artifacts-spec/specs-go/v1"
"github.com/sigstore/cosign/pkg/oci/remote"
"gopkg.in/resty.v1"
@ -51,7 +51,7 @@ func (sig *signaturesCopier) getCosignManifest(repo, digestStr string) (*ispec.M
getCosignManifestURL.RawQuery = getCosignManifestURL.Query().Encode()
resp, err := sig.client.R().
SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
SetHeader("Content-Type", ispec.MediaTypeImageManifest).
Get(getCosignManifestURL.String())
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
@ -65,13 +65,13 @@ func (sig *signaturesCopier) getCosignManifest(repo, digestStr string) (*ispec.M
sig.log.Info().Msgf("couldn't find any cosign signature from %s, status code: %d skipping",
getCosignManifestURL.String(), resp.StatusCode())
return nil, zerr.ErrSyncSignatureNotFound
return nil, zerr.ErrSyncReferrerNotFound
} else if resp.IsError() {
sig.log.Error().Str("errorType", TypeOf(zerr.ErrSyncSignature)).
Err(zerr.ErrSyncSignature).Msgf("couldn't get cosign signature from %s, status code: %d skipping",
sig.log.Error().Str("errorType", TypeOf(zerr.ErrSyncReferrer)).
Err(zerr.ErrSyncReferrer).Msgf("couldn't get cosign signature from %s, status code: %d skipping",
getCosignManifestURL.String(), resp.StatusCode())
return nil, zerr.ErrSyncSignature
return nil, zerr.ErrSyncReferrer
}
err = json.Unmarshal(resp.Body(), &cosignManifest)
@ -86,7 +86,7 @@ func (sig *signaturesCopier) getCosignManifest(repo, digestStr string) (*ispec.M
return &cosignManifest, nil
}
func (sig *signaturesCopier) getNotaryRefs(repo, digestStr string) (ReferenceList, error) {
func (sig *signaturesCopier) getNotarySignatures(repo, digestStr string) (ReferenceList, error) {
var referrers ReferenceList
getReferrersURL := sig.upstreamURL
@ -112,13 +112,13 @@ func (sig *signaturesCopier) getNotaryRefs(repo, digestStr string) (ReferenceLis
sig.log.Info().Msgf("couldn't find any notary signature from %s, status code: %d, skipping",
getReferrersURL.String(), resp.StatusCode())
return ReferenceList{}, zerr.ErrSyncSignatureNotFound
return ReferenceList{}, zerr.ErrSyncReferrerNotFound
} else if resp.IsError() {
sig.log.Error().Str("errorType", TypeOf(zerr.ErrSyncSignature)).
Err(zerr.ErrSyncSignature).Msgf("couldn't get notary signature from %s, status code: %d skipping",
sig.log.Error().Str("errorType", TypeOf(zerr.ErrSyncReferrer)).
Err(zerr.ErrSyncReferrer).Msgf("couldn't get notary signature from %s, status code: %d skipping",
getReferrersURL.String(), resp.StatusCode())
return ReferenceList{}, zerr.ErrSyncSignature
return ReferenceList{}, zerr.ErrSyncReferrer
}
err = json.Unmarshal(resp.Body(), &referrers)
@ -133,6 +133,50 @@ func (sig *signaturesCopier) getNotaryRefs(repo, digestStr string) (ReferenceLis
return referrers, nil
}
func (sig *signaturesCopier) getOCIRefs(repo, digestStr string) (ispec.Index, error) {
var index ispec.Index
getReferrersURL := sig.upstreamURL
// based on manifest digest get referrers
getReferrersURL.Path = path.Join(getReferrersURL.Path, "v2", repo, "referrers", digestStr)
getReferrersURL.RawQuery = getReferrersURL.Query().Encode()
resp, err := sig.client.R().
SetHeader("Content-Type", "application/json").
Get(getReferrersURL.String())
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("url", getReferrersURL.String()).Msg("couldn't get referrers")
return index, err
}
if resp.StatusCode() == http.StatusNotFound {
sig.log.Info().Msgf("couldn't find any oci reference from %s, status code: %d, skipping",
getReferrersURL.String(), resp.StatusCode())
return index, zerr.ErrSyncReferrerNotFound
} else if resp.IsError() {
sig.log.Error().Str("errorType", TypeOf(zerr.ErrSyncReferrer)).
Err(zerr.ErrSyncReferrer).Msgf("couldn't get oci reference from %s, status code: %d skipping",
getReferrersURL.String(), resp.StatusCode())
return index, zerr.ErrSyncReferrer
}
err = json.Unmarshal(resp.Body(), &index)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("url", getReferrersURL.String()).
Msgf("couldn't unmarshal oci reference")
return index, err
}
return index, nil
}
func (sig *signaturesCopier) syncCosignSignature(localRepo, remoteRepo, digestStr string,
cosignManifest *ispec.Manifest,
) error {
@ -158,65 +202,13 @@ func (sig *signaturesCopier) syncCosignSignature(localRepo, remoteRepo, digestSt
sig.log.Info().Msg("syncing cosign signatures")
for _, blob := range cosignManifest.Layers {
// get blob
getBlobURL := sig.upstreamURL
getBlobURL.Path = path.Join(getBlobURL.Path, "v2", remoteRepo, "blobs", blob.Digest.String())
getBlobURL.RawQuery = getBlobURL.Query().Encode()
resp, err := sig.client.R().SetDoNotParseResponse(true).Get(getBlobURL.String())
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get cosign blob: %s", blob.Digest.String())
return err
}
if resp.IsError() {
sig.log.Info().Msgf("couldn't find cosign blob from %s, status code: %d", getBlobURL.String(), resp.StatusCode())
return zerr.ErrSyncSignature
}
defer resp.RawBody().Close()
// push blob
_, _, err = imageStore.FullBlobUpload(localRepo, resp.RawBody(), blob.Digest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload cosign blob")
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, blob.Digest); err != nil {
return err
}
}
// get config blob
getBlobURL := sig.upstreamURL
getBlobURL.Path = path.Join(getBlobURL.Path, "v2", remoteRepo, "blobs", cosignManifest.Config.Digest.String())
getBlobURL.RawQuery = getBlobURL.Query().Encode()
resp, err := sig.client.R().SetDoNotParseResponse(true).Get(getBlobURL.String())
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get cosign config blob: %s", getBlobURL.String())
return err
}
if resp.IsError() {
sig.log.Info().Msgf("couldn't find cosign config blob from %s, status code: %d",
getBlobURL.String(), resp.StatusCode())
return zerr.ErrSyncSignature
}
defer resp.RawBody().Close()
// push config blob
_, _, err = imageStore.FullBlobUpload(localRepo, resp.RawBody(), cosignManifest.Config.Digest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload cosign config blob")
// sync config blob
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, cosignManifest.Config.Digest); err != nil {
return err
}
@ -248,7 +240,7 @@ func (sig *signaturesCopier) syncNotarySignature(localRepo, remoteRepo, digestSt
}
skipNotarySig, err := sig.canSkipNotarySignature(localRepo, digestStr, referrers)
if skipNotarySig || err != nil {
if err != nil {
sig.log.Error().Err(err).Msgf("couldn't check if the upstream image %s:%s notary signature can be skipped",
remoteRepo, digestStr)
}
@ -268,6 +260,7 @@ func (sig *signaturesCopier) syncNotarySignature(localRepo, remoteRepo, digestSt
getRefManifestURL.RawQuery = getRefManifestURL.Query().Encode()
resp, err := sig.client.R().
SetHeader("Content-Type", ref.MediaType).
Get(getRefManifestURL.String())
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
@ -277,7 +270,7 @@ func (sig *signaturesCopier) syncNotarySignature(localRepo, remoteRepo, digestSt
}
// read manifest
var artifactManifest artifactspec.Manifest
var artifactManifest oras.Manifest
err = json.Unmarshal(resp.Body(), &artifactManifest)
if err != nil {
@ -288,38 +281,13 @@ func (sig *signaturesCopier) syncNotarySignature(localRepo, remoteRepo, digestSt
}
for _, blob := range artifactManifest.Blobs {
getBlobURL := sig.upstreamURL
getBlobURL.Path = path.Join(getBlobURL.Path, "v2", remoteRepo, "blobs", blob.Digest.String())
getBlobURL.RawQuery = getBlobURL.Query().Encode()
resp, err := sig.client.R().SetDoNotParseResponse(true).Get(getBlobURL.String())
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get notary blob: %s", getBlobURL.String())
return err
}
defer resp.RawBody().Close()
if resp.IsError() {
sig.log.Info().Msgf("couldn't find notary blob from %s, status code: %d",
getBlobURL.String(), resp.StatusCode())
return zerr.ErrSyncSignature
}
_, _, err = imageStore.FullBlobUpload(localRepo, resp.RawBody(), blob.Digest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload notary sig blob")
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, blob.Digest); err != nil {
return err
}
}
_, err = imageStore.PutImageManifest(localRepo, ref.Digest.String(),
artifactspec.MediaTypeArtifactManifest, resp.Body())
oras.MediaTypeArtifactManifest, resp.Body())
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload notary sig manifest")
@ -333,6 +301,97 @@ func (sig *signaturesCopier) syncNotarySignature(localRepo, remoteRepo, digestSt
return nil
}
func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, digestStr string, index ispec.Index,
) error {
if len(index.Manifests) == 0 {
return nil
}
skipOCIRefs, err := sig.canSkipOCIRefs(localRepo, digestStr, index)
if err != nil {
sig.log.Error().Err(err).Msgf("couldn't check if the upstream image %s:%s oci references can be skipped",
remoteRepo, digestStr)
}
if skipOCIRefs {
return nil
}
imageStore := sig.storeController.GetImageStore(localRepo)
sig.log.Info().Msg("syncing oci references")
for _, ref := range index.Manifests {
getRefManifestURL := sig.upstreamURL
getRefManifestURL.Path = path.Join(getRefManifestURL.Path, "v2", remoteRepo, "manifests", ref.Digest.String())
getRefManifestURL.RawQuery = getRefManifestURL.Query().Encode()
resp, err := sig.client.R().
SetHeader("Content-Type", ref.MediaType).
Get(getRefManifestURL.String())
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get oci reference manifest: %s", getRefManifestURL.String())
return err
}
if ref.MediaType == ispec.MediaTypeImageManifest {
// read manifest
var manifest ispec.Manifest
err = json.Unmarshal(resp.Body(), &manifest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't unmarshal oci reference manifest: %s", getRefManifestURL.String())
return err
}
for _, layer := range manifest.Layers {
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, layer.Digest); err != nil {
return err
}
}
// sync config blob
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, manifest.Config.Digest); err != nil {
return err
}
} else if ref.MediaType == ispec.MediaTypeArtifactManifest {
// read manifest
var manifest ispec.Artifact
err = json.Unmarshal(resp.Body(), &manifest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't unmarshal oci reference manifest: %s", getRefManifestURL.String())
return err
}
for _, layer := range manifest.Blobs {
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, layer.Digest); err != nil {
return err
}
}
}
_, err = imageStore.PutImageManifest(localRepo, ref.Digest.String(),
ref.MediaType, resp.Body())
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload oci reference manifest")
return err
}
}
sig.log.Info().Msgf("successfully synced oci references for repo %s digest %s", localRepo, digestStr)
return nil
}
func (sig *signaturesCopier) canSkipNotarySignature(localRepo, digestStr string, refs ReferenceList,
) (bool, error) {
imageStore := sig.storeController.GetImageStore(localRepo)
@ -407,6 +466,72 @@ func (sig *signaturesCopier) canSkipCosignSignature(localRepo, digestStr string,
return true, nil
}
func (sig *signaturesCopier) canSkipOCIRefs(localRepo, digestStr string, index ispec.Index,
) (bool, error) {
imageStore := sig.storeController.GetImageStore(localRepo)
digest := godigest.Digest(digestStr)
// check oci references already synced
if len(index.Manifests) > 0 {
localRefs, err := imageStore.GetReferrers(localRepo, digest, "")
if err != nil {
if errors.Is(err, zerr.ErrManifestNotFound) {
return false, nil
}
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get local ocireferences for %s:%s manifest", localRepo, digestStr)
return false, err
}
if !descriptorsEqual(localRefs.Manifests, index.Manifests) {
sig.log.Info().Msgf("upstream oci references for %s:%s changed, syncing again", localRepo, digestStr)
return false, nil
}
}
sig.log.Info().Msgf("skipping oci references %s:%s, already synced", localRepo, digestStr)
return true, nil
}
func syncBlob(sig *signaturesCopier, imageStore storage.ImageStore, remoteRepo, localRepo string,
digest godigest.Digest,
) error {
getBlobURL := sig.upstreamURL
getBlobURL.Path = path.Join(getBlobURL.Path, "v2", remoteRepo, "blobs", digest.String())
getBlobURL.RawQuery = getBlobURL.Query().Encode()
resp, err := sig.client.R().SetDoNotParseResponse(true).Get(getBlobURL.String())
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).Str("url", getBlobURL.String()).
Err(err).Msgf("couldn't get blob: %s", getBlobURL.String())
return err
}
defer resp.RawBody().Close()
if resp.IsError() {
sig.log.Info().Str("url", getBlobURL.String()).Msgf("couldn't find blob from %s, status code: %d",
getBlobURL.String(), resp.StatusCode())
return zerr.ErrSyncReferrer
}
_, _, err = imageStore.FullBlobUpload(localRepo, resp.RawBody(), digest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).Str("digest", digest.String()).
Err(err).Msg("couldn't upload blob")
return err
}
return nil
}
// sync feature will try to pull cosign signature because for sync cosign signature is just an image
// this function will check if tag is a cosign tag.
func isCosignTag(tag string) bool {

18
pkg/extensions/sync/sync.go
View file

@ -309,14 +309,14 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
// get upstream signatures
cosignManifest, err := sig.getCosignManifest(upstreamRepo, upstreamImageDigest.String())
if err != nil && !errors.Is(err, zerr.ErrSyncSignatureNotFound) {
if err != nil && !errors.Is(err, zerr.ErrSyncReferrerNotFound) {
log.Error().Err(err).Msgf("couldn't get upstream image %s cosign manifest", upstreamImageRef.DockerReference())
return err
}
refs, err := sig.getNotaryRefs(upstreamRepo, upstreamImageDigest.String())
if err != nil && !errors.Is(err, zerr.ErrSyncSignatureNotFound) {
refs, err := sig.getNotarySignatures(upstreamRepo, upstreamImageDigest.String())
if err != nil && !errors.Is(err, zerr.ErrSyncReferrerNotFound) {
log.Error().Err(err).Msgf("couldn't get upstream image %s notary references", upstreamImageRef.DockerReference())
return err
@ -382,6 +382,16 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
// sync signatures
if err = retry.RetryIfNecessary(ctx, func() error {
index, err := sig.getOCIRefs(upstreamRepo, upstreamImageDigest.String())
if err != nil && !errors.Is(err, zerr.ErrSyncReferrerNotFound) {
return err
}
err = sig.syncOCIRefs(localRepo, upstreamRepo, upstreamImageDigest.String(), index)
if err != nil {
return err
}
err = sig.syncNotarySignature(localRepo, upstreamRepo, upstreamImageDigest.String(), refs)
if err != nil {
return err
@ -395,7 +405,7 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
return nil
}, retryOptions); err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't copy notary signature for %s", upstreamImageRef.DockerReference())
Err(err).Msgf("couldn't copy referrer for %s", upstreamImageRef.DockerReference())
}
}
}

11
pkg/extensions/sync/sync_internal_test.go
View file

@ -385,6 +385,17 @@ func TestSyncInternal(t *testing.T) {
So(err, ShouldNotBeNil)
So(canBeSkipped, ShouldBeFalse)
err = sig.syncOCIRefs(testImage, testImage, testImageManifestDigest.String(),
ispec.Index{Manifests: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageManifest,
},
}})
So(err, ShouldNotBeNil)
err = syncSignaturesArtifacts(sig, testImage, testImage, testImageManifestDigest.String(), OCIReference)
So(err, ShouldNotBeNil)
cosignManifest := ispec.Manifest{
Layers: []ispec.Descriptor{{Digest: "fakeDigest"}},
}

289
pkg/extensions/sync/sync_test.go
View file

@ -857,7 +857,6 @@ func TestConfigReloader(t *testing.T) {
So(string(data), ShouldContainSubstring, "reloaded params")
So(string(data), ShouldContainSubstring, "new configuration settings")
So(string(data), ShouldContainSubstring, "\"Sync\":null")
So(string(data), ShouldNotContainSubstring, "sync:")
})
}
@ -2446,9 +2445,11 @@ func TestPeriodicallySignaturesErr(t *testing.T) {
time.Sleep(2 * time.Second)
// should not be synced nor sync on demand
resp, err = resty.R().Get(destBaseURL + notaryURLPath)
resp, err = resty.R().SetHeader("Content-Type", "application/json").
SetQueryParam("artifactType", "application/vnd.cncf.notary.v2.signature").
Get(destBaseURL + notaryURLPath)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 400)
So(resp.StatusCode(), ShouldEqual, 404)
})
Convey("Trigger error on artifact references", func() {
@ -2475,27 +2476,67 @@ func TestPeriodicallySignaturesErr(t *testing.T) {
err = json.Unmarshal(resp.Body(), &referrers)
So(err, ShouldBeNil)
// read manifest
for _, ref := range referrers.Manifests {
refPath := path.Join(srcDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Encoded())
_, err = os.ReadFile(refPath)
Convey("of type OCI image", func() {
// read manifest
var artifactManifest ispec.Manifest
for _, ref := range referrers.Manifests {
refPath := path.Join(srcDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Encoded())
body, err := os.ReadFile(refPath)
So(err, ShouldBeNil)
err = json.Unmarshal(body, &artifactManifest)
So(err, ShouldBeNil)
// triggers perm denied on artifact blobs
for _, blob := range artifactManifest.Layers {
blobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Encoded())
err := os.Chmod(blobPath, 0o000)
So(err, ShouldBeNil)
}
}
// start downstream server
dctlr, destBaseURL, _, _ := startDownstreamServer(t, false, syncConfig)
defer dctlr.Shutdown()
time.Sleep(2 * time.Second)
// should not be synced nor sync on demand
resp, err = resty.R().Get(destBaseURL + artifactURLPath)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
})
// triggers perm denied on artifact blobs
err = os.Chmod(refPath, 0o000)
Convey("of type OCI artifact", func() {
// read manifest
var artifactManifest ispec.Artifact
for _, ref := range referrers.Manifests {
refPath := path.Join(srcDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Encoded())
body, err := os.ReadFile(refPath)
So(err, ShouldBeNil)
err = json.Unmarshal(body, &artifactManifest)
So(err, ShouldBeNil)
// triggers perm denied on artifact blobs
for _, blob := range artifactManifest.Blobs {
blobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Encoded())
err := os.Chmod(blobPath, 0o000)
So(err, ShouldBeNil)
}
}
// start downstream server
dctlr, destBaseURL, _, _ := startDownstreamServer(t, false, syncConfig)
defer dctlr.Shutdown()
time.Sleep(2 * time.Second)
// should not be synced nor sync on demand
resp, err = resty.R().Get(destBaseURL + artifactURLPath)
So(err, ShouldBeNil)
}
// start downstream server
dctlr, destBaseURL, _, _ := startDownstreamServer(t, false, syncConfig)
defer dctlr.Shutdown()
time.Sleep(2 * time.Second)
// should not be synced nor sync on demand
resp, err = resty.R().Get(destBaseURL + artifactURLPath)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
So(resp.StatusCode(), ShouldEqual, 404)
})
})
})
}
@ -2614,13 +2655,28 @@ func TestSignatures(t *testing.T) {
err = vrfy.Exec(context.TODO(), []string{fmt.Sprintf("localhost:%s/%s:%s", destPort, repoName, "1.0")})
So(err, ShouldBeNil)
// get oci references from downstream, should be synced
getOCIReferrersURL := srcBaseURL + path.Join("/v2", repoName, "referrers", digest.String())
resp, err := resty.R().Get(getOCIReferrersURL)
So(err, ShouldBeNil)
So(resp, ShouldNotBeEmpty)
var index ispec.Index
err = json.Unmarshal(resp.Body(), &index)
So(err, ShouldBeNil)
So(len(index.Manifests), ShouldEqual, 2)
// test negative cases (trigger errors)
// test notary signatures errors
// based on manifest digest get referrers
getReferrersURL := srcBaseURL + path.Join("/oras/artifacts/v1/", repoName, "manifests", digest.String(), "referrers")
resp, err := resty.R().
resp, err = resty.R().
SetHeader("Content-Type", "application/json").
SetQueryParam("artifactType", "application/vnd.cncf.notary.v2.signature").
Get(getReferrersURL)
@ -2821,6 +2877,92 @@ func TestSignatures(t *testing.T) {
resp, err = resty.R().Get(destBaseURL + "/v2/" + repoName + "/manifests/1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
err = os.Chmod(destManifestPath, 0o755)
So(err, ShouldBeNil)
getOCIReferrersURL = srcBaseURL + path.Join("/v2", repoName, "referrers", digest.String())
resp, err = resty.R().Get(getOCIReferrersURL)
So(err, ShouldBeNil)
So(resp, ShouldNotBeEmpty)
err = json.Unmarshal(resp.Body(), &index)
So(err, ShouldBeNil)
// remove already synced image
err = os.RemoveAll(path.Join(destDir, repoName))
So(err, ShouldBeNil)
var refManifest ispec.Manifest
for _, ref := range index.Manifests {
refPath := path.Join(srcDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Encoded())
body, err := os.ReadFile(refPath)
So(err, ShouldBeNil)
err = json.Unmarshal(body, &refManifest)
So(err, ShouldBeNil)
// triggers perm denied on notary sig blobs on downstream
for _, blob := range refManifest.Layers {
blobPath := path.Join(destDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Encoded())
err := os.MkdirAll(blobPath, 0o755)
So(err, ShouldBeNil)
err = os.Chmod(blobPath, 0o000)
So(err, ShouldBeNil)
}
}
// sync on demand
resp, err = resty.R().Get(destBaseURL + "/v2/" + repoName + "/manifests/1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
// cleanup
for _, blob := range refManifest.Layers {
blobPath := path.Join(destDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Encoded())
err = os.Chmod(blobPath, 0o755)
So(err, ShouldBeNil)
}
// remove already synced image
err = os.RemoveAll(path.Join(destDir, repoName))
So(err, ShouldBeNil)
// trigger error on reference config blob
referenceConfigBlobPath := path.Join(destDir, repoName, "blobs",
string(refManifest.Config.Digest.Algorithm()), refManifest.Config.Digest.Encoded())
err = os.MkdirAll(referenceConfigBlobPath, 0o755)
So(err, ShouldBeNil)
err = os.Chmod(referenceConfigBlobPath, 0o000)
So(err, ShouldBeNil)
// sync on demand
resp, err = resty.R().Get(destBaseURL + "/v2/" + repoName + "/manifests/1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
err = os.Chmod(referenceConfigBlobPath, 0o755)
So(err, ShouldBeNil)
// remove already synced image
err = os.RemoveAll(path.Join(destDir, repoName))
So(err, ShouldBeNil)
// trigger error on pushing oci reference manifest
for _, ref := range index.Manifests {
refPath := path.Join(destDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Encoded())
err = os.MkdirAll(refPath, 0o755)
So(err, ShouldBeNil)
err = os.Chmod(refPath, 0o000)
So(err, ShouldBeNil)
}
// sync on demand
resp, err = resty.R().Get(destBaseURL + "/v2/" + repoName + "/manifests/1.0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
})
}
@ -3359,12 +3501,15 @@ func TestSignaturesOnDemand(t *testing.T) {
err = vrfy.Exec(context.TODO(), []string{fmt.Sprintf("localhost:%s/%s:%s", destPort, repoName, "1.0")})
So(err, ShouldBeNil)
//
// test negative case
cosignEncodedDigest := strings.Replace(digest.String(), ":", "-", 1) + ".sig"
getCosignManifestURL := srcBaseURL + path.Join(constants.RoutePrefix, repoName, "manifests", cosignEncodedDigest)
mResp, err := resty.R().Get(getCosignManifestURL)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
var imageManifest ispec.Manifest
@ -3467,6 +3612,8 @@ func TestOnlySignaturesOnDemand(t *testing.T) {
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
imageManifestDigest := godigest.FromBytes(resp.Body())
splittedURL = strings.SplitAfter(destBaseURL, ":")
destPort := splittedURL[len(splittedURL)-1]
@ -3520,6 +3667,17 @@ func TestOnlySignaturesOnDemand(t *testing.T) {
err = vrfy.Exec(context.TODO(), []string{fmt.Sprintf("localhost:%s/%s:%s", destPort, repoName, "1.0")})
So(err, ShouldBeNil)
// trigger syncing OCI references on demand
artifactURLPath := path.Join("/v2", repoName, "referrers", imageManifestDigest.String())
// based on image manifest digest get referrers
resp, err = resty.R().
SetHeader("Content-Type", "application/json").
SetQueryParam("artifactType", "application/vnd.cncf.icecream").
Get(srcBaseURL + artifactURLPath)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
So(err, ShouldBeNil)
})
}
@ -4452,18 +4610,43 @@ func pushRepo(url, repoName string) godigest.Digest {
panic(err)
}
// create artifact blob
abuf := []byte("this is an artifact")
adigest := pushBlob(url, repoName, abuf)
// create artifact config blob
acbuf := []byte("{}")
acdigest := pushBlob(url, repoName, acbuf)
// push a referrer artifact
manifest = ispec.Manifest{
Config: ispec.Descriptor{
MediaType: "application/vnd.cncf.icecream",
Digest: cdigest,
Size: int64(len(cblob)),
Digest: acdigest,
Size: int64(len(acbuf)),
},
Layers: []ispec.Descriptor{
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: digest,
Size: int64(len(content)),
MediaType: "application/octet-stream",
Digest: adigest,
Size: int64(len(abuf)),
},
},
Subject: &ispec.Descriptor{
MediaType: "application/vnd.oci.image.manifest.v1+json",
Digest: digest,
Size: int64(len(content)),
},
}
artifactManifest := ispec.Artifact{
MediaType: ispec.MediaTypeArtifactManifest,
ArtifactType: "application/vnd.cncf.icecream",
Blobs: []ispec.Descriptor{
{
MediaType: "application/octet-stream",
Digest: adigest,
Size: int64(len(abuf)),
},
},
Subject: &ispec.Descriptor{
@ -4480,9 +4663,24 @@ func pushRepo(url, repoName string) godigest.Digest {
panic(err)
}
adigest := godigest.FromBytes(content)
adigest = godigest.FromBytes(content)
_, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
// put OCI reference image mediaType artifact
_, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest).
SetBody(content).Put(url + fmt.Sprintf("/v2/%s/manifests/%s", repoName, adigest.String()))
if err != nil {
panic(err)
}
content, err = json.Marshal(artifactManifest)
if err != nil {
panic(err)
}
adigest = godigest.FromBytes(content)
// put OCI reference artifact mediaType artifact
_, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeArtifactManifest).
SetBody(content).Put(url + fmt.Sprintf("/v2/%s/manifests/%s", repoName, adigest.String()))
if err != nil {
panic(err)
@ -4503,3 +4701,36 @@ func waitSync(rootDir, repoName string) {
time.Sleep(500 * time.Millisecond)
}
}
func pushBlob(url string, repoName string, buf []byte) godigest.Digest {
resp, err := resty.R().
Post(fmt.Sprintf("%s/v2/%s/blobs/uploads/", url, repoName))
if err != nil {
panic(err)
}
if resp.StatusCode() != http.StatusAccepted {
panic(perr.Wrapf(errBadStatus, "invalid status code: %d", resp.StatusCode()))
}
loc := test.Location(url, resp)
digest := godigest.FromBytes(buf)
resp, err = resty.R().
SetContentLength(true).
SetHeader("Content-Length", fmt.Sprintf("%d", len(buf))).
SetHeader("Content-Type", "application/octet-stream").
SetQueryParam("digest", digest.String()).
SetBody(buf).
Put(loc)
if err != nil {
panic(err)
}
if resp.StatusCode() != http.StatusCreated {
panic(perr.Wrapf(errBadStatus, "invalid status code: %d", resp.StatusCode()))
}
return digest
}

65
pkg/extensions/sync/utils.go
View file

@ -348,7 +348,8 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
}
// is image manifest
if mediaType == ispec.MediaTypeImageManifest {
switch mediaType {
case ispec.MediaTypeImageManifest:
if err := copyManifest(localRepo, manifestContent, reference, cacheImageStore, imageStore, log); err != nil {
if errors.Is(err, zerr.ErrImageLintAnnotations) {
log.Error().Str("errorType", TypeOf(err)).
@ -359,52 +360,50 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
return err
}
case ispec.MediaTypeImageIndex:
// is image index
var indexManifest ispec.Index
return nil
}
// is image index
var indexManifest ispec.Index
if err := json.Unmarshal(manifestContent, &indexManifest); err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("dir", path.Join(cacheImageStore.RootDir(), localRepo)).
Msg("invalid JSON")
return err
}
for _, manifest := range indexManifest.Manifests {
manifestBuf, err := cacheImageStore.GetBlobContent(localRepo, manifest.Digest)
if err != nil {
if err := json.Unmarshal(manifestContent, &indexManifest); err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("dir", path.Join(cacheImageStore.RootDir(), localRepo)).Str("digest", manifest.Digest.String()).
Msg("couldn't find manifest which is part of an image index")
Err(err).Str("dir", path.Join(cacheImageStore.RootDir(), localRepo)).
Msg("invalid JSON")
return err
}
if err := copyManifest(localRepo, manifestBuf, manifest.Digest.String(),
cacheImageStore, imageStore, log); err != nil {
if errors.Is(err, zerr.ErrImageLintAnnotations) {
for _, manifest := range indexManifest.Manifests {
manifestBuf, err := cacheImageStore.GetBlobContent(localRepo, manifest.Digest)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload manifest because of missing annotations")
Err(err).Str("dir", path.Join(cacheImageStore.RootDir(), localRepo)).Str("digest", manifest.Digest.String()).
Msg("couldn't find manifest which is part of an image index")
return nil
return err
}
if err := copyManifest(localRepo, manifestBuf, manifest.Digest.String(),
cacheImageStore, imageStore, log); err != nil {
if errors.Is(err, zerr.ErrImageLintAnnotations) {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload manifest because of missing annotations")
return nil
}
return err
}
}
_, err = imageStore.PutImageManifest(localRepo, reference, mediaType, manifestContent)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload manifest")
return err
}
}
_, err = imageStore.PutImageManifest(localRepo, reference, mediaType, manifestContent)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload manifest")
return err
}
return nil
}

203
pkg/storage/common.go
View file

@ -2,11 +2,16 @@ package storage
import (
"encoding/json"
"errors"
"os"
"path"
"strings"
"time"
"github.com/docker/distribution/registry/storage/driver"
"github.com/notaryproject/notation-go"
godigest "github.com/opencontainers/go-digest"
imeta "github.com/opencontainers/image-spec/specs-go"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
oras "github.com/oras-project/artifacts-spec/specs-go/v1"
"github.com/rs/zerolog"
@ -446,6 +451,204 @@ func ApplyLinter(imgStore ImageStore, linter Lint, repo string, manifestDesc isp
return pass, nil
}
func GetOrasReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, artifactType string,
log zerolog.Logger,
) ([]oras.Descriptor, error) {
var lockLatency time.Time
if err := gdigest.Validate(); err != nil {
return nil, err
}
dir := path.Join(imgStore.RootDir(), repo)
if !imgStore.DirExists(dir) {
return nil, zerr.ErrRepoNotFound
}
index, err := GetIndex(imgStore, repo, log)
if err != nil {
return nil, err
}
imgStore.RLock(&lockLatency)
defer imgStore.RUnlock(&lockLatency)
found := false
result := []oras.Descriptor{}
for _, manifest := range index.Manifests {
if manifest.MediaType != oras.MediaTypeArtifactManifest {
continue
}
imgStore.RUnlock(&lockLatency)
buf, err := imgStore.GetBlobContent(repo, manifest.Digest)
imgStore.RLock(&lockLatency)
if err != nil {
log.Error().Err(err).Str("blob", imgStore.BlobPath(repo, manifest.Digest)).Msg("failed to read manifest")
if os.IsNotExist(err) || errors.Is(err, driver.PathNotFoundError{}) {
return nil, zerr.ErrManifestNotFound
}
return nil, err
}
var artManifest oras.Manifest
if err := json.Unmarshal(buf, &artManifest); err != nil {
log.Error().Err(err).Str("dir", dir).Msg("invalid JSON")
return nil, err
}
if artManifest.Subject.Digest != gdigest {
continue
}
// filter by artifact type
if artifactType != "" && artManifest.ArtifactType != artifactType {
continue
}
result = append(result, oras.Descriptor{
MediaType: manifest.MediaType,
ArtifactType: artManifest.ArtifactType,
Digest: manifest.Digest,
Size: manifest.Size,
Annotations: manifest.Annotations,
})
found = true
}
if !found {
return nil, zerr.ErrManifestNotFound
}
return result, nil
}
func GetReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, artifactType string,
log zerolog.Logger,
) (ispec.Index, error) {
var lockLatency time.Time
nilIndex := ispec.Index{}
if err := gdigest.Validate(); err != nil {
return nilIndex, err
}
dir := path.Join(imgStore.RootDir(), repo)
if !imgStore.DirExists(dir) {
return nilIndex, zerr.ErrRepoNotFound
}
index, err := GetIndex(imgStore, repo, log)
if err != nil {
return nilIndex, err
}
imgStore.RLock(&lockLatency)
defer imgStore.RUnlock(&lockLatency)
found := false
result := []ispec.Descriptor{}
for _, manifest := range index.Manifests {
if manifest.Digest == gdigest {
continue
}
imgStore.RUnlock(&lockLatency)
buf, err := imgStore.GetBlobContent(repo, manifest.Digest)
imgStore.RLock(&lockLatency)
if err != nil {
log.Error().Err(err).Str("blob", imgStore.BlobPath(repo, manifest.Digest)).Msg("failed to read manifest")
if os.IsNotExist(err) || errors.Is(err, driver.PathNotFoundError{}) {
return nilIndex, zerr.ErrManifestNotFound
}
return nilIndex, err
}
if manifest.MediaType == ispec.MediaTypeImageManifest {
var mfst ispec.Manifest
if err := json.Unmarshal(buf, &mfst); err != nil {
log.Error().Err(err).Str("manifest digest", manifest.Digest.String()).Msg("invalid JSON")
return nilIndex, err
}
if mfst.Subject == nil || mfst.Subject.Digest != gdigest {
continue
}
// filter by artifact type
if artifactType != "" && mfst.Config.MediaType != artifactType {
continue
}
result = append(result, ispec.Descriptor{
MediaType: manifest.MediaType,
ArtifactType: mfst.Config.MediaType,
Size: manifest.Size,
Digest: manifest.Digest,
Annotations: mfst.Annotations,
})
} else if manifest.MediaType == ispec.MediaTypeArtifactManifest {
var art ispec.Artifact
if err := json.Unmarshal(buf, &art); err != nil {
log.Error().Err(err).Str("manifest digest", manifest.Digest.String()).Msg("invalid JSON")
return nilIndex, err
}
if art.Subject == nil || art.Subject.Digest != gdigest {
continue
}
// filter by artifact type
if artifactType != "" && art.ArtifactType != artifactType {
continue
}
result = append(result, ispec.Descriptor{
MediaType: manifest.MediaType,
ArtifactType: art.ArtifactType,
Size: manifest.Size,
Digest: manifest.Digest,
Annotations: art.Annotations,
})
}
found = true
}
if !found {
return nilIndex, zerr.ErrManifestNotFound
}
index = ispec.Index{
Versioned: imeta.Versioned{SchemaVersion: storageConstants.SchemaVersion},
MediaType: ispec.MediaTypeImageIndex,
Manifests: result,
Annotations: map[string]string{},
}
// response was filtered by artifactType
if artifactType != "" {
index.Annotations[storageConstants.ReferrerFilterAnnotation] = ""
}
return index, nil
}
func IsSupportedMediaType(mediaType string) bool {
return mediaType == ispec.MediaTypeImageIndex ||
mediaType == ispec.MediaTypeImageManifest ||

201
pkg/storage/common_test.go
View file

@ -4,19 +4,24 @@ import (
"bytes"
"encoding/json"
"os"
"path"
"testing"
"github.com/docker/distribution/registry/storage/driver"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"github.com/rs/zerolog"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
)
func TestValidateManifest(t *testing.T) {
@ -102,3 +107,199 @@ func TestValidateManifest(t *testing.T) {
})
})
}
func TestGetReferrersErrors(t *testing.T) {
Convey("make storage", t, func(c C) {
dir := t.TempDir()
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: dir,
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, storage.DefaultGCDelay, false,
true, log, metrics, nil, cacheDriver)
artifactType := "application/vnd.example.icecream.v1"
validDigest := godigest.FromBytes([]byte("blob"))
Convey("Trigger invalid digest error", func(c C) {
_, err := storage.GetReferrers(imgStore, "zot-test", "invalidDigest", artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
_, err = storage.GetOrasReferrers(imgStore, "zot-test", "invalidDigest", artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
Convey("Trigger repo not found error", func(c C) {
_, err := storage.GetReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
_, err = storage.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
err := test.CopyFiles("../../test/data/zot-test", path.Join(dir, "zot-test"))
So(err, ShouldBeNil)
digest := godigest.FromBytes([]byte("{}"))
index := ispec.Index{
Manifests: []ispec.Descriptor{
{
MediaType: artifactspec.MediaTypeArtifactManifest,
Digest: digest,
},
},
}
indexBuf, err := json.Marshal(index)
So(err, ShouldBeNil)
Convey("Trigger GetBlobContent() not found", func(c C) {
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return indexBuf, nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte{}, driver.PathNotFoundError{}
},
}
_, err = storage.GetReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
_, err = storage.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
Convey("Trigger GetBlobContent() generic error", func(c C) {
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return indexBuf, nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte{}, errors.ErrBadBlob
},
}
_, err = storage.GetReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
Convey("Trigger continue on different artifactType", func(c C) {
orasManifest := artifactspec.Manifest{
Subject: &artifactspec.Descriptor{
Digest: digest,
ArtifactType: "unknown",
},
}
orasBuf, err := json.Marshal(orasManifest)
So(err, ShouldBeNil)
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return indexBuf, nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return orasBuf, nil
},
}
_, err = storage.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
_, err = storage.GetOrasReferrers(imgStore, "zot-test", digest, artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
Convey("Trigger unmarshal error on manifest image mediaType", func(c C) {
index = ispec.Index{
Manifests: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageManifest,
Digest: digest,
},
},
}
indexBuf, err = json.Marshal(index)
So(err, ShouldBeNil)
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return indexBuf, nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte{}, nil
},
}
_, err = storage.GetReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
Convey("Trigger unmarshal error on artifact mediaType", func(c C) {
index = ispec.Index{
Manifests: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeArtifactManifest,
Digest: digest,
},
},
}
indexBuf, err = json.Marshal(index)
So(err, ShouldBeNil)
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return indexBuf, nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte{}, nil
},
}
_, err = storage.GetReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
Convey("Trigger nil subject", func(c C) {
index = ispec.Index{
Manifests: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeArtifactManifest,
Digest: digest,
},
},
}
indexBuf, err = json.Marshal(index)
So(err, ShouldBeNil)
ociManifest := ispec.Manifest{
Subject: nil,
}
ociManifestBuf, err := json.Marshal(ociManifest)
So(err, ShouldBeNil)
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return indexBuf, nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return ociManifestBuf, nil
},
}
_, err = storage.GetReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger())
So(err, ShouldNotBeNil)
})
})
}

184
pkg/storage/local/local.go
View file

@ -1341,192 +1341,12 @@ func (is *ImageStoreLocal) DeleteBlob(repo string, digest godigest.Digest) error
func (is *ImageStoreLocal) GetReferrers(repo string, gdigest godigest.Digest, artifactType string,
) (ispec.Index, error) {
var lockLatency time.Time
nilIndex := ispec.Index{}
if err := gdigest.Validate(); err != nil {
return nilIndex, err
}
dir := path.Join(is.rootDir, repo)
if !is.DirExists(dir) {
return nilIndex, zerr.ErrRepoNotFound
}
index, err := storage.GetIndex(is, repo, is.log)
if err != nil {
return nilIndex, err
}
is.RLock(&lockLatency)
defer is.RUnlock(&lockLatency)
found := false
result := []ispec.Descriptor{}
for _, manifest := range index.Manifests {
if manifest.Digest == gdigest {
continue
}
p := path.Join(dir, "blobs", manifest.Digest.Algorithm().String(), manifest.Digest.Encoded())
buf, err := os.ReadFile(p)
if err != nil {
is.log.Error().Err(err).Str("blob", p).Msg("failed to read manifest")
if os.IsNotExist(err) {
return nilIndex, zerr.ErrManifestNotFound
}
return nilIndex, err
}
if manifest.MediaType == ispec.MediaTypeImageManifest {
var mfst ispec.Manifest
if err := json.Unmarshal(buf, &mfst); err != nil {
return nilIndex, err
}
if mfst.Subject == nil || mfst.Subject.Digest != gdigest {
continue
}
// filter by artifact type
if artifactType != "" && mfst.Config.MediaType != artifactType {
continue
}
result = append(result, ispec.Descriptor{
MediaType: manifest.MediaType,
ArtifactType: mfst.Config.MediaType,
Size: manifest.Size,
Digest: manifest.Digest,
Annotations: mfst.Annotations,
})
} else if manifest.MediaType == ispec.MediaTypeArtifactManifest {
var art ispec.Artifact
if err := json.Unmarshal(buf, &art); err != nil {
return nilIndex, err
}
if art.Subject == nil || art.Subject.Digest != gdigest {
continue
}
// filter by artifact type
if artifactType != "" && art.ArtifactType != artifactType {
continue
}
result = append(result, ispec.Descriptor{
MediaType: manifest.MediaType,
ArtifactType: art.ArtifactType,
Size: manifest.Size,
Digest: manifest.Digest,
Annotations: art.Annotations,
})
}
found = true
}
if !found {
return nilIndex, zerr.ErrManifestNotFound
}
index = ispec.Index{
Versioned: imeta.Versioned{SchemaVersion: defaultSchemaVersion},
MediaType: ispec.MediaTypeImageIndex,
Manifests: result,
Annotations: map[string]string{},
}
// response was filtered by artifactType
if artifactType != "" {
index.Annotations[storageConstants.ReferrerFilterAnnotation] = artifactType
}
return index, nil
return storage.GetReferrers(is, repo, gdigest, artifactType, is.log)
}
func (is *ImageStoreLocal) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string,
) ([]oras.Descriptor, error) {
var lockLatency time.Time
if err := gdigest.Validate(); err != nil {
return nil, err
}
dir := path.Join(is.rootDir, repo)
if !is.DirExists(dir) {
return nil, zerr.ErrRepoNotFound
}
index, err := storage.GetIndex(is, repo, is.log)
if err != nil {
return nil, err
}
is.RLock(&lockLatency)
defer is.RUnlock(&lockLatency)
found := false
result := []oras.Descriptor{}
for _, manifest := range index.Manifests {
if manifest.MediaType != oras.MediaTypeArtifactManifest {
continue
}
p := path.Join(dir, "blobs", manifest.Digest.Algorithm().String(), manifest.Digest.Encoded())
buf, err := os.ReadFile(p)
if err != nil {
is.log.Error().Err(err).Str("blob", p).Msg("failed to read manifest")
if os.IsNotExist(err) {
return nil, zerr.ErrManifestNotFound
}
return nil, err
}
var artManifest oras.Manifest
if err := json.Unmarshal(buf, &artManifest); err != nil {
is.log.Error().Err(err).Str("dir", dir).Msg("invalid JSON")
return nil, err
}
if artManifest.Subject.Digest != gdigest {
continue
}
// filter by artifact type
if artifactType != "" && artManifest.ArtifactType != artifactType {
continue
}
result = append(result, oras.Descriptor{
MediaType: manifest.MediaType,
ArtifactType: artManifest.ArtifactType,
Digest: manifest.Digest,
Size: manifest.Size,
Annotations: manifest.Annotations,
})
found = true
}
if !found {
return nil, zerr.ErrManifestNotFound
}
return result, nil
return storage.GetOrasReferrers(is, repo, gdigest, artifactType, is.log)
}
func (is *ImageStoreLocal) writeFile(filename string, data []byte) error {

8
pkg/storage/s3/s3.go
View file

@ -1224,14 +1224,14 @@ func (is *ObjectStorage) GetBlobContent(repo string, digest godigest.Digest) ([]
return buf.Bytes(), nil
}
func (is *ObjectStorage) GetReferrers(repo string, digest godigest.Digest, artifactType string,
func (is *ObjectStorage) GetReferrers(repo string, gdigest godigest.Digest, artifactType string,
) (ispec.Index, error) {
return ispec.Index{}, zerr.ErrMethodNotSupported
return storage.GetReferrers(is, repo, gdigest, artifactType, is.log)
}
func (is *ObjectStorage) GetOrasReferrers(repo string, digest godigest.Digest, artifactType string,
func (is *ObjectStorage) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string,
) ([]artifactspec.Descriptor, error) {
return nil, zerr.ErrMethodNotSupported
return storage.GetOrasReferrers(is, repo, gdigest, artifactType, is.log)
}
func (is *ObjectStorage) GetIndexContent(repo string) ([]byte, error) {

219
pkg/storage/s3/s3_test.go
View file

@ -20,6 +20,7 @@ import (
guuid "github.com/gofrs/uuid"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"github.com/rs/zerolog"
. "github.com/smartystreets/goconvey/convey"
"gopkg.in/resty.v1"
@ -64,7 +65,7 @@ func createMockStorage(rootDir string, cacheDir string, dedupe bool, store drive
var cacheDriver cache.Cache
// from pkg/cli/root.go/applyDefaultValues, s3 magic
if _, err := os.Stat(cacheDir); dedupe || (!dedupe && err == nil) {
if _, err := os.Stat(path.Join(cacheDir, "s3_cache.db")); dedupe || (!dedupe && err == nil) {
cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: cacheDir,
Name: "s3_cache",
@ -116,7 +117,7 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) (
var cacheDriver cache.Cache
// from pkg/cli/root.go/applyDefaultValues, s3 magic
if _, err := os.Stat(cacheDir); dedupe || (!dedupe && err == nil) {
if _, err := os.Stat(path.Join(cacheDir, "s3_cache.db")); dedupe || (!dedupe && err == nil) {
cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: cacheDir,
Name: "s3_cache",
@ -378,6 +379,201 @@ func TestStorageDriverStatFunction(t *testing.T) {
})
}
func TestGetOrasAndOCIReferrers(t *testing.T) {
repo := "zot-test"
uuid, err := guuid.NewV4()
if err != nil {
panic(err)
}
tdir := t.TempDir()
testDir := path.Join("/oci-repo-test", uuid.String())
_, imgStore, _ := createObjectsStore(testDir, tdir, true)
Convey("Upload test image", t, func(c C) {
cfg, layers, manifest, err := test.GetImageComponents(100)
So(err, ShouldBeNil)
for _, content := range layers {
upload, err := imgStore.NewBlobUpload(repo)
So(err, ShouldBeNil)
So(upload, ShouldNotBeEmpty)
buf := bytes.NewBuffer(content)
buflen := buf.Len()
digest := godigest.FromBytes(content)
blob, err := imgStore.PutBlobChunkStreamed(repo, upload, buf)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
blobDigest1 := digest
So(blobDigest1, ShouldNotBeEmpty)
err = imgStore.FinishBlobUpload(repo, upload, buf, digest)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
}
// upload config blob
cblob, err := json.Marshal(cfg)
So(err, ShouldBeNil)
buf := bytes.NewBuffer(cblob)
buflen := buf.Len()
digest := godigest.FromBytes(cblob)
_, clen, err := imgStore.FullBlobUpload(repo, buf, digest)
So(err, ShouldBeNil)
So(clen, ShouldEqual, buflen)
// upload manifest
mblob, err := json.Marshal(manifest)
So(err, ShouldBeNil)
mbuf := bytes.NewBuffer(mblob)
mbuflen := mbuf.Len()
mdigest := godigest.FromBytes(mblob)
d, err := imgStore.PutImageManifest(repo, "1.0", ispec.MediaTypeImageManifest, mbuf.Bytes())
So(d, ShouldEqual, mdigest)
So(err, ShouldBeNil)
body := []byte("this is an artifact")
digest = godigest.FromBytes(body)
buf = bytes.NewBuffer(body)
buflen = buf.Len()
_, n, err := imgStore.FullBlobUpload(repo, buf, digest)
So(err, ShouldBeNil)
So(n, ShouldEqual, buflen)
Convey("Get oci referrers - application/vnd.oci.image.manifest.v1+json", func(c C) {
artifactType := "application/vnd.example.icecream.v1"
// push artifact config blob
configBody := []byte("{}")
configDigest := godigest.FromBytes(configBody)
configBuf := bytes.NewBuffer(configBody)
configBufLen := configBuf.Len()
_, n, err := imgStore.FullBlobUpload(repo, configBuf, configDigest)
So(err, ShouldBeNil)
So(n, ShouldEqual, configBufLen)
artifactManifest := ispec.Manifest{
MediaType: ispec.MediaTypeImageManifest,
Config: ispec.Descriptor{
MediaType: artifactType,
Size: int64(configBufLen),
Digest: configDigest,
},
Layers: []ispec.Descriptor{
{
MediaType: "application/octet-stream",
Size: int64(buflen),
Digest: digest,
},
},
Subject: &ispec.Descriptor{
MediaType: ispec.MediaTypeImageManifest,
Size: int64(mbuflen),
Digest: mdigest,
},
}
manBuf, err := json.Marshal(artifactManifest)
So(err, ShouldBeNil)
manBufLen := len(manBuf)
manDigest := godigest.FromBytes(manBuf)
_, err = imgStore.PutImageManifest(repo, manDigest.Encoded(), ispec.MediaTypeImageManifest, manBuf)
So(err, ShouldBeNil)
index, err := imgStore.GetReferrers(repo, mdigest, artifactType)
So(err, ShouldBeNil)
So(index, ShouldNotBeEmpty)
So(index.Manifests[0].ArtifactType, ShouldEqual, artifactType)
So(index.Manifests[0].MediaType, ShouldEqual, ispec.MediaTypeImageManifest)
So(index.Manifests[0].Size, ShouldEqual, manBufLen)
So(index.Manifests[0].Digest, ShouldEqual, manDigest)
})
Convey("Get oci referrers - application/vnd.oci.artifact.manifest.v1+json", func(c C) {
artifactType := "application/vnd.example.icecream.v1"
artifactManifest := ispec.Artifact{
MediaType: ispec.MediaTypeArtifactManifest,
ArtifactType: artifactType,
Blobs: []ispec.Descriptor{
{
MediaType: "application/octet-stream",
Size: int64(buflen),
Digest: digest,
},
},
Subject: &ispec.Descriptor{
MediaType: ispec.MediaTypeImageManifest,
Size: int64(mbuflen),
Digest: mdigest,
},
}
manBuf, err := json.Marshal(artifactManifest)
So(err, ShouldBeNil)
manBufLen := len(manBuf)
manDigest := godigest.FromBytes(manBuf)
_, err = imgStore.PutImageManifest(repo, manDigest.Encoded(), ispec.MediaTypeArtifactManifest, manBuf)
So(err, ShouldBeNil)
index, err := imgStore.GetReferrers(repo, mdigest, artifactType)
So(err, ShouldBeNil)
So(index, ShouldNotBeEmpty)
So(index.Manifests[1].ArtifactType, ShouldEqual, artifactType)
So(index.Manifests[1].MediaType, ShouldEqual, ispec.MediaTypeArtifactManifest)
So(index.Manifests[1].Size, ShouldEqual, manBufLen)
So(index.Manifests[1].Digest, ShouldEqual, manDigest)
})
Convey("Get oras referrers", func(c C) {
artifactManifest := artifactspec.Manifest{}
artifactManifest.ArtifactType = "signature-example"
artifactManifest.Subject = &artifactspec.Descriptor{
MediaType: ispec.MediaTypeImageManifest,
Digest: mdigest,
Size: int64(mbuflen),
}
artifactManifest.Blobs = []artifactspec.Descriptor{
{
Size: int64(buflen),
Digest: digest,
MediaType: "application/octet-stream",
},
}
manBuf, err := json.Marshal(artifactManifest)
So(err, ShouldBeNil)
manBufLen := len(manBuf)
manDigest := godigest.FromBytes(manBuf)
_, err = imgStore.PutImageManifest(repo, manDigest.Encoded(), artifactspec.MediaTypeArtifactManifest, manBuf)
So(err, ShouldBeNil)
descriptors, err := imgStore.GetOrasReferrers(repo, mdigest, "signature-example")
So(err, ShouldBeNil)
So(descriptors, ShouldNotBeEmpty)
So(descriptors[0].ArtifactType, ShouldEqual, "signature-example")
So(descriptors[0].MediaType, ShouldEqual, artifactspec.MediaTypeArtifactManifest)
So(descriptors[0].Size, ShouldEqual, manBufLen)
So(descriptors[0].Digest, ShouldEqual, manDigest)
})
})
}
func TestNegativeCasesObjectsStorage(t *testing.T) {
skipIt(t)
@ -397,11 +593,13 @@ func TestNegativeCasesObjectsStorage(t *testing.T) {
So(imgStore.InitRepo(testImage), ShouldBeNil)
objects, err := storeDriver.List(context.Background(), path.Join(imgStore.RootDir(), testImage))
So(err, ShouldBeNil)
for _, object := range objects {
t.Logf("Removing object: %s", object)
err := storeDriver.Delete(context.Background(), object)
So(err, ShouldBeNil)
}
_, err = imgStore.ValidateRepo(testImage)
So(err, ShouldNotBeNil)
_, err = imgStore.GetRepositories()
@ -467,7 +665,6 @@ func TestNegativeCasesObjectsStorage(t *testing.T) {
Convey("Without dedupe", t, func(c C) {
tdir := t.TempDir()
storeDriver, imgStore, _ := createObjectsStore(testDir, tdir, false)
defer cleanupStorage(storeDriver, testDir)
@ -907,27 +1104,19 @@ func TestNegativeCasesObjectsStorage(t *testing.T) {
})
Convey("Test GetReferrers", func(c C) {
imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
DeleteFn: func(ctx context.Context, path string) error {
return errS3
},
})
imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{})
d := godigest.FromBytes([]byte(""))
_, err := imgStore.GetReferrers(testImage, d, "application/image")
So(err, ShouldNotBeNil)
So(err, ShouldEqual, zerr.ErrMethodNotSupported)
So(err, ShouldEqual, zerr.ErrRepoBadVersion)
})
Convey("Test GetOrasReferrers", func(c C) {
imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
DeleteFn: func(ctx context.Context, path string) error {
return errS3
},
})
imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{})
d := godigest.FromBytes([]byte(""))
_, err := imgStore.GetOrasReferrers(testImage, d, "application/image")
So(err, ShouldNotBeNil)
So(err, ShouldEqual, zerr.ErrMethodNotSupported)
So(err, ShouldEqual, zerr.ErrRepoBadVersion)
})
})
}