0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-23 22:27:35 -05:00
zot/pkg/storage/common/common_test.go
peusebiu b80deb9927
refactor(storage): refactor storage into a single ImageStore (#1656)
unified both local and s3 ImageStore logic into a single ImageStore
added a new driver interface for common file/dirs manipulations
to be implemented by different storage types

refactor(gc): drop umoci dependency, implemented internal gc

added retentionDelay config option that specifies
the garbage collect delay for images without tags

this will also clean manifests which are part of an index image
(multiarch) that no longer exist.

fix(dedupe): skip blobs under .sync/ directory

if startup dedupe is running while also syncing is running
ignore blobs under sync's temporary storage

fix(storage): do not allow image indexes modifications

when deleting a manifest verify that it is not part of a multiarch image
and throw a MethodNotAllowed error to the client if it is.
we don't want to modify multiarch images

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
2023-09-01 10:54:39 -07:00

624 lines
18 KiB
Go

package storage_test
import (
"bytes"
"encoding/json"
"errors"
"os"
"path"
"testing"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"github.com/rs/zerolog"
. "github.com/smartystreets/goconvey/convey"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
common "zotregistry.io/zot/pkg/storage/common"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
)
func TestValidateManifest(t *testing.T) {
Convey("Make manifest", t, func(c C) {
dir := t.TempDir()
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: dir,
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver)
content := []byte("this is a blob")
digest := godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
_, blen, err := imgStore.FullBlobUpload("test", bytes.NewReader(content), digest)
So(err, ShouldBeNil)
So(blen, ShouldEqual, len(content))
cblob, cdigest := test.GetRandomImageConfig()
_, clen, err := imgStore.FullBlobUpload("test", bytes.NewReader(cblob), cdigest)
So(err, ShouldBeNil)
So(clen, ShouldEqual, len(cblob))
Convey("bad manifest schema version", func() {
manifest := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: ispec.MediaTypeImageConfig,
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageLayer,
Digest: digest,
Size: int64(len(content)),
},
},
}
manifest.SchemaVersion = 999
body, err := json.Marshal(manifest)
So(err, ShouldBeNil)
_, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body)
So(err, ShouldNotBeNil)
var internalErr *zerr.Error
So(errors.As(err, &internalErr), ShouldBeTrue)
So(internalErr.GetDetails(), ShouldContainKey, "jsonSchemaValidation")
So(internalErr.GetDetails()["jsonSchemaValidation"], ShouldEqual, "[schemaVersion: Must be less than or equal to 2]")
})
Convey("bad config blob", func() {
manifest := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: ispec.MediaTypeImageConfig,
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageLayer,
Digest: digest,
Size: int64(len(content)),
},
},
}
manifest.SchemaVersion = 2
configBlobPath := imgStore.BlobPath("test", cdigest)
err := os.WriteFile(configBlobPath, []byte("bad config blob"), 0o000)
So(err, ShouldBeNil)
body, err := json.Marshal(manifest)
So(err, ShouldBeNil)
// this was actually an umoci error on config blob
_, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body)
So(err, ShouldBeNil)
})
Convey("manifest with non-distributable layers", func() {
content := []byte("this blob doesn't exist")
digest := godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
manifest := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: ispec.MediaTypeImageConfig,
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageLayerNonDistributable, //nolint:staticcheck
Digest: digest,
Size: int64(len(content)),
},
},
}
manifest.SchemaVersion = 2
body, err := json.Marshal(manifest)
So(err, ShouldBeNil)
_, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body)
So(err, ShouldBeNil)
})
})
}
func TestGetReferrersErrors(t *testing.T) {
Convey("make storage", t, func(c C) {
dir := t.TempDir()
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: dir,
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, false, true, log, metrics, nil, cacheDriver)
artifactType := "application/vnd.example.icecream.v1"
validDigest := godigest.FromBytes([]byte("blob"))
Convey("Trigger invalid digest error", func(c C) {
_, err := common.GetReferrers(imgStore, "zot-test", "invalidDigest",
[]string{artifactType}, log)
So(err, ShouldNotBeNil)
_, err = common.GetOrasReferrers(imgStore, "zot-test", "invalidDigest",
artifactType, log)
So(err, ShouldNotBeNil)
})
Convey("Trigger repo not found error", func(c C) {
_, err := common.GetReferrers(imgStore, "zot-test", validDigest,
[]string{artifactType}, log)
So(err, ShouldNotBeNil)
_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
artifactType, log)
So(err, ShouldNotBeNil)
})
storageCtlr := storage.StoreController{DefaultStore: imgStore}
err := test.WriteImageToFileSystem(test.CreateDefaultImage(), "zot-test", "0.0.1", storageCtlr)
So(err, ShouldBeNil)
digest := godigest.FromBytes([]byte("{}"))
index := ispec.Index{
Manifests: []ispec.Descriptor{
{
MediaType: artifactspec.MediaTypeArtifactManifest,
Digest: digest,
},
},
}
indexBuf, err := json.Marshal(index)
So(err, ShouldBeNil)
Convey("Trigger GetBlobContent() not found", func(c C) {
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return indexBuf, nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte{}, zerr.ErrBlobNotFound
},
}
_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
[]string{artifactType}, log)
So(err, ShouldNotBeNil)
_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
artifactType, log)
So(err, ShouldNotBeNil)
})
Convey("Trigger GetBlobContent() generic error", func(c C) {
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return indexBuf, nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte{}, zerr.ErrBadBlob
},
}
_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
[]string{artifactType}, log)
So(err, ShouldNotBeNil)
_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
artifactType, log)
So(err, ShouldNotBeNil)
})
Convey("Trigger continue on different artifactType", func(c C) {
orasManifest := artifactspec.Manifest{
Subject: &artifactspec.Descriptor{
Digest: digest,
ArtifactType: "unknown",
},
}
orasBuf, err := json.Marshal(orasManifest)
So(err, ShouldBeNil)
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return indexBuf, nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return orasBuf, nil
},
}
_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest,
artifactType, log)
So(err, ShouldNotBeNil)
_, err = common.GetOrasReferrers(imgStore, "zot-test", digest,
artifactType, log)
So(err, ShouldNotBeNil)
})
Convey("Unmarshal oras artifact error", func(c C) {
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return indexBuf, nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte("wrong content"), nil
},
}
_, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log)
So(err, ShouldNotBeNil)
})
Convey("Trigger unmarshal error on manifest image mediaType", func(c C) {
index = ispec.Index{
Manifests: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageManifest,
Digest: digest,
},
},
}
indexBuf, err = json.Marshal(index)
So(err, ShouldBeNil)
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return indexBuf, nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte{}, nil
},
}
_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
[]string{artifactType}, log)
So(err, ShouldNotBeNil)
})
Convey("Trigger nil subject", func(c C) {
index = ispec.Index{
Manifests: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageManifest,
Digest: digest,
},
},
}
indexBuf, err = json.Marshal(index)
So(err, ShouldBeNil)
ociManifest := ispec.Manifest{
Subject: nil,
}
ociManifestBuf, err := json.Marshal(ociManifest)
So(err, ShouldBeNil)
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return indexBuf, nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return ociManifestBuf, nil
},
}
_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
[]string{artifactType}, log)
So(err, ShouldBeNil)
})
Convey("Index bad blob", func() {
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return []byte(`{
"manifests": [{
"digest": "digest",
"mediaType": "application/vnd.oci.image.index.v1+json"
}]
}`), nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte("bad blob"), nil
},
}
_, err = common.GetReferrers(imgStore, "zot-test", validDigest,
[]string{}, log)
So(err, ShouldNotBeNil)
})
Convey("Index bad artifac type", func() {
imgStore = &mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return []byte(`{
"manifests": [{
"digest": "digest",
"mediaType": "application/vnd.oci.image.index.v1+json"
}]
}`), nil
},
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte(`{
"subject": {"digest": "` + validDigest.String() + `"}
}`), nil
},
}
ref, err := common.GetReferrers(imgStore, "zot-test", validDigest,
[]string{"art.type"}, log)
So(err, ShouldBeNil)
So(len(ref.Manifests), ShouldEqual, 0)
})
})
}
func TestGetImageIndexErrors(t *testing.T) {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
Convey("Trigger invalid digest error", t, func(c C) {
imgStore := &mocks.MockedImageStore{}
_, err := common.GetImageIndex(imgStore, "zot-test", "invalidDigest", log)
So(err, ShouldNotBeNil)
})
Convey("Trigger GetBlobContent error", t, func(c C) {
imgStore := &mocks.MockedImageStore{
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte{}, zerr.ErrBlobNotFound
},
}
validDigest := godigest.FromBytes([]byte("blob"))
_, err := common.GetImageIndex(imgStore, "zot-test", validDigest, log)
So(err, ShouldNotBeNil)
})
Convey("Trigger unmarshal error", t, func(c C) {
imgStore := &mocks.MockedImageStore{
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte{}, nil
},
}
validDigest := godigest.FromBytes([]byte("blob"))
_, err := common.GetImageIndex(imgStore, "zot-test", validDigest, log)
So(err, ShouldNotBeNil)
})
}
func TestIsSignature(t *testing.T) {
Convey("Unknown media type", t, func(c C) {
isSingature := common.IsSignature(ispec.Descriptor{
MediaType: "unknown media type",
})
So(isSingature, ShouldBeFalse)
})
}
func TestGarbageCollectManifestErrors(t *testing.T) {
Convey("Make imagestore and upload manifest", t, func(c C) {
dir := t.TempDir()
repoName := "test"
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: dir,
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver)
Convey("trigger repo not found in GetReferencedBlobs()", func() {
err := common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log)
So(err, ShouldNotBeNil)
})
content := []byte("this is a blob")
digest := godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
_, blen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), digest)
So(err, ShouldBeNil)
So(blen, ShouldEqual, len(content))
cblob, cdigest := test.GetRandomImageConfig()
_, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest)
So(err, ShouldBeNil)
So(clen, ShouldEqual, len(cblob))
manifest := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: ispec.MediaTypeImageConfig,
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageLayer,
Digest: digest,
Size: int64(len(content)),
},
},
}
manifest.SchemaVersion = 2
body, err := json.Marshal(manifest)
So(err, ShouldBeNil)
manifestDigest := godigest.FromBytes(body)
_, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, body)
So(err, ShouldBeNil)
Convey("trigger GetIndex error in GetReferencedBlobs", func() {
err := os.Chmod(path.Join(imgStore.RootDir(), repoName), 0o000)
So(err, ShouldBeNil)
defer func() {
err := os.Chmod(path.Join(imgStore.RootDir(), repoName), 0o755)
So(err, ShouldBeNil)
}()
err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log)
So(err, ShouldNotBeNil)
})
Convey("trigger GetImageManifest error in GetReferencedBlobsInImageManifest", func() {
err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", manifestDigest.Encoded()), 0o000)
So(err, ShouldBeNil)
defer func() {
err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", manifestDigest.Encoded()), 0o755)
So(err, ShouldBeNil)
}()
err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log)
So(err, ShouldNotBeNil)
})
})
}
func TestGarbageCollectIndexErrors(t *testing.T) {
Convey("Make imagestore and upload manifest", t, func(c C) {
dir := t.TempDir()
repoName := "test"
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: dir,
Name: "cache",
UseRelPaths: true,
}, log)
imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver)
content := []byte("this is a blob")
bdgst := godigest.FromBytes(content)
So(bdgst, ShouldNotBeNil)
_, bsize, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), bdgst)
So(err, ShouldBeNil)
So(bsize, ShouldEqual, len(content))
var index ispec.Index
index.SchemaVersion = 2
index.MediaType = ispec.MediaTypeImageIndex
var digest godigest.Digest
for i := 0; i < 4; i++ {
// upload image config blob
upload, err := imgStore.NewBlobUpload(repoName)
So(err, ShouldBeNil)
So(upload, ShouldNotBeEmpty)
cblob, cdigest := test.GetRandomImageConfig()
buf := bytes.NewBuffer(cblob)
buflen := buf.Len()
blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
// create a manifest
manifest := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: ispec.MediaTypeImageConfig,
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageLayer,
Digest: bdgst,
Size: bsize,
},
},
}
manifest.SchemaVersion = 2
content, err = json.Marshal(manifest)
So(err, ShouldBeNil)
digest = godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
_, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content)
So(err, ShouldBeNil)
index.Manifests = append(index.Manifests, ispec.Descriptor{
Digest: digest,
MediaType: ispec.MediaTypeImageManifest,
Size: int64(len(content)),
})
}
// upload index image
indexContent, err := json.Marshal(index)
So(err, ShouldBeNil)
indexDigest := godigest.FromBytes(indexContent)
So(indexDigest, ShouldNotBeNil)
_, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent)
So(err, ShouldBeNil)
err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log)
So(err, ShouldBeNil)
Convey("trigger GetImageIndex error in GetReferencedBlobsInImageIndex", func() {
err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", indexDigest.Encoded()), 0o000)
So(err, ShouldBeNil)
defer func() {
err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", indexDigest.Encoded()), 0o755)
So(err, ShouldBeNil)
}()
err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log)
So(err, ShouldNotBeNil)
})
})
}