0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-23 22:27:35 -05:00
zot/pkg/meta/parse_test.go
peusebiu b80deb9927
refactor(storage): refactor storage into a single ImageStore (#1656)
unified both local and s3 ImageStore logic into a single ImageStore
added a new driver interface for common file/dirs manipulations
to be implemented by different storage types

refactor(gc): drop umoci dependency, implemented internal gc

added retentionDelay config option that specifies
the garbage collect delay for images without tags

this will also clean manifests which are part of an index image
(multiarch) that no longer exist.

fix(dedupe): skip blobs under .sync/ directory

if startup dedupe is running while also syncing is running
ignore blobs under sync's temporary storage

fix(storage): do not allow image indexes modifications

when deleting a manifest verify that it is not part of a multiarch image
and throw a MethodNotAllowed error to the client if it is.
we don't want to modify multiarch images

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
2023-09-01 10:54:39 -07:00

622 lines
19 KiB
Go

package meta_test
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"path"
"testing"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"
zerr "zotregistry.io/zot/errors"
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta"
"zotregistry.io/zot/pkg/meta/boltdb"
"zotregistry.io/zot/pkg/meta/dynamodb"
mTypes "zotregistry.io/zot/pkg/meta/types"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/local"
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
)
const repo = "repo"
func TestParseStorageErrors(t *testing.T) {
Convey("ParseStorage", t, func() {
imageStore := mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return nil, ErrTestError
},
GetRepositoriesFn: func() ([]string, error) {
return []string{"repo1", "repo2"}, nil
},
}
storeController := storage.StoreController{DefaultStore: imageStore}
metaDB := mocks.MetaDBMock{}
// sync repo fail
err := meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
Convey("getAllRepos errors", func() {
imageStore1 := mocks.MockedImageStore{
GetRepositoriesFn: func() ([]string, error) {
return []string{"repo1", "repo2"}, nil
},
}
imageStore2 := mocks.MockedImageStore{
GetRepositoriesFn: func() ([]string, error) {
return nil, ErrTestError
},
}
storeController := storage.StoreController{
DefaultStore: imageStore1,
SubStore: map[string]storageTypes.ImageStore{
"a": imageStore2,
},
}
err := meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
})
})
Convey("Parse Repo", t, func() {
imageStore := mocks.MockedImageStore{}
storeController := storage.StoreController{DefaultStore: &imageStore}
metaDB := mocks.MetaDBMock{}
log := log.NewLogger("debug", "")
Convey("imageStore.GetIndexContent errors", func() {
imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
return nil, ErrTestError
}
err := meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
Convey("json.Unmarshal errors", func() {
imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
return []byte("Invalid JSON"), nil
}
err := meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
Convey("resetRepoMetaTags errors", func() {
imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
return []byte("{}"), nil
}
Convey("metaDB.GetRepoMeta errors", func() {
metaDB.GetRepoMetaFn = func(repo string) (mTypes.RepoMetadata, error) {
return mTypes.RepoMetadata{}, ErrTestError
}
err := meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
Convey("isManifestMetaPresent errors", func() {
indexContent := ispec.Index{
Manifests: []ispec.Descriptor{
{
Digest: godigest.FromString("manifest1"),
MediaType: ispec.MediaTypeImageManifest,
Annotations: map[string]string{
ispec.AnnotationRefName: "tag1",
},
},
},
}
indexBlob, err := json.Marshal(indexContent)
So(err, ShouldBeNil)
imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
return indexBlob, nil
}
Convey("metaDB.GetManifestMeta errors", func() {
metaDB.GetManifestMetaFn = func(repo string, manifestDigest godigest.Digest) (mTypes.ManifestMetadata, error) {
return mTypes.ManifestMetadata{}, ErrTestError
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
Convey("manifestMetaIsPresent true", func() {
indexContent := ispec.Index{
Manifests: []ispec.Descriptor{
{
Digest: godigest.FromString("manifest1"),
MediaType: ispec.MediaTypeImageManifest,
Annotations: map[string]string{
ispec.AnnotationRefName: "tag1",
},
},
},
}
indexBlob, err := json.Marshal(indexContent)
So(err, ShouldBeNil)
imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
return indexBlob, nil
}
Convey("metaDB.SetRepoReference", func() {
metaDB.SetRepoReferenceFn = func(repo, tag string, manifestDigest godigest.Digest, mediaType string) error {
return ErrTestError
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
Convey("manifestMetaIsPresent false", func() {
indexContent := ispec.Index{
Manifests: []ispec.Descriptor{
{
Digest: godigest.FromString("manifest1"),
MediaType: ispec.MediaTypeImageManifest,
Annotations: map[string]string{
ispec.AnnotationRefName: "tag1",
},
},
},
}
indexBlob, err := json.Marshal(indexContent)
So(err, ShouldBeNil)
imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
return indexBlob, nil
}
metaDB.GetManifestMetaFn = func(repo string, manifestDigest godigest.Digest) (mTypes.ManifestMetadata, error) {
return mTypes.ManifestMetadata{}, zerr.ErrManifestMetaNotFound
}
Convey("GetImageManifest errors", func() {
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return nil, "", "", ErrTestError
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
Convey("CheckIsImageSignature errors", func() {
// CheckIsImageSignature will fail because of a invalid json
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return []byte("Invalid JSON"), "", "", nil
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
Convey("CheckIsImageSignature -> not signature", func() {
manifestContent := ispec.Manifest{}
manifestBlob, err := json.Marshal(manifestContent)
So(err, ShouldBeNil)
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return manifestBlob, "", "", nil
}
Convey("imgStore.GetBlobContent errors", func() {
imageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) {
return nil, ErrTestError
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
Convey("CheckIsImageSignature -> is signature", func() {
manifestContent := ispec.Manifest{
Subject: &ispec.Descriptor{
Digest: "123",
},
ArtifactType: "application/vnd.cncf.notary.signature",
Layers: []ispec.Descriptor{{MediaType: ispec.MediaTypeImageLayer}},
}
manifestBlob, err := json.Marshal(manifestContent)
So(err, ShouldBeNil)
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return manifestBlob, "", "", nil
}
metaDB.AddManifestSignatureFn = func(repo string, signedManifestDigest godigest.Digest,
sm mTypes.SignatureMetadata,
) error {
return ErrTestError
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
metaDB.AddManifestSignatureFn = func(repo string, signedManifestDigest godigest.Digest,
sm mTypes.SignatureMetadata,
) error {
return nil
}
metaDB.UpdateSignaturesValidityFn = func(repo string, signedManifestDigest godigest.Digest,
) error {
return ErrTestError
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
Convey("GetSignatureLayersInfo errors", func() {
// get notation signature layers info
badNotationManifestContent := ispec.Manifest{
Subject: &ispec.Descriptor{
Digest: "123",
},
ArtifactType: "application/vnd.cncf.notary.signature",
}
badNotationManifestBlob, err := json.Marshal(badNotationManifestContent)
So(err, ShouldBeNil)
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return badNotationManifestBlob, "", "", nil
}
// wrong number of layers of notation manifest
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
notationManifestContent := ispec.Manifest{
Subject: &ispec.Descriptor{
Digest: "123",
},
ArtifactType: "application/vnd.cncf.notary.signature",
Layers: []ispec.Descriptor{{MediaType: ispec.MediaTypeImageLayer}},
}
notationManifestBlob, err := json.Marshal(notationManifestContent)
So(err, ShouldBeNil)
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return notationManifestBlob, "", "", nil
}
imageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte{}, ErrTestError
}
// unable to get layer content
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
_, _, cosignManifestContent, _ := test.GetRandomImageComponents(10) //nolint:staticcheck
_, _, signedManifest, _ := test.GetRandomImageComponents(10) //nolint:staticcheck
signatureTag, err := test.GetCosignSignatureTagForManifest(signedManifest)
So(err, ShouldBeNil)
cosignManifestContent.Annotations = map[string]string{ispec.AnnotationRefName: signatureTag}
cosignManifestBlob, err := json.Marshal(cosignManifestContent)
So(err, ShouldBeNil)
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return cosignManifestBlob, "", "", nil
}
indexContent := ispec.Index{
Manifests: []ispec.Descriptor{
{
Digest: godigest.FromString("cosignSig"),
MediaType: ispec.MediaTypeImageManifest,
Annotations: map[string]string{
ispec.AnnotationRefName: signatureTag,
},
},
},
}
indexBlob, err := json.Marshal(indexContent)
So(err, ShouldBeNil)
imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
return indexBlob, nil
}
// unable to get layer content
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
})
}
func TestParseStorageWithBoltDB(t *testing.T) {
Convey("Boltdb", t, func() {
rootDir := t.TempDir()
boltDB, err := boltdb.GetBoltDriver(boltdb.DBParameters{
RootDir: rootDir,
})
So(err, ShouldBeNil)
metaDB, err := boltdb.New(boltDB, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
RunParseStorageTests(rootDir, metaDB)
})
}
func TestParseStorageDynamoWrapper(t *testing.T) {
skipIt(t)
Convey("Dynamodb", t, func() {
rootDir := t.TempDir()
params := dynamodb.DBDriverParameters{
Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"),
Region: "us-east-2",
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
UserDataTablename: "UserDataTable",
APIKeyTablename: "ApiKeyTable",
VersionTablename: "Version",
}
dynamoClient, err := dynamodb.GetDynamoClient(params)
So(err, ShouldBeNil)
dynamoWrapper, err := dynamodb.New(dynamoClient, params, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
err = dynamoWrapper.ResetManifestDataTable()
So(err, ShouldBeNil)
err = dynamoWrapper.ResetRepoMetaTable()
So(err, ShouldBeNil)
RunParseStorageTests(rootDir, dynamoWrapper)
})
}
func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) {
Convey("Test with simple case", func() {
imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)
storeController := storage.StoreController{DefaultStore: imageStore}
manifests := []ispec.Manifest{}
for i := 0; i < 3; i++ {
config, layers, manifest, err := test.GetRandomImageComponents(100) //nolint:staticcheck
So(err, ShouldBeNil)
manifests = append(manifests, manifest)
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
}, repo, fmt.Sprintf("tag%d", i), storeController)
So(err, ShouldBeNil)
}
// add fake signature for tag1
signatureTag, err := test.GetCosignSignatureTagForManifest(manifests[1])
So(err, ShouldBeNil)
manifestBlob, err := json.Marshal(manifests[1])
So(err, ShouldBeNil)
signedManifestDigest := godigest.FromBytes(manifestBlob)
config, layers, manifest, err := test.GetRandomImageComponents(100) //nolint:staticcheck
So(err, ShouldBeNil)
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
}, repo, signatureTag, storeController)
So(err, ShouldBeNil)
// remove tag2 from index.json
indexPath := path.Join(rootDir, repo, "index.json")
indexFile, err := os.Open(indexPath)
So(err, ShouldBeNil)
buf, err := io.ReadAll(indexFile)
So(err, ShouldBeNil)
var index ispec.Index
if err = json.Unmarshal(buf, &index); err == nil {
for _, manifest := range index.Manifests {
if val, ok := manifest.Annotations[ispec.AnnotationRefName]; ok && val == "tag2" {
delete(manifest.Annotations, ispec.AnnotationRefName)
break
}
}
}
buf, err = json.Marshal(index)
So(err, ShouldBeNil)
err = os.WriteFile(indexPath, buf, 0o600)
So(err, ShouldBeNil)
err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
repos, err := metaDB.GetMultipleRepoMeta(context.Background(),
func(repoMeta mTypes.RepoMetadata) bool { return true })
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(len(repos[0].Tags), ShouldEqual, 2)
for _, descriptor := range repos[0].Tags {
manifestMeta, err := metaDB.GetManifestMeta(repo, godigest.Digest(descriptor.Digest))
So(err, ShouldBeNil)
So(manifestMeta.ManifestBlob, ShouldNotBeNil)
So(manifestMeta.ConfigBlob, ShouldNotBeNil)
if descriptor.Digest == signedManifestDigest.String() {
So(manifestMeta.Signatures, ShouldNotBeEmpty)
}
}
})
Convey("Accept orphan signatures", func() {
imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)
storeController := storage.StoreController{DefaultStore: imageStore}
// add an image
config, layers, manifest, err := test.GetRandomImageComponents(100) //nolint:staticcheck
So(err, ShouldBeNil)
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
}, repo, "tag1", storeController)
So(err, ShouldBeNil)
// add mock cosign signature without pushing the signed image
image, err := test.GetRandomImage() //nolint:staticcheck
So(err, ShouldBeNil)
signatureTag, err := test.GetCosignSignatureTagForManifest(image.Manifest)
So(err, ShouldBeNil)
missingImageDigest := image.Digest()
// get the body of the signature
config, layers, manifest, err = test.GetRandomImageComponents(100) //nolint:staticcheck
So(err, ShouldBeNil)
err = test.WriteImageToFileSystem(
test.Image{
Config: config,
Layers: layers,
Manifest: manifest,
}, repo, signatureTag, storeController)
So(err, ShouldBeNil)
err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
repos, err := metaDB.GetMultipleRepoMeta(
context.Background(),
func(repoMeta mTypes.RepoMetadata) bool { return true },
)
So(err, ShouldBeNil)
for _, desc := range repos[0].Tags {
So(desc.Digest, ShouldNotResemble, missingImageDigest.String())
}
So(len(repos), ShouldEqual, 1)
So(repos[0].Tags, ShouldContainKey, "tag1")
So(repos[0].Tags, ShouldNotContainKey, signatureTag)
So(repos[0].Signatures, ShouldContainKey, missingImageDigest.String())
})
Convey("Check statistics after load", func() {
imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)
storeController := storage.StoreController{DefaultStore: imageStore}
// add an image
image, err := test.GetRandomImage() //nolint:staticcheck
So(err, ShouldBeNil)
manifestDigest := image.Digest()
err = test.WriteImageToFileSystem(image, repo, "tag", storeController)
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(repo, "tag", manifestDigest, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = metaDB.IncrementRepoStars(repo)
So(err, ShouldBeNil)
err = metaDB.IncrementImageDownloads(repo, "tag")
So(err, ShouldBeNil)
err = metaDB.IncrementImageDownloads(repo, "tag")
So(err, ShouldBeNil)
err = metaDB.IncrementImageDownloads(repo, "tag")
So(err, ShouldBeNil)
repoMeta, err := metaDB.GetRepoMeta(repo)
So(err, ShouldBeNil)
So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 3)
So(repoMeta.Stars, ShouldEqual, 1)
err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
repoMeta, err = metaDB.GetRepoMeta(repo)
So(err, ShouldBeNil)
So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 3)
So(repoMeta.Stars, ShouldEqual, 1)
})
}
func TestGetReferredInfo(t *testing.T) {
Convey("GetReferredInfo error", t, func() {
_, _, _, err := meta.GetReferredInfo([]byte("bad json"), "digest", ispec.MediaTypeImageManifest)
So(err, ShouldNotBeNil)
_, _, _, err = meta.GetReferredInfo([]byte("bad json"), "digest", ispec.MediaTypeImageIndex)
So(err, ShouldNotBeNil)
})
}
func skipIt(t *testing.T) {
t.Helper()
if os.Getenv("S3MOCK_ENDPOINT") == "" {
t.Skip("Skipping testing without AWS S3 mock server")
}
}
func TestGetSignatureLayersInfo(t *testing.T) {
Convey("wrong signature type", t, func() {
layers, err := meta.GetSignatureLayersInfo("repo", "tag", "123", "wrong signature type", []byte{},
nil, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
So(layers, ShouldBeEmpty)
})
Convey("error while unmarshaling manifest content", t, func() {
_, err := meta.GetSignatureLayersInfo("repo", "tag", "123", zcommon.CosignSignature, []byte("bad manifest"),
nil, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
_, err = meta.GetSignatureLayersInfo("repo", "tag", "123", zcommon.NotationSignature, []byte("bad manifest"),
nil, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
})
}