0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00
zot/pkg/meta/parse_test.go
peusebiu 1df743f173
fix(gc): sync repodb when gc'ing manifests (#1819)
fix(gc): fix cleaning deduped blobs because they have the modTime of
the original blobs, fixed by updating the modTime when hard linking
the blobs.
fix(gc): failing to parse rootDir at zot startup when using s3 storage
because there are no files under rootDir and we can not create empty dirs
on s3, fixed by creating an empty file under rootDir.

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
2023-09-22 11:51:20 -07:00

623 lines
18 KiB
Go

package meta_test
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"path"
"testing"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"
zerr "zotregistry.io/zot/errors"
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta"
"zotregistry.io/zot/pkg/meta/boltdb"
"zotregistry.io/zot/pkg/meta/dynamodb"
mTypes "zotregistry.io/zot/pkg/meta/types"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/local"
storageTypes "zotregistry.io/zot/pkg/storage/types"
"zotregistry.io/zot/pkg/test"
. "zotregistry.io/zot/pkg/test/image-utils"
"zotregistry.io/zot/pkg/test/mocks"
)
const repo = "repo"
func TestParseStorageErrors(t *testing.T) {
Convey("ParseStorage", t, func() {
imageStore := mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return nil, ErrTestError
},
GetRepositoriesFn: func() ([]string, error) {
return []string{"repo1", "repo2"}, nil
},
}
storeController := storage.StoreController{DefaultStore: imageStore}
metaDB := mocks.MetaDBMock{}
// sync repo fail
err := meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
Convey("getAllRepos errors", func() {
imageStore1 := mocks.MockedImageStore{
GetRepositoriesFn: func() ([]string, error) {
return []string{"repo1", "repo2"}, nil
},
}
imageStore2 := mocks.MockedImageStore{
GetRepositoriesFn: func() ([]string, error) {
return nil, ErrTestError
},
}
storeController := storage.StoreController{
DefaultStore: imageStore1,
SubStore: map[string]storageTypes.ImageStore{
"a": imageStore2,
},
}
err := meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
})
})
Convey("Parse Repo", t, func() {
imageStore := mocks.MockedImageStore{}
storeController := storage.StoreController{DefaultStore: &imageStore}
metaDB := mocks.MetaDBMock{}
log := log.NewLogger("debug", "")
Convey("imageStore.GetIndexContent errors", func() {
imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
return nil, ErrTestError
}
err := meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
Convey("json.Unmarshal errors", func() {
imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
return []byte("Invalid JSON"), nil
}
err := meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
Convey("resetRepoMetaTags errors", func() {
imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
return []byte("{}"), nil
}
Convey("metaDB.GetRepoMeta errors", func() {
metaDB.GetRepoMetaFn = func(repo string) (mTypes.RepoMetadata, error) {
return mTypes.RepoMetadata{}, ErrTestError
}
err := meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
Convey("isManifestMetaPresent errors", func() {
indexContent := ispec.Index{
Manifests: []ispec.Descriptor{
{
Digest: godigest.FromString("manifest1"),
MediaType: ispec.MediaTypeImageManifest,
Annotations: map[string]string{
ispec.AnnotationRefName: "tag1",
},
},
},
}
indexBlob, err := json.Marshal(indexContent)
So(err, ShouldBeNil)
imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
return indexBlob, nil
}
Convey("metaDB.GetManifestMeta errors", func() {
metaDB.GetManifestMetaFn = func(repo string, manifestDigest godigest.Digest) (mTypes.ManifestMetadata, error) {
return mTypes.ManifestMetadata{}, ErrTestError
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
Convey("manifestMetaIsPresent true", func() {
indexContent := ispec.Index{
Manifests: []ispec.Descriptor{
{
Digest: godigest.FromString("manifest1"),
MediaType: ispec.MediaTypeImageManifest,
Annotations: map[string]string{
ispec.AnnotationRefName: "tag1",
},
},
},
}
indexBlob, err := json.Marshal(indexContent)
So(err, ShouldBeNil)
imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
return indexBlob, nil
}
Convey("metaDB.SetRepoReference", func() {
metaDB.SetRepoReferenceFn = func(repo, tag string, manifestDigest godigest.Digest, mediaType string) error {
return ErrTestError
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
Convey("manifestMetaIsPresent false", func() {
indexContent := ispec.Index{
Manifests: []ispec.Descriptor{
{
Digest: godigest.FromString("manifest1"),
MediaType: ispec.MediaTypeImageManifest,
Annotations: map[string]string{
ispec.AnnotationRefName: "tag1",
},
},
},
}
indexBlob, err := json.Marshal(indexContent)
So(err, ShouldBeNil)
imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
return indexBlob, nil
}
metaDB.GetManifestMetaFn = func(repo string, manifestDigest godigest.Digest) (mTypes.ManifestMetadata, error) {
return mTypes.ManifestMetadata{}, zerr.ErrManifestMetaNotFound
}
Convey("GetImageManifest errors", func() {
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return nil, "", "", ErrTestError
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
Convey("CheckIsImageSignature errors", func() {
// CheckIsImageSignature will fail because of a invalid json
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return []byte("Invalid JSON"), "", "", nil
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
Convey("CheckIsImageSignature -> not signature", func() {
manifestContent := ispec.Manifest{}
manifestBlob, err := json.Marshal(manifestContent)
So(err, ShouldBeNil)
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return manifestBlob, "", "", nil
}
Convey("imgStore.GetBlobContent errors", func() {
imageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) {
return nil, ErrTestError
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
Convey("CheckIsImageSignature -> is signature", func() {
manifestContent := ispec.Manifest{
Subject: &ispec.Descriptor{
Digest: "123",
},
ArtifactType: "application/vnd.cncf.notary.signature",
Layers: []ispec.Descriptor{{MediaType: ispec.MediaTypeImageLayer}},
}
manifestBlob, err := json.Marshal(manifestContent)
So(err, ShouldBeNil)
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return manifestBlob, "", "", nil
}
metaDB.AddManifestSignatureFn = func(repo string, signedManifestDigest godigest.Digest,
sm mTypes.SignatureMetadata,
) error {
return ErrTestError
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
metaDB.AddManifestSignatureFn = func(repo string, signedManifestDigest godigest.Digest,
sm mTypes.SignatureMetadata,
) error {
return nil
}
metaDB.UpdateSignaturesValidityFn = func(repo string, signedManifestDigest godigest.Digest,
) error {
return ErrTestError
}
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
Convey("GetSignatureLayersInfo errors", func() {
// get notation signature layers info
badNotationManifestContent := ispec.Manifest{
Subject: &ispec.Descriptor{
Digest: "123",
},
ArtifactType: "application/vnd.cncf.notary.signature",
}
badNotationManifestBlob, err := json.Marshal(badNotationManifestContent)
So(err, ShouldBeNil)
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return badNotationManifestBlob, "", "", nil
}
// wrong number of layers of notation manifest
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
notationManifestContent := ispec.Manifest{
Subject: &ispec.Descriptor{
Digest: "123",
},
ArtifactType: "application/vnd.cncf.notary.signature",
Layers: []ispec.Descriptor{{MediaType: ispec.MediaTypeImageLayer}},
}
notationManifestBlob, err := json.Marshal(notationManifestContent)
So(err, ShouldBeNil)
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return notationManifestBlob, "", "", nil
}
imageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte{}, ErrTestError
}
// unable to get layer content
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
_, _, cosignManifestContent, _ := test.GetRandomImageComponents(10) //nolint:staticcheck
_, _, signedManifest, _ := test.GetRandomImageComponents(10) //nolint:staticcheck
signatureTag, err := test.GetCosignSignatureTagForManifest(signedManifest)
So(err, ShouldBeNil)
cosignManifestContent.Annotations = map[string]string{ispec.AnnotationRefName: signatureTag}
cosignManifestBlob, err := json.Marshal(cosignManifestContent)
So(err, ShouldBeNil)
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return cosignManifestBlob, "", "", nil
}
indexContent := ispec.Index{
Manifests: []ispec.Descriptor{
{
Digest: godigest.FromString("cosignSig"),
MediaType: ispec.MediaTypeImageManifest,
Annotations: map[string]string{
ispec.AnnotationRefName: signatureTag,
},
},
},
}
indexBlob, err := json.Marshal(indexContent)
So(err, ShouldBeNil)
imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
return indexBlob, nil
}
// unable to get layer content
err = meta.ParseRepo("repo", metaDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
})
}
func TestParseStorageWithBoltDB(t *testing.T) {
Convey("Boltdb", t, func() {
rootDir := t.TempDir()
boltDB, err := boltdb.GetBoltDriver(boltdb.DBParameters{
RootDir: rootDir,
})
So(err, ShouldBeNil)
metaDB, err := boltdb.New(boltDB, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
RunParseStorageTests(rootDir, metaDB)
})
}
func TestParseStorageDynamoWrapper(t *testing.T) {
skipIt(t)
Convey("Dynamodb", t, func() {
rootDir := t.TempDir()
params := dynamodb.DBDriverParameters{
Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"),
Region: "us-east-2",
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
UserDataTablename: "UserDataTable",
APIKeyTablename: "ApiKeyTable",
VersionTablename: "Version",
}
dynamoClient, err := dynamodb.GetDynamoClient(params)
So(err, ShouldBeNil)
dynamoWrapper, err := dynamodb.New(dynamoClient, params, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
err = dynamoWrapper.ResetManifestDataTable()
So(err, ShouldBeNil)
err = dynamoWrapper.ResetRepoMetaTable()
So(err, ShouldBeNil)
RunParseStorageTests(rootDir, dynamoWrapper)
})
}
func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) {
Convey("Test with simple case", func() {
imageStore := local.NewImageStore(rootDir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)
storeController := storage.StoreController{DefaultStore: imageStore}
manifests := []ispec.Manifest{}
for i := 0; i < 3; i++ {
config, layers, manifest, err := test.GetRandomImageComponents(100) //nolint:staticcheck
So(err, ShouldBeNil)
manifests = append(manifests, manifest)
err = test.WriteImageToFileSystem(
Image{
Config: config,
Layers: layers,
Manifest: manifest,
}, repo, fmt.Sprintf("tag%d", i), storeController)
So(err, ShouldBeNil)
}
// add fake signature for tag1
signatureTag, err := test.GetCosignSignatureTagForManifest(manifests[1])
So(err, ShouldBeNil)
manifestBlob, err := json.Marshal(manifests[1])
So(err, ShouldBeNil)
signedManifestDigest := godigest.FromBytes(manifestBlob)
config, layers, manifest, err := test.GetRandomImageComponents(100) //nolint:staticcheck
So(err, ShouldBeNil)
err = test.WriteImageToFileSystem(
Image{
Config: config,
Layers: layers,
Manifest: manifest,
}, repo, signatureTag, storeController)
So(err, ShouldBeNil)
// remove tag2 from index.json
indexPath := path.Join(rootDir, repo, "index.json")
indexFile, err := os.Open(indexPath)
So(err, ShouldBeNil)
buf, err := io.ReadAll(indexFile)
So(err, ShouldBeNil)
var index ispec.Index
if err = json.Unmarshal(buf, &index); err == nil {
for _, manifest := range index.Manifests {
if val, ok := manifest.Annotations[ispec.AnnotationRefName]; ok && val == "tag2" {
delete(manifest.Annotations, ispec.AnnotationRefName)
break
}
}
}
buf, err = json.Marshal(index)
So(err, ShouldBeNil)
err = os.WriteFile(indexPath, buf, 0o600)
So(err, ShouldBeNil)
err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
repos, err := metaDB.GetMultipleRepoMeta(context.Background(),
func(repoMeta mTypes.RepoMetadata) bool { return true })
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(len(repos[0].Tags), ShouldEqual, 2)
for _, descriptor := range repos[0].Tags {
manifestMeta, err := metaDB.GetManifestMeta(repo, godigest.Digest(descriptor.Digest))
So(err, ShouldBeNil)
So(manifestMeta.ManifestBlob, ShouldNotBeNil)
So(manifestMeta.ConfigBlob, ShouldNotBeNil)
if descriptor.Digest == signedManifestDigest.String() {
So(manifestMeta.Signatures, ShouldNotBeEmpty)
}
}
})
Convey("Accept orphan signatures", func() {
imageStore := local.NewImageStore(rootDir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)
storeController := storage.StoreController{DefaultStore: imageStore}
// add an image
config, layers, manifest, err := test.GetRandomImageComponents(100) //nolint:staticcheck
So(err, ShouldBeNil)
err = test.WriteImageToFileSystem(
Image{
Config: config,
Layers: layers,
Manifest: manifest,
}, repo, "tag1", storeController)
So(err, ShouldBeNil)
// add mock cosign signature without pushing the signed image
image, err := test.GetRandomImage() //nolint:staticcheck
So(err, ShouldBeNil)
signatureTag, err := test.GetCosignSignatureTagForManifest(image.Manifest)
So(err, ShouldBeNil)
missingImageDigest := image.Digest()
// get the body of the signature
config, layers, manifest, err = test.GetRandomImageComponents(100) //nolint:staticcheck
So(err, ShouldBeNil)
err = test.WriteImageToFileSystem(
Image{
Config: config,
Layers: layers,
Manifest: manifest,
}, repo, signatureTag, storeController)
So(err, ShouldBeNil)
err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
repos, err := metaDB.GetMultipleRepoMeta(
context.Background(),
func(repoMeta mTypes.RepoMetadata) bool { return true },
)
So(err, ShouldBeNil)
for _, desc := range repos[0].Tags {
So(desc.Digest, ShouldNotResemble, missingImageDigest.String())
}
So(len(repos), ShouldEqual, 1)
So(repos[0].Tags, ShouldContainKey, "tag1")
So(repos[0].Tags, ShouldNotContainKey, signatureTag)
So(repos[0].Signatures, ShouldContainKey, missingImageDigest.String())
})
Convey("Check statistics after load", func() {
imageStore := local.NewImageStore(rootDir, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)
storeController := storage.StoreController{DefaultStore: imageStore}
// add an image
image, err := test.GetRandomImage() //nolint:staticcheck
So(err, ShouldBeNil)
manifestDigest := image.Digest()
err = test.WriteImageToFileSystem(image, repo, "tag", storeController)
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(repo, "tag", manifestDigest, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = metaDB.IncrementRepoStars(repo)
So(err, ShouldBeNil)
err = metaDB.IncrementImageDownloads(repo, "tag")
So(err, ShouldBeNil)
err = metaDB.IncrementImageDownloads(repo, "tag")
So(err, ShouldBeNil)
err = metaDB.IncrementImageDownloads(repo, "tag")
So(err, ShouldBeNil)
repoMeta, err := metaDB.GetRepoMeta(repo)
So(err, ShouldBeNil)
So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 3)
So(repoMeta.Stars, ShouldEqual, 1)
err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
repoMeta, err = metaDB.GetRepoMeta(repo)
So(err, ShouldBeNil)
So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 3)
So(repoMeta.Stars, ShouldEqual, 1)
})
}
func TestGetReferredInfo(t *testing.T) {
Convey("GetReferredInfo error", t, func() {
_, _, _, err := meta.GetReferredInfo([]byte("bad json"), "digest", ispec.MediaTypeImageManifest)
So(err, ShouldNotBeNil)
_, _, _, err = meta.GetReferredInfo([]byte("bad json"), "digest", ispec.MediaTypeImageIndex)
So(err, ShouldNotBeNil)
})
}
func skipIt(t *testing.T) {
t.Helper()
if os.Getenv("S3MOCK_ENDPOINT") == "" {
t.Skip("Skipping testing without AWS S3 mock server")
}
}
func TestGetSignatureLayersInfo(t *testing.T) {
Convey("wrong signature type", t, func() {
layers, err := meta.GetSignatureLayersInfo("repo", "tag", "123", "wrong signature type", []byte{},
nil, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
So(layers, ShouldBeEmpty)
})
Convey("error while unmarshaling manifest content", t, func() {
_, err := meta.GetSignatureLayersInfo("repo", "tag", "123", zcommon.CosignSignature, []byte("bad manifest"),
nil, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
_, err = meta.GetSignatureLayersInfo("repo", "tag", "123", zcommon.NotationSignature, []byte("bad manifest"),
nil, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
})
}