From 8467a80a504340099ec31953a85b8629d7400be1 Mon Sep 17 00:00:00 2001 From: Andrei Aaron Date: Wed, 17 Jan 2024 20:20:07 +0200 Subject: [PATCH] refactor: update tests to use the newer API for creating test images (#2168) - update cve tests - update scrub tests - update tests for parsing storage and loading into meta DB - update controller tests Signed-off-by: Andrei Aaron --- pkg/api/controller_test.go | 185 ++++-------------- pkg/extensions/search/cve/cve_test.go | 40 ++-- .../search/cve/trivy/scanner_internal_test.go | 33 +--- .../search/cve/trivy/scanner_test.go | 5 +- pkg/meta/parse_test.go | 38 +--- pkg/storage/s3/s3_test.go | 8 +- pkg/storage/scrub_test.go | 103 +++++----- pkg/storage/storage_test.go | 9 + pkg/test/deprecated/deprecated.go | 30 --- 9 files changed, 144 insertions(+), 307 deletions(-) diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 9db8a225..f49892fa 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -64,7 +64,6 @@ import ( storageTypes "zotregistry.io/zot/pkg/storage/types" authutils "zotregistry.io/zot/pkg/test/auth" test "zotregistry.io/zot/pkg/test/common" - "zotregistry.io/zot/pkg/test/deprecated" . "zotregistry.io/zot/pkg/test/image-utils" "zotregistry.io/zot/pkg/test/inject" "zotregistry.io/zot/pkg/test/mocks" @@ -6327,26 +6326,13 @@ func TestArtifactReferences(t *testing.T) { defer cm.StopServer() repoName := "artifact-repo" - content := []byte("this is a blob") - digest := godigest.FromBytes(content) - So(digest, ShouldNotBeNil) - cfg, layers, manifest, err := deprecated.GetImageComponents(2) //nolint:staticcheck - So(err, ShouldBeNil) + image := CreateImageWith().RandomLayers(1, 2).DefaultConfig().Build() + digest := image.ManifestDescriptor.Digest - err = UploadImage( - Image{ - Config: cfg, - Layers: layers, - Manifest: manifest, - }, baseURL, repoName, "1.0") + err := UploadImage(image, baseURL, repoName, "1.0") So(err, ShouldBeNil) - content, err = json.Marshal(manifest) - So(err, ShouldBeNil) - digest = godigest.FromBytes(content) - So(digest, ShouldNotBeNil) - artifactType := "application/vnd.example.icecream.v1" Convey("Validate Image Manifest Reference", func() { @@ -6389,13 +6375,13 @@ func TestArtifactReferences(t *testing.T) { { MediaType: "application/vnd.oci.image.layer.v1.tar", Digest: digest, - Size: int64(len(content)), + Size: image.ManifestDescriptor.Size, }, }, Subject: &ispec.Descriptor{ MediaType: ispec.MediaTypeImageManifest, Digest: digest, - Size: int64(len(content)), + Size: image.ManifestDescriptor.Size, }, Annotations: map[string]string{ "key": "val", @@ -6530,7 +6516,7 @@ func TestArtifactReferences(t *testing.T) { }) Convey("Using valid content", func() { - content, err = json.Marshal(manifest) + content, err := json.Marshal(manifest) So(err, ShouldBeNil) resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). SetBody(content).Put(baseURL + fmt.Sprintf("/v2/%s/manifests/1.0", repoName)) @@ -7366,33 +7352,23 @@ func TestStorageCommit(t *testing.T) { Convey("Manifests", func() { _, _ = Print("\nManifests") - cfg, layers, manifest, err := deprecated.GetImageComponents(2) //nolint:staticcheck - So(err, ShouldBeNil) - - content := []byte("this is a blob5") - digest := godigest.FromBytes(content) - So(digest, ShouldNotBeNil) // check a non-existent manifest resp, err := resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). - SetBody(content).Head(baseURL + "/v2/unknown/manifests/test:1.0") + Head(baseURL + "/v2/unknown/manifests/test:1.0") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + image := CreateImageWith().RandomLayers(1, 2).DefaultConfig().Build() + repoName := "repo7" - err = UploadImage( - Image{ - Config: cfg, - Layers: layers, - Manifest: manifest, - }, baseURL, repoName, "test:1.0") + err = UploadImage(image, baseURL, repoName, "test:1.0") So(err, ShouldBeNil) _, err = os.Stat(path.Join(dir, "repo7")) So(err, ShouldBeNil) - content, err = json.Marshal(manifest) - So(err, ShouldBeNil) - digest = godigest.FromBytes(content) + content := image.ManifestDescriptor.Data + digest := image.ManifestDescriptor.Digest So(digest, ShouldNotBeNil) resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). SetBody(content).Put(baseURL + "/v2/repo7/manifests/test:1.0") @@ -7410,26 +7386,12 @@ func TestStorageCommit(t *testing.T) { So(digestHdr, ShouldNotBeEmpty) So(digestHdr, ShouldEqual, digest.String()) - content, err = json.Marshal(manifest) + err = UploadImage(image, baseURL, repoName, "test:1.0.1") So(err, ShouldBeNil) - err = UploadImage( - Image{ - Config: cfg, - Layers: layers, - Manifest: manifest, - }, baseURL, repoName, "test:1.0.1") - So(err, ShouldBeNil) + image = CreateImageWith().RandomLayers(1, 1).DefaultConfig().Build() - cfg, layers, manifest, err = deprecated.GetImageComponents(1) //nolint:staticcheck - So(err, ShouldBeNil) - - err = UploadImage( - Image{ - Config: cfg, - Layers: layers, - Manifest: manifest, - }, baseURL, repoName, "test:2.0") + err = UploadImage(image, baseURL, repoName, "test:2.0") So(err, ShouldBeNil) resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). @@ -7519,34 +7481,23 @@ func TestManifestImageIndex(t *testing.T) { rthdlr := api.NewRouteHandler(ctlr) - cfg, layers, manifest, err := deprecated.GetImageComponents(2) //nolint:staticcheck - So(err, ShouldBeNil) - - content := []byte("this is a blob1") - digest := godigest.FromBytes(content) - So(digest, ShouldNotBeNil) + img := CreateImageWith().RandomLayers(1, 2).DefaultConfig().Build() // check a non-existent manifest resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). - SetBody(content).Head(baseURL + "/v2/unknown/manifests/test:1.0") + Head(baseURL + "/v2/unknown/manifests/test:1.0") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) repoName := "index" - err = UploadImage( - Image{ - Config: cfg, - Layers: layers, - Manifest: manifest, - }, baseURL, repoName, "test:1.0") + err = UploadImage(img, baseURL, repoName, "test:1.0") So(err, ShouldBeNil) _, err = os.Stat(path.Join(dir, "index")) So(err, ShouldBeNil) - content, err = json.Marshal(manifest) - So(err, ShouldBeNil) - digest = godigest.FromBytes(content) + content := img.ManifestDescriptor.Data + digest := img.ManifestDescriptor.Digest So(digest, ShouldNotBeNil) m1content := content resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). @@ -7564,7 +7515,7 @@ func TestManifestImageIndex(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) - img := CreateRandomImage() + img = CreateRandomImage() err = UploadImage(img, baseURL, repoName, img.DigestStr()) So(err, ShouldBeNil) @@ -7946,41 +7897,24 @@ func TestManifestCollision(t *testing.T) { cm.StartAndWait(port) defer cm.StopServer() - cfg, layers, manifest, err := deprecated.GetImageComponents(2) //nolint:staticcheck - So(err, ShouldBeNil) + img := CreateImageWith().RandomLayers(1, 2).DefaultConfig().Build() - err = UploadImage( - Image{ - Config: cfg, - Layers: layers, - Manifest: manifest, - }, baseURL, "index", "test:1.0") + err := UploadImage(img, baseURL, "index", "test:1.0") So(err, ShouldBeNil) _, err = os.Stat(path.Join(dir, "index")) So(err, ShouldBeNil) - content := []byte("this is a blob1") - digest := godigest.FromBytes(content) - So(digest, ShouldNotBeNil) - // check a non-existent manifest resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). - SetBody(content).Head(baseURL + "/v2/unknown/manifests/test:1.0") + Head(baseURL + "/v2/unknown/manifests/test:1.0") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) - content, err = json.Marshal(manifest) - So(err, ShouldBeNil) - digest = godigest.FromBytes(content) + digest := img.ManifestDescriptor.Digest So(digest, ShouldNotBeNil) - err = UploadImage( - Image{ - Config: cfg, - Layers: layers, - Manifest: manifest, - }, baseURL, "index", "test:2.0") + err = UploadImage(img, baseURL, "index", "test:2.0") So(err, ShouldBeNil) // Deletion should fail if using digest @@ -8660,19 +8594,11 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { Convey("Overwrite original image, signatures should be garbage-collected", func() { // push an image without tag - cfg, layers, manifest, err := deprecated.GetImageComponents(2) //nolint:staticcheck - So(err, ShouldBeNil) + img := CreateImageWith().RandomLayers(1, 2).DefaultConfig().Build() - manifestBuf, err := json.Marshal(manifest) - So(err, ShouldBeNil) - untaggedManifestDigest := godigest.FromBytes(manifestBuf) + untaggedManifestDigest := img.ManifestDescriptor.Digest - err = UploadImage( - Image{ - Config: cfg, - Layers: layers, - Manifest: manifest, - }, baseURL, repoName, untaggedManifestDigest.String()) + err = UploadImage(img, baseURL, repoName, untaggedManifestDigest.String()) So(err, ShouldBeNil) // make sure repoDB reference was added @@ -8687,20 +8613,12 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { So(ok, ShouldBeTrue) // overwrite image so that signatures will get invalidated and gc'ed - cfg, layers, manifest, err = deprecated.GetImageComponents(3) //nolint:staticcheck + img = CreateImageWith().RandomLayers(1, 3).DefaultConfig().Build() + + err = UploadImage(img, baseURL, repoName, tag) So(err, ShouldBeNil) - err = UploadImage( - Image{ - Config: cfg, - Layers: layers, - Manifest: manifest, - }, baseURL, repoName, tag) - So(err, ShouldBeNil) - - manifestBuf, err = json.Marshal(manifest) - So(err, ShouldBeNil) - newManifestDigest := godigest.FromBytes(manifestBuf) + newManifestDigest := img.ManifestDescriptor.Digest err = gc.CleanRepo(ctx, repoName) //nolint: contextcheck So(err, ShouldBeNil) @@ -8804,26 +8722,17 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { // upload multiple manifests for i := 0; i < 4; i++ { - config, layers, manifest, err := deprecated.GetImageComponents(1000 + i) //nolint:staticcheck - So(err, ShouldBeNil) + img := CreateImageWith().RandomLayers(1, 1000+i).DefaultConfig().Build() - manifestContent, err := json.Marshal(manifest) - So(err, ShouldBeNil) + manifestDigest := img.ManifestDescriptor.Digest - manifestDigest := godigest.FromBytes(manifestContent) - - err = UploadImage( - Image{ - Manifest: manifest, - Config: config, - Layers: layers, - }, baseURL, repoName, manifestDigest.String()) + err = UploadImage(img, baseURL, repoName, manifestDigest.String()) So(err, ShouldBeNil) index.Manifests = append(index.Manifests, ispec.Descriptor{ Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest, - Size: int64(len(manifestContent)), + Size: img.ManifestDescriptor.Size, }) } @@ -9055,28 +8964,18 @@ func TestSearchRoutes(t *testing.T) { cm.StartAndWait(port) defer cm.StopServer() - cfg, layers, manifest, err := deprecated.GetImageComponents(10000) //nolint:staticcheck - So(err, ShouldBeNil) + img := CreateImageWith().RandomLayers(1, 10000).DefaultConfig().Build() - err = UploadImageWithBasicAuth( - Image{ - Config: cfg, - Layers: layers, - Manifest: manifest, - }, baseURL, repoName, "latest", + err := UploadImageWithBasicAuth( + img, baseURL, repoName, "latest", user1, password1) So(err, ShouldBeNil) // data for the inaccessible repo - cfg, layers, manifest, err = deprecated.GetImageComponents(10000) //nolint:staticcheck - So(err, ShouldBeNil) + img = CreateImageWith().RandomLayers(1, 10000).DefaultConfig().Build() err = UploadImageWithBasicAuth( - Image{ - Config: cfg, - Layers: layers, - Manifest: manifest, - }, baseURL, inaccessibleRepo, "latest", + img, baseURL, inaccessibleRepo, "latest", user1, password1) So(err, ShouldBeNil) diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index add2640e..0347af72 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -40,7 +40,6 @@ import ( "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/local" test "zotregistry.io/zot/pkg/test/common" - "zotregistry.io/zot/pkg/test/deprecated" . "zotregistry.io/zot/pkg/test/image-utils" "zotregistry.io/zot/pkg/test/mocks" ociutils "zotregistry.io/zot/pkg/test/oci-utils" @@ -1729,35 +1728,30 @@ func TestFixedTagsWithIndex(t *testing.T) { defer cm.StopServer() // push index with 2 manifests: one with vulns and one without vulnManifestCreated := time.Date(2010, 1, 1, 1, 1, 1, 1, time.UTC) - vulnManifest, err := deprecated.GetVulnImageWithConfig(ispec.Image{ //nolint:staticcheck - Created: &vulnManifestCreated, - Platform: ispec.Platform{OS: "linux", Architecture: "amd64"}, - }) - So(err, ShouldBeNil) + vulnImageConfig := GetDefaultConfig() + vulnImageConfig.Created = &vulnManifestCreated + vulnImageConfig.Platform = ispec.Platform{OS: "linux", Architecture: "amd64"} + vulnSingleArchImage := CreateImageWith().VulnerableLayers().VulnerableConfig(vulnImageConfig).Build() fixedManifestCreated := time.Date(2010, 1, 1, 1, 1, 1, 1, time.UTC) - fixedManifest, err := deprecated.GetImageWithConfig(ispec.Image{ //nolint:staticcheck - Created: &fixedManifestCreated, - Platform: ispec.Platform{OS: "windows", Architecture: "amd64"}, - }) - So(err, ShouldBeNil) - fixedDigest := fixedManifest.Digest() + fixedImageConfig := GetDefaultConfig() + fixedImageConfig.Created = &fixedManifestCreated + fixedImageConfig.Platform = ispec.Platform{OS: "windows", Architecture: "amd64"} + fixedSingleArchImage := CreateImageWith().DefaultLayers().ImageConfig(fixedImageConfig).Build() - multiArch := deprecated.GetMultiarchImageForImages([]Image{fixedManifest, //nolint:staticcheck - vulnManifest}) + multiArchImage := CreateMultiarchWith().Images([]Image{vulnSingleArchImage, fixedSingleArchImage}).Build() - err = UploadMultiarchImage(multiArch, baseURL, "repo", "multi-arch-tag") + err = UploadMultiarchImage(multiArchImage, baseURL, "repo", "multi-arch-tag") So(err, ShouldBeNil) // oldest vulnerability simpleVulnCreated := time.Date(2005, 1, 1, 1, 1, 1, 1, time.UTC) - simpleVulnImg, err := deprecated.GetVulnImageWithConfig(ispec.Image{ //nolint:staticcheck - Created: &simpleVulnCreated, - Platform: ispec.Platform{OS: "windows", Architecture: "amd64"}, - }) - So(err, ShouldBeNil) + singleVulnImageConfig := GetDefaultConfig() + singleVulnImageConfig.Created = &simpleVulnCreated + singleVulnImageConfig.Platform = ispec.Platform{OS: "windows", Architecture: "amd64"} + simpleVulnImage := CreateImageWith().VulnerableLayers().VulnerableConfig(singleVulnImageConfig).Build() - err = UploadImage(simpleVulnImg, baseURL, "repo", "vuln-img") + err = UploadImage(simpleVulnImage, baseURL, "repo", "vuln-img") So(err, ShouldBeNil) // Wait for trivy db to download @@ -1771,7 +1765,7 @@ func TestFixedTagsWithIndex(t *testing.T) { So(err, ShouldBeNil) So(len(tagsInfo), ShouldEqual, 1) So(len(tagsInfo[0].Manifests), ShouldEqual, 1) - So(tagsInfo[0].Manifests[0].Digest, ShouldResemble, fixedDigest) + So(tagsInfo[0].Manifests[0].Digest, ShouldResemble, fixedSingleArchImage.ManifestDescriptor.Digest) const query = ` { @@ -1794,7 +1788,7 @@ func TestFixedTagsWithIndex(t *testing.T) { So(len(responseStruct.Results), ShouldEqual, 1) So(len(responseStruct.Results[0].Manifests), ShouldEqual, 1) fixedManifestResp := responseStruct.Results[0].Manifests[0] - So(fixedManifestResp.Digest, ShouldResemble, fixedDigest.String()) + So(fixedManifestResp.Digest, ShouldResemble, fixedSingleArchImage.ManifestDescriptor.Digest.String()) }) } diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index d94a1fc8..660acd2e 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -4,9 +4,7 @@ package trivy import ( - "bytes" "context" - "encoding/json" "os" "path" "testing" @@ -29,38 +27,17 @@ import ( "zotregistry.io/zot/pkg/storage/local" storageTypes "zotregistry.io/zot/pkg/storage/types" test "zotregistry.io/zot/pkg/test/common" - "zotregistry.io/zot/pkg/test/deprecated" . "zotregistry.io/zot/pkg/test/image-utils" "zotregistry.io/zot/pkg/test/mocks" ) -func generateTestImage(storeController storage.StoreController, image string) { - repoName, tag := common.GetImageDirAndTag(image) +func generateTestImage(storeController storage.StoreController, imageName string) { + repoName, tag := common.GetImageDirAndTag(imageName) - config, layers, manifest, err := deprecated.GetImageComponents(10) //nolint:staticcheck - So(err, ShouldBeNil) + image := CreateRandomImage() - store := storeController.GetImageStore(repoName) - err = store.InitRepo(repoName) - So(err, ShouldBeNil) - - for _, layerBlob := range layers { - layerReader := bytes.NewReader(layerBlob) - layerDigest := godigest.FromBytes(layerBlob) - _, _, err = store.FullBlobUpload(repoName, layerReader, layerDigest) - So(err, ShouldBeNil) - } - - configBlob, err := json.Marshal(config) - So(err, ShouldBeNil) - configReader := bytes.NewReader(configBlob) - configDigest := godigest.FromBytes(configBlob) - _, _, err = store.FullBlobUpload(repoName, configReader, configDigest) - So(err, ShouldBeNil) - - manifestBlob, err := json.Marshal(manifest) - So(err, ShouldBeNil) - _, _, err = store.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBlob) + err := WriteImageToFileSystem( + image, repoName, tag, storeController) So(err, ShouldBeNil) } diff --git a/pkg/extensions/search/cve/trivy/scanner_test.go b/pkg/extensions/search/cve/trivy/scanner_test.go index ef69fac6..5e3af985 100644 --- a/pkg/extensions/search/cve/trivy/scanner_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_test.go @@ -25,7 +25,6 @@ import ( "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/local" . "zotregistry.io/zot/pkg/test/common" - "zotregistry.io/zot/pkg/test/deprecated" . "zotregistry.io/zot/pkg/test/image-utils" "zotregistry.io/zot/pkg/test/mocks" ) @@ -97,8 +96,8 @@ func TestScanningByDigest(t *testing.T) { simpleImage := CreateRandomImage() - multiArch := deprecated.GetMultiarchImageForImages([]Image{simpleImage, //nolint:staticcheck - vulnImage}) + multiArch := CreateMultiarchWith().Images([]Image{simpleImage, //nolint:staticcheck + vulnImage}).Build() err := UploadMultiarchImage(multiArch, baseURL, "multi-arch", "multi-arch-tag") So(err, ShouldBeNil) diff --git a/pkg/meta/parse_test.go b/pkg/meta/parse_test.go index 412a68c2..b81c74be 100644 --- a/pkg/meta/parse_test.go +++ b/pkg/meta/parse_test.go @@ -26,7 +26,6 @@ import ( "zotregistry.io/zot/pkg/storage/local" storageTypes "zotregistry.io/zot/pkg/storage/types" tcommon "zotregistry.io/zot/pkg/test/common" - "zotregistry.io/zot/pkg/test/deprecated" . "zotregistry.io/zot/pkg/test/image-utils" "zotregistry.io/zot/pkg/test/mocks" ociutils "zotregistry.io/zot/pkg/test/oci-utils" @@ -368,15 +367,9 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB, log log.Logger) signedManifestDigest := godigest.FromBytes(manifestBlob) - config, layers, manifest, err := deprecated.GetRandomImageComponents(100) //nolint:staticcheck - So(err, ShouldBeNil) + image := CreateRandomImage() - err = WriteImageToFileSystem( - Image{ - Config: config, - Layers: layers, - Manifest: manifest, - }, repo, signatureTag, storeController) + err = WriteImageToFileSystem(image, repo, signatureTag, storeController) So(err, ShouldBeNil) // remove tag2 from index.json @@ -427,37 +420,26 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB, log log.Logger) log, monitoring.NewMetricsServer(false, log), nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} - // add an image - config, layers, manifest, err := deprecated.GetRandomImageComponents(100) //nolint:staticcheck - So(err, ShouldBeNil) - err = WriteImageToFileSystem( - Image{ - Config: config, - Layers: layers, - Manifest: manifest, - }, repo, "tag1", storeController) + // add an image + image := CreateRandomImage() + + err := WriteImageToFileSystem(image, repo, "tag1", storeController) So(err, ShouldBeNil) // add mock cosign signature without pushing the signed image - image, err := deprecated.GetRandomImage() //nolint:staticcheck + image = CreateRandomImage() So(err, ShouldBeNil) signatureTag, err := signature.GetCosignSignatureTagForManifest(image.Manifest) So(err, ShouldBeNil) - missingImageDigest := image.Digest() + missingImageDigest := image.ManifestDescriptor.Digest // get the body of the signature - config, layers, manifest, err = deprecated.GetRandomImageComponents(100) //nolint:staticcheck - So(err, ShouldBeNil) + signature := CreateRandomImage() - err = WriteImageToFileSystem( - Image{ - Config: config, - Layers: layers, - Manifest: manifest, - }, repo, signatureTag, storeController) + err = WriteImageToFileSystem(signature, repo, signatureTag, storeController) So(err, ShouldBeNil) err = meta.ParseStorage(metaDB, storeController, log) //nolint: contextcheck diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 043fb465..d05e8001 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -36,7 +36,6 @@ import ( storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/s3" storageTypes "zotregistry.io/zot/pkg/storage/types" - "zotregistry.io/zot/pkg/test/deprecated" . "zotregistry.io/zot/pkg/test/image-utils" "zotregistry.io/zot/pkg/test/inject" "zotregistry.io/zot/pkg/test/mocks" @@ -467,8 +466,11 @@ func TestGetOrasAndOCIReferrers(t *testing.T) { _, imgStore, _ := createObjectsStore(testDir, tdir, true) Convey("Upload test image", t, func(c C) { - cfg, layers, manifest, err := deprecated.GetImageComponents(100) //nolint:staticcheck - So(err, ShouldBeNil) + image := CreateDefaultImage() + + manifest := image.Manifest + cfg := image.Config + layers := image.Layers for _, content := range layers { upload, err := imgStore.NewBlobUpload(repo) diff --git a/pkg/storage/scrub_test.go b/pkg/storage/scrub_test.go index d2643c58..7f96eed7 100644 --- a/pkg/storage/scrub_test.go +++ b/pkg/storage/scrub_test.go @@ -26,7 +26,6 @@ import ( "zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/storage/s3" storageTypes "zotregistry.io/zot/pkg/storage/types" - "zotregistry.io/zot/pkg/test/deprecated" . "zotregistry.io/zot/pkg/test/image-utils" "zotregistry.io/zot/pkg/test/mocks" tskip "zotregistry.io/zot/pkg/test/skip" @@ -92,24 +91,9 @@ func RunCheckAllBlobsIntegrityTests( //nolint: thelper storeCtlr.DefaultStore = imgStore So(storeCtlr.GetImageStore(repoName), ShouldResemble, imgStore) - config, layers, manifest, err := deprecated.GetImageComponents(1000) //nolint:staticcheck - So(err, ShouldBeNil) + image := CreateRandomImage() - layerReader := bytes.NewReader(layers[0]) - layerDigest := godigest.FromBytes(layers[0]) - _, _, err = imgStore.FullBlobUpload(repoName, layerReader, layerDigest) - So(err, ShouldBeNil) - - configBlob, err := json.Marshal(config) - So(err, ShouldBeNil) - configReader := bytes.NewReader(configBlob) - configDigest := godigest.FromBytes(configBlob) - _, _, err = imgStore.FullBlobUpload(repoName, configReader, configDigest) - So(err, ShouldBeNil) - - manifestBlob, err := json.Marshal(manifest) - So(err, ShouldBeNil) - manifestDigest, _, err := imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBlob) + err = WriteImageToFileSystem(image, repoName, tag, storeCtlr) So(err, ShouldBeNil) Convey("Blobs integrity not affected", func() { @@ -144,11 +128,11 @@ func RunCheckAllBlobsIntegrityTests( //nolint: thelper Convey("Manifest integrity affected", func() { // get content of manifest file - content, _, _, err := imgStore.GetImageManifest(repoName, manifestDigest.String()) + content, _, _, err := imgStore.GetImageManifest(repoName, image.ManifestDescriptor.Digest.String()) So(err, ShouldBeNil) // delete content of manifest file - manifestDig := manifestDigest.Encoded() + manifestDig := image.ManifestDescriptor.Digest.Encoded() manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", manifestDig) err = driver.Delete(manifestFile) So(err, ShouldBeNil) @@ -210,11 +194,11 @@ func RunCheckAllBlobsIntegrityTests( //nolint: thelper Convey("Config integrity affected", func() { // get content of config file - content, err := imgStore.GetBlobContent(repoName, configDigest) + content, err := imgStore.GetBlobContent(repoName, image.ConfigDescriptor.Digest) So(err, ShouldBeNil) // delete content of config file - configDig := configDigest.Encoded() + configDig := image.ConfigDescriptor.Digest.Encoded() configFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", configDig) err = driver.Delete(configFile) So(err, ShouldBeNil) @@ -254,11 +238,11 @@ func RunCheckAllBlobsIntegrityTests( //nolint: thelper Convey("Layers integrity affected", func() { // get content of layer - content, err := imgStore.GetBlobContent(repoName, layerDigest) + content, err := imgStore.GetBlobContent(repoName, image.Manifest.Layers[0].Digest) So(err, ShouldBeNil) // delete content of layer file - layerDig := layerDigest.Encoded() + layerDig := image.Manifest.Layers[0].Digest.Encoded() layerFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", layerDig) _, err = driver.WriteFile(layerFile, []byte(" ")) So(err, ShouldBeNil) @@ -284,11 +268,11 @@ func RunCheckAllBlobsIntegrityTests( //nolint: thelper Convey("Layer not found", func() { // get content of layer - content, err := imgStore.GetBlobContent(repoName, layerDigest) + content, err := imgStore.GetBlobContent(repoName, image.Manifest.Layers[0].Digest) So(err, ShouldBeNil) // change layer file permissions - layerDig := layerDigest.Encoded() + layerDig := image.Manifest.Layers[0].Digest.Encoded() repoDir := path.Join(imgStore.RootDir(), repoName) layerFile := path.Join(repoDir, "/blobs/sha256", layerDig) err = driver.Delete(layerFile) @@ -323,32 +307,16 @@ func RunCheckAllBlobsIntegrityTests( //nolint: thelper }) Convey("Scrub index", func() { - newConfig, newLayers, newManifest, err := deprecated.GetImageComponents(10) //nolint:staticcheck - So(err, ShouldBeNil) + newImage := CreateRandomImage() + newManifestDigest := newImage.ManifestDescriptor.Digest - newLayerReader := bytes.NewReader(newLayers[0]) - newLayerDigest := godigest.FromBytes(newLayers[0]) - _, _, err = imgStore.FullBlobUpload(repoName, newLayerReader, newLayerDigest) - So(err, ShouldBeNil) - - newConfigBlob, err := json.Marshal(newConfig) - So(err, ShouldBeNil) - newConfigReader := bytes.NewReader(newConfigBlob) - newConfigDigest := godigest.FromBytes(newConfigBlob) - _, _, err = imgStore.FullBlobUpload(repoName, newConfigReader, newConfigDigest) - So(err, ShouldBeNil) - - newManifestBlob, err := json.Marshal(newManifest) - So(err, ShouldBeNil) - newManifestReader := bytes.NewReader(newManifestBlob) - newManifestDigest := godigest.FromBytes(newManifestBlob) - _, _, err = imgStore.FullBlobUpload(repoName, newManifestReader, newManifestDigest) + err = WriteImageToFileSystem(newImage, repoName, "2.0", storeCtlr) So(err, ShouldBeNil) idx, err := common.GetIndex(imgStore, repoName, log) So(err, ShouldBeNil) - manifestDescriptor, ok := common.GetManifestDescByReference(idx, manifestDigest.String()) + manifestDescriptor, ok := common.GetManifestDescByReference(idx, image.ManifestDescriptor.Digest.String()) So(ok, ShouldBeTrue) var index ispec.Index @@ -358,7 +326,7 @@ func RunCheckAllBlobsIntegrityTests( //nolint: thelper { MediaType: ispec.MediaTypeImageManifest, Digest: newManifestDigest, - Size: int64(len(newManifestBlob)), + Size: newImage.ManifestDescriptor.Size, }, } @@ -472,7 +440,7 @@ func RunCheckAllBlobsIntegrityTests( //nolint: thelper Convey("Manifest not found", func() { // delete manifest file - manifestDig := manifestDigest.Encoded() + manifestDig := image.ManifestDescriptor.Digest.Encoded() manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", manifestDig) err = driver.Delete(manifestFile) So(err, ShouldBeNil) @@ -504,7 +472,7 @@ func RunCheckAllBlobsIntegrityTests( //nolint: thelper index, err := common.GetIndex(imgStore, repoName, log) So(err, ShouldBeNil) - manifestDescriptor, ok := common.GetManifestDescByReference(index, manifestDigest.String()) + manifestDescriptor, ok := common.GetManifestDescByReference(index, image.ManifestDescriptor.Digest.String()) So(ok, ShouldBeTrue) err = WriteImageToFileSystem(CreateDefaultImageWith().Subject(&manifestDescriptor).Build(), @@ -557,5 +525,42 @@ func RunCheckAllBlobsIntegrityTests( //nolint: thelper _, err = storeController.CheckAllBlobsIntegrity(context.Background()) So(err, ShouldNotBeNil) So(err, ShouldEqual, errUnexpectedError) + + manifestDigest := godigest.FromString("abcd") + + mockedImgStore = mocks.MockedImageStore{ + GetRepositoriesFn: func() ([]string, error) { + return []string{repoName}, nil + }, + GetIndexContentFn: func(repo string) ([]byte, error) { + var index ispec.Index + index.SchemaVersion = 2 + index.Manifests = []ispec.Descriptor{ + { + MediaType: "InvalidMediaType", + Digest: manifestDigest, + Size: int64(100), + Annotations: map[string]string{ispec.AnnotationRefName: "1.0"}, + }, + } + + return json.Marshal(index) + }, + } + + storeController.DefaultStore = mockedImgStore + + res, err := storeController.CheckAllBlobsIntegrity(context.Background()) + So(err, ShouldBeNil) + + buff := bytes.NewBufferString("") + res.PrintScrubResults(buff) + + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS AFFECTED BLOB ERROR") + So(actual, ShouldContainSubstring, fmt.Sprintf("%s 1.0 affected %s invalid manifest content", + repoName, manifestDigest.Encoded())) }) } diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 6b4947de..5e8ae280 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -171,6 +171,15 @@ func TestStorageAPIs(t *testing.T) { Convey("Repo layout", t, func(c C) { repoName := "test" + Convey("Get all blobs from repo without initialization", func() { + allBlobs, err := imgStore.GetAllBlobs(repoName) + So(err, ShouldBeNil) + So(allBlobs, ShouldBeEmpty) + + ok := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(ok, ShouldBeFalse) + }) + Convey("Validate repo without initialization", func() { v, err := imgStore.ValidateRepo(repoName) So(v, ShouldEqual, false) diff --git a/pkg/test/deprecated/deprecated.go b/pkg/test/deprecated/deprecated.go index 4c6ed0ad..7a79ba6b 100644 --- a/pkg/test/deprecated/deprecated.go +++ b/pkg/test/deprecated/deprecated.go @@ -121,36 +121,6 @@ func GetRandomLayer(size int) []byte { return layer } -// Deprecated: Should use the new functions starting with "Create". -func GetVulnImageWithConfig(config ispec.Image) (image.Image, error) { - vulnerableLayer, err := image.GetLayerWithVulnerability() - if err != nil { - return image.Image{}, err - } - - vulnerableConfig := ispec.Image{ - Platform: config.Platform, - Config: config.Config, - RootFS: ispec.RootFS{ - Type: "layers", - DiffIDs: []godigest.Digest{"sha256:f1417ff83b319fbdae6dd9cd6d8c9c88002dcd75ecf6ec201c8c6894681cf2b5"}, - }, - Created: config.Created, - History: config.History, - } - - img, err := GetImageWithComponents( - vulnerableConfig, - [][]byte{ - vulnerableLayer, - }) - if err != nil { - return image.Image{}, err - } - - return img, err -} - // Deprecated: Should use the new functions starting with "Create". func GetRandomImage() (image.Image, error) { const layerSize = 20