mirror of
https://github.com/project-zot/zot.git
synced 2024-12-16 21:56:37 -05:00
b80deb9927
unified both local and s3 ImageStore logic into a single ImageStore added a new driver interface for common file/dirs manipulations to be implemented by different storage types refactor(gc): drop umoci dependency, implemented internal gc added retentionDelay config option that specifies the garbage collect delay for images without tags this will also clean manifests which are part of an index image (multiarch) that no longer exist. fix(dedupe): skip blobs under .sync/ directory if startup dedupe is running while also syncing is running ignore blobs under sync's temporary storage fix(storage): do not allow image indexes modifications when deleting a manifest verify that it is not part of a multiarch image and throw a MethodNotAllowed error to the client if it is. we don't want to modify multiarch images Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
342 lines
11 KiB
Go
342 lines
11 KiB
Go
package storage_test
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"os"
|
|
"path"
|
|
"regexp"
|
|
"strings"
|
|
"testing"
|
|
|
|
godigest "github.com/opencontainers/go-digest"
|
|
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
|
. "github.com/smartystreets/goconvey/convey"
|
|
|
|
"zotregistry.io/zot/pkg/extensions/monitoring"
|
|
"zotregistry.io/zot/pkg/log"
|
|
"zotregistry.io/zot/pkg/storage"
|
|
"zotregistry.io/zot/pkg/storage/cache"
|
|
common "zotregistry.io/zot/pkg/storage/common"
|
|
storageConstants "zotregistry.io/zot/pkg/storage/constants"
|
|
"zotregistry.io/zot/pkg/storage/local"
|
|
"zotregistry.io/zot/pkg/test"
|
|
)
|
|
|
|
const (
|
|
repoName = "test"
|
|
tag = "1.0"
|
|
)
|
|
|
|
func TestCheckAllBlobsIntegrity(t *testing.T) {
|
|
dir := t.TempDir()
|
|
|
|
log := log.NewLogger("debug", "")
|
|
|
|
metrics := monitoring.NewMetricsServer(false, log)
|
|
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
|
|
RootDir: dir,
|
|
Name: "cache",
|
|
UseRelPaths: true,
|
|
}, log)
|
|
imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay,
|
|
storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver)
|
|
|
|
Convey("Scrub only one repo", t, func(c C) {
|
|
// initialize repo
|
|
err := imgStore.InitRepo(repoName)
|
|
So(err, ShouldBeNil)
|
|
ok := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName))
|
|
So(ok, ShouldBeTrue)
|
|
storeController := storage.StoreController{}
|
|
storeController.DefaultStore = imgStore
|
|
So(storeController.GetImageStore(repoName), ShouldResemble, imgStore)
|
|
|
|
storeCtlr := storage.StoreController{}
|
|
storeCtlr.DefaultStore = imgStore
|
|
|
|
config, layers, manifest, err := test.GetImageComponents(1000) //nolint:staticcheck
|
|
So(err, ShouldBeNil)
|
|
|
|
layerReader := bytes.NewReader(layers[0])
|
|
layerDigest := godigest.FromBytes(layers[0])
|
|
_, _, err = imgStore.FullBlobUpload(repoName, layerReader, layerDigest)
|
|
So(err, ShouldBeNil)
|
|
|
|
configBlob, err := json.Marshal(config)
|
|
So(err, ShouldBeNil)
|
|
configReader := bytes.NewReader(configBlob)
|
|
configDigest := godigest.FromBytes(configBlob)
|
|
_, _, err = imgStore.FullBlobUpload(repoName, configReader, configDigest)
|
|
So(err, ShouldBeNil)
|
|
|
|
manifestBlob, err := json.Marshal(manifest)
|
|
So(err, ShouldBeNil)
|
|
manifestDigest, _, err := imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBlob)
|
|
So(err, ShouldBeNil)
|
|
|
|
Convey("Blobs integrity not affected", func() {
|
|
buff := bytes.NewBufferString("")
|
|
|
|
res, err := storeCtlr.CheckAllBlobsIntegrity()
|
|
res.PrintScrubResults(buff)
|
|
So(err, ShouldBeNil)
|
|
|
|
space := regexp.MustCompile(`\s+`)
|
|
str := space.ReplaceAllString(buff.String(), " ")
|
|
actual := strings.TrimSpace(str)
|
|
So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS ERROR")
|
|
So(actual, ShouldContainSubstring, "test 1.0 ok")
|
|
})
|
|
|
|
Convey("Manifest integrity affected", func() {
|
|
// get content of manifest file
|
|
content, _, _, err := imgStore.GetImageManifest(repoName, manifestDigest.String())
|
|
So(err, ShouldBeNil)
|
|
|
|
// delete content of manifest file
|
|
manifestDig := manifestDigest.Encoded()
|
|
manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", manifestDig)
|
|
err = os.Truncate(manifestFile, 0)
|
|
So(err, ShouldBeNil)
|
|
|
|
buff := bytes.NewBufferString("")
|
|
|
|
res, err := storeCtlr.CheckAllBlobsIntegrity()
|
|
res.PrintScrubResults(buff)
|
|
So(err, ShouldBeNil)
|
|
|
|
space := regexp.MustCompile(`\s+`)
|
|
str := space.ReplaceAllString(buff.String(), " ")
|
|
actual := strings.TrimSpace(str)
|
|
So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS ERROR")
|
|
// verify error message
|
|
So(actual, ShouldContainSubstring, "test 1.0 affected parse application/vnd.oci.image.manifest.v1+json")
|
|
|
|
index, err := common.GetIndex(imgStore, repoName, log)
|
|
So(err, ShouldBeNil)
|
|
|
|
So(len(index.Manifests), ShouldEqual, 1)
|
|
manifestDescriptor := index.Manifests[0]
|
|
|
|
repoDir := path.Join(dir, repoName)
|
|
imageRes := storage.CheckLayers(repoName, tag, repoDir, manifestDescriptor)
|
|
So(imageRes.Status, ShouldEqual, "affected")
|
|
So(imageRes.Error, ShouldEqual, "unexpected end of JSON input")
|
|
|
|
// put manifest content back to file
|
|
err = os.WriteFile(manifestFile, content, 0o600)
|
|
So(err, ShouldBeNil)
|
|
})
|
|
|
|
Convey("Config integrity affected", func() {
|
|
// get content of config file
|
|
content, err := imgStore.GetBlobContent(repoName, configDigest)
|
|
So(err, ShouldBeNil)
|
|
|
|
// delete content of config file
|
|
configDig := configDigest.Encoded()
|
|
configFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", configDig)
|
|
err = os.Truncate(configFile, 0)
|
|
So(err, ShouldBeNil)
|
|
|
|
buff := bytes.NewBufferString("")
|
|
|
|
res, err := storeCtlr.CheckAllBlobsIntegrity()
|
|
res.PrintScrubResults(buff)
|
|
So(err, ShouldBeNil)
|
|
|
|
space := regexp.MustCompile(`\s+`)
|
|
str := space.ReplaceAllString(buff.String(), " ")
|
|
actual := strings.TrimSpace(str)
|
|
So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS ERROR")
|
|
So(actual, ShouldContainSubstring, "test 1.0 affected stat: parse application/vnd.oci.image.config.v1+json")
|
|
|
|
// put config content back to file
|
|
err = os.WriteFile(configFile, content, 0o600)
|
|
So(err, ShouldBeNil)
|
|
})
|
|
|
|
Convey("Layers integrity affected", func() {
|
|
// get content of layer
|
|
content, err := imgStore.GetBlobContent(repoName, layerDigest)
|
|
So(err, ShouldBeNil)
|
|
|
|
// delete content of layer file
|
|
layerDig := layerDigest.Encoded()
|
|
layerFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", layerDig)
|
|
err = os.Truncate(layerFile, 0)
|
|
So(err, ShouldBeNil)
|
|
|
|
buff := bytes.NewBufferString("")
|
|
|
|
res, err := storeCtlr.CheckAllBlobsIntegrity()
|
|
res.PrintScrubResults(buff)
|
|
So(err, ShouldBeNil)
|
|
|
|
space := regexp.MustCompile(`\s+`)
|
|
str := space.ReplaceAllString(buff.String(), " ")
|
|
actual := strings.TrimSpace(str)
|
|
So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS ERROR")
|
|
So(actual, ShouldContainSubstring, "test 1.0 affected blob: bad blob digest")
|
|
|
|
// put layer content back to file
|
|
err = os.WriteFile(layerFile, content, 0o600)
|
|
So(err, ShouldBeNil)
|
|
})
|
|
|
|
Convey("Layer not found", func() {
|
|
// change layer file permissions
|
|
layerDig := layerDigest.Encoded()
|
|
repoDir := path.Join(dir, repoName)
|
|
layerFile := path.Join(repoDir, "/blobs/sha256", layerDig)
|
|
err = os.Chmod(layerFile, 0x0200)
|
|
So(err, ShouldBeNil)
|
|
|
|
index, err := common.GetIndex(imgStore, repoName, log)
|
|
So(err, ShouldBeNil)
|
|
|
|
So(len(index.Manifests), ShouldEqual, 1)
|
|
manifestDescriptor := index.Manifests[0]
|
|
|
|
imageRes := storage.CheckLayers(repoName, tag, repoDir, manifestDescriptor)
|
|
So(imageRes.Status, ShouldEqual, "affected")
|
|
So(imageRes.Error, ShouldEqual, "blob: not found")
|
|
err = os.Chmod(layerFile, 0x0600)
|
|
So(err, ShouldBeNil)
|
|
|
|
// delete layer file
|
|
err = os.Remove(layerFile)
|
|
So(err, ShouldBeNil)
|
|
|
|
buff := bytes.NewBufferString("")
|
|
|
|
res, err := storeCtlr.CheckAllBlobsIntegrity()
|
|
res.PrintScrubResults(buff)
|
|
So(err, ShouldBeNil)
|
|
|
|
space := regexp.MustCompile(`\s+`)
|
|
str := space.ReplaceAllString(buff.String(), " ")
|
|
actual := strings.TrimSpace(str)
|
|
So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS ERROR")
|
|
So(actual, ShouldContainSubstring, "test 1.0 affected blob: not found")
|
|
})
|
|
|
|
Convey("Scrub index", func() {
|
|
newConfig, newLayers, newManifest, err := test.GetImageComponents(10) //nolint:staticcheck
|
|
So(err, ShouldBeNil)
|
|
|
|
newLayerReader := bytes.NewReader(newLayers[0])
|
|
newLayerDigest := godigest.FromBytes(newLayers[0])
|
|
_, _, err = imgStore.FullBlobUpload(repoName, newLayerReader, newLayerDigest)
|
|
So(err, ShouldBeNil)
|
|
|
|
newConfigBlob, err := json.Marshal(newConfig)
|
|
So(err, ShouldBeNil)
|
|
newConfigReader := bytes.NewReader(newConfigBlob)
|
|
newConfigDigest := godigest.FromBytes(newConfigBlob)
|
|
_, _, err = imgStore.FullBlobUpload(repoName, newConfigReader, newConfigDigest)
|
|
So(err, ShouldBeNil)
|
|
|
|
newManifestBlob, err := json.Marshal(newManifest)
|
|
So(err, ShouldBeNil)
|
|
newManifestReader := bytes.NewReader(newManifestBlob)
|
|
newManifestDigest := godigest.FromBytes(newManifestBlob)
|
|
_, _, err = imgStore.FullBlobUpload(repoName, newManifestReader, newManifestDigest)
|
|
So(err, ShouldBeNil)
|
|
|
|
var index ispec.Index
|
|
index.SchemaVersion = 2
|
|
index.Manifests = []ispec.Descriptor{
|
|
{
|
|
MediaType: ispec.MediaTypeImageManifest,
|
|
Digest: newManifestDigest,
|
|
Size: int64(len(newManifestBlob)),
|
|
},
|
|
}
|
|
|
|
indexBlob, err := json.Marshal(index)
|
|
So(err, ShouldBeNil)
|
|
indexDigest, _, err := imgStore.PutImageManifest(repoName, "", ispec.MediaTypeImageIndex, indexBlob)
|
|
So(err, ShouldBeNil)
|
|
|
|
buff := bytes.NewBufferString("")
|
|
|
|
res, err := storeCtlr.CheckAllBlobsIntegrity()
|
|
res.PrintScrubResults(buff)
|
|
So(err, ShouldBeNil)
|
|
|
|
space := regexp.MustCompile(`\s+`)
|
|
str := space.ReplaceAllString(buff.String(), " ")
|
|
actual := strings.TrimSpace(str)
|
|
So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS ERROR")
|
|
So(actual, ShouldContainSubstring, "test 1.0 ok")
|
|
So(actual, ShouldContainSubstring, "test ok")
|
|
|
|
// test scrub index - errors
|
|
indexFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", indexDigest.Encoded())
|
|
err = os.Chmod(indexFile, 0o000)
|
|
So(err, ShouldBeNil)
|
|
|
|
buff = bytes.NewBufferString("")
|
|
|
|
res, err = storeCtlr.CheckAllBlobsIntegrity()
|
|
res.PrintScrubResults(buff)
|
|
So(err, ShouldBeNil)
|
|
|
|
str = space.ReplaceAllString(buff.String(), " ")
|
|
actual = strings.TrimSpace(str)
|
|
So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS ERROR")
|
|
So(actual, ShouldContainSubstring, "test affected")
|
|
|
|
err = os.Chmod(indexFile, 0o600)
|
|
So(err, ShouldBeNil)
|
|
|
|
err = os.Truncate(indexFile, 0)
|
|
So(err, ShouldBeNil)
|
|
|
|
buff = bytes.NewBufferString("")
|
|
|
|
res, err = storeCtlr.CheckAllBlobsIntegrity()
|
|
res.PrintScrubResults(buff)
|
|
So(err, ShouldBeNil)
|
|
|
|
str = space.ReplaceAllString(buff.String(), " ")
|
|
actual = strings.TrimSpace(str)
|
|
So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS ERROR")
|
|
So(actual, ShouldContainSubstring, "test affected")
|
|
})
|
|
|
|
Convey("Manifest not found", func() {
|
|
// delete manifest file
|
|
manifestDig := manifestDigest.Encoded()
|
|
manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", manifestDig)
|
|
err = os.Remove(manifestFile)
|
|
So(err, ShouldBeNil)
|
|
|
|
buff := bytes.NewBufferString("")
|
|
|
|
res, err := storeCtlr.CheckAllBlobsIntegrity()
|
|
res.PrintScrubResults(buff)
|
|
So(err, ShouldBeNil)
|
|
|
|
space := regexp.MustCompile(`\s+`)
|
|
str := space.ReplaceAllString(buff.String(), " ")
|
|
actual := strings.TrimSpace(str)
|
|
So(actual, ShouldContainSubstring, "REPOSITORY TAG STATUS ERROR")
|
|
So(actual, ShouldContainSubstring, "test 1.0 affected")
|
|
So(actual, ShouldContainSubstring, "no such file or directory")
|
|
|
|
index, err := common.GetIndex(imgStore, repoName, log)
|
|
So(err, ShouldBeNil)
|
|
|
|
So(len(index.Manifests), ShouldEqual, 2)
|
|
manifestDescriptor := index.Manifests[0]
|
|
|
|
repoDir := path.Join(dir, repoName)
|
|
imageRes := storage.CheckLayers(repoName, tag, repoDir, manifestDescriptor)
|
|
So(imageRes.Status, ShouldEqual, "affected")
|
|
So(imageRes.Error, ShouldContainSubstring, "no such file or directory")
|
|
})
|
|
})
|
|
}
|