mirror of
https://github.com/project-zot/zot.git
synced 2024-12-16 21:56:37 -05:00
feat(scrub): add scrub logic for ImageIndex media type (#1031)
Signed-off-by: Andreea-Lupu <andreealupu1470@yahoo.com>
This commit is contained in:
Andreea Lupu
GitHub
parent
8fb11180d4
commit
ec05137eda
2 changed files with 249 additions and 132 deletions
|
|
@ -115,67 +115,103 @@ func CheckRepo(imageName string, imgStore ImageStore) ([]ScrubImageResult, error
|
|||
return results, errors.ErrRepoNotFound
|
||||
}
|
||||
|
||||
for _, m := range index.Manifests {
|
||||
tag, ok := m.Annotations[ispec.AnnotationRefName]
|
||||
if ok {
|
||||
imageResult := checkIntegrity(ctxUmoci, imageName, tag, oci, m, dir)
|
||||
results = append(results, imageResult)
|
||||
listOfManifests := []ispec.Descriptor{}
|
||||
|
||||
for _, manifest := range index.Manifests {
|
||||
if manifest.MediaType == ispec.MediaTypeImageIndex {
|
||||
buf, err := os.ReadFile(path.Join(dir, "blobs", manifest.Digest.Algorithm().String(), manifest.Digest.Encoded()))
|
||||
if err != nil {
|
||||
tagName := manifest.Annotations[ispec.AnnotationRefName]
|
||||
imgRes := getResult(imageName, tagName, errors.ErrBadBlobDigest)
|
||||
results = append(results, imgRes)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
var idx ispec.Index
|
||||
if err := json.Unmarshal(buf, &idx); err != nil {
|
||||
tagName := manifest.Annotations[ispec.AnnotationRefName]
|
||||
imgRes := getResult(imageName, tagName, errors.ErrBadBlobDigest)
|
||||
results = append(results, imgRes)
|
||||
|
||||
continue
|
||||
}
|
||||
|
||||
listOfManifests = append(listOfManifests, idx.Manifests...)
|
||||
} else if manifest.MediaType == ispec.MediaTypeImageManifest {
|
||||
listOfManifests = append(listOfManifests, manifest)
|
||||
}
|
||||
}
|
||||
|
||||
for _, m := range listOfManifests {
|
||||
tag := m.Annotations[ispec.AnnotationRefName]
|
||||
imageResult := CheckIntegrity(ctxUmoci, imageName, tag, oci, m, dir)
|
||||
results = append(results, imageResult)
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func checkIntegrity(ctx context.Context, imageName, tagName string, oci casext.Engine, manifest ispec.Descriptor, dir string) ScrubImageResult { //nolint: lll
|
||||
func CheckIntegrity(ctx context.Context, imageName, tagName string, oci casext.Engine, manifest ispec.Descriptor, dir string) ScrubImageResult { //nolint: lll
|
||||
// check manifest and config
|
||||
stat, err := umoci.Stat(ctx, oci, manifest)
|
||||
if _, err := umoci.Stat(ctx, oci, manifest); err != nil {
|
||||
return getResult(imageName, tagName, err)
|
||||
}
|
||||
|
||||
// check layers
|
||||
return CheckLayers(imageName, tagName, dir, manifest)
|
||||
}
|
||||
|
||||
func CheckLayers(imageName, tagName, dir string, manifest ispec.Descriptor) ScrubImageResult {
|
||||
imageRes := ScrubImageResult{}
|
||||
|
||||
buf, err := os.ReadFile(path.Join(dir, "blobs", manifest.Digest.Algorithm().String(), manifest.Digest.Encoded()))
|
||||
if err != nil {
|
||||
imageRes = getResult(imageName, tagName, err)
|
||||
} else {
|
||||
// check layers
|
||||
for _, s := range stat.History {
|
||||
layer := s.Layer
|
||||
if layer == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// check layer
|
||||
layerPath := path.Join(dir, "blobs", layer.Digest.Algorithm().String(), layer.Digest.Encoded())
|
||||
return imageRes
|
||||
}
|
||||
|
||||
_, err = os.Stat(layerPath)
|
||||
if err != nil {
|
||||
imageRes = getResult(imageName, tagName, errors.ErrBlobNotFound)
|
||||
var man ispec.Manifest
|
||||
if err := json.Unmarshal(buf, &man); err != nil {
|
||||
imageRes = getResult(imageName, tagName, err)
|
||||
|
||||
break
|
||||
}
|
||||
return imageRes
|
||||
}
|
||||
|
||||
layerFh, err := os.Open(layerPath)
|
||||
if err != nil {
|
||||
imageRes = getResult(imageName, tagName, errors.ErrBlobNotFound)
|
||||
for _, layer := range man.Layers {
|
||||
layerPath := path.Join(dir, "blobs", layer.Digest.Algorithm().String(), layer.Digest.Encoded())
|
||||
|
||||
break
|
||||
}
|
||||
_, err = os.Stat(layerPath)
|
||||
if err != nil {
|
||||
imageRes = getResult(imageName, tagName, errors.ErrBlobNotFound)
|
||||
|
||||
computedDigest, err := godigest.FromReader(layerFh)
|
||||
layerFh.Close()
|
||||
|
||||
if err != nil {
|
||||
imageRes = getResult(imageName, tagName, errors.ErrBadBlobDigest)
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
if computedDigest != layer.Digest {
|
||||
imageRes = getResult(imageName, tagName, errors.ErrBadBlobDigest)
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
imageRes = getResult(imageName, tagName, nil)
|
||||
break
|
||||
}
|
||||
|
||||
layerFh, err := os.Open(layerPath)
|
||||
if err != nil {
|
||||
imageRes = getResult(imageName, tagName, errors.ErrBlobNotFound)
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
computedDigest, err := godigest.FromReader(layerFh)
|
||||
layerFh.Close()
|
||||
|
||||
if err != nil {
|
||||
imageRes = getResult(imageName, tagName, errors.ErrBadBlobDigest)
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
if computedDigest != layer.Digest {
|
||||
imageRes = getResult(imageName, tagName, errors.ErrBadBlobDigest)
|
||||
|
||||
break
|
||||
}
|
||||
|
||||
imageRes = getResult(imageName, tagName, nil)
|
||||
}
|
||||
|
||||
return imageRes
|
||||
|
|
|
|||
|
|
@ -3,13 +3,11 @@ package storage_test
|
|||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
|
|
@ -20,10 +18,12 @@ import (
|
|||
"zotregistry.io/zot/pkg/storage"
|
||||
"zotregistry.io/zot/pkg/storage/cache"
|
||||
"zotregistry.io/zot/pkg/storage/local"
|
||||
"zotregistry.io/zot/pkg/test"
|
||||
)
|
||||
|
||||
const (
|
||||
repoName = "test"
|
||||
tag = "1.0"
|
||||
)
|
||||
|
||||
func TestCheckAllBlobsIntegrity(t *testing.T) {
|
||||
|
|
@ -53,89 +53,25 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
|
|||
storeCtlr := storage.StoreController{}
|
||||
storeCtlr.DefaultStore = imgStore
|
||||
|
||||
const tag = "1.0"
|
||||
|
||||
var manifestDigest godigest.Digest
|
||||
var configDigest godigest.Digest
|
||||
var layerDigest godigest.Digest
|
||||
var manifest string
|
||||
var config string
|
||||
var layer string
|
||||
|
||||
// create layer digest
|
||||
body := []byte("this is a blob")
|
||||
buf := bytes.NewBuffer(body)
|
||||
buflen := buf.Len()
|
||||
digest := godigest.FromBytes(body)
|
||||
upload, n, err := imgStore.FullBlobUpload(repoName, buf, digest)
|
||||
So(err, ShouldBeNil)
|
||||
So(n, ShouldEqual, len(body))
|
||||
So(upload, ShouldNotBeEmpty)
|
||||
layerDigest = digest
|
||||
layer = digest.String()
|
||||
|
||||
// create config digest
|
||||
created := time.Now().Format("2006-01-02T15:04:05Z")
|
||||
configBody := []byte(fmt.Sprintf(`{
|
||||
"created": "%v",
|
||||
"architecture": "amd64",
|
||||
"os": "linux",
|
||||
"rootfs": {
|
||||
"type": "layers",
|
||||
"diff_ids": [
|
||||
"",
|
||||
""
|
||||
]
|
||||
},
|
||||
"history": [
|
||||
{
|
||||
"created": "%v",
|
||||
"created_by": ""
|
||||
},
|
||||
{
|
||||
"created": "%v",
|
||||
"created_by": "",
|
||||
"empty_layer": true
|
||||
}
|
||||
]
|
||||
}`, created, created, created))
|
||||
configBuf := bytes.NewBuffer(configBody)
|
||||
configLen := configBuf.Len()
|
||||
configDigest = godigest.FromBytes(configBody)
|
||||
uConfig, nConfig, err := imgStore.FullBlobUpload(repoName, configBuf, configDigest)
|
||||
So(err, ShouldBeNil)
|
||||
So(nConfig, ShouldEqual, len(configBody))
|
||||
So(uConfig, ShouldNotBeEmpty)
|
||||
config = configDigest.String()
|
||||
|
||||
// create manifest and add it to the repository
|
||||
annotationsMap := make(map[string]string)
|
||||
annotationsMap[ispec.AnnotationRefName] = tag
|
||||
mnfst := ispec.Manifest{
|
||||
Config: ispec.Descriptor{
|
||||
MediaType: "application/vnd.oci.image.config.v1+json",
|
||||
Digest: configDigest,
|
||||
Size: int64(configLen),
|
||||
},
|
||||
Layers: []ispec.Descriptor{
|
||||
{
|
||||
MediaType: "application/vnd.oci.image.layer.v1.tar",
|
||||
Digest: digest,
|
||||
Size: int64(buflen),
|
||||
},
|
||||
},
|
||||
Annotations: annotationsMap,
|
||||
}
|
||||
|
||||
mnfst.SchemaVersion = 2
|
||||
mbytes, err := json.Marshal(mnfst)
|
||||
config, layers, manifest, err := test.GetImageComponents(1000)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestDigest, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest,
|
||||
mbytes)
|
||||
layerReader := bytes.NewReader(layers[0])
|
||||
layerDigest := godigest.FromBytes(layers[0])
|
||||
_, _, err = imgStore.FullBlobUpload(repoName, layerReader, layerDigest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifest = manifestDigest.String()
|
||||
configBlob, err := json.Marshal(config)
|
||||
So(err, ShouldBeNil)
|
||||
configReader := bytes.NewReader(configBlob)
|
||||
configDigest := godigest.FromBytes(configBlob)
|
||||
_, _, err = imgStore.FullBlobUpload(repoName, configReader, configDigest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
manifestBlob, err := json.Marshal(manifest)
|
||||
So(err, ShouldBeNil)
|
||||
manifestDigest, err := imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBlob)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
Convey("Blobs integrity not affected", func() {
|
||||
buff := bytes.NewBufferString("")
|
||||
|
|
@ -153,12 +89,12 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
|
|||
|
||||
Convey("Manifest integrity affected", func() {
|
||||
// get content of manifest file
|
||||
content, _, _, err := imgStore.GetImageManifest(repoName, manifest)
|
||||
content, _, _, err := imgStore.GetImageManifest(repoName, manifestDigest.String())
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// delete content of manifest file
|
||||
manifest = strings.ReplaceAll(manifest, "sha256:", "")
|
||||
manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", manifest)
|
||||
manifestDig := manifestDigest.Encoded()
|
||||
manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", manifestDig)
|
||||
err = os.Truncate(manifestFile, 0)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
|
|
@ -175,6 +111,17 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
|
|||
// verify error message
|
||||
So(actual, ShouldContainSubstring, "test 1.0 affected parse application/vnd.oci.image.manifest.v1+json")
|
||||
|
||||
index, err := storage.GetIndex(imgStore, repoName, log.With().Caller().Logger())
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(len(index.Manifests), ShouldEqual, 1)
|
||||
manifestDescriptor := index.Manifests[0]
|
||||
|
||||
repoDir := path.Join(dir, repoName)
|
||||
imageRes := storage.CheckLayers(repoName, tag, repoDir, manifestDescriptor)
|
||||
So(imageRes.Status, ShouldEqual, "affected")
|
||||
So(imageRes.Error, ShouldEqual, "unexpected end of JSON input")
|
||||
|
||||
// put manifest content back to file
|
||||
err = os.WriteFile(manifestFile, content, 0o600)
|
||||
So(err, ShouldBeNil)
|
||||
|
|
@ -186,8 +133,8 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
|
|||
So(err, ShouldBeNil)
|
||||
|
||||
// delete content of config file
|
||||
config = strings.ReplaceAll(config, "sha256:", "")
|
||||
configFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", config)
|
||||
configDig := configDigest.Encoded()
|
||||
configFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", configDig)
|
||||
err = os.Truncate(configFile, 0)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
|
|
@ -214,8 +161,8 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
|
|||
So(err, ShouldBeNil)
|
||||
|
||||
// delete content of layer file
|
||||
layer = strings.ReplaceAll(layer, "sha256:", "")
|
||||
layerFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", layer)
|
||||
layerDig := layerDigest.Encoded()
|
||||
layerFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", layerDig)
|
||||
err = os.Truncate(layerFile, 0)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
|
|
@ -237,9 +184,26 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
|
|||
})
|
||||
|
||||
Convey("Layer not found", func() {
|
||||
// change layer file permissions
|
||||
layerDig := layerDigest.Encoded()
|
||||
repoDir := path.Join(dir, repoName)
|
||||
layerFile := path.Join(repoDir, "/blobs/sha256", layerDig)
|
||||
err = os.Chmod(layerFile, 0x0200)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
index, err := storage.GetIndex(imgStore, repoName, log.With().Caller().Logger())
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(len(index.Manifests), ShouldEqual, 1)
|
||||
manifestDescriptor := index.Manifests[0]
|
||||
|
||||
imageRes := storage.CheckLayers(repoName, tag, repoDir, manifestDescriptor)
|
||||
So(imageRes.Status, ShouldEqual, "affected")
|
||||
So(imageRes.Error, ShouldEqual, "blob: not found")
|
||||
err = os.Chmod(layerFile, 0x0600)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// delete layer file
|
||||
layer = strings.ReplaceAll(layer, "sha256:", "")
|
||||
layerFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", layer)
|
||||
err = os.Remove(layerFile)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
|
|
@ -255,5 +219,122 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
|
|||
So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR")
|
||||
So(actual, ShouldContainSubstring, "test 1.0 affected blob: not found")
|
||||
})
|
||||
|
||||
Convey("Scrub index", func() {
|
||||
newConfig, newLayers, newManifest, err := test.GetImageComponents(10)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
newLayerReader := bytes.NewReader(newLayers[0])
|
||||
newLayerDigest := godigest.FromBytes(newLayers[0])
|
||||
_, _, err = imgStore.FullBlobUpload(repoName, newLayerReader, newLayerDigest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
newConfigBlob, err := json.Marshal(newConfig)
|
||||
So(err, ShouldBeNil)
|
||||
newConfigReader := bytes.NewReader(newConfigBlob)
|
||||
newConfigDigest := godigest.FromBytes(newConfigBlob)
|
||||
_, _, err = imgStore.FullBlobUpload(repoName, newConfigReader, newConfigDigest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
newManifestBlob, err := json.Marshal(newManifest)
|
||||
So(err, ShouldBeNil)
|
||||
newManifestReader := bytes.NewReader(newManifestBlob)
|
||||
newManifestDigest := godigest.FromBytes(newManifestBlob)
|
||||
_, _, err = imgStore.FullBlobUpload(repoName, newManifestReader, newManifestDigest)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
var index ispec.Index
|
||||
index.SchemaVersion = 2
|
||||
index.Manifests = []ispec.Descriptor{
|
||||
{
|
||||
MediaType: ispec.MediaTypeImageManifest,
|
||||
Digest: newManifestDigest,
|
||||
Size: int64(len(newManifestBlob)),
|
||||
},
|
||||
}
|
||||
|
||||
indexBlob, err := json.Marshal(index)
|
||||
So(err, ShouldBeNil)
|
||||
indexDigest, err := imgStore.PutImageManifest(repoName, "", ispec.MediaTypeImageIndex, indexBlob)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
buff := bytes.NewBufferString("")
|
||||
|
||||
res, err := storeCtlr.CheckAllBlobsIntegrity()
|
||||
res.PrintScrubResults(buff)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
space := regexp.MustCompile(`\s+`)
|
||||
str := space.ReplaceAllString(buff.String(), " ")
|
||||
actual := strings.TrimSpace(str)
|
||||
So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR")
|
||||
So(actual, ShouldContainSubstring, "test 1.0 ok")
|
||||
So(actual, ShouldContainSubstring, "test ok")
|
||||
|
||||
// test scrub index - errors
|
||||
indexFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", indexDigest.Encoded())
|
||||
err = os.Chmod(indexFile, 0o000)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
buff = bytes.NewBufferString("")
|
||||
|
||||
res, err = storeCtlr.CheckAllBlobsIntegrity()
|
||||
res.PrintScrubResults(buff)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
str = space.ReplaceAllString(buff.String(), " ")
|
||||
actual = strings.TrimSpace(str)
|
||||
So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR")
|
||||
So(actual, ShouldContainSubstring, "test affected")
|
||||
|
||||
err = os.Chmod(indexFile, 0o600)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
err = os.Truncate(indexFile, 0)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
buff = bytes.NewBufferString("")
|
||||
|
||||
res, err = storeCtlr.CheckAllBlobsIntegrity()
|
||||
res.PrintScrubResults(buff)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
str = space.ReplaceAllString(buff.String(), " ")
|
||||
actual = strings.TrimSpace(str)
|
||||
So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR")
|
||||
So(actual, ShouldContainSubstring, "test affected")
|
||||
})
|
||||
|
||||
Convey("Manifest not found", func() {
|
||||
// delete manifest file
|
||||
manifestDig := manifestDigest.Encoded()
|
||||
manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", manifestDig)
|
||||
err = os.Remove(manifestFile)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
buff := bytes.NewBufferString("")
|
||||
|
||||
res, err := storeCtlr.CheckAllBlobsIntegrity()
|
||||
res.PrintScrubResults(buff)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
space := regexp.MustCompile(`\s+`)
|
||||
str := space.ReplaceAllString(buff.String(), " ")
|
||||
actual := strings.TrimSpace(str)
|
||||
So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR")
|
||||
So(actual, ShouldContainSubstring, "test 1.0 affected")
|
||||
So(actual, ShouldContainSubstring, "no such file or directory")
|
||||
|
||||
index, err := storage.GetIndex(imgStore, repoName, log.With().Caller().Logger())
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(len(index.Manifests), ShouldEqual, 2)
|
||||
manifestDescriptor := index.Manifests[0]
|
||||
|
||||
repoDir := path.Join(dir, repoName)
|
||||
imageRes := storage.CheckLayers(repoName, tag, repoDir, manifestDescriptor)
|
||||
So(imageRes.Status, ShouldEqual, "affected")
|
||||
So(imageRes.Error, ShouldContainSubstring, "no such file or directory")
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue