0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00

feat(scrub): add scrub logic for ImageIndex media type (#1031)

Signed-off-by: Andreea-Lupu <andreealupu1470@yahoo.com>
This commit is contained in:
Andreea Lupu 2022-12-11 02:38:01 +02:00 committed by GitHub
parent 8fb11180d4
commit ec05137eda
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 249 additions and 132 deletions

View file

@ -115,34 +115,71 @@ func CheckRepo(imageName string, imgStore ImageStore) ([]ScrubImageResult, error
return results, errors.ErrRepoNotFound return results, errors.ErrRepoNotFound
} }
for _, m := range index.Manifests { listOfManifests := []ispec.Descriptor{}
tag, ok := m.Annotations[ispec.AnnotationRefName]
if ok { for _, manifest := range index.Manifests {
imageResult := checkIntegrity(ctxUmoci, imageName, tag, oci, m, dir) if manifest.MediaType == ispec.MediaTypeImageIndex {
results = append(results, imageResult) buf, err := os.ReadFile(path.Join(dir, "blobs", manifest.Digest.Algorithm().String(), manifest.Digest.Encoded()))
if err != nil {
tagName := manifest.Annotations[ispec.AnnotationRefName]
imgRes := getResult(imageName, tagName, errors.ErrBadBlobDigest)
results = append(results, imgRes)
continue
} }
var idx ispec.Index
if err := json.Unmarshal(buf, &idx); err != nil {
tagName := manifest.Annotations[ispec.AnnotationRefName]
imgRes := getResult(imageName, tagName, errors.ErrBadBlobDigest)
results = append(results, imgRes)
continue
}
listOfManifests = append(listOfManifests, idx.Manifests...)
} else if manifest.MediaType == ispec.MediaTypeImageManifest {
listOfManifests = append(listOfManifests, manifest)
}
}
for _, m := range listOfManifests {
tag := m.Annotations[ispec.AnnotationRefName]
imageResult := CheckIntegrity(ctxUmoci, imageName, tag, oci, m, dir)
results = append(results, imageResult)
} }
return results, nil return results, nil
} }
func checkIntegrity(ctx context.Context, imageName, tagName string, oci casext.Engine, manifest ispec.Descriptor, dir string) ScrubImageResult { //nolint: lll func CheckIntegrity(ctx context.Context, imageName, tagName string, oci casext.Engine, manifest ispec.Descriptor, dir string) ScrubImageResult { //nolint: lll
// check manifest and config // check manifest and config
stat, err := umoci.Stat(ctx, oci, manifest) if _, err := umoci.Stat(ctx, oci, manifest); err != nil {
return getResult(imageName, tagName, err)
imageRes := ScrubImageResult{}
if err != nil {
imageRes = getResult(imageName, tagName, err)
} else {
// check layers
for _, s := range stat.History {
layer := s.Layer
if layer == nil {
continue
} }
// check layer // check layers
return CheckLayers(imageName, tagName, dir, manifest)
}
func CheckLayers(imageName, tagName, dir string, manifest ispec.Descriptor) ScrubImageResult {
imageRes := ScrubImageResult{}
buf, err := os.ReadFile(path.Join(dir, "blobs", manifest.Digest.Algorithm().String(), manifest.Digest.Encoded()))
if err != nil {
imageRes = getResult(imageName, tagName, err)
return imageRes
}
var man ispec.Manifest
if err := json.Unmarshal(buf, &man); err != nil {
imageRes = getResult(imageName, tagName, err)
return imageRes
}
for _, layer := range man.Layers {
layerPath := path.Join(dir, "blobs", layer.Digest.Algorithm().String(), layer.Digest.Encoded()) layerPath := path.Join(dir, "blobs", layer.Digest.Algorithm().String(), layer.Digest.Encoded())
_, err = os.Stat(layerPath) _, err = os.Stat(layerPath)
@ -176,7 +213,6 @@ func checkIntegrity(ctx context.Context, imageName, tagName string, oci casext.E
imageRes = getResult(imageName, tagName, nil) imageRes = getResult(imageName, tagName, nil)
} }
}
return imageRes return imageRes
} }

View file

@ -3,13 +3,11 @@ package storage_test
import ( import (
"bytes" "bytes"
"encoding/json" "encoding/json"
"fmt"
"os" "os"
"path" "path"
"regexp" "regexp"
"strings" "strings"
"testing" "testing"
"time"
godigest "github.com/opencontainers/go-digest" godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1" ispec "github.com/opencontainers/image-spec/specs-go/v1"
@ -20,10 +18,12 @@ import (
"zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache" "zotregistry.io/zot/pkg/storage/cache"
"zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/test"
) )
const ( const (
repoName = "test" repoName = "test"
tag = "1.0"
) )
func TestCheckAllBlobsIntegrity(t *testing.T) { func TestCheckAllBlobsIntegrity(t *testing.T) {
@ -53,89 +53,25 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
storeCtlr := storage.StoreController{} storeCtlr := storage.StoreController{}
storeCtlr.DefaultStore = imgStore storeCtlr.DefaultStore = imgStore
const tag = "1.0" config, layers, manifest, err := test.GetImageComponents(1000)
var manifestDigest godigest.Digest
var configDigest godigest.Digest
var layerDigest godigest.Digest
var manifest string
var config string
var layer string
// create layer digest
body := []byte("this is a blob")
buf := bytes.NewBuffer(body)
buflen := buf.Len()
digest := godigest.FromBytes(body)
upload, n, err := imgStore.FullBlobUpload(repoName, buf, digest)
So(err, ShouldBeNil)
So(n, ShouldEqual, len(body))
So(upload, ShouldNotBeEmpty)
layerDigest = digest
layer = digest.String()
// create config digest
created := time.Now().Format("2006-01-02T15:04:05Z")
configBody := []byte(fmt.Sprintf(`{
"created": "%v",
"architecture": "amd64",
"os": "linux",
"rootfs": {
"type": "layers",
"diff_ids": [
"",
""
]
},
"history": [
{
"created": "%v",
"created_by": ""
},
{
"created": "%v",
"created_by": "",
"empty_layer": true
}
]
}`, created, created, created))
configBuf := bytes.NewBuffer(configBody)
configLen := configBuf.Len()
configDigest = godigest.FromBytes(configBody)
uConfig, nConfig, err := imgStore.FullBlobUpload(repoName, configBuf, configDigest)
So(err, ShouldBeNil)
So(nConfig, ShouldEqual, len(configBody))
So(uConfig, ShouldNotBeEmpty)
config = configDigest.String()
// create manifest and add it to the repository
annotationsMap := make(map[string]string)
annotationsMap[ispec.AnnotationRefName] = tag
mnfst := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: configDigest,
Size: int64(configLen),
},
Layers: []ispec.Descriptor{
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: digest,
Size: int64(buflen),
},
},
Annotations: annotationsMap,
}
mnfst.SchemaVersion = 2
mbytes, err := json.Marshal(mnfst)
So(err, ShouldBeNil) So(err, ShouldBeNil)
manifestDigest, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, layerReader := bytes.NewReader(layers[0])
mbytes) layerDigest := godigest.FromBytes(layers[0])
_, _, err = imgStore.FullBlobUpload(repoName, layerReader, layerDigest)
So(err, ShouldBeNil) So(err, ShouldBeNil)
manifest = manifestDigest.String() configBlob, err := json.Marshal(config)
So(err, ShouldBeNil)
configReader := bytes.NewReader(configBlob)
configDigest := godigest.FromBytes(configBlob)
_, _, err = imgStore.FullBlobUpload(repoName, configReader, configDigest)
So(err, ShouldBeNil)
manifestBlob, err := json.Marshal(manifest)
So(err, ShouldBeNil)
manifestDigest, err := imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBlob)
So(err, ShouldBeNil)
Convey("Blobs integrity not affected", func() { Convey("Blobs integrity not affected", func() {
buff := bytes.NewBufferString("") buff := bytes.NewBufferString("")
@ -153,12 +89,12 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
Convey("Manifest integrity affected", func() { Convey("Manifest integrity affected", func() {
// get content of manifest file // get content of manifest file
content, _, _, err := imgStore.GetImageManifest(repoName, manifest) content, _, _, err := imgStore.GetImageManifest(repoName, manifestDigest.String())
So(err, ShouldBeNil) So(err, ShouldBeNil)
// delete content of manifest file // delete content of manifest file
manifest = strings.ReplaceAll(manifest, "sha256:", "") manifestDig := manifestDigest.Encoded()
manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", manifest) manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", manifestDig)
err = os.Truncate(manifestFile, 0) err = os.Truncate(manifestFile, 0)
So(err, ShouldBeNil) So(err, ShouldBeNil)
@ -175,6 +111,17 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
// verify error message // verify error message
So(actual, ShouldContainSubstring, "test 1.0 affected parse application/vnd.oci.image.manifest.v1+json") So(actual, ShouldContainSubstring, "test 1.0 affected parse application/vnd.oci.image.manifest.v1+json")
index, err := storage.GetIndex(imgStore, repoName, log.With().Caller().Logger())
So(err, ShouldBeNil)
So(len(index.Manifests), ShouldEqual, 1)
manifestDescriptor := index.Manifests[0]
repoDir := path.Join(dir, repoName)
imageRes := storage.CheckLayers(repoName, tag, repoDir, manifestDescriptor)
So(imageRes.Status, ShouldEqual, "affected")
So(imageRes.Error, ShouldEqual, "unexpected end of JSON input")
// put manifest content back to file // put manifest content back to file
err = os.WriteFile(manifestFile, content, 0o600) err = os.WriteFile(manifestFile, content, 0o600)
So(err, ShouldBeNil) So(err, ShouldBeNil)
@ -186,8 +133,8 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
So(err, ShouldBeNil) So(err, ShouldBeNil)
// delete content of config file // delete content of config file
config = strings.ReplaceAll(config, "sha256:", "") configDig := configDigest.Encoded()
configFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", config) configFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", configDig)
err = os.Truncate(configFile, 0) err = os.Truncate(configFile, 0)
So(err, ShouldBeNil) So(err, ShouldBeNil)
@ -214,8 +161,8 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
So(err, ShouldBeNil) So(err, ShouldBeNil)
// delete content of layer file // delete content of layer file
layer = strings.ReplaceAll(layer, "sha256:", "") layerDig := layerDigest.Encoded()
layerFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", layer) layerFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", layerDig)
err = os.Truncate(layerFile, 0) err = os.Truncate(layerFile, 0)
So(err, ShouldBeNil) So(err, ShouldBeNil)
@ -237,9 +184,26 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
}) })
Convey("Layer not found", func() { Convey("Layer not found", func() {
// change layer file permissions
layerDig := layerDigest.Encoded()
repoDir := path.Join(dir, repoName)
layerFile := path.Join(repoDir, "/blobs/sha256", layerDig)
err = os.Chmod(layerFile, 0x0200)
So(err, ShouldBeNil)
index, err := storage.GetIndex(imgStore, repoName, log.With().Caller().Logger())
So(err, ShouldBeNil)
So(len(index.Manifests), ShouldEqual, 1)
manifestDescriptor := index.Manifests[0]
imageRes := storage.CheckLayers(repoName, tag, repoDir, manifestDescriptor)
So(imageRes.Status, ShouldEqual, "affected")
So(imageRes.Error, ShouldEqual, "blob: not found")
err = os.Chmod(layerFile, 0x0600)
So(err, ShouldBeNil)
// delete layer file // delete layer file
layer = strings.ReplaceAll(layer, "sha256:", "")
layerFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", layer)
err = os.Remove(layerFile) err = os.Remove(layerFile)
So(err, ShouldBeNil) So(err, ShouldBeNil)
@ -255,5 +219,122 @@ func TestCheckAllBlobsIntegrity(t *testing.T) {
So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR") So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR")
So(actual, ShouldContainSubstring, "test 1.0 affected blob: not found") So(actual, ShouldContainSubstring, "test 1.0 affected blob: not found")
}) })
Convey("Scrub index", func() {
newConfig, newLayers, newManifest, err := test.GetImageComponents(10)
So(err, ShouldBeNil)
newLayerReader := bytes.NewReader(newLayers[0])
newLayerDigest := godigest.FromBytes(newLayers[0])
_, _, err = imgStore.FullBlobUpload(repoName, newLayerReader, newLayerDigest)
So(err, ShouldBeNil)
newConfigBlob, err := json.Marshal(newConfig)
So(err, ShouldBeNil)
newConfigReader := bytes.NewReader(newConfigBlob)
newConfigDigest := godigest.FromBytes(newConfigBlob)
_, _, err = imgStore.FullBlobUpload(repoName, newConfigReader, newConfigDigest)
So(err, ShouldBeNil)
newManifestBlob, err := json.Marshal(newManifest)
So(err, ShouldBeNil)
newManifestReader := bytes.NewReader(newManifestBlob)
newManifestDigest := godigest.FromBytes(newManifestBlob)
_, _, err = imgStore.FullBlobUpload(repoName, newManifestReader, newManifestDigest)
So(err, ShouldBeNil)
var index ispec.Index
index.SchemaVersion = 2
index.Manifests = []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageManifest,
Digest: newManifestDigest,
Size: int64(len(newManifestBlob)),
},
}
indexBlob, err := json.Marshal(index)
So(err, ShouldBeNil)
indexDigest, err := imgStore.PutImageManifest(repoName, "", ispec.MediaTypeImageIndex, indexBlob)
So(err, ShouldBeNil)
buff := bytes.NewBufferString("")
res, err := storeCtlr.CheckAllBlobsIntegrity()
res.PrintScrubResults(buff)
So(err, ShouldBeNil)
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR")
So(actual, ShouldContainSubstring, "test 1.0 ok")
So(actual, ShouldContainSubstring, "test ok")
// test scrub index - errors
indexFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", indexDigest.Encoded())
err = os.Chmod(indexFile, 0o000)
So(err, ShouldBeNil)
buff = bytes.NewBufferString("")
res, err = storeCtlr.CheckAllBlobsIntegrity()
res.PrintScrubResults(buff)
So(err, ShouldBeNil)
str = space.ReplaceAllString(buff.String(), " ")
actual = strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR")
So(actual, ShouldContainSubstring, "test affected")
err = os.Chmod(indexFile, 0o600)
So(err, ShouldBeNil)
err = os.Truncate(indexFile, 0)
So(err, ShouldBeNil)
buff = bytes.NewBufferString("")
res, err = storeCtlr.CheckAllBlobsIntegrity()
res.PrintScrubResults(buff)
So(err, ShouldBeNil)
str = space.ReplaceAllString(buff.String(), " ")
actual = strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR")
So(actual, ShouldContainSubstring, "test affected")
})
Convey("Manifest not found", func() {
// delete manifest file
manifestDig := manifestDigest.Encoded()
manifestFile := path.Join(imgStore.RootDir(), repoName, "/blobs/sha256", manifestDig)
err = os.Remove(manifestFile)
So(err, ShouldBeNil)
buff := bytes.NewBufferString("")
res, err := storeCtlr.CheckAllBlobsIntegrity()
res.PrintScrubResults(buff)
So(err, ShouldBeNil)
space := regexp.MustCompile(`\s+`)
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG STATUS ERROR")
So(actual, ShouldContainSubstring, "test 1.0 affected")
So(actual, ShouldContainSubstring, "no such file or directory")
index, err := storage.GetIndex(imgStore, repoName, log.With().Caller().Logger())
So(err, ShouldBeNil)
So(len(index.Manifests), ShouldEqual, 2)
manifestDescriptor := index.Manifests[0]
repoDir := path.Join(dir, repoName)
imageRes := storage.CheckLayers(repoName, tag, repoDir, manifestDescriptor)
So(imageRes.Status, ShouldEqual, "affected")
So(imageRes.Error, ShouldContainSubstring, "no such file or directory")
})
}) })
} }