0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-23 22:27:35 -05:00
zot/pkg/extensions/sync/sync_internal_test.go
peusebiu 59dc4c3229
feat(scheduler): pass the shutdown/reload ctx to running tasks (#1671)
Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
2023-09-05 09:48:56 -07:00

642 lines
19 KiB
Go

//go:build sync
// +build sync
package sync
import (
"bytes"
"context"
"encoding/json"
"fmt"
"os"
"path"
"testing"
dockerManifest "github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/oci/layout"
"github.com/containers/image/v5/types"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/rs/zerolog"
. "github.com/smartystreets/goconvey/convey"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions/config"
syncconf "zotregistry.io/zot/pkg/extensions/config/sync"
"zotregistry.io/zot/pkg/extensions/lint"
"zotregistry.io/zot/pkg/extensions/monitoring"
client "zotregistry.io/zot/pkg/extensions/sync/httpclient"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/cache"
storageConstants "zotregistry.io/zot/pkg/storage/constants"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/inject"
"zotregistry.io/zot/pkg/test/mocks"
)
const (
testImage = "zot-test"
testImageTag = "0.0.1"
host = "127.0.0.1:45117"
)
var ErrTestError = fmt.Errorf("testError")
func TestInjectSyncUtils(t *testing.T) {
Convey("Inject errors in utils functions", t, func() {
repositoryReference := fmt.Sprintf("%s/%s", host, testImage)
ref, err := parseRepositoryReference(repositoryReference)
So(err, ShouldBeNil)
So(ref.Name(), ShouldEqual, repositoryReference)
injected := inject.InjectFailure(0)
if injected {
_, err = getRepoTags(context.Background(), &types.SystemContext{}, host, testImage)
So(err, ShouldNotBeNil)
}
injected = inject.InjectFailure(0)
_, err = getPolicyContext(log.NewLogger("debug", ""))
if injected {
So(err, ShouldNotBeNil)
} else {
So(err, ShouldBeNil)
}
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(t.TempDir(), false, false, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil,
)
injected = inject.InjectFailure(0)
ols := NewOciLayoutStorage(storage.StoreController{DefaultStore: imageStore})
_, err = ols.GetImageReference(testImage, testImageTag)
if injected {
So(err, ShouldNotBeNil)
} else {
So(err, ShouldBeNil)
}
})
}
func TestSyncInternal(t *testing.T) {
Convey("Verify parseRepositoryReference func", t, func() {
repositoryReference := fmt.Sprintf("%s/%s", host, testImage)
ref, err := parseRepositoryReference(repositoryReference)
So(err, ShouldBeNil)
So(ref.Name(), ShouldEqual, repositoryReference)
repositoryReference = fmt.Sprintf("%s/%s:tagged", host, testImage)
_, err = parseRepositoryReference(repositoryReference)
So(err, ShouldEqual, zerr.ErrInvalidRepositoryName)
repositoryReference = fmt.Sprintf("http://%s/%s", host, testImage)
_, err = parseRepositoryReference(repositoryReference)
So(err, ShouldNotBeNil)
repositoryReference = fmt.Sprintf("docker://%s/%s", host, testImage)
_, err = parseRepositoryReference(repositoryReference)
So(err, ShouldNotBeNil)
_, err = getFileCredentials("/path/to/inexistent/file")
So(err, ShouldNotBeNil)
tempFile, err := os.CreateTemp("", "sync-credentials-")
if err != nil {
panic(err)
}
content := []byte(`{`)
if err := os.WriteFile(tempFile.Name(), content, 0o600); err != nil {
panic(err)
}
_, err = getFileCredentials(tempFile.Name())
So(err, ShouldNotBeNil)
srcCtx := &types.SystemContext{}
_, err = getRepoTags(context.Background(), srcCtx, host, testImage)
So(err, ShouldNotBeNil)
_, err = getRepoTags(context.Background(), srcCtx, host, testImage)
So(err, ShouldNotBeNil)
_, err = getFileCredentials("/invalid/path/to/file")
So(err, ShouldNotBeNil)
ok := isSupportedMediaType("unknown")
So(ok, ShouldBeFalse)
})
}
func TestRemoteRegistry(t *testing.T) {
Convey("test remote registry", t, func() {
logger := log.NewLogger("debug", "")
cfg := client.Config{
URL: "url",
TLSVerify: false,
}
client, err := client.New(cfg, logger)
So(err, ShouldBeNil)
remote := NewRemoteRegistry(client, logger)
imageRef, err := layout.NewReference("dir", "image")
So(err, ShouldBeNil)
_, _, _, err = remote.GetManifestContent(imageRef)
So(err, ShouldNotBeNil)
tags, err := remote.GetRepoTags("repo")
So(tags, ShouldBeEmpty)
So(err, ShouldNotBeNil)
})
}
func TestService(t *testing.T) {
Convey("trigger fetch tags error", t, func() {
conf := syncconf.RegistryConfig{
URLs: []string{"http://localhost"},
}
service, err := New(conf, "", storage.StoreController{}, mocks.MetaDBMock{}, log.Logger{})
So(err, ShouldBeNil)
err = service.SyncRepo(context.Background(), "repo")
So(err, ShouldNotBeNil)
})
}
func TestLocalRegistry(t *testing.T) {
Convey("make StoreController", t, func() {
dir := t.TempDir()
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
RootDir: dir,
Name: "cache",
UseRelPaths: true,
}, log)
syncImgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver)
repoName := "repo"
registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, nil, log)
imageReference, err := registry.GetImageReference(repoName, "1.0")
So(err, ShouldBeNil)
So(imageReference, ShouldNotBeNil)
imgStore := getImageStoreFromImageReference(imageReference, repoName, "1.0")
// create a blob/layer
upload, err := imgStore.NewBlobUpload(repoName)
So(err, ShouldBeNil)
So(upload, ShouldNotBeEmpty)
content := []byte("this is a blob1")
buf := bytes.NewBuffer(content)
buflen := buf.Len()
digest := godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
bdgst1 := digest
bsize1 := len(content)
err = imgStore.FinishBlobUpload(repoName, upload, buf, digest)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
// push index image
var index ispec.Index
index.SchemaVersion = 2
index.MediaType = ispec.MediaTypeImageIndex
for i := 0; i < 4; i++ {
// upload image config blob
upload, err := imgStore.NewBlobUpload(repoName)
So(err, ShouldBeNil)
So(upload, ShouldNotBeEmpty)
cblob, cdigest := test.GetRandomImageConfig()
buf := bytes.NewBuffer(cblob)
buflen := buf.Len()
blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
// create a manifest
manifest := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: ispec.MediaTypeImageConfig,
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageLayer,
Digest: bdgst1,
Size: int64(bsize1),
},
},
}
manifest.SchemaVersion = 2
content, err = json.Marshal(manifest)
So(err, ShouldBeNil)
digest = godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
_, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content)
So(err, ShouldBeNil)
index.Manifests = append(index.Manifests, ispec.Descriptor{
Digest: digest,
MediaType: ispec.MediaTypeImageManifest,
Size: int64(len(content)),
})
}
// upload index image
indexContent, err := json.Marshal(index)
So(err, ShouldBeNil)
indexDigest := godigest.FromBytes(indexContent)
So(indexDigest, ShouldNotBeNil)
_, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent)
So(err, ShouldBeNil)
Convey("sync index image", func() {
ok, err := registry.CanSkipImage(repoName, "1.0", indexDigest)
So(ok, ShouldBeFalse)
So(err, ShouldBeNil)
err = registry.CommitImage(imageReference, repoName, "1.0")
So(err, ShouldBeNil)
})
Convey("trigger GetImageManifest error in CommitImage()", func() {
err = os.Chmod(imgStore.BlobPath(repoName, indexDigest), 0o000)
So(err, ShouldBeNil)
err = registry.CommitImage(imageReference, repoName, "1.0")
So(err, ShouldNotBeNil)
})
Convey("trigger linter error in CommitImage()", func() {
defaultVal := true
linter := lint.NewLinter(&config.LintConfig{
BaseConfig: config.BaseConfig{
Enable: &defaultVal,
},
MandatoryAnnotations: []string{"annot1"},
}, log)
syncImgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay,
storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, linter, cacheDriver)
repoName := "repo"
registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, nil, log)
err = registry.CommitImage(imageReference, repoName, "1.0")
So(err, ShouldBeNil)
})
Convey("trigger GetBlobContent on manifest error in CommitImage()", func() {
err = os.Chmod(imgStore.BlobPath(repoName, digest), 0o000)
So(err, ShouldBeNil)
err = registry.CommitImage(imageReference, repoName, "1.0")
So(err, ShouldNotBeNil)
})
Convey("trigger copyBlob() error in CommitImage()", func() {
err = os.Chmod(imgStore.BlobPath(repoName, bdgst1), 0o000)
So(err, ShouldBeNil)
err = registry.CommitImage(imageReference, repoName, "1.0")
So(err, ShouldNotBeNil)
})
Convey("trigger PutImageManifest error on index manifest in CommitImage()", func() {
err = os.MkdirAll(syncImgStore.BlobPath(repoName, indexDigest), storageConstants.DefaultDirPerms)
So(err, ShouldBeNil)
err = os.Chmod(syncImgStore.BlobPath(repoName, indexDigest), 0o000)
So(err, ShouldBeNil)
err = registry.CommitImage(imageReference, repoName, "1.0")
So(err, ShouldNotBeNil)
})
Convey("trigger metaDB error on index manifest in CommitImage()", func() {
registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, mocks.MetaDBMock{
SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error {
if Reference == "1.0" {
return zerr.ErrRepoMetaNotFound
}
return nil
},
}, log)
err = registry.CommitImage(imageReference, repoName, "1.0")
So(err, ShouldNotBeNil)
})
Convey("trigger metaDB error on image manifest in CommitImage()", func() {
registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, mocks.MetaDBMock{
SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error {
return zerr.ErrRepoMetaNotFound
},
}, log)
err = registry.CommitImage(imageReference, repoName, "1.0")
So(err, ShouldNotBeNil)
})
Convey("push image", func() {
imageReference, err := registry.GetImageReference(repoName, "2.0")
So(err, ShouldBeNil)
So(imageReference, ShouldNotBeNil)
imgStore := getImageStoreFromImageReference(imageReference, repoName, "2.0")
// upload image
// create a blob/layer
upload, err := imgStore.NewBlobUpload(repoName)
So(err, ShouldBeNil)
So(upload, ShouldNotBeEmpty)
content := []byte("this is a blob1")
buf := bytes.NewBuffer(content)
buflen := buf.Len()
digest := godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
bdgst1 := digest
bsize1 := len(content)
err = imgStore.FinishBlobUpload(repoName, upload, buf, digest)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
// upload image config blob
upload, err = imgStore.NewBlobUpload(repoName)
So(err, ShouldBeNil)
So(upload, ShouldNotBeEmpty)
cblob, cdigest := test.GetRandomImageConfig()
buf = bytes.NewBuffer(cblob)
buflen = buf.Len()
blob, err = imgStore.PutBlobChunkStreamed(repoName, upload, buf)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
// create a manifest
manifest := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: ispec.MediaTypeImageConfig,
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: ispec.MediaTypeImageLayer,
Digest: bdgst1,
Size: int64(bsize1),
},
},
}
manifest.SchemaVersion = 2
content, err = json.Marshal(manifest)
So(err, ShouldBeNil)
digest = godigest.FromBytes(content)
So(digest, ShouldNotBeNil)
_, _, err = imgStore.PutImageManifest(repoName, "2.0", ispec.MediaTypeImageManifest, content)
So(err, ShouldBeNil)
Convey("sync image", func() {
ok, err := registry.CanSkipImage(repoName, "2.0", digest)
So(ok, ShouldBeFalse)
So(err, ShouldBeNil)
err = registry.CommitImage(imageReference, repoName, "2.0")
So(err, ShouldBeNil)
})
})
})
}
func TestConvertDockerToOCI(t *testing.T) {
Convey("test converting docker to oci functions", t, func() {
dir := t.TempDir()
srcStorageCtlr := test.GetDefaultStoreController(dir, log.NewLogger("debug", ""))
err := test.WriteImageToFileSystem(test.CreateDefaultImage(), "zot-test", "0.0.1", srcStorageCtlr)
So(err, ShouldBeNil)
imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.1")
So(err, ShouldBeNil)
imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{})
So(err, ShouldBeNil)
defer imageSource.Close()
Convey("trigger Unmarshal manifest error", func() {
_, err = convertDockerManifestToOCI(imageSource, []byte{})
So(err, ShouldNotBeNil)
})
Convey("trigger getImageConfigContent() error", func() {
manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil)
So(err, ShouldBeNil)
var manifest ispec.Manifest
err = json.Unmarshal(manifestBuf, &manifest)
So(err, ShouldBeNil)
err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()), 0o000)
So(err, ShouldBeNil)
_, err = convertDockerManifestToOCI(imageSource, manifestBuf)
So(err, ShouldNotBeNil)
})
Convey("trigger Unmarshal config error", func() {
manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil)
So(err, ShouldBeNil)
var manifest ispec.Manifest
err = json.Unmarshal(manifestBuf, &manifest)
So(err, ShouldBeNil)
err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()),
[]byte{}, storageConstants.DefaultFilePerms)
So(err, ShouldBeNil)
_, err = convertDockerManifestToOCI(imageSource, manifestBuf)
So(err, ShouldNotBeNil)
})
Convey("trigger convertDockerLayersToOCI error", func() {
manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil)
So(err, ShouldBeNil)
var manifest ispec.Manifest
err = json.Unmarshal(manifestBuf, &manifest)
So(err, ShouldBeNil)
manifestDigest := godigest.FromBytes(manifestBuf)
manifest.Layers[0].MediaType = "unknown"
newManifest, err := json.Marshal(manifest)
So(err, ShouldBeNil)
err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifestDigest.Encoded()),
newManifest, storageConstants.DefaultFilePerms)
So(err, ShouldBeNil)
_, err = convertDockerManifestToOCI(imageSource, manifestBuf)
So(err, ShouldNotBeNil)
})
Convey("trigger convertDockerIndexToOCI error", func() {
manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil)
So(err, ShouldBeNil)
_, err = convertDockerIndexToOCI(imageSource, manifestBuf)
So(err, ShouldNotBeNil)
// make zot-test image an index image
var manifest ispec.Manifest
err = json.Unmarshal(manifestBuf, &manifest)
So(err, ShouldBeNil)
dockerNewManifest := ispec.Manifest{
MediaType: dockerManifest.DockerV2Schema2MediaType,
Config: manifest.Config,
Layers: manifest.Layers,
}
dockerNewManifestBuf, err := json.Marshal(dockerNewManifest)
So(err, ShouldBeNil)
dockerManifestDigest := godigest.FromBytes(manifestBuf)
err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()),
dockerNewManifestBuf, storageConstants.DefaultFilePerms)
So(err, ShouldBeNil)
var index ispec.Index
index.Manifests = append(index.Manifests, ispec.Descriptor{
Digest: dockerManifestDigest,
Size: int64(len(dockerNewManifestBuf)),
MediaType: dockerManifest.DockerV2Schema2MediaType,
})
index.MediaType = dockerManifest.DockerV2ListMediaType
dockerIndexBuf, err := json.Marshal(index)
So(err, ShouldBeNil)
dockerIndexDigest := godigest.FromBytes(dockerIndexBuf)
err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerIndexDigest.Encoded()),
dockerIndexBuf, storageConstants.DefaultFilePerms)
So(err, ShouldBeNil)
// write index.json
var indexJSON ispec.Index
indexJSONBuf, err := os.ReadFile(path.Join(dir, "zot-test", "index.json"))
So(err, ShouldBeNil)
err = json.Unmarshal(indexJSONBuf, &indexJSON)
So(err, ShouldBeNil)
indexJSON.Manifests = append(indexJSON.Manifests, ispec.Descriptor{
Digest: dockerIndexDigest,
Size: int64(len(dockerIndexBuf)),
MediaType: ispec.MediaTypeImageIndex,
Annotations: map[string]string{
ispec.AnnotationRefName: "0.0.2",
},
})
indexJSONBuf, err = json.Marshal(indexJSON)
So(err, ShouldBeNil)
err = os.WriteFile(path.Join(dir, "zot-test", "index.json"), indexJSONBuf, storageConstants.DefaultFilePerms)
So(err, ShouldBeNil)
imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.2")
So(err, ShouldBeNil)
imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{})
So(err, ShouldBeNil)
_, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf)
So(err, ShouldNotBeNil)
err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()), 0o000)
So(err, ShouldBeNil)
_, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf)
So(err, ShouldNotBeNil)
})
})
}
func TestConvertDockerLayersToOCI(t *testing.T) {
Convey("test converting docker to oci functions", t, func() {
dockerLayers := []ispec.Descriptor{
{
MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaType,
},
{
MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaTypeGzip,
},
{
MediaType: dockerManifest.DockerV2SchemaLayerMediaTypeUncompressed,
},
{
MediaType: dockerManifest.DockerV2Schema2LayerMediaType,
},
}
err := convertDockerLayersToOCI(dockerLayers)
So(err, ShouldBeNil)
So(dockerLayers[0].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributable) //nolint: staticcheck
So(dockerLayers[1].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributableGzip) //nolint: staticcheck
So(dockerLayers[2].MediaType, ShouldEqual, ispec.MediaTypeImageLayer)
So(dockerLayers[3].MediaType, ShouldEqual, ispec.MediaTypeImageLayerGzip)
})
}