//go:build sync // +build sync package sync import ( "bytes" "context" "encoding/json" "fmt" "os" "path" "testing" dockerManifest "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/oci/layout" "github.com/containers/image/v5/types" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/rs/zerolog" . "github.com/smartystreets/goconvey/convey" zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/extensions/config" syncconf "zotregistry.io/zot/pkg/extensions/config/sync" "zotregistry.io/zot/pkg/extensions/lint" "zotregistry.io/zot/pkg/extensions/monitoring" client "zotregistry.io/zot/pkg/extensions/sync/httpclient" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/cache" storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/test" "zotregistry.io/zot/pkg/test/inject" "zotregistry.io/zot/pkg/test/mocks" ) const ( testImage = "zot-test" testImageTag = "0.0.1" host = "127.0.0.1:45117" ) var ErrTestError = fmt.Errorf("testError") func TestInjectSyncUtils(t *testing.T) { Convey("Inject errors in utils functions", t, func() { repositoryReference := fmt.Sprintf("%s/%s", host, testImage) ref, err := parseRepositoryReference(repositoryReference) So(err, ShouldBeNil) So(ref.Name(), ShouldEqual, repositoryReference) injected := inject.InjectFailure(0) if injected { _, err = getRepoTags(context.Background(), &types.SystemContext{}, host, testImage) So(err, ShouldNotBeNil) } injected = inject.InjectFailure(0) _, err = getPolicyContext(log.NewLogger("debug", "")) if injected { So(err, ShouldNotBeNil) } else { So(err, ShouldBeNil) } log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) imageStore := local.NewImageStore(t.TempDir(), false, false, storageConstants.DefaultGCDelay, storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil, ) injected = inject.InjectFailure(0) ols := NewOciLayoutStorage(storage.StoreController{DefaultStore: imageStore}) _, err = ols.GetImageReference(testImage, testImageTag) if injected { So(err, ShouldNotBeNil) } else { So(err, ShouldBeNil) } }) } func TestSyncInternal(t *testing.T) { Convey("Verify parseRepositoryReference func", t, func() { repositoryReference := fmt.Sprintf("%s/%s", host, testImage) ref, err := parseRepositoryReference(repositoryReference) So(err, ShouldBeNil) So(ref.Name(), ShouldEqual, repositoryReference) repositoryReference = fmt.Sprintf("%s/%s:tagged", host, testImage) _, err = parseRepositoryReference(repositoryReference) So(err, ShouldEqual, zerr.ErrInvalidRepositoryName) repositoryReference = fmt.Sprintf("http://%s/%s", host, testImage) _, err = parseRepositoryReference(repositoryReference) So(err, ShouldNotBeNil) repositoryReference = fmt.Sprintf("docker://%s/%s", host, testImage) _, err = parseRepositoryReference(repositoryReference) So(err, ShouldNotBeNil) _, err = getFileCredentials("/path/to/inexistent/file") So(err, ShouldNotBeNil) tempFile, err := os.CreateTemp("", "sync-credentials-") if err != nil { panic(err) } content := []byte(`{`) if err := os.WriteFile(tempFile.Name(), content, 0o600); err != nil { panic(err) } _, err = getFileCredentials(tempFile.Name()) So(err, ShouldNotBeNil) srcCtx := &types.SystemContext{} _, err = getRepoTags(context.Background(), srcCtx, host, testImage) So(err, ShouldNotBeNil) _, err = getRepoTags(context.Background(), srcCtx, host, testImage) So(err, ShouldNotBeNil) _, err = getFileCredentials("/invalid/path/to/file") So(err, ShouldNotBeNil) ok := isSupportedMediaType("unknown") So(ok, ShouldBeFalse) }) } func TestRemoteRegistry(t *testing.T) { Convey("test remote registry", t, func() { logger := log.NewLogger("debug", "") cfg := client.Config{ URL: "url", TLSVerify: false, } client, err := client.New(cfg, logger) So(err, ShouldBeNil) remote := NewRemoteRegistry(client, logger) imageRef, err := layout.NewReference("dir", "image") So(err, ShouldBeNil) _, _, _, err = remote.GetManifestContent(imageRef) So(err, ShouldNotBeNil) tags, err := remote.GetRepoTags("repo") So(tags, ShouldBeEmpty) So(err, ShouldNotBeNil) }) } func TestService(t *testing.T) { Convey("trigger fetch tags error", t, func() { conf := syncconf.RegistryConfig{ URLs: []string{"http://localhost"}, } service, err := New(conf, "", storage.StoreController{}, mocks.MetaDBMock{}, log.Logger{}) So(err, ShouldBeNil) err = service.SyncRepo(context.Background(), "repo") So(err, ShouldNotBeNil) }) } func TestLocalRegistry(t *testing.T) { Convey("make StoreController", t, func() { dir := t.TempDir() log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: dir, Name: "cache", UseRelPaths: true, }, log) syncImgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) repoName := "repo" registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, nil, log) imageReference, err := registry.GetImageReference(repoName, "1.0") So(err, ShouldBeNil) So(imageReference, ShouldNotBeNil) imgStore := getImageStoreFromImageReference(imageReference, repoName, "1.0") // create a blob/layer upload, err := imgStore.NewBlobUpload(repoName) So(err, ShouldBeNil) So(upload, ShouldNotBeEmpty) content := []byte("this is a blob1") buf := bytes.NewBuffer(content) buflen := buf.Len() digest := godigest.FromBytes(content) So(digest, ShouldNotBeNil) blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) So(err, ShouldBeNil) So(blob, ShouldEqual, buflen) bdgst1 := digest bsize1 := len(content) err = imgStore.FinishBlobUpload(repoName, upload, buf, digest) So(err, ShouldBeNil) So(blob, ShouldEqual, buflen) // push index image var index ispec.Index index.SchemaVersion = 2 index.MediaType = ispec.MediaTypeImageIndex for i := 0; i < 4; i++ { // upload image config blob upload, err := imgStore.NewBlobUpload(repoName) So(err, ShouldBeNil) So(upload, ShouldNotBeEmpty) cblob, cdigest := test.GetRandomImageConfig() buf := bytes.NewBuffer(cblob) buflen := buf.Len() blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) So(err, ShouldBeNil) So(blob, ShouldEqual, buflen) err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) So(err, ShouldBeNil) So(blob, ShouldEqual, buflen) // create a manifest manifest := ispec.Manifest{ Config: ispec.Descriptor{ MediaType: ispec.MediaTypeImageConfig, Digest: cdigest, Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { MediaType: ispec.MediaTypeImageLayer, Digest: bdgst1, Size: int64(bsize1), }, }, } manifest.SchemaVersion = 2 content, err = json.Marshal(manifest) So(err, ShouldBeNil) digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) So(err, ShouldBeNil) index.Manifests = append(index.Manifests, ispec.Descriptor{ Digest: digest, MediaType: ispec.MediaTypeImageManifest, Size: int64(len(content)), }) } // upload index image indexContent, err := json.Marshal(index) So(err, ShouldBeNil) indexDigest := godigest.FromBytes(indexContent) So(indexDigest, ShouldNotBeNil) _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) So(err, ShouldBeNil) Convey("sync index image", func() { ok, err := registry.CanSkipImage(repoName, "1.0", indexDigest) So(ok, ShouldBeFalse) So(err, ShouldBeNil) err = registry.CommitImage(imageReference, repoName, "1.0") So(err, ShouldBeNil) }) Convey("trigger GetImageManifest error in CommitImage()", func() { err = os.Chmod(imgStore.BlobPath(repoName, indexDigest), 0o000) So(err, ShouldBeNil) err = registry.CommitImage(imageReference, repoName, "1.0") So(err, ShouldNotBeNil) }) Convey("trigger linter error in CommitImage()", func() { defaultVal := true linter := lint.NewLinter(&config.LintConfig{ BaseConfig: config.BaseConfig{ Enable: &defaultVal, }, MandatoryAnnotations: []string{"annot1"}, }, log) syncImgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, linter, cacheDriver) repoName := "repo" registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, nil, log) err = registry.CommitImage(imageReference, repoName, "1.0") So(err, ShouldBeNil) }) Convey("trigger GetBlobContent on manifest error in CommitImage()", func() { err = os.Chmod(imgStore.BlobPath(repoName, digest), 0o000) So(err, ShouldBeNil) err = registry.CommitImage(imageReference, repoName, "1.0") So(err, ShouldNotBeNil) }) Convey("trigger copyBlob() error in CommitImage()", func() { err = os.Chmod(imgStore.BlobPath(repoName, bdgst1), 0o000) So(err, ShouldBeNil) err = registry.CommitImage(imageReference, repoName, "1.0") So(err, ShouldNotBeNil) }) Convey("trigger PutImageManifest error on index manifest in CommitImage()", func() { err = os.MkdirAll(syncImgStore.BlobPath(repoName, indexDigest), storageConstants.DefaultDirPerms) So(err, ShouldBeNil) err = os.Chmod(syncImgStore.BlobPath(repoName, indexDigest), 0o000) So(err, ShouldBeNil) err = registry.CommitImage(imageReference, repoName, "1.0") So(err, ShouldNotBeNil) }) Convey("trigger metaDB error on index manifest in CommitImage()", func() { registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, mocks.MetaDBMock{ SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error { if Reference == "1.0" { return zerr.ErrRepoMetaNotFound } return nil }, }, log) err = registry.CommitImage(imageReference, repoName, "1.0") So(err, ShouldNotBeNil) }) Convey("trigger metaDB error on image manifest in CommitImage()", func() { registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, mocks.MetaDBMock{ SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error { return zerr.ErrRepoMetaNotFound }, }, log) err = registry.CommitImage(imageReference, repoName, "1.0") So(err, ShouldNotBeNil) }) Convey("push image", func() { imageReference, err := registry.GetImageReference(repoName, "2.0") So(err, ShouldBeNil) So(imageReference, ShouldNotBeNil) imgStore := getImageStoreFromImageReference(imageReference, repoName, "2.0") // upload image // create a blob/layer upload, err := imgStore.NewBlobUpload(repoName) So(err, ShouldBeNil) So(upload, ShouldNotBeEmpty) content := []byte("this is a blob1") buf := bytes.NewBuffer(content) buflen := buf.Len() digest := godigest.FromBytes(content) So(digest, ShouldNotBeNil) blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) So(err, ShouldBeNil) So(blob, ShouldEqual, buflen) bdgst1 := digest bsize1 := len(content) err = imgStore.FinishBlobUpload(repoName, upload, buf, digest) So(err, ShouldBeNil) So(blob, ShouldEqual, buflen) // upload image config blob upload, err = imgStore.NewBlobUpload(repoName) So(err, ShouldBeNil) So(upload, ShouldNotBeEmpty) cblob, cdigest := test.GetRandomImageConfig() buf = bytes.NewBuffer(cblob) buflen = buf.Len() blob, err = imgStore.PutBlobChunkStreamed(repoName, upload, buf) So(err, ShouldBeNil) So(blob, ShouldEqual, buflen) err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) So(err, ShouldBeNil) So(blob, ShouldEqual, buflen) // create a manifest manifest := ispec.Manifest{ Config: ispec.Descriptor{ MediaType: ispec.MediaTypeImageConfig, Digest: cdigest, Size: int64(len(cblob)), }, Layers: []ispec.Descriptor{ { MediaType: ispec.MediaTypeImageLayer, Digest: bdgst1, Size: int64(bsize1), }, }, } manifest.SchemaVersion = 2 content, err = json.Marshal(manifest) So(err, ShouldBeNil) digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) _, _, err = imgStore.PutImageManifest(repoName, "2.0", ispec.MediaTypeImageManifest, content) So(err, ShouldBeNil) Convey("sync image", func() { ok, err := registry.CanSkipImage(repoName, "2.0", digest) So(ok, ShouldBeFalse) So(err, ShouldBeNil) err = registry.CommitImage(imageReference, repoName, "2.0") So(err, ShouldBeNil) }) }) }) } func TestConvertDockerToOCI(t *testing.T) { Convey("test converting docker to oci functions", t, func() { dir := t.TempDir() srcStorageCtlr := test.GetDefaultStoreController(dir, log.NewLogger("debug", "")) err := test.WriteImageToFileSystem(test.CreateDefaultImage(), "zot-test", "0.0.1", srcStorageCtlr) So(err, ShouldBeNil) imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.1") So(err, ShouldBeNil) imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{}) So(err, ShouldBeNil) defer imageSource.Close() Convey("trigger Unmarshal manifest error", func() { _, err = convertDockerManifestToOCI(imageSource, []byte{}) So(err, ShouldNotBeNil) }) Convey("trigger getImageConfigContent() error", func() { manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) So(err, ShouldBeNil) var manifest ispec.Manifest err = json.Unmarshal(manifestBuf, &manifest) So(err, ShouldBeNil) err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()), 0o000) So(err, ShouldBeNil) _, err = convertDockerManifestToOCI(imageSource, manifestBuf) So(err, ShouldNotBeNil) }) Convey("trigger Unmarshal config error", func() { manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) So(err, ShouldBeNil) var manifest ispec.Manifest err = json.Unmarshal(manifestBuf, &manifest) So(err, ShouldBeNil) err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifest.Config.Digest.Encoded()), []byte{}, storageConstants.DefaultFilePerms) So(err, ShouldBeNil) _, err = convertDockerManifestToOCI(imageSource, manifestBuf) So(err, ShouldNotBeNil) }) Convey("trigger convertDockerLayersToOCI error", func() { manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) So(err, ShouldBeNil) var manifest ispec.Manifest err = json.Unmarshal(manifestBuf, &manifest) So(err, ShouldBeNil) manifestDigest := godigest.FromBytes(manifestBuf) manifest.Layers[0].MediaType = "unknown" newManifest, err := json.Marshal(manifest) So(err, ShouldBeNil) err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", manifestDigest.Encoded()), newManifest, storageConstants.DefaultFilePerms) So(err, ShouldBeNil) _, err = convertDockerManifestToOCI(imageSource, manifestBuf) So(err, ShouldNotBeNil) }) Convey("trigger convertDockerIndexToOCI error", func() { manifestBuf, _, err := imageSource.GetManifest(context.Background(), nil) So(err, ShouldBeNil) _, err = convertDockerIndexToOCI(imageSource, manifestBuf) So(err, ShouldNotBeNil) // make zot-test image an index image var manifest ispec.Manifest err = json.Unmarshal(manifestBuf, &manifest) So(err, ShouldBeNil) dockerNewManifest := ispec.Manifest{ MediaType: dockerManifest.DockerV2Schema2MediaType, Config: manifest.Config, Layers: manifest.Layers, } dockerNewManifestBuf, err := json.Marshal(dockerNewManifest) So(err, ShouldBeNil) dockerManifestDigest := godigest.FromBytes(manifestBuf) err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()), dockerNewManifestBuf, storageConstants.DefaultFilePerms) So(err, ShouldBeNil) var index ispec.Index index.Manifests = append(index.Manifests, ispec.Descriptor{ Digest: dockerManifestDigest, Size: int64(len(dockerNewManifestBuf)), MediaType: dockerManifest.DockerV2Schema2MediaType, }) index.MediaType = dockerManifest.DockerV2ListMediaType dockerIndexBuf, err := json.Marshal(index) So(err, ShouldBeNil) dockerIndexDigest := godigest.FromBytes(dockerIndexBuf) err = os.WriteFile(path.Join(dir, "zot-test", "blobs/sha256", dockerIndexDigest.Encoded()), dockerIndexBuf, storageConstants.DefaultFilePerms) So(err, ShouldBeNil) // write index.json var indexJSON ispec.Index indexJSONBuf, err := os.ReadFile(path.Join(dir, "zot-test", "index.json")) So(err, ShouldBeNil) err = json.Unmarshal(indexJSONBuf, &indexJSON) So(err, ShouldBeNil) indexJSON.Manifests = append(indexJSON.Manifests, ispec.Descriptor{ Digest: dockerIndexDigest, Size: int64(len(dockerIndexBuf)), MediaType: ispec.MediaTypeImageIndex, Annotations: map[string]string{ ispec.AnnotationRefName: "0.0.2", }, }) indexJSONBuf, err = json.Marshal(indexJSON) So(err, ShouldBeNil) err = os.WriteFile(path.Join(dir, "zot-test", "index.json"), indexJSONBuf, storageConstants.DefaultFilePerms) So(err, ShouldBeNil) imageRef, err := layout.NewReference(path.Join(dir, "zot-test"), "0.0.2") So(err, ShouldBeNil) imageSource, err := imageRef.NewImageSource(context.Background(), &types.SystemContext{}) So(err, ShouldBeNil) _, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf) So(err, ShouldNotBeNil) err = os.Chmod(path.Join(dir, "zot-test", "blobs/sha256", dockerManifestDigest.Encoded()), 0o000) So(err, ShouldBeNil) _, err = convertDockerIndexToOCI(imageSource, dockerIndexBuf) So(err, ShouldNotBeNil) }) }) } func TestConvertDockerLayersToOCI(t *testing.T) { Convey("test converting docker to oci functions", t, func() { dockerLayers := []ispec.Descriptor{ { MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaType, }, { MediaType: dockerManifest.DockerV2Schema2ForeignLayerMediaTypeGzip, }, { MediaType: dockerManifest.DockerV2SchemaLayerMediaTypeUncompressed, }, { MediaType: dockerManifest.DockerV2Schema2LayerMediaType, }, } err := convertDockerLayersToOCI(dockerLayers) So(err, ShouldBeNil) So(dockerLayers[0].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributable) //nolint: staticcheck So(dockerLayers[1].MediaType, ShouldEqual, ispec.MediaTypeImageLayerNonDistributableGzip) //nolint: staticcheck So(dockerLayers[2].MediaType, ShouldEqual, ispec.MediaTypeImageLayer) So(dockerLayers[3].MediaType, ShouldEqual, ispec.MediaTypeImageLayerGzip) }) }