From b2a43885227c09b4fd3a01b5bc819957bbde1ff4 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Thu, 24 Mar 2022 17:52:39 +0000 Subject: [PATCH] gc: add a unit test Signed-off-by: Ramkumar Chinchani --- Makefile | 2 +- examples/config-bench.json | 15 +++ pkg/storage/storage_fs.go | 2 +- pkg/storage/storage_fs_test.go | 187 +++++++++++++++++++++++++++++++++ 4 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 examples/config-bench.json diff --git a/Makefile b/Makefile index ae8d58a0..e215c97b 100644 --- a/Makefile +++ b/Makefile @@ -59,7 +59,7 @@ test: check-skopeo $(NOTATION) .PHONY: run-bench run-bench: binary bench - bin/zot-$(OS)-$(ARCH) serve examples/config-minimal.json & + bin/zot-$(OS)-$(ARCH) serve examples/config-bench.json & sleep 5 bin/zb-$(OS)-$(ARCH) -c 10 -n 100 -o $(BENCH_OUTPUT) http://localhost:8080 killall -r zot-* diff --git a/examples/config-bench.json b/examples/config-bench.json new file mode 100644 index 00000000..5c5fe2f7 --- /dev/null +++ b/examples/config-bench.json @@ -0,0 +1,15 @@ +{ + "distSpecVersion": "1.0.1", + "storage": { + "rootDirectory": "/tmp/zot" + }, + "http": { + "address": "127.0.0.1", + "port": "8080", + "ReadOnly": false + }, + "log": { + "level": "debug", + "output": "/dev/null" + } +} diff --git a/pkg/storage/storage_fs.go b/pkg/storage/storage_fs.go index d69d0d6f..519f98f9 100644 --- a/pkg/storage/storage_fs.go +++ b/pkg/storage/storage_fs.go @@ -1203,7 +1203,7 @@ retry: return err } - is.log.Debug().Str("blobPath", dst).Msg("dedupe: creating hard link") + is.log.Debug().Str("blobPath", dst).Str("dstRecord", dstRecord).Msg("dedupe: creating hard link") if err := os.Link(dstRecord, dst); err != nil { is.log.Error().Err(err).Str("blobPath", dst).Str("link", dstRecord).Msg("dedupe: unable to hard link") diff --git a/pkg/storage/storage_fs_test.go b/pkg/storage/storage_fs_test.go index 2e7dfad8..02ae1109 100644 --- a/pkg/storage/storage_fs_test.go +++ b/pkg/storage/storage_fs_test.go @@ -963,6 +963,193 @@ func TestGarbageCollect(t *testing.T) { So(err, ShouldNotBeNil) So(hasBlob, ShouldEqual, false) }) + + Convey("Garbage collect with dedupe", func() { + // garbage-collect is repo-local and dedupe is global and they can interact in strange ways + imgStore := storage.NewImageStore(dir, true, 5*time.Second, true, true, log, metrics) + + // first upload an image to the first repo and wait for GC timeout + + repo1Name := "gc1" + + // upload blob + upload, err := imgStore.NewBlobUpload(repo1Name) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + bdigest := godigest.FromBytes(content) + tdigest := bdigest + + blob, err := imgStore.PutBlobChunk(repo1Name, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repo1Name, upload, buf, bdigest.String()) + So(err, ShouldBeNil) + + annotationsMap := make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload(repo1Name, bytes.NewReader(cblob), cdigest.String()) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob(repo1Name, cdigest.String()) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, _ := json.Marshal(manifest) + + _, err = imgStore.PutImageManifest(repo1Name, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest.String()) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // sleep so past GC timeout + time.Sleep(10 * time.Second) + + hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest.String()) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // upload another image into a second repo with the same blob contents so dedupe is triggered + + repo2Name := "gc2" + + upload, err = imgStore.NewBlobUpload(repo2Name) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + buf = bytes.NewBuffer(content) + buflen = buf.Len() + + blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest.String()) + So(err, ShouldBeNil) + + annotationsMap = make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest = test.GetRandomImageConfig() + _, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest.String()) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest.String()) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, _ = json.Marshal(manifest) + + _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repo2Name, bdigest.String()) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // immediately upload any other image to second repo which should invoke GC inline, but expect layers to persist + + upload, err = imgStore.NewBlobUpload(repo2Name) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content = []byte("test-data-more") + buf = bytes.NewBuffer(content) + buflen = buf.Len() + bdigest = godigest.FromBytes(content) + + blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest.String()) + So(err, ShouldBeNil) + + annotationsMap = make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest = test.GetRandomImageConfig() + _, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest.String()) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest.String()) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, _ = json.Marshal(manifest) + digest := godigest.FromBytes(manifestBuf) + + _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + // original blob should exist + + hasBlob, _, err = imgStore.CheckBlob(repo2Name, tdigest.String()) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + _, _, _, err = imgStore.GetImageManifest(repo2Name, digest.String()) + So(err, ShouldBeNil) + }) }) }