Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2025-01-20 22:52:51 -05:00
Code Issues Activity

gc: add a unit test

Browse source 
Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>
This commit is contained in:
Ramkumar Chinchani 2022-03-24 17:52:39 +00:00 • committed by Ramkumar Chinchani
parent be910cf01c
commit b2a4388522
4 changed files with 204 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -59,7 +59,7 @@ test: check-skopeo $(NOTATION)
.PHONY: run-bench
run-bench: binary bench
bin/zot-$(OS)-$(ARCH) serve examples/config-minimal.json &
bin/zot-$(OS)-$(ARCH) serve examples/config-bench.json &
sleep 5
bin/zb-$(OS)-$(ARCH) -c 10 -n 100 -o $(BENCH_OUTPUT) http://localhost:8080
killall -r zot-*

15
examples/config-bench.json Normal file
View file

@ -0,0 +1,15 @@
{
"distSpecVersion": "1.0.1",
"storage": {
"rootDirectory": "/tmp/zot"
},
"http": {
"address": "127.0.0.1",
"port": "8080",
"ReadOnly": false
},
"log": {
"level": "debug",
"output": "/dev/null"
}
}

2
pkg/storage/storage_fs.go
View file

@ -1203,7 +1203,7 @@ retry:
return err
}
is.log.Debug().Str("blobPath", dst).Msg("dedupe: creating hard link")
is.log.Debug().Str("blobPath", dst).Str("dstRecord", dstRecord).Msg("dedupe: creating hard link")
if err := os.Link(dstRecord, dst); err != nil {
is.log.Error().Err(err).Str("blobPath", dst).Str("link", dstRecord).Msg("dedupe: unable to hard link")

187
pkg/storage/storage_fs_test.go
View file

@ -963,6 +963,193 @@ func TestGarbageCollect(t *testing.T) {
So(err, ShouldNotBeNil)
So(hasBlob, ShouldEqual, false)
})
Convey("Garbage collect with dedupe", func() {
// garbage-collect is repo-local and dedupe is global and they can interact in strange ways
imgStore := storage.NewImageStore(dir, true, 5*time.Second, true, true, log, metrics)
// first upload an image to the first repo and wait for GC timeout
repo1Name := "gc1"
// upload blob
upload, err := imgStore.NewBlobUpload(repo1Name)
So(err, ShouldBeNil)
So(upload, ShouldNotBeEmpty)
content := []byte("test-data")
buf := bytes.NewBuffer(content)
buflen := buf.Len()
bdigest := godigest.FromBytes(content)
tdigest := bdigest
blob, err := imgStore.PutBlobChunk(repo1Name, upload, 0, int64(buflen), buf)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
err = imgStore.FinishBlobUpload(repo1Name, upload, buf, bdigest.String())
So(err, ShouldBeNil)
annotationsMap := make(map[string]string)
annotationsMap[ispec.AnnotationRefName] = tag
cblob, cdigest := test.GetRandomImageConfig()
_, clen, err := imgStore.FullBlobUpload(repo1Name, bytes.NewReader(cblob), cdigest.String())
So(err, ShouldBeNil)
So(clen, ShouldEqual, len(cblob))
hasBlob, _, err := imgStore.CheckBlob(repo1Name, cdigest.String())
So(err, ShouldBeNil)
So(hasBlob, ShouldEqual, true)
manifest := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: bdigest,
Size: int64(buflen),
},
},
Annotations: annotationsMap,
}
manifest.SchemaVersion = 2
manifestBuf, _ := json.Marshal(manifest)
_, err = imgStore.PutImageManifest(repo1Name, tag, ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldBeNil)
hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest.String())
So(err, ShouldBeNil)
So(hasBlob, ShouldEqual, true)
// sleep so past GC timeout
time.Sleep(10 * time.Second)
hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest.String())
So(err, ShouldBeNil)
So(hasBlob, ShouldEqual, true)
// upload another image into a second repo with the same blob contents so dedupe is triggered
repo2Name := "gc2"
upload, err = imgStore.NewBlobUpload(repo2Name)
So(err, ShouldBeNil)
So(upload, ShouldNotBeEmpty)
buf = bytes.NewBuffer(content)
buflen = buf.Len()
blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest.String())
So(err, ShouldBeNil)
annotationsMap = make(map[string]string)
annotationsMap[ispec.AnnotationRefName] = tag
cblob, cdigest = test.GetRandomImageConfig()
_, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest.String())
So(err, ShouldBeNil)
So(clen, ShouldEqual, len(cblob))
hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest.String())
So(err, ShouldBeNil)
So(hasBlob, ShouldEqual, true)
manifest = ispec.Manifest{
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: bdigest,
Size: int64(buflen),
},
},
Annotations: annotationsMap,
}
manifest.SchemaVersion = 2
manifestBuf, _ = json.Marshal(manifest)
_, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldBeNil)
hasBlob, _, err = imgStore.CheckBlob(repo2Name, bdigest.String())
So(err, ShouldBeNil)
So(hasBlob, ShouldEqual, true)
// immediately upload any other image to second repo which should invoke GC inline, but expect layers to persist
upload, err = imgStore.NewBlobUpload(repo2Name)
So(err, ShouldBeNil)
So(upload, ShouldNotBeEmpty)
content = []byte("test-data-more")
buf = bytes.NewBuffer(content)
buflen = buf.Len()
bdigest = godigest.FromBytes(content)
blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf)
So(err, ShouldBeNil)
So(blob, ShouldEqual, buflen)
err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest.String())
So(err, ShouldBeNil)
annotationsMap = make(map[string]string)
annotationsMap[ispec.AnnotationRefName] = tag
cblob, cdigest = test.GetRandomImageConfig()
_, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest.String())
So(err, ShouldBeNil)
So(clen, ShouldEqual, len(cblob))
hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest.String())
So(err, ShouldBeNil)
So(hasBlob, ShouldEqual, true)
manifest = ispec.Manifest{
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: cdigest,
Size: int64(len(cblob)),
},
Layers: []ispec.Descriptor{
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: bdigest,
Size: int64(buflen),
},
},
Annotations: annotationsMap,
}
manifest.SchemaVersion = 2
manifestBuf, _ = json.Marshal(manifest)
digest := godigest.FromBytes(manifestBuf)
_, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf)
So(err, ShouldBeNil)
// original blob should exist
hasBlob, _, err = imgStore.CheckBlob(repo2Name, tdigest.String())
So(err, ShouldBeNil)
So(hasBlob, ShouldEqual, true)
_, _, _, err = imgStore.GetImageManifest(repo2Name, digest.String())
So(err, ShouldBeNil)
})
})
}