Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00
Code Issues Activity

conformance: align with upstream conformance tests

Browse source 
Upstream conformance tests are being updated, so we need to align along
with our internal GC and dedupe features.

Add a new example config file which plays nice with conformance tests.

DeleteImageManifest() updated to deal with the case where the same
manifest can be created with multiple tags and deleted with the same
digest - so all entries must be deleted.

DeleteBlob() delete the digest key (bucket) when last reference is
dropped
This commit is contained in:
Ramkumar Chinchani 2020-04-06 18:17:24 -07:00
parent ea6018b168
commit b1f882e1b8
6 changed files with 75 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
examples/config-conformance.json Normal file
View file

@ -0,0 +1,15 @@
{
"version":"0.1.0-dev",
"storage":{
"rootDirectory":"/tmp/zot",
"gc": false,
"dedupe": false
},
"http": {
"address":"127.0.0.1",
"port":"8080"
},
"log":{
"level":"debug"
}
}

7
pkg/api/routes.go
View file

@ -222,7 +222,12 @@ func (rh *RouteHandler) ListTags(w http.ResponseWriter, r *http.Request) {
pTags.Tags = tags[i+1 : i+1+n]
}
last = pTags.Tags[len(pTags.Tags)-1]
if len(pTags.Tags) == 0 {
last = ""
} else {
last = pTags.Tags[len(pTags.Tags)-1]
}
w.Header().Set("Link", fmt.Sprintf("/v2/%s/tags/list?n=%d&last=%s; rel=\"next\"", name, n, last))
WriteJSON(w, http.StatusOK, pTags)

6
pkg/compliance/v1_0_0/check.go
View file

@ -565,7 +565,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
// delete manifest by digest
resp, err = resty.R().Delete(baseURL + "/v2/repo7/manifests/" + digest.String())
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 202)
So(resp.StatusCode(), ShouldEqual, 404)
// delete again should fail
resp, err = resty.R().Delete(baseURL + "/v2/repo7/manifests/" + digest.String())
So(err, ShouldBeNil)
@ -656,6 +656,10 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
resp, err = resty.R().Get(baseURL + "/v2/page0/tags/list?n=0")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
resp, err = resty.R().Get(baseURL + "/v2/page0/tags/list?n=3")
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)

11
pkg/storage/cache.go
View file

@ -154,6 +154,17 @@ func (c *Cache) DeleteBlob(digest string, path string) error {
return err
}
cur := b.Cursor()
k, _ := cur.First()
if k == nil {
c.log.Debug().Str("digest", digest).Str("path", path).Msg("deleting empty bucket")
if err := root.DeleteBucket([]byte(digest)); err != nil {
c.log.Error().Err(err).Str("digest", digest).Str("path", path).Msg("unable to delete")
return err
}
}
return nil
}); err != nil {
return err

51
pkg/storage/storage.go
View file

@ -488,7 +488,7 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error {
}
// as per spec "reference" can only be a digest and not a tag
_, err := godigest.Parse(reference)
digest, err := godigest.Parse(reference)
if err != nil {
is.log.Error().Err(err).Msg("invalid reference")
return errors.ErrBadManifest
@ -509,33 +509,29 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error {
found := false
var digest godigest.Digest
var i int
var m ispec.Descriptor
for i, m = range index.Manifests {
if reference == m.Digest.String() {
digest = m.Digest
found = true
// we are deleting, so keep only those manifests that don't match
outIndex := index
outIndex.Manifests = []ispec.Descriptor{}
break
for _, m = range index.Manifests {
if reference == m.Digest.String() {
found = true
continue
}
outIndex.Manifests = append(outIndex.Manifests, m)
}
if !found {
return errors.ErrManifestNotFound
}
// remove the manifest entry, not preserving order
index.Manifests[i] = index.Manifests[len(index.Manifests)-1]
index.Manifests = index.Manifests[:len(index.Manifests)-1]
// now update "index.json"
dir = path.Join(is.rootDir, repo)
file := path.Join(dir, "index.json")
buf, err = json.Marshal(index)
buf, err = json.Marshal(outIndex)
if err != nil {
return err
@ -809,44 +805,53 @@ func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, digest string)
// nolint (interfacer)
func (is *ImageStore) DedupeBlob(src string, dstDigest godigest.Digest, dst string) error {
retry:
is.log.Debug().Str("src", src).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: ENTER")
dstRecord, err := is.cache.GetBlob(dstDigest.String())
if err != nil && err != errors.ErrCacheMiss {
is.log.Error().Err(err).Str("blobPath", dst).Msg("unable to lookup blob record")
is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to lookup blob record")
return err
}
if dstRecord == "" {
if err := is.cache.PutBlob(dstDigest.String(), dst); err != nil {
is.log.Error().Err(err).Str("blobPath", dst).Msg("unable to insert blob record")
is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record")
return err
}
// move the blob from uploads to final dest
if err := os.Rename(src, dst); err != nil {
is.log.Error().Err(err).Str("src", src).Str("dst", dst).Msg("unable to rename blob")
is.log.Error().Err(err).Str("src", src).Str("dst", dst).Msg("dedupe: unable to rename blob")
return err
}
is.log.Debug().Str("src", src).Str("dst", dst).Msg("dedupe: rename")
} else {
dstRecordFi, err := os.Stat(dstRecord)
if err != nil {
is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("unable to stat")
return err
is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat")
// the actual blob on disk may have been removed by GC, so sync the cache
if err := is.cache.DeleteBlob(dstDigest.String(), dst); err != nil {
is.log.Error().Err(err).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: unable to delete blob record")
return err
}
goto retry
}
dstFi, err := os.Stat(dst)
if err != nil && !os.IsNotExist(err) {
is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("unable to stat")
is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat")
return err
}
if !os.SameFile(dstFi, dstRecordFi) {
if err := os.Link(dstRecord, dst); err != nil {
is.log.Error().Err(err).Str("blobPath", dst).Str("link", dstRecord).Msg("unable to hard link")
is.log.Error().Err(err).Str("blobPath", dst).Str("link", dstRecord).Msg("dedupe: unable to hard link")
return err
}
}
if err := os.Remove(src); err != nil {
is.log.Error().Err(err).Str("src", src).Msg("uname to remove blob")
is.log.Error().Err(err).Str("src", src).Msg("dedupe: uname to remove blob")
return err
}
is.log.Debug().Str("src", src).Msg("dedupe: remove")
}
return nil

10
pkg/storage/storage_test.go
View file

@ -113,6 +113,12 @@ func TestAPIs(t *testing.T) {
mb, _ := json.Marshal(m)
Convey("Bad image manifest", func() {
_, err = il.PutImageManifest("test", d.String(), "application/json", mb)
So(err, ShouldNotBeNil)
_, err = il.PutImageManifest("test", d.String(), ispec.MediaTypeImageManifest, []byte{})
So(err, ShouldNotBeNil)
_, err = il.PutImageManifest("test", d.String(), ispec.MediaTypeImageManifest, mb)
So(err, ShouldNotBeNil)
@ -133,6 +139,7 @@ func TestAPIs(t *testing.T) {
Size: int64(l),
},
},
Annotations: map[string]string{ispec.AnnotationRefName: "1.0"},
}
m.SchemaVersion = 2
mb, _ = json.Marshal(m)
@ -140,6 +147,9 @@ func TestAPIs(t *testing.T) {
_, err = il.PutImageManifest("test", d.String(), ispec.MediaTypeImageManifest, mb)
So(err, ShouldBeNil)
_, err = il.GetImageTags("test")
So(err, ShouldBeNil)
_, _, _, err = il.GetImageManifest("test", d.String())
So(err, ShouldBeNil)