From d42ac4cd0dc1540a452c8a01204f73b984b9fafb Mon Sep 17 00:00:00 2001 From: Andrei Aaron Date: Thu, 3 Oct 2024 19:06:41 +0300 Subject: [PATCH] fix(delete manifest): distinct behaviors for delete by tag vb delete by digest (#2626) In case of delete by tag only the tag is removed, the manifest itself would continue to be accessible by digest. In case of delete by digest the manifest would be completely removed (provided it is not used by an index or another reference). Signed-off-by: Andrei Aaron --- cmd/zb/helper.go | 20 +- pkg/api/controller_test.go | 1224 +++++++++++++++++++++++++- pkg/storage/common/common.go | 24 +- pkg/storage/imagestore/imagestore.go | 21 +- pkg/storage/local/local_test.go | 22 +- pkg/storage/s3/s3_test.go | 47 +- 6 files changed, 1332 insertions(+), 26 deletions(-) diff --git a/cmd/zb/helper.go b/cmd/zb/helper.go index 5a5d93ff..e4dc27e8 100644 --- a/cmd/zb/helper.go +++ b/cmd/zb/helper.go @@ -25,25 +25,27 @@ import ( "zotregistry.dev/zot/pkg/common" ) -func makeHTTPGetRequest(url string, resultPtr interface{}, client *resty.Client) error { +func makeHTTPGetRequest(url string, resultPtr interface{}, client *resty.Client) (http.Header, error) { resp, err := client.R().Get(url) if err != nil { - return err + return http.Header{}, err } + header := resp.Header() + if resp.StatusCode() != http.StatusOK { log.Printf("unable to make GET request on %s, response status code: %d", url, resp.StatusCode()) - return fmt.Errorf("%w: Expected: %d, Got: %d, Body: '%s'", zerr.ErrBadHTTPStatusCode, http.StatusOK, + return header, fmt.Errorf("%w: Expected: %d, Got: %d, Body: '%s'", zerr.ErrBadHTTPStatusCode, http.StatusOK, resp.StatusCode(), string(resp.Body())) } err = json.Unmarshal(resp.Body(), resultPtr) if err != nil { - return err + return header, err } - return nil + return header, nil } func makeHTTPDeleteRequest(url string, client *resty.Client) error { @@ -67,7 +69,7 @@ func deleteTestRepo(repos []string, url string, client *resty.Client) error { var tags common.ImageTags // get tags - err := makeHTTPGetRequest(fmt.Sprintf("%s/v2/%s/tags/list", url, repo), &tags, client) + _, err := makeHTTPGetRequest(fmt.Sprintf("%s/v2/%s/tags/list", url, repo), &tags, client) if err != nil { return err } @@ -76,13 +78,15 @@ func deleteTestRepo(repos []string, url string, client *resty.Client) error { var manifest ispec.Manifest // first get tag manifest to get containing blobs - err := makeHTTPGetRequest(fmt.Sprintf("%s/v2/%s/manifests/%s", url, repo, tag), &manifest, client) + header, err := makeHTTPGetRequest(fmt.Sprintf("%s/v2/%s/manifests/%s", url, repo, tag), &manifest, client) if err != nil { return err } + manifestDigest := header.Get("Docker-Content-Digest") + // delete manifest so that we don't trigger BlobInUse error - err = makeHTTPDeleteRequest(fmt.Sprintf("%s/v2/%s/manifests/%s", url, repo, tag), client) + err = makeHTTPDeleteRequest(fmt.Sprintf("%s/v2/%s/manifests/%s", url, repo, manifestDigest), client) if err != nil { return err } diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 1edf8f1b..66da6687 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -35,6 +35,7 @@ import ( notreg "github.com/notaryproject/notation-go/registry" distext "github.com/opencontainers/distribution-spec/specs-go/v1/extensions" godigest "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/project-zot/mockoidc" "github.com/sigstore/cosign/v2/cmd/cosign/cli/generate" @@ -8328,7 +8329,7 @@ func TestStorageCommit(t *testing.T) { defer cm.StopServer() - Convey("Manifests", func() { + Convey("Manifest deletion deletes all tags", func() { _, _ = Print("\nManifests") // check a non-existent manifest @@ -8374,6 +8375,7 @@ func TestStorageCommit(t *testing.T) { err = UploadImage(image, baseURL, repoName, "test:2.0") So(err, ShouldBeNil) + // update tag to match the other manifest resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). SetBody(content).Put(baseURL + "/v2/repo7/manifests/test:2.0") So(err, ShouldBeNil) @@ -8405,10 +8407,35 @@ func TestStorageCommit(t *testing.T) { resp, err = resty.R().Delete(baseURL + "/v2/repo7/manifests/test:1.0") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the manifest should still be present in storage + _, err = os.Stat(path.Join(dir, "repo7", "blobs", digest.Algorithm().String(), + digest.Encoded())) + So(err, ShouldBeNil) + + // the index should not longer contain this tag + tags, err := readTagsFromStorage(dir, "repo7", digest) + So(err, ShouldBeNil) + So(tags, ShouldNotContain, "test:1.0") + So(tags, ShouldContain, "test:1.0.1") + So(tags, ShouldContain, "test:2.0") + So(len(tags), ShouldEqual, 2) + // delete manifest by digest (1.0 deleted but 1.0.1 has same reference) resp, err = resty.R().Delete(baseURL + "/v2/repo7/manifests/" + digest.String()) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the manifest should be removed from storage + _, err = os.Stat(path.Join(dir, "repo7", "blobs", digest.Algorithm().String(), + digest.Encoded())) + So(err, ShouldNotBeNil) + + // the index should not longer contain this manifest + tags, err = readTagsFromStorage(dir, "repo7", digest) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 0) + // delete manifest by digest resp, err = resty.R().Delete(baseURL + "/v2/repo7/manifests/" + digest.String()) So(err, ShouldBeNil) @@ -8442,6 +8469,1167 @@ func TestStorageCommit(t *testing.T) { So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) So(resp.Body(), ShouldNotBeEmpty) }) + + Convey("Deleting all tags does not remove the manifest", func() { + _, _ = Print("\nManifests") + + // check a non-existent manifest + resp, err := resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). + Head(baseURL + "/v2/unknown/manifests/test:1.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + + image := CreateImageWith().RandomLayers(1, 2).DefaultConfig().Build() + + repoName := "repo7" + err = UploadImage(image, baseURL, repoName, "test:1.0") + So(err, ShouldBeNil) + + _, err = os.Stat(path.Join(dir, "repo7")) + So(err, ShouldBeNil) + + content := image.ManifestDescriptor.Data + digest := image.ManifestDescriptor.Digest + So(digest, ShouldNotBeNil) + + resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). + SetBody(content).Put(baseURL + "/v2/repo7/manifests/test:1.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + digestHdr := resp.Header().Get(constants.DistContentDigestKey) + So(digestHdr, ShouldNotBeEmpty) + So(digestHdr, ShouldEqual, digest.String()) + + resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). + SetBody(content).Put(baseURL + "/v2/repo7/manifests/test:1.0.1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + digestHdr = resp.Header().Get(constants.DistContentDigestKey) + So(digestHdr, ShouldNotBeEmpty) + So(digestHdr, ShouldEqual, digest.String()) + + err = UploadImage(image, baseURL, repoName, "test:1.0.1") + So(err, ShouldBeNil) + + image = CreateImageWith().RandomLayers(1, 1).DefaultConfig().Build() + + err = UploadImage(image, baseURL, repoName, "test:2.0") + So(err, ShouldBeNil) + + // update tag to match the other manifest + resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). + SetBody(content).Put(baseURL + "/v2/repo7/manifests/test:2.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + digestHdr = resp.Header().Get(constants.DistContentDigestKey) + So(digestHdr, ShouldNotBeEmpty) + So(digestHdr, ShouldEqual, digest.String()) + + // check/get by tag + resp, err = resty.R().Head(baseURL + "/v2/repo7/manifests/test:1.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + resp, err = resty.R().Get(baseURL + "/v2/repo7/manifests/test:1.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + // check/get by reference + resp, err = resty.R().Head(baseURL + "/v2/repo7/manifests/" + digest.String()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + resp, err = resty.R().Get(baseURL + "/v2/repo7/manifests/" + digest.String()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + + // delete manifest by tag should pass + resp, err = resty.R().Delete(baseURL + "/v2/repo7/manifests/test:1.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the manifest should still be present in storage + _, err = os.Stat(path.Join(dir, "repo7", "blobs", digest.Algorithm().String(), + digest.Encoded())) + So(err, ShouldBeNil) + + // the index should not longer contain this tag + tags, err := readTagsFromStorage(dir, "repo7", digest) + So(err, ShouldBeNil) + So(tags, ShouldNotContain, "test:1.0") + So(tags, ShouldContain, "test:1.0.1") + So(tags, ShouldContain, "test:2.0") + So(len(tags), ShouldEqual, 2) + + // delete manifest by tag should pass + resp, err = resty.R().Delete(baseURL + "/v2/repo7/manifests/test:1.0.1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the manifest should still be present in storage + _, err = os.Stat(path.Join(dir, "repo7", "blobs", digest.Algorithm().String(), + digest.Encoded())) + So(err, ShouldBeNil) + + // the index should not longer contain this manifest + tags, err = readTagsFromStorage(dir, "repo7", digest) + So(err, ShouldBeNil) + So(tags, ShouldNotContain, "test:1.0") + So(tags, ShouldNotContain, "test:1.0.1") + So(tags, ShouldContain, "test:2.0") + So(len(tags), ShouldEqual, 1) + + // delete manifest by tag should pass + resp, err = resty.R().Delete(baseURL + "/v2/repo7/manifests/test:2.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the manifest should still be present in storage + _, err = os.Stat(path.Join(dir, "repo7", "blobs", digest.Algorithm().String(), + digest.Encoded())) + So(err, ShouldBeNil) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, "repo7", digest) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // check/get by tag + resp, err = resty.R().Head(baseURL + "/v2/repo7/manifests/test:1.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + resp, err = resty.R().Get(baseURL + "/v2/repo7/manifests/test:1.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + So(resp.Body(), ShouldNotBeEmpty) + resp, err = resty.R().Head(baseURL + "/v2/repo7/manifests/test:2.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + resp, err = resty.R().Get(baseURL + "/v2/repo7/manifests/test:2.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + So(resp.Body(), ShouldNotBeEmpty) + // check/get by reference + resp, err = resty.R().Head(baseURL + "/v2/repo7/manifests/" + digest.String()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + resp, err = resty.R().Get(baseURL + "/v2/repo7/manifests/" + digest.String()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + + // if the only index entry is for the empty name, + // adding a new tag would replace that index entry + // instead of having 2 entries + err = UploadImage(image, baseURL, repoName, "test:2.0.1") + So(err, ShouldBeNil) + + // update tag to match the other manifest + resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). + SetBody(content).Put(baseURL + "/v2/repo7/manifests/test:2.0.1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + digestHdr = resp.Header().Get(constants.DistContentDigestKey) + So(digestHdr, ShouldNotBeEmpty) + So(digestHdr, ShouldEqual, digest.String()) + + // the index should not longer contain this tag + tags, err = readTagsFromStorage(dir, "repo7", digest) + So(err, ShouldBeNil) + So(tags, ShouldContain, "test:2.0.1") + So(len(tags), ShouldEqual, 1) + + // delete manifest by tag should pass + resp, err = resty.R().Delete(baseURL + "/v2/repo7/manifests/test:2.0.1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the manifest should still be present in storage + _, err = os.Stat(path.Join(dir, "repo7", "blobs", digest.Algorithm().String(), + digest.Encoded())) + So(err, ShouldBeNil) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, "repo7", digest) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + resp, err = resty.R().Head(baseURL + "/v2/repo7/manifests/" + digest.String()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + }) + }) +} + +func TestMultiarchImage(t *testing.T) { + Convey("Make a new controller", t, func() { + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + + dir := t.TempDir() + ctlr := makeController(conf, dir) + + cm := test.NewControllerManager(ctlr) + cm.StartAndWait(port) + + defer cm.StopServer() + + Convey("Manifests used within an index should not be deletable", func() { + repoName := "multiarch" //nolint:goconst + + image1 := CreateRandomImage() + image2 := CreateRandomImage() + multiImage := CreateMultiarchWith().Images([]Image{image1, image2}).Build() + + err := UploadMultiarchImage(multiImage, baseURL, repoName, "index1") + So(err, ShouldBeNil) + + // add another tag for one of the manifests + manifestBlob, err := json.Marshal(image2.Manifest) + So(err, ShouldBeNil) + + resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeImageManifest). + SetBody(manifestBlob).Put(baseURL + "/v2/" + repoName + "/manifests/manifest2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/manifest2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + // the repo should contain this index with a tag + tags, err := readTagsFromStorage(dir, repoName, multiImage.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "index1") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image1.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "manifest2") + So(len(tags), ShouldEqual, 1) + + // delete manifest by digest should fail + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + image1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed) + + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed) + + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "manifest2") + So(len(tags), ShouldEqual, 1) + + // delete manifest tag should work + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/manifest2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the manifest should not be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image2.Digest().Algorithm()), + image2.Digest().Encoded())) + So(err, ShouldBeNil) + + // delete tag of index should pass, but the index should be there + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the index should not be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(multiImage.Digest().Algorithm()), + multiImage.Digest().Encoded())) + So(err, ShouldBeNil) + + // The index should not be available by tag + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + + // The index should still be available by digest + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + multiImage.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + // the repo should contain this index without a tag + tags, err = readTagsFromStorage(dir, repoName, multiImage.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image1.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // add another tag + indexBlob, err := json.Marshal(multiImage.Index) + So(err, ShouldBeNil) + + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + SetBody(indexBlob).Put(baseURL + "/v2/" + repoName + "/manifests/index2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + // the repo should contain this index with just the new tag + tags, err = readTagsFromStorage(dir, repoName, multiImage.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "index2") + So(len(tags), ShouldEqual, 1) + + // delete tag of index should pass, but the index should be there + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/index2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the repo should contain this index without a tag + tags, err = readTagsFromStorage(dir, repoName, multiImage.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // delete index by digest + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + multiImage.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the index should not be available by digest + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + multiImage.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + + // the index should be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(multiImage.Digest().Algorithm()), + multiImage.Digest().Encoded())) + So(err, ShouldNotBeNil) + + // the repo should contain this index without a tag + tags, err = readTagsFromStorage(dir, repoName, multiImage.Digest()) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 0) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + // the manifest should not be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image1.Digest().Algorithm()), + image1.Digest().Encoded())) + So(err, ShouldBeNil) + + // the manifest should not be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image2.Digest().Algorithm()), + image2.Digest().Encoded())) + So(err, ShouldBeNil) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image1.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // delete manifest should succeed + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + image1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + + tags, err = readTagsFromStorage(dir, repoName, image1.Digest()) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 0) + + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 0) + + // the manifest should be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image1.Digest().Algorithm()), + image1.Digest().Encoded())) + So(err, ShouldNotBeNil) + + // the manifest should be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image2.Digest().Algorithm()), + image2.Digest().Encoded())) + So(err, ShouldNotBeNil) + }) + + Convey("Manifests used within multiple indexes should not be deletable", func() { + repoName := "multiarch" + + image1 := CreateRandomImage() + image2 := CreateRandomImage() + image3 := CreateRandomImage() + multiImage1 := CreateMultiarchWith().Images([]Image{image1, image2}).Build() + multiImage2 := CreateMultiarchWith().Images([]Image{image2, image3}).Build() + + err := UploadMultiarchImage(multiImage1, baseURL, repoName, "index1") + So(err, ShouldBeNil) + + err = UploadMultiarchImage(multiImage2, baseURL, repoName, "index2") + So(err, ShouldBeNil) + + // add another tag for one of the indexes + indexBlob, err := json.Marshal(multiImage2.Index) + So(err, ShouldBeNil) + + resp, err := resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + SetBody(indexBlob).Put(baseURL + "/v2/" + repoName + "/manifests/index22") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/index2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/index22") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + // the repo should contain this index with a tag + tags, err := readTagsFromStorage(dir, repoName, multiImage1.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "index1") + So(len(tags), ShouldEqual, 1) + + // the repo should contain this index with multiple tags + tags, err = readTagsFromStorage(dir, repoName, multiImage2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "index2") + So(tags, ShouldContain, "index22") + So(len(tags), ShouldEqual, 2) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image1.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image3.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // delete manifest by digest should fail + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed) + + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the manifest should not be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image2.Digest().Algorithm()), + image2.Digest().Encoded())) + So(err, ShouldBeNil) + + // delete tag of index should pass, but the index should be there + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/index2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the index should not be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(multiImage2.Digest().Algorithm()), + multiImage2.Digest().Encoded())) + So(err, ShouldBeNil) + + // The index should not be available by tag + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/index2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + + // The index should still be available by digest and the other tag + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/index22") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + multiImage2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + // the repo should contain this index with just 1 tag + tags, err = readTagsFromStorage(dir, repoName, multiImage2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "index22") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image1.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image3.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // add another tag + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + SetBody(indexBlob).Put(baseURL + "/v2/" + repoName + "/manifests/index222") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + // the repo should contain this index with both old and new tags + tags, err = readTagsFromStorage(dir, repoName, multiImage2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "index22") + So(tags, ShouldContain, "index222") + So(len(tags), ShouldEqual, 2) + + // delete manifest by digest should fail + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed) + + // delete 1st index by digest + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + multiImage1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the index should not be available by digest + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + multiImage1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + + // the index should be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(multiImage1.Digest().Algorithm()), + multiImage1.Digest().Encoded())) + So(err, ShouldNotBeNil) + + // the repo should contain this index without a tag + tags, err = readTagsFromStorage(dir, repoName, multiImage1.Digest()) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 0) + + // the repo should contain this index with 2 tags + tags, err = readTagsFromStorage(dir, repoName, multiImage2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "index22") + So(tags, ShouldContain, "index222") + So(len(tags), ShouldEqual, 2) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image3.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + multiImage2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + // the manifest should not be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image2.Digest().Algorithm()), + image2.Digest().Encoded())) + So(err, ShouldBeNil) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // delete manifest should succeed for only 1 manifest + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + image1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // this manifest is referenced by the other index + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed) + + // this manifest is referenced by the other index + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + image3.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed) + + tags, err = readTagsFromStorage(dir, repoName, image1.Digest()) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 0) + + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 1) + + tags, err = readTagsFromStorage(dir, repoName, image3.Digest()) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 1) + + // the manifest should be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image1.Digest().Algorithm()), + image1.Digest().Encoded())) + So(err, ShouldNotBeNil) + + // the manifest should not be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image2.Digest().Algorithm()), + image2.Digest().Encoded())) + So(err, ShouldBeNil) + + // delete 2nd index by digest + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + multiImage2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the repo should not contain this index + tags, err = readTagsFromStorage(dir, repoName, multiImage2.Digest()) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 0) + + // the index should be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(multiImage2.Digest().Algorithm()), + multiImage2.Digest().Encoded())) + So(err, ShouldNotBeNil) + + // delete manifest should succeed + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + image3.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image3.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 0) + + tags, err = readTagsFromStorage(dir, repoName, image3.Digest()) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 0) + + // the manifest should be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image2.Digest().Algorithm()), + image2.Digest().Encoded())) + So(err, ShouldNotBeNil) + + // the manifest should be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image3.Digest().Algorithm()), + image3.Digest().Encoded())) + So(err, ShouldNotBeNil) + }) + + Convey("Indexes referenced within other indexes should not be deleted", func() { + repoName := "multiarch" + + image1 := CreateRandomImage() + image2 := CreateRandomImage() + index1 := CreateMultiarchWith().Images([]Image{image1}).Build() + index2 := CreateMultiarchWith().Images([]Image{image2}).Build() + + err := UploadMultiarchImage(index1, baseURL, repoName, "index1") + So(err, ShouldBeNil) + + err = UploadMultiarchImage(index2, baseURL, repoName, index2.Digest().String()) + So(err, ShouldBeNil) + + rootIndex := ispec.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ispec.MediaTypeImageIndex, + Manifests: []ispec.Descriptor{ + { + Digest: index1.IndexDescriptor.Digest, + Size: index1.IndexDescriptor.Size, + MediaType: ispec.MediaTypeImageIndex, + }, + { + Digest: index2.IndexDescriptor.Digest, + Size: index2.IndexDescriptor.Size, + MediaType: ispec.MediaTypeImageIndex, + }, + }, + } + + indexBlob, err := json.Marshal(rootIndex) + So(err, ShouldBeNil) + + rootIndexDigest := godigest.FromBytes(indexBlob) + + resp, err := resty.R(). + SetHeader("Content-type", ispec.MediaTypeImageIndex). + SetBody(indexBlob). + Put(baseURL + "/v2/" + repoName + "/manifests/root") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/root") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Get(baseURL + "/v2/" + repoName + "/manifests/index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Get(baseURL + "/v2/" + repoName + "/manifests/root") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + index1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + index2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + rootIndexDigest.String()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Get(baseURL + "/v2/" + repoName + "/manifests/" + rootIndexDigest.String()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + + // the repo should contain this index with a tag + tags, err := readTagsFromStorage(dir, repoName, index1.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "index1") + So(len(tags), ShouldEqual, 1) + + // the repo should contain this index with a tag + tags, err = readTagsFromStorage(dir, repoName, index2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the repo should contain this index with a tag + tags, err = readTagsFromStorage(dir, repoName, rootIndexDigest) + So(err, ShouldBeNil) + So(tags, ShouldContain, "root") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image1.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // delete manifest by digest should fail + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + image1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed) + + tags, err = readTagsFromStorage(dir, repoName, image1.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image1.Digest().Algorithm()), + image1.Digest().Encoded())) + So(err, ShouldBeNil) + + // delete index by digest should fail as it is referenced in the root index + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + index1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed) + + tags, err = readTagsFromStorage(dir, repoName, index1.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "index1") + So(len(tags), ShouldEqual, 1) + + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(index1.Digest().Algorithm()), + index1.Digest().Encoded())) + So(err, ShouldBeNil) + + // delete index by digest should fail as it is referenced in the root index + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + index2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed) + + tags, err = readTagsFromStorage(dir, repoName, index2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(index2.Digest().Algorithm()), + index2.Digest().Encoded())) + So(err, ShouldBeNil) + + // delete tag of referenced index should pass, but the index should be there + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the index should not be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(index1.Digest().Algorithm()), + index1.Digest().Encoded())) + So(err, ShouldBeNil) + + // The index should not be available by tag + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/index1") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + + // The index should still be available by digest + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + index1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + // the repo should contain this index with just 1 tag + tags, err = readTagsFromStorage(dir, repoName, index1.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, index2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, rootIndexDigest) + So(err, ShouldBeNil) + So(tags, ShouldContain, "root") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image1.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the index should contain this manifest without any tags + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // add another tag to the second index + index2Blob, err := json.Marshal(index2.Index) + So(err, ShouldBeNil) + + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + SetBody(index2Blob).Put(baseURL + "/v2/" + repoName + "/manifests/index2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + // the index should contain this manifest with a single tag + tags, err = readTagsFromStorage(dir, repoName, index2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "index2") + So(len(tags), ShouldEqual, 1) + + // delete tag of referenced index should pass, but the index should be there + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/index2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the index should not be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(index2.Digest().Algorithm()), + index2.Digest().Encoded())) + So(err, ShouldBeNil) + + // The index should not be available by tag + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/index2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + + // The index should still be available by digest + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + index2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + // the index should contain this manifest with a single tag + tags, err = readTagsFromStorage(dir, repoName, index2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // delete manifest by digest should fail + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + index2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed) + + // add another tag to the root index + resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). + SetBody(indexBlob).Put(baseURL + "/v2/" + repoName + "/manifests/root2") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusCreated) + + // the repo should contain this index with both old and new tags + tags, err = readTagsFromStorage(dir, repoName, rootIndexDigest) + So(err, ShouldBeNil) + So(tags, ShouldContain, "root") + So(tags, ShouldContain, "root2") + So(len(tags), ShouldEqual, 2) + + // delete 1st root tag + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/root") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the repo should contain this index with only the new tag + tags, err = readTagsFromStorage(dir, repoName, rootIndexDigest) + So(err, ShouldBeNil) + So(tags, ShouldContain, "root2") + So(len(tags), ShouldEqual, 1) + + // the index should be available by digest + resp, err = resty.R().Get(baseURL + "/v2/" + repoName + "/manifests/" + rootIndexDigest.String()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + So(resp.Body(), ShouldNotBeEmpty) + So(resp.Header().Get("Content-Type"), ShouldNotBeEmpty) + + // delete root index by digest + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + rootIndexDigest.String()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the index should not be available by digest + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + rootIndexDigest.String()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + + // the index should be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(rootIndexDigest.Algorithm()), + rootIndexDigest.Encoded())) + So(err, ShouldNotBeNil) + + // the repo should not contain this root index + tags, err = readTagsFromStorage(dir, repoName, rootIndexDigest) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 0) + + // the repo should contain this index with no tags + tags, err = readTagsFromStorage(dir, repoName, index1.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the repo should contain this index with no tags + tags, err = readTagsFromStorage(dir, repoName, index2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the index should not be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(index1.Digest().Algorithm()), + index1.Digest().Encoded())) + So(err, ShouldBeNil) + + // the index should not be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(index2.Digest().Algorithm()), + index2.Digest().Encoded())) + So(err, ShouldBeNil) + + // the single arch images should continue to be available and we should not be allowed to delete them + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + // the manifest should not be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image1.Digest().Algorithm()), + image1.Digest().Encoded())) + So(err, ShouldBeNil) + + // the manifest should not be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image2.Digest().Algorithm()), + image2.Digest().Encoded())) + So(err, ShouldBeNil) + + // delete index1 by digest + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + index1.Digest().String()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the index should not be available by digest + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + index1.Digest().String()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + + // the index should be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(index1.Digest().Algorithm()), + index1.Digest().Encoded())) + So(err, ShouldNotBeNil) + + // the repo should not contain this index + tags, err = readTagsFromStorage(dir, repoName, index1.Digest()) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 0) + + // delete manifest should succeed for only 1 manifest + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + image1.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // this manifest is referenced by the other index + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed) + + tags, err = readTagsFromStorage(dir, repoName, image1.Digest()) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 0) + + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(tags, ShouldContain, "") + So(len(tags), ShouldEqual, 1) + + // the manifest should be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image1.Digest().Algorithm()), + image1.Digest().Encoded())) + So(err, ShouldNotBeNil) + + // the manifest should not be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image2.Digest().Algorithm()), + image2.Digest().Encoded())) + So(err, ShouldBeNil) + + // delete 2nd index by digest + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + index2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + // the repo should not contain this index + tags, err = readTagsFromStorage(dir, repoName, index2.Digest()) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 0) + + // the index should be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(index2.Digest().Algorithm()), + index2.Digest().Encoded())) + So(err, ShouldNotBeNil) + + // delete manifest should succeed + resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + + resp, err = resty.R().Head(baseURL + "/v2/" + repoName + "/manifests/" + image2.DigestStr()) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + + tags, err = readTagsFromStorage(dir, repoName, image2.Digest()) + So(err, ShouldBeNil) + So(len(tags), ShouldEqual, 0) + + // the manifest should be removed from storage + _, err = os.Stat(path.Join(dir, repoName, "blobs", string(image2.Digest().Algorithm()), + image2.Digest().Encoded())) + So(err, ShouldNotBeNil) + }) }) } @@ -8710,6 +9898,9 @@ func TestManifestImageIndex(t *testing.T) { So(resp.Body(), ShouldNotBeEmpty) resp, err = resty.R().Delete(baseURL + "/v2/index/manifests/test:index1") So(err, ShouldBeNil) + // Manifest is in use, tag is deleted, but manifest is not + So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) + So(resp.Body(), ShouldBeEmpty) resp, err = resty.R().SetHeader("Content-Type", ispec.MediaTypeImageIndex). Get(baseURL + "/v2/index/manifests/test:index1") So(err, ShouldBeNil) @@ -11769,3 +12960,34 @@ func getNumberOfSessions(rootDir string) (int, error) { return sessionsNo, nil } + +func readTagsFromStorage(rootDir, repoName string, digest godigest.Digest) ([]string, error) { + result := []string{} + + indexJSONBuf, err := os.ReadFile(path.Join(rootDir, repoName, "index.json")) + if err != nil { + return result, err + } + + var indexJSON ispec.Index + + err = json.Unmarshal(indexJSONBuf, &indexJSON) + if err != nil { + return result, err + } + + for _, desc := range indexJSON.Manifests { + if desc.Digest != digest { + continue + } + + name := desc.Annotations[ispec.AnnotationRefName] + // There is a special case where there is an entry in + // the index.json without tags, in this case name is an empty string + // Also we should not have duplicates + // Do these checks in the actual test cases, not here + result = append(result, name) + } + + return result, nil +} diff --git a/pkg/storage/common/common.go b/pkg/storage/common/common.go index 45009019..e28239d5 100644 --- a/pkg/storage/common/common.go +++ b/pkg/storage/common/common.go @@ -330,17 +330,21 @@ func RemoveManifestDescByReference(index *ispec.Index, reference string, detectC ) (ispec.Descriptor, error) { var removedManifest ispec.Descriptor - var found bool + var found, foundAsTag bool // keep track if the manifest was found by digest or by tag + manifestDigestCounts := map[godigest.Digest]int{} // Keep track of the number of references for a specific digest foundCount := 0 var outIndex ispec.Index for _, manifest := range index.Manifests { + manifestDigestCounts[manifest.Digest]++ + tag, ok := manifest.Annotations[ispec.AnnotationRefName] if ok && tag == reference { removedManifest = manifest found = true + foundAsTag = true foundCount++ continue @@ -361,6 +365,19 @@ func RemoveManifestDescByReference(index *ispec.Index, reference string, detectC return ispec.Descriptor{}, zerr.ErrManifestNotFound } + // In case of delete by digest we remove the manifest right away from storage + // but in case of delete by tag we want to only remove the tag + // and handle the manifest based on retention rules later. + // If there are more than one tags with the same digest we want to keep the others. + // If and only if there are no other tags except the one we want to remove + // we need to add a new descriptor without a tag name to keep track of + // the manifest in index.json so it remains accessible by digest. + if foundAsTag && manifestDigestCounts[removedManifest.Digest] == 1 { + newManifest := removedManifest + delete(newManifest.Annotations, ispec.AnnotationRefName) + outIndex.Manifests = append(outIndex.Manifests, newManifest) + } + index.Manifests = outIndex.Manifests return removedManifest, nil @@ -515,6 +532,11 @@ func IsBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string, switch desc.MediaType { case ispec.MediaTypeImageIndex: + if digest == desc.Digest { + // no need to look further if we have a match + return true, nil + } + indexImage, err := GetImageIndex(imgStore, repo, desc.Digest, log) if err != nil { log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go index 64f0f5d6..6970b70e 100644 --- a/pkg/storage/imagestore/imagestore.go +++ b/pkg/storage/imagestore/imagestore.go @@ -554,10 +554,14 @@ func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //noli dir := path.Join(is.rootDir, repo, ispec.ImageBlobsDir, mDigest.Algorithm().String()) manifestPath := path.Join(dir, mDigest.Encoded()) - if _, err = is.storeDriver.WriteFile(manifestPath, body); err != nil { - is.log.Error().Err(err).Str("file", manifestPath).Msg("failed to write") + binfo, err := is.storeDriver.Stat(manifestPath) + if err != nil || binfo.Size() != desc.Size { + // The blob isn't already there, or it is corrupted, and needs a correction + if _, err = is.storeDriver.WriteFile(manifestPath, body); err != nil { + is.log.Error().Err(err).Str("file", manifestPath).Msg("failed to write") - return "", "", err + return "", "", err + } } err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) @@ -566,6 +570,14 @@ func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //noli } // now update "index.json" + for midx, manifest := range index.Manifests { + _, ok := manifest.Annotations[ispec.AnnotationRefName] + if !ok && manifest.Digest.String() == desc.Digest.String() { + // matching descriptor does not have a tag, we need to remove it and add the new descriptor + index.Manifests = append(index.Manifests[:midx], index.Manifests[midx+1:]...) + } + } + index.Manifests = append(index.Manifests, desc) // update the descriptors artifact type in order to check for signatures when applying the linter @@ -626,7 +638,8 @@ func (is *ImageStore) deleteImageManifest(repo, reference string, detectCollisio /* check if manifest is referenced in image indexes, do not allow index images manipulations (ie. remove manifest being part of an image index) */ - if manifestDesc.MediaType == ispec.MediaTypeImageManifest { + if zcommon.IsDigest(reference) && + (manifestDesc.MediaType == ispec.MediaTypeImageManifest || manifestDesc.MediaType == ispec.MediaTypeImageIndex) { for _, mDesc := range index.Manifests { if mDesc.MediaType == ispec.MediaTypeImageIndex { if ok, _ := common.IsBlobReferencedInImageIndex(is, repo, manifestDesc.Digest, ispec.Index{ diff --git a/pkg/storage/local/local_test.go b/pkg/storage/local/local_test.go index ce603fef..610b6804 100644 --- a/pkg/storage/local/local_test.go +++ b/pkg/storage/local/local_test.go @@ -1239,11 +1239,11 @@ func TestDedupeLinks(t *testing.T) { manifestBuf, err = json.Marshal(manifest) So(err, ShouldBeNil) - digest = godigest.FromBytes(manifestBuf) + manifestDigest2 := godigest.FromBytes(manifestBuf) _, _, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest, manifestBuf) So(err, ShouldBeNil) - _, _, _, err = imgStore.GetImageManifest("dedupe2", digest.String()) + _, _, _, err = imgStore.GetImageManifest("dedupe2", manifestDigest2.String()) So(err, ShouldBeNil) // verify that dedupe with hard links happened @@ -1267,6 +1267,15 @@ func TestDedupeLinks(t *testing.T) { err = imgStore.DeleteBlob("dedupe1", godigest.NewDigestFromEncoded(godigest.SHA256, blobDigest1)) So(err, ShouldBeNil) + // only the tag was removed, but not the digest, this call should error + err = imgStore.DeleteBlob("dedupe2", godigest.NewDigestFromEncoded(godigest.SHA256, blobDigest2)) + So(err, ShouldNotBeNil) + + // Delete the manifest + err = imgStore.DeleteImageManifest("dedupe2", manifestDigest2.String(), false) + So(err, ShouldBeNil) + + // The call should succeed err = imgStore.DeleteBlob("dedupe2", godigest.NewDigestFromEncoded(godigest.SHA256, blobDigest2)) So(err, ShouldBeNil) }) @@ -1422,6 +1431,15 @@ func TestDedupeLinks(t *testing.T) { err = imgStore.DeleteBlob("dedupe1", godigest.NewDigestFromEncoded(godigest.SHA256, blobDigest1)) So(err, ShouldBeNil) + // only the tag was removed, but not the digest, this call should error + err = imgStore.DeleteBlob("dedupe2", godigest.NewDigestFromEncoded(godigest.SHA256, blobDigest2)) + So(err, ShouldNotBeNil) + + // Delete the manifest + err = imgStore.DeleteImageManifest("dedupe2", manifestDigest2.String(), false) + So(err, ShouldBeNil) + + // The call should succeed err = imgStore.DeleteBlob("dedupe2", godigest.NewDigestFromEncoded(godigest.SHA256, blobDigest2)) So(err, ShouldBeNil) }) diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 85d0fbeb..39798ee1 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -1307,12 +1307,12 @@ func TestS3Dedupe(t *testing.T) { manifestBuf, err = json.Marshal(manifest) So(err, ShouldBeNil) - digest = godigest.FromBytes(manifestBuf) + manifestDigest2 := godigest.FromBytes(manifestBuf) _, _, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest, manifestBuf) So(err, ShouldBeNil) - _, _, _, err = imgStore.GetImageManifest("dedupe2", digest.String()) + _, _, _, err = imgStore.GetImageManifest("dedupe2", manifestDigest2.String()) So(err, ShouldBeNil) fi1, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256", @@ -1336,12 +1336,21 @@ func TestS3Dedupe(t *testing.T) { err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false) So(err, ShouldBeNil) + // delete tag, but not manifest err = imgStore.DeleteImageManifest("dedupe2", "1.0", false) So(err, ShouldBeNil) + // delete should succeed as the manifest was deleted err = imgStore.DeleteBlob("dedupe1", blobDigest1) So(err, ShouldBeNil) + // delete should fail, as the blob is referenced by an untagged manifest + err = imgStore.DeleteBlob("dedupe2", blobDigest2) + So(err, ShouldEqual, zerr.ErrBlobReferenced) + + err = imgStore.DeleteImageManifest("dedupe2", manifestDigest2.String(), false) + So(err, ShouldBeNil) + err = imgStore.DeleteBlob("dedupe2", blobDigest2) So(err, ShouldBeNil) }) @@ -1351,7 +1360,7 @@ func TestS3Dedupe(t *testing.T) { err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false) So(err, ShouldBeNil) - err = imgStore.DeleteImageManifest("dedupe2", "1.0", false) + err = imgStore.DeleteImageManifest("dedupe2", manifestDigest2.String(), false) So(err, ShouldBeNil) // if we delete blob1, the content should be moved to blob2 @@ -1470,12 +1479,12 @@ func TestS3Dedupe(t *testing.T) { manifestBuf, err = json.Marshal(manifest) So(err, ShouldBeNil) - digest = godigest.FromBytes(manifestBuf) + manifestDigest3 := godigest.FromBytes(manifestBuf) _, _, err = imgStore.PutImageManifest("dedupe3", "1.0", ispec.MediaTypeImageManifest, manifestBuf) So(err, ShouldBeNil) - _, _, _, err = imgStore.GetImageManifest("dedupe3", digest.String()) + _, _, _, err = imgStore.GetImageManifest("dedupe3", manifestDigest3.String()) So(err, ShouldBeNil) fi1, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256", @@ -1500,10 +1509,10 @@ func TestS3Dedupe(t *testing.T) { err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false) So(err, ShouldBeNil) - err = imgStore.DeleteImageManifest("dedupe2", "1.0", false) + err = imgStore.DeleteImageManifest("dedupe2", manifestDigest2.String(), false) So(err, ShouldBeNil) - err = imgStore.DeleteImageManifest("dedupe3", "1.0", false) + err = imgStore.DeleteImageManifest("dedupe3", manifestDigest3.String(), false) So(err, ShouldBeNil) err = imgStore.DeleteBlob("dedupe1", blobDigest1) @@ -1715,12 +1724,12 @@ func TestS3Dedupe(t *testing.T) { manifestBuf, err = json.Marshal(manifest) So(err, ShouldBeNil) - digest = godigest.FromBytes(manifestBuf) + manifestDigest2 := godigest.FromBytes(manifestBuf) _, _, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest, manifestBuf) So(err, ShouldBeNil) - _, _, _, err = imgStore.GetImageManifest("dedupe2", digest.String()) + _, _, _, err = imgStore.GetImageManifest("dedupe2", manifestDigest2.String()) So(err, ShouldBeNil) fi1, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256", @@ -1744,12 +1753,21 @@ func TestS3Dedupe(t *testing.T) { err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false) So(err, ShouldBeNil) + // delete tag, but not manifest err = imgStore.DeleteImageManifest("dedupe2", "1.0", false) So(err, ShouldBeNil) + // Delete should succeed as the manifest was deleted err = imgStore.DeleteBlob("dedupe1", blobDigest1) So(err, ShouldBeNil) + // Delete should fail, as the blob is referenced by an untagged manifest + err = imgStore.DeleteBlob("dedupe2", blobDigest2) + So(err, ShouldEqual, zerr.ErrBlobReferenced) + + err = imgStore.DeleteImageManifest("dedupe2", manifestDigest2.String(), false) + So(err, ShouldBeNil) + err = imgStore.DeleteBlob("dedupe2", blobDigest2) So(err, ShouldBeNil) }) @@ -1790,12 +1808,21 @@ func TestS3Dedupe(t *testing.T) { err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false) So(err, ShouldBeNil) + // delete tag, but not manifest err = imgStore.DeleteImageManifest("dedupe2", "1.0", false) So(err, ShouldBeNil) + // delete should succeed as the manifest was deleted err = imgStore.DeleteBlob("dedupe1", blobDigest1) So(err, ShouldBeNil) + // delete should fail, as the blob is referenced by an untagged manifest + err = imgStore.DeleteBlob("dedupe2", blobDigest2) + So(err, ShouldEqual, zerr.ErrBlobReferenced) + + err = imgStore.DeleteImageManifest("dedupe2", manifestDigest2.String(), false) + So(err, ShouldBeNil) + err = imgStore.DeleteBlob("dedupe2", blobDigest2) So(err, ShouldBeNil) }) @@ -1832,7 +1859,7 @@ func TestS3Dedupe(t *testing.T) { err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false) So(err, ShouldBeNil) - err = imgStore.DeleteImageManifest("dedupe2", "1.0", false) + err = imgStore.DeleteImageManifest("dedupe2", manifestDigest2.String(), false) So(err, ShouldBeNil) err = imgStore.DeleteBlob("dedupe1", blobDigest1)