fix: don't allow blobs to be deleted if in use (#1559)

dist-spec APIs independently allow deletion of blobs and manifests.
Doing the former when in use by an image manifest or index is simply
error-prone. So disallow it.

Fixes issue #1509

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>
Co-authored-by: Ramkumar Chinchani <rchincha@cisco.com>

.github/workflows/oci-conformance-action.yml

@@ -34,8 +34,19 @@ jobs:
           RUNNER_TRACKING_ID="" && ./bin/zot-linux-amd64 serve examples/config-conformance.json &
           IP=`hostname -I | awk '{print $1}'`
           echo "SERVER_URL=http://${IP}:8080" >> $GITHUB_ENV
-    - name: Run OCI Distribution Spec conformance tests
+    - uses: actions/checkout@v3
-      uses: opencontainers/distribution-spec@main
+      with:
+        # TODO: change to upstream once the foloowing PR is merged:
+        # https://github.com/opencontainers/distribution-spec/pull/436
+        repository: sudo-bmitch/distribution-spec
+        ref: pr-conformance-index-subject
+        path: distribution-spec
+    - name: build conformance binary from main
+      run: |
+        (cd distribution-spec/ && make conformance-binary)
+        mv distribution-spec/output/conformance.test .
+        rm -rf distribution-spec/
+    - name: run conformance
       env:
         OCI_ROOT_URL: ${{ env.SERVER_URL }}
         OCI_NAMESPACE: oci-conformance/distribution-test
@@ -44,13 +55,14 @@ jobs:
         OCI_TEST_CONTENT_DISCOVERY: 1
         OCI_TEST_CONTENT_MANAGEMENT: 1
         OCI_REFERRERS: 1
-        OCI_HIDE_SKIPPED_WORKFLOWS: 1
+        OCI_CROSSMOUNT_NAMESPACE: oci-conformance/crossmount-test
+      run: |
+        ./conformance.test 
     - run: mkdir -p .out/ && mv {report.html,junit.xml} .out/
       if: always()
-      #run: docker run --rm -v $(pwd)/results:/results -w /results -e OCI_ROOT_URL=${{ env.OCI_ROOT_URL }} -e OCI_NAMESPACE="anuvu/zot" -e OCI_TEST_PULL=1 -e OCI_TEST_PUSH=1 -e OCI_TEST_CONTENT_DISCOVERY=1 -e OCI_TEST_CONTENT_MANAGEMENT=1 -e OCI_HIDE_SKIPPED_WORKFLOWS=0 -e OCI_DEBUG="true" ghcr.io/opencontainers/distribution-spec/conformance:db4cc68
     - name: Upload test results zip as build artifact
       uses: actions/upload-artifact@v3
       with:
         name: oci-test-results-${{ github.sha }}
         path: .out/
-      if: github.event == 'push'
+      if: github.event_name == 'push'

cmd/zb/helper.go

@@ -77,6 +77,12 @@ func deleteTestRepo(repos []string, url string, client *resty.Client) error {
 				return err
 			}
 
+			// delete manifest so that we don't trigger BlobInUse error
+			err = makeHTTPDeleteRequest(fmt.Sprintf("%s/v2/%s/manifests/%s", url, repo, tag), client)
+			if err != nil {
+				return err
+			}
+
 			// delete blobs
 			for _, blob := range manifest.Layers {
 				err := makeHTTPDeleteRequest(fmt.Sprintf("%s/v2/%s/blobs/%s", url, repo, blob.Digest.String()), client)
@@ -90,12 +96,6 @@ func deleteTestRepo(repos []string, url string, client *resty.Client) error {
 			if err != nil {
 				return err
 			}
-
-			// delete manifest
-			err = makeHTTPDeleteRequest(fmt.Sprintf("%s/v2/%s/manifests/%s", url, repo, tag), client)
-			if err != nil {
-				return err
-			}
 		}
 	}
 

errors/errors.go

@@ -16,6 +16,7 @@ var (
 	ErrBlobNotFound                   = errors.New("blob: not found")
 	ErrBadBlob                        = errors.New("blob: bad blob")
 	ErrBadBlobDigest                  = errors.New("blob: bad blob digest")
+	ErrBlobReferenced                 = errors.New("blob: referenced by manifest")
 	ErrUnknownCode                    = errors.New("error: unknown error code")
 	ErrBadCACert                      = errors.New("tls: invalid ca cert")
 	ErrBadUser                        = errors.New("auth: non-existent user")

pkg/api/controller_test.go

@@ -901,6 +901,75 @@ func TestBasicAuth(t *testing.T) {
 	})
 }
 
+func TestBlobReferenced(t *testing.T) {
+	Convey("Make a new controller", t, func() {
+		port := test.GetFreePort()
+		baseURL := test.GetBaseURL(port)
+		conf := config.New()
+		conf.HTTP.Port = port
+
+		ctlr := makeController(conf, t.TempDir(), "")
+
+		cm := test.NewControllerManager(ctlr)
+		cm.StartAndWait(port)
+		defer cm.StopServer()
+
+		// without creds, should get access error
+		resp, err := resty.R().Get(baseURL + "/v2/")
+		So(err, ShouldBeNil)
+		So(resp, ShouldNotBeNil)
+		So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+
+		repoName := "repo"
+
+		cfg, layers, manifest, err := test.GetImageComponents(2)
+		So(err, ShouldBeNil)
+
+		err = test.UploadImage(
+			test.Image{
+				Config:    cfg,
+				Layers:    layers,
+				Manifest:  manifest,
+				Reference: "1.0",
+			}, baseURL, repoName)
+		So(err, ShouldBeNil)
+
+		manifestContent, err := json.Marshal(manifest)
+		So(err, ShouldBeNil)
+		manifestDigest := godigest.FromBytes(manifestContent)
+		So(manifestDigest, ShouldNotBeNil)
+
+		configContent, err := json.Marshal(cfg)
+		So(err, ShouldBeNil)
+		configDigest := godigest.FromBytes(configContent)
+		So(configDigest, ShouldNotBeNil)
+
+		// delete manifest blob
+		resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/blobs/" + manifestDigest.String())
+		So(err, ShouldBeNil)
+		So(resp, ShouldNotBeNil)
+		So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed)
+
+		// delete config blob
+		resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/blobs/" + configDigest.String())
+		So(err, ShouldBeNil)
+		So(resp, ShouldNotBeNil)
+		So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed)
+
+		// delete manifest with manifest api method
+		resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/manifests/" + manifestDigest.String())
+		So(err, ShouldBeNil)
+		So(resp, ShouldNotBeNil)
+		So(resp.StatusCode(), ShouldEqual, http.StatusAccepted)
+
+		// delete blob should work after manifest is deleted
+		resp, err = resty.R().Delete(baseURL + "/v2/" + repoName + "/blobs/" + configDigest.String())
+		So(err, ShouldBeNil)
+		So(resp, ShouldNotBeNil)
+		So(resp.StatusCode(), ShouldEqual, http.StatusAccepted)
+	})
+}
+
 func TestInterruptedBlobUpload(t *testing.T) {
 	Convey("Successfully cleaning interrupted blob uploads", t, func() {
 		port := test.GetFreePort()

pkg/api/routes.go

@@ -1091,6 +1091,10 @@ func (rh *RouteHandler) DeleteBlob(response http.ResponseWriter, request *http.R
 			zcommon.WriteJSON(response,
 				http.StatusNotFound,
 				apiErr.NewErrorList(apiErr.NewError(apiErr.BLOB_UNKNOWN, map[string]string{".String()": digest.String()})))
+		} else if errors.Is(err, zerr.ErrBlobReferenced) {
+			zcommon.WriteJSON(response,
+				http.StatusMethodNotAllowed,
+				apiErr.NewErrorList(apiErr.NewError(apiErr.DENIED, map[string]string{".String()": digest.String()})))
 		} else {
 			rh.c.Log.Error().Err(err).Msg("unexpected error")
 			response.WriteHeader(http.StatusInternalServerError)

pkg/storage/common/common.go

@@ -457,6 +457,81 @@ func PruneImageManifestsFromIndex(imgStore storageTypes.ImageStore, repo string,
 	return prunedManifests, nil
 }
 
+func isBlobReferencedInManifest(imgStore storageTypes.ImageStore, repo string,
+	bdigest, mdigest godigest.Digest, log zerolog.Logger,
+) (bool, error) {
+	if bdigest == mdigest {
+		return true, nil
+	}
+
+	manifestContent, err := GetImageManifest(imgStore, repo, mdigest, log)
+	if err != nil {
+		log.Error().Err(err).Str("repo", repo).Str("digest", mdigest.String()).
+			Msg("gc: failed to read manifest image")
+
+		return false, err
+	}
+
+	if bdigest == manifestContent.Config.Digest {
+		return true, nil
+	}
+
+	for _, layer := range manifestContent.Layers {
+		if bdigest == layer.Digest {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func isBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string,
+	digest godigest.Digest, index ispec.Index, log zerolog.Logger,
+) (bool, error) {
+	for _, desc := range index.Manifests {
+		var found bool
+
+		switch desc.MediaType {
+		case ispec.MediaTypeImageIndex:
+			/* this branch is not needed, because every manifests in index is already checked
+			when this one is hit, all manifests are referenced in index.json */
+			indexImage, err := GetImageIndex(imgStore, repo, desc.Digest, log)
+			if err != nil {
+				log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()).
+					Msg("failed to read multiarch(index) image")
+
+				return false, err
+			}
+
+			found, _ = isBlobReferencedInImageIndex(imgStore, repo, digest, indexImage, log)
+		case ispec.MediaTypeImageManifest:
+			found, _ = isBlobReferencedInManifest(imgStore, repo, digest, desc.Digest, log)
+		}
+
+		if found {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+func IsBlobReferenced(imgStore storageTypes.ImageStore, repo string,
+	digest godigest.Digest, log zerolog.Logger,
+) (bool, error) {
+	dir := path.Join(imgStore.RootDir(), repo)
+	if !imgStore.DirExists(dir) {
+		return false, zerr.ErrRepoNotFound
+	}
+
+	index, err := GetIndex(imgStore, repo, log)
+	if err != nil {
+		return false, err
+	}
+
+	return isBlobReferencedInImageIndex(imgStore, repo, digest, index, log)
+}
+
 func ApplyLinter(imgStore storageTypes.ImageStore, linter Lint, repo string, descriptor ispec.Descriptor,
 ) (bool, error) {
 	pass := true

pkg/storage/local/local.go

@@ -1408,6 +1408,11 @@ func (is *ImageStoreLocal) DeleteBlob(repo string, digest godigest.Digest) error
 		return zerr.ErrBlobNotFound
 	}
 
+	// first check if this blob is not currently in use
+	if ok, _ := common.IsBlobReferenced(is, repo, digest, is.log); ok {
+		return zerr.ErrBlobReferenced
+	}
+
 	if fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) {
 		if err := is.cache.DeleteBlob(digest, blobPath); err != nil {
 			is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", blobPath).

pkg/storage/local/local_test.go

@@ -1169,6 +1169,7 @@ func TestDedupeLinks(t *testing.T) {
 	metrics := monitoring.NewMetricsServer(false, log)
 
 	for _, testCase := range testCases {
+		Convey(fmt.Sprintf("Dedupe %t", testCase.dedupe), t, func(c C) {
 			dir := t.TempDir()
 
 			cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
@@ -1187,7 +1188,6 @@ func TestDedupeLinks(t *testing.T) {
 					testCase.dedupe, true, log, metrics, nil, nil)
 			}
 
-		Convey(fmt.Sprintf("Dedupe %t", testCase.dedupe), t, func(c C) {
 			// manifest1
 			upload, err := imgStore.NewBlobUpload("dedupe1")
 			So(err, ShouldBeNil)
@@ -1240,12 +1240,12 @@ func TestDedupeLinks(t *testing.T) {
 			manifest.SchemaVersion = 2
 			manifestBuf, err := json.Marshal(manifest)
 			So(err, ShouldBeNil)
-			digest = godigest.FromBytes(manifestBuf)
+			manifestDigest := godigest.FromBytes(manifestBuf)
-			_, _, err = imgStore.PutImageManifest("dedupe1", digest.String(),
+			_, _, err = imgStore.PutImageManifest("dedupe1", manifestDigest.String(),
 				ispec.MediaTypeImageManifest, manifestBuf)
 			So(err, ShouldBeNil)
 
-			_, _, _, err = imgStore.GetImageManifest("dedupe1", digest.String())
+			_, _, _, err = imgStore.GetImageManifest("dedupe1", manifestDigest.String())
 			So(err, ShouldBeNil)
 
 			// manifest2
@@ -1318,6 +1318,13 @@ func TestDedupeLinks(t *testing.T) {
 				Convey("delete blobs from storage/cache should work when dedupe is false", func() {
 					So(blobDigest1, ShouldEqual, blobDigest2)
 
+					// to not trigger BlobInUse err, delete manifest first
+					err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false)
+					So(err, ShouldBeNil)
+
+					err = imgStore.DeleteImageManifest("dedupe2", "1.0", false)
+					So(err, ShouldBeNil)
+
 					err = imgStore.DeleteBlob("dedupe1", godigest.NewDigestFromEncoded(godigest.SHA256, blobDigest1))
 					So(err, ShouldBeNil)
 
@@ -1466,6 +1473,13 @@ func TestDedupeLinks(t *testing.T) {
 			Convey("delete blobs from storage/cache should work when dedupe is true", func() {
 				So(blobDigest1, ShouldEqual, blobDigest2)
 
+				// to not trigger BlobInUse err, delete manifest first
+				err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false)
+				So(err, ShouldBeNil)
+
+				err = imgStore.DeleteImageManifest("dedupe2", "1.0", false)
+				So(err, ShouldBeNil)
+
 				err = imgStore.DeleteBlob("dedupe1", godigest.NewDigestFromEncoded(godigest.SHA256, blobDigest1))
 				So(err, ShouldBeNil)
 

pkg/storage/s3/s3.go

@@ -1424,6 +1424,11 @@ func (is *ObjectStorage) DeleteBlob(repo string, digest godigest.Digest) error {
 		return zerr.ErrBlobNotFound
 	}
 
+	// first check if this blob is not currently in use
+	if ok, _ := common.IsBlobReferenced(is, repo, digest, is.log); ok {
+		return zerr.ErrBlobReferenced
+	}
+
 	if fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) {
 		dstRecord, err := is.cache.GetBlob(digest)
 		if err != nil && !errors.Is(err, zerr.ErrCacheMiss) {

pkg/storage/s3/s3_test.go

@@ -1316,12 +1316,12 @@ func TestS3Dedupe(t *testing.T) {
 		manifest.SchemaVersion = 2
 		manifestBuf, err := json.Marshal(manifest)
 		So(err, ShouldBeNil)
-		digest = godigest.FromBytes(manifestBuf)
+		manifestDigest := godigest.FromBytes(manifestBuf)
-		_, _, err = imgStore.PutImageManifest("dedupe1", digest.String(),
+		_, _, err = imgStore.PutImageManifest("dedupe1", manifestDigest.String(),
 			ispec.MediaTypeImageManifest, manifestBuf)
 		So(err, ShouldBeNil)
 
-		_, _, _, err = imgStore.GetImageManifest("dedupe1", digest.String())
+		_, _, _, err = imgStore.GetImageManifest("dedupe1", manifestDigest.String())
 		So(err, ShouldBeNil)
 
 		// manifest2
@@ -1415,6 +1415,13 @@ func TestS3Dedupe(t *testing.T) {
 		Convey("delete blobs from storage/cache should work when dedupe is true", func() {
 			So(blobDigest1, ShouldEqual, blobDigest2)
 
+			// to not trigger BlobInUse err, delete manifest first
+			err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false)
+			So(err, ShouldBeNil)
+
+			err = imgStore.DeleteImageManifest("dedupe2", "1.0", false)
+			So(err, ShouldBeNil)
+
 			err = imgStore.DeleteBlob("dedupe1", blobDigest1)
 			So(err, ShouldBeNil)
 
@@ -1423,6 +1430,13 @@ func TestS3Dedupe(t *testing.T) {
 		})
 
 		Convey("Check that delete blobs moves the real content to the next contenders", func() {
+			// to not trigger BlobInUse err, delete manifest first
+			err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false)
+			So(err, ShouldBeNil)
+
+			err = imgStore.DeleteImageManifest("dedupe2", "1.0", false)
+			So(err, ShouldBeNil)
+
 			// if we delete blob1, the content should be moved to blob2
 			err = imgStore.DeleteBlob("dedupe1", blobDigest1)
 			So(err, ShouldBeNil)
@@ -1561,6 +1575,15 @@ func TestS3Dedupe(t *testing.T) {
 
 			Convey("delete blobs from storage/cache should work when dedupe is false", func() {
 				So(blobDigest1, ShouldEqual, blobDigest2)
+				// to not trigger BlobInUse err, delete manifest first
+				err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false)
+				So(err, ShouldBeNil)
+
+				err = imgStore.DeleteImageManifest("dedupe2", "1.0", false)
+				So(err, ShouldBeNil)
+
+				err = imgStore.DeleteImageManifest("dedupe3", "1.0", false)
+				So(err, ShouldBeNil)
 
 				err = imgStore.DeleteBlob("dedupe1", blobDigest1)
 				So(err, ShouldBeNil)
@@ -1696,12 +1719,12 @@ func TestS3Dedupe(t *testing.T) {
 		manifest.SchemaVersion = 2
 		manifestBuf, err := json.Marshal(manifest)
 		So(err, ShouldBeNil)
-		digest = godigest.FromBytes(manifestBuf)
+		manifestDigest := godigest.FromBytes(manifestBuf)
-		_, _, err = imgStore.PutImageManifest("dedupe1", digest.String(),
+		_, _, err = imgStore.PutImageManifest("dedupe1", manifestDigest.String(),
 			ispec.MediaTypeImageManifest, manifestBuf)
 		So(err, ShouldBeNil)
 
-		_, _, _, err = imgStore.GetImageManifest("dedupe1", digest.String())
+		_, _, _, err = imgStore.GetImageManifest("dedupe1", manifestDigest.String())
 		So(err, ShouldBeNil)
 
 		// manifest2
@@ -1787,6 +1810,13 @@ func TestS3Dedupe(t *testing.T) {
 		Convey("delete blobs from storage/cache should work when dedupe is true", func() {
 			So(blobDigest1, ShouldEqual, blobDigest2)
 
+			// to not trigger BlobInUse err, delete manifest first
+			err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false)
+			So(err, ShouldBeNil)
+
+			err = imgStore.DeleteImageManifest("dedupe2", "1.0", false)
+			So(err, ShouldBeNil)
+
 			err = imgStore.DeleteBlob("dedupe1", blobDigest1)
 			So(err, ShouldBeNil)
 
@@ -1825,6 +1855,13 @@ func TestS3Dedupe(t *testing.T) {
 			Convey("delete blobs from storage/cache should work when dedupe is false", func() {
 				So(blobDigest1, ShouldEqual, blobDigest2)
 
+				// to not trigger BlobInUse err, delete manifest first
+				err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false)
+				So(err, ShouldBeNil)
+
+				err = imgStore.DeleteImageManifest("dedupe2", "1.0", false)
+				So(err, ShouldBeNil)
+
 				err = imgStore.DeleteBlob("dedupe1", blobDigest1)
 				So(err, ShouldBeNil)
 
@@ -1859,6 +1896,13 @@ func TestS3Dedupe(t *testing.T) {
 
 		Convey("Check that delete blobs moves the real content to the next contenders", func() {
 			// if we delete blob1, the content should be moved to blob2
+			// to not trigger BlobInUse err, delete manifest first
+			err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false)
+			So(err, ShouldBeNil)
+
+			err = imgStore.DeleteImageManifest("dedupe2", "1.0", false)
+			So(err, ShouldBeNil)
+
 			err = imgStore.DeleteBlob("dedupe1", blobDigest1)
 			So(err, ShouldBeNil)
 

pkg/storage/storage_test.go

@@ -26,10 +26,12 @@ import (
 	. "github.com/smartystreets/goconvey/convey"
 	"gopkg.in/resty.v1"
 
+	zerr "zotregistry.io/zot/errors"
 	"zotregistry.io/zot/pkg/extensions/monitoring"
 	"zotregistry.io/zot/pkg/log"
 	"zotregistry.io/zot/pkg/storage"
 	"zotregistry.io/zot/pkg/storage/cache"
+	storageCommon "zotregistry.io/zot/pkg/storage/common"
 	storageConstants "zotregistry.io/zot/pkg/storage/constants"
 	"zotregistry.io/zot/pkg/storage/local"
 	"zotregistry.io/zot/pkg/storage/s3"
@@ -844,6 +846,297 @@ func TestMandatoryAnnotations(t *testing.T) {
 	}
 }
 
+func TestDeleteBlobsInUse(t *testing.T) {
+	for _, testcase := range testCases {
+		testcase := testcase
+		t.Run(testcase.testCaseName, func(t *testing.T) {
+			var imgStore storageTypes.ImageStore
+			var testDir, tdir string
+			var store driver.StorageDriver
+
+			log := log.Logger{Logger: zerolog.New(os.Stdout)}
+			metrics := monitoring.NewMetricsServer(false, log)
+
+			if testcase.storageType == "s3" {
+				skipIt(t)
+
+				uuid, err := guuid.NewV4()
+				if err != nil {
+					panic(err)
+				}
+
+				testDir = path.Join("/oci-repo-test", uuid.String())
+				tdir = t.TempDir()
+
+				store, imgStore, _ = createObjectsStore(testDir, tdir)
+
+				defer cleanupStorage(store, testDir)
+			} else {
+				tdir = t.TempDir()
+				cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{
+					RootDir:     tdir,
+					Name:        "cache",
+					UseRelPaths: true,
+				}, log)
+				imgStore = local.NewImageStore(tdir, true, storageConstants.DefaultGCDelay, true,
+					true, log, metrics, nil, cacheDriver)
+			}
+
+			Convey("Setup manifest", t, func() {
+				// put an unused blob
+				content := []byte("unused blob")
+				buf := bytes.NewBuffer(content)
+				unusedDigest := godigest.FromBytes(content)
+
+				_, _, err := imgStore.FullBlobUpload("repo", bytes.NewReader(buf.Bytes()), unusedDigest)
+				So(err, ShouldBeNil)
+
+				content = []byte("test-data1")
+				buf = bytes.NewBuffer(content)
+				buflen := buf.Len()
+				digest := godigest.FromBytes(content)
+
+				_, _, err = imgStore.FullBlobUpload("repo", bytes.NewReader(buf.Bytes()), digest)
+				So(err, ShouldBeNil)
+
+				cblob, cdigest := test.GetRandomImageConfig()
+				_, clen, err := imgStore.FullBlobUpload("repo", bytes.NewReader(cblob), cdigest)
+				So(err, ShouldBeNil)
+				So(clen, ShouldEqual, len(cblob))
+
+				annotationsMap := make(map[string]string)
+				annotationsMap[ispec.AnnotationRefName] = tag
+
+				manifest := ispec.Manifest{
+					Config: ispec.Descriptor{
+						MediaType: "application/vnd.oci.image.config.v1+json",
+						Digest:    cdigest,
+						Size:      int64(len(cblob)),
+					},
+					Layers: []ispec.Descriptor{
+						{
+							MediaType: "application/vnd.oci.image.layer.v1.tar",
+							Digest:    digest,
+							Size:      int64(buflen),
+						},
+					},
+					Annotations: annotationsMap,
+				}
+
+				manifest.SchemaVersion = 2
+				manifestBuf, err := json.Marshal(manifest)
+				So(err, ShouldBeNil)
+
+				manifestDigest, _, err := imgStore.PutImageManifest("repo", tag, ispec.MediaTypeImageManifest, manifestBuf)
+				So(err, ShouldBeNil)
+
+				Convey("Try to delete blob currently in use", func() {
+					// layer blob
+					err := imgStore.DeleteBlob("repo", digest)
+					So(err, ShouldEqual, zerr.ErrBlobReferenced)
+
+					// manifest
+					err = imgStore.DeleteBlob("repo", manifestDigest)
+					So(err, ShouldEqual, zerr.ErrBlobReferenced)
+
+					// config
+					err = imgStore.DeleteBlob("repo", cdigest)
+					So(err, ShouldEqual, zerr.ErrBlobReferenced)
+				})
+
+				Convey("Delete unused blob", func() {
+					err := imgStore.DeleteBlob("repo", unusedDigest)
+					So(err, ShouldBeNil)
+				})
+
+				Convey("Delete manifest first, then blob", func() {
+					err := imgStore.DeleteImageManifest("repo", manifestDigest.String(), false)
+					So(err, ShouldBeNil)
+
+					err = imgStore.DeleteBlob("repo", digest)
+					So(err, ShouldBeNil)
+
+					// config
+					err = imgStore.DeleteBlob("repo", cdigest)
+					So(err, ShouldBeNil)
+				})
+
+				if testcase.storageType != "s3" {
+					Convey("get image manifest error", func() {
+						err := os.Chmod(path.Join(imgStore.RootDir(), "repo", "blobs", "sha256", manifestDigest.Encoded()), 0o000)
+						So(err, ShouldBeNil)
+
+						ok, _ := storageCommon.IsBlobReferenced(imgStore, "repo", unusedDigest, log.Logger)
+						So(ok, ShouldBeFalse)
+
+						err = os.Chmod(path.Join(imgStore.RootDir(), "repo", "blobs", "sha256", manifestDigest.Encoded()), 0o755)
+						So(err, ShouldBeNil)
+					})
+				}
+			})
+
+			Convey("Setup multiarch manifest", t, func() {
+				// put an unused blob
+				content := []byte("unused blob")
+				buf := bytes.NewBuffer(content)
+				unusedDigest := godigest.FromBytes(content)
+
+				_, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(buf.Bytes()), unusedDigest)
+				So(err, ShouldBeNil)
+
+				// create a blob/layer
+				upload, err := imgStore.NewBlobUpload(repoName)
+				So(err, ShouldBeNil)
+				So(upload, ShouldNotBeEmpty)
+
+				content = []byte("this is a blob1")
+				buf = bytes.NewBuffer(content)
+				buflen := buf.Len()
+				digest := godigest.FromBytes(content)
+				So(digest, ShouldNotBeNil)
+				blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf)
+				So(err, ShouldBeNil)
+				So(blob, ShouldEqual, buflen)
+				bdgst1 := digest
+				bsize1 := len(content)
+
+				err = imgStore.FinishBlobUpload(repoName, upload, buf, digest)
+				So(err, ShouldBeNil)
+				So(blob, ShouldEqual, buflen)
+
+				var index ispec.Index
+				index.SchemaVersion = 2
+				index.MediaType = ispec.MediaTypeImageIndex
+
+				var cdigest godigest.Digest
+				var cblob []byte
+
+				for i := 0; i < 4; i++ {
+					// upload image config blob
+					upload, err = imgStore.NewBlobUpload(repoName)
+					So(err, ShouldBeNil)
+					So(upload, ShouldNotBeEmpty)
+
+					cblob, cdigest = test.GetRandomImageConfig()
+					buf = bytes.NewBuffer(cblob)
+					buflen = buf.Len()
+					blob, err = imgStore.PutBlobChunkStreamed(repoName, upload, buf)
+					So(err, ShouldBeNil)
+					So(blob, ShouldEqual, buflen)
+
+					err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest)
+					So(err, ShouldBeNil)
+					So(blob, ShouldEqual, buflen)
+
+					// create a manifest
+					manifest := ispec.Manifest{
+						Config: ispec.Descriptor{
+							MediaType: ispec.MediaTypeImageConfig,
+							Digest:    cdigest,
+							Size:      int64(len(cblob)),
+						},
+						Layers: []ispec.Descriptor{
+							{
+								MediaType: ispec.MediaTypeImageLayer,
+								Digest:    bdgst1,
+								Size:      int64(bsize1),
+							},
+						},
+					}
+					manifest.SchemaVersion = 2
+					content, err = json.Marshal(manifest)
+					So(err, ShouldBeNil)
+					digest = godigest.FromBytes(content)
+					So(digest, ShouldNotBeNil)
+					_, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content)
+					So(err, ShouldBeNil)
+
+					index.Manifests = append(index.Manifests, ispec.Descriptor{
+						Digest:    digest,
+						MediaType: ispec.MediaTypeImageManifest,
+						Size:      int64(len(content)),
+					})
+				}
+
+				// upload index image
+				indexContent, err := json.Marshal(index)
+				So(err, ShouldBeNil)
+				indexDigest := godigest.FromBytes(indexContent)
+				So(indexDigest, ShouldNotBeNil)
+
+				indexManifestDigest, _, err := imgStore.PutImageManifest(repoName, "index", ispec.MediaTypeImageIndex, indexContent)
+				So(err, ShouldBeNil)
+
+				Convey("Try to delete blob currently in use", func() {
+					// layer blob
+					err := imgStore.DeleteBlob("test", bdgst1)
+					So(err, ShouldEqual, zerr.ErrBlobReferenced)
+
+					// manifest
+					err = imgStore.DeleteBlob("test", digest)
+					So(err, ShouldEqual, zerr.ErrBlobReferenced)
+
+					// config
+					err = imgStore.DeleteBlob("test", cdigest)
+					So(err, ShouldEqual, zerr.ErrBlobReferenced)
+				})
+
+				Convey("Delete unused blob", func() {
+					err := imgStore.DeleteBlob(repoName, unusedDigest)
+					So(err, ShouldBeNil)
+				})
+
+				Convey("Delete manifests first, then blob", func() {
+					err := imgStore.DeleteImageManifest(repoName, indexManifestDigest.String(), false)
+					So(err, ShouldBeNil)
+
+					for _, manifestDesc := range index.Manifests {
+						err := imgStore.DeleteImageManifest(repoName, manifestDesc.Digest.String(), false)
+						So(err, ShouldBeNil)
+					}
+
+					err = imgStore.DeleteBlob(repoName, bdgst1)
+					So(err, ShouldBeNil)
+
+					// config
+					err = imgStore.DeleteBlob("test", cdigest)
+					So(err, ShouldBeNil)
+				})
+
+				if testcase.storageType != "s3" {
+					Convey("repo not found", func() {
+						// delete repo
+						err := os.RemoveAll(path.Join(imgStore.RootDir(), repoName))
+						So(err, ShouldBeNil)
+
+						ok, err := storageCommon.IsBlobReferenced(imgStore, repoName, bdgst1, log.Logger)
+						So(err, ShouldNotBeNil)
+						So(ok, ShouldBeFalse)
+					})
+
+					Convey("index.json not found", func() {
+						err := os.Remove(path.Join(imgStore.RootDir(), repoName, "index.json"))
+						So(err, ShouldBeNil)
+
+						ok, err := storageCommon.IsBlobReferenced(imgStore, repoName, bdgst1, log.Logger)
+						So(err, ShouldNotBeNil)
+						So(ok, ShouldBeFalse)
+					})
+
+					Convey("multiarch image not found", func() {
+						err := os.Remove(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", indexManifestDigest.Encoded()))
+						So(err, ShouldBeNil)
+
+						ok, err := storageCommon.IsBlobReferenced(imgStore, repoName, unusedDigest, log.Logger)
+						So(err, ShouldNotBeNil)
+						So(ok, ShouldBeFalse)
+					})
+				}
+			})
+		})
+	}
+}
+
 func TestStorageHandler(t *testing.T) {
 	for _, testcase := range testCases {
 		testcase := testcase