api: use blob cache path while making hard link

previously mount blob will look for blob that is provided in http request and try to hard link that path
but ideally we should look for path from our cache and do the hard link of that particular path.
this commit does the same.

pkg/api/controller_test.go

@@ -1801,6 +1801,7 @@ func TestCrossRepoMount(t *testing.T) {
 		if err != nil {
 			panic(err)
 		}
+
 		defer os.RemoveAll(dir)
 
 		c.Config.Storage.RootDirectory = dir
@@ -1825,6 +1826,9 @@ func TestCrossRepoMount(t *testing.T) {
 		params := make(map[string]string)
 
 		digest := "sha256:63a795ca90aa6e7cca60941e826810a4cd0a2e73ea02bf458241df2a5c973e29"
+
+		d := godigest.Digest(digest)
+
 		name := "zot-cve-test"
 
 		params["mount"] = digest
@@ -1836,6 +1840,7 @@ func TestCrossRepoMount(t *testing.T) {
 		So(err, ShouldBeNil)
 		So(headResponse.StatusCode(), ShouldEqual, 200)
 
+		// All invalid request of mount should return 202.
 		params["mount"] = "sha:"
 
 		postResponse, err := client.R().
@@ -1855,12 +1860,13 @@ func TestCrossRepoMount(t *testing.T) {
 		So(postResponse.StatusCode(), ShouldEqual, 202)
 
 		// Use correct request
+		// This is correct request but it will return 202 because blob is not present in cache.
 		params["mount"] = digest
 		postResponse, err = client.R().
 			SetBasicAuth(username, passphrase).SetQueryParams(params).
 			Post(baseURL + "/v2/zot-c-test/blobs/uploads/")
 		So(err, ShouldBeNil)
-		So(postResponse.StatusCode(), ShouldEqual, 201)
+		So(postResponse.StatusCode(), ShouldEqual, 202)
 
 		// Send same request again
 		postResponse, err = client.R().
@@ -1874,7 +1880,7 @@ func TestCrossRepoMount(t *testing.T) {
 			SetBasicAuth(username, passphrase).SetQueryParams(params).
 			Post(baseURL + "/v2/zot-d-test/blobs/uploads/")
 		So(err, ShouldBeNil)
-		So(postResponse.StatusCode(), ShouldEqual, 201)
+		So(postResponse.StatusCode(), ShouldEqual, 202)
 
 		headResponse, err = client.R().SetBasicAuth(username, passphrase).
 			Head(fmt.Sprintf("%s/v2/zot-cv-test/blobs/%s", baseURL, digest))
@@ -1907,6 +1913,47 @@ func TestCrossRepoMount(t *testing.T) {
 		So(err, ShouldBeNil)
 		So(postResponse.StatusCode(), ShouldEqual, 201)
 
+		// We have uploaded a blob and since we have provided digest it should be full blob upload and there should be entry
+		// in cache, now try mount blob request status and it should be 201 because now blob is present in cache
+		// and it should do hard link.
+
+		params["mount"] = digest
+		postResponse, err = client.R().
+			SetBasicAuth(username, passphrase).SetQueryParams(params).
+			Post(baseURL + "/v2/zot-mount-test/blobs/uploads/")
+		So(err, ShouldBeNil)
+		So(postResponse.StatusCode(), ShouldEqual, 201)
+
+		// Check os.SameFile here
+		cachePath := path.Join(c.Config.Storage.RootDirectory, "zot-d-test", "blobs/sha256", d.Hex())
+
+		cacheFi, err := os.Stat(cachePath)
+		So(err, ShouldBeNil)
+
+		linkPath := path.Join(c.Config.Storage.RootDirectory, "zot-mount-test", "blobs/sha256", d.Hex())
+
+		linkFi, err := os.Stat(linkPath)
+		So(err, ShouldBeNil)
+
+		So(os.SameFile(cacheFi, linkFi), ShouldEqual, true)
+
+		// Now try another mount request and this time it should be from above uploaded repo i.e zot-mount-test
+		// mount request should pass and should return 201.
+		params["mount"] = digest
+		params["from"] = "zot-mount-test"
+		postResponse, err = client.R().
+			SetBasicAuth(username, passphrase).SetQueryParams(params).
+			Post(baseURL + "/v2/zot-mount1-test/blobs/uploads/")
+		So(err, ShouldBeNil)
+		So(postResponse.StatusCode(), ShouldEqual, 201)
+
+		linkPath = path.Join(c.Config.Storage.RootDirectory, "zot-mount1-test", "blobs/sha256", d.Hex())
+
+		linkFi, err = os.Stat(linkPath)
+		So(err, ShouldBeNil)
+
+		So(os.SameFile(cacheFi, linkFi), ShouldEqual, true)
+
 		headResponse, err = client.R().SetBasicAuth(username, passphrase).
 			Head(fmt.Sprintf("%s/v2/zot-cv-test/blobs/%s", baseURL, digest))
 		So(err, ShouldBeNil)

pkg/api/routes.go

@@ -497,9 +497,7 @@ func (rh *RouteHandler) CheckBlob(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	mediaType := r.Header.Get("Accept")
+	ok, blen, err := is.CheckBlob(name, digest)
-
-	ok, blen, err := is.CheckBlob(name, digest, mediaType)
 	if err != nil {
 		switch err {
 		case errors.ErrBadBlobDigest:
@@ -662,8 +660,10 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request)
 			return
 		}
 
-		// zot does not support cross mounting directly and do a workaround by copying blob using hard link
+		// zot does not support cross mounting directly and do a workaround creating using hard link.
-		err := is.MountBlob(name, from[0], mountDigests[0])
+		// check blob looks for actual path (name+mountDigests[0]) first then look for cache and
+		// if found in cache, will do hard link and if fails we will start new upload.
+		_, _, err := is.CheckBlob(name, mountDigests[0])
 		if err != nil {
 			u, err := is.NewBlobUpload(name)
 			if err != nil {

pkg/storage/storage.go

@@ -1024,40 +1024,8 @@ func (is *ImageStore) BlobPath(repo string, digest godigest.Digest) string {
 	return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded())
 }
 
-func (is *ImageStore) MountBlob(repo string, mountRepo string, digest string) error {
-	d, err := godigest.Parse(digest)
-	if err != nil {
-		is.log.Error().Err(err).Str("digest", digest).Msg("failed to parse digest")
-
-		return errors.ErrBadBlobDigest
-	}
-
-	mountBlobPath := is.BlobPath(mountRepo, d)
-	is.log.Debug().Str("mount path", mountBlobPath)
-
-	blobPath := is.BlobPath(repo, d)
-	is.log.Debug().Str("repo path", blobPath)
-
-	_, err = os.Stat(mountBlobPath)
-	if err != nil {
-		is.log.Error().Err(err).Msg("mount: blob path not found")
-
-		return errors.ErrBlobNotFound
-	}
-
-	_, err = is.copyBlob(repo, blobPath, mountBlobPath)
-	if err != nil {
-		is.log.Error().Err(err).Msg("cache: error copying blobs from cache location")
-
-		return err
-	}
-
-	return nil
-}
-
 // CheckBlob verifies a blob and returns true if the blob is correct.
-func (is *ImageStore) CheckBlob(repo string, digest string,
+func (is *ImageStore) CheckBlob(repo string, digest string) (bool, int64, error) {
-	mediaType string) (bool, int64, error) {
 	d, err := godigest.Parse(digest)
 	if err != nil {
 		is.log.Error().Err(err).Str("digest", digest).Msg("failed to parse digest")
@@ -1097,6 +1065,12 @@ func (is *ImageStore) CheckBlob(repo string, digest string,
 		return false, -1, errors.ErrBlobNotFound
 	}
 
+	if err := is.cache.PutBlob(digest, blobPath); err != nil {
+		is.log.Error().Err(err).Str("blobPath", blobPath).Msg("dedupe: unable to insert blob record")
+
+		return false, -1, err
+	}
+
 	return true, blobSize, nil
 }
 

pkg/storage/storage_test.go

@@ -102,7 +102,7 @@ func TestAPIs(t *testing.T) {
 				So(err, ShouldBeNil)
 				So(b, ShouldEqual, l)
 
-				_, _, err = il.CheckBlob("test", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
+				_, _, err = il.CheckBlob("test", d.String())
 				So(err, ShouldBeNil)
 
 				_, _, err = il.GetBlob("test", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
@@ -201,7 +201,7 @@ func TestAPIs(t *testing.T) {
 				So(err, ShouldBeNil)
 				So(b, ShouldEqual, l)
 
-				_, _, err = il.CheckBlob("test", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
+				_, _, err = il.CheckBlob("test", d.String())
 				So(err, ShouldBeNil)
 
 				_, _, err = il.GetBlob("test", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
@@ -363,7 +363,7 @@ func TestAPIs(t *testing.T) {
 			So(err, ShouldBeNil)
 			So(b, ShouldEqual, l)
 
-			_, _, err = il.CheckBlob("dedupe1", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
+			_, _, err = il.CheckBlob("dedupe1", d.String())
 			So(err, ShouldBeNil)
 
 			_, _, err = il.GetBlob("dedupe1", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
@@ -412,7 +412,7 @@ func TestAPIs(t *testing.T) {
 			So(err, ShouldBeNil)
 			So(b, ShouldEqual, l)
 
-			_, _, err = il.CheckBlob("dedupe2", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
+			_, _, err = il.CheckBlob("dedupe2", d.String())
 			So(err, ShouldBeNil)
 
 			_, _, err = il.GetBlob("dedupe2", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")