mirror of
https://github.com/project-zot/zot.git
synced 2024-12-16 21:56:37 -05:00
api: use blob cache path while making hard link
previously mount blob will look for blob that is provided in http request and try to hard link that path but ideally we should look for path from our cache and do the hard link of that particular path. this commit does the same.
This commit is contained in:
parent
3a59b9f487
commit
af30c06aff
4 changed files with 65 additions and 44 deletions
|
@ -1801,6 +1801,7 @@ func TestCrossRepoMount(t *testing.T) {
|
|||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
defer os.RemoveAll(dir)
|
||||
|
||||
c.Config.Storage.RootDirectory = dir
|
||||
|
@ -1825,6 +1826,9 @@ func TestCrossRepoMount(t *testing.T) {
|
|||
params := make(map[string]string)
|
||||
|
||||
digest := "sha256:63a795ca90aa6e7cca60941e826810a4cd0a2e73ea02bf458241df2a5c973e29"
|
||||
|
||||
d := godigest.Digest(digest)
|
||||
|
||||
name := "zot-cve-test"
|
||||
|
||||
params["mount"] = digest
|
||||
|
@ -1836,6 +1840,7 @@ func TestCrossRepoMount(t *testing.T) {
|
|||
So(err, ShouldBeNil)
|
||||
So(headResponse.StatusCode(), ShouldEqual, 200)
|
||||
|
||||
// All invalid request of mount should return 202.
|
||||
params["mount"] = "sha:"
|
||||
|
||||
postResponse, err := client.R().
|
||||
|
@ -1855,12 +1860,13 @@ func TestCrossRepoMount(t *testing.T) {
|
|||
So(postResponse.StatusCode(), ShouldEqual, 202)
|
||||
|
||||
// Use correct request
|
||||
// This is correct request but it will return 202 because blob is not present in cache.
|
||||
params["mount"] = digest
|
||||
postResponse, err = client.R().
|
||||
SetBasicAuth(username, passphrase).SetQueryParams(params).
|
||||
Post(baseURL + "/v2/zot-c-test/blobs/uploads/")
|
||||
So(err, ShouldBeNil)
|
||||
So(postResponse.StatusCode(), ShouldEqual, 201)
|
||||
So(postResponse.StatusCode(), ShouldEqual, 202)
|
||||
|
||||
// Send same request again
|
||||
postResponse, err = client.R().
|
||||
|
@ -1874,7 +1880,7 @@ func TestCrossRepoMount(t *testing.T) {
|
|||
SetBasicAuth(username, passphrase).SetQueryParams(params).
|
||||
Post(baseURL + "/v2/zot-d-test/blobs/uploads/")
|
||||
So(err, ShouldBeNil)
|
||||
So(postResponse.StatusCode(), ShouldEqual, 201)
|
||||
So(postResponse.StatusCode(), ShouldEqual, 202)
|
||||
|
||||
headResponse, err = client.R().SetBasicAuth(username, passphrase).
|
||||
Head(fmt.Sprintf("%s/v2/zot-cv-test/blobs/%s", baseURL, digest))
|
||||
|
@ -1907,6 +1913,47 @@ func TestCrossRepoMount(t *testing.T) {
|
|||
So(err, ShouldBeNil)
|
||||
So(postResponse.StatusCode(), ShouldEqual, 201)
|
||||
|
||||
// We have uploaded a blob and since we have provided digest it should be full blob upload and there should be entry
|
||||
// in cache, now try mount blob request status and it should be 201 because now blob is present in cache
|
||||
// and it should do hard link.
|
||||
|
||||
params["mount"] = digest
|
||||
postResponse, err = client.R().
|
||||
SetBasicAuth(username, passphrase).SetQueryParams(params).
|
||||
Post(baseURL + "/v2/zot-mount-test/blobs/uploads/")
|
||||
So(err, ShouldBeNil)
|
||||
So(postResponse.StatusCode(), ShouldEqual, 201)
|
||||
|
||||
// Check os.SameFile here
|
||||
cachePath := path.Join(c.Config.Storage.RootDirectory, "zot-d-test", "blobs/sha256", d.Hex())
|
||||
|
||||
cacheFi, err := os.Stat(cachePath)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
linkPath := path.Join(c.Config.Storage.RootDirectory, "zot-mount-test", "blobs/sha256", d.Hex())
|
||||
|
||||
linkFi, err := os.Stat(linkPath)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(os.SameFile(cacheFi, linkFi), ShouldEqual, true)
|
||||
|
||||
// Now try another mount request and this time it should be from above uploaded repo i.e zot-mount-test
|
||||
// mount request should pass and should return 201.
|
||||
params["mount"] = digest
|
||||
params["from"] = "zot-mount-test"
|
||||
postResponse, err = client.R().
|
||||
SetBasicAuth(username, passphrase).SetQueryParams(params).
|
||||
Post(baseURL + "/v2/zot-mount1-test/blobs/uploads/")
|
||||
So(err, ShouldBeNil)
|
||||
So(postResponse.StatusCode(), ShouldEqual, 201)
|
||||
|
||||
linkPath = path.Join(c.Config.Storage.RootDirectory, "zot-mount1-test", "blobs/sha256", d.Hex())
|
||||
|
||||
linkFi, err = os.Stat(linkPath)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
So(os.SameFile(cacheFi, linkFi), ShouldEqual, true)
|
||||
|
||||
headResponse, err = client.R().SetBasicAuth(username, passphrase).
|
||||
Head(fmt.Sprintf("%s/v2/zot-cv-test/blobs/%s", baseURL, digest))
|
||||
So(err, ShouldBeNil)
|
||||
|
|
|
@ -497,9 +497,7 @@ func (rh *RouteHandler) CheckBlob(w http.ResponseWriter, r *http.Request) {
|
|||
return
|
||||
}
|
||||
|
||||
mediaType := r.Header.Get("Accept")
|
||||
|
||||
ok, blen, err := is.CheckBlob(name, digest, mediaType)
|
||||
ok, blen, err := is.CheckBlob(name, digest)
|
||||
if err != nil {
|
||||
switch err {
|
||||
case errors.ErrBadBlobDigest:
|
||||
|
@ -662,8 +660,10 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request)
|
|||
return
|
||||
}
|
||||
|
||||
// zot does not support cross mounting directly and do a workaround by copying blob using hard link
|
||||
err := is.MountBlob(name, from[0], mountDigests[0])
|
||||
// zot does not support cross mounting directly and do a workaround creating using hard link.
|
||||
// check blob looks for actual path (name+mountDigests[0]) first then look for cache and
|
||||
// if found in cache, will do hard link and if fails we will start new upload.
|
||||
_, _, err := is.CheckBlob(name, mountDigests[0])
|
||||
if err != nil {
|
||||
u, err := is.NewBlobUpload(name)
|
||||
if err != nil {
|
||||
|
|
|
@ -1024,40 +1024,8 @@ func (is *ImageStore) BlobPath(repo string, digest godigest.Digest) string {
|
|||
return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded())
|
||||
}
|
||||
|
||||
func (is *ImageStore) MountBlob(repo string, mountRepo string, digest string) error {
|
||||
d, err := godigest.Parse(digest)
|
||||
if err != nil {
|
||||
is.log.Error().Err(err).Str("digest", digest).Msg("failed to parse digest")
|
||||
|
||||
return errors.ErrBadBlobDigest
|
||||
}
|
||||
|
||||
mountBlobPath := is.BlobPath(mountRepo, d)
|
||||
is.log.Debug().Str("mount path", mountBlobPath)
|
||||
|
||||
blobPath := is.BlobPath(repo, d)
|
||||
is.log.Debug().Str("repo path", blobPath)
|
||||
|
||||
_, err = os.Stat(mountBlobPath)
|
||||
if err != nil {
|
||||
is.log.Error().Err(err).Msg("mount: blob path not found")
|
||||
|
||||
return errors.ErrBlobNotFound
|
||||
}
|
||||
|
||||
_, err = is.copyBlob(repo, blobPath, mountBlobPath)
|
||||
if err != nil {
|
||||
is.log.Error().Err(err).Msg("cache: error copying blobs from cache location")
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CheckBlob verifies a blob and returns true if the blob is correct.
|
||||
func (is *ImageStore) CheckBlob(repo string, digest string,
|
||||
mediaType string) (bool, int64, error) {
|
||||
func (is *ImageStore) CheckBlob(repo string, digest string) (bool, int64, error) {
|
||||
d, err := godigest.Parse(digest)
|
||||
if err != nil {
|
||||
is.log.Error().Err(err).Str("digest", digest).Msg("failed to parse digest")
|
||||
|
@ -1097,6 +1065,12 @@ func (is *ImageStore) CheckBlob(repo string, digest string,
|
|||
return false, -1, errors.ErrBlobNotFound
|
||||
}
|
||||
|
||||
if err := is.cache.PutBlob(digest, blobPath); err != nil {
|
||||
is.log.Error().Err(err).Str("blobPath", blobPath).Msg("dedupe: unable to insert blob record")
|
||||
|
||||
return false, -1, err
|
||||
}
|
||||
|
||||
return true, blobSize, nil
|
||||
}
|
||||
|
||||
|
|
|
@ -102,7 +102,7 @@ func TestAPIs(t *testing.T) {
|
|||
So(err, ShouldBeNil)
|
||||
So(b, ShouldEqual, l)
|
||||
|
||||
_, _, err = il.CheckBlob("test", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
|
||||
_, _, err = il.CheckBlob("test", d.String())
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, _, err = il.GetBlob("test", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
|
||||
|
@ -201,7 +201,7 @@ func TestAPIs(t *testing.T) {
|
|||
So(err, ShouldBeNil)
|
||||
So(b, ShouldEqual, l)
|
||||
|
||||
_, _, err = il.CheckBlob("test", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
|
||||
_, _, err = il.CheckBlob("test", d.String())
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, _, err = il.GetBlob("test", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
|
||||
|
@ -363,7 +363,7 @@ func TestAPIs(t *testing.T) {
|
|||
So(err, ShouldBeNil)
|
||||
So(b, ShouldEqual, l)
|
||||
|
||||
_, _, err = il.CheckBlob("dedupe1", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
|
||||
_, _, err = il.CheckBlob("dedupe1", d.String())
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, _, err = il.GetBlob("dedupe1", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
|
||||
|
@ -412,7 +412,7 @@ func TestAPIs(t *testing.T) {
|
|||
So(err, ShouldBeNil)
|
||||
So(b, ShouldEqual, l)
|
||||
|
||||
_, _, err = il.CheckBlob("dedupe2", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
|
||||
_, _, err = il.CheckBlob("dedupe2", d.String())
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
_, _, err = il.GetBlob("dedupe2", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
|
||||
|
|
Loading…
Reference in a new issue