spec: added support for mount request using hard link

pkg/api/controller_test.go

@@ -1452,6 +1452,64 @@ func parseBearerAuthHeader(authHeaderRaw string) *authHeader {
 	return &h
 }
 
+func TestInvalidCases(t *testing.T) {
+	Convey("Invalid repo dir", t, func() {
+		config := api.NewConfig()
+		config.HTTP.Port = SecurePort1
+		htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase))
+
+		defer os.Remove(htpasswdPath)
+
+		config.HTTP.Auth = &api.AuthConfig{
+			HTPasswd: api.AuthHTPasswd{
+				Path: htpasswdPath,
+			},
+		}
+
+		c := api.NewController(config)
+
+		err := os.Mkdir("oci-repo-test", 0000)
+		if err != nil {
+			panic(err)
+		}
+
+		defer stopServer(c)
+
+		c.Config.Storage.RootDirectory = "oci-repo-test"
+
+		go func() {
+			// this blocks
+			if err := c.Run(); err != nil {
+				return
+			}
+		}()
+
+		// wait till ready
+		for {
+			_, err := resty.R().Get(BaseURL1)
+			if err == nil {
+				break
+			}
+
+			time.Sleep(100 * time.Millisecond)
+		}
+
+		digest := "sha256:8dd57e171a61368ffcfde38045ddb6ed74a32950c271c1da93eaddfb66a77e78"
+		name := "zot-c-test"
+
+		client := resty.New()
+
+		params := make(map[string]string)
+		params["from"] = "zot-cveid-test"
+		params["mount"] = digest
+
+		postResponse, err := client.R().
+			SetBasicAuth(username, passphrase).SetQueryParams(params).
+			Post(fmt.Sprintf("%s/v2/%s/blobs/uploads/", BaseURL1, name))
+		So(err, ShouldBeNil)
+		So(postResponse.StatusCode(), ShouldEqual, 500)
+	})
+}
 func TestHTTPReadOnly(t *testing.T) {
 	Convey("Single cred", t, func() {
 		singleCredtests := []string{}
@@ -1531,7 +1589,7 @@ func TestCrossRepoMount(t *testing.T) {
 		config.HTTP.Port = SecurePort1
 		htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase))
 
-		// defer os.Remove(htpasswdPath)
+		defer os.Remove(htpasswdPath)
 
 		config.HTTP.Auth = &api.AuthConfig{
 			HTPasswd: api.AuthHTPasswd{
@@ -1550,7 +1608,7 @@ func TestCrossRepoMount(t *testing.T) {
 		if err != nil {
 			panic(err)
 		}
-		// defer os.RemoveAll(dir)
+		defer os.RemoveAll(dir)
 
 		c.Config.Storage.RootDirectory = dir
 
@@ -1573,30 +1631,171 @@ func TestCrossRepoMount(t *testing.T) {
 
 		params := make(map[string]string)
 
-		params["mount"] = "63a795ca90aa6e7cca60941e826810a4cd0a2e73ea02bf458241df2a5c973e29"
-		params["from"] = "zot-test"
+		digest := "sha256:63a795ca90aa6e7cca60941e826810a4cd0a2e73ea02bf458241df2a5c973e29"
+		name := "zot-cve-test"
+
+		params["mount"] = digest
+		params["from"] = name
 
 		client := resty.New()
+		headResponse, err := client.R().SetBasicAuth(username, passphrase).
+			Head(fmt.Sprintf("%s/v2/%s/blobs/%s", BaseURL1, name, digest))
+		So(err, ShouldBeNil)
+		So(headResponse.StatusCode(), ShouldEqual, 200)
+
+		params["mount"] = "sha:"
+
 		postResponse, err := client.R().
 			SetBasicAuth(username, passphrase).SetQueryParams(params).
 			Post(BaseURL1 + "/v2/zot-c-test/blobs/uploads/")
 		So(err, ShouldBeNil)
 		So(postResponse.StatusCode(), ShouldEqual, 202)
 
+		incorrectParams := make(map[string]string)
+		incorrectParams["mount"] = "sha256:63a795ca90aa6e7dda60941e826810a4cd0a2e73ea02bf458241df2a5c973e29"
+		incorrectParams["from"] = "zot-x-test"
+
+		postResponse, err = client.R().
+			SetBasicAuth(username, passphrase).SetQueryParams(incorrectParams).
+			Post(BaseURL1 + "/v2/zot-y-test/blobs/uploads/")
+		So(err, ShouldBeNil)
+		So(postResponse.StatusCode(), ShouldEqual, 202)
+
+		// Use correct request
+		params["mount"] = digest
 		postResponse, err = client.R().
 			SetBasicAuth(username, passphrase).SetQueryParams(params).
-			Post(BaseURL1 + "/v2/zot-cve-test/blobs/uploads/")
+			Post(BaseURL1 + "/v2/zot-c-test/blobs/uploads/")
 		So(err, ShouldBeNil)
-		So(postResponse.StatusCode(), ShouldEqual, 500)
+		So(postResponse.StatusCode(), ShouldEqual, 201)
+
+		// Send same request again
+		postResponse, err = client.R().
+			SetBasicAuth(username, passphrase).SetQueryParams(params).
+			Post(BaseURL1 + "/v2/zot-c-test/blobs/uploads/")
+		So(err, ShouldBeNil)
+		So(postResponse.StatusCode(), ShouldEqual, 202)
+
+		// Valid requests
+		postResponse, err = client.R().
+			SetBasicAuth(username, passphrase).SetQueryParams(params).
+			Post(BaseURL1 + "/v2/zot-d-test/blobs/uploads/")
+		So(err, ShouldBeNil)
+		So(postResponse.StatusCode(), ShouldEqual, 201)
+
+		headResponse, err = client.R().SetBasicAuth(username, passphrase).
+			Head(fmt.Sprintf("%s/v2/zot-cv-test/blobs/%s", BaseURL1, digest))
+		So(err, ShouldBeNil)
+		So(headResponse.StatusCode(), ShouldEqual, 404)
+
+		postResponse, err = client.R().
+			SetBasicAuth(username, passphrase).SetQueryParams(params).Post(BaseURL1 + "/v2/zot-c-test/blobs/uploads/")
+		So(err, ShouldBeNil)
+		So(postResponse.StatusCode(), ShouldEqual, 202)
 
 		postResponse, err = client.R().
 			SetBasicAuth(username, passphrase).SetQueryParams(params).
 			Post(BaseURL1 + "/v2/ /blobs/uploads/")
 		So(err, ShouldBeNil)
 		So(postResponse.StatusCode(), ShouldEqual, 404)
+
+		digest = "sha256:63a795ca90aa6e7cca60941e826810a4cd0a2e73ea02bf458241df2a5c973e29"
+
+		blob := "63a795ca90aa6e7cca60941e826810a4cd0a2e73ea02bf458241df2a5c973e29"
+
+		buf, err := ioutil.ReadFile(path.Join(c.Config.Storage.RootDirectory, "zot-cve-test/blobs/sha256/"+blob))
+		if err != nil {
+			panic(err)
+		}
+
+		postResponse, err = client.R().SetHeader("Content-type", "application/octet-stream").
+			SetBasicAuth(username, passphrase).SetQueryParam("digest", "sha256:"+blob).
+			SetBody(buf).Post(BaseURL1 + "/v2/zot-d-test/blobs/uploads/")
+		So(err, ShouldBeNil)
+		So(postResponse.StatusCode(), ShouldEqual, 201)
+
+		headResponse, err = client.R().SetBasicAuth(username, passphrase).
+			Head(fmt.Sprintf("%s/v2/zot-cv-test/blobs/%s", BaseURL1, digest))
+		So(err, ShouldBeNil)
+		So(headResponse.StatusCode(), ShouldEqual, 200)
+
+		// Invalid request
+		params = make(map[string]string)
+		params["mount"] = "sha256:"
+		postResponse, err = client.R().
+			SetBasicAuth(username, passphrase).SetQueryParams(params).
+			Post(BaseURL1 + "/v2/zot-mount-test/blobs/uploads/")
+		So(err, ShouldBeNil)
+		So(postResponse.StatusCode(), ShouldEqual, 405)
+
+		params = make(map[string]string)
+		params["from"] = "zot-cve-test"
+		postResponse, err = client.R().
+			SetBasicAuth(username, passphrase).SetQueryParams(params).
+			Post(BaseURL1 + "/v2/zot-mount-test/blobs/uploads/")
+		So(err, ShouldBeNil)
+		So(postResponse.StatusCode(), ShouldEqual, 405)
+	})
+
+	Convey("Disable dedupe and cache", t, func() {
+		config := api.NewConfig()
+		config.HTTP.Port = SecurePort1
+		htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase))
+
+		defer os.Remove(htpasswdPath)
+
+		config.HTTP.Auth = &api.AuthConfig{
+			HTPasswd: api.AuthHTPasswd{
+				Path: htpasswdPath,
+			},
+		}
+
+		c := api.NewController(config)
+
+		//defer stopServer(c)
+
+		dir, err := ioutil.TempDir("", "oci-repo-test")
+		if err != nil {
+			panic(err)
+		}
+
+		err = copyFiles("../../test/data", dir)
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(dir)
+
+		c.Config.Storage.RootDirectory = dir
+		c.Config.Storage.Dedupe = false
+		c.Config.Storage.GC = false
+
+		go func() {
+			// this blocks
+			if err := c.Run(); err != nil {
+				return
+			}
+		}()
+
+		// wait till ready
+		for {
+			_, err := resty.R().Get(BaseURL1)
+			if err == nil {
+				break
+			}
+
+			time.Sleep(100 * time.Millisecond)
+		}
+
+		digest := "sha256:7a0437f04f83f084b7ed68ad9c4a4947e12fc4e1b006b38129bac89114ec3621"
+		name := "zot-c-test"
+
+		client := resty.New()
+		headResponse, err := client.R().SetBasicAuth(username, passphrase).
+			Head(fmt.Sprintf("%s/v2/%s/blobs/%s", BaseURL1, name, digest))
+		So(err, ShouldBeNil)
+		So(headResponse.StatusCode(), ShouldEqual, 404)
 	})
 }
-
 func TestParallelRequests(t *testing.T) {
 	testCases := []struct {
 		srcImageName  string
@@ -2062,3 +2261,15 @@ func copyFiles(sourceDir string, destDir string) error {
 
 	return nil
 }
+
+func stopServer(ctrl *api.Controller) {
+	err := ctrl.Server.Shutdown(context.Background())
+	if err != nil {
+		panic(err)
+	}
+
+	err = os.RemoveAll(ctrl.Config.Storage.RootDirectory)
+	if err != nil {
+		panic(err)
+	}
+}

pkg/api/routes.go

@@ -626,22 +626,43 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	// currently zot does not support cross-repository mounting, following dist-spec and returning 202
 	if mountDigests, ok := r.URL.Query()["mount"]; ok {
 		if len(mountDigests) != 1 {
 			w.WriteHeader(http.StatusBadRequest)
 			return
 		}
 
-		u, err := rh.c.ImageStore.NewBlobUpload(name)
-		if err != nil {
-			w.WriteHeader(http.StatusInternalServerError)
+		from, ok := r.URL.Query()["from"]
+		if !ok || len(from) != 1 {
+			w.WriteHeader(http.StatusMethodNotAllowed)
 			return
 		}
 
-		w.Header().Set("Location", path.Join(r.URL.String(), u))
-		w.Header().Set(BlobUploadUUID, u)
-		w.WriteHeader(http.StatusAccepted)
+		// zot does not support cross mounting directly and do a workaround by copying blob using hard link
+		err := rh.c.ImageStore.MountBlob(name, from[0], mountDigests[0])
+		if err != nil {
+			u, err := rh.c.ImageStore.NewBlobUpload(name)
+			if err != nil {
+				switch err {
+				case errors.ErrRepoNotFound:
+					WriteJSON(w, http.StatusNotFound, NewErrorList(NewError(NAME_UNKNOWN, map[string]string{"name": name})))
+				default:
+					rh.c.Log.Error().Err(err).Msg("unexpected error")
+					w.WriteHeader(http.StatusInternalServerError)
+				}
+
+				return
+			}
+
+			w.Header().Set("Location", path.Join(r.URL.String(), u))
+			w.Header().Set("Range", "bytes=0-0")
+			w.WriteHeader(http.StatusAccepted)
+
+			return
+		}
+
+		w.Header().Set("Location", fmt.Sprintf("/v2/%s/blobs/%s", name, mountDigests[0]))
+		w.WriteHeader(http.StatusCreated)
 
 		return
 	}

pkg/extensions/search/cve/cve_test.go

@@ -501,7 +501,7 @@ func TestCVESearch(t *testing.T) {
 		}
 
 		// Wait for trivy db to download
-		time.Sleep(35 * time.Second)
+		time.Sleep(45 * time.Second)
 
 		defer func() {
 			ctx := context.Background()

pkg/storage/storage.go

@@ -114,9 +114,19 @@ func (is *ImageStore) initRepo(name string) error {
 	}
 
 	// create "blobs" subdir
-	ensureDir(path.Join(repoDir, "blobs"), is.log)
+	err := ensureDir(path.Join(repoDir, "blobs"), is.log)
+	if err != nil {
+		is.log.Error().Err(err).Msg("error creating blobs subdir")
+
+		return err
+	}
 	// create BlobUploadDir subdir
-	ensureDir(path.Join(repoDir, BlobUploadDir), is.log)
+	err = ensureDir(path.Join(repoDir, BlobUploadDir), is.log)
+	if err != nil {
+		is.log.Error().Err(err).Msg("error creating blob upload subdir")
+
+		return err
+	}
 
 	// "oci-layout" file - create if it doesn't exist
 	ilPath := path.Join(repoDir, ispec.ImageLayoutFile)
@@ -497,7 +507,7 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType
 
 	// write manifest to "blobs"
 	dir = path.Join(is.rootDir, repo, "blobs", mDigest.Algorithm().String())
-	ensureDir(dir, is.log)
+	_ = ensureDir(dir, is.log)
 	file := path.Join(dir, mDigest.Encoded())
 
 	if err := ioutil.WriteFile(file, body, 0600); err != nil {
@@ -630,6 +640,8 @@ func (is *ImageStore) BlobUploadPath(repo string, uuid string) string {
 // NewBlobUpload returns the unique ID for an upload in progress.
 func (is *ImageStore) NewBlobUpload(repo string) (string, error) {
 	if err := is.InitRepo(repo); err != nil {
+		is.log.Error().Err(err).Msg("error initializing repo")
+
 		return "", err
 	}
 
@@ -796,7 +808,13 @@ func (is *ImageStore) FinishBlobUpload(repo string, uuid string, body io.Reader,
 	is.Lock()
 	defer is.Unlock()
 
-	ensureDir(dir, is.log)
+	err = ensureDir(dir, is.log)
+	if err != nil {
+		is.log.Error().Err(err).Msg("error creating blobs/sha256 dir")
+
+		return err
+	}
+
 	dst := is.BlobPath(repo, dstDigest)
 
 	if is.dedupe && is.cache != nil {
@@ -865,7 +883,7 @@ func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, digest string)
 	is.Lock()
 	defer is.Unlock()
 
-	ensureDir(dir, is.log)
+	_ = ensureDir(dir, is.log)
 	dst := is.BlobPath(repo, dstDigest)
 
 	if is.dedupe && is.cache != nil {
@@ -967,6 +985,37 @@ func (is *ImageStore) BlobPath(repo string, digest godigest.Digest) string {
 	return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded())
 }
 
+func (is *ImageStore) MountBlob(repo string, mountRepo string, digest string) error {
+	d, err := godigest.Parse(digest)
+	if err != nil {
+		is.log.Error().Err(err).Str("digest", digest).Msg("failed to parse digest")
+
+		return errors.ErrBadBlobDigest
+	}
+
+	mountBlobPath := is.BlobPath(mountRepo, d)
+	is.log.Debug().Str("mount path", mountBlobPath)
+
+	blobPath := is.BlobPath(repo, d)
+	is.log.Debug().Str("repo path", blobPath)
+
+	_, err = os.Stat(mountBlobPath)
+	if err != nil {
+		is.log.Error().Err(err).Msg("mount: blob path not found")
+
+		return errors.ErrBlobNotFound
+	}
+
+	_, err = is.copyBlob(repo, blobPath, mountBlobPath)
+	if err != nil {
+		is.log.Error().Err(err).Msg("cache: error copying blobs from cache location")
+
+		return err
+	}
+
+	return nil
+}
+
 // CheckBlob verifies a blob and returns true if the blob is correct.
 func (is *ImageStore) CheckBlob(repo string, digest string,
 	mediaType string) (bool, int64, error) {
@@ -988,44 +1037,67 @@ func (is *ImageStore) CheckBlob(repo string, digest string,
 
 	blobInfo, err := os.Stat(blobPath)
 	if err == nil {
+		is.log.Debug().Str("blob path", blobPath).Msg("blob path found")
+
 		return true, blobInfo.Size(), nil
 	}
 
 	is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob")
 
-	if !is.dedupe || is.cache == nil {
+	// Check blobs in cache
+	dstRecord, err := is.checkCacheBlob(digest)
+	if err != nil {
+		is.log.Error().Err(err).Str("digest", digest).Msg("cache: not found")
+
 		return false, -1, errors.ErrBlobNotFound
 	}
 
-	// lookup cache and if found, dedupe here
-	dstRecord, err := is.cache.GetBlob(digest)
+	// If found copy to location
+	blobSize, err := is.copyBlob(repo, blobPath, dstRecord)
 	if err != nil {
 		return false, -1, errors.ErrBlobNotFound
 	}
 
+	return true, blobSize, nil
+}
+
+func (is *ImageStore) checkCacheBlob(digest string) (string, error) {
+	if !is.dedupe || is.cache == nil {
+		return "", errors.ErrBlobNotFound
+	}
+
+	dstRecord, err := is.cache.GetBlob(digest)
+	if err != nil {
+		return "", err
+	}
+
 	dstRecord = path.Join(is.rootDir, dstRecord)
 
 	is.log.Debug().Str("digest", digest).Str("dstRecord", dstRecord).Msg("cache: found dedupe record")
 
+	return dstRecord, nil
+}
+
+func (is *ImageStore) copyBlob(repo string, blobPath string, dstRecord string) (int64, error) {
 	if err := is.initRepo(repo); err != nil {
 		is.log.Error().Err(err).Str("repo", repo).Msg("unable to initialize an empty repo")
-		return false, -1, err
+		return -1, err
 	}
 
-	ensureDir(filepath.Dir(blobPath), is.log)
+	_ = ensureDir(filepath.Dir(blobPath), is.log)
 
 	if err := os.Link(dstRecord, blobPath); err != nil {
 		is.log.Error().Err(err).Str("blobPath", blobPath).Str("link", dstRecord).Msg("dedupe: unable to hard link")
 
-		return false, -1, errors.ErrBlobNotFound
+		return -1, errors.ErrBlobNotFound
 	}
 
-	blobInfo, err = os.Stat(blobPath)
+	blobInfo, err := os.Stat(blobPath)
 	if err == nil {
-		return true, blobInfo.Size(), nil
+		return blobInfo.Size(), nil
 	}
 
-	return false, -1, errors.ErrBlobNotFound
+	return -1, errors.ErrBlobNotFound
 }
 
 // GetBlob returns a stream to read the blob.
@@ -1115,10 +1187,14 @@ func dirExists(d string) bool {
 	return true
 }
 
-func ensureDir(dir string, log zerolog.Logger) {
+func ensureDir(dir string, log zerolog.Logger) error {
 	if err := os.MkdirAll(dir, 0755); err != nil {
-		log.Panic().Err(err).Str("dir", dir).Msg("unable to create dir")
+		log.Error().Err(err).Str("dir", dir).Msg("unable to create dir")
+
+		return err
 	}
+
+	return nil
 }
 
 func ifOlderThan(is *ImageStore, repo string, delay time.Duration) casext.GCPolicy {

pkg/storage/storage_test.go

@@ -518,7 +518,8 @@ func TestNegativeCases(t *testing.T) {
 		err = os.Chmod(dir, 0000) // remove all perms
 		So(err, ShouldBeNil)
 		if os.Geteuid() != 0 {
-			So(func() { _ = il.InitRepo("test") }, ShouldPanic)
+			err = il.InitRepo("test")
+			So(err, ShouldNotBeNil)
 		}
 	})