From 964af6ba5117dbffe0ceef547f0e24efd1fc4d59 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Thu, 9 Jan 2020 16:31:34 -0800 Subject: [PATCH] compliance: be compliant with dist-spec compliance tests dist-spec compliance tests are now becoming a part of dist-spec repo itself - we want to be compliant pkg/api/regex.go: * revert uppercasing in repository names pkg/api/routes.go: * ListTags() should support the URL params 'n' and 'last' for pagination * s/uuid/session_id/g to use the dist-spec's naming * Fix off-by-one error in GetBlobUpload()'s http response "Range" header * DeleteManifest() success status code is 202 * Fix PatchBlobUpload() to account for "streamed" use case where neither "Content-Length" nor "Content-Range" headers are set pkg/storage/storage.go: * Add a "streamed" version of PutBlobChunk() called PutBlobChunkStreamed() pkg/compliance/v1_0_0/check.go: * fix unit tests to account for changed response status codes --- examples/config-test.json | 2 +- pkg/api/regexp.go | 2 +- pkg/api/routes.go | 260 ++++++++++++++++++++++----------- pkg/compliance/v1_0_0/check.go | 82 +++++++++-- pkg/storage/storage.go | 46 +++++- 5 files changed, 295 insertions(+), 97 deletions(-) diff --git a/examples/config-test.json b/examples/config-test.json index 00519969..dd306b46 100644 --- a/examples/config-test.json +++ b/examples/config-test.json @@ -4,7 +4,7 @@ "rootDirectory":"/tmp/zot" }, "http": { - "address":"0.0.0.0", + "address":"127.0.0.1", "port":"8080" }, "log":{ diff --git a/pkg/api/regexp.go b/pkg/api/regexp.go index 45b9a86b..e2df041e 100644 --- a/pkg/api/regexp.go +++ b/pkg/api/regexp.go @@ -6,7 +6,7 @@ import "regexp" var ( // alphaNumericRegexp defines the alpha numeric atom, typically a // component of names. This only allows lower case characters and digits. - alphaNumericRegexp = match(`[a-zA-Z0-9]+`) + alphaNumericRegexp = match(`[a-z0-9]+`) // separatorRegexp defines the separators allowed to be embedded in name // components. This allow one period, one or two underscore and multiple diff --git a/pkg/api/routes.go b/pkg/api/routes.go index c75e404e..95386b40 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -17,6 +17,7 @@ import ( "io/ioutil" "net/http" "path" + "sort" "strconv" "strings" @@ -35,6 +36,7 @@ const ( DistContentDigestKey = "Docker-Content-Digest" BlobUploadUUID = "Blob-Upload-UUID" DefaultMediaType = "application/json" + BinaryMediaType = "application/octet-stream" ) type RouteHandler struct { @@ -70,13 +72,13 @@ func (rh *RouteHandler) SetupRoutes() { rh.DeleteBlob).Methods("DELETE") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/", NameRegexp.String()), rh.CreateBlobUpload).Methods("POST") - g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{uuid}", NameRegexp.String()), + g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()), rh.GetBlobUpload).Methods("GET") - g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{uuid}", NameRegexp.String()), + g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()), rh.PatchBlobUpload).Methods("PATCH") - g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{uuid}", NameRegexp.String()), + g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()), rh.UpdateBlobUpload).Methods("PUT") - g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{uuid}", NameRegexp.String()), + g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()), rh.DeleteBlobUpload).Methods("DELETE") g.HandleFunc("/_catalog", rh.ListRepositories).Methods("GET") @@ -113,8 +115,11 @@ type ImageTags struct { // @Accept json // @Produce json // @Param name path string true "test" +// @Param n query integer true "limit entries for pagination" +// @Param last query string true "last tag value for pagination" // @Success 200 {object} api.ImageTags // @Failure 404 {string} string "not found" +// @Failure 400 {string} string "bad request" func (rh *RouteHandler) ListTags(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] @@ -124,12 +129,86 @@ func (rh *RouteHandler) ListTags(w http.ResponseWriter, r *http.Request) { return } + paginate := false + n := -1 + + var err error + + nQuery, ok := r.URL.Query()["n"] + + if ok { + if len(nQuery) != 1 { + w.WriteHeader(http.StatusBadRequest) + return + } + + var n1 int64 + + if n1, err = strconv.ParseInt(nQuery[0], 10, 0); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + + n = int(n1) + paginate = true + } + + last := "" + lastQuery, ok := r.URL.Query()["last"] + + if ok { + if len(lastQuery) != 1 { + w.WriteHeader(http.StatusBadRequest) + return + } + + last = lastQuery[0] + } + tags, err := rh.c.ImageStore.GetImageTags(name) if err != nil { WriteJSON(w, http.StatusNotFound, NewError(NAME_UNKNOWN, map[string]string{"name": name})) return } + if paginate && (n < len(tags)) { + sort.Strings(tags) + + pTags := ImageTags{Name: name} + + if last == "" { + // first + pTags.Tags = tags[:n] + } else { + // next + i := -1 + tag := "" + found := false + for i, tag = range tags { + if tag == last { + found = true + break + } + } + if !found { + w.WriteHeader(http.StatusNotFound) + return + } + if n >= len(tags)-i { + pTags.Tags = tags[i+1:] + WriteJSON(w, http.StatusOK, pTags) + return + } + pTags.Tags = tags[i+1 : i+1+n] + } + + last = pTags.Tags[len(pTags.Tags)-1] + w.Header().Set("Link", fmt.Sprintf("/v2/%s/tags/list?n=%d&last=%s; rel=\"next\"", name, n, last)) + WriteJSON(w, http.StatusOK, pTags) + + return + } + WriteJSON(w, http.StatusOK, ImageTags{Name: name, Tags: tags}) } @@ -336,7 +415,7 @@ func (rh *RouteHandler) DeleteManifest(w http.ResponseWriter, r *http.Request) { return } - w.WriteHeader(http.StatusOK) + w.WriteHeader(http.StatusAccepted) } // CheckBlob godoc @@ -494,7 +573,7 @@ func (rh *RouteHandler) DeleteBlob(w http.ResponseWriter, r *http.Request) { // @Produce json // @Param name path string true "repository name" // @Success 202 {string} string "accepted" -// @Header 202 {string} Location "/v2/{name}/blobs/uploads/{uuid}" +// @Header 202 {string} Location "/v2/{name}/blobs/uploads/{session_id}" // @Header 202 {string} Range "bytes=0-0" // @Failure 404 {string} string "not found" // @Failure 500 {string} string "internal server error" @@ -540,17 +619,17 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request) // GetBlobUpload godoc // @Summary Get image blob/layer upload -// @Description Get an image's blob/layer upload given a uuid +// @Description Get an image's blob/layer upload given a session_id // @Accept json // @Produce json // @Param name path string true "repository name" -// @Param uuid path string true "upload uuid" +// @Param session_id path string true "upload session_id" // @Success 204 {string} string "no content" -// @Header 202 {string} Location "/v2/{name}/blobs/uploads/{uuid}" +// @Header 202 {string} Location "/v2/{name}/blobs/uploads/{session_id}" // @Header 202 {string} Range "bytes=0-128" // @Failure 404 {string} string "not found" // @Failure 500 {string} string "internal server error" -// @Router /v2/{name}/blobs/uploads/{uuid} [get] +// @Router /v2/{name}/blobs/uploads/{session_id} [get] func (rh *RouteHandler) GetBlobUpload(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] @@ -560,23 +639,23 @@ func (rh *RouteHandler) GetBlobUpload(w http.ResponseWriter, r *http.Request) { return } - uuid, ok := vars["uuid"] - if !ok || uuid == "" { + sessionID, ok := vars["session_id"] + if !ok || sessionID == "" { w.WriteHeader(http.StatusNotFound) return } - size, err := rh.c.ImageStore.GetBlobUpload(name, uuid) + size, err := rh.c.ImageStore.GetBlobUpload(name, sessionID) if err != nil { switch err { case errors.ErrBadUploadRange: - WriteJSON(w, http.StatusBadRequest, NewError(BLOB_UPLOAD_INVALID, map[string]string{"uuid": uuid})) + WriteJSON(w, http.StatusBadRequest, NewError(BLOB_UPLOAD_INVALID, map[string]string{"session_id": sessionID})) case errors.ErrBadBlobDigest: - WriteJSON(w, http.StatusBadRequest, NewError(BLOB_UPLOAD_INVALID, map[string]string{"uuid": uuid})) + WriteJSON(w, http.StatusBadRequest, NewError(BLOB_UPLOAD_INVALID, map[string]string{"session_id": sessionID})) case errors.ErrRepoNotFound: WriteJSON(w, http.StatusNotFound, NewError(NAME_UNKNOWN, map[string]string{"name": name})) case errors.ErrUploadNotFound: - WriteJSON(w, http.StatusNotFound, NewError(BLOB_UPLOAD_UNKNOWN, map[string]string{"uuid": uuid})) + WriteJSON(w, http.StatusNotFound, NewError(BLOB_UPLOAD_UNKNOWN, map[string]string{"session_id": sessionID})) default: rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) @@ -585,27 +664,27 @@ func (rh *RouteHandler) GetBlobUpload(w http.ResponseWriter, r *http.Request) { return } - w.Header().Set("Location", path.Join(r.URL.String(), uuid)) - w.Header().Set("Range", fmt.Sprintf("bytes=0-%d", size)) + w.Header().Set("Location", path.Join(r.URL.String(), sessionID)) + w.Header().Set("Range", fmt.Sprintf("bytes=0-%d", size-1)) w.WriteHeader(http.StatusNoContent) } // PatchBlobUpload godoc // @Summary Resume image blob/layer upload -// @Description Resume an image's blob/layer upload given an uuid +// @Description Resume an image's blob/layer upload given an session_id // @Accept json // @Produce json // @Param name path string true "repository name" -// @Param uuid path string true "upload uuid" +// @Param session_id path string true "upload session_id" // @Success 202 {string} string "accepted" -// @Header 202 {string} Location "/v2/{name}/blobs/uploads/{uuid}" +// @Header 202 {string} Location "/v2/{name}/blobs/uploads/{session_id}" // @Header 202 {string} Range "bytes=0-128" // @Header 200 {object} api.BlobUploadUUID // @Failure 400 {string} string "bad request" // @Failure 404 {string} string "not found" // @Failure 416 {string} string "range not satisfiable" // @Failure 500 {string} string "internal server error" -// @Router /v2/{name}/blobs/uploads/{uuid} [patch] +// @Router /v2/{name}/blobs/uploads/{session_id} [patch] func (rh *RouteHandler) PatchBlobUpload(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] @@ -615,53 +694,63 @@ func (rh *RouteHandler) PatchBlobUpload(w http.ResponseWriter, r *http.Request) return } - uuid, ok := vars["uuid"] - if !ok || uuid == "" { + sessionID, ok := vars["session_id"] + if !ok || sessionID == "" { w.WriteHeader(http.StatusNotFound) return } - var err error - - var contentLength int64 - - if contentLength, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil { - rh.c.Log.Warn().Str("actual", r.Header.Get("Content-Length")).Msg("invalid content length") - w.WriteHeader(http.StatusBadRequest) - - return - } - - contentRange := r.Header.Get("Content-Range") - if contentRange == "" { - rh.c.Log.Warn().Str("actual", r.Header.Get("Content-Range")).Msg("invalid content range") - w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) - - return - } - - var from, to int64 - if from, to, err = getContentRange(r); err != nil || (to-from) != contentLength { - w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) - return - } - - if contentType := r.Header.Get("Content-Type"); contentType != "application/octet-stream" { - rh.c.Log.Warn().Str("actual", contentType).Str("expected", "application/octet-stream").Msg("invalid media type") + if contentType := r.Header.Get("Content-Type"); contentType != BinaryMediaType { + rh.c.Log.Warn().Str("actual", contentType).Str("expected", BinaryMediaType).Msg("invalid media type") w.WriteHeader(http.StatusUnsupportedMediaType) return } - clen, err := rh.c.ImageStore.PutBlobChunk(name, uuid, from, to, r.Body) + var err error + + var clen int64 + + if r.Header.Get("Content-Length") == "" || r.Header.Get("Content-Range") == "" { + // streamed blob upload + clen, err = rh.c.ImageStore.PutBlobChunkStreamed(name, sessionID, r.Body) + } else { + // chunked blob upload + + var contentLength int64 + + if contentLength, err = strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64); err != nil { + rh.c.Log.Warn().Str("actual", r.Header.Get("Content-Length")).Msg("invalid content length") + w.WriteHeader(http.StatusBadRequest) + + return + } + + contentRange := r.Header.Get("Content-Range") + if contentRange == "" { + rh.c.Log.Warn().Str("actual", r.Header.Get("Content-Range")).Msg("invalid content range") + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + + return + } + + var from, to int64 + if from, to, err = getContentRange(r); err != nil || (to-from)+1 != contentLength { + w.WriteHeader(http.StatusRequestedRangeNotSatisfiable) + return + } + + clen, err = rh.c.ImageStore.PutBlobChunk(name, sessionID, from, to, r.Body) + } + if err != nil { switch err { case errors.ErrBadUploadRange: - WriteJSON(w, http.StatusBadRequest, NewError(BLOB_UPLOAD_INVALID, map[string]string{"uuid": uuid})) + WriteJSON(w, http.StatusBadRequest, NewError(BLOB_UPLOAD_INVALID, map[string]string{"session_id": sessionID})) case errors.ErrRepoNotFound: WriteJSON(w, http.StatusNotFound, NewError(NAME_UNKNOWN, map[string]string{"name": name})) case errors.ErrUploadNotFound: - WriteJSON(w, http.StatusNotFound, NewError(BLOB_UPLOAD_UNKNOWN, map[string]string{"uuid": uuid})) + WriteJSON(w, http.StatusNotFound, NewError(BLOB_UPLOAD_UNKNOWN, map[string]string{"session_id": sessionID})) default: rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) @@ -670,10 +759,10 @@ func (rh *RouteHandler) PatchBlobUpload(w http.ResponseWriter, r *http.Request) return } - w.Header().Set("Location", path.Join(r.URL.String(), uuid)) - w.Header().Set("Range", fmt.Sprintf("bytes=0-%d", clen)) + w.Header().Set("Location", r.URL.String()) + w.Header().Set("Range", fmt.Sprintf("bytes=0-%d", clen-1)) w.Header().Set("Content-Length", "0") - w.Header().Set(BlobUploadUUID, uuid) + w.Header().Set(BlobUploadUUID, sessionID) w.WriteHeader(http.StatusAccepted) } @@ -683,15 +772,16 @@ func (rh *RouteHandler) PatchBlobUpload(w http.ResponseWriter, r *http.Request) // @Accept json // @Produce json // @Param name path string true "repository name" -// @Param uuid path string true "upload uuid" +// @Param session_id path string true "upload session_id" // @Param digest query string true "blob/layer digest" // @Success 201 {string} string "created" -// @Header 202 {string} Location "/v2/{name}/blobs/{digest}" +// @Header 202 {string} Location "/v2/{name}/blobs/uploads/{digest}" // @Header 200 {object} api.DistContentDigestKey // @Failure 404 {string} string "not found" // @Failure 500 {string} string "internal server error" -// @Router /v2/{name}/blobs/uploads/{uuid} [put] +// @Router /v2/{name}/blobs/uploads/{session_id} [put] func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request) { + rh.c.Log.Info().Interface("headers", r.Header).Msg("HEADERS") vars := mux.Vars(r) name, ok := vars["name"] @@ -700,8 +790,8 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request) return } - uuid, ok := vars["uuid"] - if !ok || uuid == "" { + sessionID, ok := vars["session_id"] + if !ok || sessionID == "" { w.WriteHeader(http.StatusNotFound) return } @@ -714,10 +804,12 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request) digest := digests[0] + rh.c.Log.Info().Int64("r.ContentLength", r.ContentLength).Msg("DEBUG") + contentPresent := true contentLen, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64) - if err != nil || contentLen == 0 { + if err != nil { contentPresent = false } @@ -737,18 +829,12 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request) var from, to int64 if contentPresent { - if r.Header.Get("Content-Type") != "application/octet-stream" { - w.WriteHeader(http.StatusUnsupportedMediaType) - return - } - contentRange := r.Header.Get("Content-Range") if contentRange == "" { // monolithic upload from = 0 if contentLen == 0 { - w.WriteHeader(http.StatusBadRequest) - return + goto finish // FIXME: } to = contentLen @@ -757,15 +843,20 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request) return } - _, err = rh.c.ImageStore.PutBlobChunk(name, uuid, from, to, r.Body) + if r.Header.Get("Content-Type") != BinaryMediaType { + w.WriteHeader(http.StatusUnsupportedMediaType) + return + } + + _, err = rh.c.ImageStore.PutBlobChunk(name, sessionID, from, to, r.Body) if err != nil { switch err { case errors.ErrBadUploadRange: - WriteJSON(w, http.StatusBadRequest, NewError(BLOB_UPLOAD_INVALID, map[string]string{"uuid": uuid})) + WriteJSON(w, http.StatusBadRequest, NewError(BLOB_UPLOAD_INVALID, map[string]string{"session_id": sessionID})) case errors.ErrRepoNotFound: WriteJSON(w, http.StatusNotFound, NewError(NAME_UNKNOWN, map[string]string{"name": name})) case errors.ErrUploadNotFound: - WriteJSON(w, http.StatusNotFound, NewError(BLOB_UPLOAD_UNKNOWN, map[string]string{"uuid": uuid})) + WriteJSON(w, http.StatusNotFound, NewError(BLOB_UPLOAD_UNKNOWN, map[string]string{"session_id": sessionID})) default: rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) @@ -775,17 +866,22 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request) } } +finish: + if r.Header.Get("Content-Type") != BinaryMediaType { + w.WriteHeader(http.StatusUnsupportedMediaType) + return + } // blob chunks already transferred, just finish - if err := rh.c.ImageStore.FinishBlobUpload(name, uuid, r.Body, digest); err != nil { + if err := rh.c.ImageStore.FinishBlobUpload(name, sessionID, r.Body, digest); err != nil { switch err { case errors.ErrBadBlobDigest: WriteJSON(w, http.StatusBadRequest, NewError(DIGEST_INVALID, map[string]string{"digest": digest})) case errors.ErrBadUploadRange: - WriteJSON(w, http.StatusBadRequest, NewError(BLOB_UPLOAD_INVALID, map[string]string{"uuid": uuid})) + WriteJSON(w, http.StatusBadRequest, NewError(BLOB_UPLOAD_INVALID, map[string]string{"session_id": sessionID})) case errors.ErrRepoNotFound: WriteJSON(w, http.StatusNotFound, NewError(NAME_UNKNOWN, map[string]string{"name": name})) case errors.ErrUploadNotFound: - WriteJSON(w, http.StatusNotFound, NewError(BLOB_UPLOAD_UNKNOWN, map[string]string{"uuid": uuid})) + WriteJSON(w, http.StatusNotFound, NewError(BLOB_UPLOAD_UNKNOWN, map[string]string{"session_id": sessionID})) default: rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) @@ -806,11 +902,11 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request) // @Accept json // @Produce json // @Param name path string true "repository name" -// @Param uuid path string true "upload uuid" +// @Param session_id path string true "upload session_id" // @Success 200 {string} string "ok" // @Failure 404 {string} string "not found" // @Failure 500 {string} string "internal server error" -// @Router /v2/{name}/blobs/uploads/{uuid} [delete] +// @Router /v2/{name}/blobs/uploads/{session_id} [delete] func (rh *RouteHandler) DeleteBlobUpload(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) name, ok := vars["name"] @@ -820,18 +916,18 @@ func (rh *RouteHandler) DeleteBlobUpload(w http.ResponseWriter, r *http.Request) return } - uuid, ok := vars["uuid"] - if !ok || uuid == "" { + sessionID, ok := vars["session_id"] + if !ok || sessionID == "" { w.WriteHeader(http.StatusNotFound) return } - if err := rh.c.ImageStore.DeleteBlobUpload(name, uuid); err != nil { + if err := rh.c.ImageStore.DeleteBlobUpload(name, sessionID); err != nil { switch err { case errors.ErrRepoNotFound: WriteJSON(w, http.StatusNotFound, NewError(NAME_UNKNOWN, map[string]string{"name": name})) case errors.ErrUploadNotFound: - WriteJSON(w, http.StatusNotFound, NewError(BLOB_UPLOAD_UNKNOWN, map[string]string{"uuid": uuid})) + WriteJSON(w, http.StatusNotFound, NewError(BLOB_UPLOAD_UNKNOWN, map[string]string{"session_id": sessionID})) default: rh.c.Log.Error().Err(err).Msg("unexpected error") w.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/compliance/v1_0_0/check.go b/pkg/compliance/v1_0_0/check.go index 8d0a5541..5d3c96cf 100644 --- a/pkg/compliance/v1_0_0/check.go +++ b/pkg/compliance/v1_0_0/check.go @@ -144,7 +144,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { // without the Content-Length should fail resp, err = resty.R().SetQueryParam("digest", digest.String()).Put(loc) So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, 400) + So(resp.StatusCode(), ShouldEqual, 415) // without any data to send, should fail resp, err = resty.R().SetQueryParam("digest", digest.String()). SetHeader("Content-Type", "application/octet-stream").Put(loc) @@ -199,7 +199,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { // without the Content-Length should fail resp, err = resty.R().SetQueryParam("digest", digest.String()).Put(loc) So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, 400) + So(resp.StatusCode(), ShouldEqual, 415) // without any data to send, should fail resp, err = resty.R().SetQueryParam("digest", digest.String()). SetHeader("Content-Type", "application/octet-stream").Put(loc) @@ -239,7 +239,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(err, ShouldBeNil) // write first chunk - contentRange := fmt.Sprintf("%d-%d", 0, len(chunk1)) + contentRange := fmt.Sprintf("%d-%d", 0, len(chunk1)-1) resp, err = resty.R().SetHeader("Content-Type", "application/octet-stream"). SetHeader("Content-Range", contentRange).SetBody(chunk1).Patch(loc) So(err, ShouldBeNil) @@ -254,7 +254,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(r, ShouldEqual, "bytes="+contentRange) // write same chunk should fail - contentRange = fmt.Sprintf("%d-%d", 0, len(chunk1)) + contentRange = fmt.Sprintf("%d-%d", 0, len(chunk1)-1) resp, err = resty.R().SetHeader("Content-Type", "application/octet-stream"). SetHeader("Content-Range", contentRange).SetBody(chunk1).Patch(loc) So(err, ShouldBeNil) @@ -270,7 +270,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(digest, ShouldNotBeNil) // write final chunk - contentRange = fmt.Sprintf("%d-%d", len(chunk1), len(buf.Bytes())) + contentRange = fmt.Sprintf("%d-%d", len(chunk1), len(buf.Bytes())-1) resp, err = resty.R().SetQueryParam("digest", digest.String()). SetHeader("Content-Range", contentRange). SetHeader("Content-Type", "application/octet-stream").SetBody(chunk2).Put(loc) @@ -307,7 +307,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(err, ShouldBeNil) // write first chunk - contentRange := fmt.Sprintf("%d-%d", 0, len(chunk1)) + contentRange := fmt.Sprintf("%d-%d", 0, len(chunk1)-1) resp, err = resty.R().SetHeader("Content-Type", "application/octet-stream"). SetHeader("Content-Range", contentRange).SetBody(chunk1).Patch(loc) So(err, ShouldBeNil) @@ -322,7 +322,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(r, ShouldEqual, "bytes="+contentRange) // write same chunk should fail - contentRange = fmt.Sprintf("%d-%d", 0, len(chunk1)) + contentRange = fmt.Sprintf("%d-%d", 0, len(chunk1)-1) resp, err = resty.R().SetHeader("Content-Type", "application/octet-stream"). SetHeader("Content-Range", contentRange).SetBody(chunk1).Patch(loc) So(err, ShouldBeNil) @@ -338,7 +338,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(digest, ShouldNotBeNil) // write final chunk - contentRange = fmt.Sprintf("%d-%d", len(chunk1), len(buf.Bytes())) + contentRange = fmt.Sprintf("%d-%d", len(chunk1), len(buf.Bytes())-1) resp, err = resty.R().SetQueryParam("digest", digest.String()). SetHeader("Content-Range", contentRange). SetHeader("Content-Type", "application/octet-stream").SetBody(chunk2).Put(loc) @@ -470,7 +470,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { // delete manifest resp, err = resty.R().Delete(baseURL + "/v2/repo7/manifests/test:1.0") So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, 200) + So(resp.StatusCode(), ShouldEqual, 202) // delete again should fail resp, err = resty.R().Delete(baseURL + "/v2/repo7/manifests/" + digest.String()) So(err, ShouldBeNil) @@ -494,6 +494,67 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(resp.Body(), ShouldNotBeEmpty) }) + // pagination + Convey("Pagination", func() { + Print("\nPagination") + + for i := 0; i <= 4; i++ { + // create a blob/layer + resp, err := resty.R().Post(baseURL + "/v2/page0/blobs/uploads/") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 202) + loc := Location(baseURL, resp) + So(loc, ShouldNotBeEmpty) + + resp, err = resty.R().Get(loc) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 204) + content := []byte("this is a blob") + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + // monolithic blob upload: success + resp, err = resty.R().SetQueryParam("digest", digest.String()). + SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(loc) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 201) + blobLoc := resp.Header().Get("Location") + So(blobLoc, ShouldNotBeEmpty) + So(resp.Header().Get("Content-Length"), ShouldEqual, "0") + So(resp.Header().Get(api.DistContentDigestKey), ShouldNotBeEmpty) + + // create a manifest + m := ispec.Manifest{Layers: []ispec.Descriptor{{Digest: digest}}} + content, err = json.Marshal(m) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). + SetBody(content).Put(baseURL + fmt.Sprintf("/v2/page0/manifests/test:%d.0", i)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 201) + d := resp.Header().Get(api.DistContentDigestKey) + So(d, ShouldNotBeEmpty) + So(d, ShouldEqual, digest.String()) + } + + resp, err := resty.R().Get(baseURL + "/v2/page0/tags/list") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + resp, err = resty.R().Get(baseURL + "/v2/page0/tags/list?n=3") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + next := resp.Header().Get("Link") + So(next, ShouldNotBeEmpty) + + u := baseURL + strings.Split(next, ";")[0] + resp, err = resty.R().Get(u) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + next = resp.Header().Get("Link") + So(next, ShouldBeEmpty) + }) + // this is an additional test for repository names (alphanumeric) Convey("Repository names", func() { Print("\nRepository names") @@ -504,9 +565,6 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { resp, err = resty.R().Post(baseURL + "/v2/repotest123/blobs/uploads/") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 202) - resp, err = resty.R().Post(baseURL + "/v2/repoTest123/blobs/uploads/") - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, 202) }) }) } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 2cd35f2c..f99a30d0 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -243,6 +243,11 @@ func (is *ImageStore) GetImageManifest(repo string, reference string) ([]byte, s if err != nil { is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") + + if os.IsNotExist(err) { + return nil, "", "", errors.ErrRepoNotFound + } + return nil, "", "", err } @@ -286,8 +291,14 @@ func (is *ImageStore) GetImageManifest(repo string, reference string) ([]byte, s p = path.Join(p, digest.Encoded()) buf, err = ioutil.ReadFile(p) + if err != nil { is.log.Error().Err(err).Str("blob", p).Msg("failed to read manifest") + + if os.IsNotExist(err) { + return nil, "", "", errors.ErrManifestNotFound + } + return nil, "", "", err } @@ -548,6 +559,39 @@ func (is *ImageStore) GetBlobUpload(repo string, uuid string) (int64, error) { return fi.Size(), nil } +// PutBlobChunkStreamed appends another chunk of data to the specified blob. It returns +// the number of actual bytes to the blob. +func (is *ImageStore) PutBlobChunkStreamed(repo string, uuid string, body io.Reader) (int64, error) { + if err := is.InitRepo(repo); err != nil { + return -1, err + } + + blobUploadPath := is.BlobUploadPath(repo, uuid) + + _, err := os.Stat(blobUploadPath) + if err != nil { + return -1, errors.ErrUploadNotFound + } + + file, err := os.OpenFile( + blobUploadPath, + os.O_WRONLY|os.O_CREATE, + 0600, + ) + if err != nil { + is.log.Fatal().Err(err).Msg("failed to open file") + } + defer file.Close() + + if _, err := file.Seek(0, io.SeekEnd); err != nil { + is.log.Fatal().Err(err).Msg("failed to seek file") + } + + n, err := io.Copy(file, body) + + return n, err +} + // PutBlobChunk writes another chunk of data to the specified blob. It returns // the number of actual bytes to the blob. func (is *ImageStore) PutBlobChunk(repo string, uuid string, from int64, to int64, @@ -579,7 +623,7 @@ func (is *ImageStore) PutBlobChunk(repo string, uuid string, from int64, to int6 } defer file.Close() - if _, err := file.Seek(from, 0); err != nil { + if _, err := file.Seek(from, io.SeekStart); err != nil { is.log.Fatal().Err(err).Msg("failed to seek file") }