From 8e16c1c1c60a5101fcab67dbcc7963ce62d73a3b Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 27 Jan 2021 17:29:24 -0800 Subject: [PATCH 1/3] ci/cd: add github action to validate conformance Resolves #173 --- .github/workflows/oci-conformance-action.yml | 58 ++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/workflows/oci-conformance-action.yml diff --git a/.github/workflows/oci-conformance-action.yml b/.github/workflows/oci-conformance-action.yml new file mode 100644 index 00000000..a2b72b4b --- /dev/null +++ b/.github/workflows/oci-conformance-action.yml @@ -0,0 +1,58 @@ +# This is a conformance test workflow that is automatically triggered with each PR + +name: conformance + +# Controls when the action will run. Workflow runs when manually triggered using the UI +# or API. +on: + push: + branches: + - master + pull_request: + branches: + - master + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + run: + runs-on: ubuntu-latest + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + - name: setup docker + uses: docker-practice/actions-setup-docker@0.0.1 + with: + docker_version: 18.09 + docker_channel: stable + - name: checkout this PR + uses: actions/checkout@v2 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: zot + repository: anuvu/zot + - name: start zot server + run: | + cd ./zot + IP=`hostname -I | awk '{print $1}'` + echo "ZOT_URL=http://${IP}:5000" >> $GITHUB_ENV + ZOT_REF="local-zot:v$(date +%Y%m%d%H%M%S)" + docker build -f ./Dockerfile-conformance -t "${ZOT_REF}" . + docker run --rm -p 5000:5000 -v "$(pwd)":/go/src/github.com/anuvu/zot -idt "${ZOT_REF}" + - name: Run OCI Distribution Spec conformance tests + uses: opencontainers/distribution-spec@master + env: + OCI_ROOT_URL: ${{ env.ZOT_URL }} + OCI_NAMESPACE: oci-conformance/distribution-test + OCI_TEST_PULL: 1 + OCI_TEST_PUSH: 1 + OCI_TEST_CONTENT_DISCOVERY: 1 + OCI_TEST_CONTENT_MANGEMENT: 1 + OCI_HIDE_SKIPPED_WORKFLOWS: 1 + - run: mkdir -p .out/ && mv {report.html,junit.xml} .out/ + if: always() + #run: docker run --rm -v $(pwd)/results:/results -w /results -e OCI_ROOT_URL=${{ env.OCI_ROOT_URL }} -e OCI_NAMESPACE="anuvu/zot" -e OCI_TEST_PULL=1 -e OCI_TEST_PUSH=1 -e OCI_TEST_CONTENT_DISCOVERY=1 -e OCI_TEST_CONTENT_MANAGEMENT=1 -e OCI_HIDE_SKIPPED_WORKFLOWS=0 -e OCI_DEBUG="true" ghcr.io/opencontainers/distribution-spec/conformance:db4cc68 + - name: Upload test results zip as build artifact + uses: actions/upload-artifact@v1 + with: + name: oci-test-results-${{ github.sha }} + path: .out/ + if: github.event == 'push' From 9969ba0867a9ded31cb607925202dcf5f9c936a5 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 27 Jan 2021 17:36:33 -0800 Subject: [PATCH 2/3] conformance: update README to display conformance results --- README.md | 4 ++-- pkg/api/routes.go | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 73aab0f0..584bbff3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# zot [![Build Status](https://travis-ci.org/anuvu/zot.svg?branch=master)](https://travis-ci.org/anuvu/zot) [![codecov.io](http://codecov.io/github/anuvu/zot/coverage.svg?branch=master)](http://codecov.io/github/anuvu/zot?branch=master) +# zot [![Build Status](https://travis-ci.org/anuvu/zot.svg?branch=master)](https://travis-ci.org/anuvu/zot) [![codecov.io](http://codecov.io/github/anuvu/zot/coverage.svg?branch=master)](http://codecov.io/github/anuvu/zot?branch=master) [![Conformance Results](https://github.com/anuvu/zot/workflows/conformance/badge.svg)](https://github.com/anuvu/zot/actions?query=workflow%3Aconformance) **zot** is a vendor-neutral OCI image repository server purely based on [OCI Distribution Specification](https://github.com/opencontainers/distribution-spec). @@ -6,7 +6,7 @@ https://anuvu.github.io/zot/ # Features -* Conforms to [OCI distribution spec](https://github.com/opencontainers/distribution-spec) APIs [![zot](https://github.com/opencontainers/oci-conformance/workflows/zot-1/badge.svg)](https://github.com/opencontainers/oci-conformance/tree/master/distribution-spec#anuvu/zot) +* Conforms to [OCI distribution spec](https://github.com/opencontainers/distribution-spec) APIs * Uses [OCI image layout](https://github.com/opencontainers/image-spec/blob/master/image-layout.md) for image storage * Supports [helm charts](https://helm.sh/docs/topics/registries/) * Currently suitable for on-prem deployments (e.g. colocated with Kubernetes) diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 99817cdb..f2410699 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -906,6 +906,7 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request) rh.c.Log.Info().Int64("r.ContentLength", r.ContentLength).Msg("DEBUG") contentPresent := true + contentLen, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64) if err != nil { @@ -1067,6 +1068,7 @@ func (rh *RouteHandler) ListRepositories(w http.ResponseWriter, r *http.Request) func getContentRange(r *http.Request) (int64 /* from */, int64 /* to */, error) { contentRange := r.Header.Get("Content-Range") tokens := strings.Split(contentRange, "-") + from, err := strconv.ParseInt(tokens[0], 10, 64) if err != nil { From 2b7b57313aebdb118c902db7b5558dec338a1563 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 26 Jan 2021 14:26:18 -0800 Subject: [PATCH 3/3] conformance: fix http status code for cross-repository mounting --- Dockerfile-conformance | 29 +++++++++++++++ pkg/api/controller_test.go | 72 ++++++++++++++++++++++++++++++++++++++ pkg/api/routes.go | 25 +++++++++---- pkg/storage/storage.go | 5 +-- 4 files changed, 122 insertions(+), 9 deletions(-) create mode 100644 Dockerfile-conformance diff --git a/Dockerfile-conformance b/Dockerfile-conformance new file mode 100644 index 00000000..58eb4995 --- /dev/null +++ b/Dockerfile-conformance @@ -0,0 +1,29 @@ +# --- +# Stage 1: Install certs, build binary, create default config file +# --- +FROM docker.io/golang:1.13.6-alpine3.11 AS builder +RUN apk --update add git make ca-certificates +RUN mkdir -p /go/src/github.com/anuvu/zot +WORKDIR /go/src/github.com/anuvu/zot +COPY . . +RUN CGO_ENABLED=0 make clean binary +RUN echo -e '# Default config file for zot server\n\ +http:\n\ + address: 0.0.0.0\n\ + port: 5000\n\ +storage:\n\ + rootDirectory: /var/lib/registry\n\ + gc: false\n\ + dedupe: false' > config.yml && cat config.yml + +# --- +# Stage 2: Final image with nothing but certs, binary, and default config file +# --- +FROM scratch AS final +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=builder /go/src/github.com/anuvu/zot/bin/zot /zot +COPY --from=builder /go/src/github.com/anuvu/zot/config.yml /etc/zot/config.yml +ENTRYPOINT ["/zot"] +EXPOSE 5000 +VOLUME ["/var/lib/registry"] +CMD ["serve", "/etc/zot/config.yml"] \ No newline at end of file diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index b73b2e75..a2dfe64d 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -1525,6 +1525,78 @@ func TestHTTPReadOnly(t *testing.T) { }) } +func TestCrossRepoMount(t *testing.T) { + Convey("Cross Repo Mount", t, func() { + config := api.NewConfig() + config.HTTP.Port = SecurePort1 + htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase)) + + // defer os.Remove(htpasswdPath) + + config.HTTP.Auth = &api.AuthConfig{ + HTPasswd: api.AuthHTPasswd{ + Path: htpasswdPath, + }, + } + + c := api.NewController(config) + + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + + err = copyFiles("../../test/data", dir) + if err != nil { + panic(err) + } + // defer os.RemoveAll(dir) + + c.Config.Storage.RootDirectory = dir + + go func() { + // this blocks + if err := c.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(BaseURL1) + if err == nil { + break + } + + time.Sleep(100 * time.Millisecond) + } + + params := make(map[string]string) + + params["mount"] = "63a795ca90aa6e7cca60941e826810a4cd0a2e73ea02bf458241df2a5c973e29" + params["from"] = "zot-test" + + client := resty.New() + postResponse, err := client.R(). + SetBasicAuth(username, passphrase).SetQueryParams(params). + Post(BaseURL1 + "/v2/zot-c-test/blobs/uploads/") + So(err, ShouldBeNil) + So(postResponse.StatusCode(), ShouldEqual, 202) + + postResponse, err = client.R(). + SetBasicAuth(username, passphrase).SetQueryParams(params). + Post(BaseURL1 + "/v2/zot-cve-test/blobs/uploads/") + So(err, ShouldBeNil) + So(postResponse.StatusCode(), ShouldEqual, 500) + + postResponse, err = client.R(). + SetBasicAuth(username, passphrase).SetQueryParams(params). + Post(BaseURL1 + "/v2/ /blobs/uploads/") + So(err, ShouldBeNil) + So(postResponse.StatusCode(), ShouldEqual, 404) + }) +} + func TestParallelRequests(t *testing.T) { testCases := []struct { srcImageName string diff --git a/pkg/api/routes.go b/pkg/api/routes.go index f2410699..fa1ea675 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -626,10 +626,23 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request) return } - // blob mounts not allowed since we don't have access control yet, and this - // may be a uncommon use case, but remain compliant - if _, ok := r.URL.Query()["mount"]; ok { - w.WriteHeader(http.StatusMethodNotAllowed) + // currently zot does not support cross-repository mounting, following dist-spec and returning 202 + if mountDigests, ok := r.URL.Query()["mount"]; ok { + if len(mountDigests) != 1 { + w.WriteHeader(http.StatusBadRequest) + return + } + + u, err := rh.c.ImageStore.NewBlobUpload(name) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + + w.Header().Set("Location", path.Join(r.URL.String(), u)) + w.Header().Set(BlobUploadUUID, u) + w.WriteHeader(http.StatusAccepted) + return } @@ -908,7 +921,6 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request) contentPresent := true contentLen, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64) - if err != nil { contentPresent = false } @@ -1070,7 +1082,6 @@ func getContentRange(r *http.Request) (int64 /* from */, int64 /* to */, error) tokens := strings.Split(contentRange, "-") from, err := strconv.ParseInt(tokens[0], 10, 64) - if err != nil { return -1, -1, errors.ErrBadUploadRange } @@ -1089,8 +1100,8 @@ func getContentRange(r *http.Request) (int64 /* from */, int64 /* to */, error) func WriteJSON(w http.ResponseWriter, status int, data interface{}) { var json = jsoniter.ConfigCompatibleWithStandardLibrary - body, err := json.Marshal(data) + body, err := json.Marshal(data) if err != nil { panic(err) } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 565a7d1c..30a4744e 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -636,9 +636,10 @@ func (is *ImageStore) NewBlobUpload(repo string) (string, error) { } u := uuid.String() - blobUploadPath := is.BlobUploadPath(repo, u) - file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0600) + blobUploadPath := is.BlobUploadPath(repo, u) + + file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0600) if err != nil { return "", errors.ErrRepoNotFound }