Merge pull request #174 from shimish2/add-conformance-test

ci/cd: add github action to enable conformance test after every PR

.github/workflows/oci-conformance-action.yml

@@ -0,0 +1,58 @@
+# This is a conformance test workflow that is automatically triggered with each PR
+
+name: conformance
+
+# Controls when the action will run. Workflow runs when manually triggered using the UI
+# or API.
+on:
+  push:
+    branches: 
+      - master
+  pull_request:
+    branches:
+      - master
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+    - name: setup docker
+      uses: docker-practice/actions-setup-docker@0.0.1
+      with: 
+        docker_version: 18.09
+        docker_channel: stable
+    - name: checkout this PR 
+      uses: actions/checkout@v2
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+        path: zot
+        repository: anuvu/zot
+    - name: start zot server
+      run: |
+          cd ./zot
+          IP=`hostname -I | awk '{print $1}'`
+          echo "ZOT_URL=http://${IP}:5000" >> $GITHUB_ENV
+          ZOT_REF="local-zot:v$(date +%Y%m%d%H%M%S)"
+          docker build -f ./Dockerfile-conformance -t "${ZOT_REF}" .
+          docker run --rm -p 5000:5000 -v "$(pwd)":/go/src/github.com/anuvu/zot -idt "${ZOT_REF}"
+    - name: Run OCI Distribution Spec conformance tests
+      uses: opencontainers/distribution-spec@master
+      env:
+        OCI_ROOT_URL: ${{ env.ZOT_URL }}
+        OCI_NAMESPACE: oci-conformance/distribution-test
+        OCI_TEST_PULL: 1
+        OCI_TEST_PUSH: 1
+        OCI_TEST_CONTENT_DISCOVERY: 1
+        OCI_TEST_CONTENT_MANGEMENT: 1
+        OCI_HIDE_SKIPPED_WORKFLOWS: 1
+    - run: mkdir -p .out/ && mv {report.html,junit.xml} .out/
+      if: always()
+      #run: docker run --rm -v $(pwd)/results:/results -w /results -e OCI_ROOT_URL=${{ env.OCI_ROOT_URL }} -e OCI_NAMESPACE="anuvu/zot" -e OCI_TEST_PULL=1 -e OCI_TEST_PUSH=1 -e OCI_TEST_CONTENT_DISCOVERY=1 -e OCI_TEST_CONTENT_MANAGEMENT=1 -e OCI_HIDE_SKIPPED_WORKFLOWS=0 -e OCI_DEBUG="true" ghcr.io/opencontainers/distribution-spec/conformance:db4cc68
+    - name: Upload test results zip as build artifact
+      uses: actions/upload-artifact@v1
+      with:
+        name: oci-test-results-${{ github.sha }}
+        path: .out/
+      if: github.event == 'push'

Dockerfile-conformance

@@ -0,0 +1,29 @@
+# ---
+# Stage 1: Install certs, build binary, create default config file
+# ---
+FROM docker.io/golang:1.13.6-alpine3.11 AS builder
+RUN apk --update add git make ca-certificates
+RUN mkdir -p /go/src/github.com/anuvu/zot
+WORKDIR /go/src/github.com/anuvu/zot
+COPY . .
+RUN CGO_ENABLED=0 make clean binary
+RUN echo -e '# Default config file for zot server\n\
+http:\n\
+  address: 0.0.0.0\n\
+  port: 5000\n\
+storage:\n\
+  rootDirectory: /var/lib/registry\n\
+  gc: false\n\
+  dedupe: false' > config.yml && cat config.yml
+
+# ---
+# Stage 2: Final image with nothing but certs, binary, and default config file
+# ---
+FROM scratch AS final
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=builder /go/src/github.com/anuvu/zot/bin/zot /zot
+COPY --from=builder /go/src/github.com/anuvu/zot/config.yml /etc/zot/config.yml
+ENTRYPOINT ["/zot"]
+EXPOSE 5000
+VOLUME ["/var/lib/registry"]
+CMD ["serve", "/etc/zot/config.yml"]

README.md

@@ -1,4 +1,4 @@
-# zot [![Build Status](https://travis-ci.org/anuvu/zot.svg?branch=master)](https://travis-ci.org/anuvu/zot) [![codecov.io](http://codecov.io/github/anuvu/zot/coverage.svg?branch=master)](http://codecov.io/github/anuvu/zot?branch=master)
+# zot [![Build Status](https://travis-ci.org/anuvu/zot.svg?branch=master)](https://travis-ci.org/anuvu/zot) [![codecov.io](http://codecov.io/github/anuvu/zot/coverage.svg?branch=master)](http://codecov.io/github/anuvu/zot?branch=master) [![Conformance Results](https://github.com/anuvu/zot/workflows/conformance/badge.svg)](https://github.com/anuvu/zot/actions?query=workflow%3Aconformance)
 
 **zot** is a vendor-neutral OCI image repository server purely based on 
 [OCI Distribution Specification](https://github.com/opencontainers/distribution-spec).
@@ -6,7 +6,7 @@
 https://anuvu.github.io/zot/
 
 # Features
-* Conforms to [OCI distribution spec](https://github.com/opencontainers/distribution-spec) APIs [![zot](https://github.com/opencontainers/oci-conformance/workflows/zot-1/badge.svg)](https://github.com/opencontainers/oci-conformance/tree/master/distribution-spec#anuvu/zot)
+* Conforms to [OCI distribution spec](https://github.com/opencontainers/distribution-spec) APIs
 * Uses [OCI image layout](https://github.com/opencontainers/image-spec/blob/master/image-layout.md) for image storage
 * Supports [helm charts](https://helm.sh/docs/topics/registries/)
 * Currently suitable for on-prem deployments (e.g. colocated with Kubernetes)

pkg/api/controller_test.go

@@ -1525,6 +1525,78 @@ func TestHTTPReadOnly(t *testing.T) {
 	})
 }
 
+func TestCrossRepoMount(t *testing.T) {
+	Convey("Cross Repo Mount", t, func() {
+		config := api.NewConfig()
+		config.HTTP.Port = SecurePort1
+		htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase))
+
+		// defer os.Remove(htpasswdPath)
+
+		config.HTTP.Auth = &api.AuthConfig{
+			HTPasswd: api.AuthHTPasswd{
+				Path: htpasswdPath,
+			},
+		}
+
+		c := api.NewController(config)
+
+		dir, err := ioutil.TempDir("", "oci-repo-test")
+		if err != nil {
+			panic(err)
+		}
+
+		err = copyFiles("../../test/data", dir)
+		if err != nil {
+			panic(err)
+		}
+		// defer os.RemoveAll(dir)
+
+		c.Config.Storage.RootDirectory = dir
+
+		go func() {
+			// this blocks
+			if err := c.Run(); err != nil {
+				return
+			}
+		}()
+
+		// wait till ready
+		for {
+			_, err := resty.R().Get(BaseURL1)
+			if err == nil {
+				break
+			}
+
+			time.Sleep(100 * time.Millisecond)
+		}
+
+		params := make(map[string]string)
+
+		params["mount"] = "63a795ca90aa6e7cca60941e826810a4cd0a2e73ea02bf458241df2a5c973e29"
+		params["from"] = "zot-test"
+
+		client := resty.New()
+		postResponse, err := client.R().
+			SetBasicAuth(username, passphrase).SetQueryParams(params).
+			Post(BaseURL1 + "/v2/zot-c-test/blobs/uploads/")
+		So(err, ShouldBeNil)
+		So(postResponse.StatusCode(), ShouldEqual, 202)
+
+		postResponse, err = client.R().
+			SetBasicAuth(username, passphrase).SetQueryParams(params).
+			Post(BaseURL1 + "/v2/zot-cve-test/blobs/uploads/")
+		So(err, ShouldBeNil)
+		So(postResponse.StatusCode(), ShouldEqual, 500)
+
+		postResponse, err = client.R().
+			SetBasicAuth(username, passphrase).SetQueryParams(params).
+			Post(BaseURL1 + "/v2/ /blobs/uploads/")
+		So(err, ShouldBeNil)
+		So(postResponse.StatusCode(), ShouldEqual, 404)
+	})
+}
+
 func TestParallelRequests(t *testing.T) {
 	testCases := []struct {
 		srcImageName  string

pkg/api/routes.go

@@ -626,10 +626,23 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	// blob mounts not allowed since we don't have access control yet, and this
-	// may be a uncommon use case, but remain compliant
-	if _, ok := r.URL.Query()["mount"]; ok {
-		w.WriteHeader(http.StatusMethodNotAllowed)
+	// currently zot does not support cross-repository mounting, following dist-spec and returning 202
+	if mountDigests, ok := r.URL.Query()["mount"]; ok {
+		if len(mountDigests) != 1 {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		u, err := rh.c.ImageStore.NewBlobUpload(name)
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+
+		w.Header().Set("Location", path.Join(r.URL.String(), u))
+		w.Header().Set(BlobUploadUUID, u)
+		w.WriteHeader(http.StatusAccepted)
+
 		return
 	}
 
@@ -906,8 +919,8 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request)
 	rh.c.Log.Info().Int64("r.ContentLength", r.ContentLength).Msg("DEBUG")
 
 	contentPresent := true
-	contentLen, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64)
 
+	contentLen, err := strconv.ParseInt(r.Header.Get("Content-Length"), 10, 64)
 	if err != nil {
 		contentPresent = false
 	}
@@ -1067,8 +1080,8 @@ func (rh *RouteHandler) ListRepositories(w http.ResponseWriter, r *http.Request)
 func getContentRange(r *http.Request) (int64 /* from */, int64 /* to */, error) {
 	contentRange := r.Header.Get("Content-Range")
 	tokens := strings.Split(contentRange, "-")
-	from, err := strconv.ParseInt(tokens[0], 10, 64)
 
+	from, err := strconv.ParseInt(tokens[0], 10, 64)
 	if err != nil {
 		return -1, -1, errors.ErrBadUploadRange
 	}
@@ -1087,8 +1100,8 @@ func getContentRange(r *http.Request) (int64 /* from */, int64 /* to */, error)
 
 func WriteJSON(w http.ResponseWriter, status int, data interface{}) {
 	var json = jsoniter.ConfigCompatibleWithStandardLibrary
-	body, err := json.Marshal(data)
 
+	body, err := json.Marshal(data)
 	if err != nil {
 		panic(err)
 	}

pkg/storage/storage.go

@@ -636,9 +636,10 @@ func (is *ImageStore) NewBlobUpload(repo string) (string, error) {
 	}
 
 	u := uuid.String()
-	blobUploadPath := is.BlobUploadPath(repo, u)
-	file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0600)
 
+	blobUploadPath := is.BlobUploadPath(repo, u)
+
+	file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0600)
 	if err != nil {
 		return "", errors.ErrRepoNotFound
 	}