From 25ad71787ad31d56c2f28be0cedf0da0a71a4f67 Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Wed, 14 Oct 2020 10:29:17 -0700 Subject: [PATCH 1/4] test: minimize trivy db download tests to avoid api rate limit --- go.mod | 1 + pkg/extensions/search/cve/cve_test.go | 61 +++------------------------ 2 files changed, 8 insertions(+), 54 deletions(-) diff --git a/go.mod b/go.mod index b3ee67ff..20316b5f 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/smartystreets/goconvey v1.6.4 github.com/spf13/cobra v0.0.5 github.com/spf13/viper v1.6.1 + github.com/stretchr/testify v1.6.1 github.com/swaggo/http-swagger v0.0.0-20190614090009-c2865af9083e github.com/swaggo/swag v1.6.3 github.com/vektah/gqlparser/v2 v2.0.1 diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index d88390c4..cb7ad7b9 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -365,12 +365,10 @@ func makeHtpasswdFile() string { } func TestDownloadDB(t *testing.T) { - Convey("Download DB", t, func() { + Convey("Download DB passing invalid dir", t, func() { err := testSetup() So(err, ShouldBeNil) - err = cveinfo.UpdateCVEDb(dbDir, cve.Log) - So(err, ShouldBeNil) - + // Test Invalid dir download err = cveinfo.UpdateCVEDb("./testdata1", cve.Log) So(err, ShouldNotBeNil) }) @@ -462,6 +460,8 @@ func TestImageTag(t *testing.T) { func TestCVESearch(t *testing.T) { Convey("Test image vulenrability scanning", t, func() { + updateDuration, _ := time.ParseDuration("1h") + expectedDuration, _ := time.ParseDuration("2h") config := api.NewConfig() config.HTTP.Port = SecurePort1 htpasswdPath := makeHtpasswdFile() @@ -476,7 +476,7 @@ func TestCVESearch(t *testing.T) { defer os.RemoveAll(dbDir) c.Config.Storage.RootDirectory = dbDir cveConfig := &api.CVEConfig{ - UpdateInterval: 2, + UpdateInterval: updateDuration, } searchConfig := &api.SearchConfig{ CVE: cveConfig, @@ -508,6 +508,8 @@ func TestCVESearch(t *testing.T) { _ = c.Server.Shutdown(ctx) }() + So(c.Config.Extensions.Search.CVE.UpdateInterval, ShouldEqual, expectedDuration) + // without creds, should get access error resp, err := resty.R().Get(BaseURL1 + "/v2/") So(err, ShouldBeNil) @@ -684,52 +686,3 @@ func TestCVESearch(t *testing.T) { So(resp.StatusCode(), ShouldEqual, 200) }) } - -func TestCveConfig(t *testing.T) { - updateDuration, _ := time.ParseDuration("1h") - expectedDuration, _ := time.ParseDuration("2h") - - Convey("Make a new cve config", t, func() { - config := api.NewConfig() - config.HTTP.Port = SecurePort1 - cveConfig := &api.CVEConfig{ - UpdateInterval: updateDuration, - } - searchConfig := &api.SearchConfig{ - CVE: cveConfig, - } - config.Extensions = &api.ExtensionConfig{ - Search: searchConfig, - } - c := api.NewController(config) - dir, err := ioutil.TempDir("", "oci-repo-test") - if err != nil { - panic(err) - } - defer os.RemoveAll(dir) - c.Config.Storage.RootDirectory = dir - - go func() { - // this blocks - if err := c.Run(); err != nil { - return - } - }() - - // wait till ready - for { - _, err := resty.R().Get(BaseURL1) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - - So(c.Config.Extensions.Search.CVE.UpdateInterval, ShouldEqual, expectedDuration) - - defer func() { - ctx := context.Background() - _ = c.Server.Shutdown(ctx) - }() - }) -} From 386c72d332018a3e0deae50ce51c2f4cff3ee8ff Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Wed, 30 Sep 2020 17:16:34 -0700 Subject: [PATCH 2/4] routes: refactor locks to handle large file uploads The storage layer is protected with read-write locks. However, we may be holding the locks over unnecessarily large critical sections. The typical workflow is that a blob is first uploaded via a per-client private session-id meaning the blob is not publicly visible yet. When the blob being uploaded is very large, the transfer takes a long time while holding the lock. Private session-id based uploads don't really need locks, and hold locks only when blobs are published after the upload is complete. --- pkg/api/routes.go | 44 ++++++++++++------------------------------ pkg/storage/storage.go | 35 +++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 32 deletions(-) diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 74992ff3..dc88e062 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -57,26 +57,6 @@ func (rh *RouteHandler) searchHandler() *gqlHandler.Server { return gqlHandler.NewDefaultServer(search.NewExecutableSchema(resConfig)) } -// blobRLockWrapper calls the real handler with read-lock held. -func (rh *RouteHandler) blobRLockWrapper(f func(w http.ResponseWriter, - r *http.Request)) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - rh.c.ImageStore.RLock() - f(w, r) - rh.c.ImageStore.RUnlock() - } -} - -// blobLockWrapper calls the real handler with write-lock held. -func (rh *RouteHandler) blobLockWrapper(f func(w http.ResponseWriter, - r *http.Request)) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - rh.c.ImageStore.Lock() - f(w, r) - rh.c.ImageStore.Unlock() - } -} - func (rh *RouteHandler) SetupRoutes() { rh.c.Router.Use(AuthHandler(rh.c)) g := rh.c.Router.PathPrefix(RoutePrefix).Subrouter() @@ -84,29 +64,29 @@ func (rh *RouteHandler) SetupRoutes() { g.HandleFunc(fmt.Sprintf("/{name:%s}/tags/list", NameRegexp.String()), rh.ListTags).Methods("GET") g.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()), - rh.blobRLockWrapper(rh.CheckManifest)).Methods("HEAD") + rh.CheckManifest).Methods("HEAD") g.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()), - rh.blobRLockWrapper(rh.GetManifest)).Methods("GET") + rh.GetManifest).Methods("GET") g.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()), - rh.blobLockWrapper(rh.UpdateManifest)).Methods("PUT") + rh.UpdateManifest).Methods("PUT") g.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()), - rh.blobLockWrapper(rh.DeleteManifest)).Methods("DELETE") + rh.DeleteManifest).Methods("DELETE") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()), - rh.blobRLockWrapper(rh.CheckBlob)).Methods("HEAD") + rh.CheckBlob).Methods("HEAD") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()), - rh.blobRLockWrapper(rh.GetBlob)).Methods("GET") + rh.GetBlob).Methods("GET") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()), - rh.blobLockWrapper(rh.DeleteBlob)).Methods("DELETE") + rh.DeleteBlob).Methods("DELETE") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/", NameRegexp.String()), - rh.blobLockWrapper(rh.CreateBlobUpload)).Methods("POST") + rh.CreateBlobUpload).Methods("POST") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()), - rh.blobRLockWrapper(rh.GetBlobUpload)).Methods("GET") + rh.GetBlobUpload).Methods("GET") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()), - rh.blobLockWrapper(rh.PatchBlobUpload)).Methods("PATCH") + rh.PatchBlobUpload).Methods("PATCH") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()), - rh.blobLockWrapper(rh.UpdateBlobUpload)).Methods("PUT") + rh.UpdateBlobUpload).Methods("PUT") g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()), - rh.blobLockWrapper(rh.DeleteBlobUpload)).Methods("DELETE") + rh.DeleteBlobUpload).Methods("DELETE") g.HandleFunc("/_catalog", rh.ListRepositories).Methods("GET") g.HandleFunc("/", diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 917cab03..565a7d1c 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -111,6 +111,9 @@ func (is *ImageStore) Unlock() { func (is *ImageStore) InitRepo(name string) error { repoDir := path.Join(is.rootDir, name) + is.Lock() + defer is.Unlock() + if fi, err := os.Stat(repoDir); err == nil && fi.IsDir() { return nil } @@ -217,6 +220,9 @@ func (is *ImageStore) ValidateRepo(name string) (bool, error) { func (is *ImageStore) GetRepositories() ([]string, error) { dir := is.rootDir + is.RLock() + defer is.RUnlock() + _, err := ioutil.ReadDir(dir) if err != nil { is.log.Error().Err(err).Msg("failure walking storage root-dir") @@ -258,6 +264,9 @@ func (is *ImageStore) GetImageTags(repo string) ([]string, error) { return nil, errors.ErrRepoNotFound } + is.RLock() + defer is.RUnlock() + buf, err := ioutil.ReadFile(path.Join(dir, "index.json")) if err != nil { is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") @@ -289,6 +298,9 @@ func (is *ImageStore) GetImageManifest(repo string, reference string) ([]byte, s return nil, "", "", errors.ErrRepoNotFound } + is.RLock() + defer is.RUnlock() + buf, err := ioutil.ReadFile(path.Join(dir, "index.json")) if err != nil { @@ -414,6 +426,9 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType refIsDigest = true } + is.Lock() + defer is.Unlock() + dir := path.Join(is.rootDir, repo) buf, err := ioutil.ReadFile(path.Join(dir, "index.json")) @@ -532,6 +547,9 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error { return errors.ErrBadManifest } + is.Lock() + defer is.Unlock() + buf, err := ioutil.ReadFile(path.Join(dir, "index.json")) if err != nil { @@ -770,6 +788,10 @@ func (is *ImageStore) FinishBlobUpload(repo string, uuid string, body io.Reader, } dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) + + is.Lock() + defer is.Unlock() + ensureDir(dir, is.log) dst := is.BlobPath(repo, dstDigest) @@ -835,6 +857,10 @@ func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, digest string) } dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) + + is.Lock() + defer is.Unlock() + ensureDir(dir, is.log) dst := is.BlobPath(repo, dstDigest) @@ -948,6 +974,9 @@ func (is *ImageStore) CheckBlob(repo string, digest string, blobPath := is.BlobPath(repo, d) + is.RLock() + defer is.RUnlock() + blobInfo, err := os.Stat(blobPath) if err != nil { is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") @@ -969,6 +998,9 @@ func (is *ImageStore) GetBlob(repo string, digest string, mediaType string) (io. blobPath := is.BlobPath(repo, d) + is.RLock() + defer is.RUnlock() + blobInfo, err := os.Stat(blobPath) if err != nil { is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") @@ -994,6 +1026,9 @@ func (is *ImageStore) DeleteBlob(repo string, digest string) error { blobPath := is.BlobPath(repo, d) + is.Lock() + defer is.Unlock() + _, err = os.Stat(blobPath) if err != nil { is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") From 14214a57941e41168560893ed70a82c2a89b879a Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Tue, 13 Oct 2020 12:03:47 -0700 Subject: [PATCH 3/4] test: add unit test to verify lock changes --- pkg/api/BUILD.bazel | 4 +- pkg/api/controller_test.go | 511 +++++++++++++++++++++++++- pkg/cli/cve_cmd_test.go | 2 +- pkg/extensions/search/cve/cve_test.go | 2 +- 4 files changed, 515 insertions(+), 4 deletions(-) diff --git a/pkg/api/BUILD.bazel b/pkg/api/BUILD.bazel index ca6c8c37..7ce00ca1 100644 --- a/pkg/api/BUILD.bazel +++ b/pkg/api/BUILD.bazel @@ -36,7 +36,7 @@ go_library( go_test( name = "go_default_test", - timeout = "moderate", + timeout = "long", srcs = ["controller_test.go"], data = [ "//:exported_testdata", @@ -49,7 +49,9 @@ go_test( "@com_github_mitchellh_mapstructure//:go_default_library", "@com_github_nmcclain_ldap//:go_default_library", "@com_github_opencontainers_go_digest//:go_default_library", + "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library", "@com_github_smartystreets_goconvey//convey:go_default_library", + "@com_github_stretchr_testify//assert:go_default_library", "@in_gopkg_resty_v1//:go_default_library", "@org_golang_x_crypto//bcrypt:go_default_library", ], diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 93896374..141cda73 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -1,17 +1,20 @@ package api_test import ( + "bufio" "context" "crypto/tls" "crypto/x509" "encoding/json" "fmt" + "io" "io/ioutil" "net" "net/http" "net/http/httptest" "net/url" "os" + "path" "regexp" "strings" "testing" @@ -23,8 +26,11 @@ import ( "github.com/anuvu/zot/pkg/api" "github.com/chartmuseum/auth" "github.com/mitchellh/mapstructure" - vldap "github.com/nmcclain/ldap" godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/stretchr/testify/assert" + + vldap "github.com/nmcclain/ldap" . "github.com/smartystreets/goconvey/convey" "gopkg.in/resty.v1" ) @@ -101,6 +107,7 @@ func getCredString(username, password string) string { return usernameAndHash } + func TestNew(t *testing.T) { Convey("Make a new controller", t, func() { config := api.NewConfig() @@ -1515,3 +1522,505 @@ func TestHTTPReadOnly(t *testing.T) { } }) } + +func TestParallelRequests(t *testing.T) { + testCases := []struct { + srcImageName string + srcImageTag string + destImageName string + destImageTag string + testCaseName string + }{ + { + srcImageName: "zot-test", + srcImageTag: "0.0.1", + destImageName: "zot-1-test", + destImageTag: "0.0.1", + testCaseName: "Request-1", + }, + { + srcImageName: "zot-test", + srcImageTag: "0.0.1", + destImageName: "zot-2-test", + testCaseName: "Request-2", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-3-test", + testCaseName: "Request-3", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-4-test", + testCaseName: "Request-4", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-5-test", + testCaseName: "Request-5", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-1-test", + testCaseName: "Request-6", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-2-test", + testCaseName: "Request-7", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-3-test", + testCaseName: "Request-8", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-4-test", + testCaseName: "Request-9", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-5-test", + testCaseName: "Request-10", + }, + { + srcImageName: "zot-test", + srcImageTag: "0.0.1", + destImageName: "zot-1-test", + destImageTag: "0.0.1", + testCaseName: "Request-11", + }, + { + srcImageName: "zot-test", + srcImageTag: "0.0.1", + destImageName: "zot-2-test", + testCaseName: "Request-12", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-3-test", + testCaseName: "Request-13", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-4-test", + testCaseName: "Request-14", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-5-test", + testCaseName: "Request-15", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-1-test", + testCaseName: "Request-16", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-2-test", + testCaseName: "Request-17", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-3-test", + testCaseName: "Request-18", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-4-test", + testCaseName: "Request-19", + }, + { + srcImageName: "zot-cve-test", + srcImageTag: "0.0.1", + destImageName: "zot-5-test", + testCaseName: "Request-20", + }, + } + + config := api.NewConfig() + config.HTTP.Port = SecurePort1 + htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase)) + + // defer os.Remove(htpasswdPath) + + config.HTTP.Auth = &api.AuthConfig{ + HTPasswd: api.AuthHTPasswd{ + Path: htpasswdPath, + }, + } + + c := api.NewController(config) + + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + + err = copyFiles("../../test/data", dir) + if err != nil { + panic(err) + } + //defer os.RemoveAll(dir) + c.Config.Storage.RootDirectory = dir + + go func() { + // this blocks + if err := c.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(BaseURL1) + if err == nil { + break + } + + time.Sleep(100 * time.Millisecond) + } + + // without creds, should get access error + for i, testcase := range testCases { + testcase := testcase + j := i + //println(i) + t.Run(testcase.testCaseName, func(t *testing.T) { + t.Parallel() + client := resty.New() + + tagResponse, err := client.R().SetBasicAuth(username, passphrase). + Get(BaseURL1 + "/v2/" + testcase.destImageName + "/tags/list") + assert.Equal(t, err, nil, "Error should be nil") + assert.NotEqual(t, tagResponse.StatusCode(), 400, "bad request") + + manifestList := getAllManifests(path.Join(c.Config.Storage.RootDirectory, testcase.srcImageName)) + + for _, manifest := range manifestList { + headResponse, err := client.R().SetBasicAuth(username, passphrase). + Head(BaseURL1 + "/v2/" + testcase.destImageName + "/manifests/" + manifest) + assert.Equal(t, err, nil, "Error should be nil") + assert.Equal(t, headResponse.StatusCode(), 404, "response status code should return 404") + + getResponse, err := client.R().SetBasicAuth(username, passphrase). + Get(BaseURL1 + "/v2/" + testcase.destImageName + "/manifests/" + manifest) + assert.Equal(t, err, nil, "Error should be nil") + assert.Equal(t, getResponse.StatusCode(), 404, "response status code should return 404") + } + + blobList := getAllBlobs(path.Join(c.Config.Storage.RootDirectory, testcase.srcImageName)) + + for _, blob := range blobList { + // Get request of blob + headResponse, err := client.R(). + SetBasicAuth(username, passphrase). + Head(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/sha256:" + blob) + + assert.Equal(t, err, nil, "Should not be nil") + assert.NotEqual(t, headResponse.StatusCode(), 500, "internal server error should not occurred") + + getResponse, err := client.R(). + SetBasicAuth(username, passphrase). + Get(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/sha256:" + blob) + + assert.Equal(t, err, nil, "Should not be nil") + assert.NotEqual(t, getResponse.StatusCode(), 500, "internal server error should not occurred") + + blobPath := path.Join(c.Config.Storage.RootDirectory, testcase.srcImageName, "blobs/sha256", blob) + + buf, err := ioutil.ReadFile(blobPath) + if err != nil { + panic(err) + } + + // Post request of blob + postResponse, err := client.R(). + SetHeader("Content-type", "application/octet-stream"). + SetBasicAuth(username, passphrase). + SetBody(buf).Post(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/uploads/") + + assert.Equal(t, err, nil, "Error should be nil") + assert.NotEqual(t, postResponse.StatusCode(), 500, "response status code should not return 500") + + // Post request with query parameter + + if j%2 == 0 { + postResponse, err = client.R(). + SetHeader("Content-type", "application/octet-stream"). + SetBasicAuth(username, passphrase). + SetBody(buf). + Post(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/uploads/") + + assert.Equal(t, err, nil, "Error should be nil") + assert.NotEqual(t, postResponse.StatusCode(), 500, "response status code should not return 500") + + var sessionID string + sessionIDList := postResponse.Header().Values("Blob-Upload-UUID") + if len(sessionIDList) == 0 { + location := postResponse.Header().Values("Location") + firstLocation := location[0] + splitLocation := strings.Split(firstLocation, "/") + sessionID = splitLocation[len(splitLocation)-1] + } else { + sessionID = sessionIDList[0] + } + + file, err := os.Open(blobPath) + if err != nil { + panic(err) + } + + defer file.Close() + + reader := bufio.NewReader(file) + + b := make([]byte, 1024*1024) + + if j%4 == 0 { + readContent := 0 + for { + n, err := reader.Read(b) + if err != nil { + if err == io.EOF { + break + } + panic(err) + } + // Patch request of blob + + patchResponse, err := client.R(). + SetBody(b[0:n]). + SetHeader("Content-Type", "application/octet-stream"). + SetHeader("Content-Length", fmt.Sprintf("%d", n)). + SetHeader("Content-Range", fmt.Sprintf("%d", readContent)+"-"+fmt.Sprintf("%d", readContent+n-1)). + SetBasicAuth(username, passphrase). + Patch(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/uploads/" + sessionID) + if err != nil { + panic(err) + } + + assert.Equal(t, err, nil, "Error should be nil") + assert.NotEqual(t, patchResponse.StatusCode(), 500, "response status code should not return 500") + + readContent += n + } + } else { + for { + n, err := reader.Read(b) + if err != nil { + if err == io.EOF { + break + } + panic(err) + } + // Patch request of blob + + patchResponse, err := client.R().SetBody(b[0:n]).SetHeader("Content-type", "application/octet-stream"). + SetBasicAuth(username, passphrase). + Patch(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/uploads/" + sessionID) + + if err != nil { + panic(err) + } + + assert.Equal(t, err, nil, "Error should be nil") + assert.NotEqual(t, patchResponse.StatusCode(), 500, "response status code should not return 500") + } + } + } else { + postResponse, err = client.R(). + SetHeader("Content-type", "application/octet-stream"). + SetBasicAuth(username, passphrase). + SetBody(buf).SetQueryParam("digest", "sha256:"+blob). + Post(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/uploads/") + + assert.Equal(t, err, nil, "Error should be nil") + assert.NotEqual(t, postResponse.StatusCode(), 500, "response status code should not return 500") + } + + headResponse, err = client.R(). + SetBasicAuth(username, passphrase). + Head(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/sha256:" + blob) + + assert.Equal(t, err, nil, "Should not be nil") + assert.NotEqual(t, headResponse.StatusCode(), 500, "response should return success code") + + getResponse, err = client.R(). + SetBasicAuth(username, passphrase). + Get(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/sha256:" + blob) + + assert.Equal(t, err, nil, "Should not be nil") + assert.NotEqual(t, getResponse.StatusCode(), 500, "response should return success code") + + if i < 5 { // nolint: scopelint + deleteResponse, err := client.R(). + SetBasicAuth(username, passphrase). + Delete(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/sha256:" + blob) + + assert.Equal(t, err, nil, "Should not be nil") + assert.Equal(t, deleteResponse.StatusCode(), 202, "response should return success code") + } + } + + tagResponse, err = client.R().SetBasicAuth(username, passphrase). + Get(BaseURL1 + "/v2/" + testcase.destImageName + "/tags/list") + assert.Equal(t, err, nil, "Error should be nil") + assert.Equal(t, tagResponse.StatusCode(), 200, "response status code should return success code") + + repoResponse, err := client.R().SetBasicAuth(username, passphrase). + Get(BaseURL1 + "/v2/_catalog") + assert.Equal(t, err, nil, "Error should be nil") + assert.Equal(t, repoResponse.StatusCode(), 200, "response status code should return success code") + }) + } +} + +func getAllBlobs(imagePath string) []string { + blobList := make([]string, 0) + + if !dirExists(imagePath) { + return []string{} + } + + buf, err := ioutil.ReadFile(path.Join(imagePath, "index.json")) + + if err != nil { + panic(err) + } + + var index ispec.Index + if err := json.Unmarshal(buf, &index); err != nil { + panic(err) + } + + var digest godigest.Digest + + for _, m := range index.Manifests { + digest = m.Digest + blobList = append(blobList, digest.Encoded()) + p := path.Join(imagePath, "blobs", digest.Algorithm().String(), digest.Encoded()) + + buf, err = ioutil.ReadFile(p) + + if err != nil { + panic(err) + } + + var manifest ispec.Manifest + if err := json.Unmarshal(buf, &manifest); err != nil { + panic(err) + } + + blobList = append(blobList, manifest.Config.Digest.Encoded()) + + for _, layer := range manifest.Layers { + blobList = append(blobList, layer.Digest.Encoded()) + } + } + + return blobList +} + +func getAllManifests(imagePath string) []string { + manifestList := make([]string, 0) + + if !dirExists(imagePath) { + return []string{} + } + + buf, err := ioutil.ReadFile(path.Join(imagePath, "index.json")) + + if err != nil { + panic(err) + } + + var index ispec.Index + if err := json.Unmarshal(buf, &index); err != nil { + panic(err) + } + + var digest godigest.Digest + + for _, m := range index.Manifests { + digest = m.Digest + manifestList = append(manifestList, digest.Encoded()) + } + + return manifestList +} + +func dirExists(d string) bool { + fi, err := os.Stat(d) + if err != nil && os.IsNotExist(err) { + return false + } + + if !fi.IsDir() { + return false + } + + return true +} + +func copyFiles(sourceDir string, destDir string) error { + sourceMeta, err := os.Stat(sourceDir) + if err != nil { + return err + } + + if err := os.MkdirAll(destDir, sourceMeta.Mode()); err != nil { + return err + } + + files, err := ioutil.ReadDir(sourceDir) + if err != nil { + return err + } + + for _, file := range files { + sourceFilePath := path.Join(sourceDir, file.Name()) + destFilePath := path.Join(destDir, file.Name()) + + if file.IsDir() { + if err = copyFiles(sourceFilePath, destFilePath); err != nil { + return err + } + } else { + sourceFile, err := os.Open(sourceFilePath) + if err != nil { + return err + } + defer sourceFile.Close() + + destFile, err := os.Create(destFilePath) + if err != nil { + return err + } + defer destFile.Close() + + if _, err = io.Copy(destFile, sourceFile); err != nil { + return err + } + } + } + + return nil +} diff --git a/pkg/cli/cve_cmd_test.go b/pkg/cli/cve_cmd_test.go index 824aeb21..3a4b9652 100644 --- a/pkg/cli/cve_cmd_test.go +++ b/pkg/cli/cve_cmd_test.go @@ -326,7 +326,7 @@ func TestServerCVEResponse(t *testing.T) { time.Sleep(100 * time.Millisecond) } - time.Sleep(25 * time.Second) + time.Sleep(35 * time.Second) defer func(controller *api.Controller) { ctx := context.Background() diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index cb7ad7b9..d1e88fa5 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -501,7 +501,7 @@ func TestCVESearch(t *testing.T) { } // Wait for trivy db to download - time.Sleep(30 * time.Second) + time.Sleep(35 * time.Second) defer func() { ctx := context.Background() From 7439feb1c2e6bc4ff83060480ba229f829e52ecf Mon Sep 17 00:00:00 2001 From: Shivam Mishra Date: Fri, 16 Oct 2020 15:30:08 -0700 Subject: [PATCH 4/4] build: set timeout in travis make build process to avoid timeout failure --- .travis.yml | 2 +- pkg/api/controller_test.go | 38 +------------------------------------- 2 files changed, 2 insertions(+), 38 deletions(-) diff --git a/.travis.yml b/.travis.yml index 634464ed..e73c222a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,7 +22,7 @@ install: - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then wget -N https://github.com/bazelbuild/bazel/releases/download/3.0.0/bazel-3.0.0-installer-linux-x86_64.sh && chmod +x bazel-3.0.0-installer-linux-x86_64.sh && ./bazel-3.0.0-installer-linux-x86_64.sh --user; go get -u github.com/swaggo/swag/cmd/swag; go mod download; sudo apt-get update; sudo apt-get install rpm; sudo apt install snapd; sudo snap install skopeo --edge --devmode; fi script: - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make && travis_wait make -f Makefile.bazel build; fi + - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then travis_wait make && travis_wait make -f Makefile.bazel build; fi after_success: - bash <(curl -s https://codecov.io/bash) diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 141cda73..7885aec3 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -1617,42 +1617,6 @@ func TestParallelRequests(t *testing.T) { destImageName: "zot-4-test", testCaseName: "Request-14", }, - { - srcImageName: "zot-cve-test", - srcImageTag: "0.0.1", - destImageName: "zot-5-test", - testCaseName: "Request-15", - }, - { - srcImageName: "zot-cve-test", - srcImageTag: "0.0.1", - destImageName: "zot-1-test", - testCaseName: "Request-16", - }, - { - srcImageName: "zot-cve-test", - srcImageTag: "0.0.1", - destImageName: "zot-2-test", - testCaseName: "Request-17", - }, - { - srcImageName: "zot-cve-test", - srcImageTag: "0.0.1", - destImageName: "zot-3-test", - testCaseName: "Request-18", - }, - { - srcImageName: "zot-cve-test", - srcImageTag: "0.0.1", - destImageName: "zot-4-test", - testCaseName: "Request-19", - }, - { - srcImageName: "zot-cve-test", - srcImageTag: "0.0.1", - destImageName: "zot-5-test", - testCaseName: "Request-20", - }, } config := api.NewConfig() @@ -1792,7 +1756,7 @@ func TestParallelRequests(t *testing.T) { reader := bufio.NewReader(file) - b := make([]byte, 1024*1024) + b := make([]byte, 5*1024*1024) if j%4 == 0 { readContent := 0