mirror of
https://github.com/project-zot/zot.git
synced 2025-02-17 23:45:36 -05:00
Merge pull request #146 from shimish2/rchincha-origin-locks
routes: refactor locks to handle large file uploads
This commit is contained in:
Ramkumar Chinchani
GitHub
commit
17dce7e63b
8 changed files with 535 additions and 91 deletions
|
|
@ -22,7 +22,7 @@ install:
|
|||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then wget -N https://github.com/bazelbuild/bazel/releases/download/3.0.0/bazel-3.0.0-installer-linux-x86_64.sh && chmod +x bazel-3.0.0-installer-linux-x86_64.sh && ./bazel-3.0.0-installer-linux-x86_64.sh --user; go get -u github.com/swaggo/swag/cmd/swag; go mod download; sudo apt-get update; sudo apt-get install rpm; sudo apt install snapd; sudo snap install skopeo --edge --devmode; fi
|
||||
|
||||
script:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then make && travis_wait make -f Makefile.bazel build; fi
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then travis_wait make && travis_wait make -f Makefile.bazel build; fi
|
||||
|
||||
after_success:
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
|
|
|||
1
go.mod
1
go.mod
|
|
@ -31,6 +31,7 @@ require (
|
|||
github.com/smartystreets/goconvey v1.6.4
|
||||
github.com/spf13/cobra v0.0.5
|
||||
github.com/spf13/viper v1.6.1
|
||||
github.com/stretchr/testify v1.6.1
|
||||
github.com/swaggo/http-swagger v0.0.0-20190614090009-c2865af9083e
|
||||
github.com/swaggo/swag v1.6.3
|
||||
github.com/vektah/gqlparser/v2 v2.0.1
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ go_library(
|
|||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
timeout = "moderate",
|
||||
timeout = "long",
|
||||
srcs = ["controller_test.go"],
|
||||
data = [
|
||||
"//:exported_testdata",
|
||||
|
|
@ -49,7 +49,9 @@ go_test(
|
|||
"@com_github_mitchellh_mapstructure//:go_default_library",
|
||||
"@com_github_nmcclain_ldap//:go_default_library",
|
||||
"@com_github_opencontainers_go_digest//:go_default_library",
|
||||
"@com_github_opencontainers_image_spec//specs-go/v1:go_default_library",
|
||||
"@com_github_smartystreets_goconvey//convey:go_default_library",
|
||||
"@com_github_stretchr_testify//assert:go_default_library",
|
||||
"@in_gopkg_resty_v1//:go_default_library",
|
||||
"@org_golang_x_crypto//bcrypt:go_default_library",
|
||||
],
|
||||
|
|
|
|||
|
|
@ -1,17 +1,20 @@
|
|||
package api_test
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
"testing"
|
||||
|
|
@ -23,8 +26,11 @@ import (
|
|||
"github.com/anuvu/zot/pkg/api"
|
||||
"github.com/chartmuseum/auth"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
vldap "github.com/nmcclain/ldap"
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
ispec "github.com/opencontainers/image-spec/specs-go/v1"
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
vldap "github.com/nmcclain/ldap"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
"gopkg.in/resty.v1"
|
||||
)
|
||||
|
|
@ -101,6 +107,7 @@ func getCredString(username, password string) string {
|
|||
|
||||
return usernameAndHash
|
||||
}
|
||||
|
||||
func TestNew(t *testing.T) {
|
||||
Convey("Make a new controller", t, func() {
|
||||
config := api.NewConfig()
|
||||
|
|
@ -1515,3 +1522,469 @@ func TestHTTPReadOnly(t *testing.T) {
|
|||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestParallelRequests(t *testing.T) {
|
||||
testCases := []struct {
|
||||
srcImageName string
|
||||
srcImageTag string
|
||||
destImageName string
|
||||
destImageTag string
|
||||
testCaseName string
|
||||
}{
|
||||
{
|
||||
srcImageName: "zot-test",
|
||||
srcImageTag: "0.0.1",
|
||||
destImageName: "zot-1-test",
|
||||
destImageTag: "0.0.1",
|
||||
testCaseName: "Request-1",
|
||||
},
|
||||
{
|
||||
srcImageName: "zot-test",
|
||||
srcImageTag: "0.0.1",
|
||||
destImageName: "zot-2-test",
|
||||
testCaseName: "Request-2",
|
||||
},
|
||||
{
|
||||
srcImageName: "zot-cve-test",
|
||||
srcImageTag: "0.0.1",
|
||||
destImageName: "zot-3-test",
|
||||
testCaseName: "Request-3",
|
||||
},
|
||||
{
|
||||
srcImageName: "zot-cve-test",
|
||||
srcImageTag: "0.0.1",
|
||||
destImageName: "zot-4-test",
|
||||
testCaseName: "Request-4",
|
||||
},
|
||||
{
|
||||
srcImageName: "zot-cve-test",
|
||||
srcImageTag: "0.0.1",
|
||||
destImageName: "zot-5-test",
|
||||
testCaseName: "Request-5",
|
||||
},
|
||||
{
|
||||
srcImageName: "zot-cve-test",
|
||||
srcImageTag: "0.0.1",
|
||||
destImageName: "zot-1-test",
|
||||
testCaseName: "Request-6",
|
||||
},
|
||||
{
|
||||
srcImageName: "zot-cve-test",
|
||||
srcImageTag: "0.0.1",
|
||||
destImageName: "zot-2-test",
|
||||
testCaseName: "Request-7",
|
||||
},
|
||||
{
|
||||
srcImageName: "zot-cve-test",
|
||||
srcImageTag: "0.0.1",
|
||||
destImageName: "zot-3-test",
|
||||
testCaseName: "Request-8",
|
||||
},
|
||||
{
|
||||
srcImageName: "zot-cve-test",
|
||||
srcImageTag: "0.0.1",
|
||||
destImageName: "zot-4-test",
|
||||
testCaseName: "Request-9",
|
||||
},
|
||||
{
|
||||
srcImageName: "zot-cve-test",
|
||||
srcImageTag: "0.0.1",
|
||||
destImageName: "zot-5-test",
|
||||
testCaseName: "Request-10",
|
||||
},
|
||||
{
|
||||
srcImageName: "zot-test",
|
||||
srcImageTag: "0.0.1",
|
||||
destImageName: "zot-1-test",
|
||||
destImageTag: "0.0.1",
|
||||
testCaseName: "Request-11",
|
||||
},
|
||||
{
|
||||
srcImageName: "zot-test",
|
||||
srcImageTag: "0.0.1",
|
||||
destImageName: "zot-2-test",
|
||||
testCaseName: "Request-12",
|
||||
},
|
||||
{
|
||||
srcImageName: "zot-cve-test",
|
||||
srcImageTag: "0.0.1",
|
||||
destImageName: "zot-3-test",
|
||||
testCaseName: "Request-13",
|
||||
},
|
||||
{
|
||||
srcImageName: "zot-cve-test",
|
||||
srcImageTag: "0.0.1",
|
||||
destImageName: "zot-4-test",
|
||||
testCaseName: "Request-14",
|
||||
},
|
||||
}
|
||||
|
||||
config := api.NewConfig()
|
||||
config.HTTP.Port = SecurePort1
|
||||
htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase))
|
||||
|
||||
// defer os.Remove(htpasswdPath)
|
||||
|
||||
config.HTTP.Auth = &api.AuthConfig{
|
||||
HTPasswd: api.AuthHTPasswd{
|
||||
Path: htpasswdPath,
|
||||
},
|
||||
}
|
||||
|
||||
c := api.NewController(config)
|
||||
|
||||
dir, err := ioutil.TempDir("", "oci-repo-test")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
err = copyFiles("../../test/data", dir)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
//defer os.RemoveAll(dir)
|
||||
c.Config.Storage.RootDirectory = dir
|
||||
|
||||
go func() {
|
||||
// this blocks
|
||||
if err := c.Run(); err != nil {
|
||||
return
|
||||
}
|
||||
}()
|
||||
|
||||
// wait till ready
|
||||
for {
|
||||
_, err := resty.R().Get(BaseURL1)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
// without creds, should get access error
|
||||
for i, testcase := range testCases {
|
||||
testcase := testcase
|
||||
j := i
|
||||
//println(i)
|
||||
t.Run(testcase.testCaseName, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
client := resty.New()
|
||||
|
||||
tagResponse, err := client.R().SetBasicAuth(username, passphrase).
|
||||
Get(BaseURL1 + "/v2/" + testcase.destImageName + "/tags/list")
|
||||
assert.Equal(t, err, nil, "Error should be nil")
|
||||
assert.NotEqual(t, tagResponse.StatusCode(), 400, "bad request")
|
||||
|
||||
manifestList := getAllManifests(path.Join(c.Config.Storage.RootDirectory, testcase.srcImageName))
|
||||
|
||||
for _, manifest := range manifestList {
|
||||
headResponse, err := client.R().SetBasicAuth(username, passphrase).
|
||||
Head(BaseURL1 + "/v2/" + testcase.destImageName + "/manifests/" + manifest)
|
||||
assert.Equal(t, err, nil, "Error should be nil")
|
||||
assert.Equal(t, headResponse.StatusCode(), 404, "response status code should return 404")
|
||||
|
||||
getResponse, err := client.R().SetBasicAuth(username, passphrase).
|
||||
Get(BaseURL1 + "/v2/" + testcase.destImageName + "/manifests/" + manifest)
|
||||
assert.Equal(t, err, nil, "Error should be nil")
|
||||
assert.Equal(t, getResponse.StatusCode(), 404, "response status code should return 404")
|
||||
}
|
||||
|
||||
blobList := getAllBlobs(path.Join(c.Config.Storage.RootDirectory, testcase.srcImageName))
|
||||
|
||||
for _, blob := range blobList {
|
||||
// Get request of blob
|
||||
headResponse, err := client.R().
|
||||
SetBasicAuth(username, passphrase).
|
||||
Head(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/sha256:" + blob)
|
||||
|
||||
assert.Equal(t, err, nil, "Should not be nil")
|
||||
assert.NotEqual(t, headResponse.StatusCode(), 500, "internal server error should not occurred")
|
||||
|
||||
getResponse, err := client.R().
|
||||
SetBasicAuth(username, passphrase).
|
||||
Get(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/sha256:" + blob)
|
||||
|
||||
assert.Equal(t, err, nil, "Should not be nil")
|
||||
assert.NotEqual(t, getResponse.StatusCode(), 500, "internal server error should not occurred")
|
||||
|
||||
blobPath := path.Join(c.Config.Storage.RootDirectory, testcase.srcImageName, "blobs/sha256", blob)
|
||||
|
||||
buf, err := ioutil.ReadFile(blobPath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Post request of blob
|
||||
postResponse, err := client.R().
|
||||
SetHeader("Content-type", "application/octet-stream").
|
||||
SetBasicAuth(username, passphrase).
|
||||
SetBody(buf).Post(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/uploads/")
|
||||
|
||||
assert.Equal(t, err, nil, "Error should be nil")
|
||||
assert.NotEqual(t, postResponse.StatusCode(), 500, "response status code should not return 500")
|
||||
|
||||
// Post request with query parameter
|
||||
|
||||
if j%2 == 0 {
|
||||
postResponse, err = client.R().
|
||||
SetHeader("Content-type", "application/octet-stream").
|
||||
SetBasicAuth(username, passphrase).
|
||||
SetBody(buf).
|
||||
Post(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/uploads/")
|
||||
|
||||
assert.Equal(t, err, nil, "Error should be nil")
|
||||
assert.NotEqual(t, postResponse.StatusCode(), 500, "response status code should not return 500")
|
||||
|
||||
var sessionID string
|
||||
sessionIDList := postResponse.Header().Values("Blob-Upload-UUID")
|
||||
if len(sessionIDList) == 0 {
|
||||
location := postResponse.Header().Values("Location")
|
||||
firstLocation := location[0]
|
||||
splitLocation := strings.Split(firstLocation, "/")
|
||||
sessionID = splitLocation[len(splitLocation)-1]
|
||||
} else {
|
||||
sessionID = sessionIDList[0]
|
||||
}
|
||||
|
||||
file, err := os.Open(blobPath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
defer file.Close()
|
||||
|
||||
reader := bufio.NewReader(file)
|
||||
|
||||
b := make([]byte, 5*1024*1024)
|
||||
|
||||
if j%4 == 0 {
|
||||
readContent := 0
|
||||
for {
|
||||
n, err := reader.Read(b)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
panic(err)
|
||||
}
|
||||
// Patch request of blob
|
||||
|
||||
patchResponse, err := client.R().
|
||||
SetBody(b[0:n]).
|
||||
SetHeader("Content-Type", "application/octet-stream").
|
||||
SetHeader("Content-Length", fmt.Sprintf("%d", n)).
|
||||
SetHeader("Content-Range", fmt.Sprintf("%d", readContent)+"-"+fmt.Sprintf("%d", readContent+n-1)).
|
||||
SetBasicAuth(username, passphrase).
|
||||
Patch(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/uploads/" + sessionID)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
assert.Equal(t, err, nil, "Error should be nil")
|
||||
assert.NotEqual(t, patchResponse.StatusCode(), 500, "response status code should not return 500")
|
||||
|
||||
readContent += n
|
||||
}
|
||||
} else {
|
||||
for {
|
||||
n, err := reader.Read(b)
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
panic(err)
|
||||
}
|
||||
// Patch request of blob
|
||||
|
||||
patchResponse, err := client.R().SetBody(b[0:n]).SetHeader("Content-type", "application/octet-stream").
|
||||
SetBasicAuth(username, passphrase).
|
||||
Patch(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/uploads/" + sessionID)
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
assert.Equal(t, err, nil, "Error should be nil")
|
||||
assert.NotEqual(t, patchResponse.StatusCode(), 500, "response status code should not return 500")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
postResponse, err = client.R().
|
||||
SetHeader("Content-type", "application/octet-stream").
|
||||
SetBasicAuth(username, passphrase).
|
||||
SetBody(buf).SetQueryParam("digest", "sha256:"+blob).
|
||||
Post(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/uploads/")
|
||||
|
||||
assert.Equal(t, err, nil, "Error should be nil")
|
||||
assert.NotEqual(t, postResponse.StatusCode(), 500, "response status code should not return 500")
|
||||
}
|
||||
|
||||
headResponse, err = client.R().
|
||||
SetBasicAuth(username, passphrase).
|
||||
Head(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/sha256:" + blob)
|
||||
|
||||
assert.Equal(t, err, nil, "Should not be nil")
|
||||
assert.NotEqual(t, headResponse.StatusCode(), 500, "response should return success code")
|
||||
|
||||
getResponse, err = client.R().
|
||||
SetBasicAuth(username, passphrase).
|
||||
Get(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/sha256:" + blob)
|
||||
|
||||
assert.Equal(t, err, nil, "Should not be nil")
|
||||
assert.NotEqual(t, getResponse.StatusCode(), 500, "response should return success code")
|
||||
|
||||
if i < 5 { // nolint: scopelint
|
||||
deleteResponse, err := client.R().
|
||||
SetBasicAuth(username, passphrase).
|
||||
Delete(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/sha256:" + blob)
|
||||
|
||||
assert.Equal(t, err, nil, "Should not be nil")
|
||||
assert.Equal(t, deleteResponse.StatusCode(), 202, "response should return success code")
|
||||
}
|
||||
}
|
||||
|
||||
tagResponse, err = client.R().SetBasicAuth(username, passphrase).
|
||||
Get(BaseURL1 + "/v2/" + testcase.destImageName + "/tags/list")
|
||||
assert.Equal(t, err, nil, "Error should be nil")
|
||||
assert.Equal(t, tagResponse.StatusCode(), 200, "response status code should return success code")
|
||||
|
||||
repoResponse, err := client.R().SetBasicAuth(username, passphrase).
|
||||
Get(BaseURL1 + "/v2/_catalog")
|
||||
assert.Equal(t, err, nil, "Error should be nil")
|
||||
assert.Equal(t, repoResponse.StatusCode(), 200, "response status code should return success code")
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func getAllBlobs(imagePath string) []string {
|
||||
blobList := make([]string, 0)
|
||||
|
||||
if !dirExists(imagePath) {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadFile(path.Join(imagePath, "index.json"))
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
var index ispec.Index
|
||||
if err := json.Unmarshal(buf, &index); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
var digest godigest.Digest
|
||||
|
||||
for _, m := range index.Manifests {
|
||||
digest = m.Digest
|
||||
blobList = append(blobList, digest.Encoded())
|
||||
p := path.Join(imagePath, "blobs", digest.Algorithm().String(), digest.Encoded())
|
||||
|
||||
buf, err = ioutil.ReadFile(p)
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
var manifest ispec.Manifest
|
||||
if err := json.Unmarshal(buf, &manifest); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
blobList = append(blobList, manifest.Config.Digest.Encoded())
|
||||
|
||||
for _, layer := range manifest.Layers {
|
||||
blobList = append(blobList, layer.Digest.Encoded())
|
||||
}
|
||||
}
|
||||
|
||||
return blobList
|
||||
}
|
||||
|
||||
func getAllManifests(imagePath string) []string {
|
||||
manifestList := make([]string, 0)
|
||||
|
||||
if !dirExists(imagePath) {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadFile(path.Join(imagePath, "index.json"))
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
var index ispec.Index
|
||||
if err := json.Unmarshal(buf, &index); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
var digest godigest.Digest
|
||||
|
||||
for _, m := range index.Manifests {
|
||||
digest = m.Digest
|
||||
manifestList = append(manifestList, digest.Encoded())
|
||||
}
|
||||
|
||||
return manifestList
|
||||
}
|
||||
|
||||
func dirExists(d string) bool {
|
||||
fi, err := os.Stat(d)
|
||||
if err != nil && os.IsNotExist(err) {
|
||||
return false
|
||||
}
|
||||
|
||||
if !fi.IsDir() {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func copyFiles(sourceDir string, destDir string) error {
|
||||
sourceMeta, err := os.Stat(sourceDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(destDir, sourceMeta.Mode()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
files, err := ioutil.ReadDir(sourceDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
sourceFilePath := path.Join(sourceDir, file.Name())
|
||||
destFilePath := path.Join(destDir, file.Name())
|
||||
|
||||
if file.IsDir() {
|
||||
if err = copyFiles(sourceFilePath, destFilePath); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
sourceFile, err := os.Open(sourceFilePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer sourceFile.Close()
|
||||
|
||||
destFile, err := os.Create(destFilePath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer destFile.Close()
|
||||
|
||||
if _, err = io.Copy(destFile, sourceFile); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,26 +57,6 @@ func (rh *RouteHandler) searchHandler() *gqlHandler.Server {
|
|||
return gqlHandler.NewDefaultServer(search.NewExecutableSchema(resConfig))
|
||||
}
|
||||
|
||||
// blobRLockWrapper calls the real handler with read-lock held.
|
||||
func (rh *RouteHandler) blobRLockWrapper(f func(w http.ResponseWriter,
|
||||
r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
rh.c.ImageStore.RLock()
|
||||
f(w, r)
|
||||
rh.c.ImageStore.RUnlock()
|
||||
}
|
||||
}
|
||||
|
||||
// blobLockWrapper calls the real handler with write-lock held.
|
||||
func (rh *RouteHandler) blobLockWrapper(f func(w http.ResponseWriter,
|
||||
r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
rh.c.ImageStore.Lock()
|
||||
f(w, r)
|
||||
rh.c.ImageStore.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (rh *RouteHandler) SetupRoutes() {
|
||||
rh.c.Router.Use(AuthHandler(rh.c))
|
||||
g := rh.c.Router.PathPrefix(RoutePrefix).Subrouter()
|
||||
|
|
@ -84,29 +64,29 @@ func (rh *RouteHandler) SetupRoutes() {
|
|||
g.HandleFunc(fmt.Sprintf("/{name:%s}/tags/list", NameRegexp.String()),
|
||||
rh.ListTags).Methods("GET")
|
||||
g.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()),
|
||||
rh.blobRLockWrapper(rh.CheckManifest)).Methods("HEAD")
|
||||
rh.CheckManifest).Methods("HEAD")
|
||||
g.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()),
|
||||
rh.blobRLockWrapper(rh.GetManifest)).Methods("GET")
|
||||
rh.GetManifest).Methods("GET")
|
||||
g.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()),
|
||||
rh.blobLockWrapper(rh.UpdateManifest)).Methods("PUT")
|
||||
rh.UpdateManifest).Methods("PUT")
|
||||
g.HandleFunc(fmt.Sprintf("/{name:%s}/manifests/{reference}", NameRegexp.String()),
|
||||
rh.blobLockWrapper(rh.DeleteManifest)).Methods("DELETE")
|
||||
rh.DeleteManifest).Methods("DELETE")
|
||||
g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()),
|
||||
rh.blobRLockWrapper(rh.CheckBlob)).Methods("HEAD")
|
||||
rh.CheckBlob).Methods("HEAD")
|
||||
g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()),
|
||||
rh.blobRLockWrapper(rh.GetBlob)).Methods("GET")
|
||||
rh.GetBlob).Methods("GET")
|
||||
g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/{digest}", NameRegexp.String()),
|
||||
rh.blobLockWrapper(rh.DeleteBlob)).Methods("DELETE")
|
||||
rh.DeleteBlob).Methods("DELETE")
|
||||
g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/", NameRegexp.String()),
|
||||
rh.blobLockWrapper(rh.CreateBlobUpload)).Methods("POST")
|
||||
rh.CreateBlobUpload).Methods("POST")
|
||||
g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()),
|
||||
rh.blobRLockWrapper(rh.GetBlobUpload)).Methods("GET")
|
||||
rh.GetBlobUpload).Methods("GET")
|
||||
g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()),
|
||||
rh.blobLockWrapper(rh.PatchBlobUpload)).Methods("PATCH")
|
||||
rh.PatchBlobUpload).Methods("PATCH")
|
||||
g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()),
|
||||
rh.blobLockWrapper(rh.UpdateBlobUpload)).Methods("PUT")
|
||||
rh.UpdateBlobUpload).Methods("PUT")
|
||||
g.HandleFunc(fmt.Sprintf("/{name:%s}/blobs/uploads/{session_id}", NameRegexp.String()),
|
||||
rh.blobLockWrapper(rh.DeleteBlobUpload)).Methods("DELETE")
|
||||
rh.DeleteBlobUpload).Methods("DELETE")
|
||||
g.HandleFunc("/_catalog",
|
||||
rh.ListRepositories).Methods("GET")
|
||||
g.HandleFunc("/",
|
||||
|
|
|
|||
|
|
@ -326,7 +326,7 @@ func TestServerCVEResponse(t *testing.T) {
|
|||
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
time.Sleep(25 * time.Second)
|
||||
time.Sleep(35 * time.Second)
|
||||
|
||||
defer func(controller *api.Controller) {
|
||||
ctx := context.Background()
|
||||
|
|
|
|||
|
|
@ -365,12 +365,10 @@ func makeHtpasswdFile() string {
|
|||
}
|
||||
|
||||
func TestDownloadDB(t *testing.T) {
|
||||
Convey("Download DB", t, func() {
|
||||
Convey("Download DB passing invalid dir", t, func() {
|
||||
err := testSetup()
|
||||
So(err, ShouldBeNil)
|
||||
err = cveinfo.UpdateCVEDb(dbDir, cve.Log)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
// Test Invalid dir download
|
||||
err = cveinfo.UpdateCVEDb("./testdata1", cve.Log)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
|
|
@ -462,6 +460,8 @@ func TestImageTag(t *testing.T) {
|
|||
|
||||
func TestCVESearch(t *testing.T) {
|
||||
Convey("Test image vulenrability scanning", t, func() {
|
||||
updateDuration, _ := time.ParseDuration("1h")
|
||||
expectedDuration, _ := time.ParseDuration("2h")
|
||||
config := api.NewConfig()
|
||||
config.HTTP.Port = SecurePort1
|
||||
htpasswdPath := makeHtpasswdFile()
|
||||
|
|
@ -476,7 +476,7 @@ func TestCVESearch(t *testing.T) {
|
|||
defer os.RemoveAll(dbDir)
|
||||
c.Config.Storage.RootDirectory = dbDir
|
||||
cveConfig := &api.CVEConfig{
|
||||
UpdateInterval: 2,
|
||||
UpdateInterval: updateDuration,
|
||||
}
|
||||
searchConfig := &api.SearchConfig{
|
||||
CVE: cveConfig,
|
||||
|
|
@ -501,13 +501,15 @@ func TestCVESearch(t *testing.T) {
|
|||
}
|
||||
|
||||
// Wait for trivy db to download
|
||||
time.Sleep(30 * time.Second)
|
||||
time.Sleep(35 * time.Second)
|
||||
|
||||
defer func() {
|
||||
ctx := context.Background()
|
||||
_ = c.Server.Shutdown(ctx)
|
||||
}()
|
||||
|
||||
So(c.Config.Extensions.Search.CVE.UpdateInterval, ShouldEqual, expectedDuration)
|
||||
|
||||
// without creds, should get access error
|
||||
resp, err := resty.R().Get(BaseURL1 + "/v2/")
|
||||
So(err, ShouldBeNil)
|
||||
|
|
@ -684,52 +686,3 @@ func TestCVESearch(t *testing.T) {
|
|||
So(resp.StatusCode(), ShouldEqual, 200)
|
||||
})
|
||||
}
|
||||
|
||||
func TestCveConfig(t *testing.T) {
|
||||
updateDuration, _ := time.ParseDuration("1h")
|
||||
expectedDuration, _ := time.ParseDuration("2h")
|
||||
|
||||
Convey("Make a new cve config", t, func() {
|
||||
config := api.NewConfig()
|
||||
config.HTTP.Port = SecurePort1
|
||||
cveConfig := &api.CVEConfig{
|
||||
UpdateInterval: updateDuration,
|
||||
}
|
||||
searchConfig := &api.SearchConfig{
|
||||
CVE: cveConfig,
|
||||
}
|
||||
config.Extensions = &api.ExtensionConfig{
|
||||
Search: searchConfig,
|
||||
}
|
||||
c := api.NewController(config)
|
||||
dir, err := ioutil.TempDir("", "oci-repo-test")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
c.Config.Storage.RootDirectory = dir
|
||||
|
||||
go func() {
|
||||
// this blocks
|
||||
if err := c.Run(); err != nil {
|
||||
return
|
||||
}
|
||||
}()
|
||||
|
||||
// wait till ready
|
||||
for {
|
||||
_, err := resty.R().Get(BaseURL1)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
|
||||
So(c.Config.Extensions.Search.CVE.UpdateInterval, ShouldEqual, expectedDuration)
|
||||
|
||||
defer func() {
|
||||
ctx := context.Background()
|
||||
_ = c.Server.Shutdown(ctx)
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -111,6 +111,9 @@ func (is *ImageStore) Unlock() {
|
|||
func (is *ImageStore) InitRepo(name string) error {
|
||||
repoDir := path.Join(is.rootDir, name)
|
||||
|
||||
is.Lock()
|
||||
defer is.Unlock()
|
||||
|
||||
if fi, err := os.Stat(repoDir); err == nil && fi.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -217,6 +220,9 @@ func (is *ImageStore) ValidateRepo(name string) (bool, error) {
|
|||
func (is *ImageStore) GetRepositories() ([]string, error) {
|
||||
dir := is.rootDir
|
||||
|
||||
is.RLock()
|
||||
defer is.RUnlock()
|
||||
|
||||
_, err := ioutil.ReadDir(dir)
|
||||
if err != nil {
|
||||
is.log.Error().Err(err).Msg("failure walking storage root-dir")
|
||||
|
|
@ -258,6 +264,9 @@ func (is *ImageStore) GetImageTags(repo string) ([]string, error) {
|
|||
return nil, errors.ErrRepoNotFound
|
||||
}
|
||||
|
||||
is.RLock()
|
||||
defer is.RUnlock()
|
||||
|
||||
buf, err := ioutil.ReadFile(path.Join(dir, "index.json"))
|
||||
if err != nil {
|
||||
is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json")
|
||||
|
|
@ -289,6 +298,9 @@ func (is *ImageStore) GetImageManifest(repo string, reference string) ([]byte, s
|
|||
return nil, "", "", errors.ErrRepoNotFound
|
||||
}
|
||||
|
||||
is.RLock()
|
||||
defer is.RUnlock()
|
||||
|
||||
buf, err := ioutil.ReadFile(path.Join(dir, "index.json"))
|
||||
|
||||
if err != nil {
|
||||
|
|
@ -414,6 +426,9 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType
|
|||
refIsDigest = true
|
||||
}
|
||||
|
||||
is.Lock()
|
||||
defer is.Unlock()
|
||||
|
||||
dir := path.Join(is.rootDir, repo)
|
||||
buf, err := ioutil.ReadFile(path.Join(dir, "index.json"))
|
||||
|
||||
|
|
@ -532,6 +547,9 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error {
|
|||
return errors.ErrBadManifest
|
||||
}
|
||||
|
||||
is.Lock()
|
||||
defer is.Unlock()
|
||||
|
||||
buf, err := ioutil.ReadFile(path.Join(dir, "index.json"))
|
||||
|
||||
if err != nil {
|
||||
|
|
@ -770,6 +788,10 @@ func (is *ImageStore) FinishBlobUpload(repo string, uuid string, body io.Reader,
|
|||
}
|
||||
|
||||
dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String())
|
||||
|
||||
is.Lock()
|
||||
defer is.Unlock()
|
||||
|
||||
ensureDir(dir, is.log)
|
||||
dst := is.BlobPath(repo, dstDigest)
|
||||
|
||||
|
|
@ -835,6 +857,10 @@ func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, digest string)
|
|||
}
|
||||
|
||||
dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String())
|
||||
|
||||
is.Lock()
|
||||
defer is.Unlock()
|
||||
|
||||
ensureDir(dir, is.log)
|
||||
dst := is.BlobPath(repo, dstDigest)
|
||||
|
||||
|
|
@ -948,6 +974,9 @@ func (is *ImageStore) CheckBlob(repo string, digest string,
|
|||
|
||||
blobPath := is.BlobPath(repo, d)
|
||||
|
||||
is.RLock()
|
||||
defer is.RUnlock()
|
||||
|
||||
blobInfo, err := os.Stat(blobPath)
|
||||
if err != nil {
|
||||
is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob")
|
||||
|
|
@ -969,6 +998,9 @@ func (is *ImageStore) GetBlob(repo string, digest string, mediaType string) (io.
|
|||
|
||||
blobPath := is.BlobPath(repo, d)
|
||||
|
||||
is.RLock()
|
||||
defer is.RUnlock()
|
||||
|
||||
blobInfo, err := os.Stat(blobPath)
|
||||
if err != nil {
|
||||
is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob")
|
||||
|
|
@ -994,6 +1026,9 @@ func (is *ImageStore) DeleteBlob(repo string, digest string) error {
|
|||
|
||||
blobPath := is.BlobPath(repo, d)
|
||||
|
||||
is.Lock()
|
||||
defer is.Unlock()
|
||||
|
||||
_, err = os.Stat(blobPath)
|
||||
if err != nil {
|
||||
is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue