From e900b09cfb7ad31e27faf1d67588731c41f27223 Mon Sep 17 00:00:00 2001 From: Alexei Dodon Date: Wed, 10 Nov 2021 14:31:03 +0000 Subject: [PATCH] Fix data races in tests, closes #255 Signed-off-by: Alexei Dodon --- pkg/api/controller_test.go | 448 ++++++++------------ pkg/cli/client.go | 18 +- pkg/cli/client_test.go | 73 +--- pkg/cli/cve_cmd_test.go | 10 +- pkg/cli/image_cmd_test.go | 30 +- pkg/compliance/v1_0_0/check.go | 15 +- pkg/compliance/v1_0_0/check_test.go | 22 +- pkg/exporter/api/controller_test.go | 16 +- pkg/extensions/search/common/common_test.go | 80 +--- pkg/extensions/search/cve/cve_test.go | 117 +---- pkg/extensions/search/digest/digest_test.go | 101 ++--- pkg/extensions/sync/sync_internal_test.go | 58 +-- pkg/extensions/sync/sync_test.go | 227 ++++------ pkg/log/log_test.go | 106 +---- pkg/storage/storage_fs.go | 24 +- pkg/storage/storage_fs_test.go | 19 +- pkg/storage/storage_test.go | 9 + test/common.go | 110 +++++ test/common_test.go | 81 ++++ 19 files changed, 589 insertions(+), 975 deletions(-) create mode 100644 test/common.go create mode 100644 test/common_test.go diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index f2cd55c6..0f30fc65 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -22,27 +22,23 @@ import ( "testing" "time" - "golang.org/x/crypto/bcrypt" - "github.com/anuvu/zot/errors" "github.com/anuvu/zot/pkg/api" "github.com/anuvu/zot/pkg/api/config" "github.com/anuvu/zot/pkg/storage" + . "github.com/anuvu/zot/test" "github.com/chartmuseum/auth" "github.com/mitchellh/mapstructure" + vldap "github.com/nmcclain/ldap" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/phayes/freeport" - "github.com/stretchr/testify/assert" - - vldap "github.com/nmcclain/ldap" . "github.com/smartystreets/goconvey/convey" + "github.com/stretchr/testify/assert" + "golang.org/x/crypto/bcrypt" "gopkg.in/resty.v1" ) const ( - BaseURL = "http://127.0.0.1:%s" - BaseSecureURL = "https://127.0.0.1:%s" username = "test" passphrase = "test" ServerCert = "../../test/data/server.cert" @@ -66,53 +62,6 @@ type ( } ) -func getFreePort() string { - port, err := freeport.GetFreePort() - if err != nil { - panic(err) - } - - return fmt.Sprint(port) -} - -func getBaseURL(port string, secure bool) string { - if secure { - return fmt.Sprintf(BaseSecureURL, port) - } - - return fmt.Sprintf(BaseURL, port) -} - -func makeHtpasswdFile() string { - f, err := ioutil.TempFile("", "htpasswd-") - if err != nil { - panic(err) - } - - // bcrypt(username="test", passwd="test") - content := []byte("test:$2y$05$hlbSXDp6hzDLu6VwACS39ORvVRpr3OMR4RlJ31jtlaOEGnPjKZI1m\n") - if err := ioutil.WriteFile(f.Name(), content, 0600); err != nil { - panic(err) - } - - return f.Name() -} - -func makeHtpasswdFileFromString(fileContent string) string { - f, err := ioutil.TempFile("", "htpasswd-") - if err != nil { - panic(err) - } - - // bcrypt(username="test", passwd="test") - content := []byte(fileContent) - if err := ioutil.WriteFile(f.Name(), content, 0600); err != nil { - panic(err) - } - - return f.Name() -} - func getCredString(username, password string) string { hash, err := bcrypt.GenerateFromPassword([]byte(password), 10) if err != nil { @@ -141,7 +90,7 @@ func TestNew(t *testing.T) { func TestObjectStorageController(t *testing.T) { skipIt(t) Convey("Negative make a new object storage controller", t, func() { - port := getFreePort() + port := GetFreePort() conf := config.New() conf.HTTP.Port = port storageDriverParams := map[string]interface{}{ @@ -159,8 +108,8 @@ func TestObjectStorageController(t *testing.T) { }) Convey("Make a new object storage controller", t, func() { - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -208,8 +157,8 @@ func TestObjectStorageController(t *testing.T) { func TestObjectStorageControllerSubPaths(t *testing.T) { skipIt(t) Convey("Make a new object storage controller", t, func() { - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -260,8 +209,8 @@ func TestObjectStorageControllerSubPaths(t *testing.T) { func TestHtpasswdSingleCred(t *testing.T) { Convey("Single cred", t, func() { - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) singleCredtests := []string{} user := ALICE password := ALICE @@ -273,7 +222,7 @@ func TestHtpasswdSingleCred(t *testing.T) { conf := config.New() conf.HTTP.Port = port - htpasswdPath := makeHtpasswdFileFromString(testString) + htpasswdPath := MakeHtpasswdFileFromString(testString) defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ HTPasswd: config.AuthHTPasswd{ @@ -337,11 +286,11 @@ func TestHtpasswdTwoCreds(t *testing.T) { for _, testString := range twoCredTests { func() { - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := makeHtpasswdFileFromString(testString) + htpasswdPath := MakeHtpasswdFileFromString(testString) defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ HTPasswd: config.AuthHTPasswd{ @@ -406,11 +355,11 @@ func TestHtpasswdFiveCreds(t *testing.T) { } func() { - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := makeHtpasswdFileFromString(credString.String()) + htpasswdPath := MakeHtpasswdFileFromString(credString.String()) defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ HTPasswd: config.AuthHTPasswd{ @@ -459,11 +408,11 @@ func TestHtpasswdFiveCreds(t *testing.T) { } func TestBasicAuth(t *testing.T) { Convey("Make a new controller", t, func() { - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := makeHtpasswdFile() + htpasswdPath := MakeHtpasswdFile() defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ @@ -521,8 +470,8 @@ func TestBasicAuth(t *testing.T) { func TestInterruptedBlobUpload(t *testing.T) { Convey("Successfully cleaning interrupted blob uploads", t, func() { - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -761,11 +710,11 @@ func TestInterruptedBlobUpload(t *testing.T) { func TestMultipleInstance(t *testing.T) { Convey("Negative test zot multiple instance", t, func() { - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := makeHtpasswdFile() + htpasswdPath := MakeHtpasswdFile() defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ @@ -783,7 +732,7 @@ func TestMultipleInstance(t *testing.T) { } defer os.RemoveAll(globalDir) - subDir, err := ioutil.TempDir("/tmp", "oci-sub-test") + subDir, err := ioutil.TempDir("", "oci-sub-test") if err != nil { panic(err) } @@ -823,11 +772,11 @@ func TestMultipleInstance(t *testing.T) { }) Convey("Test zot multiple instance", t, func() { - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := makeHtpasswdFile() + htpasswdPath := MakeHtpasswdFile() defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ @@ -842,7 +791,7 @@ func TestMultipleInstance(t *testing.T) { } defer os.RemoveAll(globalDir) - subDir, err := ioutil.TempDir("/tmp", "oci-sub-test") + subDir, err := ioutil.TempDir("", "oci-sub-test") if err != nil { panic(err) } @@ -899,12 +848,12 @@ func TestTLSWithBasicAuth(t *testing.T) { So(err, ShouldBeNil) caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) - htpasswdPath := makeHtpasswdFile() + htpasswdPath := MakeHtpasswdFile() defer os.Remove(htpasswdPath) - port := getFreePort() - baseURL := getBaseURL(port, false) - secureBaseURL := getBaseURL(port, true) + port := GetFreePort() + baseURL := GetBaseURL(port) + secureBaseURL := GetSecureBaseURL(port) resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() @@ -980,12 +929,12 @@ func TestTLSWithBasicAuthAllowReadAccess(t *testing.T) { So(err, ShouldBeNil) caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) - htpasswdPath := makeHtpasswdFile() + htpasswdPath := MakeHtpasswdFile() defer os.Remove(htpasswdPath) - port := getFreePort() - baseURL := getBaseURL(port, false) - secureBaseURL := getBaseURL(port, true) + port := GetFreePort() + baseURL := GetBaseURL(port) + secureBaseURL := GetSecureBaseURL(port) resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() @@ -1064,9 +1013,9 @@ func TestTLSMutualAuth(t *testing.T) { caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) - port := getFreePort() - baseURL := getBaseURL(port, false) - secureBaseURL := getBaseURL(port, true) + port := GetFreePort() + baseURL := GetBaseURL(port) + secureBaseURL := GetSecureBaseURL(port) resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() @@ -1152,9 +1101,9 @@ func TestTLSMutualAuthAllowReadAccess(t *testing.T) { caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) - port := getFreePort() - baseURL := getBaseURL(port, false) - secureBaseURL := getBaseURL(port, true) + port := GetFreePort() + baseURL := GetBaseURL(port) + secureBaseURL := GetSecureBaseURL(port) resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() @@ -1247,12 +1196,12 @@ func TestTLSMutualAndBasicAuth(t *testing.T) { So(err, ShouldBeNil) caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) - htpasswdPath := makeHtpasswdFile() + htpasswdPath := MakeHtpasswdFile() defer os.Remove(htpasswdPath) - port := getFreePort() - baseURL := getBaseURL(port, false) - secureBaseURL := getBaseURL(port, true) + port := GetFreePort() + baseURL := GetBaseURL(port) + secureBaseURL := GetSecureBaseURL(port) resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() @@ -1345,12 +1294,12 @@ func TestTLSMutualAndBasicAuthAllowReadAccess(t *testing.T) { So(err, ShouldBeNil) caCertPool := x509.NewCertPool() caCertPool.AppendCertsFromPEM(caCert) - htpasswdPath := makeHtpasswdFile() + htpasswdPath := MakeHtpasswdFile() defer os.Remove(htpasswdPath) - port := getFreePort() - baseURL := getBaseURL(port, false) - secureBaseURL := getBaseURL(port, true) + port := GetFreePort() + baseURL := GetBaseURL(port) + secureBaseURL := GetSecureBaseURL(port) resty.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) defer func() { resty.SetTLSClientConfig(nil) }() @@ -1525,8 +1474,8 @@ func TestBasicAuthWithLDAP(t *testing.T) { l.Start() defer l.Stop() - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -1594,8 +1543,8 @@ func TestBearerAuth(t *testing.T) { authTestServer := makeAuthTestServer() defer authTestServer.Close() - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -1776,8 +1725,8 @@ func TestBearerAuthWithAllowReadAccess(t *testing.T) { authTestServer := makeAuthTestServer() defer authTestServer.Close() - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port @@ -2010,12 +1959,12 @@ func parseBearerAuthHeader(authHeaderRaw string) *authHeader { func TestAuthorizationWithBasicAuth(t *testing.T) { Convey("Make a new controller", t, func() { - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := makeHtpasswdFile() + htpasswdPath := MakeHtpasswdFile() defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ @@ -2047,7 +1996,7 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { panic(err) } defer os.RemoveAll(dir) - err = copyFiles("../../test/data", dir) + err = CopyFiles("../../test/data", dir) if err != nil { panic(err) } @@ -2399,12 +2348,12 @@ func TestAuthorizationWithBasicAuth(t *testing.T) { func TestInvalidCases(t *testing.T) { Convey("Invalid repo dir", t, func() { - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase)) + htpasswdPath := MakeHtpasswdFileFromString(getCredString(username, passphrase)) defer os.Remove(htpasswdPath) @@ -2466,8 +2415,8 @@ func TestHTTPReadOnly(t *testing.T) { singleCredtests = append(singleCredtests, getCredString(user, password)) singleCredtests = append(singleCredtests, getCredString(user, password)+"\n") - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) for _, testString := range singleCredtests { func() { @@ -2476,7 +2425,7 @@ func TestHTTPReadOnly(t *testing.T) { // enable read-only mode conf.HTTP.ReadOnly = true - htpasswdPath := makeHtpasswdFileFromString(testString) + htpasswdPath := MakeHtpasswdFileFromString(testString) defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ HTPasswd: config.AuthHTPasswd{ @@ -2533,12 +2482,12 @@ func TestHTTPReadOnly(t *testing.T) { func TestCrossRepoMount(t *testing.T) { Convey("Cross Repo Mount", t, func() { - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase)) + htpasswdPath := MakeHtpasswdFileFromString(getCredString(username, passphrase)) defer os.Remove(htpasswdPath) @@ -2555,7 +2504,7 @@ func TestCrossRepoMount(t *testing.T) { panic(err) } - err = copyFiles("../../test/data", dir) + err = CopyFiles("../../test/data", dir) if err != nil { panic(err) } @@ -2736,12 +2685,12 @@ func TestCrossRepoMount(t *testing.T) { }) Convey("Disable dedupe and cache", t, func() { - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase)) + htpasswdPath := MakeHtpasswdFileFromString(getCredString(username, passphrase)) defer os.Remove(htpasswdPath) @@ -2760,7 +2709,7 @@ func TestCrossRepoMount(t *testing.T) { panic(err) } - err = copyFiles("../../test/data", dir) + err = CopyFiles("../../test/data", dir) if err != nil { panic(err) } @@ -2797,6 +2746,7 @@ func TestCrossRepoMount(t *testing.T) { So(headResponse.StatusCode(), ShouldEqual, 404) }) } + func TestParallelRequests(t *testing.T) { testCases := []struct { srcImageName string @@ -2893,12 +2843,12 @@ func TestParallelRequests(t *testing.T) { }, } - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase)) + htpasswdPath := MakeHtpasswdFileFromString(getCredString(username, passphrase)) conf.HTTP.Auth = &config.AuthConfig{ HTPasswd: config.AuthHTPasswd{ @@ -3141,158 +3091,14 @@ func TestParallelRequests(t *testing.T) { } } -func getAllBlobs(imagePath string) []string { - blobList := make([]string, 0) - - if !dirExists(imagePath) { - return []string{} - } - - buf, err := ioutil.ReadFile(path.Join(imagePath, "index.json")) - - if err != nil { - panic(err) - } - - var index ispec.Index - if err := json.Unmarshal(buf, &index); err != nil { - panic(err) - } - - var digest godigest.Digest - - for _, m := range index.Manifests { - digest = m.Digest - blobList = append(blobList, digest.Encoded()) - p := path.Join(imagePath, "blobs", digest.Algorithm().String(), digest.Encoded()) - - buf, err = ioutil.ReadFile(p) - - if err != nil { - panic(err) - } - - var manifest ispec.Manifest - if err := json.Unmarshal(buf, &manifest); err != nil { - panic(err) - } - - blobList = append(blobList, manifest.Config.Digest.Encoded()) - - for _, layer := range manifest.Layers { - blobList = append(blobList, layer.Digest.Encoded()) - } - } - - return blobList -} - -func getAllManifests(imagePath string) []string { - manifestList := make([]string, 0) - - if !dirExists(imagePath) { - return []string{} - } - - buf, err := ioutil.ReadFile(path.Join(imagePath, "index.json")) - - if err != nil { - panic(err) - } - - var index ispec.Index - if err := json.Unmarshal(buf, &index); err != nil { - panic(err) - } - - var digest godigest.Digest - - for _, m := range index.Manifests { - digest = m.Digest - manifestList = append(manifestList, digest.Encoded()) - } - - return manifestList -} - -func dirExists(d string) bool { - fi, err := os.Stat(d) - if err != nil && os.IsNotExist(err) { - return false - } - - if !fi.IsDir() { - return false - } - - return true -} - -func copyFiles(sourceDir string, destDir string) error { - sourceMeta, err := os.Stat(sourceDir) - if err != nil { - return err - } - - if err := os.MkdirAll(destDir, sourceMeta.Mode()); err != nil { - return err - } - - files, err := ioutil.ReadDir(sourceDir) - if err != nil { - return err - } - - for _, file := range files { - sourceFilePath := path.Join(sourceDir, file.Name()) - destFilePath := path.Join(destDir, file.Name()) - - if file.IsDir() { - if err = copyFiles(sourceFilePath, destFilePath); err != nil { - return err - } - } else { - sourceFile, err := os.Open(sourceFilePath) - if err != nil { - return err - } - defer sourceFile.Close() - - destFile, err := os.Create(destFilePath) - if err != nil { - return err - } - defer destFile.Close() - - if _, err = io.Copy(destFile, sourceFile); err != nil { - return err - } - } - } - - return nil -} - -func stopServer(ctrl *api.Controller) { - err := ctrl.Server.Shutdown(context.Background()) - if err != nil { - panic(err) - } - - err = os.RemoveAll(ctrl.Config.Storage.RootDirectory) - if err != nil { - panic(err) - } -} - func TestHardLink(t *testing.T) { Convey("Validate hard link", t, func() { - port := getFreePort() - baseURL := getBaseURL(port, false) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase)) + htpasswdPath := MakeHtpasswdFileFromString(getCredString(username, passphrase)) conf.HTTP.Auth = &config.AuthConfig{ HTPasswd: config.AuthHTPasswd{ @@ -3363,3 +3169,89 @@ func TestHardLink(t *testing.T) { So(c.Config.Storage.Dedupe, ShouldEqual, false) }) } + +func getAllBlobs(imagePath string) []string { + blobList := make([]string, 0) + + if !storage.DirExists(imagePath) { + return []string{} + } + + buf, err := ioutil.ReadFile(path.Join(imagePath, "index.json")) + + if err != nil { + panic(err) + } + + var index ispec.Index + if err := json.Unmarshal(buf, &index); err != nil { + panic(err) + } + + var digest godigest.Digest + + for _, m := range index.Manifests { + digest = m.Digest + blobList = append(blobList, digest.Encoded()) + p := path.Join(imagePath, "blobs", digest.Algorithm().String(), digest.Encoded()) + + buf, err = ioutil.ReadFile(p) + + if err != nil { + panic(err) + } + + var manifest ispec.Manifest + if err := json.Unmarshal(buf, &manifest); err != nil { + panic(err) + } + + blobList = append(blobList, manifest.Config.Digest.Encoded()) + + for _, layer := range manifest.Layers { + blobList = append(blobList, layer.Digest.Encoded()) + } + } + + return blobList +} + +func getAllManifests(imagePath string) []string { + manifestList := make([]string, 0) + + if !storage.DirExists(imagePath) { + return []string{} + } + + buf, err := ioutil.ReadFile(path.Join(imagePath, "index.json")) + + if err != nil { + panic(err) + } + + var index ispec.Index + if err := json.Unmarshal(buf, &index); err != nil { + panic(err) + } + + var digest godigest.Digest + + for _, m := range index.Manifests { + digest = m.Digest + manifestList = append(manifestList, digest.Encoded()) + } + + return manifestList +} + +func stopServer(ctrl *api.Controller) { + err := ctrl.Server.Shutdown(context.Background()) + if err != nil { + panic(err) + } + + err = os.RemoveAll(ctrl.Config.Storage.RootDirectory) + if err != nil { + panic(err) + } +} diff --git a/pkg/cli/client.go b/pkg/cli/client.go index da3af454..730d7ec9 100644 --- a/pkg/cli/client.go +++ b/pkg/cli/client.go @@ -19,6 +19,7 @@ import ( "time" zotErrors "github.com/anuvu/zot/errors" + "github.com/anuvu/zot/pkg/storage" ) var httpClientsMap = make(map[string]*http.Client) //nolint: gochecknoglobals @@ -141,7 +142,7 @@ func loadPerHostCerts(caCertPool *x509.CertPool, host string) *tls.Config { home := os.Getenv("HOME") clientCertsDir := filepath.Join(home, homeCertsDir, host) - if dirExists(clientCertsDir) { + if storage.DirExists(clientCertsDir) { tlsConfig, err := getTLSConfig(clientCertsDir, caCertPool) if err == nil { @@ -151,7 +152,7 @@ func loadPerHostCerts(caCertPool *x509.CertPool, host string) *tls.Config { // Check if the /etc/containers/certs.d/$IP:$PORT dir exists clientCertsDir = filepath.Join(certsPath, host) - if dirExists(clientCertsDir) { + if storage.DirExists(clientCertsDir) { tlsConfig, err := getTLSConfig(clientCertsDir, caCertPool) if err == nil { @@ -185,19 +186,6 @@ func getTLSConfig(certsPath string, caCertPool *x509.CertPool) (*tls.Config, err }, nil } -func dirExists(d string) bool { - fi, err := os.Stat(d) - if err != nil && os.IsNotExist(err) { - return false - } - - if !fi.IsDir() { - return false - } - - return true -} - func isURL(str string) bool { u, err := url.Parse(str) return err == nil && u.Scheme != "" && u.Host != "" diff --git a/pkg/cli/client_test.go b/pkg/cli/client_test.go index a5fbd822..5e924ef5 100644 --- a/pkg/cli/client_test.go +++ b/pkg/cli/client_test.go @@ -8,20 +8,17 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "io" "io/ioutil" "os" - "path" "path/filepath" - - "gopkg.in/resty.v1" - "testing" "time" "github.com/anuvu/zot/pkg/api" "github.com/anuvu/zot/pkg/api/config" + . "github.com/anuvu/zot/test" . "github.com/smartystreets/goconvey/convey" + "gopkg.in/resty.v1" ) const ( @@ -44,21 +41,6 @@ const ( certsDir1 = "/.config/containers/certs.d/127.0.0.1:8088/" ) -func makeHtpasswdFile() string { - f, err := ioutil.TempFile("", "htpasswd-") - if err != nil { - panic(err) - } - - // bcrypt(username="test", passwd="test") - content := []byte("test:$2y$05$hlbSXDp6hzDLu6VwACS39ORvVRpr3OMR4RlJ31jtlaOEGnPjKZI1m\n") - if err := ioutil.WriteFile(f.Name(), content, 0600); err != nil { - panic(err) - } - - return f.Name() -} - func TestTLSWithAuth(t *testing.T) { Convey("Make a new controller", t, func() { caCert, err := ioutil.ReadFile(CACert) @@ -70,7 +52,7 @@ func TestTLSWithAuth(t *testing.T) { defer func() { resty.SetTLSClientConfig(nil) }() conf := config.New() conf.HTTP.Port = SecurePort1 - htpasswdPath := makeHtpasswdFile() + htpasswdPath := MakeHtpasswdFile() defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ @@ -119,7 +101,7 @@ func TestTLSWithAuth(t *testing.T) { home := os.Getenv("HOME") destCertsDir := filepath.Join(home, certsDir1) - if err = copyFiles(sourceCertsDir, destCertsDir); err != nil { + if err = CopyFiles(sourceCertsDir, destCertsDir); err != nil { panic(err) } defer os.RemoveAll(destCertsDir) @@ -218,7 +200,7 @@ func TestTLSWithoutAuth(t *testing.T) { home := os.Getenv("HOME") destCertsDir := filepath.Join(home, certsDir1) - if err = copyFiles(sourceCertsDir, destCertsDir); err != nil { + if err = CopyFiles(sourceCertsDir, destCertsDir); err != nil { panic(err) } defer os.RemoveAll(destCertsDir) @@ -359,48 +341,3 @@ func TestTLSBadCerts(t *testing.T) { }) }) } - -func copyFiles(sourceDir string, destDir string) error { - sourceMeta, err := os.Stat(sourceDir) - if err != nil { - return err - } - - if err := os.MkdirAll(destDir, sourceMeta.Mode()); err != nil { - return err - } - - files, err := ioutil.ReadDir(sourceDir) - if err != nil { - return err - } - - for _, file := range files { - sourceFilePath := path.Join(sourceDir, file.Name()) - destFilePath := path.Join(destDir, file.Name()) - - if file.IsDir() { - if err = copyFiles(sourceFilePath, destFilePath); err != nil { - return err - } - } else { - sourceFile, err := os.Open(sourceFilePath) - if err != nil { - return err - } - defer sourceFile.Close() - - destFile, err := os.Create(destFilePath) - if err != nil { - return err - } - defer destFile.Close() - - if _, err = io.Copy(destFile, sourceFile); err != nil { - return err - } - } - } - - return nil -} diff --git a/pkg/cli/cve_cmd_test.go b/pkg/cli/cve_cmd_test.go index fc566efb..2ee46a63 100644 --- a/pkg/cli/cve_cmd_test.go +++ b/pkg/cli/cve_cmd_test.go @@ -18,9 +18,9 @@ import ( "github.com/anuvu/zot/pkg/api" "github.com/anuvu/zot/pkg/api/config" extconf "github.com/anuvu/zot/pkg/extensions/config" - "gopkg.in/resty.v1" - + . "github.com/anuvu/zot/test" . "github.com/smartystreets/goconvey/convey" + "gopkg.in/resty.v1" ) func TestSearchCVECmd(t *testing.T) { @@ -285,8 +285,8 @@ func TestSearchCVECmd(t *testing.T) { } func TestServerCVEResponse(t *testing.T) { - port := getFreePort() - url := getBaseURL(port) + port := GetFreePort() + url := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port c := api.NewController(conf) @@ -296,7 +296,7 @@ func TestServerCVEResponse(t *testing.T) { panic(err) } - err = copyFiles("../../test/data/zot-cve-test", path.Join(dir, "zot-cve-test")) + err = CopyFiles("../../test/data/zot-cve-test", path.Join(dir, "zot-cve-test")) if err != nil { panic(err) } diff --git a/pkg/cli/image_cmd_test.go b/pkg/cli/image_cmd_test.go index 063deb45..29894eb3 100644 --- a/pkg/cli/image_cmd_test.go +++ b/pkg/cli/image_cmd_test.go @@ -13,40 +13,20 @@ import ( "regexp" "strings" "sync" - - "gopkg.in/resty.v1" - "testing" "time" zotErrors "github.com/anuvu/zot/errors" "github.com/anuvu/zot/pkg/api" "github.com/anuvu/zot/pkg/api/config" - "github.com/anuvu/zot/pkg/compliance/v1_0_0" extconf "github.com/anuvu/zot/pkg/extensions/config" + . "github.com/anuvu/zot/test" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/phayes/freeport" . "github.com/smartystreets/goconvey/convey" + "gopkg.in/resty.v1" ) -const ( - BaseURL = "http://127.0.0.1:%s" -) - -func getBaseURL(port string) string { - return fmt.Sprintf(BaseURL, port) -} - -func getFreePort() string { - port, err := freeport.GetFreePort() - if err != nil { - panic(err) - } - - return fmt.Sprint(port) -} - func TestSearchImageCmd(t *testing.T) { Convey("Test image help", t, func() { args := []string{"--help"} @@ -301,8 +281,8 @@ func TestOutputFormat(t *testing.T) { func TestServerResponse(t *testing.T) { Convey("Test from real server", t, func() { - port := getFreePort() - url := getBaseURL(port) + port := GetFreePort() + url := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port conf.Extensions = &extconf.ExtensionConfig{ @@ -481,7 +461,7 @@ func TestServerResponse(t *testing.T) { func uploadManifest(url string) { // create a blob/layer resp, _ := resty.R().Post(url + "/v2/repo7/blobs/uploads/") - loc := v1_0_0.Location(url, resp) + loc := Location(url, resp) content := []byte("this is a blob5") digest := godigest.FromBytes(content) diff --git a/pkg/compliance/v1_0_0/check.go b/pkg/compliance/v1_0_0/check.go index 9124d1e8..23275034 100644 --- a/pkg/compliance/v1_0_0/check.go +++ b/pkg/compliance/v1_0_0/check.go @@ -13,6 +13,7 @@ import ( "github.com/anuvu/zot/pkg/api" "github.com/anuvu/zot/pkg/compliance" + . "github.com/anuvu/zot/test" // nolint:golint,stylecheck godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" // nolint:golint,stylecheck @@ -20,20 +21,6 @@ import ( "gopkg.in/resty.v1" ) -func Location(baseURL string, resp *resty.Response) string { - // For some API responses, the Location header is set and is supposed to - // indicate an opaque value. However, it is not clear if this value is an - // absolute URL (https://server:port/v2/...) or just a path (/v2/...) - // zot implements the latter as per the spec, but some registries appear to - // return the former - this needs to be clarified - loc := resp.Header().Get("Location") - if loc[0] == '/' { - return baseURL + loc - } - - return loc -} - func CheckWorkflows(t *testing.T, config *compliance.Config) { if config == nil || config.Address == "" || config.Port == "" { panic("insufficient config") diff --git a/pkg/compliance/v1_0_0/check_test.go b/pkg/compliance/v1_0_0/check_test.go index 74d59df7..b2a6b283 100644 --- a/pkg/compliance/v1_0_0/check_test.go +++ b/pkg/compliance/v1_0_0/check_test.go @@ -2,8 +2,8 @@ package v1_0_0_test import ( "context" - "fmt" "io/ioutil" + "net/http" "os" "testing" "time" @@ -12,7 +12,7 @@ import ( "github.com/anuvu/zot/pkg/api/config" "github.com/anuvu/zot/pkg/compliance" "github.com/anuvu/zot/pkg/compliance/v1_0_0" - "github.com/phayes/freeport" + . "github.com/anuvu/zot/test" "gopkg.in/resty.v1" ) @@ -53,17 +53,11 @@ func TestWorkflowsOutputJSON(t *testing.T) { // start local server on random open port. func startServer() (*api.Controller, string) { - portInt, err := freeport.GetFreePort() - if err != nil { - panic(err) - } - - randomPort := fmt.Sprintf("%d", portInt) - fmt.Println(randomPort) - + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Address = listenAddress - conf.HTTP.Port = randomPort + conf.HTTP.Port = port ctrl := api.NewController(conf) dir, err := ioutil.TempDir("", "oci-repo-test") @@ -103,19 +97,17 @@ func startServer() (*api.Controller, string) { } }() - baseURL := fmt.Sprintf("http://%s:%s", listenAddress, randomPort) - for { // poll until ready resp, _ := resty.R().Get(baseURL) - if resp.StatusCode() == 404 { + if resp.StatusCode() == http.StatusNotFound { break } time.Sleep(100 * time.Millisecond) } - return ctrl, randomPort + return ctrl, port } func stopServer(ctrl *api.Controller) { diff --git a/pkg/exporter/api/controller_test.go b/pkg/exporter/api/controller_test.go index cdedff0f..95d519f2 100644 --- a/pkg/exporter/api/controller_test.go +++ b/pkg/exporter/api/controller_test.go @@ -19,8 +19,8 @@ import ( zotcfg "github.com/anuvu/zot/pkg/api/config" "github.com/anuvu/zot/pkg/exporter/api" "github.com/anuvu/zot/pkg/extensions/monitoring" + . "github.com/anuvu/zot/test" jsoniter "github.com/json-iterator/go" - "github.com/phayes/freeport" "github.com/prometheus/client_golang/prometheus" dto "github.com/prometheus/client_model/go" . "github.com/smartystreets/goconvey/convey" @@ -28,7 +28,6 @@ import ( ) const ( - BaseURL = "http://127.0.0.1:%s" SleepTime = 50 * time.Millisecond SecondToNanoseconds = 1000000000 ) @@ -42,15 +41,6 @@ func getRandomLatency() time.Duration { return getRandomLatencyN(120 * SecondToNanoseconds) // a random latency (in nanoseconds) that can be up to 2 minutes } -func getFreePort() string { - port, err := freeport.GetFreePort() - if err != nil { - panic(err) - } - - return fmt.Sprint(port) -} - func TestNew(t *testing.T) { Convey("Make a new controller", t, func() { config := api.DefaultConfig() @@ -91,8 +81,8 @@ func TestNewExporter(t *testing.T) { Convey("Make an exporter controller", t, func() { exporterConfig := api.DefaultConfig() So(exporterConfig, ShouldNotBeNil) - exporterPort := getFreePort() - serverPort := getFreePort() + exporterPort := GetFreePort() + serverPort := GetFreePort() exporterConfig.Exporter.Port = exporterPort dir, _ := ioutil.TempDir("", "metrics") exporterConfig.Exporter.Metrics.Path = strings.TrimPrefix(dir, "/tmp/") diff --git a/pkg/extensions/search/common/common_test.go b/pkg/extensions/search/common/common_test.go index 78d04289..e4bbf2ef 100644 --- a/pkg/extensions/search/common/common_test.go +++ b/pkg/extensions/search/common/common_test.go @@ -5,7 +5,6 @@ package common_test import ( "context" "encoding/json" - "io" "io/ioutil" "os" "path" @@ -19,6 +18,7 @@ import ( "github.com/anuvu/zot/pkg/extensions/search/common" "github.com/anuvu/zot/pkg/log" "github.com/anuvu/zot/pkg/storage" + . "github.com/anuvu/zot/test" ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" "gopkg.in/resty.v1" @@ -30,11 +30,6 @@ var ( subRootDir string ) -const ( - BaseURL1 = "http://127.0.0.1:8085" - Port1 = "8085" -) - type ImgResponsWithLatestTag struct { ImgListWithLatestTag ImgListWithLatestTag `json:"data"` Errors []ErrorGQL `json:"errors"` @@ -75,12 +70,12 @@ func testSetup() error { subRootDir = subDir - err = copyFiles("../../../../test/data", rootDir) + err = CopyFiles("../../../../test/data", rootDir) if err != nil { return err } - err = copyFiles("../../../../test/data", subDir) + err = CopyFiles("../../../../test/data", subDir) if err != nil { return err } @@ -112,51 +107,6 @@ func getTags() ([]common.TagInfo, []common.TagInfo) { return tags, infectedTags } -func copyFiles(sourceDir string, destDir string) error { - sourceMeta, err := os.Stat(sourceDir) - if err != nil { - return err - } - - if err := os.MkdirAll(destDir, sourceMeta.Mode()); err != nil { - return err - } - - files, err := ioutil.ReadDir(sourceDir) - if err != nil { - return err - } - - for _, file := range files { - sourceFilePath := path.Join(sourceDir, file.Name()) - destFilePath := path.Join(destDir, file.Name()) - - if file.IsDir() { - if err = copyFiles(sourceFilePath, destFilePath); err != nil { - return err - } - } else { - sourceFile, err := os.Open(sourceFilePath) - if err != nil { - return err - } - defer sourceFile.Close() - - destFile, err := os.Create(destFilePath) - if err != nil { - return err - } - defer destFile.Close() - - if _, err = io.Copy(destFile, sourceFile); err != nil { - return err - } - } - } - - return nil -} - func TestImageFormat(t *testing.T) { Convey("Test valid image", t, func() { log := log.NewLogger("debug", "") @@ -219,8 +169,10 @@ func TestLatestTagSearchHTTP(t *testing.T) { if err != nil { panic(err) } + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() - conf.HTTP.Port = Port1 + conf.HTTP.Port = port conf.Storage.RootDirectory = rootDir conf.Storage.SubPaths = make(map[string]config.StorageConfig) conf.Storage.SubPaths["/a"] = config.StorageConfig{RootDirectory: subRootDir} @@ -241,7 +193,7 @@ func TestLatestTagSearchHTTP(t *testing.T) { // wait till ready for { - _, err := resty.R().Get(BaseURL1) + _, err := resty.R().Get(baseURL) if err == nil { break } @@ -254,17 +206,17 @@ func TestLatestTagSearchHTTP(t *testing.T) { _ = c.Server.Shutdown(ctx) }() - resp, err := resty.R().Get(BaseURL1 + "/v2/") + resp, err := resty.R().Get(baseURL + "/v2/") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) - resp, err = resty.R().Get(BaseURL1 + "/query") + resp, err = resty.R().Get(baseURL + "/query") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) - resp, err = resty.R().Get(BaseURL1 + "/query?query={ImageListWithLatestTag(){Name%20Latest}}") + resp, err = resty.R().Get(baseURL + "/query?query={ImageListWithLatestTag(){Name%20Latest}}") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -277,7 +229,7 @@ func TestLatestTagSearchHTTP(t *testing.T) { images := responseStruct.ImgListWithLatestTag.Images So(images[0].Latest, ShouldEqual, "0.0.1") - resp, err = resty.R().Get(BaseURL1 + "/query?query={ImageListWithLatestTag(){Name%20Latest}}") + resp, err = resty.R().Get(baseURL + "/query?query={ImageListWithLatestTag(){Name%20Latest}}") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) @@ -286,7 +238,7 @@ func TestLatestTagSearchHTTP(t *testing.T) { panic(err) } - resp, err = resty.R().Get(BaseURL1 + "/query?query={ImageListWithLatestTag(){Name%20Latest}}") + resp, err = resty.R().Get(baseURL + "/query?query={ImageListWithLatestTag(){Name%20Latest}}") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -307,7 +259,7 @@ func TestLatestTagSearchHTTP(t *testing.T) { panic(err) } - resp, err = resty.R().Get(BaseURL1 + "/query?query={ImageListWithLatestTag(){Name%20Latest}}") + resp, err = resty.R().Get(baseURL + "/query?query={ImageListWithLatestTag(){Name%20Latest}}") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -318,7 +270,7 @@ func TestLatestTagSearchHTTP(t *testing.T) { panic(err) } - resp, err = resty.R().Get(BaseURL1 + "/query?query={ImageListWithLatestTag(){Name%20Latest}}") + resp, err = resty.R().Get(baseURL + "/query?query={ImageListWithLatestTag(){Name%20Latest}}") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -329,7 +281,7 @@ func TestLatestTagSearchHTTP(t *testing.T) { panic(err) } - resp, err = resty.R().Get(BaseURL1 + "/query?query={ImageListWithLatestTag(){Name%20Latest}}") + resp, err = resty.R().Get(baseURL + "/query?query={ImageListWithLatestTag(){Name%20Latest}}") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -341,7 +293,7 @@ func TestLatestTagSearchHTTP(t *testing.T) { panic(err) } - resp, err = resty.R().Get(BaseURL1 + "/query?query={ImageListWithLatestTag(){Name%20Latest}}") + resp, err = resty.R().Get(baseURL + "/query?query={ImageListWithLatestTag(){Name%20Latest}}") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index d29cbdc0..8f13c221 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -7,7 +7,6 @@ import ( "context" "encoding/json" "fmt" - "io" "io/ioutil" "os" "path" @@ -22,8 +21,8 @@ import ( cveinfo "github.com/anuvu/zot/pkg/extensions/search/cve" "github.com/anuvu/zot/pkg/log" "github.com/anuvu/zot/pkg/storage" + . "github.com/anuvu/zot/test" ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/phayes/freeport" . "github.com/smartystreets/goconvey/convey" "gopkg.in/resty.v1" ) @@ -36,7 +35,6 @@ var ( ) const ( - BaseURL = "http://127.0.0.1:%s" username = "test" passphrase = "test" ) @@ -77,19 +75,6 @@ type CVE struct { Severity string `json:"Severity"` } -func getFreePort() string { - port, err := freeport.GetFreePort() - if err != nil { - panic(err) - } - - return fmt.Sprint(port) -} - -func getBaseURL(port string) string { - return fmt.Sprintf(BaseURL, port) -} - func testSetup() error { dir, err := ioutil.TempDir("", "util_test") if err != nil { @@ -112,7 +97,7 @@ func testSetup() error { return err } - err = copyFiles("../../../../test/data", dbDir) + err = CopyFiles("../../../../test/data", dbDir) if err != nil { return err } @@ -122,12 +107,12 @@ func testSetup() error { func generateTestData() error { // nolint: gocyclo // Image dir with no files - err := os.Mkdir(path.Join(dbDir, "zot-noindex-test"), 0o755) + err := os.Mkdir(path.Join(dbDir, "zot-noindex-test"), 0755) if err != nil { return err } - err = os.Mkdir(path.Join(dbDir, "zot-nonreadable-test"), 0o755) + err = os.Mkdir(path.Join(dbDir, "zot-nonreadable-test"), 0755) if err != nil { return err } @@ -140,12 +125,12 @@ func generateTestData() error { // nolint: gocyclo return err } - if err = ioutil.WriteFile(path.Join(dbDir, "zot-nonreadable-test", "index.json"), buf, 0o111); err != nil { + if err = ioutil.WriteFile(path.Join(dbDir, "zot-nonreadable-test", "index.json"), buf, 0111); err != nil { return err } // Image dir with invalid index.json - err = os.Mkdir(path.Join(dbDir, "zot-squashfs-invalid-index"), 0o755) + err = os.Mkdir(path.Join(dbDir, "zot-squashfs-invalid-index"), 0755) if err != nil { return err } @@ -158,7 +143,7 @@ func generateTestData() error { // nolint: gocyclo } // Image dir with no blobs - err = os.Mkdir(path.Join(dbDir, "zot-squashfs-noblobs"), 0o755) + err = os.Mkdir(path.Join(dbDir, "zot-squashfs-noblobs"), 0755) if err != nil { return err } @@ -172,7 +157,7 @@ func generateTestData() error { // nolint: gocyclo } // Image dir with invalid blob - err = os.MkdirAll(path.Join(dbDir, "zot-squashfs-invalid-blob", "blobs/sha256"), 0o755) + err = os.MkdirAll(path.Join(dbDir, "zot-squashfs-invalid-blob", "blobs/sha256"), 0755) if err != nil { return err } @@ -195,7 +180,7 @@ func generateTestData() error { // nolint: gocyclo // Create a squashfs image - err = os.MkdirAll(path.Join(dbDir, "zot-squashfs-test", "blobs/sha256"), 0o755) + err = os.MkdirAll(path.Join(dbDir, "zot-squashfs-test", "blobs/sha256"), 0755) if err != nil { return err } @@ -207,11 +192,11 @@ func generateTestData() error { // nolint: gocyclo return err } - if err = ioutil.WriteFile(path.Join(dbDir, "zot-squashfs-test", "oci-layout"), buf, 0o644); err != nil { //nolint: gosec + if err = ioutil.WriteFile(path.Join(dbDir, "zot-squashfs-test", "oci-layout"), buf, 0644); err != nil { //nolint: gosec return err } - err = os.Mkdir(path.Join(dbDir, "zot-squashfs-test", ".uploads"), 0o755) + err = os.Mkdir(path.Join(dbDir, "zot-squashfs-test", ".uploads"), 0755) if err != nil { return err } @@ -267,7 +252,7 @@ func generateTestData() error { // nolint: gocyclo // Create a image with invalid layer blob - err = os.MkdirAll(path.Join(dbDir, "zot-invalid-layer", "blobs/sha256"), 0o755) + err = os.MkdirAll(path.Join(dbDir, "zot-invalid-layer", "blobs/sha256"), 0755) if err != nil { return err } @@ -295,7 +280,7 @@ func generateTestData() error { // nolint: gocyclo // Create a image with no layer blob - err = os.MkdirAll(path.Join(dbDir, "zot-no-layer", "blobs/sha256"), 0o755) + err = os.MkdirAll(path.Join(dbDir, "zot-no-layer", "blobs/sha256"), 0755) if err != nil { return err } @@ -325,73 +310,13 @@ func generateTestData() error { // nolint: gocyclo } func makeTestFile(fileName string, content string) error { - if err := ioutil.WriteFile(fileName, []byte(content), 0o600); err != nil { + if err := ioutil.WriteFile(fileName, []byte(content), 0600); err != nil { panic(err) } return nil } -func copyFiles(sourceDir string, destDir string) error { - sourceMeta, err := os.Stat(sourceDir) - if err != nil { - return err - } - - if err := os.MkdirAll(destDir, sourceMeta.Mode()); err != nil { - return err - } - - files, err := ioutil.ReadDir(sourceDir) - if err != nil { - return err - } - - for _, file := range files { - sourceFilePath := path.Join(sourceDir, file.Name()) - destFilePath := path.Join(destDir, file.Name()) - - if file.IsDir() { - if err = copyFiles(sourceFilePath, destFilePath); err != nil { - return err - } - } else { - sourceFile, err := os.Open(sourceFilePath) - if err != nil { - return err - } - defer sourceFile.Close() - - destFile, err := os.Create(destFilePath) - if err != nil { - return err - } - defer destFile.Close() - - if _, err = io.Copy(destFile, sourceFile); err != nil { - return err - } - } - } - - return nil -} - -func makeHtpasswdFile() string { - f, err := ioutil.TempFile("", "htpasswd-") - if err != nil { - panic(err) - } - - // bcrypt(username="test", passwd="test") - content := []byte("test:$2y$05$hlbSXDp6hzDLu6VwACS39ORvVRpr3OMR4RlJ31jtlaOEGnPjKZI1m\n") - if err := ioutil.WriteFile(f.Name(), content, 0o600); err != nil { - panic(err) - } - - return f.Name() -} - func TestMultipleStoragePath(t *testing.T) { Convey("Test multiple storage path", t, func() { // Create temporary directory @@ -452,11 +377,11 @@ func TestDownloadDB(t *testing.T) { func TestCVESearch(t *testing.T) { Convey("Test image vulenrability scanning", t, func() { updateDuration, _ = time.ParseDuration("1h") - port := getFreePort() - baseURL := getBaseURL(port) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() conf.HTTP.Port = port - htpasswdPath := makeHtpasswdFile() + htpasswdPath := MakeHtpasswdFile() defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ @@ -680,10 +605,10 @@ func TestCVESearch(t *testing.T) { func TestCVEConfig(t *testing.T) { Convey("Verify CVE config", t, func() { conf := config.New() - port := getFreePort() + port := GetFreePort() conf.HTTP.Port = port - baseURL := getBaseURL(port) - htpasswdPath := makeHtpasswdFile() + baseURL := GetBaseURL(port) + htpasswdPath := MakeHtpasswdFile() defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ @@ -704,7 +629,7 @@ func TestCVEConfig(t *testing.T) { defer os.RemoveAll(firstDir) defer os.RemoveAll(secondDir) - err = copyFiles("../../../../test/data", path.Join(secondDir, "a")) + err = CopyFiles("../../../../test/data", path.Join(secondDir, "a")) if err != nil { panic(err) } diff --git a/pkg/extensions/search/digest/digest_test.go b/pkg/extensions/search/digest/digest_test.go index e5ea22c8..7543faf0 100644 --- a/pkg/extensions/search/digest/digest_test.go +++ b/pkg/extensions/search/digest/digest_test.go @@ -6,10 +6,8 @@ package digestinfo_test import ( "context" "encoding/json" - "io" "io/ioutil" "os" - "path" "testing" "time" @@ -20,6 +18,7 @@ import ( digestinfo "github.com/anuvu/zot/pkg/extensions/search/digest" "github.com/anuvu/zot/pkg/log" "github.com/anuvu/zot/pkg/storage" + . "github.com/anuvu/zot/test" . "github.com/smartystreets/goconvey/convey" "gopkg.in/resty.v1" ) @@ -31,11 +30,6 @@ var ( subRootDir string ) -const ( - BaseURL1 = "http://127.0.0.1:8085" - Port1 = "8085" -) - type ImgResponseForDigest struct { ImgListForDigest ImgListForDigest `json:"data"` Errors []ErrorGQL `json:"errors"` @@ -89,12 +83,12 @@ func testSetup() error { return err } - err = copyFiles("../../../../test/data", rootDir) + err = CopyFiles("../../../../test/data", rootDir) if err != nil { return err } - err = copyFiles("../../../../test/data", subDir+"/a/") + err = CopyFiles("../../../../test/data", subDir+"/a/") if err != nil { return err } @@ -108,51 +102,6 @@ func testSetup() error { return nil } -func copyFiles(sourceDir string, destDir string) error { - sourceMeta, err := os.Stat(sourceDir) - if err != nil { - return err - } - - if err := os.MkdirAll(destDir, sourceMeta.Mode()); err != nil { - return err - } - - files, err := ioutil.ReadDir(sourceDir) - if err != nil { - return err - } - - for _, file := range files { - sourceFilePath := path.Join(sourceDir, file.Name()) - destFilePath := path.Join(destDir, file.Name()) - - if file.IsDir() { - if err = copyFiles(sourceFilePath, destFilePath); err != nil { - return err - } - } else { - sourceFile, err := os.Open(sourceFilePath) - if err != nil { - return err - } - defer sourceFile.Close() - - destFile, err := os.Create(destFilePath) - if err != nil { - return err - } - defer destFile.Close() - - if _, err = io.Copy(destFile, sourceFile); err != nil { - return err - } - } - } - - return nil -} - func TestDigestInfo(t *testing.T) { Convey("Test image tag", t, func() { // Search by manifest digest @@ -187,8 +136,10 @@ func TestDigestInfo(t *testing.T) { func TestDigestSearchHTTP(t *testing.T) { Convey("Test image search by digest scanning", t, func() { + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() - conf.HTTP.Port = Port1 + conf.HTTP.Port = port conf.Storage.RootDirectory = rootDir conf.Extensions = &extconf.ExtensionConfig{ Search: &extconf.SearchConfig{Enable: true}, @@ -205,7 +156,7 @@ func TestDigestSearchHTTP(t *testing.T) { // wait till ready for { - _, err := resty.R().Get(BaseURL1) + _, err := resty.R().Get(baseURL) if err == nil { break } @@ -218,18 +169,18 @@ func TestDigestSearchHTTP(t *testing.T) { _ = c.Server.Shutdown(ctx) }() - resp, err := resty.R().Get(BaseURL1 + "/v2/") + resp, err := resty.R().Get(baseURL + "/v2/") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) - resp, err = resty.R().Get(BaseURL1 + "/query") + resp, err = resty.R().Get(baseURL + "/query") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) // "sha" should match all digests in all images - resp, err = resty.R().Get(BaseURL1 + "/query?query={ImageListForDigest(id:\"sha\"){Name%20Tags}}") + resp, err = resty.R().Get(baseURL + "/query?query={ImageListForDigest(id:\"sha\"){Name%20Tags}}") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -244,7 +195,7 @@ func TestDigestSearchHTTP(t *testing.T) { // Call should return {"data":{"ImageListForDigest":[{"Name":"zot-test","Tags":["0.0.1"]}]}} // "2bacca16" should match the manifest of 1 image - resp, err = resty.R().Get(BaseURL1 + "/query?query={ImageListForDigest(id:\"2bacca16\"){Name%20Tags}}") + resp, err = resty.R().Get(baseURL + "/query?query={ImageListForDigest(id:\"2bacca16\"){Name%20Tags}}") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -259,7 +210,7 @@ func TestDigestSearchHTTP(t *testing.T) { // Call should return {"data":{"ImageListForDigest":[{"Name":"zot-test","Tags":["0.0.1"]}]}} // "adf3bb6c" should match the config of 1 image - resp, err = resty.R().Get(BaseURL1 + "/query?query={ImageListForDigest(id:\"adf3bb6c\"){Name%20Tags}}") + resp, err = resty.R().Get(baseURL + "/query?query={ImageListForDigest(id:\"adf3bb6c\"){Name%20Tags}}") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -274,7 +225,7 @@ func TestDigestSearchHTTP(t *testing.T) { // Call should return {"data":{"ImageListForDigest":[{"Name":"zot-cve-test","Tags":["0.0.1"]}]}} // "7a0437f0" should match the layer of 1 image - resp, err = resty.R().Get(BaseURL1 + "/query?query={ImageListForDigest(id:\"7a0437f0\"){Name%20Tags}}") + resp, err = resty.R().Get(baseURL + "/query?query={ImageListForDigest(id:\"7a0437f0\"){Name%20Tags}}") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -289,7 +240,7 @@ func TestDigestSearchHTTP(t *testing.T) { // Call should return {"data":{"ImageListForDigest":[]}} // "1111111" should match 0 images - resp, err = resty.R().Get(BaseURL1 + "/query?query={ImageListForDigest(id:\"1111111\"){Name%20Tags}}") + resp, err = resty.R().Get(baseURL + "/query?query={ImageListForDigest(id:\"1111111\"){Name%20Tags}}") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -300,7 +251,7 @@ func TestDigestSearchHTTP(t *testing.T) { So(len(responseStruct.ImgListForDigest.Images), ShouldEqual, 0) // Call should return {"errors": [{....}]", data":null}} - resp, err = resty.R().Get(BaseURL1 + "/query?query={ImageListForDigest(id:\"1111111\"){Name%20Tag343s}}") + resp, err = resty.R().Get(baseURL + "/query?query={ImageListForDigest(id:\"1111111\"){Name%20Tag343s}}") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 422) @@ -313,8 +264,10 @@ func TestDigestSearchHTTP(t *testing.T) { func TestDigestSearchHTTPSubPaths(t *testing.T) { Convey("Test image search by digest scanning using storage subpaths", t, func() { + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() - conf.HTTP.Port = Port1 + conf.HTTP.Port = port conf.Extensions = &extconf.ExtensionConfig{ Search: &extconf.SearchConfig{Enable: true}, } @@ -344,7 +297,7 @@ func TestDigestSearchHTTPSubPaths(t *testing.T) { // wait till ready for { - _, err := resty.R().Get(BaseURL1) + _, err := resty.R().Get(baseURL) if err == nil { break } @@ -357,17 +310,17 @@ func TestDigestSearchHTTPSubPaths(t *testing.T) { _ = c.Server.Shutdown(ctx) }() - resp, err := resty.R().Get(BaseURL1 + "/v2/") + resp, err := resty.R().Get(baseURL + "/v2/") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) - resp, err = resty.R().Get(BaseURL1 + "/query") + resp, err = resty.R().Get(baseURL + "/query") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) - resp, err = resty.R().Get(BaseURL1 + "/query?query={ImageListForDigest(id:\"sha\"){Name%20Tags}}") + resp, err = resty.R().Get(baseURL + "/query?query={ImageListForDigest(id:\"sha\"){Name%20Tags}}") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -384,8 +337,10 @@ func TestDigestSearchDisabled(t *testing.T) { Convey("Test disabling image search", t, func() { dir, err := ioutil.TempDir("", "digest_test") So(err, ShouldBeNil) + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() - conf.HTTP.Port = Port1 + conf.HTTP.Port = port conf.Storage.RootDirectory = dir conf.Extensions = &extconf.ExtensionConfig{ Search: &extconf.SearchConfig{Enable: false}, @@ -402,7 +357,7 @@ func TestDigestSearchDisabled(t *testing.T) { // wait till ready for { - _, err := resty.R().Get(BaseURL1) + _, err := resty.R().Get(baseURL) if err == nil { break } @@ -415,12 +370,12 @@ func TestDigestSearchDisabled(t *testing.T) { _ = c.Server.Shutdown(ctx) }() - resp, err := resty.R().Get(BaseURL1 + "/v2/") + resp, err := resty.R().Get(baseURL + "/v2/") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) - resp, err = resty.R().Get(BaseURL1 + "/query") + resp, err = resty.R().Get(baseURL + "/query") So(resp, ShouldNotBeNil) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 404) diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index 925fa38f..88fb7317 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "fmt" - "io" "io/ioutil" "os" "path" @@ -15,6 +14,7 @@ import ( "github.com/anuvu/zot/pkg/extensions/monitoring" "github.com/anuvu/zot/pkg/log" "github.com/anuvu/zot/pkg/storage" + . "github.com/anuvu/zot/test" "github.com/containers/image/v5/docker" "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/types" @@ -25,62 +25,12 @@ import ( ) const ( - BaseURL = "http://127.0.0.1:5001" - ServerCert = "../../../test/data/server.cert" - ServerKey = "../../../test/data/server.key" - CACert = "../../../test/data/ca.crt" - testImage = "zot-test" testImageTag = "0.0.1" host = "127.0.0.1:45117" ) -func copyFiles(sourceDir string, destDir string) error { - sourceMeta, err := os.Stat(sourceDir) - if err != nil { - return err - } - - if err := os.MkdirAll(destDir, sourceMeta.Mode()); err != nil { - return err - } - - files, err := ioutil.ReadDir(sourceDir) - if err != nil { - return err - } - - for _, file := range files { - sourceFilePath := path.Join(sourceDir, file.Name()) - destFilePath := path.Join(destDir, file.Name()) - - if file.IsDir() { - if err = copyFiles(sourceFilePath, destFilePath); err != nil { - return err - } - } else { - sourceFile, err := os.Open(sourceFilePath) - if err != nil { - return err - } - defer sourceFile.Close() - - destFile, err := os.Create(destFilePath) - if err != nil { - return err - } - defer destFile.Close() - - if _, err = io.Copy(destFile, sourceFile); err != nil { - return err - } - } - } - - return nil -} - func TestSyncInternal(t *testing.T) { Convey("Verify parseRepositoryReference func", t, func() { repositoryReference := fmt.Sprintf("%s/%s", host, testImage) @@ -135,13 +85,15 @@ func TestSyncInternal(t *testing.T) { var tlsVerify bool updateDuration := time.Microsecond + port := GetFreePort() + baseURL := GetBaseURL(port) syncRegistryConfig := RegistryConfig{ Content: []Content{ { Prefix: testImage, }, }, - URL: BaseURL, + URL: baseURL, PollInterval: updateDuration, TLSVerify: &tlsVerify, CertDir: "", @@ -243,7 +195,7 @@ func TestSyncInternal(t *testing.T) { panic(err) } - err = copyFiles("../../../test/data", testRootDir) + err = CopyFiles("../../../test/data", testRootDir) if err != nil { panic(err) } diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 983df1b1..af8d5df7 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -22,20 +22,18 @@ import ( "github.com/anuvu/zot/pkg/api/config" extconf "github.com/anuvu/zot/pkg/extensions/config" "github.com/anuvu/zot/pkg/extensions/sync" + . "github.com/anuvu/zot/test" ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/phayes/freeport" . "github.com/smartystreets/goconvey/convey" "gopkg.in/resty.v1" ) const ( - BaseURL = "http://127.0.0.1:%s" - BaseSecureURL = "https://127.0.0.1:%s" - ServerCert = "../../../test/data/server.cert" - ServerKey = "../../../test/data/server.key" - CACert = "../../../test/data/ca.crt" - ClientCert = "../../../test/data/client.cert" - ClientKey = "../../../test/data/client.key" + ServerCert = "../../../test/data/server.cert" + ServerKey = "../../../test/data/server.key" + CACert = "../../../test/data/ca.crt" + ClientCert = "../../../test/data/client.cert" + ClientKey = "../../../test/data/client.key" testImage = "zot-test" testImageTag = "0.0.1" @@ -53,23 +51,6 @@ type catalog struct { Repositories []string `json:"repositories"` } -func getFreePort() string { - port, err := freeport.GetFreePort() - if err != nil { - panic(err) - } - - return fmt.Sprint(port) -} - -func getBaseURL(port string, secure bool) string { - if secure { - return fmt.Sprintf(BaseSecureURL, port) - } - - return fmt.Sprintf(BaseURL, port) -} - func copyFile(sourceFilePath, destFilePath string) error { destFile, err := os.Create(destFilePath) if err != nil { @@ -90,72 +71,12 @@ func copyFile(sourceFilePath, destFilePath string) error { return nil } -func copyFiles(sourceDir string, destDir string) error { - sourceMeta, err := os.Stat(sourceDir) - if err != nil { - return err - } - - if err := os.MkdirAll(destDir, sourceMeta.Mode()); err != nil { - return err - } - - files, err := ioutil.ReadDir(sourceDir) - if err != nil { - return err - } - - for _, file := range files { - sourceFilePath := path.Join(sourceDir, file.Name()) - destFilePath := path.Join(destDir, file.Name()) - - if file.IsDir() { - if err = copyFiles(sourceFilePath, destFilePath); err != nil { - return err - } - } else { - sourceFile, err := os.Open(sourceFilePath) - if err != nil { - return err - } - defer sourceFile.Close() - - destFile, err := os.Create(destFilePath) - if err != nil { - return err - } - defer destFile.Close() - - if _, err = io.Copy(destFile, sourceFile); err != nil { - return err - } - } - } - - return nil -} - -func makeHtpasswdFile() string { - f, err := ioutil.TempFile("", "htpasswd-") - if err != nil { - panic(err) - } - - // bcrypt(username="test", passwd="test") - content := []byte("test:$2y$05$hlbSXDp6hzDLu6VwACS39ORvVRpr3OMR4RlJ31jtlaOEGnPjKZI1m\n") - if err := ioutil.WriteFile(f.Name(), content, 0600); err != nil { - panic(err) - } - - return f.Name() -} - func TestSyncOnDemand(t *testing.T) { Convey("Verify sync on demand feature", t, func() { updateDuration, _ := time.ParseDuration("30m") - srcPort := getFreePort() - srcBaseURL := getBaseURL(srcPort, false) + srcPort := GetFreePort() + srcBaseURL := GetBaseURL(srcPort) srcConfig := config.New() srcConfig.HTTP.Port = srcPort @@ -167,7 +88,7 @@ func TestSyncOnDemand(t *testing.T) { defer os.RemoveAll(srcDir) - err = copyFiles("../../../test/data", srcDir) + err = CopyFiles("../../../test/data", srcDir) if err != nil { panic(err) } @@ -198,8 +119,8 @@ func TestSyncOnDemand(t *testing.T) { time.Sleep(100 * time.Millisecond) } - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, false) + destPort := GetFreePort() + destBaseURL := GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -357,8 +278,8 @@ func TestSync(t *testing.T) { Convey("Verify sync feature", t, func() { updateDuration, _ := time.ParseDuration("30m") - srcPort := getFreePort() - srcBaseURL := getBaseURL(srcPort, false) + srcPort := GetFreePort() + srcBaseURL := GetBaseURL(srcPort) srcConfig := config.New() srcConfig.HTTP.Port = srcPort @@ -370,7 +291,7 @@ func TestSync(t *testing.T) { defer os.RemoveAll(srcDir) - err = copyFiles("../../../test/data", srcDir) + err = CopyFiles("../../../test/data", srcDir) if err != nil { panic(err) } @@ -401,8 +322,8 @@ func TestSync(t *testing.T) { time.Sleep(100 * time.Millisecond) } - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, false) + destPort := GetFreePort() + destBaseURL := GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -505,8 +426,8 @@ func TestSync(t *testing.T) { }) Convey("Test sync with more contents", func() { - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, false) + destPort := GetFreePort() + destBaseURL := GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -625,8 +546,8 @@ func TestSyncPermsDenied(t *testing.T) { Convey("Verify sync feature without perm on sync cache", t, func() { updateDuration, _ := time.ParseDuration("30m") - srcPort := getFreePort() - srcBaseURL := getBaseURL(srcPort, false) + srcPort := GetFreePort() + srcBaseURL := GetBaseURL(srcPort) srcConfig := config.New() srcConfig.HTTP.Port = srcPort @@ -638,7 +559,7 @@ func TestSyncPermsDenied(t *testing.T) { defer os.RemoveAll(srcDir) - err = copyFiles("../../../test/data", srcDir) + err = CopyFiles("../../../test/data", srcDir) if err != nil { panic(err) } @@ -669,8 +590,8 @@ func TestSyncPermsDenied(t *testing.T) { time.Sleep(100 * time.Millisecond) } - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, false) + destPort := GetFreePort() + destBaseURL := GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -759,8 +680,8 @@ func TestSyncBadTLS(t *testing.T) { updateDuration, _ := time.ParseDuration("1h") - srcPort := getFreePort() - srcBaseURL := getBaseURL(srcPort, true) + srcPort := GetFreePort() + srcBaseURL := GetSecureBaseURL(srcPort) srcConfig := config.New() srcConfig.HTTP.Port = srcPort @@ -814,15 +735,15 @@ func TestSyncBadTLS(t *testing.T) { panic(err) } - err = copyFiles("../../../test/data", destDir) + err = CopyFiles("../../../test/data", destDir) if err != nil { panic(err) } defer os.RemoveAll(destDir) - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, true) + destPort := GetFreePort() + destBaseURL := GetSecureBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -906,8 +827,8 @@ func TestSyncTLS(t *testing.T) { updateDuration, _ := time.ParseDuration("1h") - srcPort := getFreePort() - srcBaseURL := getBaseURL(srcPort, true) + srcPort := GetFreePort() + srcBaseURL := GetSecureBaseURL(srcPort) srcConfig := config.New() srcConfig.HTTP.Port = srcPort @@ -925,7 +846,7 @@ func TestSyncTLS(t *testing.T) { defer os.RemoveAll(srcDir) - err = copyFiles("../../../test/data", srcDir) + err = CopyFiles("../../../test/data", srcDir) if err != nil { panic(err) } @@ -971,8 +892,8 @@ func TestSyncTLS(t *testing.T) { panic(err) } - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, true) + destPort := GetFreePort() + destBaseURL := GetSecureBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -1090,13 +1011,13 @@ func TestSyncBasicAuth(t *testing.T) { Convey("Verify sync basic auth", t, func() { updateDuration, _ := time.ParseDuration("1h") - srcPort := getFreePort() - srcBaseURL := getBaseURL(srcPort, false) + srcPort := GetFreePort() + srcBaseURL := GetBaseURL(srcPort) srcConfig := config.New() srcConfig.HTTP.Port = srcPort - htpasswdPath := makeHtpasswdFile() + htpasswdPath := MakeHtpasswdFile() defer os.Remove(htpasswdPath) srcConfig.HTTP.Auth = &config.AuthConfig{ @@ -1112,7 +1033,7 @@ func TestSyncBasicAuth(t *testing.T) { defer os.RemoveAll(srcDir) - err = copyFiles("../../../test/data", srcDir) + err = CopyFiles("../../../test/data", srcDir) if err != nil { panic(err) } @@ -1145,8 +1066,8 @@ func TestSyncBasicAuth(t *testing.T) { } Convey("Verify sync basic auth with file credentials", func() { - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, false) + destPort := GetFreePort() + destBaseURL := GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -1243,8 +1164,8 @@ func TestSyncBasicAuth(t *testing.T) { }) Convey("Verify sync basic auth with wrong file credentials", func() { - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, false) + destPort := GetFreePort() + destBaseURL := GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -1335,8 +1256,8 @@ func TestSyncBasicAuth(t *testing.T) { }) Convey("Verify sync basic auth with bad file credentials", func() { - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, false) + destPort := GetFreePort() + destBaseURL := GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -1425,8 +1346,8 @@ func TestSyncBasicAuth(t *testing.T) { }) Convey("Verify on demand sync with basic auth", func() { - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, false) + destPort := GetFreePort() + destBaseURL := GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -1552,8 +1473,8 @@ func TestSyncBadUrl(t *testing.T) { Convey("Verify sync with bad url", t, func() { updateDuration, _ := time.ParseDuration("1h") - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, false) + destPort := GetFreePort() + destBaseURL := GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -1627,8 +1548,8 @@ func TestSyncNoImagesByRegex(t *testing.T) { Convey("Verify sync with no images on source based on regex", t, func() { updateDuration, _ := time.ParseDuration("1h") - srcPort := getFreePort() - srcBaseURL := getBaseURL(srcPort, false) + srcPort := GetFreePort() + srcBaseURL := GetBaseURL(srcPort) srcConfig := config.New() srcConfig.HTTP.Port = srcPort @@ -1638,7 +1559,7 @@ func TestSyncNoImagesByRegex(t *testing.T) { panic(err) } - err = copyFiles("../../../test/data", srcDir) + err = CopyFiles("../../../test/data", srcDir) if err != nil { panic(err) } @@ -1671,8 +1592,8 @@ func TestSyncNoImagesByRegex(t *testing.T) { time.Sleep(100 * time.Millisecond) } - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, false) + destPort := GetFreePort() + destBaseURL := GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -1758,8 +1679,8 @@ func TestSyncInvalidRegex(t *testing.T) { Convey("Verify sync with invalid regex", t, func() { updateDuration, _ := time.ParseDuration("1h") - srcPort := getFreePort() - srcBaseURL := getBaseURL(srcPort, false) + srcPort := GetFreePort() + srcBaseURL := GetBaseURL(srcPort) srcConfig := config.New() srcConfig.HTTP.Port = srcPort @@ -1769,7 +1690,7 @@ func TestSyncInvalidRegex(t *testing.T) { panic(err) } - err = copyFiles("../../../test/data", srcDir) + err = CopyFiles("../../../test/data", srcDir) if err != nil { panic(err) } @@ -1802,8 +1723,8 @@ func TestSyncInvalidRegex(t *testing.T) { time.Sleep(100 * time.Millisecond) } - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, false) + destPort := GetFreePort() + destBaseURL := GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -1876,8 +1797,8 @@ func TestSyncNotSemver(t *testing.T) { Convey("Verify sync feature semver compliant", t, func() { updateDuration, _ := time.ParseDuration("30m") - srcPort := getFreePort() - srcBaseURL := getBaseURL(srcPort, false) + srcPort := GetFreePort() + srcBaseURL := GetBaseURL(srcPort) srcConfig := config.New() srcConfig.HTTP.Port = srcPort @@ -1889,7 +1810,7 @@ func TestSyncNotSemver(t *testing.T) { defer os.RemoveAll(srcDir) - err = copyFiles("../../../test/data", srcDir) + err = CopyFiles("../../../test/data", srcDir) if err != nil { panic(err) } @@ -1935,8 +1856,8 @@ func TestSyncNotSemver(t *testing.T) { So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 201) - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, false) + destPort := GetFreePort() + destBaseURL := GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -2032,8 +1953,8 @@ func TestSyncInvalidCerts(t *testing.T) { defer func() { client.SetTLSClientConfig(nil) }() updateDuration, _ := time.ParseDuration("1h") - srcPort := getFreePort() - srcBaseURL := getBaseURL(srcPort, true) + srcPort := GetFreePort() + srcBaseURL := GetSecureBaseURL(srcPort) srcConfig := config.New() srcConfig.HTTP.Port = srcPort @@ -2051,7 +1972,7 @@ func TestSyncInvalidCerts(t *testing.T) { defer os.RemoveAll(srcDir) - err = copyFiles("../../../test/data", srcDir) + err = CopyFiles("../../../test/data", srcDir) if err != nil { panic(err) } @@ -2088,8 +2009,8 @@ func TestSyncInvalidCerts(t *testing.T) { time.Sleep(100 * time.Millisecond) } - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, false) + destPort := GetFreePort() + destBaseURL := GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -2208,8 +2129,8 @@ func TestSyncInvalidUrl(t *testing.T) { Convey("Verify sync invalid url", t, func() { updateDuration, _ := time.ParseDuration("30m") - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, false) + destPort := GetFreePort() + destBaseURL := GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort @@ -2283,8 +2204,8 @@ func TestSyncInvalidTags(t *testing.T) { Convey("Verify sync invalid url", t, func() { updateDuration, _ := time.ParseDuration("30m") - srcPort := getFreePort() - srcBaseURL := getBaseURL(srcPort, false) + srcPort := GetFreePort() + srcBaseURL := GetBaseURL(srcPort) srcConfig := config.New() srcConfig.HTTP.Port = srcPort @@ -2296,7 +2217,7 @@ func TestSyncInvalidTags(t *testing.T) { defer os.RemoveAll(srcDir) - err = copyFiles("../../../test/data", srcDir) + err = CopyFiles("../../../test/data", srcDir) if err != nil { panic(err) } @@ -2326,8 +2247,8 @@ func TestSyncInvalidTags(t *testing.T) { time.Sleep(100 * time.Millisecond) } - destPort := getFreePort() - destBaseURL := getBaseURL(destPort, false) + destPort := GetFreePort() + destBaseURL := GetBaseURL(destPort) destConfig := config.New() destConfig.HTTP.Port = destPort diff --git a/pkg/log/log_test.go b/pkg/log/log_test.go index fbc80819..f98460d5 100644 --- a/pkg/log/log_test.go +++ b/pkg/log/log_test.go @@ -7,28 +7,24 @@ import ( "context" "encoding/json" "fmt" - "io" "io/ioutil" "net/http" "os" - "path" "strings" "testing" "time" "github.com/anuvu/zot/pkg/api" "github.com/anuvu/zot/pkg/api/config" + . "github.com/anuvu/zot/test" godigest "github.com/opencontainers/go-digest" . "github.com/smartystreets/goconvey/convey" "gopkg.in/resty.v1" ) const ( - BaseURL = "http://127.0.0.1:8086" - SecurePort = "8086" username = "test" passphrase = "test" - ServerCert = "../../test/data/server.cert" AuthorizedNamespace = "everyone/isallowed" UnauthorizedNamespace = "fortknox/notallowed" ) @@ -44,80 +40,6 @@ type AuditLog struct { Message string `json:"message"` } -func makeHtpasswdFile() string { - f, err := ioutil.TempFile("", "htpasswd-") - if err != nil { - panic(err) - } - - // bcrypt(username="test", passwd="test") - content := []byte("test:$2y$05$hlbSXDp6hzDLu6VwACS39ORvVRpr3OMR4RlJ31jtlaOEGnPjKZI1m\n") - if err := ioutil.WriteFile(f.Name(), content, 0600); err != nil { - panic(err) - } - - return f.Name() -} - -func copyFiles(sourceDir string, destDir string) error { - sourceMeta, err := os.Stat(sourceDir) - if err != nil { - return err - } - - if err := os.MkdirAll(destDir, sourceMeta.Mode()); err != nil { - return err - } - - files, err := ioutil.ReadDir(sourceDir) - if err != nil { - return err - } - - for _, file := range files { - sourceFilePath := path.Join(sourceDir, file.Name()) - destFilePath := path.Join(destDir, file.Name()) - - if file.IsDir() { - if err = copyFiles(sourceFilePath, destFilePath); err != nil { - return err - } - } else { - sourceFile, err := os.Open(sourceFilePath) - if err != nil { - return err - } - defer sourceFile.Close() - - destFile, err := os.Create(destFilePath) - if err != nil { - return err - } - defer destFile.Close() - - if _, err = io.Copy(destFile, sourceFile); err != nil { - return err - } - } - } - - return nil -} - -func Location(baseURL string, resp *resty.Response) string { - // For some API responses, the Location header is set and is supposed to - // indicate an opaque value. However, it is not clear if this value is an - // absolute URL (https://server:port/v2/...) or just a path (/v2/...) - // zot implements the latter as per the spec, but some registries appear to - // return the former - this needs to be clarified - loc := resp.Header().Get("Location") - if loc[0] == '/' { - return baseURL + loc - } - - return loc -} - func TestAuditLogMessages(t *testing.T) { Convey("Make a new controller", t, func() { dir, err := ioutil.TempDir("", "oci-repo-test") @@ -125,20 +47,22 @@ func TestAuditLogMessages(t *testing.T) { panic(err) } defer os.RemoveAll(dir) - err = copyFiles("../../test/data", dir) + err = CopyFiles("../../test/data", dir) if err != nil { panic(err) } + port := GetFreePort() + baseURL := GetBaseURL(port) conf := config.New() outputPath := dir + "/zot.log" auditPath := dir + "/zot-audit.log" conf.Log = &config.LogConfig{Level: "debug", Output: outputPath, Audit: auditPath} - conf.HTTP.Port = SecurePort + conf.HTTP.Port = port - htpasswdPath := makeHtpasswdFile() + htpasswdPath := MakeHtpasswdFile() defer os.Remove(htpasswdPath) conf.HTTP.Auth = &config.AuthConfig{ HTPasswd: config.AuthHTPasswd{ @@ -157,7 +81,7 @@ func TestAuditLogMessages(t *testing.T) { // wait till ready for { - _, err := resty.R().Get(BaseURL) + _, err := resty.R().Get(baseURL) if err == nil { break } @@ -178,8 +102,7 @@ func TestAuditLogMessages(t *testing.T) { defer auditFile.Close() Convey("Test GET request", func() { - resp, err := resty.R().SetBasicAuth(username, passphrase). - Get(BaseURL + "/v2/") + resp, err := resty.R().SetBasicAuth(username, passphrase).Get(baseURL + "/v2/") So(err, ShouldBeNil) So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusOK) @@ -190,8 +113,7 @@ func TestAuditLogMessages(t *testing.T) { Convey("Test POST request", func() { path := "/v2/" + AuthorizedNamespace + "/blobs/uploads/" - resp, err := resty.R().SetBasicAuth(username, passphrase). - Post(BaseURL + path) + resp, err := resty.R().SetBasicAuth(username, passphrase).Post(baseURL + path) So(err, ShouldBeNil) So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) @@ -221,10 +143,10 @@ func TestAuditLogMessages(t *testing.T) { Convey("Test PUT and DELETE request", func() { // create upload path := "/v2/repo/blobs/uploads/" - resp, err := resty.R().SetBasicAuth(username, passphrase).Post(BaseURL + path) + resp, err := resty.R().SetBasicAuth(username, passphrase).Post(baseURL + path) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) - loc := Location(BaseURL, resp) + loc := Location(baseURL, resp) So(loc, ShouldNotBeEmpty) location := resp.Header().Get("Location") So(location, ShouldNotBeEmpty) @@ -260,7 +182,7 @@ func TestAuditLogMessages(t *testing.T) { SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(loc) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusCreated) - blobLoc := Location(BaseURL, resp) + blobLoc := Location(baseURL, resp) So(blobLoc, ShouldNotBeEmpty) So(resp.Header().Get(api.DistContentDigestKey), ShouldNotBeEmpty) @@ -317,10 +239,10 @@ func TestAuditLogMessages(t *testing.T) { Convey("Test PATCH request", func() { path := "/v2/repo/blobs/uploads/" - resp, err := resty.R().SetBasicAuth(username, passphrase).Post(BaseURL + path) + resp, err := resty.R().SetBasicAuth(username, passphrase).Post(baseURL + path) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusAccepted) - loc := Location(BaseURL, resp) + loc := Location(baseURL, resp) So(loc, ShouldNotBeEmpty) location := resp.Header().Get("Location") So(location, ShouldNotBeEmpty) diff --git a/pkg/storage/storage_fs.go b/pkg/storage/storage_fs.go index 96805e76..64f8f494 100644 --- a/pkg/storage/storage_fs.go +++ b/pkg/storage/storage_fs.go @@ -62,16 +62,7 @@ func (is *ImageStoreFS) RootDir() string { } func (is *ImageStoreFS) DirExists(d string) bool { - fi, err := os.Stat(d) - if err != nil && os.IsNotExist(err) { - return false - } - - if !fi.IsDir() { - return false - } - - return true + return DirExists(d) } func getRoutePrefix(name string) string { @@ -1341,3 +1332,16 @@ func ifOlderThan(is *ImageStoreFS, repo string, delay time.Duration) casext.GCPo return true, nil } } + +func DirExists(d string) bool { + fi, err := os.Stat(d) + if err != nil && os.IsNotExist(err) { + return false + } + + if !fi.IsDir() { + return false + } + + return true +} diff --git a/pkg/storage/storage_fs_test.go b/pkg/storage/storage_fs_test.go index f1c05554..0bc1a347 100644 --- a/pkg/storage/storage_fs_test.go +++ b/pkg/storage/storage_fs_test.go @@ -595,7 +595,7 @@ func TestNegativeCases(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("Invalid dedupe sceanrios", t, func() { + Convey("Invalid dedupe scenarios", t, func() { dir, err := ioutil.TempDir("", "oci-repo-test") if err != nil { panic(err) @@ -671,6 +671,23 @@ func TestNegativeCases(t *testing.T) { So(err, ShouldBeNil) So(b, ShouldEqual, l) }) + + Convey("DirExists call with a filename as argument", t, func(c C) { + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + + filePath := path.Join(dir, "file.txt") + err = ioutil.WriteFile(filePath, []byte("some dummy file content"), 0644) //nolint: gosec + if err != nil { + panic(err) + } + + ok := storage.DirExists(filePath) + So(ok, ShouldBeFalse) + }) } func TestHardLink(t *testing.T) { diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 64e2ea5d..2a19b3a6 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -475,6 +475,15 @@ func TestStorageAPIs(t *testing.T) { indexContent, err := il.GetIndexContent("test") So(err, ShouldBeNil) + if testcase.storageType == "fs" { + err = os.Chmod(path.Join(il.RootDir(), "test", "index.json"), 0000) + So(err, ShouldBeNil) + _, err = il.GetIndexContent("test") + So(err, ShouldNotBeNil) + err = os.Chmod(path.Join(il.RootDir(), "test", "index.json"), 0644) + So(err, ShouldBeNil) + } + var index ispec.Index err = json.Unmarshal(indexContent, &index) diff --git a/test/common.go b/test/common.go new file mode 100644 index 00000000..61e0f9dc --- /dev/null +++ b/test/common.go @@ -0,0 +1,110 @@ +package test + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path" + + "github.com/phayes/freeport" + "gopkg.in/resty.v1" +) + +const ( + BaseURL = "http://127.0.0.1:%s" + BaseSecureURL = "https://127.0.0.1:%s" +) + +func GetFreePort() string { + port, err := freeport.GetFreePort() + if err != nil { + panic(err) + } + + return fmt.Sprint(port) +} + +func GetBaseURL(port string) string { + return fmt.Sprintf(BaseURL, port) +} + +func GetSecureBaseURL(port string) string { + return fmt.Sprintf(BaseSecureURL, port) +} + +func MakeHtpasswdFile() string { + // bcrypt(username="test", passwd="test") + content := "test:$2y$05$hlbSXDp6hzDLu6VwACS39ORvVRpr3OMR4RlJ31jtlaOEGnPjKZI1m\n" + return MakeHtpasswdFileFromString(content) +} + +func MakeHtpasswdFileFromString(fileContent string) string { + f, err := ioutil.TempFile("", "htpasswd-") + if err != nil { + panic(err) + } + + // bcrypt(username="test", passwd="test") + content := []byte(fileContent) + if err := ioutil.WriteFile(f.Name(), content, 0600); err != nil { + panic(err) + } + + return f.Name() +} + +func Location(baseURL string, resp *resty.Response) string { + // For some API responses, the Location header is set and is supposed to + // indicate an opaque value. However, it is not clear if this value is an + // absolute URL (https://server:port/v2/...) or just a path (/v2/...) + // zot implements the latter as per the spec, but some registries appear to + // return the former - this needs to be clarified + loc := resp.Header().Get("Location") + return baseURL + loc +} + +func CopyFiles(sourceDir string, destDir string) error { + sourceMeta, err := os.Stat(sourceDir) + if err != nil { + return err + } + + if err := os.MkdirAll(destDir, sourceMeta.Mode()); err != nil { + return err + } + + files, err := ioutil.ReadDir(sourceDir) + if err != nil { + return err + } + + for _, file := range files { + sourceFilePath := path.Join(sourceDir, file.Name()) + destFilePath := path.Join(destDir, file.Name()) + + if file.IsDir() { + if err = CopyFiles(sourceFilePath, destFilePath); err != nil { + return err + } + } else { + sourceFile, err := os.Open(sourceFilePath) + if err != nil { + return err + } + defer sourceFile.Close() + + destFile, err := os.Create(destFilePath) + if err != nil { + return err + } + defer destFile.Close() + + if _, err = io.Copy(destFile, sourceFile); err != nil { + return err + } + } + } + + return nil +} diff --git a/test/common_test.go b/test/common_test.go new file mode 100644 index 00000000..b6850763 --- /dev/null +++ b/test/common_test.go @@ -0,0 +1,81 @@ +// +build extended + +package test_test + +import ( + "io/ioutil" + "os" + "path" + "testing" + + . "github.com/anuvu/zot/test" + . "github.com/smartystreets/goconvey/convey" +) + +func TestCopyFiles(t *testing.T) { + Convey("sourceDir does not exist", t, func() { + err := CopyFiles("/path/to/some/unexisting/directory", os.TempDir()) + So(err, ShouldNotBeNil) + }) + Convey("destDir is a file", t, func() { + dir, err := ioutil.TempDir("", "copy-files-test") + if err != nil { + panic(err) + } + + err = CopyFiles("data", dir) + if err != nil { + panic(err) + } + + defer os.RemoveAll(dir) + err = CopyFiles(dir, "/etc/passwd") + So(err, ShouldNotBeNil) + }) + Convey("sourceDir does not have read permissions", t, func() { + dir, err := ioutil.TempDir("", "copy-files-test") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + + err = os.Chmod(dir, 0300) + So(err, ShouldBeNil) + + err = CopyFiles(dir, os.TempDir()) + So(err, ShouldNotBeNil) + }) + Convey("sourceDir has a subfolder that does not have read permissions", t, func() { + dir, err := ioutil.TempDir("", "copy-files-test") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + + sdir := "subdir" + err = os.Mkdir(path.Join(dir, sdir), 0300) + So(err, ShouldBeNil) + + err = CopyFiles(dir, os.TempDir()) + So(err, ShouldNotBeNil) + }) + Convey("sourceDir has a file that does not have read permissions", t, func() { + dir, err := ioutil.TempDir("", "copy-files-test") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + + filePath := path.Join(dir, "file.txt") + err = ioutil.WriteFile(filePath, []byte("some dummy file content"), 0644) //nolint: gosec + if err != nil { + panic(err) + } + + err = os.Chmod(filePath, 0300) + So(err, ShouldBeNil) + + err = CopyFiles(dir, os.TempDir()) + So(err, ShouldNotBeNil) + }) +}