From fff6107310654d1950ae1eecf016ed4006422374 Mon Sep 17 00:00:00 2001 From: Petu Eusebiu Date: Thu, 25 Nov 2021 14:04:39 +0200 Subject: [PATCH] Sync prefix can be an exact match or a glob pattern, closes #297 Signed-off-by: Petu Eusebiu --- examples/README.md | 10 +- go.mod | 1 + go.sum | 2 + pkg/cli/root.go | 16 + pkg/cli/root_test.go | 17 + pkg/extensions/extensions.go | 8 +- pkg/extensions/sync/http_handler.go | 5 + pkg/extensions/sync/sync.go | 14 +- pkg/extensions/sync/sync_internal_test.go | 28 + pkg/extensions/sync/sync_test.go | 1407 +++++---------------- pkg/extensions/sync/utils.go | 31 +- 11 files changed, 400 insertions(+), 1139 deletions(-) diff --git a/examples/README.md b/examples/README.md index e2f6c4a3..23d7322f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -358,18 +358,21 @@ Configure each registry sync: "url": "https://registry1:5000", "onDemand": false, # pull any image which the local registry doesn't have "pollInterval": "6h", # polling interval - "tlsVerify": true, # wheather or not to verify tls + "tlsVerify": true, # whether or not to verify tls "certDir": "/home/user/certs", # use certificates at certDir path, if not specified then use the default certs dir "content":[ # which content to periodically pull { - "prefix":"/repo1/repo", # pull all images under /repo1/repo + "prefix":"/repo1/repo", # pull image repo1/repo "tags":{ # filter by tags "regex":"4.*", # filter tags by regex "semver":true # filter tags by semver compliance } }, { - "prefix":"/repo2/repo" # pull all images under /repo2/repo + "prefix":"/repo2/repo*" # pull all images that matches repo2/repo.* + }, + { + "prefix":"/repo3/**" # pull all images under repo3/ (matches recursively all repos under repo3/) } ] }, @@ -396,3 +399,4 @@ Configure each registry sync: } ``` +Prefixes can be strings that exactly match repositories or they can be glob patterns. diff --git a/go.mod b/go.mod index 8d77c29f..fc8a34eb 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/apex/log v1.9.0 github.com/aquasecurity/trivy v0.0.0-00010101000000-000000000000 github.com/aquasecurity/trivy-db v0.0.0-20210916043317-726b7b72a47b + github.com/bmatcuk/doublestar/v4 v4.0.2 github.com/briandowns/spinner v1.16.0 github.com/chartmuseum/auth v0.5.0 github.com/containerd/containerd v1.5.8 // indirect diff --git a/go.sum b/go.sum index b7d9f2e6..78bf83a2 100644 --- a/go.sum +++ b/go.sum @@ -304,6 +304,8 @@ github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= +github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA= +github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U= github.com/bombsimon/wsl/v2 v2.2.0/go.mod h1:Azh8c3XGEJl9LyX0/sFC+CKMc7Ssgua0g+6abzXN4Pg= diff --git a/pkg/cli/root.go b/pkg/cli/root.go index 238fecf7..16a9f3aa 100644 --- a/pkg/cli/root.go +++ b/pkg/cli/root.go @@ -5,6 +5,7 @@ import ( "github.com/anuvu/zot/pkg/api" "github.com/anuvu/zot/pkg/api/config" "github.com/anuvu/zot/pkg/storage" + glob "github.com/bmatcuk/doublestar/v4" "github.com/fsnotify/fsnotify" "github.com/mitchellh/mapstructure" distspec "github.com/opencontainers/distribution-spec/specs-go" @@ -188,6 +189,21 @@ func LoadConfiguration(config *config.Config, configPath string) { } } + // check glob patterns in sync are compilable + if config.Extensions != nil && config.Extensions.Sync != nil { + for _, regCfg := range config.Extensions.Sync.Registries { + if regCfg.Content != nil { + for _, content := range regCfg.Content { + ok := glob.ValidatePattern(content.Prefix) + if !ok { + log.Error().Err(glob.ErrBadPattern).Str("pattern", content.Prefix).Msg("pattern could not be compiled") + panic(errors.ErrBadConfig) + } + } + } + } + } + // enforce s3 driver on subpaths in case of using storage driver if config.Storage.SubPaths != nil { if len(config.Storage.SubPaths) > 0 { diff --git a/pkg/cli/root_test.go b/pkg/cli/root_test.go index 9f995e48..abe253c3 100644 --- a/pkg/cli/root_test.go +++ b/pkg/cli/root_test.go @@ -134,6 +134,23 @@ func TestVerify(t *testing.T) { So(func() { _ = cli.NewRootCmd().Execute() }, ShouldPanic) }) + Convey("Test verify with bad sync prefixes", t, func(c C) { + tmpfile, err := ioutil.TempFile("", "zot-test*.json") + So(err, ShouldBeNil) + defer os.Remove(tmpfile.Name()) // clean up + content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + "http":{"address":"127.0.0.1","port":"8080","realm":"zot", + "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, + "extensions":{"sync": {"registries": [{"url":"localhost:9999", + "content": [{"prefix":"[repo%^&"}]}]}}}`) + _, err = tmpfile.Write(content) + So(err, ShouldBeNil) + err = tmpfile.Close() + So(err, ShouldBeNil) + os.Args = []string{"cli_test", "verify", tmpfile.Name()} + So(func() { _ = cli.NewRootCmd().Execute() }, ShouldPanic) + }) + Convey("Test verify good config", t, func(c C) { tmpfile, err := ioutil.TempFile("", "zot-test*.json") So(err, ShouldBeNil) diff --git a/pkg/extensions/extensions.go b/pkg/extensions/extensions.go index 69c64842..666946a9 100644 --- a/pkg/extensions/extensions.go +++ b/pkg/extensions/extensions.go @@ -72,10 +72,14 @@ func EnableSyncExtension(config *config.Config, log log.Logger, storeController if config.Extensions.Sync != nil { defaultPollInterval, _ := time.ParseDuration("1h") for id, registryCfg := range config.Extensions.Sync.Registries { - if registryCfg.PollInterval < defaultPollInterval { + if registryCfg.Content != nil && + len(registryCfg.Content) > 0 && + registryCfg.PollInterval < defaultPollInterval { config.Extensions.Sync.Registries[id].PollInterval = defaultPollInterval - log.Warn().Msg("Sync registries interval set to too-short interval < 1h, changing update duration to 1 hour and continuing.") // nolint: lll + log.Warn().Str("registry", registryCfg.URL). + Msg("Sync registries interval set to too-short interval < 1h," + + "changing update duration to 1 hour and continuing.") } } diff --git a/pkg/extensions/sync/http_handler.go b/pkg/extensions/sync/http_handler.go index 787b209a..08bda90d 100644 --- a/pkg/extensions/sync/http_handler.go +++ b/pkg/extensions/sync/http_handler.go @@ -48,6 +48,11 @@ func (h *PostHandler) Handler(w http.ResponseWriter, r *http.Request) { } for _, regCfg := range h.Cfg.Registries { + if len(regCfg.Content) == 0 { + h.Log.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL) + continue + } + upstreamRegistryName := strings.Replace(strings.Replace(regCfg.URL, "http://", "", 1), "https://", "", 1) if err := syncRegistry(regCfg, h.StoreController, h.Log, localCtx, policyCtx, diff --git a/pkg/extensions/sync/sync.go b/pkg/extensions/sync/sync.go index 5ed08ca7..86a12a2f 100644 --- a/pkg/extensions/sync/sync.go +++ b/pkg/extensions/sync/sync.go @@ -310,11 +310,6 @@ func getUpstreamContext(regCfg *RegistryConfig, credentials Credentials) *types. func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController, log log.Logger, localCtx *types.SystemContext, policyCtx *signature.PolicyContext, credentials Credentials, uuid string) error { - if len(regCfg.Content) == 0 { - log.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL) - return nil - } - log.Info().Msgf("syncing registry: %s", regCfg.URL) var err error @@ -341,9 +336,9 @@ func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController upstreamRegistryName := strings.Replace(strings.Replace(regCfg.URL, "http://", "", 1), "https://", "", 1) - log.Info().Msg("filtering repos based on sync prefixes") + log.Info().Msgf("filtering %d repos based on sync prefixes", len(catalog.Repositories)) - repos := filterRepos(catalog.Repositories, regCfg.Content) + repos := filterRepos(catalog.Repositories, regCfg.Content, log) log.Info().Msgf("got repos: %v", repos) @@ -467,6 +462,11 @@ func Run(cfg Config, storeController storage.StoreController, logger log.Logger) // for each upstream registry, start a go routine. for _, regCfg := range cfg.Registries { + if len(regCfg.Content) == 0 { + logger.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL) + continue + } + // schedule each registry sync ticker := time.NewTicker(regCfg.PollInterval) diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index 88fb7317..9343e2cc 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -168,6 +168,34 @@ func TestSyncInternal(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("Test filterRepos()", t, func() { + repos := []string{"repo", "repo1", "repo2", "repo/repo2", "repo/repo2/repo3/repo4"} + contents := []Content{ + { + Prefix: "repo", + }, + { + Prefix: "/repo/**", + }, + { + Prefix: "repo*", + }, + } + filteredRepos := filterRepos(repos, contents, log.NewLogger("", "")) + So(filteredRepos[0], ShouldResemble, []string{"repo"}) + So(filteredRepos[1], ShouldResemble, []string{"repo/repo2", "repo/repo2/repo3/repo4"}) + So(filteredRepos[2], ShouldResemble, []string{"repo1", "repo2"}) + + contents = []Content{ + { + Prefix: "[repo%#@", + }, + } + + filteredRepos = filterRepos(repos, contents, log.NewLogger("", "")) + So(len(filteredRepos), ShouldEqual, 0) + }) + Convey("Verify pushSyncedLocalImage func", t, func() { storageDir, err := ioutil.TempDir("", "oci-dest-repo-test") if err != nil { diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 96e347d7..2c633a81 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -37,7 +37,7 @@ const ( testImage = "zot-test" testImageTag = "0.0.1" - testCveImage = "/zot-cve-test" + testCveImage = "zot-cve-test" ) var errSync = errors.New("sync error, src oci repo differs from dest one") @@ -71,69 +71,174 @@ func copyFile(sourceFilePath, destFilePath string) error { return nil } +func startUpstreamServer(secure, basicAuth bool) (*api.Controller, string, string, string, *resty.Client) { + srcPort := GetFreePort() + + srcConfig := config.New() + + client := resty.New() + + var srcBaseURL string + if secure { + srcBaseURL = GetSecureBaseURL(srcPort) + + srcConfig.HTTP.TLS = &config.TLSConfig{ + Cert: ServerCert, + Key: ServerKey, + CACert: CACert, + } + + caCert, err := ioutil.ReadFile(CACert) + if err != nil { + panic(err) + } + + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + + client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) + + cert, err := tls.LoadX509KeyPair("../../../test/data/client.cert", "../../../test/data/client.key") + if err != nil { + panic(err) + } + + client.SetCertificates(cert) + } else { + srcBaseURL = GetBaseURL(srcPort) + } + + var htpasswdPath string + if basicAuth { + htpasswdPath = MakeHtpasswdFile() + srcConfig.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ + Path: htpasswdPath, + }, + } + } + + srcConfig.HTTP.Port = srcPort + + srcDir, err := ioutil.TempDir("", "oci-src-repo-test") + if err != nil { + panic(err) + } + + err = CopyFiles("../../../test/data", srcDir) + if err != nil { + panic(err) + } + + srcConfig.Storage.RootDirectory = srcDir + + sc := api.NewController(srcConfig) + + go func() { + // this blocks + if err := sc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := client.R().Get(srcBaseURL) + if err == nil { + break + } + + time.Sleep(100 * time.Millisecond) + } + + return sc, srcBaseURL, srcDir, htpasswdPath, client +} + +func startDownstreamServer(secure bool, syncConfig *sync.Config) (*api.Controller, string, string, *resty.Client) { + destPort := GetFreePort() + + destConfig := config.New() + + client := resty.New() + + var destBaseURL string + if secure { + destBaseURL = GetSecureBaseURL(destPort) + + destConfig.HTTP.TLS = &config.TLSConfig{ + Cert: ServerCert, + Key: ServerKey, + CACert: CACert, + } + + caCert, err := ioutil.ReadFile(CACert) + if err != nil { + panic(err) + } + + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + + client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) + + cert, err := tls.LoadX509KeyPair("../../../test/data/client.cert", "../../../test/data/client.key") + if err != nil { + panic(err) + } + + client.SetCertificates(cert) + } else { + destBaseURL = GetBaseURL(destPort) + } + + destConfig.HTTP.Port = destPort + + destDir, err := ioutil.TempDir("", "oci-dest-repo-test") + if err != nil { + panic(err) + } + + destConfig.Storage.RootDirectory = destDir + + destConfig.Extensions = &extconf.ExtensionConfig{} + destConfig.Extensions.Search = nil + destConfig.Extensions.Sync = syncConfig + + dc := api.NewController(destConfig) + + go func() { + // this blocks + if err := dc.Run(); err != nil { + return + } + }() + + // wait till ready + for { + _, err := client.R().Get(destBaseURL) + if err == nil { + break + } + + time.Sleep(100 * time.Millisecond) + } + + return dc, destBaseURL, destDir, client +} + func TestSyncOnDemand(t *testing.T) { Convey("Verify sync on demand feature", t, func() { updateDuration, _ := time.ParseDuration("30m") - srcPort := GetFreePort() - srcBaseURL := GetBaseURL(srcPort) - - srcConfig := config.New() - srcConfig.HTTP.Port = srcPort - - srcDir, err := ioutil.TempDir("", "oci-src-repo-test") - if err != nil { - panic(err) - } - + sc, srcBaseURL, srcDir, _, srcClient := startUpstreamServer(false, false) defer os.RemoveAll(srcDir) - err = CopyFiles("../../../test/data", srcDir) - if err != nil { - panic(err) - } - - srcConfig.Storage.RootDirectory = srcDir - - sc := api.NewController(srcConfig) - - go func() { - // this blocks - if err := sc.Run(); err != nil { - return - } - }() - defer func() { ctx := context.Background() _ = sc.Server.Shutdown(ctx) time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(srcBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - - destPort := GetFreePort() - destBaseURL := GetBaseURL(destPort) - - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destConfig.Storage.RootDirectory = destDir - regex := ".*" var semver bool var tlsVerify bool @@ -156,19 +261,11 @@ func TestSyncOnDemand(t *testing.T) { OnDemand: true, } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = &sync.Config{ + syncConfig := &sync.Config{ Registries: []sync.RegistryConfig{syncRegistryConfig}} - dc := api.NewController(destConfig) - - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() @@ -176,32 +273,23 @@ func TestSyncOnDemand(t *testing.T) { time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(destBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - var srcTagsList TagsList var destTagsList TagsList - resp, _ := resty.R().Get(srcBaseURL + "/v2/" + testImage + "/tags/list") + resp, _ := srcClient.R().Get(srcBaseURL + "/v2/" + testImage + "/tags/list") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 200) - err = json.Unmarshal(resp.Body(), &srcTagsList) + err := json.Unmarshal(resp.Body(), &srcTagsList) if err != nil { panic(err) } - resp, err = resty.R().Get(destBaseURL + "/v2/" + "inexistent" + "/manifests/" + testImageTag) + resp, err = destClient.R().Get(destBaseURL + "/v2/" + "inexistent" + "/manifests/" + testImageTag) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 404) - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "inexistent") + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "inexistent") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 404) @@ -210,7 +298,7 @@ func TestSyncOnDemand(t *testing.T) { panic(err) } - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 500) @@ -219,7 +307,7 @@ func TestSyncOnDemand(t *testing.T) { panic(err) } - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "1.1.1") + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "1.1.1") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 404) @@ -228,11 +316,11 @@ func TestSyncOnDemand(t *testing.T) { panic(err) } - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "1.1.1") + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "1.1.1") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 404) - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 404) @@ -246,7 +334,7 @@ func TestSyncOnDemand(t *testing.T) { panic(err) } - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 404) @@ -255,11 +343,11 @@ func TestSyncOnDemand(t *testing.T) { panic(err) } - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -268,9 +356,7 @@ func TestSyncOnDemand(t *testing.T) { panic(err) } - if eq := reflect.DeepEqual(destTagsList.Tags, srcTagsList.Tags); eq == false { - panic(errSync) - } + So(destTagsList, ShouldResemble, srcTagsList) }) } @@ -278,65 +364,15 @@ func TestSync(t *testing.T) { Convey("Verify sync feature", t, func() { updateDuration, _ := time.ParseDuration("30m") - srcPort := GetFreePort() - srcBaseURL := GetBaseURL(srcPort) - - srcConfig := config.New() - srcConfig.HTTP.Port = srcPort - - srcDir, err := ioutil.TempDir("", "oci-src-repo-test") - if err != nil { - panic(err) - } - + sc, srcBaseURL, srcDir, _, srcClient := startUpstreamServer(false, false) defer os.RemoveAll(srcDir) - err = CopyFiles("../../../test/data", srcDir) - if err != nil { - panic(err) - } - - srcConfig.Storage.RootDirectory = srcDir - - sc := api.NewController(srcConfig) - - go func() { - // this blocks - if err := sc.Run(); err != nil { - return - } - }() - defer func() { ctx := context.Background() _ = sc.Server.Shutdown(ctx) time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(srcBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - - destPort := GetFreePort() - destBaseURL := GetBaseURL(destPort) - - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destConfig.Storage.RootDirectory = destDir - regex := ".*" semver := true var tlsVerify bool @@ -357,18 +393,10 @@ func TestSync(t *testing.T) { CertDir: "", } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} - dc := api.NewController(destConfig) - - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() @@ -376,29 +404,20 @@ func TestSync(t *testing.T) { time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(destBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - var srcTagsList TagsList var destTagsList TagsList - resp, _ := resty.R().Get(srcBaseURL + "/v2/" + testImage + "/tags/list") + resp, _ := srcClient.R().Get(srcBaseURL + "/v2/" + testImage + "/tags/list") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 200) - err = json.Unmarshal(resp.Body(), &srcTagsList) + err := json.Unmarshal(resp.Body(), &srcTagsList) if err != nil { panic(err) } for { - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") if err != nil { panic(err) } @@ -415,32 +434,15 @@ func TestSync(t *testing.T) { time.Sleep(500 * time.Millisecond) } - if eq := reflect.DeepEqual(destTagsList.Tags, srcTagsList.Tags); eq == false { - panic(errSync) - } + So(destTagsList, ShouldResemble, srcTagsList) Convey("Test sync on POST request on /sync", func() { - resp, _ := resty.R().Post(destBaseURL + "/sync") + resp, _ := destClient.R().Post(destBaseURL + "/sync") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 200) }) Convey("Test sync with more contents", func() { - destPort := GetFreePort() - destBaseURL := GetBaseURL(destPort) - - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destConfig.Storage.RootDirectory = destDir - regex := ".*" semver := true @@ -471,18 +473,10 @@ func TestSync(t *testing.T) { CertDir: "", } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} - dc := api.NewController(destConfig) - - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() @@ -490,19 +484,11 @@ func TestSync(t *testing.T) { time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(destBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - var srcTagsList TagsList var destTagsList TagsList - resp, _ := resty.R().Get(srcBaseURL + "/v2/" + testImage + "/tags/list") + resp, err := srcClient.R().Get(srcBaseURL + "/v2/" + testImage + "/tags/list") + So(err, ShouldBeNil) So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 200) @@ -512,7 +498,7 @@ func TestSync(t *testing.T) { } for { - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") if err != nil { panic(err) } @@ -529,12 +515,26 @@ func TestSync(t *testing.T) { time.Sleep(500 * time.Millisecond) } - if eq := reflect.DeepEqual(destTagsList.Tags, srcTagsList.Tags); eq == false { - panic(errSync) - } + So(destTagsList, ShouldResemble, srcTagsList) + + // testCveImage should not be synced because of regex being "invalid", shouldn't match anything + resp, _ = srcClient.R().Get(srcBaseURL + "/v2/" + testCveImage + "/tags/list") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + err = json.Unmarshal(resp.Body(), &srcTagsList) + So(err, ShouldBeNil) + + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testCveImage + "/tags/list") + So(err, ShouldBeNil) + + err = json.Unmarshal(resp.Body(), &destTagsList) + So(err, ShouldBeNil) + + So(destTagsList, ShouldNotResemble, srcTagsList) Convey("Test sync on POST request on /sync", func() { - resp, _ := resty.R().Post(destBaseURL + "/sync") + resp, _ := destClient.R().Post(destBaseURL + "/sync") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 200) }) @@ -546,65 +546,15 @@ func TestSyncPermsDenied(t *testing.T) { Convey("Verify sync feature without perm on sync cache", t, func() { updateDuration, _ := time.ParseDuration("30m") - srcPort := GetFreePort() - srcBaseURL := GetBaseURL(srcPort) - - srcConfig := config.New() - srcConfig.HTTP.Port = srcPort - - srcDir, err := ioutil.TempDir("", "oci-src-repo-test") - if err != nil { - panic(err) - } - + sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false) defer os.RemoveAll(srcDir) - err = CopyFiles("../../../test/data", srcDir) - if err != nil { - panic(err) - } - - srcConfig.Storage.RootDirectory = srcDir - - sc := api.NewController(srcConfig) - - go func() { - // this blocks - if err := sc.Run(); err != nil { - return - } - }() - defer func() { ctx := context.Background() _ = sc.Server.Shutdown(ctx) time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(srcBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - - destPort := GetFreePort() - destBaseURL := GetBaseURL(destPort) - - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destConfig.Storage.RootDirectory = destDir - regex := ".*" semver := true var tlsVerify bool @@ -625,18 +575,10 @@ func TestSyncPermsDenied(t *testing.T) { CertDir: "", } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} - dc := api.NewController(destConfig) - - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() @@ -644,22 +586,13 @@ func TestSyncPermsDenied(t *testing.T) { time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(destBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - - err = os.Chmod(path.Join(destDir, testImage, sync.SyncBlobUploadDir), 0000) + err := os.Chmod(path.Join(destDir, testImage, sync.SyncBlobUploadDir), 0000) if err != nil { panic(err) } Convey("Test sync on POST request on /sync", func() { - resp, _ := resty.R().Post(destBaseURL + "/sync") + resp, _ := destClient.R().Post(destBaseURL + "/sync") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 500) }) @@ -668,93 +601,17 @@ func TestSyncPermsDenied(t *testing.T) { func TestSyncBadTLS(t *testing.T) { Convey("Verify sync TLS feature", t, func() { - caCert, err := ioutil.ReadFile(CACert) - So(err, ShouldBeNil) - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - - client := resty.New() - - client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) - defer func() { client.SetTLSClientConfig(nil) }() - - updateDuration, _ := time.ParseDuration("1h") - - srcPort := GetFreePort() - srcBaseURL := GetSecureBaseURL(srcPort) - - srcConfig := config.New() - srcConfig.HTTP.Port = srcPort - - srcConfig.HTTP.TLS = &config.TLSConfig{ - Cert: ServerCert, - Key: ServerKey, - CACert: CACert, - } - - srcDir, err := ioutil.TempDir("", "oci-src-repo-test") - if err != nil { - panic(err) - } + updateDuration, _ := time.ParseDuration("30m") + sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(true, false) defer os.RemoveAll(srcDir) - srcConfig.Storage.RootDirectory = srcDir - - sc := api.NewController(srcConfig) - - go func() { - // this blocks - if err := sc.Run(); err != nil { - return - } - }() - defer func() { ctx := context.Background() _ = sc.Server.Shutdown(ctx) + time.Sleep(500 * time.Millisecond) }() - cert, err := tls.LoadX509KeyPair("../../../test/data/client.cert", "../../../test/data/client.key") - if err != nil { - panic(err) - } - - client.SetCertificates(cert) - // wait till ready - for { - _, err := client.R().Get(srcBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - err = CopyFiles("../../../test/data", destDir) - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destPort := GetFreePort() - destBaseURL := GetSecureBaseURL(destPort) - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destConfig.HTTP.TLS = &config.TLSConfig{ - Cert: ServerCert, - Key: ServerKey, - CACert: CACert, - } - - destConfig.Storage.RootDirectory = destDir - regex := ".*" var semver bool tlsVerify := true @@ -775,36 +632,29 @@ func TestSyncBadTLS(t *testing.T) { TLSVerify: &tlsVerify, } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} - dc := api.NewController(destConfig) - - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(true, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() _ = dc.Server.Shutdown(ctx) + time.Sleep(500 * time.Millisecond) }() // give it time to set up sync time.Sleep(2 * time.Second) - resp, _ := client.R().Post(destBaseURL + "/sync") + resp, _ := destClient.R().Post(destBaseURL + "/sync") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 500) - resp, _ = client.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "invalid") + resp, _ = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "invalid") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 404) - resp, _ = client.R().Get(destBaseURL + "/v2/" + "invalid" + "/manifests/" + testImageTag) + resp, _ = destClient.R().Get(destBaseURL + "/v2/" + "invalid" + "/manifests/" + testImageTag) So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 404) }) @@ -812,76 +662,19 @@ func TestSyncBadTLS(t *testing.T) { func TestSyncTLS(t *testing.T) { Convey("Verify sync TLS feature", t, func() { - caCert, err := ioutil.ReadFile(CACert) - So(err, ShouldBeNil) - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - - client := resty.New() - - client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) - defer func() { client.SetTLSClientConfig(nil) }() - - var srcIndex ispec.Index - var destIndex ispec.Index - updateDuration, _ := time.ParseDuration("1h") - srcPort := GetFreePort() - srcBaseURL := GetSecureBaseURL(srcPort) - - srcConfig := config.New() - srcConfig.HTTP.Port = srcPort - - srcConfig.HTTP.TLS = &config.TLSConfig{ - Cert: ServerCert, - Key: ServerKey, - CACert: CACert, - } - - srcDir, err := ioutil.TempDir("", "oci-src-repo-test") - if err != nil { - panic(err) - } - + sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(true, false) defer os.RemoveAll(srcDir) - err = CopyFiles("../../../test/data", srcDir) - if err != nil { - panic(err) - } - - srcConfig.Storage.RootDirectory = srcDir - - sc := api.NewController(srcConfig) - - go func() { - // this blocks - if err := sc.Run(); err != nil { - return - } - }() - defer func() { ctx := context.Background() _ = sc.Server.Shutdown(ctx) time.Sleep(500 * time.Millisecond) }() - cert, err := tls.LoadX509KeyPair("../../../test/data/client.cert", "../../../test/data/client.key") - if err != nil { - panic(err) - } - - client.SetCertificates(cert) - // wait till ready - for { - _, err := client.R().Get(srcBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } + var srcIndex ispec.Index + var destIndex ispec.Index srcBuf, err := ioutil.ReadFile(path.Join(srcDir, testImage, "index.json")) if err != nil { @@ -892,26 +685,6 @@ func TestSyncTLS(t *testing.T) { panic(err) } - destPort := GetFreePort() - destBaseURL := GetSecureBaseURL(destPort) - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destConfig.HTTP.TLS = &config.TLSConfig{ - Cert: ServerCert, - Key: ServerKey, - CACert: CACert, - } - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destConfig.Storage.RootDirectory = destDir - // copy upstream client certs, use them in sync config destClientCertDir, err := ioutil.TempDir("", "destCerts") if err != nil { @@ -958,18 +731,10 @@ func TestSyncTLS(t *testing.T) { CertDir: destClientCertDir, } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} - dc := api.NewController(destConfig) - - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(true, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() @@ -999,90 +764,29 @@ func TestSyncTLS(t *testing.T) { } Convey("Test sync on POST request on /sync", func() { - resp, _ := client.R().SetBasicAuth("test", "test").Post(destBaseURL + "/sync") + resp, _ := destClient.R().SetBasicAuth("test", "test").Post(destBaseURL + "/sync") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 200) }) }) } -// nolint: gocyclo func TestSyncBasicAuth(t *testing.T) { Convey("Verify sync basic auth", t, func() { updateDuration, _ := time.ParseDuration("1h") - srcPort := GetFreePort() - srcBaseURL := GetBaseURL(srcPort) - - srcConfig := config.New() - srcConfig.HTTP.Port = srcPort - - htpasswdPath := MakeHtpasswdFile() + sc, srcBaseURL, srcDir, htpasswdPath, srcClient := startUpstreamServer(false, true) defer os.Remove(htpasswdPath) - - srcConfig.HTTP.Auth = &config.AuthConfig{ - HTPasswd: config.AuthHTPasswd{ - Path: htpasswdPath, - }, - } - - srcDir, err := ioutil.TempDir("", "oci-src-repo-test") - if err != nil { - panic(err) - } - defer os.RemoveAll(srcDir) - err = CopyFiles("../../../test/data", srcDir) - if err != nil { - panic(err) - } - - srcConfig.Storage.RootDirectory = srcDir - - sc := api.NewController(srcConfig) - - go func() { - // this blocks - if err := sc.Run(); err != nil { - return - } - }() - defer func() { ctx := context.Background() _ = sc.Server.Shutdown(ctx) time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(srcBaseURL) - - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - Convey("Verify sync basic auth with file credentials", func() { - destPort := GetFreePort() - destBaseURL := GetBaseURL(destPort) - - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destConfig.Storage.RootDirectory = destDir - registryName := strings.Replace(strings.Replace(srcBaseURL, "http://", "", 1), "https://", "", 1) - credentialsFile := makeCredentialsFile(fmt.Sprintf(`{"%s":{"username": "test", "password": "test"}}`, registryName)) var tlsVerify bool @@ -1099,19 +803,11 @@ func TestSyncBasicAuth(t *testing.T) { CertDir: "", } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = &sync.Config{CredentialsFile: credentialsFile, + syncConfig := &sync.Config{CredentialsFile: credentialsFile, Registries: []sync.RegistryConfig{syncRegistryConfig}} - dc := api.NewController(destConfig) - - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() @@ -1119,29 +815,20 @@ func TestSyncBasicAuth(t *testing.T) { time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(destBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - var srcTagsList TagsList var destTagsList TagsList - resp, _ := resty.R().SetBasicAuth("test", "test").Get(srcBaseURL + "/v2/" + testImage + "/tags/list") + resp, _ := srcClient.R().SetBasicAuth("test", "test").Get(srcBaseURL + "/v2/" + testImage + "/tags/list") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 200) - err = json.Unmarshal(resp.Body(), &srcTagsList) + err := json.Unmarshal(resp.Body(), &srcTagsList) if err != nil { panic(err) } for { - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") if err != nil { panic(err) } @@ -1158,9 +845,7 @@ func TestSyncBasicAuth(t *testing.T) { time.Sleep(500 * time.Millisecond) } - if eq := reflect.DeepEqual(destTagsList.Tags, srcTagsList.Tags); eq == false { - panic(errSync) - } + So(destTagsList, ShouldResemble, srcTagsList) }) Convey("Verify sync basic auth with wrong file credentials", func() { @@ -1256,30 +941,12 @@ func TestSyncBasicAuth(t *testing.T) { }) Convey("Verify sync basic auth with bad file credentials", func() { - destPort := GetFreePort() - destBaseURL := GetBaseURL(destPort) - - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destConfig.Storage.RootDirectory = destDir - - regex := ".*" - var semver bool - registryName := strings.Replace(strings.Replace(srcBaseURL, "http://", "", 1), "https://", "", 1) credentialsFile := makeCredentialsFile(fmt.Sprintf(`{"%s":{"username": "test", "password": "test"}}`, registryName)) - err = os.Chmod(credentialsFile, 0000) + err := os.Chmod(credentialsFile, 0000) So(err, ShouldBeNil) defer func() { @@ -1287,6 +954,8 @@ func TestSyncBasicAuth(t *testing.T) { So(os.RemoveAll(credentialsFile), ShouldBeNil) }() + regex := ".*" + var semver bool var tlsVerify bool syncRegistryConfig := sync.RegistryConfig{ @@ -1305,40 +974,24 @@ func TestSyncBasicAuth(t *testing.T) { CertDir: "", } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = &sync.Config{CredentialsFile: credentialsFile, + syncConfig := &sync.Config{CredentialsFile: credentialsFile, Registries: []sync.RegistryConfig{syncRegistryConfig}} - dc := api.NewController(destConfig) - - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() _ = dc.Server.Shutdown(ctx) + time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(destBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - - resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + resp, err := destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 404) Convey("Test sync on POST request on /sync", func() { - resp, _ := resty.R().Post(destBaseURL + "/sync") + resp, _ := destClient.R().Post(destBaseURL + "/sync") So(resp, ShouldNotBeNil) So(string(resp.Body()), ShouldContainSubstring, "permission denied") So(resp.StatusCode(), ShouldEqual, 500) @@ -1346,21 +999,6 @@ func TestSyncBasicAuth(t *testing.T) { }) Convey("Verify on demand sync with basic auth", func() { - destPort := GetFreePort() - destBaseURL := GetBaseURL(destPort) - - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destConfig.Storage.RootDirectory = destDir - registryName := strings.Replace(strings.Replace(srcBaseURL, "http://", "", 1), "https://", "", 1) credentialsFile := makeCredentialsFile(fmt.Sprintf(`{"%s":{"username": "test", "password": "test"}}`, registryName)) @@ -1379,22 +1017,14 @@ func TestSyncBasicAuth(t *testing.T) { OnDemand: false, } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil // add file path to the credentials - destConfig.Extensions.Sync = &sync.Config{CredentialsFile: credentialsFile, + syncConfig := &sync.Config{CredentialsFile: credentialsFile, Registries: []sync.RegistryConfig{unreacheableSyncRegistryConfig1, unreacheableSyncRegistryConfig2, syncRegistryConfig}} - dc := api.NewController(destConfig) - - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() @@ -1402,51 +1032,42 @@ func TestSyncBasicAuth(t *testing.T) { time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(destBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - var srcTagsList TagsList var destTagsList TagsList - resp, _ := resty.R().SetBasicAuth("test", "test").Get(srcBaseURL + "/v2/" + testImage + "/tags/list") + resp, _ := srcClient.R().SetBasicAuth("test", "test").Get(srcBaseURL + "/v2/" + testImage + "/tags/list") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 200) - err = json.Unmarshal(resp.Body(), &srcTagsList) + err := json.Unmarshal(resp.Body(), &srcTagsList) if err != nil { panic(err) } - resp, err = resty.R().Get(destBaseURL + "/v2/" + "inexistent" + "/manifests/" + testImageTag) + resp, err = destClient.R().Get(destBaseURL + "/v2/" + "inexistent" + "/manifests/" + testImageTag) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 404) - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "inexistent") + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "inexistent") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 404) - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) err = dc.StoreController.DefaultStore.DeleteImageManifest(testImage, testImageTag) So(err, ShouldBeNil) - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "1.1.1") + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "1.1.1") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 404) - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 200) - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") if err != nil { panic(err) } @@ -1456,12 +1077,10 @@ func TestSyncBasicAuth(t *testing.T) { panic(err) } - if eq := reflect.DeepEqual(destTagsList.Tags, srcTagsList.Tags); eq == false { - panic(errSync) - } + So(destTagsList, ShouldResemble, srcTagsList) Convey("Test sync on POST request on /sync", func() { - resp, _ := resty.R().Post(destBaseURL + "/sync") + resp, _ := destClient.R().Post(destBaseURL + "/sync") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 200) }) @@ -1469,25 +1088,10 @@ func TestSyncBasicAuth(t *testing.T) { }) } -func TestSyncBadUrl(t *testing.T) { +func TestSyncBadURL(t *testing.T) { Convey("Verify sync with bad url", t, func() { updateDuration, _ := time.ParseDuration("1h") - destPort := GetFreePort() - destBaseURL := GetBaseURL(destPort) - - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destConfig.Storage.RootDirectory = destDir - regex := ".*" var semver bool var tlsVerify bool @@ -1508,17 +1112,10 @@ func TestSyncBadUrl(t *testing.T) { CertDir: "", } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} - dc := api.NewController(destConfig) + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() @@ -1526,17 +1123,8 @@ func TestSyncBadUrl(t *testing.T) { time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(destBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - Convey("Test sync on POST request on /sync", func() { - resp, _ := resty.R().Post(destBaseURL + "/sync") + resp, _ := destClient.R().Post(destBaseURL + "/sync") So(resp, ShouldNotBeNil) So(string(resp.Body()), ShouldContainSubstring, "unsupported protocol scheme") So(resp.StatusCode(), ShouldEqual, 500) @@ -1548,65 +1136,15 @@ func TestSyncNoImagesByRegex(t *testing.T) { Convey("Verify sync with no images on source based on regex", t, func() { updateDuration, _ := time.ParseDuration("1h") - srcPort := GetFreePort() - srcBaseURL := GetBaseURL(srcPort) - - srcConfig := config.New() - srcConfig.HTTP.Port = srcPort - - srcDir, err := ioutil.TempDir("", "oci-src-repo-test") - if err != nil { - panic(err) - } - - err = CopyFiles("../../../test/data", srcDir) - if err != nil { - panic(err) - } - + sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false) defer os.RemoveAll(srcDir) - srcConfig.Storage.RootDirectory = srcDir - - sc := api.NewController(srcConfig) - - go func() { - // this blocks - if err := sc.Run(); err != nil { - return - } - }() - defer func() { ctx := context.Background() _ = sc.Server.Shutdown(ctx) time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(srcBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - - destPort := GetFreePort() - destBaseURL := GetBaseURL(destPort) - - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destConfig.Storage.RootDirectory = destDir - regex := "9.9.9" var tlsVerify bool @@ -1625,18 +1163,10 @@ func TestSyncNoImagesByRegex(t *testing.T) { CertDir: "", } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} - dc := api.NewController(destConfig) - - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() @@ -1644,25 +1174,16 @@ func TestSyncNoImagesByRegex(t *testing.T) { time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(destBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - Convey("Test sync on POST request on /sync", func() { - resp, _ := resty.R().Post(destBaseURL + "/sync") + resp, err := destClient.R().Post(destBaseURL + "/sync") + So(err, ShouldBeNil) So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 200) - So(err, ShouldBeNil) - resp, err := resty.R().Get(destBaseURL + "/v2/_catalog") - if err != nil { - panic(err) - } + resp, err = destClient.R().Get(destBaseURL + "/v2/_catalog") + So(err, ShouldBeNil) + So(resp, ShouldNotBeEmpty) + So(resp.StatusCode(), ShouldEqual, 200) var c catalog err = json.Unmarshal(resp.Body(), &c) @@ -1679,65 +1200,15 @@ func TestSyncInvalidRegex(t *testing.T) { Convey("Verify sync with invalid regex", t, func() { updateDuration, _ := time.ParseDuration("1h") - srcPort := GetFreePort() - srcBaseURL := GetBaseURL(srcPort) - - srcConfig := config.New() - srcConfig.HTTP.Port = srcPort - - srcDir, err := ioutil.TempDir("", "oci-src-repo-test") - if err != nil { - panic(err) - } - - err = CopyFiles("../../../test/data", srcDir) - if err != nil { - panic(err) - } - + sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false) defer os.RemoveAll(srcDir) - srcConfig.Storage.RootDirectory = srcDir - - sc := api.NewController(srcConfig) - - go func() { - // this blocks - if err := sc.Run(); err != nil { - return - } - }() - defer func() { ctx := context.Background() _ = sc.Server.Shutdown(ctx) time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(srcBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - - destPort := GetFreePort() - destBaseURL := GetBaseURL(destPort) - - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destConfig.Storage.RootDirectory = destDir - regex := "[" var tlsVerify bool @@ -1756,18 +1227,10 @@ func TestSyncInvalidRegex(t *testing.T) { CertDir: "", } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} - dc := api.NewController(destConfig) - - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() @@ -1775,17 +1238,8 @@ func TestSyncInvalidRegex(t *testing.T) { time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(destBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - Convey("Test sync on POST request on /sync", func() { - resp, _ := resty.R().Post(destBaseURL + "/sync") + resp, _ := destClient.R().Post(destBaseURL + "/sync") So(resp, ShouldNotBeNil) So(string(resp.Body()), ShouldContainSubstring, "error parsing regexp") So(resp.StatusCode(), ShouldEqual, 500) @@ -1797,50 +1251,15 @@ func TestSyncNotSemver(t *testing.T) { Convey("Verify sync feature semver compliant", t, func() { updateDuration, _ := time.ParseDuration("30m") - srcPort := GetFreePort() - srcBaseURL := GetBaseURL(srcPort) - - srcConfig := config.New() - srcConfig.HTTP.Port = srcPort - - srcDir, err := ioutil.TempDir("", "oci-src-repo-test") - if err != nil { - panic(err) - } - + sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false) defer os.RemoveAll(srcDir) - err = CopyFiles("../../../test/data", srcDir) - if err != nil { - panic(err) - } - - srcConfig.Storage.RootDirectory = srcDir - - sc := api.NewController(srcConfig) - - go func() { - // this blocks - if err := sc.Run(); err != nil { - return - } - }() - defer func() { ctx := context.Background() _ = sc.Server.Shutdown(ctx) time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(srcBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - // get manifest so we can update it with a semver non compliant tag resp, err := resty.R().Get(srcBaseURL + "/v2/zot-test/manifests/0.0.1") So(err, ShouldBeNil) @@ -1856,21 +1275,6 @@ func TestSyncNotSemver(t *testing.T) { So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 201) - destPort := GetFreePort() - destBaseURL := GetBaseURL(destPort) - - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destConfig.Storage.RootDirectory = destDir - semver := true var tlsVerify bool @@ -1889,18 +1293,10 @@ func TestSyncNotSemver(t *testing.T) { CertDir: "", } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} - dc := api.NewController(destConfig) - - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() @@ -1908,26 +1304,16 @@ func TestSyncNotSemver(t *testing.T) { time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(destBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - Convey("Test sync on POST request on /sync", func() { - resp, _ := resty.R().Post(destBaseURL + "/sync") + resp, _ := destClient.R().Post(destBaseURL + "/sync") So(resp, ShouldNotBeNil) So(resp.StatusCode(), ShouldEqual, 200) var destTagsList TagsList - resp, err = resty.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") - if err != nil { - panic(err) - } + resp, err = destClient.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) err = json.Unmarshal(resp.Body(), &destTagsList) if err != nil { @@ -1942,87 +1328,17 @@ func TestSyncNotSemver(t *testing.T) { func TestSyncInvalidCerts(t *testing.T) { Convey("Verify sync with bad certs", t, func() { - caCert, err := ioutil.ReadFile(CACert) - So(err, ShouldBeNil) - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - - client := resty.New() - - client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool}) - defer func() { client.SetTLSClientConfig(nil) }() updateDuration, _ := time.ParseDuration("1h") - srcPort := GetFreePort() - srcBaseURL := GetSecureBaseURL(srcPort) - - srcConfig := config.New() - srcConfig.HTTP.Port = srcPort - - srcConfig.HTTP.TLS = &config.TLSConfig{ - Cert: ServerCert, - Key: ServerKey, - CACert: CACert, - } - - srcDir, err := ioutil.TempDir("", "oci-src-repo-test") - if err != nil { - panic(err) - } - + sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(true, false) defer os.RemoveAll(srcDir) - err = CopyFiles("../../../test/data", srcDir) - if err != nil { - panic(err) - } - - srcConfig.Storage.RootDirectory = srcDir - - sc := api.NewController(srcConfig) - - go func() { - // this blocks - if err := sc.Run(); err != nil { - return - } - }() - defer func() { ctx := context.Background() _ = sc.Server.Shutdown(ctx) time.Sleep(500 * time.Millisecond) }() - cert, err := tls.LoadX509KeyPair("../../../test/data/client.cert", "../../../test/data/client.key") - if err != nil { - panic(err) - } - - client.SetCertificates(cert) - // wait till ready - for { - _, err := client.R().Get(srcBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - - destPort := GetFreePort() - destBaseURL := GetBaseURL(destPort) - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destConfig.Storage.RootDirectory = destDir - // copy client certs, use them in sync config clientCertDir, err := ioutil.TempDir("", "certs") if err != nil { @@ -2074,18 +1390,10 @@ func TestSyncInvalidCerts(t *testing.T) { CertDir: clientCertDir, } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} - dc := api.NewController(destConfig) - - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() @@ -2093,17 +1401,8 @@ func TestSyncInvalidCerts(t *testing.T) { time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(destBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - Convey("Test sync on POST request on /sync", func() { - resp, _ := resty.R().Post(destBaseURL + "/sync") + resp, _ := destClient.R().Post(destBaseURL + "/sync") So(resp, ShouldNotBeNil) So(string(resp.Body()), ShouldContainSubstring, "bad certificate") So(resp.StatusCode(), ShouldEqual, 500) @@ -2128,22 +1427,6 @@ func makeCredentialsFile(fileContent string) string { func TestSyncInvalidUrl(t *testing.T) { Convey("Verify sync invalid url", t, func() { updateDuration, _ := time.ParseDuration("30m") - - destPort := GetFreePort() - destBaseURL := GetBaseURL(destPort) - - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destConfig.Storage.RootDirectory = destDir - regex := ".*" var semver bool var tlsVerify bool @@ -2166,102 +1449,36 @@ func TestSyncInvalidUrl(t *testing.T) { OnDemand: true, } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} - dc := api.NewController(destConfig) - - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() _ = dc.Server.Shutdown(ctx) - time.Sleep(1 * time.Second) + time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(destBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - - resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + resp, err := destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 404) }) } func TestSyncInvalidTags(t *testing.T) { - Convey("Verify sync invalid url", t, func() { + Convey("Verify sync invalid tags", t, func() { updateDuration, _ := time.ParseDuration("30m") - srcPort := GetFreePort() - srcBaseURL := GetBaseURL(srcPort) - - srcConfig := config.New() - srcConfig.HTTP.Port = srcPort - - srcDir, err := ioutil.TempDir("", "oci-src-repo-test") - if err != nil { - panic(err) - } - + sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false) defer os.RemoveAll(srcDir) - err = CopyFiles("../../../test/data", srcDir) - if err != nil { - panic(err) - } - - srcConfig.Storage.RootDirectory = srcDir - - sc := api.NewController(srcConfig) - - go func() { - // this blocks - if err := sc.Run(); err != nil { - return - } - }() - defer func() { ctx := context.Background() _ = sc.Server.Shutdown(ctx) + time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(srcBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - - destPort := GetFreePort() - destBaseURL := GetBaseURL(destPort) - - destConfig := config.New() - destConfig.HTTP.Port = destPort - - destDir, err := ioutil.TempDir("", "oci-dest-repo-test") - if err != nil { - panic(err) - } - - defer os.RemoveAll(destDir) - - destConfig.Storage.RootDirectory = destDir - regex := ".*" var semver bool var tlsVerify bool @@ -2284,35 +1501,19 @@ func TestSyncInvalidTags(t *testing.T) { OnDemand: true, } - destConfig.Extensions = &extconf.ExtensionConfig{} - destConfig.Extensions.Search = nil - destConfig.Extensions.Sync = &sync.Config{ + syncConfig := &sync.Config{ Registries: []sync.RegistryConfig{syncRegistryConfig}} - dc := api.NewController(destConfig) - - go func() { - // this blocks - if err := dc.Run(); err != nil { - return - } - }() + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) defer func() { ctx := context.Background() _ = dc.Server.Shutdown(ctx) + time.Sleep(500 * time.Millisecond) }() - // wait till ready - for { - _, err := resty.R().Get(destBaseURL) - if err == nil { - break - } - time.Sleep(100 * time.Millisecond) - } - - resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "invalid:tag") + resp, err := destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + "invalid:tag") So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 404) }) diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go index 51f769aa..b8378edd 100644 --- a/pkg/extensions/sync/utils.go +++ b/pkg/extensions/sync/utils.go @@ -11,6 +11,7 @@ import ( "github.com/anuvu/zot/pkg/extensions/monitoring" "github.com/anuvu/zot/pkg/log" "github.com/anuvu/zot/pkg/storage" + glob "github.com/bmatcuk/doublestar/v4" "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/types" ispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -42,44 +43,26 @@ func parseRepositoryReference(input string) (reference.Named, error) { } // filterRepos filters repos based on prefix given in the config. -func filterRepos(repos []string, content []Content) map[int][]string { - // prefix: repo +func filterRepos(repos []string, content []Content, log log.Logger) map[int][]string { filtered := make(map[int][]string) for _, repo := range repos { - matched := false - // we use contentID to figure out tags filtering for contentID, c := range content { - // handle prefixes starting with '/' var prefix string + // handle prefixes starting with '/' if strings.HasPrefix(c.Prefix, "/") { prefix = c.Prefix[1:] } else { prefix = c.Prefix } - // split both prefix and repository and compare each part - splittedPrefix := strings.Split(prefix, "/") - // split at most n + 1 - splittedRepo := strings.SplitN(repo, "/", len(splittedPrefix)+1) - - // if prefix is longer than a repository, no match - if len(splittedPrefix) > len(splittedRepo) { + matched, err := glob.Match(prefix, repo) + if err != nil { + log.Error().Err(err).Str("pattern", + prefix).Msg("error while parsing glob pattern, skipping it...") continue } - // check if matched each part of prefix and repository - for i := 0; i < len(splittedPrefix); i++ { - if splittedRepo[i] == splittedPrefix[i] { - matched = true - } else { - // if a part doesn't match, check next prefix - matched = false - break - } - } - - // if matched no need to check the next prefixes if matched { filtered[contentID] = append(filtered[contentID], repo) break