Disable sync periodically polling when pollInterval is not configured

Filtering out sync on demand images based on content configuration

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

examples/README.md

@@ -357,10 +357,10 @@ Configure each registry sync:
 			"registries": [{
 				"url": "https://registry1:5000",
 				"onDemand": false,                  # pull any image which the local registry doesn't have
-				"pollInterval": "6h",               # polling interval
+				"pollInterval": "6h",               # polling interval, if not set then periodically polling will not run
 				"tlsVerify": true,                  # whether or not to verify tls
 				"certDir": "/home/user/certs",      # use certificates at certDir path, if not specified then use the default certs dir
-				"content":[                         # which content to periodically pull
+				"content":[                         # which content to periodically pull, also it's used for filtering ondemand images, if not set then periodically polling will not run
 					{
 						"prefix":"/repo1/repo",         # pull image repo1/repo
 						"tags":{                        # filter by tags

pkg/cli/root_test.go

@@ -93,6 +93,37 @@ func TestVerify(t *testing.T) {
 		So(func() { _ = cli.NewRootCmd().Execute() }, ShouldPanic)
 	})
 
+	Convey("Test verify storage driver different than s3", t, func(c C) {
+		tmpfile, err := ioutil.TempFile("", "zot-test*.json")
+		So(err, ShouldBeNil)
+		defer os.Remove(tmpfile.Name()) // clean up
+		content := []byte(`{"storage":{"rootDirectory":"/tmp/zot", "storageDriver": {"name": "gcs"}},
+							"http":{"address":"127.0.0.1","port":"8080","realm":"zot",
+							"auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`)
+		_, err = tmpfile.Write(content)
+		So(err, ShouldBeNil)
+		err = tmpfile.Close()
+		So(err, ShouldBeNil)
+		os.Args = []string{"cli_test", "verify", tmpfile.Name()}
+		So(func() { _ = cli.NewRootCmd().Execute() }, ShouldPanic)
+	})
+
+	Convey("Test verify subpath storage driver different than s3", t, func(c C) {
+		tmpfile, err := ioutil.TempFile("", "zot-test*.json")
+		So(err, ShouldBeNil)
+		defer os.Remove(tmpfile.Name()) // clean up
+		content := []byte(`{"storage":{"rootDirectory":"/tmp/zot", "storageDriver": {"name": "s3"},
+							"subPaths": {"/a": {"rootDirectory": "/zot-a","storageDriver": {"name": "gcs"}}}},
+							"http":{"address":"127.0.0.1","port":"8080","realm":"zot",
+							"auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`)
+		_, err = tmpfile.Write(content)
+		So(err, ShouldBeNil)
+		err = tmpfile.Close()
+		So(err, ShouldBeNil)
+		os.Args = []string{"cli_test", "verify", tmpfile.Name()}
+		So(func() { _ = cli.NewRootCmd().Execute() }, ShouldPanic)
+	})
+
 	Convey("Test verify w/ authorization and w/o authentication", t, func(c C) {
 		tmpfile, err := ioutil.TempFile("", "zot-test*.json")
 		So(err, ShouldBeNil)

pkg/extensions/extensions.go

@@ -72,19 +72,6 @@ func EnableExtensions(config *config.Config, log log.Logger, rootDir string) {
 func EnableSyncExtension(config *config.Config, wg *goSync.WaitGroup,
 	storeController storage.StoreController, log log.Logger) {
 	if config.Extensions.Sync != nil {
-		defaultPollInterval, _ := time.ParseDuration("1h")
-		for id, registryCfg := range config.Extensions.Sync.Registries {
-			if registryCfg.Content != nil &&
-				len(registryCfg.Content) > 0 &&
-				registryCfg.PollInterval < defaultPollInterval {
-				config.Extensions.Sync.Registries[id].PollInterval = defaultPollInterval
-
-				log.Warn().Str("registry", registryCfg.URL).
-					Msg("Sync registries interval set to too-short interval < 1h," +
-						"changing update duration to 1 hour and continuing.")
-			}
-		}
-
 		if err := sync.Run(*config.Extensions.Sync, storeController, wg, log); err != nil {
 			log.Error().Err(err).Msg("Error encountered while setting up syncing")
 		}

pkg/extensions/sync/http_handler.go

@@ -48,8 +48,15 @@ func (h *PostHandler) Handler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	for _, regCfg := range h.Cfg.Registries {
+		// if content not provided, don't run periodically sync
 		if len(regCfg.Content) == 0 {
-			h.Log.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL)
+			h.Log.Info().Msgf("sync config content not configured for %s, will not run periodically sync", regCfg.URL)
+			continue
+		}
+
+		// if pollInterval is not provided, don't run periodically sync
+		if regCfg.PollInterval == 0 {
+			h.Log.Warn().Msgf("sync config PollInterval not configured for %s, will not run periodically sync", regCfg.URL)
 			continue
 		}
 

pkg/extensions/sync/on_demand.go

@@ -51,6 +51,16 @@ func OneImage(cfg Config, storeController storage.StoreController,
 			continue
 		}
 
+		// if content config is not specified, then don't filter, just sync demanded image
+		if len(regCfg.Content) != 0 {
+			repos := filterRepos([]string{repo}, regCfg.Content, log)
+			if len(repos) == 0 {
+				log.Info().Msgf("skipping syncing on demand %s from %s registry because it's filtered out by content config",
+					repo, regCfg.URL)
+				continue
+			}
+		}
+
 		registryConfig := regCfg
 		log.Info().Msgf("syncing on demand with %s", registryConfig.URL)
 

pkg/extensions/sync/sync.go

@@ -464,15 +464,21 @@ func Run(cfg Config, storeController storage.StoreController, wg *goSync.WaitGro
 
 	// for each upstream registry, start a go routine.
 	for _, regCfg := range cfg.Registries {
+		// if content not provided, don't run periodically sync
 		if len(regCfg.Content) == 0 {
-			logger.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL)
+			logger.Info().Msgf("sync config content not configured for %s, will not run periodically sync", regCfg.URL)
+			continue
+		}
+
+		// if pollInterval is not provided, don't run periodically sync
+		if regCfg.PollInterval == 0 {
+			logger.Warn().Msgf("sync config PollInterval not configured for %s, will not run periodically sync", regCfg.URL)
 			continue
 		}
 
 		// increment reference since will be busy, so shutdown has to wait
 		wg.Add(1)
 
-		// schedule each registry sync
 		ticker := time.NewTicker(regCfg.PollInterval)
 
 		// fork a new zerolog child to avoid data race
@@ -480,6 +486,7 @@ func Run(cfg Config, storeController storage.StoreController, wg *goSync.WaitGro
 
 		upstreamRegistry := strings.Replace(strings.Replace(regCfg.URL, "http://", "", 1), "https://", "", 1)
 
+		// schedule each registry sync
 		go func(regCfg RegistryConfig, l log.Logger) {
 			// run on intervals
 			for ; true; <-ticker.C {

pkg/extensions/sync/sync_test.go

@@ -228,8 +228,6 @@ func startDownstreamServer(secure bool, syncConfig *sync.Config) (*api.Controlle
 
 func TestSyncOnDemand(t *testing.T) {
 	Convey("Verify sync on demand feature", t, func() {
-		updateDuration, _ := time.ParseDuration("30m")
-
 		sc, srcBaseURL, srcDir, _, srcClient := startUpstreamServer(false, false)
 		defer os.RemoveAll(srcDir)
 
@@ -237,26 +235,25 @@ func TestSyncOnDemand(t *testing.T) {
 			sc.Shutdown()
 		}()
 
-		regex := ".*"
-		var semver bool
 		var tlsVerify bool
 
+		regex := ".*"
+		semver := true
+
 		syncRegistryConfig := sync.RegistryConfig{
 			Content: []sync.Content{
 				{
-					// won't match any image on source registry, we will sync on demand
-					Prefix: "dummy",
+					Prefix: testImage,
 					Tags: &sync.Tags{
 						Regex:  &regex,
 						Semver: &semver,
 					},
 				},
 			},
-			URL:          srcBaseURL,
-			PollInterval: updateDuration,
-			TLSVerify:    &tlsVerify,
-			CertDir:      "",
-			OnDemand:     true,
+			URL:       srcBaseURL,
+			TLSVerify: &tlsVerify,
+			CertDir:   "",
+			OnDemand:  true,
 		}
 
 		syncConfig := &sync.Config{
@@ -1621,3 +1618,193 @@ func TestSyncSubPaths(t *testing.T) {
 		So(err, ShouldNotBeNil)
 	})
 }
+
+func TestSyncOnDemandContentFiltering(t *testing.T) {
+	Convey("Verify sync on demand feature", t, func() {
+		sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false)
+		defer os.RemoveAll(srcDir)
+
+		defer func() {
+			sc.Shutdown()
+		}()
+
+		Convey("Test image is filtered out by content", func() {
+			regex := ".*"
+			var semver bool
+			var tlsVerify bool
+
+			syncRegistryConfig := sync.RegistryConfig{
+				Content: []sync.Content{
+					{
+						//should be filtered out
+						Prefix: "dummy",
+						Tags: &sync.Tags{
+							Regex:  &regex,
+							Semver: &semver,
+						},
+					},
+				},
+				URL:       srcBaseURL,
+				TLSVerify: &tlsVerify,
+				CertDir:   "",
+				OnDemand:  true,
+			}
+
+			syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}}
+
+			dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig)
+			defer os.RemoveAll(destDir)
+
+			defer func() {
+				dc.Shutdown()
+			}()
+
+			resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+		})
+
+		Convey("Test image is not filtered out by content", func() {
+			regex := ".*"
+			semver := true
+			var tlsVerify bool
+
+			syncRegistryConfig := sync.RegistryConfig{
+				Content: []sync.Content{
+					{
+						// will sync on demand, should not be filtered out
+						Prefix: testImage,
+						Tags: &sync.Tags{
+							Regex:  &regex,
+							Semver: &semver,
+						},
+					},
+				},
+				URL:       srcBaseURL,
+				TLSVerify: &tlsVerify,
+				CertDir:   "",
+				OnDemand:  true,
+			}
+
+			syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}}
+
+			dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig)
+			defer os.RemoveAll(destDir)
+
+			defer func() {
+				dc.Shutdown()
+			}()
+
+			resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+		})
+	})
+}
+
+func TestSyncConfigRules(t *testing.T) {
+	Convey("Verify sync config rules", t, func() {
+		sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false)
+		defer os.RemoveAll(srcDir)
+
+		defer func() {
+			sc.Shutdown()
+		}()
+
+		Convey("Test periodically sync is disabled when pollInterval is not set", func() {
+			regex := ".*"
+			var semver bool
+			var tlsVerify bool
+
+			syncRegistryConfig := sync.RegistryConfig{
+				Content: []sync.Content{
+					{
+						Prefix: testImage,
+						Tags: &sync.Tags{
+							Regex:  &regex,
+							Semver: &semver,
+						},
+					},
+				},
+				URL:       srcBaseURL,
+				TLSVerify: &tlsVerify,
+				CertDir:   "",
+				OnDemand:  false,
+			}
+
+			syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}}
+
+			dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig)
+			defer os.RemoveAll(destDir)
+
+			defer func() {
+				dc.Shutdown()
+			}()
+
+			// trigger sync, this way we can be sure periodically sync ran
+			resp, _ := resty.R().Post(destBaseURL + "/sync")
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+
+			// image should not be synced
+			resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+		})
+
+		Convey("Test periodically sync is disabled when content is not set", func() {
+			var tlsVerify bool
+			updateDuration, _ := time.ParseDuration("30m")
+
+			syncRegistryConfig := sync.RegistryConfig{
+				PollInterval: updateDuration,
+				URL:          srcBaseURL,
+				TLSVerify:    &tlsVerify,
+				CertDir:      "",
+				OnDemand:     false,
+			}
+
+			syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}}
+
+			dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig)
+			defer os.RemoveAll(destDir)
+
+			defer func() {
+				dc.Shutdown()
+			}()
+
+			// trigger sync, this way we can be sure periodically sync ran
+			resp, _ := resty.R().Post(destBaseURL + "/sync")
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+
+			resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+		})
+
+		Convey("Test ondemand sync is disabled when ondemand is false", func() {
+			var tlsVerify bool
+
+			syncRegistryConfig := sync.RegistryConfig{
+				URL:       srcBaseURL,
+				TLSVerify: &tlsVerify,
+				CertDir:   "",
+				OnDemand:  false,
+			}
+
+			syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}}
+
+			dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig)
+			defer os.RemoveAll(destDir)
+
+			defer func() {
+				dc.Shutdown()
+			}()
+
+			resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+		})
+	})
+}