diff --git a/Makefile b/Makefile index 231d4217..23ae8f3d 100644 --- a/Makefile +++ b/Makefile @@ -386,12 +386,14 @@ test-bats-sync: BUILD_LABELS=sync test-bats-sync: binary binary-minimal bench check-skopeo $(BATS) $(NOTATION) $(COSIGN) $(BATS) --trace --print-output-on-failure test/blackbox/sync.bats $(BATS) --trace --print-output-on-failure test/blackbox/sync_docker.bats + $(BATS) --trace --print-output-on-failure test/blackbox/sync_replica_cluster.bats .PHONY: test-bats-sync-verbose test-bats-sync-verbose: BUILD_LABELS=sync test-bats-sync-verbose: binary binary-minimal bench check-skopeo $(BATS) $(NOTATION) $(COSIGN) $(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync.bats $(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_docker.bats + $(BATS) --trace -t -x -p --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/sync_replica_cluster.bats .PHONY: test-bats-cve test-bats-cve: BUILD_LABELS=search diff --git a/errors/errors.go b/errors/errors.go index fde86690..a07dc66b 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -108,4 +108,5 @@ var ( ErrInvalidTruststoreName = errors.New("signatures: invalid truststore name") ErrInvalidCertificateContent = errors.New("signatures: invalid certificate content") ErrInvalidStateCookie = errors.New("auth: state cookie not present or differs from original state") + ErrSyncNoURLsLeft = errors.New("sync: no valid registry urls left after filtering local ones") ) diff --git a/pkg/extensions/extension_sync.go b/pkg/extensions/extension_sync.go index f87f372b..716e2d35 100644 --- a/pkg/extensions/extension_sync.go +++ b/pkg/extensions/extension_sync.go @@ -4,7 +4,13 @@ package extensions import ( + "net" + "net/url" + "strings" + + zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/api/config" + syncconf "zotregistry.io/zot/pkg/extensions/config/sync" "zotregistry.io/zot/pkg/extensions/sync" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/repodb" @@ -19,6 +25,19 @@ func EnableSyncExtension(config *config.Config, repoDB repodb.RepoDB, onDemand := sync.NewOnDemand(log) for _, registryConfig := range config.Extensions.Sync.Registries { + registryConfig := registryConfig + if len(registryConfig.URLs) > 1 { + if err := removeSelfURLs(config, ®istryConfig, log); err != nil { + return nil, err + } + } + + if len(registryConfig.URLs) == 0 { + log.Error().Err(zerr.ErrSyncNoURLsLeft).Msg("unable to start sync extension") + + return nil, zerr.ErrSyncNoURLsLeft + } + isPeriodical := len(registryConfig.Content) != 0 && registryConfig.PollInterval != 0 isOnDemand := registryConfig.OnDemand @@ -49,3 +68,106 @@ func EnableSyncExtension(config *config.Config, repoDB repodb.RepoDB, return nil, nil //nolint: nilnil } + +func getLocalIPs() ([]string, error) { + var localIPs []string + + ifaces, err := net.Interfaces() + if err != nil { + return []string{}, err + } + + for _, i := range ifaces { + addrs, err := i.Addrs() + if err != nil { + return localIPs, err + } + + for _, addr := range addrs { + if localIP, ok := addr.(*net.IPNet); ok { + localIPs = append(localIPs, localIP.IP.String()) + } + } + } + + return localIPs, nil +} + +func getIPFromHostName(host string) ([]string, error) { + addrs, err := net.LookupIP(host) + if err != nil { + return []string{}, err + } + + ips := make([]string, 0, len(addrs)) + + for _, ip := range addrs { + ips = append(ips, ip.String()) + } + + return ips, nil +} + +func removeSelfURLs(config *config.Config, registryConfig *syncconf.RegistryConfig, log log.Logger) error { + // get IP from config + port := config.HTTP.Port + selfAddress := net.JoinHostPort(config.HTTP.Address, port) + + // get all local IPs from interfaces + localIPs, err := getLocalIPs() + if err != nil { + return err + } + + for idx := len(registryConfig.URLs) - 1; idx >= 0; idx-- { + registryURL := registryConfig.URLs[idx] + + url, err := url.Parse(registryURL) + if err != nil { + log.Error().Str("url", registryURL).Msg("failed to parse sync registry url, removing it") + + registryConfig.URLs = append(registryConfig.URLs[:idx], registryConfig.URLs[idx+1:]...) + + continue + } + + // check self address + if strings.Contains(registryURL, selfAddress) { + log.Info().Str("url", registryURL).Msg("removing local registry url") + + registryConfig.URLs = append(registryConfig.URLs[:idx], registryConfig.URLs[idx+1:]...) + + continue + } + + // check dns + ips, err := getIPFromHostName(url.Hostname()) + if err != nil { + // will not remove, maybe it will get resolved later after multiple retries + log.Warn().Str("url", registryURL).Msg("failed to lookup sync registry url's hostname") + + continue + } + + var removed bool + + for _, localIP := range localIPs { + // if ip resolved from hostname/dns is equal with any local ip + for _, ip := range ips { + if net.JoinHostPort(ip, url.Port()) == net.JoinHostPort(localIP, port) { + registryConfig.URLs = append(registryConfig.URLs[:idx], registryConfig.URLs[idx+1:]...) + + removed = true + + break + } + } + + if removed { + break + } + } + } + + return nil +} diff --git a/pkg/extensions/sync/service.go b/pkg/extensions/sync/service.go index 5477c55f..ee90dfc4 100644 --- a/pkg/extensions/sync/service.go +++ b/pkg/extensions/sync/service.go @@ -125,7 +125,7 @@ func (service *BaseService) SetNextAvailableClient() error { } if err != nil { - return err + continue } if !service.client.IsAvailable() { diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 9c6b1c2f..1d15bd17 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -802,6 +802,14 @@ func TestOnDemand(t *testing.T) { regex := ".*" semver := true + destPort := test.GetFreePort() + destConfig := config.New() + + destBaseURL := test.GetBaseURL(destPort) + + hostname, err := os.Hostname() + So(err, ShouldBeNil) + syncRegistryConfig := syncconf.RegistryConfig{ Content: []syncconf.Content{ { @@ -812,7 +820,11 @@ func TestOnDemand(t *testing.T) { }, }, }, - URLs: []string{srcBaseURL}, + // include self url, should be ignored + URLs: []string{ + fmt.Sprintf("http://%s", hostname), destBaseURL, + srcBaseURL, fmt.Sprintf("http://localhost:%s", destPort), + }, TLSVerify: &tlsVerify, CertDir: "", OnDemand: true, @@ -824,11 +836,6 @@ func TestOnDemand(t *testing.T) { Registries: []syncconf.RegistryConfig{syncRegistryConfig}, } - destPort := test.GetFreePort() - destConfig := config.New() - - destBaseURL := test.GetBaseURL(destPort) - destConfig.HTTP.Port = destPort destDir := t.TempDir() @@ -3384,7 +3391,7 @@ func TestMultipleURLs(t *testing.T) { }, }, }, - URLs: []string{"badURL", "http://invalid.invalid/invalid/", srcBaseURL}, + URLs: []string{"badURL", "@!#!$#@%", "http://invalid.invalid/invalid/", srcBaseURL}, PollInterval: updateDuration, TLSVerify: &tlsVerify, CertDir: "", @@ -3438,6 +3445,49 @@ func TestMultipleURLs(t *testing.T) { }) } +func TestNoURLsLeftInConfig(t *testing.T) { + Convey("Verify sync feature", t, func() { + updateDuration, _ := time.ParseDuration("30m") + + regex := ".*" + semver := true + var tlsVerify bool + + syncRegistryConfig := syncconf.RegistryConfig{ + Content: []syncconf.Content{ + { + Prefix: testImage, + Tags: &syncconf.Tags{ + Regex: ®ex, + Semver: &semver, + }, + }, + }, + URLs: []string{"@!#!$#@%", "@!#!$#@%"}, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: "", + } + + defaultVal := true + syncConfig := &syncconf.Config{ + Enable: &defaultVal, + Registries: []syncconf.RegistryConfig{syncRegistryConfig}, + } + + dctlr, destBaseURL, _, destClient := makeDownstreamServer(t, false, syncConfig) + + dcm := test.NewControllerManager(dctlr) + dcm.StartAndWait(dctlr.Config.HTTP.Port) + defer dcm.StopServer() + + resp, err := destClient.R().Get(destBaseURL + "/v2/" + testImage + "/tags/list") + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + }) +} + func TestPeriodicallySignaturesErr(t *testing.T) { Convey("Verify sync periodically signatures errors", t, func() { updateDuration, _ := time.ParseDuration("30m") diff --git a/test/blackbox/sync_replica_cluster.bats b/test/blackbox/sync_replica_cluster.bats new file mode 100644 index 00000000..6b4e2eb3 --- /dev/null +++ b/test/blackbox/sync_replica_cluster.bats @@ -0,0 +1,155 @@ +load helpers_sync + +function setup_file() { + # Verify prerequisites are available + if ! verify_prerequisites; then + exit 1 + fi + + # Download test data to folder common for the entire suite, not just this file + skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20 + # Setup zot server + local zot_sync_one_root_dir=${BATS_FILE_TMPDIR}/zot-one + local zot_sync_two_root_dir=${BATS_FILE_TMPDIR}/zot-two + + local zot_sync_one_config_file=${BATS_FILE_TMPDIR}/zot_sync_one_config.json + local zot_sync_two_config_file=${BATS_FILE_TMPDIR}/zot_sync_two_config.json + + mkdir -p ${zot_sync_one_root_dir} + mkdir -p ${zot_sync_two_root_dir} + + cat >${zot_sync_one_config_file} <${zot_sync_two_config_file} <