Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2025-02-17 23:45:36 -05:00
Code Issues Activity

Sync prefix can be an exact match or a glob pattern, closes #297

Browse source 
Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
Petu Eusebiu 2021-11-25 14:04:39 +02:00 • committed by Ramkumar Chinchani
parent f0ef10fa50
commit fff6107310
11 changed files with 400 additions and 1139 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
examples/README.md
View file

@ -358,18 +358,21 @@ Configure each registry sync:
"url": "https://registry1:5000",
"onDemand": false, # pull any image which the local registry doesn't have
"pollInterval": "6h", # polling interval
"tlsVerify": true, # wheather or not to verify tls
"tlsVerify": true, # whether or not to verify tls
"certDir": "/home/user/certs", # use certificates at certDir path, if not specified then use the default certs dir
"content":[ # which content to periodically pull
{
"prefix":"/repo1/repo", # pull all images under /repo1/repo
"prefix":"/repo1/repo", # pull image repo1/repo
"tags":{ # filter by tags
"regex":"4.*", # filter tags by regex
"semver":true # filter tags by semver compliance
}
},
{
"prefix":"/repo2/repo" # pull all images under /repo2/repo
"prefix":"/repo2/repo*" # pull all images that matches repo2/repo.*
},
{
"prefix":"/repo3/**" # pull all images under repo3/ (matches recursively all repos under repo3/)
}
]
},
@ -396,3 +399,4 @@ Configure each registry sync:
}
```
Prefixes can be strings that exactly match repositories or they can be glob patterns.

1
go.mod
View file

@ -10,6 +10,7 @@ require (
github.com/apex/log v1.9.0
github.com/aquasecurity/trivy v0.0.0-00010101000000-000000000000
github.com/aquasecurity/trivy-db v0.0.0-20210916043317-726b7b72a47b
github.com/bmatcuk/doublestar/v4 v4.0.2
github.com/briandowns/spinner v1.16.0
github.com/chartmuseum/auth v0.5.0
github.com/containerd/containerd v1.5.8 // indirect

2
go.sum
View file

@ -304,6 +304,8 @@ github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0=
github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE=
github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA=
github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/bombsimon/wsl/v2 v2.0.0/go.mod h1:mf25kr/SqFEPhhcxW1+7pxzGlW+hIl/hYTKY95VwV8U=
github.com/bombsimon/wsl/v2 v2.2.0/go.mod h1:Azh8c3XGEJl9LyX0/sFC+CKMc7Ssgua0g+6abzXN4Pg=

16
pkg/cli/root.go
View file

@ -5,6 +5,7 @@ import (
"github.com/anuvu/zot/pkg/api"
"github.com/anuvu/zot/pkg/api/config"
"github.com/anuvu/zot/pkg/storage"
glob "github.com/bmatcuk/doublestar/v4"
"github.com/fsnotify/fsnotify"
"github.com/mitchellh/mapstructure"
distspec "github.com/opencontainers/distribution-spec/specs-go"
@ -188,6 +189,21 @@ func LoadConfiguration(config *config.Config, configPath string) {
}
}
// check glob patterns in sync are compilable
if config.Extensions != nil && config.Extensions.Sync != nil {
for _, regCfg := range config.Extensions.Sync.Registries {
if regCfg.Content != nil {
for _, content := range regCfg.Content {
ok := glob.ValidatePattern(content.Prefix)
if !ok {
log.Error().Err(glob.ErrBadPattern).Str("pattern", content.Prefix).Msg("pattern could not be compiled")
panic(errors.ErrBadConfig)
}
}
}
}
}
// enforce s3 driver on subpaths in case of using storage driver
if config.Storage.SubPaths != nil {
if len(config.Storage.SubPaths) > 0 {

17
pkg/cli/root_test.go
View file

@ -134,6 +134,23 @@ func TestVerify(t *testing.T) {
So(func() { _ = cli.NewRootCmd().Execute() }, ShouldPanic)
})
Convey("Test verify with bad sync prefixes", t, func(c C) {
tmpfile, err := ioutil.TempFile("", "zot-test*.json")
So(err, ShouldBeNil)
defer os.Remove(tmpfile.Name()) // clean up
content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"},
"http":{"address":"127.0.0.1","port":"8080","realm":"zot",
"auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}},
"extensions":{"sync": {"registries": [{"url":"localhost:9999",
"content": [{"prefix":"[repo%^&"}]}]}}}`)
_, err = tmpfile.Write(content)
So(err, ShouldBeNil)
err = tmpfile.Close()
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "verify", tmpfile.Name()}
So(func() { _ = cli.NewRootCmd().Execute() }, ShouldPanic)
})
Convey("Test verify good config", t, func(c C) {
tmpfile, err := ioutil.TempFile("", "zot-test*.json")
So(err, ShouldBeNil)

8
pkg/extensions/extensions.go
View file

@ -72,10 +72,14 @@ func EnableSyncExtension(config *config.Config, log log.Logger, storeController
if config.Extensions.Sync != nil {
defaultPollInterval, _ := time.ParseDuration("1h")
for id, registryCfg := range config.Extensions.Sync.Registries {
if registryCfg.PollInterval < defaultPollInterval {
if registryCfg.Content != nil &&
len(registryCfg.Content) > 0 &&
registryCfg.PollInterval < defaultPollInterval {
config.Extensions.Sync.Registries[id].PollInterval = defaultPollInterval
log.Warn().Msg("Sync registries interval set to too-short interval < 1h, changing update duration to 1 hour and continuing.") // nolint: lll
log.Warn().Str("registry", registryCfg.URL).
Msg("Sync registries interval set to too-short interval < 1h," +
"changing update duration to 1 hour and continuing.")
}
}

5
pkg/extensions/sync/http_handler.go
View file

@ -48,6 +48,11 @@ func (h *PostHandler) Handler(w http.ResponseWriter, r *http.Request) {
}
for _, regCfg := range h.Cfg.Registries {
if len(regCfg.Content) == 0 {
h.Log.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL)
continue
}
upstreamRegistryName := strings.Replace(strings.Replace(regCfg.URL, "http://", "", 1), "https://", "", 1)
if err := syncRegistry(regCfg, h.StoreController, h.Log, localCtx, policyCtx,

14
pkg/extensions/sync/sync.go
View file

@ -310,11 +310,6 @@ func getUpstreamContext(regCfg *RegistryConfig, credentials Credentials) *types.
func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController,
log log.Logger, localCtx *types.SystemContext,
policyCtx *signature.PolicyContext, credentials Credentials, uuid string) error {
if len(regCfg.Content) == 0 {
log.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL)
return nil
}
log.Info().Msgf("syncing registry: %s", regCfg.URL)
var err error
@ -341,9 +336,9 @@ func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController
upstreamRegistryName := strings.Replace(strings.Replace(regCfg.URL, "http://", "", 1), "https://", "", 1)
log.Info().Msg("filtering repos based on sync prefixes")
log.Info().Msgf("filtering %d repos based on sync prefixes", len(catalog.Repositories))
repos := filterRepos(catalog.Repositories, regCfg.Content)
repos := filterRepos(catalog.Repositories, regCfg.Content, log)
log.Info().Msgf("got repos: %v", repos)
@ -467,6 +462,11 @@ func Run(cfg Config, storeController storage.StoreController, logger log.Logger)
// for each upstream registry, start a go routine.
for _, regCfg := range cfg.Registries {
if len(regCfg.Content) == 0 {
logger.Info().Msgf("no content found for %s, will not run periodically sync", regCfg.URL)
continue
}
// schedule each registry sync
ticker := time.NewTicker(regCfg.PollInterval)

28
pkg/extensions/sync/sync_internal_test.go
View file

@ -168,6 +168,34 @@ func TestSyncInternal(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("Test filterRepos()", t, func() {
repos := []string{"repo", "repo1", "repo2", "repo/repo2", "repo/repo2/repo3/repo4"}
contents := []Content{
{
Prefix: "repo",
},
{
Prefix: "/repo/**",
},
{
Prefix: "repo*",
},
}
filteredRepos := filterRepos(repos, contents, log.NewLogger("", ""))
So(filteredRepos[0], ShouldResemble, []string{"repo"})
So(filteredRepos[1], ShouldResemble, []string{"repo/repo2", "repo/repo2/repo3/repo4"})
So(filteredRepos[2], ShouldResemble, []string{"repo1", "repo2"})
contents = []Content{
{
Prefix: "[repo%#@",
},
}
filteredRepos = filterRepos(repos, contents, log.NewLogger("", ""))
So(len(filteredRepos), ShouldEqual, 0)
})
Convey("Verify pushSyncedLocalImage func", t, func() {
storageDir, err := ioutil.TempDir("", "oci-dest-repo-test")
if err != nil {

1339
pkg/extensions/sync/sync_test.go
View file

File diff suppressed because it is too large Load diff

31
pkg/extensions/sync/utils.go
View file

@ -11,6 +11,7 @@ import (
"github.com/anuvu/zot/pkg/extensions/monitoring"
"github.com/anuvu/zot/pkg/log"
"github.com/anuvu/zot/pkg/storage"
glob "github.com/bmatcuk/doublestar/v4"
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/types"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
@ -42,44 +43,26 @@ func parseRepositoryReference(input string) (reference.Named, error) {
}
// filterRepos filters repos based on prefix given in the config.
func filterRepos(repos []string, content []Content) map[int][]string {
// prefix: repo
func filterRepos(repos []string, content []Content, log log.Logger) map[int][]string {
filtered := make(map[int][]string)
for _, repo := range repos {
matched := false
// we use contentID to figure out tags filtering
for contentID, c := range content {
// handle prefixes starting with '/'
var prefix string
// handle prefixes starting with '/'
if strings.HasPrefix(c.Prefix, "/") {
prefix = c.Prefix[1:]
} else {
prefix = c.Prefix
}
// split both prefix and repository and compare each part
splittedPrefix := strings.Split(prefix, "/")
// split at most n + 1
splittedRepo := strings.SplitN(repo, "/", len(splittedPrefix)+1)
// if prefix is longer than a repository, no match
if len(splittedPrefix) > len(splittedRepo) {
matched, err := glob.Match(prefix, repo)
if err != nil {
log.Error().Err(err).Str("pattern",
prefix).Msg("error while parsing glob pattern, skipping it...")
continue
}
// check if matched each part of prefix and repository
for i := 0; i < len(splittedPrefix); i++ {
if splittedRepo[i] == splittedPrefix[i] {
matched = true
} else {
// if a part doesn't match, check next prefix
matched = false
break
}
}
// if matched no need to check the next prefixes
if matched {
filtered[contentID] = append(filtered[contentID], repo)
break