0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-30 22:34:13 -05:00

add enable/disable option for scrub extension (#827)

Signed-off-by: Andreea-Lupu <andreealupu1470@yahoo.com>
This commit is contained in:
Andreea Lupu 2022-09-28 04:06:50 +03:00 committed by GitHub
parent 18d17f5d4c
commit 5ef023dbc1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 96 additions and 5 deletions

View file

@ -46,6 +46,7 @@
}
},
"scrub": {
"enable": true,
"interval": "24h"
}
}

View file

@ -12,6 +12,7 @@
},
"extensions": {
"scrub": {
"enable": true,
"interval": "24h"
}
}

View file

@ -413,7 +413,38 @@ func TestServeScrubExtension(t *testing.T) {
defer func() { os.Args = oldArgs }()
Convey("scrub enabled by scrub interval param set", t, func(c C) {
Convey("scrub implicitly enabled", t, func(c C) {
content := `{
"storage": {
"rootDirectory": "%s"
},
"http": {
"address": "127.0.0.1",
"port": "%s"
},
"log": {
"level": "debug",
"output": "%s"
},
"extensions": {
"scrub": {
}
}
}`
logPath, err := runCLIWithConfig(t.TempDir(), content)
So(err, ShouldBeNil)
data, err := os.ReadFile(logPath)
So(err, ShouldBeNil)
defer os.Remove(logPath) // clean up
dataStr := string(data)
So(dataStr, ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":{\"Enable\":true,\"Interval\":86400000000000},\"Lint\":null") //nolint:lll // gofumpt conflicts with lll
So(dataStr, ShouldNotContainSubstring,
"Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.")
})
Convey("scrub implicitly enabled, but with scrub interval param set", t, func(c C) {
content := `{
"storage": {
"rootDirectory": "%s"
@ -441,12 +472,44 @@ func TestServeScrubExtension(t *testing.T) {
// Even if in config we specified scrub interval=1h, the minimum interval is 2h
dataStr := string(data)
So(dataStr, ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":{\"Interval\":3600000000000},\"Lint\":null") //nolint:lll // gofumpt conflicts with lll
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":{\"Enable\":true,\"Interval\":3600000000000},\"Lint\":null") //nolint:lll // gofumpt conflicts with lll
So(dataStr, ShouldContainSubstring,
"Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.")
})
Convey("scrub not enabled - scrub interval param not set", t, func(c C) {
Convey("scrub explicitly enabled, but without scrub interval param set", t, func(c C) {
content := `{
"storage": {
"rootDirectory": "%s"
},
"http": {
"address": "127.0.0.1",
"port": "%s"
},
"log": {
"level": "debug",
"output": "%s"
},
"extensions": {
"scrub": {
"enable": true
}
}
}`
logPath, err := runCLIWithConfig(t.TempDir(), content)
So(err, ShouldBeNil)
data, err := os.ReadFile(logPath)
So(err, ShouldBeNil)
defer os.Remove(logPath) // clean up
dataStr := string(data)
So(dataStr, ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":{\"Enable\":true,\"Interval\":86400000000000},\"Lint\":null") //nolint:lll // gofumpt conflicts with lll
So(dataStr, ShouldNotContainSubstring,
"Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.")
})
Convey("scrub explicitly disabled", t, func(c C) {
content := `{
"storage": {
"rootDirectory": "%s"
@ -461,6 +524,7 @@ func TestServeScrubExtension(t *testing.T) {
},
"extensions": {
"scrub": {
"enable": false
}
}
}`
@ -472,7 +536,7 @@ func TestServeScrubExtension(t *testing.T) {
defer os.Remove(logPath) // clean up
dataStr := string(data)
So(dataStr, ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}")
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":{\"Enable\":false,\"Interval\":86400000000000},\"Lint\":null}") //nolint:lll // gofumpt conflicts with lll
So(dataStr, ShouldContainSubstring, "Scrub config not provided, skipping scrub")
So(dataStr, ShouldNotContainSubstring,
"Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.")

View file

@ -344,6 +344,13 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper) {
// Note: In case search is not empty the config.Extensions will not be nil and we will not reach here
config.Extensions.Search = &extconf.SearchConfig{}
}
_, ok = extMap["scrub"]
if ok {
// we found a config like `"extensions": {"scrub:": {}}`
// Note: In case scrub is not empty the config.Extensions will not be nil and we will not reach here
config.Extensions.Scrub = &extconf.ScrubConfig{}
}
}
if config.Extensions != nil {
@ -378,6 +385,16 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper) {
config.Extensions.Metrics.Prometheus = &extconf.PrometheusConfig{Path: constants.DefaultMetricsExtensionRoute}
}
}
if config.Extensions.Scrub != nil {
if config.Extensions.Scrub.Enable == nil {
config.Extensions.Scrub.Enable = &defaultVal
}
if config.Extensions.Scrub.Interval == 0 {
config.Extensions.Scrub.Interval = 24 * time.Hour // nolint: gomnd
}
}
}
if !config.Storage.GC && viperInstance.Get("storage::gcdelay") == nil {

View file

@ -39,5 +39,6 @@ type PrometheusConfig struct {
}
type ScrubConfig struct {
Enable *bool
Interval time.Duration
}

View file

@ -20,7 +20,7 @@ func EnableScrubExtension(config *config.Config, log log.Logger, storeController
sch *scheduler.Scheduler,
) {
if config.Extensions.Scrub != nil &&
config.Extensions.Scrub.Interval != 0 {
*config.Extensions.Scrub.Enable {
minScrubInterval, _ := time.ParseDuration("2h")
if config.Extensions.Scrub.Interval < minScrubInterval {

View file

@ -48,7 +48,9 @@ func TestScrubExtension(t *testing.T) {
substore := config.StorageConfig{RootDirectory: subdir}
conf.Storage.SubPaths = map[string]config.StorageConfig{"/a": substore}
conf.Log.Output = logFile.Name()
trueValue := true
scrubConfig := &extconf.ScrubConfig{
Enable: &trueValue,
Interval: 2,
}
conf.Extensions = &extconf.ExtensionConfig{
@ -106,7 +108,9 @@ func TestScrubExtension(t *testing.T) {
conf.Storage.RootDirectory = dir
conf.Log.Output = logFile.Name()
trueValue := true
scrubConfig := &extconf.ScrubConfig{
Enable: &trueValue,
Interval: 2,
}
conf.Extensions = &extconf.ExtensionConfig{
@ -171,7 +175,9 @@ func TestScrubExtension(t *testing.T) {
conf.Storage.RootDirectory = dir
conf.Log.Output = logFile.Name()
trueValue := true
scrubConfig := &extconf.ScrubConfig{
Enable: &trueValue,
Interval: 2,
}
conf.Extensions = &extconf.ExtensionConfig{

View file

@ -35,6 +35,7 @@ function setup() {
},
"extensions": {
"scrub": {
"enable": true,
"interval": "2h"
}
}