diff --git a/examples/config-allextensions.json b/examples/config-allextensions.json index 294896da..f960a6b3 100644 --- a/examples/config-allextensions.json +++ b/examples/config-allextensions.json @@ -41,6 +41,9 @@ "cve": { "updateInterval": "2h" } + }, + "scrub": { + "interval": "24h" } } } diff --git a/examples/config-scrub.json b/examples/config-scrub.json new file mode 100644 index 00000000..6df19468 --- /dev/null +++ b/examples/config-scrub.json @@ -0,0 +1,18 @@ +{ + "distSpecVersion":"1.0.1", + "storage": { + "rootDirectory": "/tmp/zot" + }, + "http": { + "address": "127.0.0.1", + "port": "8080" + }, + "log": { + "level": "debug" + }, + "extensions": { + "scrub": { + "interval": "24h" + } + } +} diff --git a/pkg/api/controller.go b/pkg/api/controller.go index c0429c81..c602b40d 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -325,6 +325,10 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error { ext.EnableSyncExtension(reloadCtx, c.Config, c.wgShutDown, c.StoreController, c.Log) } + if c.Config.Extensions != nil { + ext.EnableScrubExtension(c.Config, c.StoreController, c.Log) + } + return nil } diff --git a/pkg/cli/extensions_test.go b/pkg/cli/extensions_test.go index a8836ac1..9fbdbd1f 100644 --- a/pkg/cli/extensions_test.go +++ b/pkg/cli/extensions_test.go @@ -102,7 +102,7 @@ func TestServeExtensions(t *testing.T) { WaitTillServerReady(baseURL) data, err := os.ReadFile(logFile.Name()) So(err, ShouldBeNil) - So(string(data), ShouldContainSubstring, "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null") + So(string(data), ShouldContainSubstring, "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null") // nolint:lll }) } @@ -143,7 +143,7 @@ func testWithMetricsEnabled(cfgContentFormat string) { data, err := os.ReadFile(logFile.Name()) So(err, ShouldBeNil) So(string(data), ShouldContainSubstring, - "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":true,\"Prometheus\":{\"Path\":\"/metrics\"}}}") // nolint:lll + "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":true,\"Prometheus\":{\"Path\":\"/metrics\"}},\"Scrub\":null}") // nolint:lll } func TestServeMetricsExtension(t *testing.T) { @@ -267,7 +267,7 @@ func TestServeMetricsExtension(t *testing.T) { data, err := os.ReadFile(logFile.Name()) So(err, ShouldBeNil) So(string(data), ShouldContainSubstring, - "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":false,\"Prometheus\":{\"Path\":\"/metrics\"}}}") // nolint:lll + "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":false,\"Prometheus\":{\"Path\":\"/metrics\"}},\"Scrub\":null}") // nolint:lll }) } @@ -458,6 +458,112 @@ func TestServeSyncExtension(t *testing.T) { }) } +func TestServeScrubExtension(t *testing.T) { + oldArgs := os.Args + + defer func() { os.Args = oldArgs }() + + Convey("scrub enabled by scrub interval param set", t, func(c C) { + port := GetFreePort() + baseURL := GetBaseURL(port) + logFile, err := ioutil.TempFile("", "zot-log*.txt") + So(err, ShouldBeNil) + defer os.Remove(logFile.Name()) // clean up + + content := fmt.Sprintf(`{ + "storage": { + "rootDirectory": "/tmp/zot" + }, + "http": { + "address": "127.0.0.1", + "port": "%s" + }, + "log": { + "level": "debug", + "output": "%s" + }, + "extensions": { + "scrub": { + "interval": "1h" + } + } + }`, port, logFile.Name()) + + cfgfile, err := ioutil.TempFile("", "zot-test*.json") + So(err, ShouldBeNil) + defer os.Remove(cfgfile.Name()) // clean up + _, err = cfgfile.Write([]byte(content)) + So(err, ShouldBeNil) + err = cfgfile.Close() + So(err, ShouldBeNil) + + os.Args = []string{"cli_test", "serve", cfgfile.Name()} + go func() { + err = cli.NewServerRootCmd().Execute() + So(err, ShouldBeNil) + }() + WaitTillServerReady(baseURL) + + data, err := os.ReadFile(logFile.Name()) + So(err, ShouldBeNil) + // Even if in config we specified scrub interval=1h, the minimum interval is 2h + So(string(data), ShouldContainSubstring, + "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":{\"Interval\":3600000000000}") // nolint:lll + So(string(data), ShouldContainSubstring, "executing scrub to check manifest/blob integrity") + So(string(data), ShouldContainSubstring, + "Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") + }) + + Convey("scrub not enabled - scrub interval param not set", t, func(c C) { + port := GetFreePort() + baseURL := GetBaseURL(port) + logFile, err := ioutil.TempFile("", "zot-log*.txt") + So(err, ShouldBeNil) + defer os.Remove(logFile.Name()) // clean up + + content := fmt.Sprintf(`{ + "storage": { + "rootDirectory": "/tmp/zot" + }, + "http": { + "address": "127.0.0.1", + "port": "%s" + }, + "log": { + "level": "debug", + "output": "%s" + }, + "extensions": { + "scrub": { + } + } + }`, port, logFile.Name()) + + cfgfile, err := ioutil.TempFile("", "zot-test*.json") + So(err, ShouldBeNil) + defer os.Remove(cfgfile.Name()) // clean up + _, err = cfgfile.Write([]byte(content)) + So(err, ShouldBeNil) + err = cfgfile.Close() + So(err, ShouldBeNil) + + os.Args = []string{"cli_test", "serve", cfgfile.Name()} + go func() { + err = cli.NewServerRootCmd().Execute() + So(err, ShouldBeNil) + }() + WaitTillServerReady(baseURL) + + data, err := os.ReadFile(logFile.Name()) + So(err, ShouldBeNil) + So(string(data), ShouldContainSubstring, + "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") + So(string(data), ShouldContainSubstring, "Scrub config not provided, skipping scrub") + So(string(data), ShouldNotContainSubstring, + "Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") + }) +} + func TestServeSearchExtension(t *testing.T) { oldArgs := os.Args @@ -506,7 +612,7 @@ func TestServeSearchExtension(t *testing.T) { data, err := os.ReadFile(logFile.Name()) So(err, ShouldBeNil) So(string(data), ShouldContainSubstring, - "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null}") // nolint:lll + "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") // nolint:lll So(string(data), ShouldContainSubstring, "updating the CVE database") }) @@ -557,7 +663,7 @@ func TestServeSearchExtension(t *testing.T) { So(err, ShouldBeNil) // Even if in config we specified updateInterval=1h, the minimum interval is 2h So(string(data), ShouldContainSubstring, - "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":3600000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null}") // nolint:lll + "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":3600000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") // nolint:lll So(string(data), ShouldContainSubstring, "updating the CVE database") So(string(data), ShouldContainSubstring, "CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.") @@ -607,7 +713,7 @@ func TestServeSearchExtension(t *testing.T) { data, err := os.ReadFile(logFile.Name()) So(err, ShouldBeNil) So(string(data), ShouldContainSubstring, - "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null}") // nolint:lll + "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") // nolint:lll So(string(data), ShouldContainSubstring, "updating the CVE database") }) @@ -658,7 +764,7 @@ func TestServeSearchExtension(t *testing.T) { data, err := os.ReadFile(logFile.Name()) So(err, ShouldBeNil) So(string(data), ShouldContainSubstring, - "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":10800000000000},\"Enable\":false},\"Sync\":null,\"Metrics\":null}") // nolint:lll + "\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":10800000000000},\"Enable\":false},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") // nolint:lll So(string(data), ShouldContainSubstring, "CVE config not provided, skipping CVE update") So(string(data), ShouldNotContainSubstring, "CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.") diff --git a/pkg/cli/root.go b/pkg/cli/root.go index 8b7a779e..0ce72804 100644 --- a/pkg/cli/root.go +++ b/pkg/cli/root.go @@ -96,7 +96,7 @@ func newScrubCmd(conf *config.Config) *cobra.Command { response, err := http.DefaultClient.Do(req) if err == nil { response.Body.Close() - log.Info().Msg("The server is running, in order to perform the scrub command the server should be shut down") + log.Warn().Msg("The server is running, in order to perform the scrub command the server should be shut down") panic("Error: server is running") } else { // server is down diff --git a/pkg/extensions/config/config.go b/pkg/extensions/config/config.go index bc91891d..3000c313 100644 --- a/pkg/extensions/config/config.go +++ b/pkg/extensions/config/config.go @@ -10,6 +10,7 @@ type ExtensionConfig struct { Search *SearchConfig Sync *sync.Config Metrics *MetricsConfig + Scrub *ScrubConfig } type SearchConfig struct { @@ -30,3 +31,7 @@ type MetricsConfig struct { type PrometheusConfig struct { Path string // default is "/metrics" } + +type ScrubConfig struct { + Interval time.Duration +} diff --git a/pkg/extensions/extensions.go b/pkg/extensions/extensions.go index ea511e37..f75af40e 100644 --- a/pkg/extensions/extensions.go +++ b/pkg/extensions/extensions.go @@ -12,6 +12,7 @@ import ( "github.com/gorilla/mux" "github.com/prometheus/client_golang/prometheus/promhttp" "zotregistry.io/zot/pkg/api/config" + "zotregistry.io/zot/pkg/extensions/scrub" "zotregistry.io/zot/pkg/extensions/search" cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" "zotregistry.io/zot/pkg/extensions/sync" @@ -81,6 +82,30 @@ func EnableSyncExtension(ctx context.Context, config *config.Config, wg *goSync. } } +// EnableScrubExtension enables scrub extension. +func EnableScrubExtension(config *config.Config, storeController storage.StoreController, + log log.Logger) { + if config.Extensions.Scrub != nil && + config.Extensions.Scrub.Interval != 0 { + minScrubInterval, _ := time.ParseDuration("2h") + + if config.Extensions.Scrub.Interval < minScrubInterval { + config.Extensions.Scrub.Interval = minScrubInterval + + log.Warn().Msg("Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") // nolint: lll + } + + go func() { + err := scrub.Run(log, config.Extensions.Scrub.Interval, storeController) + if err != nil { + log.Error().Err(err).Msg("error while trying to scrub") + } + }() + } else { + log.Info().Msg("Scrub config not provided, skipping scrub") + } +} + // SetupRoutes ... func SetupRoutes(config *config.Config, router *mux.Router, storeController storage.StoreController, l log.Logger) { diff --git a/pkg/extensions/minimal.go b/pkg/extensions/minimal.go index 02e1b9f9..cf0d5055 100644 --- a/pkg/extensions/minimal.go +++ b/pkg/extensions/minimal.go @@ -32,6 +32,13 @@ func EnableSyncExtension(ctx context.Context, config *config.Config, wg *goSync. "please build zot full binary for this feature") } +// EnableScrubExtension ... +func EnableScrubExtension(config *config.Config, storeController storage.StoreController, + log log.Logger) { + log.Warn().Msg("skipping enabling scrub extension because given zot binary doesn't support any extensions," + + "please build zot full binary for this feature") +} + // SetupRoutes ... func SetupRoutes(conf *config.Config, router *mux.Router, storeController storage.StoreController, log log.Logger) { log.Warn().Msg("skipping setting up extensions routes because given zot binary doesn't support " + diff --git a/pkg/extensions/scrub/scrub.go b/pkg/extensions/scrub/scrub.go new file mode 100644 index 00000000..2ac0e527 --- /dev/null +++ b/pkg/extensions/scrub/scrub.go @@ -0,0 +1,44 @@ +//go:build extended +// +build extended + +package scrub + +import ( + "time" + + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/storage" +) + +// Scrub Extension... +func Run(log log.Logger, scrubInterval time.Duration, storeController storage.StoreController) error { + for { + log.Info().Msg("executing scrub to check manifest/blob integrity") + + results, err := storeController.CheckAllBlobsIntegrity() + if err != nil { + return err + } + + for _, result := range results.ScrubResults { + if result.Status == "ok" { + log.Info(). + Str("image", result.ImageName). + Str("tag", result.Tag). + Str("status", result.Status). + Msg("scrub: blobs/manifest ok") + } else { + log.Warn(). + Str("image", result.ImageName). + Str("tag", result.Tag). + Str("status", result.Status). + Str("error", result.Error). + Msg("scrub: blobs/manifest affected") + } + } + + log.Info().Str("Scrub completed, next scrub scheduled after", scrubInterval.String()).Msg("") + + time.Sleep(scrubInterval) + } +} diff --git a/pkg/extensions/scrub/scrub_test.go b/pkg/extensions/scrub/scrub_test.go new file mode 100644 index 00000000..069b0bb8 --- /dev/null +++ b/pkg/extensions/scrub/scrub_test.go @@ -0,0 +1,210 @@ +//go:build extended +// +build extended + +package scrub_test + +import ( + "context" + "io/ioutil" + "os" + "path" + "testing" + "time" + + . "github.com/smartystreets/goconvey/convey" + "gopkg.in/resty.v1" + "zotregistry.io/zot/pkg/api" + "zotregistry.io/zot/pkg/api/config" + extconf "zotregistry.io/zot/pkg/extensions/config" + "zotregistry.io/zot/pkg/test" +) + +const ( + repoName = "test" +) + +func TestScrubExtension(t *testing.T) { + Convey("Blobs integrity not affected", t, func(c C) { + port := test.GetFreePort() + url := test.GetBaseURL(port) + + logFile, err := ioutil.TempFile("", "zot-log*.txt") + So(err, ShouldBeNil) + + defer os.Remove(logFile.Name()) // clean up + + conf := config.New() + conf.HTTP.Port = port + + dir := t.TempDir() + + conf.Storage.RootDirectory = dir + conf.Log.Output = logFile.Name() + scrubConfig := &extconf.ScrubConfig{ + Interval: 2, + } + conf.Extensions = &extconf.ExtensionConfig{ + Scrub: scrubConfig, + } + + ctlr := api.NewController(conf) + + err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName)) + if err != nil { + panic(err) + } + + go func(controller *api.Controller) { + // this blocks + if err := controller.Run(context.Background()); err != nil { + return + } + }(ctlr) + + // wait till ready + for { + _, err := resty.R().Get(url) + if err == nil { + break + } + + time.Sleep(100 * time.Millisecond) + } + time.Sleep(1 * time.Second) + + defer func(controller *api.Controller) { + ctx := context.Background() + _ = controller.Server.Shutdown(ctx) + }(ctlr) + + data, err := os.ReadFile(logFile.Name()) + So(err, ShouldBeNil) + So(string(data), ShouldContainSubstring, "scrub: blobs/manifest ok") + }) + + Convey("Blobs integrity affected", t, func(c C) { + port := test.GetFreePort() + url := test.GetBaseURL(port) + + logFile, err := ioutil.TempFile("", "zot-log*.txt") + So(err, ShouldBeNil) + + defer os.Remove(logFile.Name()) // clean up + + conf := config.New() + conf.HTTP.Port = port + + dir := t.TempDir() + + conf.Storage.RootDirectory = dir + conf.Log.Output = logFile.Name() + scrubConfig := &extconf.ScrubConfig{ + Interval: 2, + } + conf.Extensions = &extconf.ExtensionConfig{ + Scrub: scrubConfig, + } + + ctlr := api.NewController(conf) + + err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName)) + if err != nil { + panic(err) + } + + err = os.Remove(path.Join(dir, repoName, "blobs/sha256", + "2bacca16b9df395fc855c14ccf50b12b58d35d468b8e7f25758aff90f89bf396")) + if err != nil { + panic(err) + } + + go func(controller *api.Controller) { + // this blocks + if err := controller.Run(context.Background()); err != nil { + return + } + }(ctlr) + + // wait till ready + for { + _, err := resty.R().Get(url) + if err == nil { + break + } + + time.Sleep(100 * time.Millisecond) + } + time.Sleep(500 * time.Millisecond) + + defer func(controller *api.Controller) { + ctx := context.Background() + _ = controller.Server.Shutdown(ctx) + }(ctlr) + + data, err := os.ReadFile(logFile.Name()) + So(err, ShouldBeNil) + So(string(data), ShouldContainSubstring, "scrub: blobs/manifest affected") + }) + + Convey("CheckAllBlobsIntegrity error - not enough permissions to access root directory", t, func(c C) { + port := test.GetFreePort() + url := test.GetBaseURL(port) + + logFile, err := ioutil.TempFile("", "zot-log*.txt") + So(err, ShouldBeNil) + + defer os.Remove(logFile.Name()) // clean up + + conf := config.New() + conf.HTTP.Port = port + + dir := t.TempDir() + + conf.Storage.RootDirectory = dir + conf.Log.Output = logFile.Name() + scrubConfig := &extconf.ScrubConfig{ + Interval: 2, + } + conf.Extensions = &extconf.ExtensionConfig{ + Scrub: scrubConfig, + } + + ctlr := api.NewController(conf) + + err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName)) + if err != nil { + panic(err) + } + + So(os.Chmod(path.Join(dir, repoName), 0o000), ShouldBeNil) + + go func(controller *api.Controller) { + // this blocks + if err := controller.Run(context.Background()); err != nil { + return + } + }(ctlr) + + // wait till ready + for { + _, err := resty.R().Get(url) + if err == nil { + break + } + + time.Sleep(100 * time.Millisecond) + } + time.Sleep(500 * time.Millisecond) + + defer func(controller *api.Controller) { + ctx := context.Background() + _ = controller.Server.Shutdown(ctx) + }(ctlr) + + data, err := os.ReadFile(logFile.Name()) + So(err, ShouldBeNil) + So(string(data), ShouldContainSubstring, "error while trying to scrub") + + So(os.Chmod(path.Join(dir, repoName), 0o755), ShouldBeNil) + }) +} diff --git a/pkg/storage/scrub.go b/pkg/storage/scrub.go index aa80a031..be417687 100644 --- a/pkg/storage/scrub.go +++ b/pkg/storage/scrub.go @@ -53,25 +53,38 @@ func (sc StoreController) CheckAllBlobsIntegrity() (ScrubResults, error) { imageStoreList[""] = sc.DefaultStore for _, imgStore := range imageStoreList { - images, err := imgStore.GetRepositories() + imgStoreResults, err := CheckImageStoreBlobsIntegrity(imgStore) if err != nil { return results, err } - for _, repo := range images { - imageResults, err := checkImage(repo, imgStore) - if err != nil { - return results, err - } - - results.ScrubResults = append(results.ScrubResults, imageResults...) - } + results.ScrubResults = append(results.ScrubResults, imgStoreResults...) } return results, nil } -func checkImage(imageName string, imgStore ImageStore) ([]ScrubImageResult, error) { +func CheckImageStoreBlobsIntegrity(imgStore ImageStore) ([]ScrubImageResult, error) { + results := []ScrubImageResult{} + + repos, err := imgStore.GetRepositories() + if err != nil { + return results, err + } + + for _, repo := range repos { + imageResults, err := checkRepo(repo, imgStore) + if err != nil { + return results, err + } + + results = append(results, imageResults...) + } + + return results, nil +} + +func checkRepo(imageName string, imgStore ImageStore) ([]ScrubImageResult, error) { results := []ScrubImageResult{} dir := path.Join(imgStore.RootDir(), imageName)