Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-30 22:34:13 -05:00
Code Issues Activity

make scrub inline and periodic

Browse source 
Signed-off-by: Andreea-Lupu <andreealupu1470@yahoo.com>
This commit is contained in:
Andreea-Lupu 2022-03-04 09:37:06 +02:00 committed by Ramkumar Chinchani
parent ad519e2d3e
commit 9454c77be2
11 changed files with 453 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
examples/config-allextensions.json
View file

@ -41,6 +41,9 @@
"cve": {
"updateInterval": "2h"
}
},
"scrub": {
"interval": "24h"
}
}
}

18
examples/config-scrub.json Normal file
View file

@ -0,0 +1,18 @@
{
"distSpecVersion":"1.0.1",
"storage": {
"rootDirectory": "/tmp/zot"
},
"http": {
"address": "127.0.0.1",
"port": "8080"
},
"log": {
"level": "debug"
},
"extensions": {
"scrub": {
"interval": "24h"
}
}
}

4
pkg/api/controller.go
View file

@ -325,6 +325,10 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error {
ext.EnableSyncExtension(reloadCtx, c.Config, c.wgShutDown, c.StoreController, c.Log)
}
if c.Config.Extensions != nil {
ext.EnableScrubExtension(c.Config, c.StoreController, c.Log)
}
return nil
}

120
pkg/cli/extensions_test.go
View file

@ -102,7 +102,7 @@ func TestServeExtensions(t *testing.T) {
WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring, "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null")
So(string(data), ShouldContainSubstring, "\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null") // nolint:lll
})
}
@ -143,7 +143,7 @@ func testWithMetricsEnabled(cfgContentFormat string) {
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":true,\"Prometheus\":{\"Path\":\"/metrics\"}}}") // nolint:lll
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":true,\"Prometheus\":{\"Path\":\"/metrics\"}},\"Scrub\":null}") // nolint:lll
}
func TestServeMetricsExtension(t *testing.T) {
@ -267,7 +267,7 @@ func TestServeMetricsExtension(t *testing.T) {
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":false,\"Prometheus\":{\"Path\":\"/metrics\"}}}") // nolint:lll
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":{\"Enable\":false,\"Prometheus\":{\"Path\":\"/metrics\"}},\"Scrub\":null}") // nolint:lll
})
}
@ -458,6 +458,112 @@ func TestServeSyncExtension(t *testing.T) {
})
}
func TestServeScrubExtension(t *testing.T) {
oldArgs := os.Args
defer func() { os.Args = oldArgs }()
Convey("scrub enabled by scrub interval param set", t, func(c C) {
port := GetFreePort()
baseURL := GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
content := fmt.Sprintf(`{
"storage": {
"rootDirectory": "/tmp/zot"
},
"http": {
"address": "127.0.0.1",
"port": "%s"
},
"log": {
"level": "debug",
"output": "%s"
},
"extensions": {
"scrub": {
"interval": "1h"
}
}
}`, port, logFile.Name())
cfgfile, err := ioutil.TempFile("", "zot-test*.json")
So(err, ShouldBeNil)
defer os.Remove(cfgfile.Name()) // clean up
_, err = cfgfile.Write([]byte(content))
So(err, ShouldBeNil)
err = cfgfile.Close()
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "serve", cfgfile.Name()}
go func() {
err = cli.NewServerRootCmd().Execute()
So(err, ShouldBeNil)
}()
WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
// Even if in config we specified scrub interval=1h, the minimum interval is 2h
So(string(data), ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":{\"Interval\":3600000000000}") // nolint:lll
So(string(data), ShouldContainSubstring, "executing scrub to check manifest/blob integrity")
So(string(data), ShouldContainSubstring,
"Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.")
})
Convey("scrub not enabled - scrub interval param not set", t, func(c C) {
port := GetFreePort()
baseURL := GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
content := fmt.Sprintf(`{
"storage": {
"rootDirectory": "/tmp/zot"
},
"http": {
"address": "127.0.0.1",
"port": "%s"
},
"log": {
"level": "debug",
"output": "%s"
},
"extensions": {
"scrub": {
}
}
}`, port, logFile.Name())
cfgfile, err := ioutil.TempFile("", "zot-test*.json")
So(err, ShouldBeNil)
defer os.Remove(cfgfile.Name()) // clean up
_, err = cfgfile.Write([]byte(content))
So(err, ShouldBeNil)
err = cfgfile.Close()
So(err, ShouldBeNil)
os.Args = []string{"cli_test", "serve", cfgfile.Name()}
go func() {
err = cli.NewServerRootCmd().Execute()
So(err, ShouldBeNil)
}()
WaitTillServerReady(baseURL)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
"\"Extensions\":{\"Search\":null,\"Sync\":null,\"Metrics\":null,\"Scrub\":null}")
So(string(data), ShouldContainSubstring, "Scrub config not provided, skipping scrub")
So(string(data), ShouldNotContainSubstring,
"Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.")
})
}
func TestServeSearchExtension(t *testing.T) {
oldArgs := os.Args
@ -506,7 +612,7 @@ func TestServeSearchExtension(t *testing.T) {
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null}") // nolint:lll
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") // nolint:lll
So(string(data), ShouldContainSubstring, "updating the CVE database")
})
@ -557,7 +663,7 @@ func TestServeSearchExtension(t *testing.T) {
So(err, ShouldBeNil)
// Even if in config we specified updateInterval=1h, the minimum interval is 2h
So(string(data), ShouldContainSubstring,
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":3600000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null}") // nolint:lll
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":3600000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") // nolint:lll
So(string(data), ShouldContainSubstring, "updating the CVE database")
So(string(data), ShouldContainSubstring,
"CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.")
@ -607,7 +713,7 @@ func TestServeSearchExtension(t *testing.T) {
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null}") // nolint:lll
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":86400000000000},\"Enable\":true},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") // nolint:lll
So(string(data), ShouldContainSubstring, "updating the CVE database")
})
@ -658,7 +764,7 @@ func TestServeSearchExtension(t *testing.T) {
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring,
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":10800000000000},\"Enable\":false},\"Sync\":null,\"Metrics\":null}") // nolint:lll
"\"Extensions\":{\"Search\":{\"CVE\":{\"UpdateInterval\":10800000000000},\"Enable\":false},\"Sync\":null,\"Metrics\":null,\"Scrub\":null}") // nolint:lll
So(string(data), ShouldContainSubstring, "CVE config not provided, skipping CVE update")
So(string(data), ShouldNotContainSubstring,
"CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.")

2
pkg/cli/root.go
View file

@ -96,7 +96,7 @@ func newScrubCmd(conf *config.Config) *cobra.Command {
response, err := http.DefaultClient.Do(req)
if err == nil {
response.Body.Close()
log.Info().Msg("The server is running, in order to perform the scrub command the server should be shut down")
log.Warn().Msg("The server is running, in order to perform the scrub command the server should be shut down")
panic("Error: server is running")
} else {
// server is down

5
pkg/extensions/config/config.go
View file

@ -10,6 +10,7 @@ type ExtensionConfig struct {
Search *SearchConfig
Sync *sync.Config
Metrics *MetricsConfig
Scrub *ScrubConfig
}
type SearchConfig struct {
@ -30,3 +31,7 @@ type MetricsConfig struct {
type PrometheusConfig struct {
Path string // default is "/metrics"
}
type ScrubConfig struct {
Interval time.Duration
}

25
pkg/extensions/extensions.go
View file

@ -12,6 +12,7 @@ import (
"github.com/gorilla/mux"
"github.com/prometheus/client_golang/prometheus/promhttp"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/extensions/scrub"
"zotregistry.io/zot/pkg/extensions/search"
cveinfo "zotregistry.io/zot/pkg/extensions/search/cve"
"zotregistry.io/zot/pkg/extensions/sync"
@ -81,6 +82,30 @@ func EnableSyncExtension(ctx context.Context, config *config.Config, wg *goSync.
}
}
// EnableScrubExtension enables scrub extension.
func EnableScrubExtension(config *config.Config, storeController storage.StoreController,
log log.Logger) {
if config.Extensions.Scrub != nil &&
config.Extensions.Scrub.Interval != 0 {
minScrubInterval, _ := time.ParseDuration("2h")
if config.Extensions.Scrub.Interval < minScrubInterval {
config.Extensions.Scrub.Interval = minScrubInterval
log.Warn().Msg("Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") // nolint: lll
}
go func() {
err := scrub.Run(log, config.Extensions.Scrub.Interval, storeController)
if err != nil {
log.Error().Err(err).Msg("error while trying to scrub")
}
}()
} else {
log.Info().Msg("Scrub config not provided, skipping scrub")
}
}
// SetupRoutes ...
func SetupRoutes(config *config.Config, router *mux.Router, storeController storage.StoreController,
l log.Logger) {

7
pkg/extensions/minimal.go
View file

@ -32,6 +32,13 @@ func EnableSyncExtension(ctx context.Context, config *config.Config, wg *goSync.
"please build zot full binary for this feature")
}
// EnableScrubExtension ...
func EnableScrubExtension(config *config.Config, storeController storage.StoreController,
log log.Logger) {
log.Warn().Msg("skipping enabling scrub extension because given zot binary doesn't support any extensions," +
"please build zot full binary for this feature")
}
// SetupRoutes ...
func SetupRoutes(conf *config.Config, router *mux.Router, storeController storage.StoreController, log log.Logger) {
log.Warn().Msg("skipping setting up extensions routes because given zot binary doesn't support " +

44
pkg/extensions/scrub/scrub.go Normal file
View file

@ -0,0 +1,44 @@
//go:build extended
// +build extended
package scrub
import (
"time"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
// Scrub Extension...
func Run(log log.Logger, scrubInterval time.Duration, storeController storage.StoreController) error {
for {
log.Info().Msg("executing scrub to check manifest/blob integrity")
results, err := storeController.CheckAllBlobsIntegrity()
if err != nil {
return err
}
for _, result := range results.ScrubResults {
if result.Status == "ok" {
log.Info().
Str("image", result.ImageName).
Str("tag", result.Tag).
Str("status", result.Status).
Msg("scrub: blobs/manifest ok")
} else {
log.Warn().
Str("image", result.ImageName).
Str("tag", result.Tag).
Str("status", result.Status).
Str("error", result.Error).
Msg("scrub: blobs/manifest affected")
}
}
log.Info().Str("Scrub completed, next scrub scheduled after", scrubInterval.String()).Msg("")
time.Sleep(scrubInterval)
}
}

210
pkg/extensions/scrub/scrub_test.go Normal file
View file

@ -0,0 +1,210 @@
//go:build extended
// +build extended
package scrub_test
import (
"context"
"io/ioutil"
"os"
"path"
"testing"
"time"
. "github.com/smartystreets/goconvey/convey"
"gopkg.in/resty.v1"
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/test"
)
const (
repoName = "test"
)
func TestScrubExtension(t *testing.T) {
Convey("Blobs integrity not affected", t, func(c C) {
port := test.GetFreePort()
url := test.GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
conf := config.New()
conf.HTTP.Port = port
dir := t.TempDir()
conf.Storage.RootDirectory = dir
conf.Log.Output = logFile.Name()
scrubConfig := &extconf.ScrubConfig{
Interval: 2,
}
conf.Extensions = &extconf.ExtensionConfig{
Scrub: scrubConfig,
}
ctlr := api.NewController(conf)
err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName))
if err != nil {
panic(err)
}
go func(controller *api.Controller) {
// this blocks
if err := controller.Run(context.Background()); err != nil {
return
}
}(ctlr)
// wait till ready
for {
_, err := resty.R().Get(url)
if err == nil {
break
}
time.Sleep(100 * time.Millisecond)
}
time.Sleep(1 * time.Second)
defer func(controller *api.Controller) {
ctx := context.Background()
_ = controller.Server.Shutdown(ctx)
}(ctlr)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring, "scrub: blobs/manifest ok")
})
Convey("Blobs integrity affected", t, func(c C) {
port := test.GetFreePort()
url := test.GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
conf := config.New()
conf.HTTP.Port = port
dir := t.TempDir()
conf.Storage.RootDirectory = dir
conf.Log.Output = logFile.Name()
scrubConfig := &extconf.ScrubConfig{
Interval: 2,
}
conf.Extensions = &extconf.ExtensionConfig{
Scrub: scrubConfig,
}
ctlr := api.NewController(conf)
err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName))
if err != nil {
panic(err)
}
err = os.Remove(path.Join(dir, repoName, "blobs/sha256",
"2bacca16b9df395fc855c14ccf50b12b58d35d468b8e7f25758aff90f89bf396"))
if err != nil {
panic(err)
}
go func(controller *api.Controller) {
// this blocks
if err := controller.Run(context.Background()); err != nil {
return
}
}(ctlr)
// wait till ready
for {
_, err := resty.R().Get(url)
if err == nil {
break
}
time.Sleep(100 * time.Millisecond)
}
time.Sleep(500 * time.Millisecond)
defer func(controller *api.Controller) {
ctx := context.Background()
_ = controller.Server.Shutdown(ctx)
}(ctlr)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring, "scrub: blobs/manifest affected")
})
Convey("CheckAllBlobsIntegrity error - not enough permissions to access root directory", t, func(c C) {
port := test.GetFreePort()
url := test.GetBaseURL(port)
logFile, err := ioutil.TempFile("", "zot-log*.txt")
So(err, ShouldBeNil)
defer os.Remove(logFile.Name()) // clean up
conf := config.New()
conf.HTTP.Port = port
dir := t.TempDir()
conf.Storage.RootDirectory = dir
conf.Log.Output = logFile.Name()
scrubConfig := &extconf.ScrubConfig{
Interval: 2,
}
conf.Extensions = &extconf.ExtensionConfig{
Scrub: scrubConfig,
}
ctlr := api.NewController(conf)
err = test.CopyFiles("../../../test/data/zot-test", path.Join(dir, repoName))
if err != nil {
panic(err)
}
So(os.Chmod(path.Join(dir, repoName), 0o000), ShouldBeNil)
go func(controller *api.Controller) {
// this blocks
if err := controller.Run(context.Background()); err != nil {
return
}
}(ctlr)
// wait till ready
for {
_, err := resty.R().Get(url)
if err == nil {
break
}
time.Sleep(100 * time.Millisecond)
}
time.Sleep(500 * time.Millisecond)
defer func(controller *api.Controller) {
ctx := context.Background()
_ = controller.Server.Shutdown(ctx)
}(ctlr)
data, err := os.ReadFile(logFile.Name())
So(err, ShouldBeNil)
So(string(data), ShouldContainSubstring, "error while trying to scrub")
So(os.Chmod(path.Join(dir, repoName), 0o755), ShouldBeNil)
})
}

33
pkg/storage/scrub.go
View file

@ -53,25 +53,38 @@ func (sc StoreController) CheckAllBlobsIntegrity() (ScrubResults, error) {
imageStoreList[""] = sc.DefaultStore
for _, imgStore := range imageStoreList {
images, err := imgStore.GetRepositories()
imgStoreResults, err := CheckImageStoreBlobsIntegrity(imgStore)
if err != nil {
return results, err
}
for _, repo := range images {
imageResults, err := checkImage(repo, imgStore)
if err != nil {
return results, err
}
results.ScrubResults = append(results.ScrubResults, imageResults...)
}
results.ScrubResults = append(results.ScrubResults, imgStoreResults...)
}
return results, nil
}
func checkImage(imageName string, imgStore ImageStore) ([]ScrubImageResult, error) {
func CheckImageStoreBlobsIntegrity(imgStore ImageStore) ([]ScrubImageResult, error) {
results := []ScrubImageResult{}
repos, err := imgStore.GetRepositories()
if err != nil {
return results, err
}
for _, repo := range repos {
imageResults, err := checkRepo(repo, imgStore)
if err != nil {
return results, err
}
results = append(results, imageResults...)
}
return results, nil
}
func checkRepo(imageName string, imgStore ImageStore) ([]ScrubImageResult, error) {
results := []ScrubImageResult{}
dir := path.Join(imgStore.RootDir(), imageName)