From d960c2e23f9774dc8db40b9d7c3852f638ce24e2 Mon Sep 17 00:00:00 2001 From: Andrei Aaron Date: Thu, 19 Jan 2023 00:18:03 +0200 Subject: [PATCH] fix(ci): do not use "ghcr.io/aquasecurity/trivy-db" for ci (#1107) Signed-off-by: Andrei Aaron --- pkg/cli/cve_cmd_test.go | 16 ++++++ pkg/cli/extensions_test.go | 4 +- pkg/extensions/config/config.go | 5 ++ pkg/extensions/extension_search.go | 15 ++++- pkg/extensions/search/common/common_test.go | 16 +++++- pkg/extensions/search/cve/cve.go | 4 +- pkg/extensions/search/cve/cve_test.go | 11 ++-- pkg/extensions/search/cve/trivy/scanner.go | 18 ++++-- .../search/cve/trivy/scanner_internal_test.go | 55 +++++++++++++++---- 9 files changed, 113 insertions(+), 31 deletions(-) diff --git a/pkg/cli/cve_cmd_test.go b/pkg/cli/cve_cmd_test.go index f478f9f8..284e2e1f 100644 --- a/pkg/cli/cve_cmd_test.go +++ b/pkg/cli/cve_cmd_test.go @@ -358,8 +358,12 @@ func TestServerCVEResponseGQL(t *testing.T) { } conf.Storage.RootDirectory = dir + trivyConfig := &extconf.TrivyConfig{ + DBRepository: "ghcr.io/project-zot/trivy-db", + } cveConfig := &extconf.CVEConfig{ UpdateInterval: 2, + Trivy: trivyConfig, } defaultVal := true searchConfig := &extconf.SearchConfig{ @@ -647,8 +651,12 @@ func TestNegativeServerResponse(t *testing.T) { } conf.Storage.RootDirectory = dir + trivyConfig := &extconf.TrivyConfig{ + DBRepository: "ghcr.io/project-zot/trivy-db", + } cveConfig := &extconf.CVEConfig{ UpdateInterval: 2, + Trivy: trivyConfig, } defaultVal := false searchConfig := &extconf.SearchConfig{ @@ -734,8 +742,12 @@ func TestNegativeServerResponse(t *testing.T) { } conf.Storage.RootDirectory = dir + trivyConfig := &extconf.TrivyConfig{ + DBRepository: "ghcr.io/project-zot/trivy-db", + } cveConfig := &extconf.CVEConfig{ UpdateInterval: 2, + Trivy: trivyConfig, } defaultVal := true searchConfig := &extconf.SearchConfig{ @@ -812,8 +824,12 @@ func TestServerCVEResponse(t *testing.T) { } conf.Storage.RootDirectory = dir + trivyConfig := &extconf.TrivyConfig{ + DBRepository: "ghcr.io/project-zot/trivy-db", + } cveConfig := &extconf.CVEConfig{ UpdateInterval: 2, + Trivy: trivyConfig, } defaultVal := true searchConfig := &extconf.SearchConfig{ diff --git a/pkg/cli/extensions_test.go b/pkg/cli/extensions_test.go index d82b3831..2b714e94 100644 --- a/pkg/cli/extensions_test.go +++ b/pkg/cli/extensions_test.go @@ -680,7 +680,7 @@ func TestServeSearchEnabledCVE(t *testing.T) { // to avoid data race when multiple go routines write to trivy DB instance. WaitTillTrivyDBDownloadStarted(tempDir) - substring := "\"Extensions\":{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000}},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}" //nolint:lll // gofumpt conflicts with lll + substring := "\"Extensions\":{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000,\"Trivy\":null}},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}" //nolint:lll // gofumpt conflicts with lll found, err := readLogFileAndSearchString(logPath, substring, readLogFileTimeout) So(found, ShouldBeTrue) So(err, ShouldBeNil) @@ -768,7 +768,7 @@ func TestServeSearchDisabled(t *testing.T) { defer os.Remove(logPath) // clean up dataStr := string(data) So(dataStr, ShouldContainSubstring, - "\"Extensions\":{\"Search\":{\"Enable\":false,\"CVE\":{\"UpdateInterval\":10800000000000}},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}") //nolint:lll // gofumpt conflicts with lll + "\"Extensions\":{\"Search\":{\"Enable\":false,\"CVE\":{\"UpdateInterval\":10800000000000,\"Trivy\":null}},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}") //nolint:lll // gofumpt conflicts with lll So(dataStr, ShouldContainSubstring, "CVE config not provided, skipping CVE update") So(dataStr, ShouldNotContainSubstring, "CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.") diff --git a/pkg/extensions/config/config.go b/pkg/extensions/config/config.go index 3527756e..7dffea8b 100644 --- a/pkg/extensions/config/config.go +++ b/pkg/extensions/config/config.go @@ -32,6 +32,11 @@ type SearchConfig struct { type CVEConfig struct { UpdateInterval time.Duration // should be 2 hours or more, if not specified default be kept as 24 hours + Trivy *TrivyConfig +} + +type TrivyConfig struct { + DBRepository string // default is "ghcr.io/aquasecurity/trivy-db" } type MetricsConfig struct { diff --git a/pkg/extensions/extension_search.go b/pkg/extensions/extension_search.go index a42571cf..f005b716 100644 --- a/pkg/extensions/extension_search.go +++ b/pkg/extensions/extension_search.go @@ -37,7 +37,12 @@ func EnableSearchExtension(config *config.Config, storeController storage.StoreC log.Warn().Msg("CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.") //nolint:lll // gofumpt conflicts with lll } - cveInfo = cveinfo.NewCVEInfo(storeController, repoDB, log) + dbRepository := "" + if config.Extensions.Search.CVE.Trivy != nil { + dbRepository = config.Extensions.Search.CVE.Trivy.DBRepository + } + + cveInfo = cveinfo.NewCVEInfo(storeController, repoDB, dbRepository, log) go func() { err := downloadTrivyDB(log, config.Extensions.Search.CVE.UpdateInterval) @@ -77,7 +82,13 @@ func SetupSearchRoutes(config *config.Config, router *mux.Router, storeControlle // cveinfo should already be initialized by this time // as EnableSearchExtension is supposed to be called earlier, but let's be sure if cveInfo == nil { - cveInfo = cveinfo.NewCVEInfo(storeController, repoDB, log) + dbRepository := "" + + if config.Extensions.Search.CVE.Trivy != nil { + dbRepository = config.Extensions.Search.CVE.Trivy.DBRepository + } + + cveInfo = cveinfo.NewCVEInfo(storeController, repoDB, dbRepository, log) } resConfig = search.GetResolverConfig(log, storeController, repoDB, cveInfo) diff --git a/pkg/extensions/search/common/common_test.go b/pkg/extensions/search/common/common_test.go index 704fcd88..79905f41 100644 --- a/pkg/extensions/search/common/common_test.go +++ b/pkg/extensions/search/common/common_test.go @@ -480,8 +480,12 @@ func TestRepoListWithNewestImage(t *testing.T) { defaultVal := true updateDuration, _ := time.ParseDuration("1h") + trivyConfig := &extconf.TrivyConfig{ + DBRepository: "ghcr.io/project-zot/trivy-db", + } cveConfig := &extconf.CVEConfig{ UpdateInterval: updateDuration, + Trivy: trivyConfig, } searchConfig := &extconf.SearchConfig{ BaseConfig: extconf.BaseConfig{Enable: &defaultVal}, @@ -507,7 +511,7 @@ func TestRepoListWithNewestImage(t *testing.T) { defer stopServer(ctlr) WaitTillServerReady(baseURL) - substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000}}" + substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000,\"Trivy\":{\"DBRepository\":\"ghcr.io/project-zot/trivy-db\"}}}" //nolint: lll found, err := readFileAndSearchString(logPath, substring, 2*time.Minute) So(found, ShouldBeTrue) So(err, ShouldBeNil) @@ -2507,8 +2511,12 @@ func TestGlobalSearch(t *testing.T) { defaultVal := true updateDuration, _ := time.ParseDuration("1h") + trivyConfig := &extconf.TrivyConfig{ + DBRepository: "ghcr.io/project-zot/trivy-db", + } cveConfig := &extconf.CVEConfig{ UpdateInterval: updateDuration, + Trivy: trivyConfig, } searchConfig := &extconf.SearchConfig{ BaseConfig: extconf.BaseConfig{Enable: &defaultVal}, @@ -2535,7 +2543,7 @@ func TestGlobalSearch(t *testing.T) { WaitTillServerReady(baseURL) // Wait for trivy db to download - substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000}}" + substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000,\"Trivy\":{\"DBRepository\":\"ghcr.io/project-zot/trivy-db\"}}}" //nolint: lll found, err := readFileAndSearchString(logPath, substring, 2*time.Minute) So(found, ShouldBeTrue) So(err, ShouldBeNil) @@ -4731,8 +4739,12 @@ func TestImageSummary(t *testing.T) { defaultVal := true updateDuration, _ := time.ParseDuration("1h") + trivyConfig := &extconf.TrivyConfig{ + DBRepository: "ghcr.io/project-zot/trivy-db", + } cveConfig := &extconf.CVEConfig{ UpdateInterval: updateDuration, + Trivy: trivyConfig, } searchConfig := &extconf.SearchConfig{ BaseConfig: extconf.BaseConfig{Enable: &defaultVal}, diff --git a/pkg/extensions/search/cve/cve.go b/pkg/extensions/search/cve/cve.go index 75f789ab..92088872 100644 --- a/pkg/extensions/search/cve/cve.go +++ b/pkg/extensions/search/cve/cve.go @@ -48,9 +48,9 @@ type BaseCveInfo struct { } func NewCVEInfo(storeController storage.StoreController, repoDB repodb.RepoDB, - log log.Logger, + dbRepository string, log log.Logger, ) *BaseCveInfo { - scanner := trivy.NewScanner(storeController, repoDB, log) + scanner := trivy.NewScanner(storeController, repoDB, dbRepository, log) return &BaseCveInfo{ Log: log, diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 6c8ae633..0a056f11 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -311,10 +311,6 @@ func TestImageFormat(t *testing.T) { imgDir := "../../../../test/data" dbDir := t.TempDir() - conf := config.New() - conf.Extensions = &extconf.ExtensionConfig{} - conf.Extensions.Lint = &extconf.LintConfig{} - metrics := monitoring.NewMetricsServer(false, log) defaultStore := local.NewImageStore(imgDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil) @@ -328,7 +324,7 @@ func TestImageFormat(t *testing.T) { err = repodb.SyncRepoDB(repoDB, storeController, log) So(err, ShouldBeNil) - cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, log) + cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", log) isValidImage, err := cveInfo.Scanner.IsImageFormatScannable("zot-test") So(err, ShouldNotBeNil) @@ -477,8 +473,13 @@ func TestCVESearch(t *testing.T) { } conf.Storage.RootDirectory = dbDir + + trivyConfig := &extconf.TrivyConfig{ + DBRepository: "ghcr.io/project-zot/trivy-db", + } cveConfig := &extconf.CVEConfig{ UpdateInterval: updateDuration, + Trivy: trivyConfig, } defaultVal := true searchConfig := &extconf.SearchConfig{ diff --git a/pkg/extensions/search/cve/trivy/scanner.go b/pkg/extensions/search/cve/trivy/scanner.go index 9a90368a..baab325c 100644 --- a/pkg/extensions/search/cve/trivy/scanner.go +++ b/pkg/extensions/search/cve/trivy/scanner.go @@ -23,11 +23,11 @@ import ( "zotregistry.io/zot/pkg/storage" ) -const dbRepository = "ghcr.io/aquasecurity/trivy-db" +const defaultDBRepository = "ghcr.io/aquasecurity/trivy-db" // getNewScanOptions sets trivy configuration values for our scans and returns them as // a trivy Options structure. -func getNewScanOptions(dir string) *flag.Options { +func getNewScanOptions(dir, dbRepository string) *flag.Options { scanOptions := flag.Options{ GlobalOptions: flag.GlobalOptions{ CacheDir: dir, @@ -70,22 +70,27 @@ type Scanner struct { log log.Logger dbLock *sync.Mutex cache *CveCache + dbRepository string } func NewScanner(storeController storage.StoreController, - repoDB repodb.RepoDB, log log.Logger, + repoDB repodb.RepoDB, dbRepository string, log log.Logger, ) *Scanner { cveController := cveTrivyController{} subCveConfig := make(map[string]*flag.Options) + if dbRepository == "" { + dbRepository = defaultDBRepository + } + if storeController.DefaultStore != nil { imageStore := storeController.DefaultStore rootDir := imageStore.RootDir() cacheDir := path.Join(rootDir, "_trivy") - opts := getNewScanOptions(cacheDir) + opts := getNewScanOptions(cacheDir, dbRepository) cveController.DefaultCveConfig = opts } @@ -95,7 +100,7 @@ func NewScanner(storeController storage.StoreController, rootDir := storage.RootDir() cacheDir := path.Join(rootDir, "_trivy") - opts := getNewScanOptions(cacheDir) + opts := getNewScanOptions(cacheDir, dbRepository) subCveConfig[route] = opts } @@ -110,6 +115,7 @@ func NewScanner(storeController storage.StoreController, storeController: storeController, dbLock: &sync.Mutex{}, cache: NewCveCache(10000, log), //nolint:gomnd + dbRepository: dbRepository, } } @@ -333,7 +339,7 @@ func (scanner Scanner) UpdateDB() error { func (scanner Scanner) updateDB(dbDir string) error { scanner.log.Debug().Msgf("Download Trivy DB to destination dir: %s", dbDir) - err := operation.DownloadDB("dev", dbDir, dbRepository, false, false, false) + err := operation.DownloadDB("dev", dbDir, scanner.dbRepository, false, false, false) if err != nil { scanner.log.Error().Err(err).Msgf("Error downloading Trivy DB to destination dir: %s", dbDir) diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index 6d377670..0cb596da 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -11,8 +11,6 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" - "zotregistry.io/zot/pkg/api/config" - extconf "zotregistry.io/zot/pkg/extensions/config" "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/extensions/search/common" "zotregistry.io/zot/pkg/log" @@ -63,10 +61,6 @@ func TestMultipleStoragePath(t *testing.T) { log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - conf := config.New() - conf.Extensions = &extconf.ExtensionConfig{} - conf.Extensions.Lint = &extconf.LintConfig{} - // Create ImageStore firstStore := local.NewImageStore(firstRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil) @@ -93,7 +87,7 @@ func TestMultipleStoragePath(t *testing.T) { err = repodb.SyncRepoDB(repoDB, storeController, log) So(err, ShouldBeNil) - scanner := NewScanner(storeController, repoDB, log) + scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log) So(scanner.storeController.DefaultStore, ShouldNotBeNil) So(scanner.storeController.SubStore, ShouldNotBeNil) @@ -168,10 +162,6 @@ func TestTrivyLibraryErrors(t *testing.T) { log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - conf := config.New() - conf.Extensions = &extconf.ExtensionConfig{} - conf.Extensions.Lint = &extconf.LintConfig{} - // Create ImageStore store := local.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil) @@ -186,7 +176,7 @@ func TestTrivyLibraryErrors(t *testing.T) { err = repodb.SyncRepoDB(repoDB, storeController, log) So(err, ShouldBeNil) - scanner := NewScanner(storeController, repoDB, log) + scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log) // Download DB since DB download on scan is disabled err = scanner.UpdateDB() @@ -218,3 +208,44 @@ func TestTrivyLibraryErrors(t *testing.T) { So(err, ShouldNotBeNil) }) } + +func TestDefaultTrivyDBUrl(t *testing.T) { + Convey("Test trivy DB download from default location", t, func() { + // Create temporary directory + rootDir := t.TempDir() + + err := test.CopyFiles("../../../../../test/data/zot-test", path.Join(rootDir, "zot-test")) + So(err, ShouldBeNil) + + log := log.NewLogger("debug", "") + metrics := monitoring.NewMetricsServer(false, log) + + // Create ImageStore + store := local.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil) + + storeController := storage.StoreController{} + storeController.DefaultStore = store + + repoDB, err := bolt.NewBoltDBWrapper(bolt.DBParameters{ + RootDir: rootDir, + }) + So(err, ShouldBeNil) + + err = repodb.SyncRepoDB(repoDB, storeController, log) + So(err, ShouldBeNil) + + // Use empty string for DB repository, the default url would be used internally + scanner := NewScanner(storeController, repoDB, "", log) + + // Download DB since DB download on scan is disabled + err = scanner.UpdateDB() + So(err, ShouldBeNil) + + img := "zot-test:0.0.1" + + // Scanning image + opts := scanner.getTrivyOptions(img) + _, err = scanner.runTrivy(opts) + So(err, ShouldBeNil) + }) +}