Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2025-01-20 22:52:51 -05:00
Code Issues Activity

fix(ci): do not use "ghcr.io/aquasecurity/trivy-db" for ci (#1107)

Browse source 
Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
This commit is contained in:
Andrei Aaron 2023-01-19 00:18:03 +02:00 • committed by GitHub
parent fac1d1d05d
commit d960c2e23f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 113 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
pkg/cli/cve_cmd_test.go
View file

@ -358,8 +358,12 @@ func TestServerCVEResponseGQL(t *testing.T) {
}
conf.Storage.RootDirectory = dir
trivyConfig := &extconf.TrivyConfig{
DBRepository: "ghcr.io/project-zot/trivy-db",
}
cveConfig := &extconf.CVEConfig{
UpdateInterval: 2,
Trivy: trivyConfig,
}
defaultVal := true
searchConfig := &extconf.SearchConfig{
@ -647,8 +651,12 @@ func TestNegativeServerResponse(t *testing.T) {
}
conf.Storage.RootDirectory = dir
trivyConfig := &extconf.TrivyConfig{
DBRepository: "ghcr.io/project-zot/trivy-db",
}
cveConfig := &extconf.CVEConfig{
UpdateInterval: 2,
Trivy: trivyConfig,
}
defaultVal := false
searchConfig := &extconf.SearchConfig{
@ -734,8 +742,12 @@ func TestNegativeServerResponse(t *testing.T) {
}
conf.Storage.RootDirectory = dir
trivyConfig := &extconf.TrivyConfig{
DBRepository: "ghcr.io/project-zot/trivy-db",
}
cveConfig := &extconf.CVEConfig{
UpdateInterval: 2,
Trivy: trivyConfig,
}
defaultVal := true
searchConfig := &extconf.SearchConfig{
@ -812,8 +824,12 @@ func TestServerCVEResponse(t *testing.T) {
}
conf.Storage.RootDirectory = dir
trivyConfig := &extconf.TrivyConfig{
DBRepository: "ghcr.io/project-zot/trivy-db",
}
cveConfig := &extconf.CVEConfig{
UpdateInterval: 2,
Trivy: trivyConfig,
}
defaultVal := true
searchConfig := &extconf.SearchConfig{

4
pkg/cli/extensions_test.go
View file

@ -680,7 +680,7 @@ func TestServeSearchEnabledCVE(t *testing.T) {
// to avoid data race when multiple go routines write to trivy DB instance.
WaitTillTrivyDBDownloadStarted(tempDir)
substring := "\"Extensions\":{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000}},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}" //nolint:lll // gofumpt conflicts with lll
substring := "\"Extensions\":{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000,\"Trivy\":null}},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}" //nolint:lll // gofumpt conflicts with lll
found, err := readLogFileAndSearchString(logPath, substring, readLogFileTimeout)
So(found, ShouldBeTrue)
So(err, ShouldBeNil)
@ -768,7 +768,7 @@ func TestServeSearchDisabled(t *testing.T) {
defer os.Remove(logPath) // clean up
dataStr := string(data)
So(dataStr, ShouldContainSubstring,
"\"Extensions\":{\"Search\":{\"Enable\":false,\"CVE\":{\"UpdateInterval\":10800000000000}},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}") //nolint:lll // gofumpt conflicts with lll
"\"Extensions\":{\"Search\":{\"Enable\":false,\"CVE\":{\"UpdateInterval\":10800000000000,\"Trivy\":null}},\"Sync\":null,\"Metrics\":null,\"Scrub\":null,\"Lint\":null}") //nolint:lll // gofumpt conflicts with lll
So(dataStr, ShouldContainSubstring, "CVE config not provided, skipping CVE update")
So(dataStr, ShouldNotContainSubstring,
"CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.")

5
pkg/extensions/config/config.go
View file

@ -32,6 +32,11 @@ type SearchConfig struct {
type CVEConfig struct {
UpdateInterval time.Duration // should be 2 hours or more, if not specified default be kept as 24 hours
Trivy *TrivyConfig
}
type TrivyConfig struct {
DBRepository string // default is "ghcr.io/aquasecurity/trivy-db"
}
type MetricsConfig struct {

15
pkg/extensions/extension_search.go
View file

@ -37,7 +37,12 @@ func EnableSearchExtension(config *config.Config, storeController storage.StoreC
log.Warn().Msg("CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.") //nolint:lll // gofumpt conflicts with lll
}
cveInfo = cveinfo.NewCVEInfo(storeController, repoDB, log)
dbRepository := ""
if config.Extensions.Search.CVE.Trivy != nil {
dbRepository = config.Extensions.Search.CVE.Trivy.DBRepository
}
cveInfo = cveinfo.NewCVEInfo(storeController, repoDB, dbRepository, log)
go func() {
err := downloadTrivyDB(log, config.Extensions.Search.CVE.UpdateInterval)
@ -77,7 +82,13 @@ func SetupSearchRoutes(config *config.Config, router *mux.Router, storeControlle
// cveinfo should already be initialized by this time
// as EnableSearchExtension is supposed to be called earlier, but let's be sure
if cveInfo == nil {
cveInfo = cveinfo.NewCVEInfo(storeController, repoDB, log)
dbRepository := ""
if config.Extensions.Search.CVE.Trivy != nil {
dbRepository = config.Extensions.Search.CVE.Trivy.DBRepository
}
cveInfo = cveinfo.NewCVEInfo(storeController, repoDB, dbRepository, log)
}
resConfig = search.GetResolverConfig(log, storeController, repoDB, cveInfo)

16
pkg/extensions/search/common/common_test.go
View file

@ -480,8 +480,12 @@ func TestRepoListWithNewestImage(t *testing.T) {
defaultVal := true
updateDuration, _ := time.ParseDuration("1h")
trivyConfig := &extconf.TrivyConfig{
DBRepository: "ghcr.io/project-zot/trivy-db",
}
cveConfig := &extconf.CVEConfig{
UpdateInterval: updateDuration,
Trivy: trivyConfig,
}
searchConfig := &extconf.SearchConfig{
BaseConfig: extconf.BaseConfig{Enable: &defaultVal},
@ -507,7 +511,7 @@ func TestRepoListWithNewestImage(t *testing.T) {
defer stopServer(ctlr)
WaitTillServerReady(baseURL)
substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000}}"
substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000,\"Trivy\":{\"DBRepository\":\"ghcr.io/project-zot/trivy-db\"}}}" //nolint: lll
found, err := readFileAndSearchString(logPath, substring, 2*time.Minute)
So(found, ShouldBeTrue)
So(err, ShouldBeNil)
@ -2507,8 +2511,12 @@ func TestGlobalSearch(t *testing.T) {
defaultVal := true
updateDuration, _ := time.ParseDuration("1h")
trivyConfig := &extconf.TrivyConfig{
DBRepository: "ghcr.io/project-zot/trivy-db",
}
cveConfig := &extconf.CVEConfig{
UpdateInterval: updateDuration,
Trivy: trivyConfig,
}
searchConfig := &extconf.SearchConfig{
BaseConfig: extconf.BaseConfig{Enable: &defaultVal},
@ -2535,7 +2543,7 @@ func TestGlobalSearch(t *testing.T) {
WaitTillServerReady(baseURL)
// Wait for trivy db to download
substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000}}"
substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000,\"Trivy\":{\"DBRepository\":\"ghcr.io/project-zot/trivy-db\"}}}" //nolint: lll
found, err := readFileAndSearchString(logPath, substring, 2*time.Minute)
So(found, ShouldBeTrue)
So(err, ShouldBeNil)
@ -4731,8 +4739,12 @@ func TestImageSummary(t *testing.T) {
defaultVal := true
updateDuration, _ := time.ParseDuration("1h")
trivyConfig := &extconf.TrivyConfig{
DBRepository: "ghcr.io/project-zot/trivy-db",
}
cveConfig := &extconf.CVEConfig{
UpdateInterval: updateDuration,
Trivy: trivyConfig,
}
searchConfig := &extconf.SearchConfig{
BaseConfig: extconf.BaseConfig{Enable: &defaultVal},

4
pkg/extensions/search/cve/cve.go
View file

@ -48,9 +48,9 @@ type BaseCveInfo struct {
}
func NewCVEInfo(storeController storage.StoreController, repoDB repodb.RepoDB,
log log.Logger,
dbRepository string, log log.Logger,
) *BaseCveInfo {
scanner := trivy.NewScanner(storeController, repoDB, log)
scanner := trivy.NewScanner(storeController, repoDB, dbRepository, log)
return &BaseCveInfo{
Log: log,

11
pkg/extensions/search/cve/cve_test.go
View file

@ -311,10 +311,6 @@ func TestImageFormat(t *testing.T) {
imgDir := "../../../../test/data"
dbDir := t.TempDir()
conf := config.New()
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Lint = &extconf.LintConfig{}
metrics := monitoring.NewMetricsServer(false, log)
defaultStore := local.NewImageStore(imgDir, false, storage.DefaultGCDelay,
false, false, log, metrics, nil, nil)
@ -328,7 +324,7 @@ func TestImageFormat(t *testing.T) {
err = repodb.SyncRepoDB(repoDB, storeController, log)
So(err, ShouldBeNil)
cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, log)
cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", log)
isValidImage, err := cveInfo.Scanner.IsImageFormatScannable("zot-test")
So(err, ShouldNotBeNil)
@ -477,8 +473,13 @@ func TestCVESearch(t *testing.T) {
}
conf.Storage.RootDirectory = dbDir
trivyConfig := &extconf.TrivyConfig{
DBRepository: "ghcr.io/project-zot/trivy-db",
}
cveConfig := &extconf.CVEConfig{
UpdateInterval: updateDuration,
Trivy: trivyConfig,
}
defaultVal := true
searchConfig := &extconf.SearchConfig{

18
pkg/extensions/search/cve/trivy/scanner.go
View file

@ -23,11 +23,11 @@ import (
"zotregistry.io/zot/pkg/storage"
)
const dbRepository = "ghcr.io/aquasecurity/trivy-db"
const defaultDBRepository = "ghcr.io/aquasecurity/trivy-db"
// getNewScanOptions sets trivy configuration values for our scans and returns them as
// a trivy Options structure.
func getNewScanOptions(dir string) *flag.Options {
func getNewScanOptions(dir, dbRepository string) *flag.Options {
scanOptions := flag.Options{
GlobalOptions: flag.GlobalOptions{
CacheDir: dir,
@ -70,22 +70,27 @@ type Scanner struct {
log log.Logger
dbLock *sync.Mutex
cache *CveCache
dbRepository string
}
func NewScanner(storeController storage.StoreController,
repoDB repodb.RepoDB, log log.Logger,
repoDB repodb.RepoDB, dbRepository string, log log.Logger,
) *Scanner {
cveController := cveTrivyController{}
subCveConfig := make(map[string]*flag.Options)
if dbRepository == "" {
dbRepository = defaultDBRepository
}
if storeController.DefaultStore != nil {
imageStore := storeController.DefaultStore
rootDir := imageStore.RootDir()
cacheDir := path.Join(rootDir, "_trivy")
opts := getNewScanOptions(cacheDir)
opts := getNewScanOptions(cacheDir, dbRepository)
cveController.DefaultCveConfig = opts
}
@ -95,7 +100,7 @@ func NewScanner(storeController storage.StoreController,
rootDir := storage.RootDir()
cacheDir := path.Join(rootDir, "_trivy")
opts := getNewScanOptions(cacheDir)
opts := getNewScanOptions(cacheDir, dbRepository)
subCveConfig[route] = opts
}
@ -110,6 +115,7 @@ func NewScanner(storeController storage.StoreController,
storeController: storeController,
dbLock: &sync.Mutex{},
cache: NewCveCache(10000, log), //nolint:gomnd
dbRepository: dbRepository,
}
}
@ -333,7 +339,7 @@ func (scanner Scanner) UpdateDB() error {
func (scanner Scanner) updateDB(dbDir string) error {
scanner.log.Debug().Msgf("Download Trivy DB to destination dir: %s", dbDir)
err := operation.DownloadDB("dev", dbDir, dbRepository, false, false, false)
err := operation.DownloadDB("dev", dbDir, scanner.dbRepository, false, false, false)
if err != nil {
scanner.log.Error().Err(err).Msgf("Error downloading Trivy DB to destination dir: %s", dbDir)

55
pkg/extensions/search/cve/trivy/scanner_internal_test.go
View file

@ -11,8 +11,6 @@ import (
ispec "github.com/opencontainers/image-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/pkg/api/config"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/extensions/search/common"
"zotregistry.io/zot/pkg/log"
@ -63,10 +61,6 @@ func TestMultipleStoragePath(t *testing.T) {
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
conf := config.New()
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Lint = &extconf.LintConfig{}
// Create ImageStore
firstStore := local.NewImageStore(firstRootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
@ -93,7 +87,7 @@ func TestMultipleStoragePath(t *testing.T) {
err = repodb.SyncRepoDB(repoDB, storeController, log)
So(err, ShouldBeNil)
scanner := NewScanner(storeController, repoDB, log)
scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log)
So(scanner.storeController.DefaultStore, ShouldNotBeNil)
So(scanner.storeController.SubStore, ShouldNotBeNil)
@ -168,10 +162,6 @@ func TestTrivyLibraryErrors(t *testing.T) {
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
conf := config.New()
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Lint = &extconf.LintConfig{}
// Create ImageStore
store := local.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
@ -186,7 +176,7 @@ func TestTrivyLibraryErrors(t *testing.T) {
err = repodb.SyncRepoDB(repoDB, storeController, log)
So(err, ShouldBeNil)
scanner := NewScanner(storeController, repoDB, log)
scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log)
// Download DB since DB download on scan is disabled
err = scanner.UpdateDB()
@ -218,3 +208,44 @@ func TestTrivyLibraryErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
}
func TestDefaultTrivyDBUrl(t *testing.T) {
Convey("Test trivy DB download from default location", t, func() {
// Create temporary directory
rootDir := t.TempDir()
err := test.CopyFiles("../../../../../test/data/zot-test", path.Join(rootDir, "zot-test"))
So(err, ShouldBeNil)
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
// Create ImageStore
store := local.NewImageStore(rootDir, false, storage.DefaultGCDelay, false, false, log, metrics, nil, nil)
storeController := storage.StoreController{}
storeController.DefaultStore = store
repoDB, err := bolt.NewBoltDBWrapper(bolt.DBParameters{
RootDir: rootDir,
})
So(err, ShouldBeNil)
err = repodb.SyncRepoDB(repoDB, storeController, log)
So(err, ShouldBeNil)
// Use empty string for DB repository, the default url would be used internally
scanner := NewScanner(storeController, repoDB, "", log)
// Download DB since DB download on scan is disabled
err = scanner.UpdateDB()
So(err, ShouldBeNil)
img := "zot-test:0.0.1"
// Scanning image
opts := scanner.getTrivyOptions(img)
_, err = scanner.runTrivy(opts)
So(err, ShouldBeNil)
})
}