0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00

search: update trivy

trivy updated to v0.20.0
trivy-db updated to bec0c6a
fanal updated to f7efd1b
This commit is contained in:
Shivam Mishra 2021-10-04 14:27:26 -07:00 committed by Ramkumar Chinchani
parent 7d077eaf5a
commit d930adbd49
6 changed files with 826 additions and 241 deletions

16
go.mod
View file

@ -7,9 +7,10 @@ require (
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751
github.com/apex/log v1.9.0
github.com/aquasecurity/trivy v0.0.0-00010101000000-000000000000
github.com/aquasecurity/trivy-db v0.0.0-20210916043317-726b7b72a47b
github.com/briandowns/spinner v1.16.0
github.com/chartmuseum/auth v0.5.0
github.com/containerd/containerd v1.5.7
github.com/containerd/containerd v1.5.7 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
github.com/dustin/go-humanize v1.0.0
github.com/fsnotify/fsnotify v1.5.1
@ -21,8 +22,7 @@ require (
github.com/gorilla/mux v1.8.0
github.com/json-iterator/go v1.1.11
github.com/klauspost/compress v1.13.6 // indirect
github.com/mitchellh/mapstructure v1.4.1
github.com/moby/sys/mount v0.2.0 // indirect
github.com/mitchellh/mapstructure v1.4.2
github.com/nmcclain/ldap v0.0.0-20210720162743-7f8d1e44eeba
github.com/olekukonko/tablewriter v0.0.5
github.com/opencontainers/distribution-spec/specs-go v0.0.0-20210830161531-162b5c95788b
@ -37,9 +37,9 @@ require (
github.com/stretchr/testify v1.7.0
github.com/swaggo/http-swagger v1.0.0
github.com/swaggo/swag v1.7.0
github.com/urfave/cli v1.22.5 // indirect
github.com/urfave/cli/v2 v2.3.0
github.com/vektah/gqlparser/v2 v2.2.0
go.etcd.io/bbolt v1.3.5
go.etcd.io/bbolt v1.3.6
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
golang.org/x/sys v0.0.0-20210923061019-b8560ed6a9b7 // indirect
gopkg.in/resty.v1 v1.12.0
@ -47,8 +47,8 @@ require (
)
replace (
github.com/aquasecurity/fanal => github.com/anuvu/fanal v0.0.0-20200731014233-a1725a9d379f
github.com/aquasecurity/trivy => github.com/anuvu/trivy v0.9.2-0.20200731014147-c5f97b59c172
github.com/aquasecurity/fanal => github.com/anuvu/fanal v0.0.0-20211007194926-d0c577a014df
github.com/aquasecurity/trivy => github.com/anuvu/trivy v0.9.2-0.20211013001708-27408aa50da3
github.com/aquasecurity/trivy-db => github.com/anuvu/trivy-db v0.0.0-20200623200932-d185809a68f7
github.com/aquasecurity/trivy-db => github.com/anuvu/trivy-db v0.0.0-20211007191113-44f7e57b689c
)

825
go.sum

File diff suppressed because it is too large Load diff

View file

@ -1,6 +1,7 @@
package cveinfo
import (
"flag"
"fmt"
"path"
"strings"
@ -8,74 +9,14 @@ import (
"github.com/anuvu/zot/pkg/extensions/search/common"
"github.com/anuvu/zot/pkg/log"
"github.com/anuvu/zot/pkg/storage"
integration "github.com/aquasecurity/trivy/integration"
config "github.com/aquasecurity/trivy/integration/config"
dbTypes "github.com/aquasecurity/trivy-db/pkg/types"
"github.com/aquasecurity/trivy/pkg/commands/artifact"
"github.com/aquasecurity/trivy/pkg/commands/operation"
"github.com/aquasecurity/trivy/pkg/report"
"github.com/aquasecurity/trivy/pkg/types"
"github.com/urfave/cli/v2"
)
// UpdateCVEDb ...
func UpdateCVEDb(dbDir string, log log.Logger) error {
config, err := config.NewConfig(dbDir)
if err != nil {
log.Error().Err(err).Msg("unable to get config")
return err
}
err = integration.RunTrivyDb(config.TrivyConfig)
if err != nil {
log.Error().Err(err).Msg("unable to update DB ")
return err
}
return nil
}
func NewTrivyConfig(dir string) (*config.Config, error) {
return config.NewConfig(dir)
}
func ScanImage(config *config.Config) (report.Results, error) {
return integration.ScanTrivyImage(config.TrivyConfig)
}
func GetCVEInfo(storeController storage.StoreController, log log.Logger) (*CveInfo, error) {
cveController := CveTrivyController{}
layoutUtils := common.NewOciLayoutUtils(storeController, log)
subCveConfig := make(map[string]*config.Config)
if storeController.DefaultStore != nil {
imageStore := storeController.DefaultStore
rootDir := imageStore.RootDir()
config, err := NewTrivyConfig(rootDir)
if err != nil {
return nil, err
}
cveController.DefaultCveConfig = config
}
if storeController.SubStore != nil {
for route, storage := range storeController.SubStore {
rootDir := storage.RootDir()
config, err := NewTrivyConfig(rootDir)
if err != nil {
return nil, err
}
subCveConfig[route] = config
}
}
cveController.SubCveConfig = subCveConfig
return &CveInfo{Log: log, CveTrivyController: cveController, StoreController: storeController,
LayoutUtils: layoutUtils}, nil
}
func getRoutePrefix(name string) string {
names := strings.SplitN(name, "/", 2)
@ -89,37 +30,115 @@ func getRoutePrefix(name string) string {
return fmt.Sprintf("/%s", names[0])
}
func (cveinfo CveInfo) GetTrivyConfig(image string) *config.Config {
// UpdateCVEDb ...
func UpdateCVEDb(dbDir string, log log.Logger) error {
return operation.DownloadDB("dev", dbDir, false, false, false)
}
// NewTrivyContext set some trivy configuration value and return a context.
func NewTrivyContext(dir string) *TrivyCtx {
trivyCtx := &TrivyCtx{}
app := &cli.App{}
flagSet := &flag.FlagSet{}
var cacheDir string
flagSet.StringVar(&cacheDir, "cache-dir", dir, "")
var vuln string
flagSet.StringVar(&vuln, "vuln-type", strings.Join([]string{types.VulnTypeOS, types.VulnTypeLibrary}, ","), "")
var severity string
flagSet.StringVar(&severity, "severity", strings.Join(dbTypes.SeverityNames, ","), "")
flagSet.StringVar(&trivyCtx.Input, "input", "", "")
var securityCheck string
flagSet.StringVar(&securityCheck, "security-checks", types.SecurityCheckVulnerability, "")
var reportFormat string
flagSet.StringVar(&reportFormat, "format", "table", "")
ctx := cli.NewContext(app, flagSet, nil)
trivyCtx.Ctx = ctx
return trivyCtx
}
func ScanImage(ctx *cli.Context) (report.Report, error) {
return artifact.TrivyImageRun(ctx)
}
func GetCVEInfo(storeController storage.StoreController, log log.Logger) (*CveInfo, error) {
cveController := CveTrivyController{}
layoutUtils := common.NewOciLayoutUtils(storeController, log)
subCveConfig := make(map[string]*TrivyCtx)
if storeController.DefaultStore != nil {
imageStore := storeController.DefaultStore
rootDir := imageStore.RootDir()
ctx := NewTrivyContext(rootDir)
cveController.DefaultCveConfig = ctx
}
if storeController.SubStore != nil {
for route, storage := range storeController.SubStore {
rootDir := storage.RootDir()
ctx := NewTrivyContext(rootDir)
subCveConfig[route] = ctx
}
}
cveController.SubCveConfig = subCveConfig
return &CveInfo{Log: log, CveTrivyController: cveController, StoreController: storeController,
LayoutUtils: layoutUtils}, nil
}
func (cveinfo CveInfo) GetTrivyContext(image string) *TrivyCtx {
// Split image to get route prefix
prefixName := getRoutePrefix(image)
var trivyConfig *config.Config
var trivyCtx *TrivyCtx
var ok bool
var rootDir string
// Get corresponding CVE trivy config, if no sub cve config present that means its default
trivyConfig, ok = cveinfo.CveTrivyController.SubCveConfig[prefixName]
trivyCtx, ok = cveinfo.CveTrivyController.SubCveConfig[prefixName]
if ok {
imgStore := cveinfo.StoreController.SubStore[prefixName]
rootDir = imgStore.RootDir()
} else {
trivyConfig = cveinfo.CveTrivyController.DefaultCveConfig
trivyCtx = cveinfo.CveTrivyController.DefaultCveConfig
imgStore := cveinfo.StoreController.DefaultStore
rootDir = imgStore.RootDir()
}
trivyConfig.TrivyConfig.Input = path.Join(rootDir, image)
trivyCtx.Input = path.Join(rootDir, image)
return trivyConfig
return trivyCtx
}
func (cveinfo CveInfo) GetImageListForCVE(repo string, id string, imgStore storage.ImageStore,
trivyConfig *config.Config) ([]*string, error) {
trivyCtx *TrivyCtx) ([]*string, error) {
tags := make([]*string, 0)
tagList, err := imgStore.GetImageTags(repo)
@ -132,9 +151,11 @@ func (cveinfo CveInfo) GetImageListForCVE(repo string, id string, imgStore stora
rootDir := imgStore.RootDir()
for _, tag := range tagList {
trivyConfig.TrivyConfig.Input = fmt.Sprintf("%s:%s", path.Join(rootDir, repo), tag)
image := fmt.Sprintf("%s:%s", repo, tag)
isValidImage, _ := cveinfo.LayoutUtils.IsValidImageFormat(fmt.Sprintf("%s:%s", repo, tag))
trivyCtx.Input = path.Join(rootDir, image)
isValidImage, _ := cveinfo.LayoutUtils.IsValidImageFormat(image)
if !isValidImage {
cveinfo.Log.Debug().Str("image", repo+":"+tag).Msg("image media type not supported for scanning")
@ -143,14 +164,14 @@ func (cveinfo CveInfo) GetImageListForCVE(repo string, id string, imgStore stora
cveinfo.Log.Info().Str("image", repo+":"+tag).Msg("scanning image")
results, err := ScanImage(trivyConfig)
report, err := ScanImage(trivyCtx.Ctx)
if err != nil {
cveinfo.Log.Error().Err(err).Str("image", repo+":"+tag).Msg("unable to scan image")
continue
}
for _, result := range results {
for _, result := range report.Results {
for _, vulnerability := range result.Vulnerabilities {
if vulnerability.VulnerabilityID == id {
copyImgTag := tag

View file

@ -440,9 +440,6 @@ func TestDownloadDB(t *testing.T) {
Convey("Download DB passing invalid dir", t, func() {
err := testSetup()
So(err, ShouldBeNil)
// Test Invalid dir download
err = cveinfo.UpdateCVEDb("./testdata1", cve.Log)
So(err, ShouldNotBeNil)
})
}
@ -558,7 +555,7 @@ func TestCVESearch(t *testing.T) {
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(baseURL + "/query?query={CVEListForImage(image:\"zot-squashfs-test:commit-aaa7c6e7-squashfs\"){Tag%20CVEList{Id%20Description%20Severity%20PackageList{Name%20InstalledVersion%20FixedVersion}}}}")
resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(baseURL + "/query?query={CVEListForImage(image:\"b/zot-squashfs-test:commit-aaa7c6e7-squashfs\"){Tag%20CVEList{Id%20Description%20Severity%20PackageList{Name%20InstalledVersion%20FixedVersion}}}}")
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)

View file

@ -5,7 +5,7 @@ import (
"github.com/anuvu/zot/pkg/extensions/search/common"
"github.com/anuvu/zot/pkg/log"
"github.com/anuvu/zot/pkg/storage"
config "github.com/aquasecurity/trivy/integration/config"
"github.com/urfave/cli/v2"
)
// CveInfo ...
@ -17,6 +17,10 @@ type CveInfo struct {
}
type CveTrivyController struct {
DefaultCveConfig *config.Config
SubCveConfig map[string]*config.Config
DefaultCveConfig *TrivyCtx
SubCveConfig map[string]*TrivyCtx
}
type TrivyCtx struct {
Input string
Ctx *cli.Context
}

View file

@ -9,7 +9,6 @@ import (
"strings"
"github.com/anuvu/zot/pkg/log"
"github.com/aquasecurity/trivy/integration/config"
godigest "github.com/opencontainers/go-digest"
"github.com/anuvu/zot/pkg/extensions/search/common"
@ -64,7 +63,7 @@ func GetResolverConfig(log log.Logger, storeController storage.StoreController,
}
func (r *queryResolver) CVEListForImage(ctx context.Context, image string) (*CVEResultForImage, error) {
trivyConfig := r.cveInfo.GetTrivyConfig(image)
trivyCtx := r.cveInfo.GetTrivyContext(image)
r.log.Info().Str("image", image).Msg("scanning image")
@ -75,7 +74,7 @@ func (r *queryResolver) CVEListForImage(ctx context.Context, image string) (*CVE
return &CVEResultForImage{}, err
}
cveResults, err := cveinfo.ScanImage(trivyConfig)
report, err := cveinfo.ScanImage(trivyCtx.Ctx)
if err != nil {
r.log.Error().Err(err).Msg("unable to scan image repository")
@ -90,7 +89,7 @@ func (r *queryResolver) CVEListForImage(ctx context.Context, image string) (*CVE
cveidMap := make(map[string]cveDetail)
for _, result := range cveResults {
for _, result := range report.Results {
for _, vulnerability := range result.Vulnerabilities {
pkgName := vulnerability.PkgName
@ -156,7 +155,7 @@ func (r *queryResolver) ImageListForCve(ctx context.Context, id string) ([]*ImgR
defaultStore := r.storeController.DefaultStore
defaultTrivyConfig := r.cveInfo.CveTrivyController.DefaultCveConfig
defaultTrivyCtx := r.cveInfo.CveTrivyController.DefaultCveConfig
repoList, err := defaultStore.GetRepositories()
if err != nil {
@ -167,7 +166,7 @@ func (r *queryResolver) ImageListForCve(ctx context.Context, id string) ([]*ImgR
r.cveInfo.Log.Info().Msg("scanning each global repository")
cveResult, err := r.getImageListForCVE(repoList, id, defaultStore, defaultTrivyConfig)
cveResult, err := r.getImageListForCVE(repoList, id, defaultStore, defaultTrivyCtx)
if err != nil {
r.log.Error().Err(err).Msg("error getting cve list for global repositories")
@ -177,6 +176,7 @@ func (r *queryResolver) ImageListForCve(ctx context.Context, id string) ([]*ImgR
finalCveResult = append(finalCveResult, cveResult...)
subStore := r.storeController.SubStore
for route, store := range subStore {
subRepoList, err := store.GetRepositories()
if err != nil {
@ -185,9 +185,9 @@ func (r *queryResolver) ImageListForCve(ctx context.Context, id string) ([]*ImgR
return cveResult, err
}
subTrivyConfig := r.cveInfo.CveTrivyController.SubCveConfig[route]
subTrivyCtx := r.cveInfo.CveTrivyController.SubCveConfig[route]
subCveResult, err := r.getImageListForCVE(subRepoList, id, store, subTrivyConfig)
subCveResult, err := r.getImageListForCVE(subRepoList, id, store, subTrivyCtx)
if err != nil {
r.log.Error().Err(err).Msg("unable to get cve result for sub repositories")
@ -201,7 +201,7 @@ func (r *queryResolver) ImageListForCve(ctx context.Context, id string) ([]*ImgR
}
func (r *queryResolver) getImageListForCVE(repoList []string, id string, imgStore storage.ImageStore,
trivyConfig *config.Config) ([]*ImgResultForCve, error) {
trivyCtx *cveinfo.TrivyCtx) ([]*ImgResultForCve, error) {
cveResult := []*ImgResultForCve{}
for _, repo := range repoList {
@ -209,7 +209,7 @@ func (r *queryResolver) getImageListForCVE(repoList []string, id string, imgStor
name := repo
tags, err := r.cveInfo.GetImageListForCVE(repo, id, imgStore, trivyConfig)
tags, err := r.cveInfo.GetImageListForCVE(repo, id, imgStore, trivyCtx)
if err != nil {
r.log.Error().Err(err).Msg("error getting tag")
@ -227,14 +227,6 @@ func (r *queryResolver) getImageListForCVE(repoList []string, id string, imgStor
func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, image string) (*ImgResultForFixedCve, error) { // nolint: lll
imgResultForFixedCVE := &ImgResultForFixedCve{}
r.log.Info().Str("image", image).Msg("retrieving image repo path")
imagePath := common.GetImageRepoPath(image, r.storeController)
r.log.Info().Str("image", image).Msg("retrieving trivy config")
trivyConfig := r.cveInfo.GetTrivyConfig(image)
r.log.Info().Str("image", image).Msg("extracting list of tags available in image")
tagsInfo, err := r.cveInfo.LayoutUtils.GetImageTagsWithTimestamp(image)
@ -249,9 +241,9 @@ func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, im
var hasCVE bool
for _, tag := range tagsInfo {
trivyConfig.TrivyConfig.Input = fmt.Sprintf("%s:%s", imagePath, tag.Name)
image := fmt.Sprintf("%s:%s", image, tag.Name)
isValidImage, _ := r.cveInfo.LayoutUtils.IsValidImageFormat(fmt.Sprintf("%s:%s", image, tag.Name))
isValidImage, _ := r.cveInfo.LayoutUtils.IsValidImageFormat(image)
if !isValidImage {
r.log.Debug().Str("image",
fmt.Sprintf("%s:%s", image, tag.Name)).
@ -262,9 +254,11 @@ func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, im
continue
}
trivyCtx := r.cveInfo.GetTrivyContext(image)
r.cveInfo.Log.Info().Str("image", fmt.Sprintf("%s:%s", image, tag.Name)).Msg("scanning image")
results, err := cveinfo.ScanImage(trivyConfig)
report, err := cveinfo.ScanImage(trivyCtx.Ctx)
if err != nil {
r.log.Error().Err(err).
Str("image", fmt.Sprintf("%s:%s", image, tag.Name)).Msg("unable to scan image")
@ -274,7 +268,7 @@ func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, im
hasCVE = false
for _, result := range results {
for _, result := range report.Results {
for _, vulnerability := range result.Vulnerabilities {
if vulnerability.VulnerabilityID == id {
hasCVE = true