From 96d00cd0ef0c884fcd5325cb9277437fbc6f5aeb Mon Sep 17 00:00:00 2001 From: Andrei Aaron Date: Thu, 1 Jun 2023 00:37:46 +0300 Subject: [PATCH] fix(cve): Fix CVE scanning in images containing Jar files (#1475) --- Makefile | 11 ++- go.mod | 12 ++++ go.sum | 19 +++++- pkg/cli/extensions_test.go | 32 +++++++-- pkg/cli/root.go | 31 ++++++++- pkg/extensions/config/config.go | 3 +- pkg/extensions/extension_search.go | 18 ++--- pkg/extensions/extension_search_test.go | 2 +- pkg/extensions/search/cve/cve.go | 4 +- pkg/extensions/search/cve/cve_test.go | 4 +- pkg/extensions/search/cve/trivy/scanner.go | 67 +++++++++++-------- .../search/cve/trivy/scanner_internal_test.go | 39 +++++++++-- pkg/extensions/search/search_test.go | 6 +- pkg/storage/local/local.go | 2 +- pkg/storage/s3/s3.go | 2 +- pkg/test/common_test.go | 4 +- 16 files changed, 185 insertions(+), 71 deletions(-) diff --git a/Makefile b/Makefile index b6d29c2c..c16967d1 100644 --- a/Makefile +++ b/Makefile @@ -117,8 +117,14 @@ privileged-test: check-skopeo $(TESTDATA) go test -failfast -tags needprivileges,$(BUILD_LABELS),containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-dev-needprivileges.txt -covermode=atomic ./pkg/storage/... ./pkg/cli/... -run ^TestElevatedPrivileges $(TESTDATA): check-skopeo - $(shell mkdir -p ${TESTDATA}; cd ${TESTDATA}; mkdir -p noidentity; ../scripts/gen_certs.sh; cd ${TESTDATA}/noidentity; ../../scripts/gen_nameless_certs.sh; cd ${TOP_LEVEL}; skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:7 oci:${TESTDATA}/zot-test:0.0.1;skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:8 oci:${TESTDATA}/zot-cve-test:0.0.1) - $(shell chmod -R a=rwx ${TESTDATA}) + mkdir -p ${TESTDATA}; \ + cd ${TESTDATA}; ../scripts/gen_certs.sh; \ + mkdir -p noidentity; cd ${TESTDATA}/noidentity; ../../scripts/gen_nameless_certs.sh; \ + cd ${TOP_LEVEL}; \ + skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:7 oci:${TESTDATA}/zot-test:0.0.1; \ + skopeo --insecure-policy copy -q docker://public.ecr.aws/t0x7q1g8/centos:8 oci:${TESTDATA}/zot-cve-test:0.0.1; \ + skopeo --insecure-policy copy -q docker://ghcr.io/project-zot/test-images/java:0.0.1 oci:${TESTDATA}/zot-cve-java-test:0.0.1; \ + chmod -R a=rwx ${TESTDATA} .PHONY: run-bench run-bench: binary bench @@ -242,6 +248,7 @@ clean: rm -rf hack rm -rf test/data/zot-test rm -rf test/data/zot-cve-test + rm -rf test/data/zot-cve-java-test rm -rf pkg/extensions/build .PHONY: run diff --git a/go.mod b/go.mod index 4886f515..414be095 100644 --- a/go.mod +++ b/go.mod @@ -58,6 +58,7 @@ require ( github.com/opencontainers/distribution-spec/specs-go v0.0.0-20230117141039-067a0f5b0e25 github.com/sigstore/cosign/v2 v2.0.2 github.com/swaggo/http-swagger v1.3.4 + modernc.org/sqlite v1.20.3 oras.land/oras-go/v2 v2.2.0 ) @@ -111,6 +112,7 @@ require ( github.com/gosuri/uitable v0.0.4 // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/jmoiron/sqlx v1.3.5 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect github.com/liamg/iamgo v0.0.9 // indirect @@ -142,6 +144,7 @@ require ( github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 // indirect github.com/rubenv/sql-migrate v1.2.0 // indirect github.com/samber/lo v1.37.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect @@ -164,6 +167,15 @@ require ( k8s.io/cli-runtime v0.26.3 // indirect k8s.io/component-base v0.26.3 // indirect k8s.io/kubectl v0.26.3 // indirect + lukechampine.com/uint128 v1.2.0 // indirect + modernc.org/cc/v3 v3.40.0 // indirect + modernc.org/ccgo/v3 v3.16.13 // indirect + modernc.org/libc v1.22.2 // indirect + modernc.org/mathutil v1.5.0 // indirect + modernc.org/memory v1.5.0 // indirect + modernc.org/opt v0.1.3 // indirect + modernc.org/strutil v1.1.3 // indirect + modernc.org/token v1.0.1 // indirect oras.land/oras-go v1.2.3 // indirect sigs.k8s.io/kustomize/api v0.12.1 // indirect sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect diff --git a/go.sum b/go.sum index 983d9bce..101fd37d 100644 --- a/go.sum +++ b/go.sum @@ -937,7 +937,7 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c h1:lvddKcYTQ545ADhBujtIJmqQrZBDsGo7XIMbAQe/sNY= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.3 h1:FAgZmpLl/SXurPEZyCMPBIiiYeTbqfjlbdnCNTAkbGE= github.com/google/s2a-go v0.1.3/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= @@ -1094,6 +1094,7 @@ github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0Lh github.com/karrick/godirwalk v1.16.1 h1:DynhcF+bztK8gooS0+NDJFrdNZjJ3gzVzC545UNA9iw= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -1416,7 +1417,9 @@ github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b h1:zd/2RN github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b/go.mod h1:KjY0wibdYKc4DYkerHSbguaf3JeIPGhNJBp2BNiFH78= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578 h1:VstopitMQi3hZP0fzvnsLmzXZdQGc4bEcgu24cp+d4M= +github.com/remyoudompheng/bigfft v0.0.0-20230126093431-47fa9a501578/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= @@ -2431,15 +2434,29 @@ k8s.io/kubectl v0.26.3/go.mod h1:02+gv7Qn4dupzN3fi/9OvqqdW+uG/4Zi56vc4Zmsp1g= k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 h1:kmDqav+P+/5e1i9tFfHq1qcF3sOrDp+YEkVDAHu7Jwk= k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= +modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= +modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= modernc.org/libc v1.22.2 h1:4U7v51GyhlWqQmwCHj28Rdq2Yzwk55ovjFrdPjs8Hb0= +modernc.org/libc v1.22.2/go.mod h1:uvQavJ1pZ0hIoC/jfqNoMLURIMhKzINIWypNM17puug= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= +modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= +modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/sqlite v1.20.3 h1:SqGJMMxjj1PHusLxdYxeQSodg7Jxn9WWkaAQjKrntZs= +modernc.org/sqlite v1.20.3/go.mod h1:zKcGyrICaxNTMEHSr1HQ2GUraP0j+845GYw37+EyT6A= modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.15.0 h1:oY+JeD11qVVSgVvodMJsu7Edf8tr5E/7tuhF5cNYz34= modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg= +modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.7.0 h1:xkDw/KepgEjeizO2sNco+hqYkU12taxQFqPEmgm1GWE= oras.land/oras-go v1.2.3 h1:v8PJl+gEAntI1pJ/LCrDgsuk+1PKVavVEPsYIHFE5uY= oras.land/oras-go v1.2.3/go.mod h1:M/uaPdYklze0Vf3AakfarnpoEckvw0ESbRdN8Z1vdJg= oras.land/oras-go/v2 v2.2.0 h1:E1fqITD56Eg5neZbxBtAdZVgDHD6wBabJo6xESTcQyo= diff --git a/pkg/cli/extensions_test.go b/pkg/cli/extensions_test.go index 7d935103..b28b6d01 100644 --- a/pkg/cli/extensions_test.go +++ b/pkg/cli/extensions_test.go @@ -641,6 +641,13 @@ func TestServeSearchEnabled(t *testing.T) { substring := `"Extensions":{"Search":{"Enable":true,"CVE":null}` found, err := readLogFileAndSearchString(logPath, substring, readLogFileTimeout) + + if !found { + data, err := os.ReadFile(logPath) + So(err, ShouldBeNil) + t.Log(string(data)) + } + So(found, ShouldBeTrue) So(err, ShouldBeNil) }) @@ -680,20 +687,26 @@ func TestServeSearchEnabledCVE(t *testing.T) { // to avoid data race when multiple go routines write to trivy DB instance. WaitTillTrivyDBDownloadStarted(tempDir) - substring := "\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000,\"Trivy\":null}}" + // The default config handling logic will convert the 1h interval to a 2h interval + substring := "\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":7200000000000,\"Trivy\":" + + "{\"DBRepository\":\"ghcr.io/aquasecurity/trivy-db\",\"JavaDBRepository\":\"ghcr.io/aquasecurity/trivy-java-db\"}}}" found, err := readLogFileAndSearchString(logPath, substring, readLogFileTimeout) + + defer func() { + if !found { + data, err := os.ReadFile(logPath) + So(err, ShouldBeNil) + t.Log(string(data)) + } + }() + So(found, ShouldBeTrue) So(err, ShouldBeNil) found, err = readLogFileAndSearchString(logPath, "updating the CVE database", readLogFileTimeout) So(found, ShouldBeTrue) So(err, ShouldBeNil) - - substring = "CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing." //nolint:lll // gofumpt conflicts with lll - found, err = readLogFileAndSearchString(logPath, substring, readLogFileTimeout) - So(found, ShouldBeTrue) - So(err, ShouldBeNil) }) } @@ -729,6 +742,13 @@ func TestServeSearchEnabledNoCVE(t *testing.T) { substring := `"Extensions":{"Search":{"Enable":true,"CVE":null}` //nolint:lll // gofumpt conflicts with lll found, err := readLogFileAndSearchString(logPath, substring, readLogFileTimeout) + + if !found { + data, err := os.ReadFile(logPath) + So(err, ShouldBeNil) + t.Log(string(data)) + } + So(found, ShouldBeTrue) So(err, ShouldBeNil) }) diff --git a/pkg/cli/root.go b/pkg/cli/root.go index a1bb4bf4..744f43c0 100644 --- a/pkg/cli/root.go +++ b/pkg/cli/root.go @@ -448,7 +448,7 @@ func validateAuthzPolicies(config *config.Config) error { return nil } -//nolint:gocyclo +//nolint:gocyclo,cyclop,nestif func applyDefaultValues(config *config.Config, viperInstance *viper.Viper) { defaultVal := true @@ -503,6 +503,35 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper) { if config.Extensions.Search.Enable == nil { config.Extensions.Search.Enable = &defaultVal } + + if *config.Extensions.Search.Enable && config.Extensions.Search.CVE != nil { + defaultUpdateInterval, _ := time.ParseDuration("2h") + + if config.Extensions.Search.CVE.UpdateInterval < defaultUpdateInterval { + config.Extensions.Search.CVE.UpdateInterval = defaultUpdateInterval + + log.Warn().Msg("CVE update interval set to too-short interval < 2h, " + + "changing update duration to 2 hours and continuing.") + } + + if config.Extensions.Search.CVE.Trivy == nil { + config.Extensions.Search.CVE.Trivy = &extconf.TrivyConfig{} + } + + if config.Extensions.Search.CVE.Trivy.DBRepository == "" { + defaultDBDownloadURL := "ghcr.io/aquasecurity/trivy-db" + log.Info().Str("trivyDownloadURL", defaultDBDownloadURL). + Msg("Config: using default Trivy DB download URL.") + config.Extensions.Search.CVE.Trivy.DBRepository = defaultDBDownloadURL + } + + if config.Extensions.Search.CVE.Trivy.JavaDBRepository == "" { + defaultJavaDBDownloadURL := "ghcr.io/aquasecurity/trivy-java-db" + log.Info().Str("trivyJavaDownloadURL", defaultJavaDBDownloadURL). + Msg("Config: using default Trivy Java DB download URL.") + config.Extensions.Search.CVE.Trivy.JavaDBRepository = defaultJavaDBDownloadURL + } + } } if config.Extensions.Metrics != nil { diff --git a/pkg/extensions/config/config.go b/pkg/extensions/config/config.go index d750f1d9..c84de457 100644 --- a/pkg/extensions/config/config.go +++ b/pkg/extensions/config/config.go @@ -42,7 +42,8 @@ type CVEConfig struct { } type TrivyConfig struct { - DBRepository string // default is "ghcr.io/aquasecurity/trivy-db" + DBRepository string // default is "ghcr.io/aquasecurity/trivy-db" + JavaDBRepository string // default is "ghcr.io/aquasecurity/trivy-java-db" } type MetricsConfig struct { diff --git a/pkg/extensions/extension_search.go b/pkg/extensions/extension_search.go index 1a105497..aefa104b 100644 --- a/pkg/extensions/extension_search.go +++ b/pkg/extensions/extension_search.go @@ -45,27 +45,16 @@ func GetCVEInfo(config *config.Config, storeController storage.StoreController, return nil } - dbRepository := "" + dbRepository := config.Extensions.Search.CVE.Trivy.DBRepository + javaDBRepository := config.Extensions.Search.CVE.Trivy.JavaDBRepository - if config.Extensions.Search.CVE.Trivy != nil { - dbRepository = config.Extensions.Search.CVE.Trivy.DBRepository - } - - return cveinfo.NewCVEInfo(storeController, repoDB, dbRepository, log) + return cveinfo.NewCVEInfo(storeController, repoDB, dbRepository, javaDBRepository, log) } func EnableSearchExtension(config *config.Config, storeController storage.StoreController, repoDB repodb.RepoDB, taskScheduler *scheduler.Scheduler, cveInfo CveInfo, log log.Logger, ) { if config.Extensions.Search != nil && *config.Extensions.Search.Enable && config.Extensions.Search.CVE != nil { - defaultUpdateInterval, _ := time.ParseDuration("2h") - - if config.Extensions.Search.CVE.UpdateInterval < defaultUpdateInterval { - config.Extensions.Search.CVE.UpdateInterval = defaultUpdateInterval - - log.Warn().Msg("CVE update interval set to too-short interval < 2h, changing update duration to 2 hours and continuing.") //nolint:lll // gofumpt conflicts with lll - } - updateInterval := config.Extensions.Search.CVE.UpdateInterval downloadTrivyDB(updateInterval, taskScheduler, cveInfo, log) @@ -77,6 +66,7 @@ func EnableSearchExtension(config *config.Config, storeController storage.StoreC func downloadTrivyDB(interval time.Duration, sch *scheduler.Scheduler, cveInfo CveInfo, log log.Logger) { generator := NewTrivyTaskGenerator(interval, cveInfo, log) + log.Info().Msg("Submitting CVE DB update scheduler") sch.SubmitGenerator(generator, interval, scheduler.HighPriority) } diff --git a/pkg/extensions/extension_search_test.go b/pkg/extensions/extension_search_test.go index 793172ff..6dae4efb 100644 --- a/pkg/extensions/extension_search_test.go +++ b/pkg/extensions/extension_search_test.go @@ -46,7 +46,7 @@ func TestTrivyDBGenerator(t *testing.T) { DefaultStore: mocks.MockedImageStore{}, } - cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", logger) + cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "ghcr.io/project-zot/trivy-db", "", logger) generator := NewTrivyTaskGenerator(time.Minute, cveInfo, logger) sch.SubmitGenerator(generator, 12000*time.Millisecond, scheduler.HighPriority) diff --git a/pkg/extensions/search/cve/cve.go b/pkg/extensions/search/cve/cve.go index 694c343c..8f96c5e7 100644 --- a/pkg/extensions/search/cve/cve.go +++ b/pkg/extensions/search/cve/cve.go @@ -46,9 +46,9 @@ type BaseCveInfo struct { } func NewCVEInfo(storeController storage.StoreController, repoDB repodb.RepoDB, - dbRepository string, log log.Logger, + dbRepository, javaDBRepository string, log log.Logger, ) *BaseCveInfo { - scanner := trivy.NewScanner(storeController, repoDB, dbRepository, log) + scanner := trivy.NewScanner(storeController, repoDB, dbRepository, javaDBRepository, log) return &BaseCveInfo{ Log: log, diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 1317c0ce..45711447 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -327,7 +327,7 @@ func TestImageFormat(t *testing.T) { err = repodb.ParseStorage(repoDB, storeController, log) So(err, ShouldBeNil) - cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", log) + cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "ghcr.io/project-zot/trivy-db", "", log) isValidImage, err := cveInfo.Scanner.IsImageFormatScannable("zot-test", "") So(err, ShouldNotBeNil) @@ -390,7 +390,7 @@ func TestImageFormat(t *testing.T) { DefaultStore: mocks.MockedImageStore{}, } - cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", log) + cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "ghcr.io/project-zot/trivy-db", "", log) isScanable, err := cveInfo.Scanner.IsImageFormatScannable("repo", "tag") So(err, ShouldBeNil) diff --git a/pkg/extensions/search/cve/trivy/scanner.go b/pkg/extensions/search/cve/trivy/scanner.go index 994b81ac..bfb25950 100644 --- a/pkg/extensions/search/cve/trivy/scanner.go +++ b/pkg/extensions/search/cve/trivy/scanner.go @@ -12,10 +12,12 @@ import ( "github.com/aquasecurity/trivy/pkg/commands/operation" fanalTypes "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/flag" + "github.com/aquasecurity/trivy/pkg/javadb" "github.com/aquasecurity/trivy/pkg/types" regTypes "github.com/google/go-containerregistry/pkg/v1/types" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" + _ "modernc.org/sqlite" zerr "zotregistry.io/zot/errors" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" @@ -24,11 +26,9 @@ import ( "zotregistry.io/zot/pkg/storage" ) -const defaultDBRepository = "ghcr.io/aquasecurity/trivy-db" - // getNewScanOptions sets trivy configuration values for our scans and returns them as // a trivy Options structure. -func getNewScanOptions(dir, dbRepository string) *flag.Options { +func getNewScanOptions(dir, dbRepository, javaDBRepository string) *flag.Options { scanOptions := flag.Options{ GlobalOptions: flag.GlobalOptions{ CacheDir: dir, @@ -41,8 +41,10 @@ func getNewScanOptions(dir, dbRepository string) *flag.Options { VulnType: []string{types.VulnTypeOS, types.VulnTypeLibrary}, }, DBOptions: flag.DBOptions{ - DBRepository: dbRepository, - SkipDBUpdate: true, + DBRepository: dbRepository, + JavaDBRepository: javaDBRepository, + SkipDBUpdate: true, + SkipJavaDBUpdate: true, }, ReportOptions: flag.ReportOptions{ Format: "table", @@ -65,33 +67,30 @@ type cveTrivyController struct { } type Scanner struct { - repoDB repodb.RepoDB - cveController cveTrivyController - storeController storage.StoreController - log log.Logger - dbLock *sync.Mutex - cache *CveCache - dbRepository string + repoDB repodb.RepoDB + cveController cveTrivyController + storeController storage.StoreController + log log.Logger + dbLock *sync.Mutex + cache *CveCache + dbRepository string + javaDBRepository string } func NewScanner(storeController storage.StoreController, - repoDB repodb.RepoDB, dbRepository string, log log.Logger, + repoDB repodb.RepoDB, dbRepository, javaDBRepository string, log log.Logger, ) *Scanner { cveController := cveTrivyController{} subCveConfig := make(map[string]*flag.Options) - if dbRepository == "" { - dbRepository = defaultDBRepository - } - if storeController.DefaultStore != nil { imageStore := storeController.DefaultStore rootDir := imageStore.RootDir() cacheDir := path.Join(rootDir, "_trivy") - opts := getNewScanOptions(cacheDir, dbRepository) + opts := getNewScanOptions(cacheDir, dbRepository, javaDBRepository) cveController.DefaultCveConfig = opts } @@ -101,7 +100,7 @@ func NewScanner(storeController storage.StoreController, rootDir := storage.RootDir() cacheDir := path.Join(rootDir, "_trivy") - opts := getNewScanOptions(cacheDir, dbRepository) + opts := getNewScanOptions(cacheDir, dbRepository, javaDBRepository) subCveConfig[route] = opts } @@ -110,13 +109,14 @@ func NewScanner(storeController storage.StoreController, cveController.SubCveConfig = subCveConfig return &Scanner{ - log: log, - repoDB: repoDB, - cveController: cveController, - storeController: storeController, - dbLock: &sync.Mutex{}, - cache: NewCveCache(10000, log), //nolint:gomnd - dbRepository: dbRepository, + log: log, + repoDB: repoDB, + cveController: cveController, + storeController: storeController, + dbLock: &sync.Mutex{}, + cache: NewCveCache(10000, log), //nolint:gomnd + dbRepository: dbRepository, + javaDBRepository: javaDBRepository, } } @@ -371,14 +371,25 @@ func (scanner Scanner) updateDB(dbDir string) error { ctx := context.Background() - err := operation.DownloadDB(ctx, "dev", dbDir, scanner.dbRepository, false, false, - fanalTypes.RegistryOptions{Insecure: false}) + registryOpts := fanalTypes.RegistryOptions{Insecure: false} + + err := operation.DownloadDB(ctx, "dev", dbDir, scanner.dbRepository, false, false, registryOpts) if err != nil { scanner.log.Error().Err(err).Str("dbDir", dbDir).Msg("Error downloading Trivy DB to destination dir") return err } + if scanner.javaDBRepository != "" { + javadb.Init(dbDir, scanner.javaDBRepository, false, false, registryOpts.Insecure) + + if err := javadb.Update(); err != nil { + scanner.log.Error().Err(err).Str("dbDir", dbDir).Msg("Error downloading Trivy Java DB to destination dir") + + return err + } + } + scanner.log.Debug().Str("dbDir", dbDir).Msg("Finished downloading Trivy DB to destination dir") return nil diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index c73dce06..189d983b 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -102,7 +102,7 @@ func TestMultipleStoragePath(t *testing.T) { err = repodb.ParseStorage(repoDB, storeController, log) So(err, ShouldBeNil) - scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log) + scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", "", log) So(scanner.storeController.DefaultStore, ShouldNotBeNil) So(scanner.storeController.SubStore, ShouldNotBeNil) @@ -197,9 +197,23 @@ func TestTrivyLibraryErrors(t *testing.T) { err = repodb.ParseStorage(repoDB, storeController, log) So(err, ShouldBeNil) - scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log) + // Download DB fails for missing DB url + scanner := NewScanner(storeController, repoDB, "", "", log) + + err = scanner.UpdateDB() + So(err, ShouldNotBeNil) + + // Download DB fails for invalid Java DB + scanner = NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", + "ghcr.io/project-zot/trivy-not-db", log) + + err = scanner.UpdateDB() + So(err, ShouldNotBeNil) + + // Download DB passes for valid Trivy DB url, and missing Trivy Java DB url + // Download DB is necessary since DB download on scan is disabled + scanner = NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", "", log) - // Download DB since DB download on scan is disabled err = scanner.UpdateDB() So(err, ShouldBeNil) @@ -381,7 +395,8 @@ func TestImageScannable(t *testing.T) { storeController := storage.StoreController{} storeController.DefaultStore = store - scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log) + scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", + "ghcr.io/aquasecurity/trivy-java-db", log) Convey("Valid image should be scannable", t, func() { result, err := scanner.IsImageFormatScannable("repo1", "valid") @@ -434,6 +449,9 @@ func TestDefaultTrivyDBUrl(t *testing.T) { err := test.CopyFiles("../../../../../test/data/zot-test", path.Join(rootDir, "zot-test")) So(err, ShouldBeNil) + err = test.CopyFiles("../../../../../test/data/zot-cve-java-test", path.Join(rootDir, "zot-cve-java-test")) + So(err, ShouldBeNil) + log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) @@ -456,18 +474,25 @@ func TestDefaultTrivyDBUrl(t *testing.T) { err = repodb.ParseStorage(repoDB, storeController, log) So(err, ShouldBeNil) - // Use empty string for DB repository, the default url would be used internally - scanner := NewScanner(storeController, repoDB, "", log) + scanner := NewScanner(storeController, repoDB, "ghcr.io/aquasecurity/trivy-db", + "ghcr.io/aquasecurity/trivy-java-db", log) // Download DB since DB download on scan is disabled err = scanner.UpdateDB() So(err, ShouldBeNil) + // Scanning image img := "zot-test:0.0.1" - // Scanning image opts := scanner.getTrivyOptions(img) _, err = scanner.runTrivy(opts) So(err, ShouldBeNil) + + // Scanning image containing a jar file + img = "zot-cve-java-test:0.0.1" + + opts = scanner.getTrivyOptions(img) + _, err = scanner.runTrivy(opts) + So(err, ShouldBeNil) }) } diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index d76791b6..6834e774 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -712,7 +712,8 @@ func TestRepoListWithNewestImage(t *testing.T) { defer ctlr.Shutdown() - substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000,\"Trivy\":{\"DBRepository\":\"ghcr.io/project-zot/trivy-db\"}}}" //nolint: lll + substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000," + + "\"Trivy\":{\"DBRepository\":\"ghcr.io/project-zot/trivy-db\",\"JavaDBRepository\":\"\"}}}" found, err := readFileAndSearchString(logPath, substring, 2*time.Minute) So(found, ShouldBeTrue) So(err, ShouldBeNil) @@ -3384,7 +3385,8 @@ func TestGlobalSearch(t *testing.T) { defer ctlr.Shutdown() // Wait for trivy db to download - substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000,\"Trivy\":{\"DBRepository\":\"ghcr.io/project-zot/trivy-db\"}}}" //nolint: lll + substring := "{\"Search\":{\"Enable\":true,\"CVE\":{\"UpdateInterval\":3600000000000," + + "\"Trivy\":{\"DBRepository\":\"ghcr.io/project-zot/trivy-db\",\"JavaDBRepository\":\"\"}}}" found, err := readFileAndSearchString(logPath, substring, 2*time.Minute) So(found, ShouldBeTrue) So(err, ShouldBeNil) diff --git a/pkg/storage/local/local.go b/pkg/storage/local/local.go index d5b5450a..7788a697 100644 --- a/pkg/storage/local/local.go +++ b/pkg/storage/local/local.go @@ -1968,6 +1968,6 @@ func (is *ImageStoreLocal) RunDedupeBlobs(interval time.Duration, sch *scheduler Log: is.log, } - sch.SubmitGenerator(generator, interval, scheduler.HighPriority) + sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) } } diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index 6837386e..4e47752d 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -1644,5 +1644,5 @@ func (is *ObjectStorage) RunDedupeBlobs(interval time.Duration, sch *scheduler.S Log: is.log, } - sch.SubmitGenerator(generator, interval, scheduler.HighPriority) + sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) } diff --git a/pkg/test/common_test.go b/pkg/test/common_test.go index 544bcd3d..3675a9ab 100644 --- a/pkg/test/common_test.go +++ b/pkg/test/common_test.go @@ -723,7 +723,7 @@ func TestReadLogFileAndSearchString(t *testing.T) { defer os.Remove(logPath) Convey("Invalid path", t, func() { - _, err = test.ReadLogFileAndSearchString("invalidPath", "DB update completed, next update scheduled", 90*time.Second) + _, err = test.ReadLogFileAndSearchString("invalidPath", "DB update completed, next update scheduled", 1*time.Second) So(err, ShouldNotBeNil) }) @@ -750,7 +750,7 @@ func TestReadLogFileAndCountStringOccurence(t *testing.T) { Convey("Invalid path", t, func() { _, err = test.ReadLogFileAndCountStringOccurence("invalidPath", - "DB update completed, next update scheduled", 90*time.Second, 1) + "DB update completed, next update scheduled", 1*time.Second, 1) So(err, ShouldNotBeNil) })