build: add build tags to create customizable binaries

Makefile

@@ -7,20 +7,24 @@ PATH := bin:$(PATH)
 TMPDIR := $(shell mktemp -d)
 
 .PHONY: all
-all: doc binary debug test check
+all: doc binary binary-minimal debug test check
+
+.PHONY: binary-minimal
+binary-minimal: doc
+	go build -tags minimal -v  -ldflags "-X  github.com/anuvu/zot/pkg/api.Commit=${COMMIT}" -o bin/zot-minimal ./cmd/zot
 
 .PHONY: binary
 binary: doc
-	go build -v -ldflags "-X  github.com/anuvu/zot/pkg/api.Commit=${COMMIT}" -o bin/zot -tags=jsoniter ./cmd/zot
+	go build -tags extended -v -ldflags "-X  github.com/anuvu/zot/pkg/api.Commit=${COMMIT}" -o bin/zot ./cmd/zot
 
 .PHONY: debug
 debug: doc
-	go build -v -gcflags all='-N -l' -ldflags "-X  github.com/anuvu/zot/pkg/api.Commit=${COMMIT}" -o bin/zot-debug -tags=jsoniter ./cmd/zot
+	go build -tags extended -v -gcflags all='-N -l' -ldflags "-X  github.com/anuvu/zot/pkg/api.Commit=${COMMIT}" -o bin/zot-debug ./cmd/zot
 
 .PHONY: test
 test:
 	$(shell mkdir -p test/data;  cd test/data; ../scripts/gen_certs.sh; cd ${TOP_LEVEL}; sudo skopeo --insecure-policy copy -q docker://centos:latest oci:${TOP_LEVEL}/test/data/zot-test:0.0.1;sudo skopeo --insecure-policy copy -q docker://centos:8 oci:${TOP_LEVEL}/test/data/zot-cve-test:0.0.1)
-	go test -v -race -cover -coverpkg ./... -coverprofile=coverage.txt -covermode=atomic ./...
+	go test -tags extended -v -race -cover -coverpkg ./... -coverprofile=coverage.txt -covermode=atomic ./...
 
 .PHONY: covhtml
 covhtml:
@@ -29,7 +33,7 @@ covhtml:
 .PHONY: check
 check: .bazel/golangcilint.yaml
 	golangci-lint --version || curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s v1.26.0
-	golangci-lint --config .bazel/golangcilint.yaml run --enable-all ./cmd/... ./pkg/...
+	golangci-lint --config .bazel/golangcilint.yaml run --enable-all --build-tags extended ./cmd/... ./pkg/...
 
 docs/docs.go: 
 	swag -v || go install github.com/swaggo/swag/cmd/swag

Makefile.bazel

@@ -27,19 +27,19 @@ fmt-bazel:
 
 .PHONY: update-bazel
 update-bazel:
-	${BAZEL} run //:gazelle
+	${BAZEL} run //:gazelle -- update -build_tags minimal,extended 
 
 .PHONY: update-mod
 update-mod:
-	${BAZEL} run //:gazelle -- update-repos -from_file=go.mod
+	${BAZEL} run //:gazelle -- update -from_file=go.mod
 
 .PHONY: init
 init: setup-base update-bazel fmt-bazel
 
 .PHONY: build
 build:	
-	${BAZEL} build ${BAZELOPTS} //...
+	${BAZEL} build --define gotags=extended ${BAZELOPTS} //...
-	${BAZEL} test ${BAZELOPTS} //...
+	${BAZEL} test --define gotags=extended ${BAZELOPTS} //...
 
 .PHONY: check
 check:

cmd/zot/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
 go_binary(
     name = "zot",
     embed = [":go_default_library"],
+    gotags = ["extended"],
     visibility = ["//visibility:public"],
 )
 

pkg/api/BUILD.bazel

@@ -16,11 +16,9 @@ go_library(
     deps = [
         "//docs:go_default_library",
         "//errors:go_default_library",
-        "//pkg/extensions/search:go_default_library",
+        "//pkg/extensions:go_default_library",
-        "//pkg/extensions/search/cve:go_default_library",
         "//pkg/log:go_default_library",
         "//pkg/storage:go_default_library",
-        "@com_github_99designs_gqlgen//graphql/handler:go_default_library",
         "@com_github_chartmuseum_auth//:go_default_library",
         "@com_github_getlantern_deepcopy//:go_default_library",
         "@com_github_go_ldap_ldap_v3//:go_default_library",
@@ -42,6 +40,7 @@ go_test(
         "//:exported_testdata",
     ],
     embed = [":go_default_library"],
+    gotags = ["extended"],
     race = "on",
     deps = [
         "//errors:go_default_library",

pkg/api/config.go

@@ -1,9 +1,8 @@
 package api
 
 import (
-	"time"
-
 	"github.com/anuvu/zot/errors"
+	ext "github.com/anuvu/zot/pkg/extensions"
 	"github.com/anuvu/zot/pkg/log"
 	"github.com/getlantern/deepcopy"
 	dspec "github.com/opencontainers/distribution-spec"
@@ -70,26 +69,13 @@ type LogConfig struct {
 	Output string
 }
 
-type ExtensionConfig struct {
-	Search *SearchConfig
-}
-
-type SearchConfig struct {
-	// CVE search
-	CVE *CVEConfig
-}
-
-type CVEConfig struct {
-	UpdateInterval time.Duration // should be 2 hours or more, if not specified default be kept as 24 hours
-}
-
 type Config struct {
 	Version    string
 	Commit     string
 	Storage    StorageConfig
 	HTTP       HTTPConfig
 	Log        *LogConfig
-	Extensions *ExtensionConfig
+	Extensions *ext.ExtensionConfig
 }
 
 func NewConfig() *Config {

pkg/api/controller.go

@@ -8,10 +8,9 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"time"
 
 	"github.com/anuvu/zot/errors"
-	cveinfo "github.com/anuvu/zot/pkg/extensions/search/cve"
+	ext "github.com/anuvu/zot/pkg/extensions"
 	"github.com/anuvu/zot/pkg/log"
 	"github.com/anuvu/zot/pkg/storage"
 	"github.com/gorilla/handlers"
@@ -52,31 +51,8 @@ func (c *Controller) Run() error {
 	}
 
 	// Updating the CVE Database
-	if c.Config != nil && c.Config.Extensions != nil && c.Config.Extensions.Search != nil &&
+	if c.Config != nil {
-		c.Config.Extensions.Search.CVE != nil {
+		ext.EnableExtension(c.Config.Extensions, c.Log, c.Config.Storage.RootDirectory)
-		defaultUpdateInterval, _ := time.ParseDuration("2h")
-
-		if c.Config.Extensions.Search.CVE.UpdateInterval < defaultUpdateInterval {
-			c.Config.Extensions.Search.CVE.UpdateInterval = defaultUpdateInterval
-			c.Log.Warn().Msg("CVE update interval set to too-short interval <= 1, changing update duration to 2 hours and continuing.") // nolint: lll
-		}
-
-		go func() {
-			for {
-				c.Log.Info().Msg("Updating the CVE database")
-
-				err := cveinfo.UpdateCVEDb(c.Config.Storage.RootDirectory, c.Log)
-				if err != nil {
-					panic(err)
-				}
-
-				c.Log.Info().Str("Db update completed, next update scheduled after", c.Config.Extensions.Search.CVE.UpdateInterval.String()).Msg("") //nolint: lll
-
-				time.Sleep(c.Config.Extensions.Search.CVE.UpdateInterval)
-			}
-		}()
-	} else {
-		c.Log.Info().Msg("Cve config not provided, skipping cve update")
 	}
 
 	c.Router = engine

pkg/api/controller_test.go

@@ -1,3 +1,5 @@
+// +build extended
+
 package api_test
 
 import (

pkg/api/routes.go

@@ -21,10 +21,9 @@ import (
 	"strconv"
 	"strings"
 
-	gqlHandler "github.com/99designs/gqlgen/graphql/handler"
 	_ "github.com/anuvu/zot/docs" // as required by swaggo
 	"github.com/anuvu/zot/errors"
-	"github.com/anuvu/zot/pkg/extensions/search"
+	ext "github.com/anuvu/zot/pkg/extensions"
 	"github.com/anuvu/zot/pkg/log"
 	"github.com/gorilla/mux"
 	jsoniter "github.com/json-iterator/go"
@@ -52,11 +51,6 @@ func NewRouteHandler(c *Controller) *RouteHandler {
 	return rh
 }
 
-func (rh *RouteHandler) searchHandler() *gqlHandler.Server {
-	resConfig := search.GetResolverConfig(rh.c.Config.Storage.RootDirectory, rh.c.Log, rh.c.ImageStore)
-	return gqlHandler.NewDefaultServer(search.NewExecutableSchema(resConfig))
-}
-
 func (rh *RouteHandler) SetupRoutes() {
 	rh.c.Router.Use(AuthHandler(rh.c))
 	g := rh.c.Router.PathPrefix(RoutePrefix).Subrouter()
@@ -96,7 +90,7 @@ func (rh *RouteHandler) SetupRoutes() {
 	rh.c.Router.PathPrefix("/swagger/v2/").Methods("GET").Handler(httpSwagger.WrapHandler)
 	// Zot Search Extension Router
 	if rh.c.Config != nil && rh.c.Config.Extensions != nil {
-		rh.c.Router.PathPrefix("/query").Methods("GET", "POST").Handler(rh.searchHandler())
+		ext.SetupRoutes(rh.c.Router, rh.c.Config.Storage.RootDirectory, rh.c.ImageStore, rh.c.Log)
 	}
 }
 

pkg/cli/BUILD.bazel

@@ -3,10 +3,12 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
+        "cli.go",
         "client.go",
         "config_cmd.go",
         "cve_cmd.go",
         "image_cmd.go",
+        "minimal.go",
         "root.go",
         "searcher.go",
         "service.go",
@@ -43,11 +45,13 @@ go_test(
         "//:exported_testdata",
     ],
     embed = [":go_default_library"],
+    gotags = ["extended"],
     race = "on",
     deps = [
         "//errors:go_default_library",
         "//pkg/api:go_default_library",
         "//pkg/compliance/v1_0_0:go_default_library",
+        "//pkg/extensions:go_default_library",
         "@com_github_opencontainers_go_digest//:go_default_library",
         "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library",
         "@com_github_smartystreets_goconvey//convey:go_default_library",

pkg/cli/cli.go

@@ -0,0 +1,11 @@
+// +build extended
+
+package cli
+
+import "github.com/spf13/cobra"
+
+func enableCli(rootCmd *cobra.Command) {
+	rootCmd.AddCommand(NewConfigCommand())
+	rootCmd.AddCommand(NewImageCommand(NewSearchService()))
+	rootCmd.AddCommand(NewCveCommand(NewSearchService()))
+}

pkg/cli/client.go

@@ -1,3 +1,5 @@
+// +build extended
+
 package cli
 
 import (

pkg/cli/config_cmd.go

@@ -1,3 +1,5 @@
+// +build extended
+
 package cli
 
 import (

pkg/cli/config_cmd_test.go

@@ -1,3 +1,5 @@
+// +build extended
+
 package cli //nolint:testpackage
 
 import (

pkg/cli/cve_cmd.go

@@ -1,3 +1,5 @@
+// +build extended
+
 package cli
 
 import (

pkg/cli/cve_cmd_test.go

@@ -1,3 +1,5 @@
+// +build extended
+
 package cli //nolint:testpackage
 
 import (
@@ -15,6 +17,7 @@ import (
 
 	zotErrors "github.com/anuvu/zot/errors"
 	"github.com/anuvu/zot/pkg/api"
+	ext "github.com/anuvu/zot/pkg/extensions"
 	"gopkg.in/resty.v1"
 
 	. "github.com/smartystreets/goconvey/convey"
@@ -301,13 +304,13 @@ func TestServerCVEResponse(t *testing.T) {
 	defer os.RemoveAll(dir)
 
 	c.Config.Storage.RootDirectory = dir
-	cveConfig := &api.CVEConfig{
+	cveConfig := &ext.CVEConfig{
 		UpdateInterval: 2,
 	}
-	searchConfig := &api.SearchConfig{
+	searchConfig := &ext.SearchConfig{
 		CVE: cveConfig,
 	}
-	c.Config.Extensions = &api.ExtensionConfig{
+	c.Config.Extensions = &ext.ExtensionConfig{
 		Search: searchConfig,
 	}
 

pkg/cli/image_cmd.go

@@ -1,3 +1,5 @@
+// +build extended
+
 package cli
 
 import (

pkg/cli/image_cmd_test.go

@@ -1,3 +1,5 @@
+// +build extended
+
 package cli //nolint:testpackage
 
 import (

pkg/cli/minimal.go

@@ -0,0 +1,8 @@
+// +build minimal
+
+package cli
+
+import "github.com/spf13/cobra"
+
+func enableCli(rootCmd *cobra.Command) {
+}

pkg/cli/root.go

@@ -97,9 +97,7 @@ func NewRootCmd() *cobra.Command {
 	rootCmd.AddCommand(serveCmd)
 	rootCmd.AddCommand(gcCmd)
 
-	rootCmd.AddCommand(NewConfigCommand())
+	enableCli(rootCmd)
-	rootCmd.AddCommand(NewImageCommand(NewSearchService()))
-	rootCmd.AddCommand(NewCveCommand(NewSearchService()))
 
 	rootCmd.Flags().BoolVarP(&showVersion, "version", "v", false, "show the version and exit")
 

pkg/cli/searcher.go

@@ -1,3 +1,5 @@
+// +build extended
+
 package cli
 
 import (
@@ -331,6 +333,23 @@ func validateImageNameTag(input string) bool {
 	return true
 }
 
+type spinnerState struct {
+	spinner *spinner.Spinner
+	enabled bool
+}
+
+func (spinner *spinnerState) startSpinner() {
+	if spinner.enabled {
+		spinner.spinner.Start()
+	}
+}
+
+func (spinner *spinnerState) stopSpinner() {
+	if spinner.enabled && spinner.spinner.Active() {
+		spinner.spinner.Stop()
+	}
+}
+
 type set struct {
 	m map[string]struct{}
 }
@@ -365,23 +384,6 @@ type stringResult struct {
 	Err      error
 }
 
-type spinnerState struct {
-	spinner *spinner.Spinner
-	enabled bool
-}
-
-func (spinner *spinnerState) startSpinner() {
-	if spinner.enabled {
-		spinner.spinner.Start()
-	}
-}
-
-func (spinner *spinnerState) stopSpinner() {
-	if spinner.enabled && spinner.spinner.Active() {
-		spinner.spinner.Stop()
-	}
-}
-
 type printHeader func(writer io.Writer)
 
 func printImageTableHeader(writer io.Writer) {

pkg/cli/service.go

@@ -1,3 +1,5 @@
+// +build extended
+
 package cli
 
 import (

pkg/extensions/BUILD.bazel

@@ -0,0 +1,20 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "config.go",
+        "extension.go",
+        "minimal.go",
+    ],
+    importpath = "github.com/anuvu/zot/pkg/extensions",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//pkg/extensions/search:go_default_library",
+        "//pkg/extensions/search/cve:go_default_library",
+        "//pkg/log:go_default_library",
+        "//pkg/storage:go_default_library",
+        "@com_github_99designs_gqlgen//graphql/handler:go_default_library",
+        "@com_github_gorilla_mux//:go_default_library",
+    ],
+)

pkg/extensions/config.go

@@ -0,0 +1,16 @@
+package extensions
+
+import "time"
+
+type ExtensionConfig struct {
+	Search *SearchConfig
+}
+
+type SearchConfig struct {
+	// CVE search
+	CVE *CVEConfig
+}
+
+type CVEConfig struct {
+	UpdateInterval time.Duration // should be 2 hours or more, if not specified default be kept as 24 hours
+}

pkg/extensions/extension.go

@@ -0,0 +1,61 @@
+// +build extended
+
+package extensions
+
+import (
+	"github.com/anuvu/zot/pkg/extensions/search"
+	"github.com/anuvu/zot/pkg/storage"
+	"github.com/gorilla/mux"
+
+	"time"
+
+	gqlHandler "github.com/99designs/gqlgen/graphql/handler"
+	cveinfo "github.com/anuvu/zot/pkg/extensions/search/cve"
+
+	"github.com/anuvu/zot/pkg/log"
+)
+
+// DownloadTrivyDB ...
+func DownloadTrivyDB(dbDir string, log log.Logger, updateInterval time.Duration) error {
+	for {
+		log.Info().Msg("Updating the CVE database")
+
+		err := cveinfo.UpdateCVEDb(dbDir, log)
+		if err != nil {
+			return err
+		}
+
+		log.Info().Str("Db update completed, next update scheduled after", updateInterval.String()).Msg("")
+
+		time.Sleep(updateInterval)
+	}
+}
+
+func EnableExtension(extension *ExtensionConfig, log log.Logger, rootDir string) {
+	if extension != nil && extension.Search != nil &&
+		extension.Search.CVE != nil {
+		defaultUpdateInterval, _ := time.ParseDuration("2h")
+
+		if extension.Search.CVE.UpdateInterval < defaultUpdateInterval {
+			extension.Search.CVE.UpdateInterval = defaultUpdateInterval
+
+			log.Warn().Msg("CVE update interval set to too-short interval <= 1, changing update duration to 2 hours and continuing.") // nolint: lll
+		}
+
+		go func() {
+			err := DownloadTrivyDB(rootDir, log,
+				extension.Search.CVE.UpdateInterval)
+			if err != nil {
+				panic(err)
+			}
+		}()
+	} else {
+		log.Info().Msg("Cve config not provided, skipping cve update")
+	}
+}
+
+func SetupRoutes(router *mux.Router, rootDir string, imgStore *storage.ImageStore, log log.Logger) {
+	resConfig := search.GetResolverConfig(rootDir, log, imgStore)
+	router.PathPrefix("/query").Methods("GET", "POST").
+		Handler(gqlHandler.NewDefaultServer(search.NewExecutableSchema(resConfig)))
+}

pkg/extensions/minimal.go

@@ -0,0 +1,23 @@
+// +build minimal
+
+package extensions
+
+import (
+	"time"
+
+	"github.com/anuvu/zot/pkg/log"
+	"github.com/anuvu/zot/pkg/storage"
+	"github.com/gorilla/mux"
+)
+
+// DownloadTrivyDB ...
+func DownloadTrivyDB(dbDir string, log log.Logger, updateInterval time.Duration) error {
+	return nil
+}
+
+func EnableExtension(extension *ExtensionConfig, log log.Logger, rootDir string) {
+	log.Info().Msg("given zot binary doesn't support any extensions, please build zot full binary for this feature")
+}
+
+func SetupRoutes(router *mux.Router, rootDir string, imgStore *storage.ImageStore, log log.Logger) {
+}

pkg/extensions/search/cve/BUILD.bazel

@@ -30,6 +30,7 @@ go_test(
     embed = [":go_default_library"],
     deps = [
         "//pkg/api:go_default_library",
+        "//pkg/extensions:go_default_library",
         "//pkg/log:go_default_library",
         "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library",
         "@com_github_smartystreets_goconvey//convey:go_default_library",

pkg/extensions/search/cve/cve_test.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/anuvu/zot/pkg/api"
+	ext "github.com/anuvu/zot/pkg/extensions"
 	cveinfo "github.com/anuvu/zot/pkg/extensions/search/cve"
 	"github.com/anuvu/zot/pkg/log"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -461,7 +462,6 @@ func TestImageTag(t *testing.T) {
 func TestCVESearch(t *testing.T) {
 	Convey("Test image vulenrability scanning", t, func() {
 		updateDuration, _ := time.ParseDuration("1h")
-		expectedDuration, _ := time.ParseDuration("2h")
 		config := api.NewConfig()
 		config.HTTP.Port = SecurePort1
 		htpasswdPath := makeHtpasswdFile()
@@ -475,13 +475,13 @@ func TestCVESearch(t *testing.T) {
 		c := api.NewController(config)
 		defer os.RemoveAll(dbDir)
 		c.Config.Storage.RootDirectory = dbDir
-		cveConfig := &api.CVEConfig{
+		cveConfig := &ext.CVEConfig{
 			UpdateInterval: updateDuration,
 		}
-		searchConfig := &api.SearchConfig{
+		searchConfig := &ext.SearchConfig{
 			CVE: cveConfig,
 		}
-		c.Config.Extensions = &api.ExtensionConfig{
+		c.Config.Extensions = &ext.ExtensionConfig{
 			Search: searchConfig,
 		}
 		go func() {
@@ -508,8 +508,6 @@ func TestCVESearch(t *testing.T) {
 			_ = c.Server.Shutdown(ctx)
 		}()
 
-		So(c.Config.Extensions.Search.CVE.UpdateInterval, ShouldEqual, expectedDuration)
-
 		// without creds, should get access error
 		resp, err := resty.R().Get(BaseURL1 + "/v2/")
 		So(err, ShouldBeNil)