From 271b916a26be6322a89d96f6d7a72131deb028a2 Mon Sep 17 00:00:00 2001 From: Josh Dolitsky <393494+jdolitsky@users.noreply.github.com> Date: Fri, 13 Dec 2019 14:57:51 -0600 Subject: [PATCH] feat(compliance): Add JSON output option This adds a new --json flag to the compliance subcommand, which will output the compliance test results as minified JSON to stdout. Also a few other small additions: - Exit 1 if compliance tests fail - Use random port for test server using freeport library (added) Signed-off-by: Josh Dolitsky <393494+jdolitsky@users.noreply.github.com> --- README.md | 2 +- WORKSPACE | 7 +++ go.mod | 1 + go.sum | 2 + pkg/cli/root.go | 6 +++ pkg/compliance/config.go | 7 +-- pkg/compliance/v1_0_0/BUILD.bazel | 2 + pkg/compliance/v1_0_0/check.go | 67 +++++++++++++++++++++++++++++ pkg/compliance/v1_0_0/check_test.go | 66 ++++++++++++++++++---------- 9 files changed, 133 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index f9ef95ea..78dfccd0 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ Examples of config files are available in [examples/](examples/) dir. # Compliance checks ``` -bin/zot compliance -H hostIP -P port [-V "all"] +bin/zot compliance -H hostIP -P port [-V "all"] [--json] ``` # Ecosystem diff --git a/WORKSPACE b/WORKSPACE index 5d8fd27b..d254e94f 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1072,6 +1072,13 @@ go_repository( version = "v0.0.0-20160315200505-970db520ece7", ) +go_repository( + name = "com_github_phayes_freeport", + importpath = "github.com/phayes/freeport", + sum = "h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=", + version = "v0.0.0-20180830031419-95f893ade6f2", +) + go_repository( name = "com_github_pquerna_otp", importpath = "github.com/pquerna/otp", diff --git a/go.mod b/go.mod index 135f3d26..54c2370c 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/opencontainers/distribution-spec v1.0.0-rc0 github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/image-spec v1.0.1 + github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/rs/zerolog v1.17.2 github.com/smartystreets/goconvey v1.6.4 github.com/spf13/cobra v0.0.5 diff --git a/go.sum b/go.sum index 8ac37311..849c1d95 100644 --- a/go.sum +++ b/go.sum @@ -134,6 +134,8 @@ github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVo github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc= +github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/pkg/cli/root.go b/pkg/cli/root.go index d5e969dc..c06906bb 100644 --- a/pkg/cli/root.go +++ b/pkg/cli/root.go @@ -1,6 +1,7 @@ package cli import ( + "os" "testing" "github.com/anuvu/zot/errors" @@ -102,6 +103,9 @@ func NewRootCmd() *cobra.Command { default: v1_0_0.CheckWorkflows(t, complianceConfig) } + if t.Failed() { + os.Exit(1) + } }, } @@ -121,6 +125,8 @@ func NewRootCmd() *cobra.Command { complianceCmd.Flags().StringVarP(&complianceConfig.Version, "version", "V", "all", "OCI dist-spec version to check") + complianceCmd.Flags().BoolVarP(&complianceConfig.OutputJSON, "json", "j", false, + "output test results as JSON") rootCmd := &cobra.Command{ Use: "zot", diff --git a/pkg/compliance/config.go b/pkg/compliance/config.go index 75605bda..721d7312 100644 --- a/pkg/compliance/config.go +++ b/pkg/compliance/config.go @@ -1,9 +1,10 @@ package compliance type Config struct { - Address string - Port string - Version string + Address string + Port string + Version string + OutputJSON bool } func NewConfig() *Config { diff --git a/pkg/compliance/v1_0_0/BUILD.bazel b/pkg/compliance/v1_0_0/BUILD.bazel index 2a773aca..32d30ff6 100644 --- a/pkg/compliance/v1_0_0/BUILD.bazel +++ b/pkg/compliance/v1_0_0/BUILD.bazel @@ -11,6 +11,7 @@ go_library( "@com_github_opencontainers_go_digest//:go_default_library", "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library", "@com_github_smartystreets_goconvey//convey:go_default_library", + "@com_github_smartystreets_goconvey//convey/reporting:go_default_library", "@in_gopkg_resty_v1//:go_default_library", ], ) @@ -23,6 +24,7 @@ go_test( deps = [ "//pkg/api:go_default_library", "//pkg/compliance:go_default_library", + "@com_github_phayes_freeport//:go_default_library", "@in_gopkg_resty_v1//:go_default_library", ], ) diff --git a/pkg/compliance/v1_0_0/check.go b/pkg/compliance/v1_0_0/check.go index cedea1a9..a367d8c5 100644 --- a/pkg/compliance/v1_0_0/check.go +++ b/pkg/compliance/v1_0_0/check.go @@ -5,6 +5,9 @@ import ( "bytes" "encoding/json" "fmt" + "io" + "os" + "strings" "testing" "github.com/anuvu/zot/pkg/api" @@ -12,6 +15,7 @@ import ( godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" + "github.com/smartystreets/goconvey/convey/reporting" "gopkg.in/resty.v1" ) @@ -19,6 +23,12 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { if config == nil || config.Address == "" || config.Port == "" { panic("insufficient config") } + + if config.OutputJSON { + outputJSONEnter() + defer outputJSONExit() + } + baseURL := fmt.Sprintf("http://%s:%s", config.Address, config.Port) fmt.Println("------------------------------") @@ -451,3 +461,60 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { }) }) } + +var ( + old *os.File + r *os.File + w *os.File + outC chan string +) + +func outputJSONEnter() { + // this env var instructs goconvey to output results to JSON (stdout) + os.Setenv("GOCONVEY_REPORTER", "json") + + // stdout capture copied from: https://stackoverflow.com/a/29339052 + old = os.Stdout + // keep backup of the real stdout + r, w, _ = os.Pipe() + outC = make(chan string) + os.Stdout = w + + // copy the output in a separate goroutine so printing can't block indefinitely + go func() { + var buf bytes.Buffer + io.Copy(&buf, r) + outC <- buf.String() + }() +} + +func outputJSONExit() { + // back to normal state + w.Close() + os.Stdout = old // restoring the real stdout + out := <-outC + + // The output of JSON is combined with regular output, so we look for the + // first occurrence of the "{" character and take everything after that + rawJSON := "[{" + strings.Join(strings.Split(out, "{")[1:], "{") + rawJSON = strings.Replace(rawJSON, reporting.OpenJson, "", 1) + rawJSON = strings.Replace(rawJSON, reporting.CloseJson, "", 1) + tmp := strings.Split(rawJSON, ",") + rawJSON = strings.Join(tmp[0:len(tmp)-1], ",") + "]" + + rawJSONMinified := validateMinifyRawJSON(rawJSON) + fmt.Println(rawJSONMinified) +} + +func validateMinifyRawJSON(rawJSON string) string { + var j interface{} + err := json.Unmarshal([]byte(rawJSON), &j) + if err != nil { + panic(err) + } + rawJSONBytesMinified, err := json.Marshal(j) + if err != nil { + panic(err) + } + return string(rawJSONBytesMinified) +} diff --git a/pkg/compliance/v1_0_0/check_test.go b/pkg/compliance/v1_0_0/check_test.go index be866fb6..d3f034b2 100644 --- a/pkg/compliance/v1_0_0/check_test.go +++ b/pkg/compliance/v1_0_0/check_test.go @@ -1,63 +1,83 @@ +//nolint (dupl) package v1_0_0_test import ( "context" "fmt" "io/ioutil" - "os" "testing" "time" "github.com/anuvu/zot/pkg/api" "github.com/anuvu/zot/pkg/compliance" "github.com/anuvu/zot/pkg/compliance/v1_0_0" + "github.com/phayes/freeport" "gopkg.in/resty.v1" ) -const ( - Address = "127.0.0.1" - Port = "8080" +var ( + listenAddress = "127.0.0.1" ) func TestWorkflows(t *testing.T) { - v1_0_0.CheckWorkflows(t, &compliance.Config{Address: Address, Port: Port}) + ctrl, randomPort := startServer() + defer stopServer(ctrl) + v1_0_0.CheckWorkflows(t, &compliance.Config{ + Address: listenAddress, + Port: randomPort, + }) } -func TestMain(m *testing.M) { - config := api.NewConfig() - config.HTTP.Address = Address - config.HTTP.Port = Port - c := api.NewController(config) - dir, err := ioutil.TempDir("", "oci-repo-test") +func TestWorkflowsOutputJSON(t *testing.T) { + ctrl, randomPort := startServer() + defer stopServer(ctrl) + v1_0_0.CheckWorkflows(t, &compliance.Config{ + Address: listenAddress, + Port: randomPort, + OutputJSON: true, + }) +} +// start local server on random open port +func startServer() (*api.Controller, string) { + portInt, err := freeport.GetFreePort() if err != nil { panic(err) } - //defer os.RemoveAll(dir) - c.Config.Storage.RootDirectory = dir + randomPort := fmt.Sprintf("%d", portInt) + fmt.Println(randomPort) + config := api.NewConfig() + config.HTTP.Address = listenAddress + config.HTTP.Port = randomPort + ctrl := api.NewController(config) + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + + //defer os.RemoveAll(dir) + ctrl.Config.Storage.RootDirectory = dir go func() { // this blocks - if err := c.Run(); err != nil { + if err := ctrl.Run(); err != nil { return } }() - BaseURL := fmt.Sprintf("http://%s:%s", Address, Port) - + baseURL := fmt.Sprintf("http://%s:%s", listenAddress, randomPort) for { // poll until ready - resp, _ := resty.R().Get(BaseURL) + resp, _ := resty.R().Get(baseURL) if resp.StatusCode() == 404 { break } - time.Sleep(100 * time.Millisecond) } - status := m.Run() - ctx := context.Background() - _ = c.Server.Shutdown(ctx) - - os.Exit(status) + return ctrl, randomPort +} + +func stopServer(ctrl *api.Controller) { + ctrl.Server.Shutdown(context.Background()) }