feat(compliance): Add JSON output option

This adds a new --json flag to the compliance subcommand, which
will output the compliance test results as minified JSON to stdout.

Also a few other small additions:
- Exit 1 if compliance tests fail
- Use random port for test server using freeport library (added)

Signed-off-by: Josh Dolitsky <393494+jdolitsky@users.noreply.github.com>

README.md

@@ -56,7 +56,7 @@ Examples of config files are available in [examples/](examples/) dir.
 # Compliance checks
 
 ```
-bin/zot compliance -H hostIP -P port [-V "all"]
+bin/zot compliance -H hostIP -P port [-V "all"] [--json]
 ```
 
 # Ecosystem

WORKSPACE

@@ -1072,6 +1072,13 @@ go_repository(
     version = "v0.0.0-20160315200505-970db520ece7",
 )
 
+go_repository(
+    name = "com_github_phayes_freeport",
+    importpath = "github.com/phayes/freeport",
+    sum = "h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=",
+    version = "v0.0.0-20180830031419-95f893ade6f2",
+)
+
 go_repository(
     name = "com_github_pquerna_otp",
     importpath = "github.com/pquerna/otp",

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/opencontainers/distribution-spec v1.0.0-rc0
 	github.com/opencontainers/go-digest v1.0.0-rc1
 	github.com/opencontainers/image-spec v1.0.1
+	github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2
 	github.com/rs/zerolog v1.17.2
 	github.com/smartystreets/goconvey v1.6.4
 	github.com/spf13/cobra v0.0.5

go.sum

@@ -134,6 +134,8 @@ github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVo
 github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 h1:JhzVVoYvbOACxoUmOs6V/G4D5nPVUW73rKvXxP4XUJc=
+github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2/go.mod h1:iIss55rKnNBTvrwdmkUpLnDpZoAHvWaiq5+iMmen4AE=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

pkg/cli/root.go

@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"os"
 	"testing"
 
 	"github.com/anuvu/zot/errors"
@@ -102,6 +103,9 @@ func NewRootCmd() *cobra.Command {
 			default:
 				v1_0_0.CheckWorkflows(t, complianceConfig)
 			}
+			if t.Failed() {
+				os.Exit(1)
+			}
 		},
 	}
 
@@ -121,6 +125,8 @@ func NewRootCmd() *cobra.Command {
 
 	complianceCmd.Flags().StringVarP(&complianceConfig.Version, "version", "V", "all",
 		"OCI dist-spec version to check")
+	complianceCmd.Flags().BoolVarP(&complianceConfig.OutputJSON, "json", "j", false,
+		"output test results as JSON")
 
 	rootCmd := &cobra.Command{
 		Use:   "zot",

pkg/compliance/config.go

@@ -4,6 +4,7 @@ type Config struct {
 	Address    string
 	Port       string
 	Version    string
+	OutputJSON bool
 }
 
 func NewConfig() *Config {

pkg/compliance/v1_0_0/BUILD.bazel

@@ -11,6 +11,7 @@ go_library(
         "@com_github_opencontainers_go_digest//:go_default_library",
         "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library",
         "@com_github_smartystreets_goconvey//convey:go_default_library",
+        "@com_github_smartystreets_goconvey//convey/reporting:go_default_library",
         "@in_gopkg_resty_v1//:go_default_library",
     ],
 )
@@ -23,6 +24,7 @@ go_test(
     deps = [
         "//pkg/api:go_default_library",
         "//pkg/compliance:go_default_library",
+        "@com_github_phayes_freeport//:go_default_library",
         "@in_gopkg_resty_v1//:go_default_library",
     ],
 )

pkg/compliance/v1_0_0/check.go

@@ -5,6 +5,9 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
+	"os"
+	"strings"
 	"testing"
 
 	"github.com/anuvu/zot/pkg/api"
@@ -12,6 +15,7 @@ import (
 	godigest "github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	. "github.com/smartystreets/goconvey/convey"
+	"github.com/smartystreets/goconvey/convey/reporting"
 	"gopkg.in/resty.v1"
 )
 
@@ -19,6 +23,12 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
 	if config == nil || config.Address == "" || config.Port == "" {
 		panic("insufficient config")
 	}
+
+	if config.OutputJSON {
+		outputJSONEnter()
+		defer outputJSONExit()
+	}
+
 	baseURL := fmt.Sprintf("http://%s:%s", config.Address, config.Port)
 
 	fmt.Println("------------------------------")
@@ -451,3 +461,60 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
 		})
 	})
 }
+
+var (
+	old  *os.File
+	r    *os.File
+	w    *os.File
+	outC chan string
+)
+
+func outputJSONEnter() {
+	// this env var instructs goconvey to output results to JSON (stdout)
+	os.Setenv("GOCONVEY_REPORTER", "json")
+
+	// stdout capture copied from: https://stackoverflow.com/a/29339052
+	old = os.Stdout
+	// keep backup of the real stdout
+	r, w, _ = os.Pipe()
+	outC = make(chan string)
+	os.Stdout = w
+
+	// copy the output in a separate goroutine so printing can't block indefinitely
+	go func() {
+		var buf bytes.Buffer
+		io.Copy(&buf, r)
+		outC <- buf.String()
+	}()
+}
+
+func outputJSONExit() {
+	// back to normal state
+	w.Close()
+	os.Stdout = old // restoring the real stdout
+	out := <-outC
+
+	// The output of JSON is combined with regular output, so we look for the
+	// first occurrence of the "{" character and take everything after that
+	rawJSON := "[{" + strings.Join(strings.Split(out, "{")[1:], "{")
+	rawJSON = strings.Replace(rawJSON, reporting.OpenJson, "", 1)
+	rawJSON = strings.Replace(rawJSON, reporting.CloseJson, "", 1)
+	tmp := strings.Split(rawJSON, ",")
+	rawJSON = strings.Join(tmp[0:len(tmp)-1], ",") + "]"
+
+	rawJSONMinified := validateMinifyRawJSON(rawJSON)
+	fmt.Println(rawJSONMinified)
+}
+
+func validateMinifyRawJSON(rawJSON string) string {
+	var j interface{}
+	err := json.Unmarshal([]byte(rawJSON), &j)
+	if err != nil {
+		panic(err)
+	}
+	rawJSONBytesMinified, err := json.Marshal(j)
+	if err != nil {
+		panic(err)
+	}
+	return string(rawJSONBytesMinified)
+}

pkg/compliance/v1_0_0/check_test.go

@@ -1,63 +1,83 @@
+//nolint (dupl)
 package v1_0_0_test
 
 import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"os"
 	"testing"
 	"time"
 
 	"github.com/anuvu/zot/pkg/api"
 	"github.com/anuvu/zot/pkg/compliance"
 	"github.com/anuvu/zot/pkg/compliance/v1_0_0"
+	"github.com/phayes/freeport"
 	"gopkg.in/resty.v1"
 )
 
-const (
-	Address = "127.0.0.1"
-	Port    = "8080"
+var (
+	listenAddress = "127.0.0.1"
 )
 
 func TestWorkflows(t *testing.T) {
-	v1_0_0.CheckWorkflows(t, &compliance.Config{Address: Address, Port: Port})
+	ctrl, randomPort := startServer()
+	defer stopServer(ctrl)
+	v1_0_0.CheckWorkflows(t, &compliance.Config{
+		Address: listenAddress,
+		Port:    randomPort,
+	})
 }
 
-func TestMain(m *testing.M) {
-	config := api.NewConfig()
-	config.HTTP.Address = Address
-	config.HTTP.Port = Port
-	c := api.NewController(config)
-	dir, err := ioutil.TempDir("", "oci-repo-test")
+func TestWorkflowsOutputJSON(t *testing.T) {
+	ctrl, randomPort := startServer()
+	defer stopServer(ctrl)
+	v1_0_0.CheckWorkflows(t, &compliance.Config{
+		Address:    listenAddress,
+		Port:       randomPort,
+		OutputJSON: true,
+	})
+}
 
+// start local server on random open port
+func startServer() (*api.Controller, string) {
+	portInt, err := freeport.GetFreePort()
 	if err != nil {
 		panic(err)
 	}
-	//defer os.RemoveAll(dir)
-	c.Config.Storage.RootDirectory = dir
+	randomPort := fmt.Sprintf("%d", portInt)
+	fmt.Println(randomPort)
 
+	config := api.NewConfig()
+	config.HTTP.Address = listenAddress
+	config.HTTP.Port = randomPort
+	ctrl := api.NewController(config)
+	dir, err := ioutil.TempDir("", "oci-repo-test")
+	if err != nil {
+		panic(err)
+	}
+
+	//defer os.RemoveAll(dir)
+	ctrl.Config.Storage.RootDirectory = dir
 	go func() {
 		// this blocks
-		if err := c.Run(); err != nil {
+		if err := ctrl.Run(); err != nil {
 			return
 		}
 	}()
 
-	BaseURL := fmt.Sprintf("http://%s:%s", Address, Port)
-
+	baseURL := fmt.Sprintf("http://%s:%s", listenAddress, randomPort)
 	for {
 		// poll until ready
-		resp, _ := resty.R().Get(BaseURL)
+		resp, _ := resty.R().Get(baseURL)
 		if resp.StatusCode() == 404 {
 			break
 		}
-
 		time.Sleep(100 * time.Millisecond)
 	}
 
-	status := m.Run()
-	ctx := context.Background()
-	_ = c.Server.Shutdown(ctx)
-
-	os.Exit(status)
+	return ctrl, randomPort
+}
+
+func stopServer(ctrl *api.Controller) {
+	ctrl.Server.Shutdown(context.Background())
 }