ci/cd: add continuous benchmark action

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>

.github/workflows/benchmark.yaml

@@ -0,0 +1,40 @@
+name: "Minimal Continuous Benchmark"
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [main]
+
+jobs:
+  benchmark:
+    name: Performance regression check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: 1.17.x
+      # Run benchmark with `go test -bench` and stores the output to a file
+      - name: Run benchmark
+        run: make BENCH_OUTPUT=ci-cd run-bench
+      # Download previous benchmark result from cache (if exists)
+      - name: Download previous benchmark data
+        uses: actions/cache@v1
+        with:
+          path: ./cache
+          key: ${{ runner.os }}-benchmark
+      # Run `github-action-benchmark` action
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          # What benchmark tool the output.txt came from
+          tool: 'customBiggerIsBetter'
+          # Where the output from the benchmark tool is stored
+          output-file-path: ci-cd.json
+          # Where the previous data file is stored
+          external-data-json-path: ./cache/benchmark-data.json
+          # Workflow will fail when an alert happens
+          fail-on-alert: true
+      # Upload the updated cache file for the next job by actions/cache

Makefile

@@ -12,6 +12,7 @@ GOLINTER := $(TOOLSDIR)/bin/golangci-lint
 NOTATION := $(TOOLSDIR)/bin/notation
 OS ?= linux
 ARCH ?= amd64
+BENCH_OUTPUT ?= stdout
 
 .PHONY: all
 all: swagger binary binary-minimal binary-debug binary-arch binary-arch-minimal cli cli-arch bench bench-arch exporter-minimal verify-config test test-clean check
@@ -64,6 +65,13 @@ test: check-skopeo $(NOTATION)
 	go test -tags extended,containers_image_openpgp -v -trimpath -race -timeout 15m -cover -coverpkg ./... -coverprofile=coverage-extended.txt -covermode=atomic ./...
 	go test -tags minimal,containers_image_openpgp -v -trimpath -race -cover -coverpkg ./... -coverprofile=coverage-minimal.txt -covermode=atomic ./...
 
+.PHONY: run-bench
+run-bench: binary bench
+	bin/zot serve examples/config-minimal.json &
+	sleep 5
+	bin/zb -c 10 -n 100 -o $(BENCH_OUTPUT) http://localhost:8080
+	killall zot
+
 .PHONY: test-clean
 test-clean:
 	$(shell sudo rm -rf /etc/containers/certs.d/127.0.0.1:8089/)

cmd/zb/main.go

@@ -13,7 +13,7 @@ import (
 func NewPerfRootCmd() *cobra.Command {
 	showVersion := false
 
-	var auth, workdir, repo, output string
+	var auth, workdir, repo, outFmt string
 
 	var concurrency, requests int
 
@@ -45,7 +45,7 @@ func NewPerfRootCmd() *cobra.Command {
 
 			requests = concurrency * (requests / concurrency)
 
-			Perf(workdir, url, auth, repo, concurrency, requests)
+			Perf(workdir, url, auth, repo, concurrency, requests, outFmt)
 		},
 	}
 
@@ -59,8 +59,8 @@ func NewPerfRootCmd() *cobra.Command {
 		"Number of multiple requests to make at a time")
 	rootCmd.Flags().IntVarP(&requests, "requests", "n", 1,
 		"Number of requests to perform")
-	rootCmd.Flags().StringVarP(&output, "output-format", "o", "",
+	rootCmd.Flags().StringVarP(&outFmt, "output-format", "o", "",
-		"Output format of test results [default: stdout]")
+		"Output format of test results: stdout (default), json, ci-cd")
 
 	// "version"
 	rootCmd.Flags().BoolVarP(&showVersion, "version", "v", false, "show the version and exit")

cmd/zb/perf.go

@@ -4,6 +4,7 @@ import (
 	"crypto/rand"
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"net/url"
@@ -33,6 +34,7 @@ const (
 	smallBlob            = 1 * MiB
 	mediumBlob           = 10 * MiB
 	largeBlob            = 100 * MiB
+	cicdFmt              = "ci-cd"
 )
 
 // helper routines
@@ -198,7 +200,17 @@ func updateStats(summary *statsSummary, record statsRecord) {
 	summary.latencies = append(summary.latencies, record.latency)
 }
 
-func printStats(requests int, summary *statsSummary) {
+type cicdTestSummary struct {
+	Name  string      `json:"name"`
+	Unit  string      `json:"unit"`
+	Value interface{} `json:"value"`
+	Range string      `json:"range,omitempty"`
+}
+
+//nolint:gochecknoglobals // used only in this test
+var cicdSummary = []cicdTestSummary{}
+
+func printStats(requests int, summary *statsSummary, outFmt string) {
 	log.Printf("============\n")
 	log.Printf("Test name:\t%s", summary.name)
 	log.Printf("Time taken for tests:\t%v", summary.total)
@@ -219,6 +231,18 @@ func printStats(requests int, summary *statsSummary) {
 	log.Printf("%s:\t%v", "p90", summary.latencies[requests*9/10])
 	log.Printf("%s:\t%v", "p99", summary.latencies[requests*99/100])
 	log.Printf("\n")
+
+	// ci/cd
+	if outFmt == cicdFmt {
+		cicdSummary = append(cicdSummary,
+			cicdTestSummary{
+				Name:  summary.name,
+				Unit:  "requests per sec",
+				Value: summary.rps,
+				Range: "3",
+			},
+		)
+	}
 }
 
 // test suites/funcs.
@@ -644,7 +668,7 @@ var testSuite = []testConfig{ // nolint:gochecknoglobals // used only in this te
 	},
 }
 
-func Perf(workdir, url, auth, repo string, concurrency int, requests int) {
+func Perf(workdir, url, auth, repo string, concurrency int, requests int, outFmt string) {
 	// logging
 	log.SetFlags(0)
 	log.SetOutput(tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', tabwriter.TabIndent))
@@ -691,6 +715,17 @@ func Perf(workdir, url, auth, repo string, concurrency int, requests int) {
 
 		sort.Sort(Durations(summary.latencies))
 
-		printStats(requests, &summary)
+		printStats(requests, &summary, outFmt)
+	}
+
+	if outFmt == cicdFmt {
+		jsonOut, err := json.Marshal(cicdSummary)
+		if err != nil {
+			log.Fatal(err) //nolint:gocritic // file closed on exit
+		}
+
+		if err := ioutil.WriteFile(fmt.Sprintf("%s.json", outFmt), jsonOut, defaultFilePerms); err != nil {
+			log.Fatal(err)
+		}
 	}
 }