feat(pprof): add profiling route handler to debug runtime (#1818)

(cherry picked from commit 56ddb70f624e7070ad0d3531d498675f9f82c664)

Signed-off-by: Alex Stan <alexandrustan96@yahoo.ro>
Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
Co-authored-by: Alex Stan <alexandrustan96@yahoo.ro>

Makefile

@@ -33,8 +33,8 @@ OS ?= $(shell go env GOOS)
 ARCH ?= $(shell go env GOARCH)
 
 BENCH_OUTPUT ?= stdout
-ALL_EXTENSIONS = debug,imagetrust,lint,metrics,mgmt,scrub,search,sync,ui,userprefs
+ALL_EXTENSIONS = debug,imagetrust,lint,metrics,mgmt,profile,scrub,search,sync,ui,userprefs
-EXTENSIONS ?= sync,search,scrub,metrics,lint,ui,mgmt,userprefs,imagetrust
+EXTENSIONS ?= sync,search,scrub,metrics,lint,ui,mgmt,profile,userprefs,imagetrust
 UI_DEPENDENCIES := search,mgmt,userprefs
 # freebsd/arm64 not supported for pie builds
 BUILDMODE_FLAGS := -buildmode=pie

pkg/api/routes.go

@@ -39,6 +39,7 @@ import (
 	apiErr "zotregistry.io/zot/pkg/api/errors"
 	zcommon "zotregistry.io/zot/pkg/common"
 	gqlPlayground "zotregistry.io/zot/pkg/debug/gqlplayground"
+	pprof "zotregistry.io/zot/pkg/debug/pprof"
 	debug "zotregistry.io/zot/pkg/debug/swagger"
 	ext "zotregistry.io/zot/pkg/extensions"
 	syncConstants "zotregistry.io/zot/pkg/extensions/sync/constants"
@@ -178,6 +179,8 @@ func (rh *RouteHandler) SetupRoutes() {
 	debug.SetupSwaggerRoutes(rh.c.Config, rh.c.Router, authHandler, rh.c.Log)
 	// gql playground
 	gqlPlayground.SetupGQLPlaygroundRoutes(prefixedRouter, rh.c.StoreController, rh.c.Log)
+	// pprof
+	pprof.SetupPprofRoutes(rh.c.Config, prefixedRouter, authHandler, rh.c.Log)
 
 	// Preconditions for enabling the actual extension routes are part of extensions themselves
 	ext.SetupMetricsRoutes(rh.c.Config, rh.c.Router, authHandler, rh.c.Log, rh.c.Metrics)

pkg/debug/constants/consts.go

@@ -3,4 +3,5 @@ package constants
 const (
 	Debug                 = "/_zot/debug"
 	GQLPlaygroundEndpoint = Debug + "/graphql-playground"
+	ProfilingEndpoint     = "/_zot/pprof/"
 )

pkg/debug/pprof/pprof.go

@@ -0,0 +1,153 @@
+//go:build profile
+// +build profile
+
+package pprof
+
+import (
+	"bytes"
+	"fmt"
+	"html"
+	"io"
+	"net/http"
+	"net/http/pprof"
+	"net/url"
+	runPprof "runtime/pprof"
+	"sort"
+	"strings"
+
+	"github.com/gorilla/mux"
+
+	"zotregistry.io/zot/pkg/api/config"
+	registryConst "zotregistry.io/zot/pkg/api/constants"
+	zcommon "zotregistry.io/zot/pkg/common"
+	"zotregistry.io/zot/pkg/debug/constants"
+	"zotregistry.io/zot/pkg/log"
+)
+
+type profileEntry struct {
+	Name  string
+	Href  string
+	Desc  string
+	Count int
+}
+
+var profileDescriptions = map[string]string{ //nolint: gochecknoglobals
+	"allocs":       "A sampling of all past memory allocations",
+	"block":        "Stack traces that led to blocking on synchronization primitives",
+	"cmdline":      "The command line invocation of the current program",
+	"goroutine":    "Stack traces of all current goroutines. Use debug=2 as a query parameter to export in the same format as an unrecovered panic.",  //nolint: lll
+	"heap":         "A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.", //nolint: lll
+	"mutex":        "Stack traces of holders of contended mutexes",
+	"profile":      "CPU profile. You can specify the duration in the seconds GET parameter. After you get the profile file, use the go tool pprof command to investigate the profile.", //nolint: lll
+	"threadcreate": "Stack traces that led to the creation of new OS threads",
+	"trace":        "A trace of execution of the current program. You can specify the duration in the seconds GET parameter. After you get the trace file, use the go tool trace command to investigate the trace.", //nolint: lll
+}
+
+func SetupPprofRoutes(conf *config.Config, router *mux.Router, authFunc mux.MiddlewareFunc,
+	log log.Logger,
+) {
+	// If authn/authz are enabled the endpoints for pprof should be available only to admins
+	pprofRouter := router.PathPrefix(constants.ProfilingEndpoint).Subrouter()
+	pprofRouter.Use(zcommon.AuthzOnlyAdminsMiddleware(conf))
+	pprofRouter.Methods(http.MethodGet).Handler(http.HandlerFunc(
+		func(w http.ResponseWriter, r *http.Request) {
+			if name, found := strings.CutPrefix(r.URL.Path,
+				registryConst.RoutePrefix+constants.ProfilingEndpoint); found {
+				if name != "" {
+					switch name {
+					case "profile": // not available through pprof.Handler
+						pprof.Profile(w, r)
+
+						return
+					case "trace": // not available through pprof.Handler
+						pprof.Trace(w, r)
+
+						return
+					default:
+						pprof.Handler(name).ServeHTTP(w, r)
+
+						return
+					}
+				}
+			}
+
+			var profiles []profileEntry
+			for _, p := range runPprof.Profiles() {
+				profiles = append(profiles, profileEntry{
+					Name:  p.Name(),
+					Href:  p.Name(),
+					Desc:  profileDescriptions[p.Name()],
+					Count: p.Count(),
+				})
+			}
+
+			// Adding other profiles exposed from within this package
+			for _, p := range []string{"cmdline", "profile", "trace"} {
+				profiles = append(profiles, profileEntry{
+					Name: p,
+					Href: p,
+					Desc: profileDescriptions[p],
+				})
+			}
+
+			sort.Slice(profiles, func(i, j int) bool {
+				return profiles[i].Name < profiles[j].Name
+			})
+
+			if err := indexTmplExecute(w, profiles); err != nil {
+				log.Print(err)
+			}
+		}))
+}
+
+func indexTmplExecute(writer io.Writer, profiles []profileEntry) error {
+	var buff bytes.Buffer
+
+	buff.WriteString(`<html>
+<head>
+<title>/v2/_zot/pprof/</title>
+<style>
+.profile-name{
+	display:inline-block;
+	width:6rem;
+}
+</style>
+</head>
+<body>
+/debug/pprof/
+<br>
+<p>Set debug=1 as a query parameter to export in legacy text format</p>
+<br>
+Types of profiles available:
+<table>
+<thead><td>Count</td><td>Profile</td></thead>
+`)
+
+	for _, profile := range profiles {
+		link := &url.URL{Path: profile.Href, RawQuery: "debug=1"}
+		fmt.Fprintf(&buff, "<tr><td>%d</td><td><a href='%s'>%s</a></td></tr>\n",
+			profile.Count, link, html.EscapeString(profile.Name))
+	}
+
+	buff.WriteString(`</table>
+<a href="goroutine?debug=2">full goroutine stack dump</a>
+<br>
+<p>
+Profile Descriptions:
+<ul>
+`)
+
+	for _, profile := range profiles {
+		fmt.Fprintf(&buff, "<li><div class=profile-name>%s: </div> %s</li>\n",
+			html.EscapeString(profile.Name), html.EscapeString(profile.Desc))
+	}
+
+	buff.WriteString(`</ul>
+</p>
+</body>
+</html>`)
+
+	_, err := writer.Write(buff.Bytes())
+
+	return err
+}

pkg/debug/pprof/pprof.md

@@ -0,0 +1,33 @@
+# Profiling in Zot
+
+This project gives the user the posibility to debug and profile the runtime to find relevant data such as CPU intensive function calls. An in-depth article on profiling in Go can be found [here](https://go.dev/blog/pprof).
+
+A call to http://localhost:8080/v2/_zot/pprof/ would list the following available profiles, wrapped in an HTML file, with count values prior to change due to the runtime:
+
+```
+Types of profiles available:
+Count	Profile
+95	allocs
+0	block
+0	cmdline
+11	goroutine
+95	heap
+0	mutex
+0	profile
+13	threadcreate
+0	trace
+full goroutine stack dump
+```
+
+For example, the following can be used to gather the cpu profile for the amount of seconds specified as a query parameter, and then the results are stored in `cpu.prof` file:
+```
+curl -sK -v http://localhost:8080/v2/_zot/pprof/profile?seconds=30 > cpu.prof
+```
+
+Then, the user can use the `go tool pprof` to analyze the information generated previously in `cpu.prof`. The following command boots up an http server with a GUI and multiple charts that represent the data.
+```
+go tool pprof -http=:9090 cpu.prof
+```
+A flamegraph example would look like the following:
+
+<img src="flamegraph.png" height="50%">

pkg/debug/pprof/pprof_disabled.go

@@ -0,0 +1,18 @@
+//go:build !profile
+// +build !profile
+
+package pprof
+
+import (
+	"github.com/gorilla/mux"
+
+	"zotregistry.io/zot/pkg/api/config"
+	"zotregistry.io/zot/pkg/log" //nolint:goimports
+)
+
+func SetupPprofRoutes(conf *config.Config, router *mux.Router, authFunc mux.MiddlewareFunc,
+	log log.Logger,
+) {
+	log.Warn().Msg("skipping enabling pprof extension because given zot binary " +
+		"doesn't include this feature, please build a binary that does so")
+}

pkg/debug/pprof/pprof_test.go

@@ -0,0 +1,189 @@
+//go:build profile
+// +build profile
+
+package pprof_test
+
+import (
+	"net/http"
+	"os"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+	"gopkg.in/resty.v1"
+
+	"zotregistry.io/zot/pkg/api"
+	"zotregistry.io/zot/pkg/api/config"
+	"zotregistry.io/zot/pkg/api/constants"
+	debugConstants "zotregistry.io/zot/pkg/debug/constants"
+	"zotregistry.io/zot/pkg/test"
+)
+
+func TestProfilingAuthz(t *testing.T) {
+	Convey("Make a new controller", t, func() {
+		port := test.GetFreePort()
+		baseURL := test.GetBaseURL(port)
+		adminUsername := "admin"
+		adminPassword := "admin"
+		username := "test"
+		password := "test"
+		authorizationAllRepos := "**"
+
+		testCreds := test.GetCredString(adminUsername, adminPassword) +
+			"\n" + test.GetCredString(username, password)
+		htpasswdPath := test.MakeHtpasswdFileFromString(testCreds)
+		defer os.Remove(htpasswdPath)
+
+		conf := config.New()
+		conf.HTTP.Port = port
+		conf.Storage.RootDirectory = t.TempDir()
+
+		Convey("Test with no access control", func() {
+			ctlr := api.NewController(conf)
+			cm := test.NewControllerManager(ctlr)
+			cm.StartAndWait(port)
+			defer cm.StopServer()
+
+			// unauthenticated clients should have access to /v2/
+			resp, err := resty.R().Get(baseURL + "/v2/")
+			So(err, ShouldBeNil)
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+
+			// unauthenticated clients should have access to the profiling endpoints
+			resp, err = resty.R().Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "trace")
+			So(err, ShouldBeNil)
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+
+			resp, err = resty.R().SetQueryParam("seconds", "1").
+				Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "profile")
+			So(err, ShouldBeNil)
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+
+			resp, err = resty.R().Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "goroutine")
+			So(err, ShouldBeNil)
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+
+			// test building the index
+			resp, err = resty.R().Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint)
+			So(err, ShouldBeNil)
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+		})
+
+		Convey("Test with authenticated users and no anonymous policy", func() {
+			conf.HTTP.Auth = &config.AuthConfig{
+				HTPasswd: config.AuthHTPasswd{
+					Path: htpasswdPath,
+				},
+			}
+			conf.HTTP.AccessControl = &config.AccessControlConfig{
+				Repositories: config.Repositories{
+					authorizationAllRepos: config.PolicyGroup{
+						Policies: []config.Policy{
+							{
+								Users:   []string{username},
+								Actions: []string{"read", "create"},
+							},
+						},
+						DefaultPolicy: []string{},
+					},
+				},
+				AdminPolicy: config.Policy{
+					Users:   []string{adminUsername},
+					Actions: []string{},
+				},
+			}
+
+			ctlr := api.NewController(conf)
+			cm := test.NewControllerManager(ctlr)
+			cm.StartAndWait(port)
+			defer cm.StopServer()
+
+			// unauthenticated clients should not have access to /v2/
+			resp, err := resty.R().Get(baseURL + "/v2/")
+			So(err, ShouldBeNil)
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusUnauthorized)
+
+			// unauthenticated clients should not have access to the profiling endpoint
+			resp, err = resty.R().Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "trace")
+			So(err, ShouldBeNil)
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusUnauthorized)
+
+			// authenticated clients without permissions should not have access to the profiling endpoint
+			resp, err = resty.R().SetBasicAuth(username, password).
+				Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "trace")
+			So(err, ShouldBeNil)
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusForbidden)
+
+			// authenticated clients with admin permissions should have access to the profiling endpoint
+			resp, err = resty.R().SetBasicAuth(adminUsername, adminPassword).
+				Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "trace")
+			So(err, ShouldBeNil)
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+		})
+
+		Convey("Test with authenticated users and anonymous policy", func() {
+			conf.HTTP.Auth = &config.AuthConfig{
+				HTPasswd: config.AuthHTPasswd{
+					Path: htpasswdPath,
+				},
+			}
+			conf.HTTP.AccessControl = &config.AccessControlConfig{
+				Repositories: config.Repositories{
+					authorizationAllRepos: config.PolicyGroup{
+						Policies: []config.Policy{
+							{
+								Users:   []string{username},
+								Actions: []string{"read", "create"},
+							},
+						},
+						DefaultPolicy:   []string{},
+						AnonymousPolicy: []string{"read"},
+					},
+				},
+				AdminPolicy: config.Policy{
+					Users:   []string{adminUsername},
+					Actions: []string{},
+				},
+			}
+
+			ctlr := api.NewController(conf)
+			cm := test.NewControllerManager(ctlr)
+			cm.StartAndWait(port)
+			defer cm.StopServer()
+
+			// unauthenticated clients should have access to /v2/
+			resp, err := resty.R().Get(baseURL + "/v2/")
+			So(err, ShouldBeNil)
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+
+			// unauthenticated clients should not have access to the profiling endpoint
+			resp, err = resty.R().Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "trace")
+			So(err, ShouldBeNil)
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusUnauthorized)
+
+			// authenticated clients without permissions should not have access to the profiling endpoint
+			resp, err = resty.R().SetBasicAuth(username, password).
+				Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "trace")
+			So(err, ShouldBeNil)
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusForbidden)
+
+			// authenticated clients with admin permissions should have access to the profiling endpoint
+			resp, err = resty.R().SetBasicAuth(adminUsername, adminPassword).
+				Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "trace")
+			So(err, ShouldBeNil)
+			So(resp, ShouldNotBeNil)
+			So(resp.StatusCode(), ShouldEqual, http.StatusOK)
+		})
+	})
+}

pkg/extensions/extension_image_trust_test.go

@@ -728,6 +728,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[
 		port := test.GetFreePort()
 		testCreds := test.GetCredString("admin", "admin") + "\n" + test.GetCredString("test", "test")
 		htpasswdPath := test.MakeHtpasswdFileFromString(testCreds)
+		defer os.Remove(htpasswdPath)
 
 		conf := config.New()
 		conf.HTTP.Port = port