mirror of
https://github.com/project-zot/zot.git
synced 2024-12-30 22:34:13 -05:00
feat(pprof): add profiling route handler to debug runtime (#1818)
(cherry picked from commit 56ddb70f624e7070ad0d3531d498675f9f82c664) Signed-off-by: Alex Stan <alexandrustan96@yahoo.ro> Signed-off-by: Andrei Aaron <aaaron@luxoft.com> Co-authored-by: Alex Stan <alexandrustan96@yahoo.ro>
This commit is contained in:
Andrei Aaron
GitHub
parent
f8002c7dd3
commit
a11fe2d195
8 changed files with 400 additions and 2 deletions
4
Makefile
4
Makefile
|
|
@ -33,8 +33,8 @@ OS ?= $(shell go env GOOS)
|
|||
ARCH ?= $(shell go env GOARCH)
|
||||
|
||||
BENCH_OUTPUT ?= stdout
|
||||
ALL_EXTENSIONS = debug,imagetrust,lint,metrics,mgmt,scrub,search,sync,ui,userprefs
|
||||
EXTENSIONS ?= sync,search,scrub,metrics,lint,ui,mgmt,userprefs,imagetrust
|
||||
ALL_EXTENSIONS = debug,imagetrust,lint,metrics,mgmt,profile,scrub,search,sync,ui,userprefs
|
||||
EXTENSIONS ?= sync,search,scrub,metrics,lint,ui,mgmt,profile,userprefs,imagetrust
|
||||
UI_DEPENDENCIES := search,mgmt,userprefs
|
||||
# freebsd/arm64 not supported for pie builds
|
||||
BUILDMODE_FLAGS := -buildmode=pie
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ import (
|
|||
apiErr "zotregistry.io/zot/pkg/api/errors"
|
||||
zcommon "zotregistry.io/zot/pkg/common"
|
||||
gqlPlayground "zotregistry.io/zot/pkg/debug/gqlplayground"
|
||||
pprof "zotregistry.io/zot/pkg/debug/pprof"
|
||||
debug "zotregistry.io/zot/pkg/debug/swagger"
|
||||
ext "zotregistry.io/zot/pkg/extensions"
|
||||
syncConstants "zotregistry.io/zot/pkg/extensions/sync/constants"
|
||||
|
|
@ -178,6 +179,8 @@ func (rh *RouteHandler) SetupRoutes() {
|
|||
debug.SetupSwaggerRoutes(rh.c.Config, rh.c.Router, authHandler, rh.c.Log)
|
||||
// gql playground
|
||||
gqlPlayground.SetupGQLPlaygroundRoutes(prefixedRouter, rh.c.StoreController, rh.c.Log)
|
||||
// pprof
|
||||
pprof.SetupPprofRoutes(rh.c.Config, prefixedRouter, authHandler, rh.c.Log)
|
||||
|
||||
// Preconditions for enabling the actual extension routes are part of extensions themselves
|
||||
ext.SetupMetricsRoutes(rh.c.Config, rh.c.Router, authHandler, rh.c.Log, rh.c.Metrics)
|
||||
|
|
|
|||
|
|
@ -3,4 +3,5 @@ package constants
|
|||
const (
|
||||
Debug = "/_zot/debug"
|
||||
GQLPlaygroundEndpoint = Debug + "/graphql-playground"
|
||||
ProfilingEndpoint = "/_zot/pprof/"
|
||||
)
|
||||
|
|
|
|||
153
pkg/debug/pprof/pprof.go
Normal file
153
pkg/debug/pprof/pprof.go
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
//go:build profile
|
||||
// +build profile
|
||||
|
||||
package pprof
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/pprof"
|
||||
"net/url"
|
||||
runPprof "runtime/pprof"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
|
||||
"zotregistry.io/zot/pkg/api/config"
|
||||
registryConst "zotregistry.io/zot/pkg/api/constants"
|
||||
zcommon "zotregistry.io/zot/pkg/common"
|
||||
"zotregistry.io/zot/pkg/debug/constants"
|
||||
"zotregistry.io/zot/pkg/log"
|
||||
)
|
||||
|
||||
type profileEntry struct {
|
||||
Name string
|
||||
Href string
|
||||
Desc string
|
||||
Count int
|
||||
}
|
||||
|
||||
var profileDescriptions = map[string]string{ //nolint: gochecknoglobals
|
||||
"allocs": "A sampling of all past memory allocations",
|
||||
"block": "Stack traces that led to blocking on synchronization primitives",
|
||||
"cmdline": "The command line invocation of the current program",
|
||||
"goroutine": "Stack traces of all current goroutines. Use debug=2 as a query parameter to export in the same format as an unrecovered panic.", //nolint: lll
|
||||
"heap": "A sampling of memory allocations of live objects. You can specify the gc GET parameter to run GC before taking the heap sample.", //nolint: lll
|
||||
"mutex": "Stack traces of holders of contended mutexes",
|
||||
"profile": "CPU profile. You can specify the duration in the seconds GET parameter. After you get the profile file, use the go tool pprof command to investigate the profile.", //nolint: lll
|
||||
"threadcreate": "Stack traces that led to the creation of new OS threads",
|
||||
"trace": "A trace of execution of the current program. You can specify the duration in the seconds GET parameter. After you get the trace file, use the go tool trace command to investigate the trace.", //nolint: lll
|
||||
}
|
||||
|
||||
func SetupPprofRoutes(conf *config.Config, router *mux.Router, authFunc mux.MiddlewareFunc,
|
||||
log log.Logger,
|
||||
) {
|
||||
// If authn/authz are enabled the endpoints for pprof should be available only to admins
|
||||
pprofRouter := router.PathPrefix(constants.ProfilingEndpoint).Subrouter()
|
||||
pprofRouter.Use(zcommon.AuthzOnlyAdminsMiddleware(conf))
|
||||
pprofRouter.Methods(http.MethodGet).Handler(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
if name, found := strings.CutPrefix(r.URL.Path,
|
||||
registryConst.RoutePrefix+constants.ProfilingEndpoint); found {
|
||||
if name != "" {
|
||||
switch name {
|
||||
case "profile": // not available through pprof.Handler
|
||||
pprof.Profile(w, r)
|
||||
|
||||
return
|
||||
case "trace": // not available through pprof.Handler
|
||||
pprof.Trace(w, r)
|
||||
|
||||
return
|
||||
default:
|
||||
pprof.Handler(name).ServeHTTP(w, r)
|
||||
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var profiles []profileEntry
|
||||
for _, p := range runPprof.Profiles() {
|
||||
profiles = append(profiles, profileEntry{
|
||||
Name: p.Name(),
|
||||
Href: p.Name(),
|
||||
Desc: profileDescriptions[p.Name()],
|
||||
Count: p.Count(),
|
||||
})
|
||||
}
|
||||
|
||||
// Adding other profiles exposed from within this package
|
||||
for _, p := range []string{"cmdline", "profile", "trace"} {
|
||||
profiles = append(profiles, profileEntry{
|
||||
Name: p,
|
||||
Href: p,
|
||||
Desc: profileDescriptions[p],
|
||||
})
|
||||
}
|
||||
|
||||
sort.Slice(profiles, func(i, j int) bool {
|
||||
return profiles[i].Name < profiles[j].Name
|
||||
})
|
||||
|
||||
if err := indexTmplExecute(w, profiles); err != nil {
|
||||
log.Print(err)
|
||||
}
|
||||
}))
|
||||
}
|
||||
|
||||
func indexTmplExecute(writer io.Writer, profiles []profileEntry) error {
|
||||
var buff bytes.Buffer
|
||||
|
||||
buff.WriteString(`<html>
|
||||
<head>
|
||||
<title>/v2/_zot/pprof/</title>
|
||||
<style>
|
||||
.profile-name{
|
||||
display:inline-block;
|
||||
width:6rem;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
/debug/pprof/
|
||||
<br>
|
||||
<p>Set debug=1 as a query parameter to export in legacy text format</p>
|
||||
<br>
|
||||
Types of profiles available:
|
||||
<table>
|
||||
<thead><td>Count</td><td>Profile</td></thead>
|
||||
`)
|
||||
|
||||
for _, profile := range profiles {
|
||||
link := &url.URL{Path: profile.Href, RawQuery: "debug=1"}
|
||||
fmt.Fprintf(&buff, "<tr><td>%d</td><td><a href='%s'>%s</a></td></tr>\n",
|
||||
profile.Count, link, html.EscapeString(profile.Name))
|
||||
}
|
||||
|
||||
buff.WriteString(`</table>
|
||||
<a href="goroutine?debug=2">full goroutine stack dump</a>
|
||||
<br>
|
||||
<p>
|
||||
Profile Descriptions:
|
||||
<ul>
|
||||
`)
|
||||
|
||||
for _, profile := range profiles {
|
||||
fmt.Fprintf(&buff, "<li><div class=profile-name>%s: </div> %s</li>\n",
|
||||
html.EscapeString(profile.Name), html.EscapeString(profile.Desc))
|
||||
}
|
||||
|
||||
buff.WriteString(`</ul>
|
||||
</p>
|
||||
</body>
|
||||
</html>`)
|
||||
|
||||
_, err := writer.Write(buff.Bytes())
|
||||
|
||||
return err
|
||||
}
|
||||
33
pkg/debug/pprof/pprof.md
Normal file
33
pkg/debug/pprof/pprof.md
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# Profiling in Zot
|
||||
|
||||
This project gives the user the posibility to debug and profile the runtime to find relevant data such as CPU intensive function calls. An in-depth article on profiling in Go can be found [here](https://go.dev/blog/pprof).
|
||||
|
||||
A call to http://localhost:8080/v2/_zot/pprof/ would list the following available profiles, wrapped in an HTML file, with count values prior to change due to the runtime:
|
||||
|
||||
```
|
||||
Types of profiles available:
|
||||
Count Profile
|
||||
95 allocs
|
||||
0 block
|
||||
0 cmdline
|
||||
11 goroutine
|
||||
95 heap
|
||||
0 mutex
|
||||
0 profile
|
||||
13 threadcreate
|
||||
0 trace
|
||||
full goroutine stack dump
|
||||
```
|
||||
|
||||
For example, the following can be used to gather the cpu profile for the amount of seconds specified as a query parameter, and then the results are stored in `cpu.prof` file:
|
||||
```
|
||||
curl -sK -v http://localhost:8080/v2/_zot/pprof/profile?seconds=30 > cpu.prof
|
||||
```
|
||||
|
||||
Then, the user can use the `go tool pprof` to analyze the information generated previously in `cpu.prof`. The following command boots up an http server with a GUI and multiple charts that represent the data.
|
||||
```
|
||||
go tool pprof -http=:9090 cpu.prof
|
||||
```
|
||||
A flamegraph example would look like the following:
|
||||
|
||||
<img src="flamegraph.png" height="50%">
|
||||
18
pkg/debug/pprof/pprof_disabled.go
Normal file
18
pkg/debug/pprof/pprof_disabled.go
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
//go:build !profile
|
||||
// +build !profile
|
||||
|
||||
package pprof
|
||||
|
||||
import (
|
||||
"github.com/gorilla/mux"
|
||||
|
||||
"zotregistry.io/zot/pkg/api/config"
|
||||
"zotregistry.io/zot/pkg/log" //nolint:goimports
|
||||
)
|
||||
|
||||
func SetupPprofRoutes(conf *config.Config, router *mux.Router, authFunc mux.MiddlewareFunc,
|
||||
log log.Logger,
|
||||
) {
|
||||
log.Warn().Msg("skipping enabling pprof extension because given zot binary " +
|
||||
"doesn't include this feature, please build a binary that does so")
|
||||
}
|
||||
189
pkg/debug/pprof/pprof_test.go
Normal file
189
pkg/debug/pprof/pprof_test.go
Normal file
|
|
@ -0,0 +1,189 @@
|
|||
//go:build profile
|
||||
// +build profile
|
||||
|
||||
package pprof_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
"gopkg.in/resty.v1"
|
||||
|
||||
"zotregistry.io/zot/pkg/api"
|
||||
"zotregistry.io/zot/pkg/api/config"
|
||||
"zotregistry.io/zot/pkg/api/constants"
|
||||
debugConstants "zotregistry.io/zot/pkg/debug/constants"
|
||||
"zotregistry.io/zot/pkg/test"
|
||||
)
|
||||
|
||||
func TestProfilingAuthz(t *testing.T) {
|
||||
Convey("Make a new controller", t, func() {
|
||||
port := test.GetFreePort()
|
||||
baseURL := test.GetBaseURL(port)
|
||||
adminUsername := "admin"
|
||||
adminPassword := "admin"
|
||||
username := "test"
|
||||
password := "test"
|
||||
authorizationAllRepos := "**"
|
||||
|
||||
testCreds := test.GetCredString(adminUsername, adminPassword) +
|
||||
"\n" + test.GetCredString(username, password)
|
||||
htpasswdPath := test.MakeHtpasswdFileFromString(testCreds)
|
||||
defer os.Remove(htpasswdPath)
|
||||
|
||||
conf := config.New()
|
||||
conf.HTTP.Port = port
|
||||
conf.Storage.RootDirectory = t.TempDir()
|
||||
|
||||
Convey("Test with no access control", func() {
|
||||
ctlr := api.NewController(conf)
|
||||
cm := test.NewControllerManager(ctlr)
|
||||
cm.StartAndWait(port)
|
||||
defer cm.StopServer()
|
||||
|
||||
// unauthenticated clients should have access to /v2/
|
||||
resp, err := resty.R().Get(baseURL + "/v2/")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
|
||||
|
||||
// unauthenticated clients should have access to the profiling endpoints
|
||||
resp, err = resty.R().Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "trace")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
|
||||
|
||||
resp, err = resty.R().SetQueryParam("seconds", "1").
|
||||
Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "profile")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
|
||||
|
||||
resp, err = resty.R().Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "goroutine")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
|
||||
|
||||
// test building the index
|
||||
resp, err = resty.R().Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint)
|
||||
So(err, ShouldBeNil)
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
|
||||
})
|
||||
|
||||
Convey("Test with authenticated users and no anonymous policy", func() {
|
||||
conf.HTTP.Auth = &config.AuthConfig{
|
||||
HTPasswd: config.AuthHTPasswd{
|
||||
Path: htpasswdPath,
|
||||
},
|
||||
}
|
||||
conf.HTTP.AccessControl = &config.AccessControlConfig{
|
||||
Repositories: config.Repositories{
|
||||
authorizationAllRepos: config.PolicyGroup{
|
||||
Policies: []config.Policy{
|
||||
{
|
||||
Users: []string{username},
|
||||
Actions: []string{"read", "create"},
|
||||
},
|
||||
},
|
||||
DefaultPolicy: []string{},
|
||||
},
|
||||
},
|
||||
AdminPolicy: config.Policy{
|
||||
Users: []string{adminUsername},
|
||||
Actions: []string{},
|
||||
},
|
||||
}
|
||||
|
||||
ctlr := api.NewController(conf)
|
||||
cm := test.NewControllerManager(ctlr)
|
||||
cm.StartAndWait(port)
|
||||
defer cm.StopServer()
|
||||
|
||||
// unauthenticated clients should not have access to /v2/
|
||||
resp, err := resty.R().Get(baseURL + "/v2/")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusUnauthorized)
|
||||
|
||||
// unauthenticated clients should not have access to the profiling endpoint
|
||||
resp, err = resty.R().Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "trace")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusUnauthorized)
|
||||
|
||||
// authenticated clients without permissions should not have access to the profiling endpoint
|
||||
resp, err = resty.R().SetBasicAuth(username, password).
|
||||
Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "trace")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusForbidden)
|
||||
|
||||
// authenticated clients with admin permissions should have access to the profiling endpoint
|
||||
resp, err = resty.R().SetBasicAuth(adminUsername, adminPassword).
|
||||
Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "trace")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
|
||||
})
|
||||
|
||||
Convey("Test with authenticated users and anonymous policy", func() {
|
||||
conf.HTTP.Auth = &config.AuthConfig{
|
||||
HTPasswd: config.AuthHTPasswd{
|
||||
Path: htpasswdPath,
|
||||
},
|
||||
}
|
||||
conf.HTTP.AccessControl = &config.AccessControlConfig{
|
||||
Repositories: config.Repositories{
|
||||
authorizationAllRepos: config.PolicyGroup{
|
||||
Policies: []config.Policy{
|
||||
{
|
||||
Users: []string{username},
|
||||
Actions: []string{"read", "create"},
|
||||
},
|
||||
},
|
||||
DefaultPolicy: []string{},
|
||||
AnonymousPolicy: []string{"read"},
|
||||
},
|
||||
},
|
||||
AdminPolicy: config.Policy{
|
||||
Users: []string{adminUsername},
|
||||
Actions: []string{},
|
||||
},
|
||||
}
|
||||
|
||||
ctlr := api.NewController(conf)
|
||||
cm := test.NewControllerManager(ctlr)
|
||||
cm.StartAndWait(port)
|
||||
defer cm.StopServer()
|
||||
|
||||
// unauthenticated clients should have access to /v2/
|
||||
resp, err := resty.R().Get(baseURL + "/v2/")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
|
||||
|
||||
// unauthenticated clients should not have access to the profiling endpoint
|
||||
resp, err = resty.R().Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "trace")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusUnauthorized)
|
||||
|
||||
// authenticated clients without permissions should not have access to the profiling endpoint
|
||||
resp, err = resty.R().SetBasicAuth(username, password).
|
||||
Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "trace")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusForbidden)
|
||||
|
||||
// authenticated clients with admin permissions should have access to the profiling endpoint
|
||||
resp, err = resty.R().SetBasicAuth(adminUsername, adminPassword).
|
||||
Get(baseURL + constants.RoutePrefix + debugConstants.ProfilingEndpoint + "trace")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
|
@ -728,6 +728,7 @@ func RunSignatureUploadAndVerificationTests(t *testing.T, cacheDriverParams map[
|
|||
port := test.GetFreePort()
|
||||
testCreds := test.GetCredString("admin", "admin") + "\n" + test.GetCredString("test", "test")
|
||||
htpasswdPath := test.MakeHtpasswdFileFromString(testCreds)
|
||||
defer os.Remove(htpasswdPath)
|
||||
|
||||
conf := config.New()
|
||||
conf.HTTP.Port = port
|
||||
|
|
|
|||
Loading…
Reference in a new issue