0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2025-01-06 22:40:28 -05:00
zot/pkg/extensions/monitoring/monitoring_test.go
peusebiu 7642e5af98
fix(scheduler): fix data race (#2085)
* fix(scheduler): data race when pushing new tasks

the problem here is that scheduler can be closed in two ways:
- canceling the context given as argument to scheduler.RunScheduler()
- running scheduler.Shutdown()

because of this shutdown can trigger a data race between calling scheduler.inShutdown()
and actually pushing tasks into the pool workers

solved that by keeping a quit channel and listening on both quit channel and ctx.Done()
and closing the worker chan and scheduler afterwards.

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

* refactor(scheduler): refactor into a single shutdown

before this we could stop scheduler either by closing the context
provided to RunScheduler(ctx) or by running Shutdown().

simplify things by getting rid of the external context in RunScheduler().
keep an internal context in the scheduler itself and pass it down to all tasks.

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>

---------

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
2023-12-11 10:00:34 -08:00

516 lines
16 KiB
Go

//go:build metrics
// +build metrics
package monitoring_test
import (
"fmt"
"io"
"math/rand"
"net/http"
"os"
"path"
"testing"
"time"
. "github.com/smartystreets/goconvey/convey"
"gopkg.in/resty.v1"
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/scheduler"
common "zotregistry.io/zot/pkg/storage/common"
test "zotregistry.io/zot/pkg/test/common"
. "zotregistry.io/zot/pkg/test/image-utils"
ociutils "zotregistry.io/zot/pkg/test/oci-utils"
)
func TestExtensionMetrics(t *testing.T) {
Convey("Make a new controller with explicitly enabled metrics", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
rootDir := t.TempDir()
conf.Storage.RootDirectory = rootDir
conf.Extensions = &extconf.ExtensionConfig{}
enabled := true
conf.Extensions.Metrics = &extconf.MetricsConfig{
BaseConfig: extconf.BaseConfig{Enable: &enabled},
Prometheus: &extconf.PrometheusConfig{Path: "/metrics"},
}
ctlr := api.NewController(conf)
So(ctlr, ShouldNotBeNil)
cm := test.NewControllerManager(ctlr)
cm.StartAndWait(port)
defer cm.StopServer()
// improve code coverage
ctlr.Metrics.SendMetric(baseURL)
ctlr.Metrics.ForceSendMetric(baseURL)
So(ctlr.Metrics.IsEnabled(), ShouldBeTrue)
So(ctlr.Metrics.ReceiveMetrics(), ShouldBeNil)
monitoring.ObserveHTTPRepoLatency(ctlr.Metrics,
"/v2/alpine/blobs/uploads/299148f0-0e32-4830-90d2-a3fa744137d9", time.Millisecond)
monitoring.IncDownloadCounter(ctlr.Metrics, "alpine")
monitoring.IncUploadCounter(ctlr.Metrics, "alpine")
srcStorageCtlr := ociutils.GetDefaultStoreController(rootDir, ctlr.Log)
err := WriteImageToFileSystem(CreateDefaultImage(), "alpine", "0.0.1", srcStorageCtlr)
So(err, ShouldBeNil)
monitoring.SetStorageUsage(ctlr.Metrics, rootDir, "alpine")
monitoring.ObserveStorageLockLatency(ctlr.Metrics, time.Millisecond, rootDir, "RWLock")
resp, err := resty.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
respStr := string(resp.Body())
So(respStr, ShouldContainSubstring, "zot_repo_downloads_total{repo=\"alpine\"} 1")
So(respStr, ShouldContainSubstring, "zot_repo_uploads_total{repo=\"alpine\"} 1")
So(respStr, ShouldContainSubstring, "zot_repo_storage_bytes{repo=\"alpine\"}")
So(respStr, ShouldContainSubstring, "zot_storage_lock_latency_seconds_bucket")
So(respStr, ShouldContainSubstring, "zot_storage_lock_latency_seconds_sum")
So(respStr, ShouldContainSubstring, "zot_storage_lock_latency_seconds_bucket")
})
Convey("Make a new controller with disabled metrics extension", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
conf.Storage.RootDirectory = t.TempDir()
conf.Extensions = &extconf.ExtensionConfig{}
var disabled bool
conf.Extensions.Metrics = &extconf.MetricsConfig{BaseConfig: extconf.BaseConfig{Enable: &disabled}}
ctlr := api.NewController(conf)
So(ctlr, ShouldNotBeNil)
cm := test.NewControllerManager(ctlr)
cm.StartAndWait(port)
defer cm.StopServer()
So(ctlr.Metrics.IsEnabled(), ShouldBeFalse)
resp, err := resty.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
})
}
func TestMetricsAuthentication(t *testing.T) {
Convey("test metrics without authentication and metrics enabled", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
ctlr := api.NewController(conf)
ctlr.Config.Storage.RootDirectory = t.TempDir()
cm := test.NewControllerManager(ctlr)
cm.StartAndWait(port)
defer cm.StopServer()
// metrics endpoint not available
resp, err := resty.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusNotFound)
})
Convey("test metrics without authentication and with metrics enabled", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
enabled := true
metricsConfig := &extconf.MetricsConfig{
BaseConfig: extconf.BaseConfig{Enable: &enabled},
Prometheus: &extconf.PrometheusConfig{Path: "/metrics"},
}
conf.Extensions = &extconf.ExtensionConfig{
Metrics: metricsConfig,
}
ctlr := api.NewController(conf)
ctlr.Config.Storage.RootDirectory = t.TempDir()
cm := test.NewControllerManager(ctlr)
cm.StartAndWait(port)
defer cm.StopServer()
// without auth set metrics endpoint is available
resp, err := resty.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
})
Convey("test metrics with authentication and metrics enabled", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
username := generateRandomString()
password := generateRandomString()
metricsuser := generateRandomString()
metricspass := generateRandomString()
content := test.GetCredString(username, password) + "\n" + test.GetCredString(metricsuser, metricspass)
htpasswdPath := test.MakeHtpasswdFileFromString(content)
defer os.Remove(htpasswdPath)
conf.HTTP.Auth = &config.AuthConfig{
HTPasswd: config.AuthHTPasswd{
Path: htpasswdPath,
},
}
enabled := true
metricsConfig := &extconf.MetricsConfig{
BaseConfig: extconf.BaseConfig{Enable: &enabled},
Prometheus: &extconf.PrometheusConfig{Path: "/metrics"},
}
conf.Extensions = &extconf.ExtensionConfig{
Metrics: metricsConfig,
}
ctlr := api.NewController(conf)
ctlr.Config.Storage.RootDirectory = t.TempDir()
cm := test.NewControllerManager(ctlr)
cm.StartAndWait(port)
defer cm.StopServer()
// without credentials
resp, err := resty.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusUnauthorized)
// with wrong credentials
resp, err = resty.R().SetBasicAuth("atacker", "wrongpassword").Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusUnauthorized)
// authenticated users
resp, err = resty.R().SetBasicAuth(username, password).Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
resp, err = resty.R().SetBasicAuth(metricsuser, metricspass).Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
})
}
func TestMetricsAuthorization(t *testing.T) {
const AuthorizationAllRepos = "**"
Convey("Make a new controller with auth & metrics enabled", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
username := generateRandomString()
password := generateRandomString()
metricsuser := generateRandomString()
metricspass := generateRandomString()
content := test.GetCredString(username, password) + "\n" + test.GetCredString(metricsuser, metricspass)
htpasswdPath := test.MakeHtpasswdFileFromString(content)
defer os.Remove(htpasswdPath)
conf.HTTP.Auth = &config.AuthConfig{
HTPasswd: config.AuthHTPasswd{
Path: htpasswdPath,
},
}
enabled := true
metricsConfig := &extconf.MetricsConfig{
BaseConfig: extconf.BaseConfig{Enable: &enabled},
Prometheus: &extconf.PrometheusConfig{Path: "/metrics"},
}
conf.Extensions = &extconf.ExtensionConfig{
Metrics: metricsConfig,
}
Convey("with basic auth: no metrics users in accessControl", func() {
conf.HTTP.AccessControl = &config.AccessControlConfig{
Metrics: config.Metrics{
Users: []string{},
},
}
ctlr := api.NewController(conf)
ctlr.Config.Storage.RootDirectory = t.TempDir()
cm := test.NewControllerManager(ctlr)
cm.StartAndWait(port)
defer cm.StopServer()
// authenticated but not authorized user should not have access to/metrics
client := resty.New()
client.SetBasicAuth(username, password)
resp, err := client.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusUnauthorized)
// authenticated but not authorized user should not have access to/metrics
client.SetBasicAuth(metricsuser, metricspass)
resp, err = client.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusUnauthorized)
})
Convey("with basic auth: metrics users in accessControl", func() {
conf.HTTP.AccessControl = &config.AccessControlConfig{
Metrics: config.Metrics{
Users: []string{metricsuser},
},
}
ctlr := api.NewController(conf)
ctlr.Config.Storage.RootDirectory = t.TempDir()
cm := test.NewControllerManager(ctlr)
cm.StartAndWait(port)
defer cm.StopServer()
// authenticated but not authorized user should not have access to/metrics
client := resty.New()
client.SetBasicAuth(username, password)
resp, err := client.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusForbidden)
// authenticated & authorized user should have access to/metrics
client.SetBasicAuth(metricsuser, metricspass)
resp, err = client.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
})
Convey("with basic auth: with anonymousPolicy in accessControl", func() {
conf.HTTP.AccessControl = &config.AccessControlConfig{
Metrics: config.Metrics{
Users: []string{metricsuser},
},
Repositories: config.Repositories{
AuthorizationAllRepos: config.PolicyGroup{
Policies: []config.Policy{
{
Users: []string{},
Actions: []string{},
},
},
AnonymousPolicy: []string{"read"},
DefaultPolicy: []string{},
},
},
}
ctlr := api.NewController(conf)
ctlr.Config.Storage.RootDirectory = t.TempDir()
cm := test.NewControllerManager(ctlr)
cm.StartAndWait(port)
defer cm.StopServer()
// unauthenticated clients should not have access to /metrics
resp, err := resty.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusUnauthorized)
// unauthenticated clients should not have access to /metrics
resp, err = resty.R().SetBasicAuth("hacker", "trywithwrongpass").Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusUnauthorized)
// authenticated but not authorized user should not have access to/metrics
client := resty.New()
client.SetBasicAuth(username, password)
resp, err = client.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusForbidden)
// authenticated & authorized user should have access to/metrics
client.SetBasicAuth(metricsuser, metricspass)
resp, err = client.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
})
Convey("with basic auth: with adminPolicy in accessControl", func() {
conf.HTTP.AccessControl = &config.AccessControlConfig{
Metrics: config.Metrics{
Users: []string{metricsuser},
},
Repositories: config.Repositories{
AuthorizationAllRepos: config.PolicyGroup{
Policies: []config.Policy{
{
Users: []string{},
Actions: []string{},
},
},
DefaultPolicy: []string{},
},
},
AdminPolicy: config.Policy{
Users: []string{"test"},
Groups: []string{"admins"},
Actions: []string{"read", "create", "update", "delete"},
},
}
ctlr := api.NewController(conf)
ctlr.Config.Storage.RootDirectory = t.TempDir()
cm := test.NewControllerManager(ctlr)
cm.StartAndWait(port)
defer cm.StopServer()
// unauthenticated clients should not have access to /metrics
resp, err := resty.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusUnauthorized)
// unauthenticated clients should not have access to /metrics
resp, err = resty.R().SetBasicAuth("hacker", "trywithwrongpass").Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusUnauthorized)
// authenticated admin user (but not authorized) should not have access to/metrics
client := resty.New()
client.SetBasicAuth(username, password)
resp, err = client.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusForbidden)
// authenticated & authorized user should have access to/metrics
client.SetBasicAuth(metricsuser, metricspass)
resp, err = client.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
})
})
}
func TestPopulateStorageMetrics(t *testing.T) {
Convey("Start a scheduler when metrics enabled", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
rootDir := t.TempDir()
conf.Storage.RootDirectory = rootDir
conf.Extensions = &extconf.ExtensionConfig{}
enabled := true
conf.Extensions.Metrics = &extconf.MetricsConfig{
BaseConfig: extconf.BaseConfig{Enable: &enabled},
Prometheus: &extconf.PrometheusConfig{Path: "/metrics"},
}
logFile, err := os.CreateTemp(t.TempDir(), "zot-log*.txt")
if err != nil {
panic(err)
}
logPath := logFile.Name()
defer os.Remove(logPath)
writers := io.MultiWriter(os.Stdout, logFile)
ctlr := api.NewController(conf)
So(ctlr, ShouldNotBeNil)
ctlr.Log.Logger = ctlr.Log.Output(writers)
cm := test.NewControllerManager(ctlr)
cm.StartAndWait(port)
defer cm.StopServer()
// write a couple of images
srcStorageCtlr := ociutils.GetDefaultStoreController(rootDir, ctlr.Log)
err = WriteImageToFileSystem(CreateDefaultImage(), "alpine", "0.0.1", srcStorageCtlr)
So(err, ShouldBeNil)
err = WriteImageToFileSystem(CreateDefaultImage(), "busybox", "0.0.1", srcStorageCtlr)
So(err, ShouldBeNil)
metrics := monitoring.NewMetricsServer(true, ctlr.Log)
sch := scheduler.NewScheduler(conf, metrics, ctlr.Log)
sch.RunScheduler()
generator := &common.StorageMetricsInitGenerator{
ImgStore: ctlr.StoreController.DefaultStore,
Metrics: ctlr.Metrics,
Log: ctlr.Log,
MaxDelay: 1, // maximum delay between jobs (each job computes repo's storage size)
}
sch.SubmitGenerator(generator, time.Duration(0), scheduler.LowPriority)
// Wait for storage metrics to update
found, err := test.ReadLogFileAndSearchString(logPath,
"computed storage usage for repo alpine", time.Minute)
So(err, ShouldBeNil)
So(found, ShouldBeTrue)
found, err = test.ReadLogFileAndSearchString(logPath,
"computed storage usage for repo busybox", time.Minute)
So(err, ShouldBeNil)
So(found, ShouldBeTrue)
sch.Shutdown()
alpineSize, err := monitoring.GetDirSize(path.Join(rootDir, "alpine"))
So(err, ShouldBeNil)
busyboxSize, err := monitoring.GetDirSize(path.Join(rootDir, "busybox"))
So(err, ShouldBeNil)
resp, err := resty.R().Get(baseURL + "/metrics")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
alpineMetric := fmt.Sprintf("zot_repo_storage_bytes{repo=\"alpine\"} %d", alpineSize)
busyboxMetric := fmt.Sprintf("zot_repo_storage_bytes{repo=\"busybox\"} %d", busyboxSize)
respStr := string(resp.Body())
So(respStr, ShouldContainSubstring, alpineMetric)
So(respStr, ShouldContainSubstring, busyboxMetric)
})
}
func generateRandomString() string {
//nolint: gosec
seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
charset := "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
randomBytes := make([]byte, 10)
for i := range randomBytes {
randomBytes[i] = charset[seededRand.Intn(len(charset))]
}
return string(randomBytes)
}