mirror of
https://github.com/project-zot/zot.git
synced 2024-12-16 21:56:37 -05:00
fix: more accurate storage metrics after zot restart (#1972)
Signed-off-by: Alexei Dodon <adodon@cisco.com>
This commit is contained in:
Alexei Dodon
GitHub
parent
3e6053e1db
commit
a79d79a03a
10 changed files with 195 additions and 20 deletions
|
|
@ -6,7 +6,7 @@ cloud.google.com/go/internal|https://github.com/googleapis/google-cloud-go/blob/
|
|||
cloud.google.com/go/storage|https://github.com/googleapis/google-cloud-go/blob/storage/v1.31.0/storage/LICENSE|Apache-2.0
|
||||
cloud.google.com/go|https://github.com/googleapis/google-cloud-go/blob/v0.110.7/LICENSE|Apache-2.0
|
||||
filippo.io/edwards25519|https://github.com/FiloSottile/edwards25519/blob/v1.0.0/LICENSE|BSD-3-Clause
|
||||
github.com/99designs/gqlgen|https://github.com/99designs/gqlgen/blob/v0.17.39/LICENSE|MIT
|
||||
github.com/99designs/gqlgen|https://github.com/99designs/gqlgen/blob/v0.17.40/LICENSE|MIT
|
||||
github.com/AdaLogics/go-fuzz-headers|https://github.com/AdaLogics/go-fuzz-headers/blob/ced1acdcaa24/LICENSE|Apache-2.0
|
||||
github.com/AdamKorcz/go-118-fuzz-build|https://github.com/AdamKorcz/go-118-fuzz-build/blob/8075edf89bb0/LICENSE|Apache-2.0
|
||||
github.com/AliyunContainerService/ack-ram-tool/pkg/credentials/alibabacloudsdkgo/helper|https://github.com/AliyunContainerService/ack-ram-tool/blob/pkg/credentials/alibabacloudsdkgo/helper/v0.2.0/pkg/credentials/alibabacloudsdkgo/helper/LICENSE|Apache-2.0
|
||||
|
|
@ -63,13 +63,13 @@ github.com/aquasecurity/go-version/pkg/part|https://github.com/aquasecurity/go-v
|
|||
github.com/aquasecurity/table|https://github.com/aquasecurity/table/blob/v1.8.0/LICENSE|MIT
|
||||
github.com/aquasecurity/tml|https://github.com/aquasecurity/tml/blob/v0.6.1/LICENSE|Unlicense
|
||||
github.com/asaskevich/govalidator|https://github.com/asaskevich/govalidator/blob/a9d515a09cc2/LICENSE|MIT
|
||||
github.com/aws/aws-sdk-go-v2/config|https://github.com/aws/aws-sdk-go-v2/blob/config/v1.18.44/config/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/credentials|https://github.com/aws/aws-sdk-go-v2/blob/credentials/v1.13.42/credentials/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/config|https://github.com/aws/aws-sdk-go-v2/blob/config/v1.19.1/config/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/credentials|https://github.com/aws/aws-sdk-go-v2/blob/credentials/v1.13.43/credentials/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue|https://github.com/aws/aws-sdk-go-v2/blob/feature/dynamodb/attributevalue/v1.10.43/feature/dynamodb/attributevalue/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds|https://github.com/aws/aws-sdk-go-v2/blob/feature/ec2/imds/v1.13.12/feature/ec2/imds/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds|https://github.com/aws/aws-sdk-go-v2/blob/feature/ec2/imds/v1.13.13/feature/ec2/imds/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources|https://github.com/aws/aws-sdk-go-v2/blob/internal/configsources/v1.1.43/internal/configsources/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2|https://github.com/aws/aws-sdk-go-v2/blob/internal/endpoints/v2.4.37/internal/endpoints/v2/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini|https://github.com/aws/aws-sdk-go-v2/blob/internal/ini/v1.3.44/internal/ini/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini|https://github.com/aws/aws-sdk-go-v2/blob/internal/ini/v1.3.45/internal/ini/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/internal/sync/singleflight|https://github.com/aws/aws-sdk-go-v2/blob/v1.21.2/internal/sync/singleflight/LICENSE|BSD-3-Clause
|
||||
github.com/aws/aws-sdk-go-v2/service/dynamodb/types|https://github.com/aws/aws-sdk-go-v2/blob/service/dynamodb/v1.23.0/service/dynamodb/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/service/dynamodbstreams/types|https://github.com/aws/aws-sdk-go-v2/blob/service/dynamodbstreams/v1.15.7/service/dynamodbstreams/LICENSE.txt|Apache-2.0
|
||||
|
|
@ -81,14 +81,14 @@ github.com/aws/aws-sdk-go-v2/service/ecrpublic|https://github.com/aws/aws-sdk-go
|
|||
github.com/aws/aws-sdk-go-v2/service/ecr|https://github.com/aws/aws-sdk-go-v2/blob/service/ecr/v1.17.18/service/ecr/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding|https://github.com/aws/aws-sdk-go-v2/blob/service/internal/accept-encoding/v1.9.15/service/internal/accept-encoding/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery|https://github.com/aws/aws-sdk-go-v2/blob/service/internal/endpoint-discovery/v1.7.37/service/internal/endpoint-discovery/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url|https://github.com/aws/aws-sdk-go-v2/blob/service/internal/presigned-url/v1.9.36/service/internal/presigned-url/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url|https://github.com/aws/aws-sdk-go-v2/blob/service/internal/presigned-url/v1.9.37/service/internal/presigned-url/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/service/secretsmanager|https://github.com/aws/aws-sdk-go-v2/blob/service/secretsmanager/v1.21.6/service/secretsmanager/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc|https://github.com/aws/aws-sdk-go-v2/blob/service/ssooidc/v1.17.2/service/ssooidc/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/service/sso|https://github.com/aws/aws-sdk-go-v2/blob/service/sso/v1.15.1/service/sso/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/service/sts|https://github.com/aws/aws-sdk-go-v2/blob/service/sts/v1.23.1/service/sts/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc|https://github.com/aws/aws-sdk-go-v2/blob/service/ssooidc/v1.17.3/service/ssooidc/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/service/sso|https://github.com/aws/aws-sdk-go-v2/blob/service/sso/v1.15.2/service/sso/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2/service/sts|https://github.com/aws/aws-sdk-go-v2/blob/service/sts/v1.23.2/service/sts/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go-v2|https://github.com/aws/aws-sdk-go-v2/blob/v1.21.2/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go/internal/sync/singleflight|https://github.com/aws/aws-sdk-go/blob/v1.46.1/internal/sync/singleflight/LICENSE|BSD-3-Clause
|
||||
github.com/aws/aws-sdk-go|https://github.com/aws/aws-sdk-go/blob/v1.46.1/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/aws-sdk-go/internal/sync/singleflight|https://github.com/aws/aws-sdk-go/blob/v1.46.7/internal/sync/singleflight/LICENSE|BSD-3-Clause
|
||||
github.com/aws/aws-sdk-go|https://github.com/aws/aws-sdk-go/blob/v1.46.7/LICENSE.txt|Apache-2.0
|
||||
github.com/aws/smithy-go/internal/sync/singleflight|https://github.com/aws/smithy-go/blob/v1.15.0/internal/sync/singleflight/LICENSE|BSD-3-Clause
|
||||
github.com/aws/smithy-go|https://github.com/aws/smithy-go/blob/v1.15.0/LICENSE|Apache-2.0
|
||||
github.com/awslabs/amazon-ecr-credential-helper/ecr-login|https://github.com/awslabs/amazon-ecr-credential-helper/blob/396b2034c795/ecr-login/LICENSE|Apache-2.0
|
||||
|
|
@ -198,7 +198,7 @@ github.com/google/gofuzz|https://github.com/google/gofuzz/blob/v1.2.0/LICENSE|Ap
|
|||
github.com/google/licenseclassifier/v2|https://github.com/google/licenseclassifier/blob/v2.0.0/v2/LICENSE|Apache-2.0
|
||||
github.com/google/s2a-go|https://github.com/google/s2a-go/blob/v0.1.7/LICENSE.md|Apache-2.0
|
||||
github.com/google/shlex|https://github.com/google/shlex/blob/e7afc7fbc510/COPYING|Apache-2.0
|
||||
github.com/google/uuid|https://github.com/google/uuid/blob/v1.3.1/LICENSE|BSD-3-Clause
|
||||
github.com/google/uuid|https://github.com/google/uuid/blob/v1.4.0/LICENSE|BSD-3-Clause
|
||||
github.com/google/wire|https://github.com/google/wire/blob/v0.5.0/LICENSE|Apache-2.0
|
||||
github.com/googleapis/enterprise-certificate-proxy/client|https://github.com/googleapis/enterprise-certificate-proxy/blob/v0.3.1/LICENSE|Apache-2.0
|
||||
github.com/googleapis/gax-go/v2|https://github.com/googleapis/gax-go/blob/v2.12.0/v2/LICENSE|BSD-3-Clause
|
||||
|
|
@ -295,7 +295,7 @@ github.com/mpvl/unique|https://github.com/mpvl/unique/blob/cbe035fff7de/LICENSE|
|
|||
github.com/munnerz/goautoneg|https://github.com/munnerz/goautoneg/blob/a7dc8b61c822/LICENSE|BSD-3-Clause
|
||||
github.com/nmcclain/asn1-ber|https://github.com/nmcclain/asn1-ber/blob/2661553a0484/LICENSE|BSD-3-Clause
|
||||
github.com/nmcclain/ldap|https://github.com/nmcclain/ldap/blob/7f8d1e44eeba/LICENSE|BSD-3-Clause
|
||||
github.com/notaryproject/notation-core-go|https://github.com/notaryproject/notation-core-go/blob/v1.0.0/LICENSE|Apache-2.0
|
||||
github.com/notaryproject/notation-core-go|https://github.com/notaryproject/notation-core-go/blob/v1.0.1/LICENSE|Apache-2.0
|
||||
github.com/notaryproject/notation-go|https://github.com/notaryproject/notation-go/blob/v1.0.0/LICENSE|Apache-2.0
|
||||
github.com/nozzle/throttler|https://github.com/nozzle/throttler/blob/2ea982251481/LICENSE|Apache-2.0
|
||||
github.com/oklog/ulid|https://github.com/oklog/ulid/blob/v1.3.1/LICENSE|Apache-2.0
|
||||
|
|
@ -392,7 +392,7 @@ github.com/zclconf/go-cty-yaml|https://github.com/zclconf/go-cty-yaml/blob/v1.0.
|
|||
github.com/zclconf/go-cty/cty|https://github.com/zclconf/go-cty/blob/v1.13.0/LICENSE|MIT
|
||||
github.com/zeebo/errs|https://github.com/zeebo/errs/blob/v1.3.0/LICENSE|MIT
|
||||
github.com/zitadel/oidc|https://github.com/zitadel/oidc/blob/v1.13.5/LICENSE|Apache-2.0
|
||||
go.etcd.io/bbolt|https://github.com/etcd-io/bbolt/blob/v1.3.7/LICENSE|MIT
|
||||
go.etcd.io/bbolt|https://github.com/etcd-io/bbolt/blob/v1.3.8/LICENSE|MIT
|
||||
go.mongodb.org/mongo-driver|https://github.com/mongodb/mongo-go-driver/blob/v1.11.3/LICENSE|Apache-2.0
|
||||
go.mozilla.org/pkcs7|https://github.com/mozilla-services/pkcs7/blob/33d05740a352/LICENSE|MIT
|
||||
go.opencensus.io|https://github.com/census-instrumentation/opencensus-go/blob/v0.24.0/LICENSE|Apache-2.0
|
||||
|
|
@ -441,7 +441,7 @@ google.golang.org/genproto/googleapis/rpc|https://github.com/googleapis/go-genpr
|
|||
google.golang.org/genproto/googleapis/type/expr|https://github.com/googleapis/go-genproto/blob/007df8e322eb/LICENSE|Apache-2.0
|
||||
google.golang.org/genproto/googleapis/type|https://github.com/googleapis/go-genproto/blob/007df8e322eb/LICENSE|Apache-2.0
|
||||
google.golang.org/genproto/protobuf/field_mask|https://github.com/googleapis/go-genproto/blob/007df8e322eb/LICENSE|Apache-2.0
|
||||
google.golang.org/grpc|https://github.com/grpc/grpc-go/blob/v1.58.2/LICENSE|Apache-2.0
|
||||
google.golang.org/grpc|https://github.com/grpc/grpc-go/blob/v1.58.3/LICENSE|Apache-2.0
|
||||
google.golang.org/protobuf|https://github.com/protocolbuffers/protobuf-go/blob/v1.31.0/LICENSE|BSD-3-Clause
|
||||
gopkg.in/cheggaaa/pb.v1|https://github.com/cheggaaa/pb/blob/v1.0.28/LICENSE|BSD-3-Clause
|
||||
gopkg.in/go-jose/go-jose.v2/json|https://github.com/go-jose/go-jose/blob/v2.6.1/json/LICENSE|BSD-3-Clause
|
||||
|
|
|
|||
|
|
@ -372,6 +372,10 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
|
|||
ext.EnableMetricsExtension(c.Config, c.Log, c.Config.Storage.RootDirectory)
|
||||
ext.EnableSearchExtension(c.Config, c.StoreController, c.MetaDB, taskScheduler, c.CveScanner, c.Log)
|
||||
}
|
||||
// runs once if metrics are enabled & imagestore is local
|
||||
if c.Config.IsMetricsEnabled() && c.Config.Storage.StorageDriver == nil {
|
||||
c.StoreController.DefaultStore.PopulateStorageMetrics(time.Duration(0), taskScheduler)
|
||||
}
|
||||
|
||||
if c.Config.Storage.SubPaths != nil {
|
||||
for route, storageConfig := range c.Config.Storage.SubPaths {
|
||||
|
|
@ -396,6 +400,10 @@ func (c *Controller) StartBackgroundTasks(reloadCtx context.Context) {
|
|||
substore := c.StoreController.SubStore[route]
|
||||
if substore != nil {
|
||||
substore.RunDedupeBlobs(time.Duration(0), taskScheduler)
|
||||
|
||||
if c.Config.IsMetricsEnabled() && c.Config.Storage.StorageDriver == nil {
|
||||
substore.PopulateStorageMetrics(time.Duration(0), taskScheduler)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ type MetricServer interface {
|
|||
IsEnabled() bool
|
||||
}
|
||||
|
||||
func getDirSize(path string) (int64, error) {
|
||||
func GetDirSize(path string) (int64, error) {
|
||||
var size int64
|
||||
|
||||
err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
|
||||
|
|
|
|||
|
|
@ -171,7 +171,7 @@ func IncDownloadCounter(ms MetricServer, repo string) {
|
|||
func SetStorageUsage(ms MetricServer, rootDir, repo string) {
|
||||
ms.SendMetric(func() {
|
||||
dir := path.Join(rootDir, repo)
|
||||
repoSize, err := getDirSize(dir)
|
||||
repoSize, err := GetDirSize(dir)
|
||||
|
||||
if err == nil {
|
||||
repoStorageBytes.WithLabelValues(repo).Set(float64(repoSize))
|
||||
|
|
|
|||
|
|
@ -486,7 +486,7 @@ func IncUploadCounter(ms MetricServer, repo string) {
|
|||
func SetStorageUsage(ms MetricServer, rootDir, repo string) {
|
||||
dir := path.Join(rootDir, repo)
|
||||
|
||||
repoSize, err := getDirSize(dir)
|
||||
repoSize, err := GetDirSize(dir)
|
||||
if err != nil {
|
||||
ms.(*metricServer).log.Error().Err(err).Msg("failed to set storage usage")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,9 +4,12 @@
|
|||
package monitoring_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
|
|
@ -17,6 +20,8 @@ import (
|
|||
"zotregistry.io/zot/pkg/api/config"
|
||||
extconf "zotregistry.io/zot/pkg/extensions/config"
|
||||
"zotregistry.io/zot/pkg/extensions/monitoring"
|
||||
"zotregistry.io/zot/pkg/scheduler"
|
||||
common "zotregistry.io/zot/pkg/storage/common"
|
||||
test "zotregistry.io/zot/pkg/test/common"
|
||||
. "zotregistry.io/zot/pkg/test/image-utils"
|
||||
ociutils "zotregistry.io/zot/pkg/test/oci-utils"
|
||||
|
|
@ -413,6 +418,70 @@ func TestMetricsAuthorization(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestPopulateStorageMetrics(t *testing.T) {
|
||||
Convey("Start a scheduler when metrics enabled", t, func() {
|
||||
port := test.GetFreePort()
|
||||
baseURL := test.GetBaseURL(port)
|
||||
conf := config.New()
|
||||
conf.HTTP.Port = port
|
||||
|
||||
rootDir := t.TempDir()
|
||||
|
||||
conf.Storage.RootDirectory = rootDir
|
||||
conf.Extensions = &extconf.ExtensionConfig{}
|
||||
enabled := true
|
||||
conf.Extensions.Metrics = &extconf.MetricsConfig{
|
||||
BaseConfig: extconf.BaseConfig{Enable: &enabled},
|
||||
Prometheus: &extconf.PrometheusConfig{Path: "/metrics"},
|
||||
}
|
||||
|
||||
ctlr := api.NewController(conf)
|
||||
So(ctlr, ShouldNotBeNil)
|
||||
|
||||
cm := test.NewControllerManager(ctlr)
|
||||
cm.StartAndWait(port)
|
||||
defer cm.StopServer()
|
||||
|
||||
// write a couple of images
|
||||
srcStorageCtlr := ociutils.GetDefaultStoreController(rootDir, ctlr.Log)
|
||||
err := WriteImageToFileSystem(CreateDefaultImage(), "alpine", "0.0.1", srcStorageCtlr)
|
||||
So(err, ShouldBeNil)
|
||||
err = WriteImageToFileSystem(CreateDefaultImage(), "busybox", "0.0.1", srcStorageCtlr)
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
sch := scheduler.NewScheduler(conf, ctlr.Log)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
sch.RunScheduler(ctx)
|
||||
|
||||
generator := &common.StorageMetricsInitGenerator{
|
||||
ImgStore: ctlr.StoreController.DefaultStore,
|
||||
Metrics: ctlr.Metrics,
|
||||
Log: ctlr.Log,
|
||||
MaxDelay: 1, // maximum delay between jobs (each job computes repo's storage size)
|
||||
}
|
||||
|
||||
sch.SubmitGenerator(generator, time.Duration(0), scheduler.LowPriority)
|
||||
|
||||
time.Sleep(5 * time.Second)
|
||||
cancel()
|
||||
alpineSize, err := monitoring.GetDirSize(path.Join(rootDir, "alpine"))
|
||||
So(err, ShouldBeNil)
|
||||
busyboxSize, err := monitoring.GetDirSize(path.Join(rootDir, "busybox"))
|
||||
So(err, ShouldBeNil)
|
||||
|
||||
resp, err := resty.R().Get(baseURL + "/metrics")
|
||||
So(err, ShouldBeNil)
|
||||
So(resp, ShouldNotBeNil)
|
||||
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
|
||||
|
||||
alpineMetric := fmt.Sprintf("zot_repo_storage_bytes{repo=\"alpine\"} %d", alpineSize)
|
||||
busyboxMetric := fmt.Sprintf("zot_repo_storage_bytes{repo=\"busybox\"} %d", busyboxSize)
|
||||
respStr := string(resp.Body())
|
||||
So(respStr, ShouldContainSubstring, alpineMetric)
|
||||
So(respStr, ShouldContainSubstring, busyboxMetric)
|
||||
})
|
||||
}
|
||||
|
||||
func generateRandomString() string {
|
||||
//nolint: gosec
|
||||
seededRand := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
|
|
|
|||
|
|
@ -6,8 +6,10 @@ import (
|
|||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/docker/distribution/registry/storage/driver"
|
||||
godigest "github.com/opencontainers/go-digest"
|
||||
|
|
@ -18,6 +20,7 @@ import (
|
|||
|
||||
zerr "zotregistry.io/zot/errors"
|
||||
zcommon "zotregistry.io/zot/pkg/common"
|
||||
"zotregistry.io/zot/pkg/extensions/monitoring"
|
||||
zlog "zotregistry.io/zot/pkg/log"
|
||||
"zotregistry.io/zot/pkg/scheduler"
|
||||
storageConstants "zotregistry.io/zot/pkg/storage/constants"
|
||||
|
|
@ -1052,3 +1055,72 @@ func (dt *dedupeTask) DoWork(ctx context.Context) error {
|
|||
|
||||
return err
|
||||
}
|
||||
|
||||
type StorageMetricsInitGenerator struct {
|
||||
ImgStore storageTypes.ImageStore
|
||||
done bool
|
||||
Metrics monitoring.MetricServer
|
||||
lastRepo string
|
||||
nextRun time.Time
|
||||
rand *rand.Rand
|
||||
Log zlog.Logger
|
||||
MaxDelay int
|
||||
}
|
||||
|
||||
func (gen *StorageMetricsInitGenerator) Next() (scheduler.Task, error) {
|
||||
if gen.lastRepo == "" && gen.nextRun.IsZero() {
|
||||
gen.rand = rand.New(rand.NewSource(time.Now().UTC().UnixNano())) //nolint: gosec
|
||||
}
|
||||
|
||||
delay := gen.rand.Intn(gen.MaxDelay)
|
||||
|
||||
gen.nextRun = time.Now().Add(time.Duration(delay) * time.Second)
|
||||
|
||||
repo, err := gen.ImgStore.GetNextRepository(gen.lastRepo)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
gen.Log.Debug().Str("repo", repo).Int("randomDelay", delay).Msg("StorageMetricsInitGenerator")
|
||||
|
||||
if repo == "" {
|
||||
gen.done = true
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
gen.lastRepo = repo
|
||||
|
||||
return NewStorageMetricsTask(gen.ImgStore, gen.Metrics, repo), nil
|
||||
}
|
||||
|
||||
func (gen *StorageMetricsInitGenerator) IsDone() bool {
|
||||
return gen.done
|
||||
}
|
||||
|
||||
func (gen *StorageMetricsInitGenerator) IsReady() bool {
|
||||
return time.Now().After(gen.nextRun)
|
||||
}
|
||||
|
||||
func (gen *StorageMetricsInitGenerator) Reset() {
|
||||
gen.lastRepo = ""
|
||||
gen.done = false
|
||||
gen.nextRun = time.Time{}
|
||||
}
|
||||
|
||||
type smTask struct {
|
||||
imgStore storageTypes.ImageStore
|
||||
metrics monitoring.MetricServer
|
||||
repo string
|
||||
}
|
||||
|
||||
func NewStorageMetricsTask(imgStore storageTypes.ImageStore, metrics monitoring.MetricServer, repo string,
|
||||
) *smTask {
|
||||
return &smTask{imgStore, metrics, repo}
|
||||
}
|
||||
|
||||
func (smt *smTask) DoWork(ctx context.Context) error {
|
||||
// run task
|
||||
monitoring.SetStorageUsage(smt.metrics, smt.imgStore.RootDir(), smt.repo)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -488,7 +488,10 @@ func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //noli
|
|||
is.Unlock(&lockLatency)
|
||||
|
||||
if err == nil {
|
||||
monitoring.SetStorageUsage(is.metrics, is.rootDir, repo)
|
||||
if is.storeDriver.Name() == storageConstants.LocalStorageDriverName {
|
||||
monitoring.SetStorageUsage(is.metrics, is.rootDir, repo)
|
||||
}
|
||||
|
||||
monitoring.IncUploadCounter(is.metrics, repo)
|
||||
}
|
||||
}()
|
||||
|
|
@ -621,7 +624,11 @@ func (is *ImageStore) DeleteImageManifest(repo, reference string, detectCollisio
|
|||
}
|
||||
|
||||
func (is *ImageStore) deleteImageManifest(repo, reference string, detectCollisions bool) error {
|
||||
defer monitoring.SetStorageUsage(is.metrics, is.rootDir, repo)
|
||||
defer func() {
|
||||
if is.storeDriver.Name() == storageConstants.LocalStorageDriverName {
|
||||
monitoring.SetStorageUsage(is.metrics, is.rootDir, repo)
|
||||
}
|
||||
}()
|
||||
|
||||
index, err := common.GetIndex(is, repo, is.log)
|
||||
if err != nil {
|
||||
|
|
@ -1929,6 +1936,17 @@ func (is *ImageStore) RunDedupeBlobs(interval time.Duration, sch *scheduler.Sche
|
|||
sch.SubmitGenerator(generator, interval, scheduler.MediumPriority)
|
||||
}
|
||||
|
||||
func (is *ImageStore) PopulateStorageMetrics(interval time.Duration, sch *scheduler.Scheduler) {
|
||||
generator := &common.StorageMetricsInitGenerator{
|
||||
ImgStore: is,
|
||||
Metrics: is.metrics,
|
||||
Log: is.log,
|
||||
MaxDelay: 15, //nolint:gomnd
|
||||
}
|
||||
|
||||
sch.SubmitGenerator(generator, interval, scheduler.LowPriority)
|
||||
}
|
||||
|
||||
type blobStream struct {
|
||||
reader io.Reader
|
||||
closer io.Closer
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ type ImageStore interface { //nolint:interfacebloat
|
|||
RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error
|
||||
GetNextDigestWithBlobPaths(repos []string, lastDigests []godigest.Digest) (godigest.Digest, []string, error)
|
||||
GetAllBlobs(repo string) ([]string, error)
|
||||
PopulateStorageMetrics(interval time.Duration, sch *scheduler.Scheduler)
|
||||
}
|
||||
|
||||
type Driver interface { //nolint:interfacebloat
|
||||
|
|
|
|||
|
|
@ -55,6 +55,7 @@ type MockedImageStore struct {
|
|||
GetAllBlobsFn func(repo string) ([]string, error)
|
||||
CleanupRepoFn func(repo string, blobs []godigest.Digest, removeRepo bool) (int, error)
|
||||
PutIndexContentFn func(repo string, index ispec.Index) error
|
||||
PopulateStorageMetricsFn func(interval time.Duration, sch *scheduler.Scheduler)
|
||||
}
|
||||
|
||||
func (is MockedImageStore) Lock(t *time.Time) {
|
||||
|
|
@ -405,3 +406,9 @@ func (is MockedImageStore) PutIndexContent(repo string, index ispec.Index) error
|
|||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (is MockedImageStore) PopulateStorageMetrics(interval time.Duration, sch *scheduler.Scheduler) {
|
||||
if is.PopulateStorageMetricsFn != nil {
|
||||
is.PopulateStorageMetricsFn(interval, sch)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue