From 8ed34608e48247e73d2265963075bf67df6bd062 Mon Sep 17 00:00:00 2001 From: Roxana Nemulescu Date: Wed, 24 Aug 2022 19:15:46 +0300 Subject: [PATCH] GetCatalog() - Populate _catalog with mix of many small, medium and large images Signed-off-by: Roxana Nemulescu --- .github/workflows/benchmark.yaml | 2 +- .github/workflows/cluster.yaml | 2 +- cmd/zb/helper.go | 136 +++++++++++++++++-------------- cmd/zb/perf.go | 79 ++++++++---------- 4 files changed, 114 insertions(+), 105 deletions(-) diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index fa967e00..35059f7c 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -26,7 +26,7 @@ jobs: uses: actions/cache@v3 with: path: ./cache - key: ${{ runner.os }}-benchmark + key: $${runner.os }}-gen1-benchmark # Run `github-action-benchmark` action - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1.14.0 diff --git a/.github/workflows/cluster.yaml b/.github/workflows/cluster.yaml index 60ec56eb..a28c667a 100644 --- a/.github/workflows/cluster.yaml +++ b/.github/workflows/cluster.yaml @@ -167,7 +167,7 @@ jobs: uses: actions/cache@v3 with: path: ./cache - key: ${{ runner.os }}-benchmark-stateless-cluster + key: $${runner.os }}-gen1-benchmark-stateless-cluster # Run `github-action-benchmark` action - name: Store benchmark result uses: benchmark-action/github-action-benchmark@v1.14.0 diff --git a/cmd/zb/helper.go b/cmd/zb/helper.go index f5826ed1..e9684d14 100644 --- a/cmd/zb/helper.go +++ b/cmd/zb/helper.go @@ -7,6 +7,7 @@ import ( "io" "io/ioutil" "log" + mrand "math/rand" "net/http" "os" "path" @@ -64,23 +65,7 @@ func pullAndCollect(url string, repos []string, manifestItem manifestStruct, }() if config.mixedSize { - smallSizeIdx := 0 - mediumSizeIdx := 1 - largeSizeIdx := 2 - - idx := flipFunc(config.probabilityRange) - - switch idx { - case smallSizeIdx: - current := loadOrStore(&statusRequests, "1MB", 0) - statusRequests.Store("1MB", current+1) - case mediumSizeIdx: - current := loadOrStore(&statusRequests, "10MB", 0) - statusRequests.Store("10MB", current+1) - case largeSizeIdx: - current := loadOrStore(&statusRequests, "100MB", 0) - statusRequests.Store("100MB", current+1) - } + _, idx := getRandomSize(config.probabilityRange) manifestHash = manifestBySizeHash[idx] } @@ -256,7 +241,7 @@ func pullAndCollect(url string, repos []string, manifestItem manifestStruct, return repos } -func pushMonolithImage(workdir, url, trepo string, repos []string, size int, +func pushMonolithImage(workdir, url, trepo string, repos []string, config testConfig, client *resty.Client, ) (map[string]string, []string, error) { var statusCode int @@ -292,6 +277,15 @@ func pushMonolithImage(workdir, url, trepo string, repos []string, size int, } loc := test.Location(url, resp) + + var size int + + if config.size == 0 { + size, _ = getRandomSize(config.probabilityRange) + } else { + size = config.size + } + blob := path.Join(workdir, fmt.Sprintf("%d.blob", size)) fhandle, err := os.OpenFile(blob, os.O_RDONLY, defaultFilePerms) @@ -465,27 +459,7 @@ func pushMonolithAndCollect(workdir, url, trepo string, count int, var size int if config.mixedSize { - idx := flipFunc(config.probabilityRange) - smallSizeIdx := 0 - mediumSizeIdx := 1 - largeSizeIdx := 2 - - switch idx { - case smallSizeIdx: - size = smallBlob - current := loadOrStore(&statusRequests, "1MB", 0) - statusRequests.Store("1MB", current+1) - case mediumSizeIdx: - size = mediumBlob - current := loadOrStore(&statusRequests, "10MB", 0) - statusRequests.Store("10MB", current+1) - case largeSizeIdx: - size = largeBlob - current := loadOrStore(&statusRequests, "100MB", 0) - statusRequests.Store("100MB", current+1) - default: - size = config.size - } + size, _ = getRandomSize(config.probabilityRange) } else { size = config.size } @@ -689,27 +663,7 @@ func pushChunkAndCollect(workdir, url, trepo string, count int, var size int if config.mixedSize { - idx := flipFunc(config.probabilityRange) - smallSizeIdx := 0 - mediumSizeIdx := 1 - largeSizeIdx := 2 - - switch idx { - case smallSizeIdx: - size = smallBlob - current := loadOrStore(&statusRequests, "1MB", 0) - statusRequests.Store("1MB", current+1) - case mediumSizeIdx: - size = mediumBlob - current := loadOrStore(&statusRequests, "10MB", 0) - statusRequests.Store("10MB", current+1) - case largeSizeIdx: - size = largeBlob - current := loadOrStore(&statusRequests, "100MB", 0) - statusRequests.Store("100MB", current+1) - default: - size = config.size - } + size, _ = getRandomSize(config.probabilityRange) } else { size = config.size } @@ -920,6 +874,68 @@ func pushChunkAndCollect(workdir, url, trepo string, count int, return repos } +func getRandomSize(probabilityRange []float64) (int, int) { + var size int + + idx := flipFunc(probabilityRange) + smallSizeIdx := 0 + mediumSizeIdx := 1 + largeSizeIdx := 2 + + switch idx { + case smallSizeIdx: + size = smallBlob + current := loadOrStore(&statusRequests, "1MB", 0) + statusRequests.Store("1MB", current+1) + case mediumSizeIdx: + size = mediumBlob + current := loadOrStore(&statusRequests, "10MB", 0) + statusRequests.Store("10MB", current+1) + case largeSizeIdx: + size = largeBlob + current := loadOrStore(&statusRequests, "100MB", 0) + statusRequests.Store("100MB", current+1) + default: + size = 0 + } + + return size, idx +} + +// nolint:gosec +func flipFunc(probabilityRange []float64) int { + mrand.Seed(time.Now().UTC().UnixNano()) + toss := mrand.Float64() + + for idx, r := range probabilityRange { + if toss < r { + return idx + } + } + + return len(probabilityRange) - 1 +} + +// pbty - probabilities. +func normalizeProbabilityRange(pbty []float64) []float64 { + dim := len(pbty) + + // npd - normalized probability density + npd := make([]float64, dim) + + for idx := range pbty { + npd[idx] = 0.0 + } + + // [0.2, 0.7, 0.1] -> [0.2, 0.9, 1] + npd[0] = pbty[0] + for i := 1; i < dim; i++ { + npd[i] = npd[i-1] + pbty[i] + } + + return npd +} + func loadOrStore(statusRequests *sync.Map, key string, value int) int { val, _ := statusRequests.LoadOrStore(key, value) diff --git a/cmd/zb/perf.go b/cmd/zb/perf.go index 5eae70d2..790c1046 100644 --- a/cmd/zb/perf.go +++ b/cmd/zb/perf.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "log" "math/big" - mrand "math/rand" "net" "net/http" urlparser "net/url" @@ -263,40 +262,6 @@ func printStats(requests int, summary *statsSummary, outFmt string) { } } -// nolint:gosec -func flipFunc(probabilityRange []float64) int { - mrand.Seed(time.Now().UTC().UnixNano()) - toss := mrand.Float64() - - for idx, r := range probabilityRange { - if toss < r { - return idx - } - } - - return len(probabilityRange) - 1 -} - -// pbty - probabilities. -func normalizeProbabilityRange(pbty []float64) []float64 { - dim := len(pbty) - - // npd - normalized probability density - npd := make([]float64, dim) - - for idx := range pbty { - npd[idx] = 0.0 - } - - // [0.2, 0.7, 0.1] -> [0.2, 0.9, 1] - npd[0] = pbty[0] - for i := 1; i < dim; i++ { - npd[i] = npd[i-1] + pbty[i] - } - - return npd -} - // test suites/funcs. type testFunc func( @@ -307,6 +272,7 @@ type testFunc func( client *resty.Client, ) error +// nolint:gosec func GetCatalog( workdir, url, repo string, requests int, @@ -314,6 +280,20 @@ func GetCatalog( statsCh chan statsRecord, client *resty.Client, ) error { + var repos []string + + var err error + + statusRequests = sync.Map{} + + for count := 0; count < requests; count++ { + // Push random blob + _, repos, err = pushMonolithImage(workdir, url, repo, repos, config, client) + if err != nil { + return err + } + } + for count := 0; count < requests; count++ { func() { start := time.Now() @@ -355,6 +335,12 @@ func GetCatalog( }() } + // clean up + err = deleteTestRepo(repos, url, client) + if err != nil { + return err + } + return nil } @@ -436,25 +422,31 @@ func Pull( mediumSizeIdx := 1 largeSizeIdx := 2 + config.size = smallBlob + // Push small blob - manifestBySize, repos, err := pushMonolithImage(workdir, url, trepo, repos, smallBlob, client) + manifestBySize, repos, err := pushMonolithImage(workdir, url, trepo, repos, config, client) if err != nil { return err } manifestBySizeHash[smallSizeIdx] = manifestBySize + config.size = mediumBlob + // Push medium blob - manifestBySize, repos, err = pushMonolithImage(workdir, url, trepo, repos, mediumBlob, client) + manifestBySize, repos, err = pushMonolithImage(workdir, url, trepo, repos, config, client) if err != nil { return err } manifestBySizeHash[mediumSizeIdx] = manifestBySize + config.size = largeBlob + // Push large blob // nolint: ineffassign, staticcheck, wastedassign - manifestBySize, repos, err = pushMonolithImage(workdir, url, trepo, repos, largeBlob, client) + manifestBySize, repos, err = pushMonolithImage(workdir, url, trepo, repos, config, client) if err != nil { return err } @@ -463,7 +455,7 @@ func Pull( } else { // Push blob given size var err error - manifestHash, repos, err = pushMonolithImage(workdir, url, trepo, repos, config.size, client) + manifestHash, repos, err = pushMonolithImage(workdir, url, trepo, repos, config, client) if err != nil { return err } @@ -500,7 +492,7 @@ func MixedPullAndPush( statusRequests = sync.Map{} // Push blob given size - manifestHash, repos, err := pushMonolithImage(workdir, url, trepo, repos, config.size, client) + manifestHash, repos, err := pushMonolithImage(workdir, url, trepo, repos, config, client) if err != nil { return err } @@ -548,8 +540,9 @@ type testConfig struct { var testSuite = []testConfig{ // nolint:gochecknoglobals // used only in this test { - name: "Get Catalog", - tfunc: GetCatalog, + name: "Get Catalog", + tfunc: GetCatalog, + probabilityRange: normalizeProbabilityRange([]float64{0.7, 0.2, 0.1}), }, { name: "Push Monolith 1MB", @@ -703,7 +696,7 @@ func Perf( summary.total = time.Since(start) summary.rps = float32(requests) / float32(summary.total.Seconds()) - if tconfig.mixedSize { + if tconfig.mixedSize || tconfig.size == 0 { summary.mixedSize = true }