GetCatalog() - Populate _catalog with mix of many small, medium and large images

Signed-off-by: Roxana Nemulescu <roxana.nemulescu@gmail.com>

.github/workflows/benchmark.yaml

@@ -26,7 +26,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ./cache
-          key: ${{ runner.os }}-benchmark
+          key: $${runner.os }}-gen1-benchmark
       # Run `github-action-benchmark` action
       - name: Store benchmark result
         uses: benchmark-action/github-action-benchmark@v1.14.0

.github/workflows/cluster.yaml

@@ -167,7 +167,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ./cache
-          key: ${{ runner.os }}-benchmark-stateless-cluster
+          key: $${runner.os }}-gen1-benchmark-stateless-cluster
       # Run `github-action-benchmark` action
       - name: Store benchmark result
         uses: benchmark-action/github-action-benchmark@v1.14.0

cmd/zb/helper.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"io/ioutil"
 	"log"
+	mrand "math/rand"
 	"net/http"
 	"os"
 	"path"
@@ -64,23 +65,7 @@ func pullAndCollect(url string, repos []string, manifestItem manifestStruct,
 		}()
 
 		if config.mixedSize {
-			smallSizeIdx := 0
+			_, idx := getRandomSize(config.probabilityRange)
-			mediumSizeIdx := 1
-			largeSizeIdx := 2
-
-			idx := flipFunc(config.probabilityRange)
-
-			switch idx {
-			case smallSizeIdx:
-				current := loadOrStore(&statusRequests, "1MB", 0)
-				statusRequests.Store("1MB", current+1)
-			case mediumSizeIdx:
-				current := loadOrStore(&statusRequests, "10MB", 0)
-				statusRequests.Store("10MB", current+1)
-			case largeSizeIdx:
-				current := loadOrStore(&statusRequests, "100MB", 0)
-				statusRequests.Store("100MB", current+1)
-			}
 
 			manifestHash = manifestBySizeHash[idx]
 		}
@@ -256,7 +241,7 @@ func pullAndCollect(url string, repos []string, manifestItem manifestStruct,
 	return repos
 }
 
-func pushMonolithImage(workdir, url, trepo string, repos []string, size int,
+func pushMonolithImage(workdir, url, trepo string, repos []string, config testConfig,
 	client *resty.Client,
 ) (map[string]string, []string, error) {
 	var statusCode int
@@ -292,6 +277,15 @@ func pushMonolithImage(workdir, url, trepo string, repos []string, size int,
 	}
 
 	loc := test.Location(url, resp)
+
+	var size int
+
+	if config.size == 0 {
+		size, _ = getRandomSize(config.probabilityRange)
+	} else {
+		size = config.size
+	}
+
 	blob := path.Join(workdir, fmt.Sprintf("%d.blob", size))
 
 	fhandle, err := os.OpenFile(blob, os.O_RDONLY, defaultFilePerms)
@@ -465,27 +459,7 @@ func pushMonolithAndCollect(workdir, url, trepo string, count int,
 		var size int
 
 		if config.mixedSize {
-			idx := flipFunc(config.probabilityRange)
+			size, _ = getRandomSize(config.probabilityRange)
-			smallSizeIdx := 0
-			mediumSizeIdx := 1
-			largeSizeIdx := 2
-
-			switch idx {
-			case smallSizeIdx:
-				size = smallBlob
-				current := loadOrStore(&statusRequests, "1MB", 0)
-				statusRequests.Store("1MB", current+1)
-			case mediumSizeIdx:
-				size = mediumBlob
-				current := loadOrStore(&statusRequests, "10MB", 0)
-				statusRequests.Store("10MB", current+1)
-			case largeSizeIdx:
-				size = largeBlob
-				current := loadOrStore(&statusRequests, "100MB", 0)
-				statusRequests.Store("100MB", current+1)
-			default:
-				size = config.size
-			}
 		} else {
 			size = config.size
 		}
@@ -689,27 +663,7 @@ func pushChunkAndCollect(workdir, url, trepo string, count int,
 		var size int
 
 		if config.mixedSize {
-			idx := flipFunc(config.probabilityRange)
+			size, _ = getRandomSize(config.probabilityRange)
-			smallSizeIdx := 0
-			mediumSizeIdx := 1
-			largeSizeIdx := 2
-
-			switch idx {
-			case smallSizeIdx:
-				size = smallBlob
-				current := loadOrStore(&statusRequests, "1MB", 0)
-				statusRequests.Store("1MB", current+1)
-			case mediumSizeIdx:
-				size = mediumBlob
-				current := loadOrStore(&statusRequests, "10MB", 0)
-				statusRequests.Store("10MB", current+1)
-			case largeSizeIdx:
-				size = largeBlob
-				current := loadOrStore(&statusRequests, "100MB", 0)
-				statusRequests.Store("100MB", current+1)
-			default:
-				size = config.size
-			}
 		} else {
 			size = config.size
 		}
@@ -920,6 +874,68 @@ func pushChunkAndCollect(workdir, url, trepo string, count int,
 	return repos
 }
 
+func getRandomSize(probabilityRange []float64) (int, int) {
+	var size int
+
+	idx := flipFunc(probabilityRange)
+	smallSizeIdx := 0
+	mediumSizeIdx := 1
+	largeSizeIdx := 2
+
+	switch idx {
+	case smallSizeIdx:
+		size = smallBlob
+		current := loadOrStore(&statusRequests, "1MB", 0)
+		statusRequests.Store("1MB", current+1)
+	case mediumSizeIdx:
+		size = mediumBlob
+		current := loadOrStore(&statusRequests, "10MB", 0)
+		statusRequests.Store("10MB", current+1)
+	case largeSizeIdx:
+		size = largeBlob
+		current := loadOrStore(&statusRequests, "100MB", 0)
+		statusRequests.Store("100MB", current+1)
+	default:
+		size = 0
+	}
+
+	return size, idx
+}
+
+// nolint:gosec
+func flipFunc(probabilityRange []float64) int {
+	mrand.Seed(time.Now().UTC().UnixNano())
+	toss := mrand.Float64()
+
+	for idx, r := range probabilityRange {
+		if toss < r {
+			return idx
+		}
+	}
+
+	return len(probabilityRange) - 1
+}
+
+// pbty - probabilities.
+func normalizeProbabilityRange(pbty []float64) []float64 {
+	dim := len(pbty)
+
+	// npd - normalized probability density
+	npd := make([]float64, dim)
+
+	for idx := range pbty {
+		npd[idx] = 0.0
+	}
+
+	// [0.2, 0.7, 0.1] -> [0.2, 0.9, 1]
+	npd[0] = pbty[0]
+	for i := 1; i < dim; i++ {
+		npd[i] = npd[i-1] + pbty[i]
+	}
+
+	return npd
+}
+
 func loadOrStore(statusRequests *sync.Map, key string, value int) int {
 	val, _ := statusRequests.LoadOrStore(key, value)
 

cmd/zb/perf.go

@@ -7,7 +7,6 @@ import (
 	"io/ioutil"
 	"log"
 	"math/big"
-	mrand "math/rand"
 	"net"
 	"net/http"
 	urlparser "net/url"
@@ -263,40 +262,6 @@ func printStats(requests int, summary *statsSummary, outFmt string) {
 	}
 }
 
-// nolint:gosec
-func flipFunc(probabilityRange []float64) int {
-	mrand.Seed(time.Now().UTC().UnixNano())
-	toss := mrand.Float64()
-
-	for idx, r := range probabilityRange {
-		if toss < r {
-			return idx
-		}
-	}
-
-	return len(probabilityRange) - 1
-}
-
-// pbty - probabilities.
-func normalizeProbabilityRange(pbty []float64) []float64 {
-	dim := len(pbty)
-
-	// npd - normalized probability density
-	npd := make([]float64, dim)
-
-	for idx := range pbty {
-		npd[idx] = 0.0
-	}
-
-	// [0.2, 0.7, 0.1] -> [0.2, 0.9, 1]
-	npd[0] = pbty[0]
-	for i := 1; i < dim; i++ {
-		npd[i] = npd[i-1] + pbty[i]
-	}
-
-	return npd
-}
-
 // test suites/funcs.
 
 type testFunc func(
@@ -307,6 +272,7 @@ type testFunc func(
 	client *resty.Client,
 ) error
 
+// nolint:gosec
 func GetCatalog(
 	workdir, url, repo string,
 	requests int,
@@ -314,6 +280,20 @@ func GetCatalog(
 	statsCh chan statsRecord,
 	client *resty.Client,
 ) error {
+	var repos []string
+
+	var err error
+
+	statusRequests = sync.Map{}
+
+	for count := 0; count < requests; count++ {
+		// Push random blob
+		_, repos, err = pushMonolithImage(workdir, url, repo, repos, config, client)
+		if err != nil {
+			return err
+		}
+	}
+
 	for count := 0; count < requests; count++ {
 		func() {
 			start := time.Now()
@@ -355,6 +335,12 @@ func GetCatalog(
 		}()
 	}
 
+	// clean up
+	err = deleteTestRepo(repos, url, client)
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -436,25 +422,31 @@ func Pull(
 		mediumSizeIdx := 1
 		largeSizeIdx := 2
 
+		config.size = smallBlob
+
 		// Push small blob
-		manifestBySize, repos, err := pushMonolithImage(workdir, url, trepo, repos, smallBlob, client)
+		manifestBySize, repos, err := pushMonolithImage(workdir, url, trepo, repos, config, client)
 		if err != nil {
 			return err
 		}
 
 		manifestBySizeHash[smallSizeIdx] = manifestBySize
 
+		config.size = mediumBlob
+
 		// Push medium blob
-		manifestBySize, repos, err = pushMonolithImage(workdir, url, trepo, repos, mediumBlob, client)
+		manifestBySize, repos, err = pushMonolithImage(workdir, url, trepo, repos, config, client)
 		if err != nil {
 			return err
 		}
 
 		manifestBySizeHash[mediumSizeIdx] = manifestBySize
 
+		config.size = largeBlob
+
 		// Push large blob
 		// nolint: ineffassign, staticcheck, wastedassign
-		manifestBySize, repos, err = pushMonolithImage(workdir, url, trepo, repos, largeBlob, client)
+		manifestBySize, repos, err = pushMonolithImage(workdir, url, trepo, repos, config, client)
 		if err != nil {
 			return err
 		}
@@ -463,7 +455,7 @@ func Pull(
 	} else {
 		// Push blob given size
 		var err error
-		manifestHash, repos, err = pushMonolithImage(workdir, url, trepo, repos, config.size, client)
+		manifestHash, repos, err = pushMonolithImage(workdir, url, trepo, repos, config, client)
 		if err != nil {
 			return err
 		}
@@ -500,7 +492,7 @@ func MixedPullAndPush(
 	statusRequests = sync.Map{}
 
 	// Push blob given size
-	manifestHash, repos, err := pushMonolithImage(workdir, url, trepo, repos, config.size, client)
+	manifestHash, repos, err := pushMonolithImage(workdir, url, trepo, repos, config, client)
 	if err != nil {
 		return err
 	}
@@ -548,8 +540,9 @@ type testConfig struct {
 
 var testSuite = []testConfig{ // nolint:gochecknoglobals // used only in this test
 	{
-		name:  "Get Catalog",
+		name:             "Get Catalog",
-		tfunc: GetCatalog,
+		tfunc:            GetCatalog,
+		probabilityRange: normalizeProbabilityRange([]float64{0.7, 0.2, 0.1}),
 	},
 	{
 		name:  "Push Monolith 1MB",
@@ -703,7 +696,7 @@ func Perf(
 		summary.total = time.Since(start)
 		summary.rps = float32(requests) / float32(summary.total.Seconds())
 
-		if tconfig.mixedSize {
+		if tconfig.mixedSize || tconfig.size == 0 {
 			summary.mixedSize = true
 		}