Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00
Code Issues Activity

test calculated size

Browse source 
Signed-off-by: Laurentiu Niculae <themelopeus@gmail.com>
This commit is contained in:
Laurentiu Niculae 2022-07-20 12:30:34 +03:00 committed by Ramkumar Chinchani
parent 80369140f1
commit 58f8cd5d7d
3 changed files with 289 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

266
pkg/extensions/search/common/common_test.go
View file

@ -9,14 +9,17 @@ import (
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"os/exec"
"path"
"strconv"
"testing"
"time"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sigstore/cosign/cmd/cosign/cli/generate"
"github.com/sigstore/cosign/cmd/cosign/cli/options"
@ -39,7 +42,12 @@ const (
graphqlQueryPrefix = constants.ExtSearchPrefix
)
var ErrTestError = errors.New("test error")
var (
ErrTestError = errors.New("test error")
ErrPutBlob = errors.New("can't put blob")
ErrPostBlob = errors.New("can't post blob")
ErrPutManifest = errors.New("can't put manifest")
)
// nolint:gochecknoglobals
var (
@ -912,3 +920,259 @@ func TestBaseOciLayoutUtils(t *testing.T) {
So(err, ShouldNotBeNil)
})
}
func TestSearchSize(t *testing.T) {
Convey("Repo sizes", t, func() {
port := GetFreePort()
baseURL := GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
tr := true
conf.Extensions = &extconf.ExtensionConfig{
Search: &extconf.SearchConfig{Enable: &tr},
}
ctlr := api.NewController(conf)
dir := t.TempDir()
ctlr.Config.Storage.RootDirectory = dir
go startServer(ctlr)
defer stopServer(ctlr)
WaitTillServerReady(baseURL)
repoName := "testrepo"
config, layers, manifest, err := getImageComponents(10000)
So(err, ShouldBeNil)
configBlob, err := json.Marshal(config)
So(err, ShouldBeNil)
configSize := len(configBlob)
layersSize := 0
for _, l := range layers {
layersSize += len(l)
}
manifestBlob, err := json.Marshal(manifest)
So(err, ShouldBeNil)
manifestSize := len(manifestBlob)
err = UploadImage(
uploadImage{
Manifest: manifest,
Config: config,
Layers: layers,
Tag: "latest",
},
baseURL,
repoName,
)
So(err, ShouldBeNil)
query := `
{
GlobalSearch(query:"test"){
Images { RepoName Tag LastUpdated Size Score }
Repos {
Name LastUpdated Size Vendors Score
Platforms {
Os
Arch
}
}
Layers { Digest Size }
}
}`
resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query))
So(err, ShouldBeNil)
So(configSize+layersSize+manifestSize, ShouldNotBeZeroValue)
responseStruct := &GlobalSearchResultResp{}
err = json.Unmarshal(resp.Body(), responseStruct)
So(err, ShouldBeNil)
image := responseStruct.GlobalSearchResult.GlobalSearch.Images[0]
So(image.Tag, ShouldResemble, "latest")
size, err := strconv.Atoi(image.Size)
So(err, ShouldBeNil)
So(size, ShouldAlmostEqual, configSize+layersSize+manifestSize)
repo := responseStruct.GlobalSearchResult.GlobalSearch.Repos[0]
size, err = strconv.Atoi(repo.Size)
So(err, ShouldBeNil)
So(size, ShouldAlmostEqual, configSize+layersSize+manifestSize)
// add the same image with different tag
err = UploadImage(
uploadImage{
Manifest: manifest,
Config: config,
Layers: layers,
Tag: "10.2.14",
},
baseURL,
repoName,
)
So(err, ShouldBeNil)
resp, err = resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query))
So(err, ShouldBeNil)
So(configSize+layersSize+manifestSize, ShouldNotBeZeroValue)
responseStruct = &GlobalSearchResultResp{}
err = json.Unmarshal(resp.Body(), responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.GlobalSearchResult.GlobalSearch.Images), ShouldEqual, 2)
// check that the repo size is the same
repo = responseStruct.GlobalSearchResult.GlobalSearch.Repos[0]
size, err = strconv.Atoi(repo.Size)
So(err, ShouldBeNil)
So(size, ShouldAlmostEqual, configSize+layersSize+manifestSize)
})
}
func getImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manifest, error) {
config := ispec.Image{
Architecture: "amd64",
OS: "linux",
RootFS: ispec.RootFS{
Type: "layers",
DiffIDs: []digest.Digest{},
},
Author: "ZotUser",
}
configBlob, err := json.Marshal(config)
if err != nil {
return ispec.Image{}, [][]byte{}, ispec.Manifest{}, err
}
configDigest := digest.FromBytes(configBlob)
layers := [][]byte{
make([]byte, layerSize),
}
manifest := ispec.Manifest{
Versioned: specs.Versioned{
SchemaVersion: 2,
},
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: configDigest,
Size: int64(len(configBlob)),
},
Layers: []ispec.Descriptor{
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: digest.FromBytes(layers[0]),
Size: int64(len(layers[0])),
},
},
}
return config, layers, manifest, nil
}
type uploadImage struct {
Manifest ispec.Manifest
Config ispec.Image
Layers [][]byte
Tag string
}
func UploadImage(img uploadImage, baseURL, repo string) error {
for _, blob := range img.Layers {
resp, err := resty.R().Post(baseURL + "/v2/" + repo + "/blobs/uploads/")
if err != nil {
return err
}
if resp.StatusCode() != http.StatusAccepted {
return ErrPostBlob
}
loc := resp.Header().Get("Location")
digest := digest.FromBytes(blob).String()
resp, err = resty.R().
SetHeader("Content-Length", fmt.Sprintf("%d", len(blob))).
SetHeader("Content-Type", "application/octet-stream").
SetQueryParam("digest", digest).
SetBody(blob).
Put(baseURL + loc)
if resp.StatusCode() != http.StatusCreated {
return ErrPutBlob
}
if err != nil {
return err
}
}
// upload config
cblob, err := json.Marshal(img.Config)
if err != nil {
return err
}
cdigest := digest.FromBytes(cblob)
resp, err := resty.R().
Post(baseURL + "/v2/" + repo + "/blobs/uploads/")
if err != nil {
return err
}
if resp.StatusCode() != http.StatusAccepted {
return ErrPostBlob
}
loc := Location(baseURL, resp)
// uploading blob should get 201
resp, err = resty.R().
SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))).
SetHeader("Content-Type", "application/octet-stream").
SetQueryParam("digest", cdigest.String()).
SetBody(cblob).
Put(loc)
if err != nil {
return err
}
if resp.StatusCode() != http.StatusCreated {
return ErrPutBlob
}
// put manifest
manifestBlob, err := json.Marshal(img.Manifest)
if err != nil {
return err
}
_, err = resty.R().
SetHeader("Content-type", "application/vnd.oci.image.manifest.v1+json").
SetBody(manifestBlob).
Put(baseURL + "/v2/" + repo + "/manifests/" + img.Tag)
return err
}
func startServer(c *api.Controller) {
// this blocks
ctx := context.Background()
if err := c.Run(ctx); err != nil {
return
}
}
func stopServer(c *api.Controller) {
ctx := context.Background()
_ = c.Server.Shutdown(ctx)
}

23
pkg/extensions/search/common/oci_layout.go
View file

@ -30,7 +30,6 @@ type OciLayoutUtils interface {
GetImagePlatform(imageInfo ispec.Image) (string, string)
GetImageVendor(imageInfo ispec.Image) string
GetImageManifestSize(repo string, manifestDigest godigest.Digest) int64
GetImageConfigSize(repo string, manifestDigest godigest.Digest) int64
GetRepoLastUpdated(repo string) (time.Time, error)
GetExpandedRepoInfo(name string) (RepoInfo, error)
GetImageConfigInfo(repo string, manifestDigest godigest.Digest) (ispec.Image, error)
@ -301,14 +300,16 @@ func (olu BaseOciLayoutUtils) GetImageVendor(imageConfig ispec.Image) string {
}
func (olu BaseOciLayoutUtils) GetImageManifestSize(repo string, manifestDigest godigest.Digest) int64 {
imageBlobManifest, err := olu.GetImageBlobManifest(repo, manifestDigest)
if err != nil {
olu.Log.Error().Err(err).Msg("can't get image blob manifest")
imageStore := olu.StoreController.GetImageStore(repo)
return 0
manifestBlob, err := imageStore.GetBlobContent(repo, manifestDigest.String())
if err != nil {
olu.Log.Error().Err(err).Msg("error when getting manifest blob content")
return int64(len(manifestBlob))
}
return imageBlobManifest.Config.Size
return int64(len(manifestBlob))
}
func (olu BaseOciLayoutUtils) GetImageConfigSize(repo string, manifestDigest godigest.Digest) int64 {
@ -318,16 +319,8 @@ func (olu BaseOciLayoutUtils) GetImageConfigSize(repo string, manifestDigest god
return 0
}
imageStore := olu.StoreController.GetImageStore(repo)
buf, err := imageStore.GetBlobContent(repo, imageBlobManifest.Config.Digest.String())
if err != nil {
olu.Log.Error().Err(err).Msg("error when getting blob content")
return int64(len(buf))
}
return int64(len(buf))
return imageBlobManifest.Config.Size
}
func (olu BaseOciLayoutUtils) GetRepoLastUpdated(repo string) (time.Time, error) {

22
pkg/extensions/search/resolver.go
View file

@ -206,7 +206,7 @@ func globalSearch(repoList []string, name, tag string, olu common.OciLayoutUtils
repo := repo
// map used for dedube if 2 images reference the same blob
repoLayerBlob2Size := make(map[string]int64, 10)
repoBlob2Size := make(map[string]int64, 10)
// made up of all manifests, configs and image layers
repoSize := int64(0)
@ -235,8 +235,19 @@ func globalSearch(repoList []string, name, tag string, olu common.OciLayoutUtils
for i, manifest := range repoInfo.Manifests {
imageLayersSize := int64(0)
imageBlobManifest, err := olu.GetImageBlobManifest(repo, godigest.Digest(tagsInfo[i].Digest))
if err != nil {
log.Error().Err(err).Msgf("can't read manifest for repo %s %s", repo, manifest.Tag)
continue
}
manifestSize := olu.GetImageManifestSize(repo, godigest.Digest(tagsInfo[i].Digest))
configSize := olu.GetImageConfigSize(repo, godigest.Digest(tagsInfo[i].Digest))
configSize := imageBlobManifest.Config.Size
repoBlob2Size[tagsInfo[i].Digest] = manifestSize
repoBlob2Size[imageBlobManifest.Config.Digest.Hex] = configSize
for _, layer := range manifest.Layers {
layer := layer
@ -248,7 +259,7 @@ func globalSearch(repoList []string, name, tag string, olu common.OciLayoutUtils
continue
}
repoLayerBlob2Size[layer.Digest] = layerSize
repoBlob2Size[layer.Digest] = layerSize
imageLayersSize += layerSize
// if we have a tag we won't match a layer
@ -266,7 +277,6 @@ func globalSearch(repoList []string, name, tag string, olu common.OciLayoutUtils
}
imageSize := imageLayersSize + manifestSize + configSize
repoSize += manifestSize + configSize
index := strings.Index(repo, name)
matchesTag := strings.HasPrefix(manifest.Tag, tag)
@ -310,8 +320,8 @@ func globalSearch(repoList []string, name, tag string, olu common.OciLayoutUtils
}
}
for layerBlob := range repoLayerBlob2Size {
repoSize += repoLayerBlob2Size[layerBlob]
for blob := range repoBlob2Size {
repoSize += repoBlob2Size[blob]
}
if index := strings.Index(repo, name); index != -1 {