diff --git a/pkg/extensions/search/common/common_test.go b/pkg/extensions/search/common/common_test.go index dbde9c1d..c0c9a6de 100644 --- a/pkg/extensions/search/common/common_test.go +++ b/pkg/extensions/search/common/common_test.go @@ -9,14 +9,17 @@ import ( "errors" "fmt" "io/ioutil" + "net/http" "net/url" "os" "os/exec" "path" + "strconv" "testing" "time" "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sigstore/cosign/cmd/cosign/cli/generate" "github.com/sigstore/cosign/cmd/cosign/cli/options" @@ -39,7 +42,12 @@ const ( graphqlQueryPrefix = constants.ExtSearchPrefix ) -var ErrTestError = errors.New("test error") +var ( + ErrTestError = errors.New("test error") + ErrPutBlob = errors.New("can't put blob") + ErrPostBlob = errors.New("can't post blob") + ErrPutManifest = errors.New("can't put manifest") +) // nolint:gochecknoglobals var ( @@ -912,3 +920,259 @@ func TestBaseOciLayoutUtils(t *testing.T) { So(err, ShouldNotBeNil) }) } + +func TestSearchSize(t *testing.T) { + Convey("Repo sizes", t, func() { + port := GetFreePort() + baseURL := GetBaseURL(port) + + conf := config.New() + conf.HTTP.Port = port + tr := true + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{Enable: &tr}, + } + + ctlr := api.NewController(conf) + dir := t.TempDir() + ctlr.Config.Storage.RootDirectory = dir + + go startServer(ctlr) + defer stopServer(ctlr) + WaitTillServerReady(baseURL) + + repoName := "testrepo" + config, layers, manifest, err := getImageComponents(10000) + So(err, ShouldBeNil) + + configBlob, err := json.Marshal(config) + So(err, ShouldBeNil) + configSize := len(configBlob) + + layersSize := 0 + for _, l := range layers { + layersSize += len(l) + } + + manifestBlob, err := json.Marshal(manifest) + So(err, ShouldBeNil) + manifestSize := len(manifestBlob) + + err = UploadImage( + uploadImage{ + Manifest: manifest, + Config: config, + Layers: layers, + Tag: "latest", + }, + baseURL, + repoName, + ) + So(err, ShouldBeNil) + + query := ` + { + GlobalSearch(query:"test"){ + Images { RepoName Tag LastUpdated Size Score } + Repos { + Name LastUpdated Size Vendors Score + Platforms { + Os + Arch + } + } + Layers { Digest Size } + } + }` + resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query)) + So(err, ShouldBeNil) + So(configSize+layersSize+manifestSize, ShouldNotBeZeroValue) + + responseStruct := &GlobalSearchResultResp{} + err = json.Unmarshal(resp.Body(), responseStruct) + So(err, ShouldBeNil) + + image := responseStruct.GlobalSearchResult.GlobalSearch.Images[0] + So(image.Tag, ShouldResemble, "latest") + + size, err := strconv.Atoi(image.Size) + So(err, ShouldBeNil) + So(size, ShouldAlmostEqual, configSize+layersSize+manifestSize) + + repo := responseStruct.GlobalSearchResult.GlobalSearch.Repos[0] + size, err = strconv.Atoi(repo.Size) + So(err, ShouldBeNil) + So(size, ShouldAlmostEqual, configSize+layersSize+manifestSize) + + // add the same image with different tag + err = UploadImage( + uploadImage{ + Manifest: manifest, + Config: config, + Layers: layers, + Tag: "10.2.14", + }, + baseURL, + repoName, + ) + So(err, ShouldBeNil) + + resp, err = resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query)) + So(err, ShouldBeNil) + So(configSize+layersSize+manifestSize, ShouldNotBeZeroValue) + + responseStruct = &GlobalSearchResultResp{} + err = json.Unmarshal(resp.Body(), responseStruct) + So(err, ShouldBeNil) + + So(len(responseStruct.GlobalSearchResult.GlobalSearch.Images), ShouldEqual, 2) + // check that the repo size is the same + repo = responseStruct.GlobalSearchResult.GlobalSearch.Repos[0] + size, err = strconv.Atoi(repo.Size) + So(err, ShouldBeNil) + So(size, ShouldAlmostEqual, configSize+layersSize+manifestSize) + }) +} + +func getImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manifest, error) { + config := ispec.Image{ + Architecture: "amd64", + OS: "linux", + RootFS: ispec.RootFS{ + Type: "layers", + DiffIDs: []digest.Digest{}, + }, + Author: "ZotUser", + } + + configBlob, err := json.Marshal(config) + if err != nil { + return ispec.Image{}, [][]byte{}, ispec.Manifest{}, err + } + + configDigest := digest.FromBytes(configBlob) + + layers := [][]byte{ + make([]byte, layerSize), + } + + manifest := ispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: configDigest, + Size: int64(len(configBlob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[0]), + Size: int64(len(layers[0])), + }, + }, + } + + return config, layers, manifest, nil +} + +type uploadImage struct { + Manifest ispec.Manifest + Config ispec.Image + Layers [][]byte + Tag string +} + +func UploadImage(img uploadImage, baseURL, repo string) error { + for _, blob := range img.Layers { + resp, err := resty.R().Post(baseURL + "/v2/" + repo + "/blobs/uploads/") + if err != nil { + return err + } + + if resp.StatusCode() != http.StatusAccepted { + return ErrPostBlob + } + + loc := resp.Header().Get("Location") + + digest := digest.FromBytes(blob).String() + + resp, err = resty.R(). + SetHeader("Content-Length", fmt.Sprintf("%d", len(blob))). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", digest). + SetBody(blob). + Put(baseURL + loc) + + if resp.StatusCode() != http.StatusCreated { + return ErrPutBlob + } + + if err != nil { + return err + } + } + + // upload config + cblob, err := json.Marshal(img.Config) + if err != nil { + return err + } + + cdigest := digest.FromBytes(cblob) + + resp, err := resty.R(). + Post(baseURL + "/v2/" + repo + "/blobs/uploads/") + if err != nil { + return err + } + + if resp.StatusCode() != http.StatusAccepted { + return ErrPostBlob + } + + loc := Location(baseURL, resp) + + // uploading blob should get 201 + resp, err = resty.R(). + SetHeader("Content-Length", fmt.Sprintf("%d", len(cblob))). + SetHeader("Content-Type", "application/octet-stream"). + SetQueryParam("digest", cdigest.String()). + SetBody(cblob). + Put(loc) + if err != nil { + return err + } + + if resp.StatusCode() != http.StatusCreated { + return ErrPutBlob + } + + // put manifest + manifestBlob, err := json.Marshal(img.Manifest) + if err != nil { + return err + } + + _, err = resty.R(). + SetHeader("Content-type", "application/vnd.oci.image.manifest.v1+json"). + SetBody(manifestBlob). + Put(baseURL + "/v2/" + repo + "/manifests/" + img.Tag) + + return err +} + +func startServer(c *api.Controller) { + // this blocks + ctx := context.Background() + if err := c.Run(ctx); err != nil { + return + } +} + +func stopServer(c *api.Controller) { + ctx := context.Background() + _ = c.Server.Shutdown(ctx) +} diff --git a/pkg/extensions/search/common/oci_layout.go b/pkg/extensions/search/common/oci_layout.go index 2ee3c864..40d1d2b7 100644 --- a/pkg/extensions/search/common/oci_layout.go +++ b/pkg/extensions/search/common/oci_layout.go @@ -30,7 +30,6 @@ type OciLayoutUtils interface { GetImagePlatform(imageInfo ispec.Image) (string, string) GetImageVendor(imageInfo ispec.Image) string GetImageManifestSize(repo string, manifestDigest godigest.Digest) int64 - GetImageConfigSize(repo string, manifestDigest godigest.Digest) int64 GetRepoLastUpdated(repo string) (time.Time, error) GetExpandedRepoInfo(name string) (RepoInfo, error) GetImageConfigInfo(repo string, manifestDigest godigest.Digest) (ispec.Image, error) @@ -301,14 +300,16 @@ func (olu BaseOciLayoutUtils) GetImageVendor(imageConfig ispec.Image) string { } func (olu BaseOciLayoutUtils) GetImageManifestSize(repo string, manifestDigest godigest.Digest) int64 { - imageBlobManifest, err := olu.GetImageBlobManifest(repo, manifestDigest) - if err != nil { - olu.Log.Error().Err(err).Msg("can't get image blob manifest") + imageStore := olu.StoreController.GetImageStore(repo) - return 0 + manifestBlob, err := imageStore.GetBlobContent(repo, manifestDigest.String()) + if err != nil { + olu.Log.Error().Err(err).Msg("error when getting manifest blob content") + + return int64(len(manifestBlob)) } - return imageBlobManifest.Config.Size + return int64(len(manifestBlob)) } func (olu BaseOciLayoutUtils) GetImageConfigSize(repo string, manifestDigest godigest.Digest) int64 { @@ -318,16 +319,8 @@ func (olu BaseOciLayoutUtils) GetImageConfigSize(repo string, manifestDigest god return 0 } - imageStore := olu.StoreController.GetImageStore(repo) - buf, err := imageStore.GetBlobContent(repo, imageBlobManifest.Config.Digest.String()) - if err != nil { - olu.Log.Error().Err(err).Msg("error when getting blob content") - - return int64(len(buf)) - } - - return int64(len(buf)) + return imageBlobManifest.Config.Size } func (olu BaseOciLayoutUtils) GetRepoLastUpdated(repo string) (time.Time, error) { diff --git a/pkg/extensions/search/resolver.go b/pkg/extensions/search/resolver.go index d6c633e6..6b21939b 100644 --- a/pkg/extensions/search/resolver.go +++ b/pkg/extensions/search/resolver.go @@ -206,7 +206,7 @@ func globalSearch(repoList []string, name, tag string, olu common.OciLayoutUtils repo := repo // map used for dedube if 2 images reference the same blob - repoLayerBlob2Size := make(map[string]int64, 10) + repoBlob2Size := make(map[string]int64, 10) // made up of all manifests, configs and image layers repoSize := int64(0) @@ -235,8 +235,19 @@ func globalSearch(repoList []string, name, tag string, olu common.OciLayoutUtils for i, manifest := range repoInfo.Manifests { imageLayersSize := int64(0) + + imageBlobManifest, err := olu.GetImageBlobManifest(repo, godigest.Digest(tagsInfo[i].Digest)) + if err != nil { + log.Error().Err(err).Msgf("can't read manifest for repo %s %s", repo, manifest.Tag) + + continue + } + manifestSize := olu.GetImageManifestSize(repo, godigest.Digest(tagsInfo[i].Digest)) - configSize := olu.GetImageConfigSize(repo, godigest.Digest(tagsInfo[i].Digest)) + configSize := imageBlobManifest.Config.Size + + repoBlob2Size[tagsInfo[i].Digest] = manifestSize + repoBlob2Size[imageBlobManifest.Config.Digest.Hex] = configSize for _, layer := range manifest.Layers { layer := layer @@ -248,7 +259,7 @@ func globalSearch(repoList []string, name, tag string, olu common.OciLayoutUtils continue } - repoLayerBlob2Size[layer.Digest] = layerSize + repoBlob2Size[layer.Digest] = layerSize imageLayersSize += layerSize // if we have a tag we won't match a layer @@ -266,7 +277,6 @@ func globalSearch(repoList []string, name, tag string, olu common.OciLayoutUtils } imageSize := imageLayersSize + manifestSize + configSize - repoSize += manifestSize + configSize index := strings.Index(repo, name) matchesTag := strings.HasPrefix(manifest.Tag, tag) @@ -310,8 +320,8 @@ func globalSearch(repoList []string, name, tag string, olu common.OciLayoutUtils } } - for layerBlob := range repoLayerBlob2Size { - repoSize += repoLayerBlob2Size[layerBlob] + for blob := range repoBlob2Size { + repoSize += repoBlob2Size[blob] } if index := strings.Index(repo, name); index != -1 {