Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2025-04-08 02:54:41 -05:00
Code Issues Activity

feat(repodb): implement pagination for ImageList and integrate it with RepoDB (#1129)

Browse source 
* feat(repodb): implement pagination for ImageList and integrate it with RepoDB

- it can now return all images from all repos, when provided repo parameter is ""

Signed-off-by: Alex Stan <alexandrustan96@yahoo.ro>
(cherry picked from commit c003dcec9f805564946935e7eb091632f605035e)
(cherry picked from commit 72feba979b9ddd452465a652bb31f439584a046c)
Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

* ci(timeouts): increase ci-cd workflow timeout for the build and test step

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

Signed-off-by: Alex Stan <alexandrustan96@yahoo.ro>
Signed-off-by: Andrei Aaron <aaaron@luxoft.com>
Co-authored-by: Alex Stan <alexandrustan96@yahoo.ro>
This commit is contained in:
Andrei Aaron 2023-01-23 19:45:11 +02:00 • committed by GitHub
parent 3caa0f3253
commit 08983a845a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 569 additions and 397 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/ci-cd.yml vendored
View file

@ -87,7 +87,7 @@ jobs:
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake
- name: Run build and test
timeout-minutes: 70
timeout-minutes: 80
run: |
echo "Building for $OS:$ARCH"
cd $GITHUB_WORKSPACE

44
pkg/cli/image_cmd_test.go
View file

@ -368,7 +368,7 @@ func TestSignature(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:1.0 6742241d true 1B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 6742241d true 447B")
t.Log("Test getting all images using rest calls to get catalog and individual manifests")
cmd = MockNewImageCommand(new(searchService))
@ -445,7 +445,7 @@ func TestSignature(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 0.0.1 6742241d true 1B")
So(actual, ShouldContainSubstring, "repo7 0.0.1 6742241d true 447B")
t.Log("Test getting all images using rest calls to get catalog and individual manifests")
cmd = MockNewImageCommand(new(searchService))
@ -913,8 +913,8 @@ func TestServerResponseGQL(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 15B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 15B")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B")
Convey("Test all images invalid output format", func() {
args := []string{"imagetest", "-o", "random"}
configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"imagetest","url":"%s","showspinner":false}]}`, url))
@ -945,14 +945,14 @@ func TestServerResponseGQL(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
// Actual cli output should be something similar to (order of images may differ):
// IMAGE NAME TAG DIGEST CONFIG LAYERS SIZE
// repo7 test:2.0 a0ca253b b8781e88 15B
// b8781e88 15B
// repo7 test:1.0 a0ca253b b8781e88 15B
// b8781e88 15B
// IMAGE NAME TAG DIGEST CONFIG SIGNED LAYERS SIZE
// repo7 test:2.0 a0ca253b b8781e88 false 492B
// b8781e88 15B
// repo7 test:1.0 a0ca253b b8781e88 false 492B
// b8781e88 15B
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST CONFIG SIGNED LAYERS SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 3a1d2d0c false 15B b8781e88 15B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 3a1d2d0c false 15B b8781e88 15B")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 3a1d2d0c false 492B b8781e88 15B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 3a1d2d0c false 492B b8781e88 15B")
})
Convey("Test all images with debug flag", func() {
@ -971,8 +971,8 @@ func TestServerResponseGQL(t *testing.T) {
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "GET")
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 15B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 15B")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B")
})
Convey("Test image by name config url", func() {
@ -990,8 +990,8 @@ func TestServerResponseGQL(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 15B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 15B")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B")
Convey("with shorthand", func() {
args := []string{"imagetest", "-n", "repo7"}
@ -1008,8 +1008,8 @@ func TestServerResponseGQL(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIGNED SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 15B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 15B")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 false 492B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 false 492B")
})
Convey("invalid output format", func() {
@ -1193,11 +1193,11 @@ func TestServerResponse(t *testing.T) {
str := space.ReplaceAllString(buff.String(), " ")
actual := strings.TrimSpace(str)
// Actual cli output should be something similar to (order of images may differ):
// IMAGE NAME TAG DIGEST CONFIG LAYERS SIZE
// repo7 test:2.0 a0ca253b b8781e88 492B
// b8781e88 15B
// repo7 test:1.0 a0ca253b b8781e88 492B
// b8781e88 15B
// IMAGE NAME TAG DIGEST CONFIG SIGNED LAYERS SIZE
// repo7 test:2.0 a0ca253b b8781e88 false 492B
// b8781e88 15B
// repo7 test:1.0 a0ca253b b8781e88 false 492B
// b8781e88 15B
So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST CONFIG SIGNED LAYERS SIZE")
So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 3a1d2d0c false 492B b8781e88 15B")
So(actual, ShouldContainSubstring, "repo7 test:1.0 883fc0c5 3a1d2d0c false 492B b8781e88 15B")

350
pkg/extensions/search/common/common_test.go
View file

@ -34,7 +34,6 @@ import (
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/extensions/search/common"
"zotregistry.io/zot/pkg/extensions/search/convert"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
@ -3169,146 +3168,69 @@ func TestImageList(t *testing.T) {
err = json.Unmarshal(imageConfigBuf, &imageConfigInfo)
So(err, ShouldBeNil)
query := fmt.Sprintf(`{
ImageList(repo:"%s"){
History{
HistoryDescription{
Author
Comment
Created
CreatedBy
EmptyLayer
},
Layer{
Digest
Size
Convey("without pagination, valid response", func() {
query := fmt.Sprintf(`{
ImageList(repo:"%s"){
History{
HistoryDescription{
Author
Comment
Created
CreatedBy
EmptyLayer
},
Layer{
Digest
Size
}
}
}
}
}`, repos[0])
}`, repos[0])
resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
So(resp, ShouldNotBeNil)
resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
So(resp, ShouldNotBeNil)
var responseStruct ImageListResponse
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
var responseStruct ImageListResponse
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.ImageList.SummaryList[0].History), ShouldEqual, len(imageConfigInfo.History))
})
So(len(responseStruct.ImageList.SummaryList), ShouldEqual, len(tags))
So(len(responseStruct.ImageList.SummaryList[0].History), ShouldEqual, len(imageConfigInfo.History))
})
Convey("Test ImageSummary retuned by ImageList when getting tags timestamp info fails", t, func() {
invalid := "test"
tempDir := t.TempDir()
port := GetFreePort()
baseURL := GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
conf.Storage.RootDirectory = tempDir
defaultVal := true
conf.Extensions = &extconf.ExtensionConfig{
Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}},
}
conf.Extensions.Search.CVE = nil
ctlr := api.NewController(conf)
ctlrManager := NewControllerManager(ctlr)
ctlrManager.StartAndWait(port)
defer ctlrManager.StopServer()
config := ispec.Image{
Platform: ispec.Platform{
Architecture: "amd64",
OS: "linux",
},
RootFS: ispec.RootFS{
Type: "layers",
DiffIDs: []godigest.Digest{},
},
Author: "ZotUser",
History: []ispec.History{},
}
configBlob, err := json.Marshal(config)
So(err, ShouldBeNil)
configDigest := godigest.FromBytes(configBlob)
layerDigest := godigest.FromString(invalid)
layerblob := []byte(invalid)
schemaVersion := 2
ispecManifest := ispec.Manifest{
Versioned: specs.Versioned{
SchemaVersion: schemaVersion,
},
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: configDigest,
Size: int64(len(configBlob)),
},
Layers: []ispec.Descriptor{ // just 1 layer in manifest
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: layerDigest,
Size: int64(len(layerblob)),
},
},
Annotations: map[string]string{
ispec.AnnotationRefName: "1.0",
},
}
err = UploadImage(
Image{
Manifest: ispecManifest,
Config: config,
Layers: [][]byte{
layerblob,
},
Tag: "0.0.1",
},
baseURL,
invalid,
)
So(err, ShouldBeNil)
configPath := path.Join(conf.Storage.RootDirectory, invalid, "blobs",
configDigest.Algorithm().String(), configDigest.Encoded())
err = os.Remove(configPath)
So(err, ShouldBeNil)
query := fmt.Sprintf(`{
ImageList(repo:"%s"){
History{
HistoryDescription{
Author
Comment
Created
CreatedBy
EmptyLayer
},
Layer{
Digest
Size
Convey("Pagination with valid params", func() {
limit := 1
query := fmt.Sprintf(`{
ImageList(repo:"%s", requestedPage:{limit: %d, offset: 0, sortBy:RELEVANCE}){
History{
HistoryDescription{
Author
Comment
Created
CreatedBy
EmptyLayer
},
Layer{
Digest
Size
}
}
}
}
}`, invalid)
}`, repos[0], limit)
resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
So(resp, ShouldNotBeNil)
resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
So(resp, ShouldNotBeNil)
var responseStruct ImageListResponse
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.ImageList.SummaryList), ShouldBeZeroValue)
var responseStruct ImageListResponse
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.ImageList.SummaryList), ShouldEqual, limit)
})
})
}
@ -3504,114 +3426,6 @@ func TestGlobalSearchPagination(t *testing.T) {
})
}
func TestBuildImageInfo(t *testing.T) {
Convey("Check image summary when layer count does not match history", t, func() {
invalid := "invalid"
port := GetFreePort()
baseURL := GetBaseURL(port)
rootDir = t.TempDir()
conf := config.New()
conf.HTTP.Port = port
conf.Storage.RootDirectory = rootDir
defaultVal := true
conf.Extensions = &extconf.ExtensionConfig{
Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}},
}
conf.Extensions.Search.CVE = nil
ctlr := api.NewController(conf)
ctlrManager := NewControllerManager(ctlr)
ctlrManager.StartAndWait(port)
defer ctlrManager.StopServer()
olu := &common.BaseOciLayoutUtils{
StoreController: ctlr.StoreController,
Log: ctlr.Log,
}
config := ispec.Image{
Platform: ispec.Platform{
OS: "linux",
Architecture: "amd64",
},
RootFS: ispec.RootFS{
Type: "layers",
DiffIDs: []godigest.Digest{},
},
Author: "ZotUser",
History: []ispec.History{ // should contain 3 elements, 2 of which corresponding to layers
{
EmptyLayer: false,
},
{
EmptyLayer: false,
},
{
EmptyLayer: true,
},
},
}
configBlob, err := json.Marshal(config)
So(err, ShouldBeNil)
configDigest := godigest.FromBytes(configBlob)
layerDigest := godigest.FromString(invalid)
layerblob := []byte(invalid)
schemaVersion := 2
ispecManifest := ispec.Manifest{
Versioned: specs.Versioned{
SchemaVersion: schemaVersion,
},
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: configDigest,
Size: int64(len(configBlob)),
},
Layers: []ispec.Descriptor{ // just 1 layer in manifest
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: layerDigest,
Size: int64(len(layerblob)),
},
},
}
manifestLayersSize := ispecManifest.Layers[0].Size
manifestBlob, err := json.Marshal(ispecManifest)
So(err, ShouldBeNil)
manifestDigest := godigest.FromBytes(manifestBlob)
err = UploadImage(
Image{
Manifest: ispecManifest,
Config: config,
Layers: [][]byte{
layerblob,
},
Tag: "0.0.1",
},
baseURL,
invalid,
)
So(err, ShouldBeNil)
imageConfig, err := olu.GetImageConfigInfo(invalid, manifestDigest)
So(err, ShouldBeNil)
isSigned := false
imageSummary := convert.BuildImageInfo(invalid, invalid, manifestDigest, ispecManifest,
imageConfig, isSigned)
So(len(imageSummary.Layers), ShouldEqual, len(ispecManifest.Layers))
imageSummaryLayerSize, err := strconv.Atoi(*imageSummary.Size)
So(err, ShouldBeNil)
So(imageSummaryLayerSize, ShouldEqual, manifestLayersSize)
})
}
func TestRepoDBWhenSigningImages(t *testing.T) {
Convey("SigningImages", t, func() {
subpath := "/a"
@ -4498,6 +4312,50 @@ func TestBaseOciLayoutUtils(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("GetImageTagsWithTimestamp: GetImageInfo fails", t, func() {
index := ispec.Index{
Manifests: []ispec.Descriptor{
{Annotations: map[string]string{ispec.AnnotationRefName: "w"}}, {},
},
}
indexBlob, err := json.Marshal(index)
So(err, ShouldBeNil)
manifest := ispec.Manifest{
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: "configDigest",
},
Layers: []ispec.Descriptor{
{},
{},
},
}
manifestBlob, err := json.Marshal(manifest)
So(err, ShouldBeNil)
mockStoreController := mocks.MockedImageStore{
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
if digest.String() == "configDigest" {
return nil, ErrTestError
}
return manifestBlob, nil
},
GetIndexContentFn: func(repo string) ([]byte, error) {
return indexBlob, nil
},
}
storeController := storage.StoreController{DefaultStore: mockStoreController}
olu := common.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", ""))
_, err = olu.GetImageTagsWithTimestamp("repo")
So(err, ShouldNotBeNil)
})
Convey("GetExpandedRepoInfo: fails", t, func() {
index := ispec.Index{
Manifests: []ispec.Descriptor{
@ -4570,6 +4428,20 @@ func TestBaseOciLayoutUtils(t *testing.T) {
_, err = olu.GetExpandedRepoInfo("rep")
So(err, ShouldBeNil)
})
Convey("GetImageInfo fail", t, func() {
mockStoreController := mocks.MockedImageStore{
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte{}, ErrTestError
},
}
storeController := storage.StoreController{DefaultStore: mockStoreController}
olu := common.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", ""))
_, err := olu.GetImageInfo("", "")
So(err, ShouldNotBeNil)
})
}
func TestSearchSize(t *testing.T) {

278
pkg/extensions/search/convert/convert_test.go
View file

@ -4,17 +4,24 @@ import (
"context"
"encoding/json"
"errors"
"strconv"
"testing"
"github.com/99designs/gqlgen/graphql"
godigest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/extensions/search/common"
"zotregistry.io/zot/pkg/extensions/search/convert"
cveinfo "zotregistry.io/zot/pkg/extensions/search/cve"
"zotregistry.io/zot/pkg/meta/repodb"
bolt "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper"
. "zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
)
@ -74,3 +81,274 @@ func TestConvertErrors(t *testing.T) {
So(graphql.GetErrors(ctx).Error(), ShouldContainSubstring, "unable to run vulnerability scan on tag")
})
}
func TestBuildImageInfo(t *testing.T) {
rootDir := t.TempDir()
port := GetFreePort()
baseURL := GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
conf.Storage.RootDirectory = rootDir
defaultVal := true
conf.Extensions = &extconf.ExtensionConfig{
Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}},
}
conf.Extensions.Search.CVE = nil
ctlr := api.NewController(conf)
ctlrManager := NewControllerManager(ctlr)
ctlrManager.StartAndWait(port)
defer ctlrManager.StopServer()
olu := &common.BaseOciLayoutUtils{
StoreController: ctlr.StoreController,
Log: ctlr.Log,
}
Convey("Check image summary when the image has no history", t, func() {
imageName := "nohistory"
config := ispec.Image{
Platform: ispec.Platform{
OS: "linux",
Architecture: "amd64",
},
RootFS: ispec.RootFS{
Type: "layers",
DiffIDs: []godigest.Digest{},
},
Author: "ZotUser",
}
configBlob, err := json.Marshal(config)
So(err, ShouldBeNil)
configDigest := godigest.FromBytes(configBlob)
layerDigest := godigest.FromString(imageName)
layerblob := []byte(imageName)
schemaVersion := 2
ispecManifest := ispec.Manifest{
Versioned: specs.Versioned{
SchemaVersion: schemaVersion,
},
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: configDigest,
Size: int64(len(configBlob)),
},
Layers: []ispec.Descriptor{ // just 1 layer in manifest
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: layerDigest,
Size: int64(len(layerblob)),
},
},
}
manifestLayersSize := ispecManifest.Layers[0].Size
manifestBlob, err := json.Marshal(ispecManifest)
So(err, ShouldBeNil)
manifestDigest := godigest.FromBytes(manifestBlob)
err = UploadImage(
Image{
Manifest: ispecManifest,
Config: config,
Layers: [][]byte{
layerblob,
},
Tag: "0.0.1",
},
baseURL,
imageName,
)
So(err, ShouldBeNil)
imageConfig, err := olu.GetImageConfigInfo(imageName, manifestDigest)
So(err, ShouldBeNil)
isSigned := false
imageSummary := convert.BuildImageInfo(imageName, imageName, manifestDigest, ispecManifest,
imageConfig, isSigned)
So(len(imageSummary.Layers), ShouldEqual, len(ispecManifest.Layers))
imageSummaryLayerSize, err := strconv.Atoi(*imageSummary.Size)
So(err, ShouldBeNil)
So(imageSummaryLayerSize, ShouldEqual, manifestLayersSize)
})
Convey("Check image summary when layer count matche history entries", t, func() {
imageName := "valid"
config := ispec.Image{
Platform: ispec.Platform{
OS: "linux",
Architecture: "amd64",
},
RootFS: ispec.RootFS{
Type: "layers",
DiffIDs: []godigest.Digest{},
},
Author: "ZotUser",
History: []ispec.History{ // should contain 3 elements, 2 of which corresponding to layers
{
EmptyLayer: false,
},
{
EmptyLayer: false,
},
{
EmptyLayer: true,
},
},
}
configBlob, err := json.Marshal(config)
So(err, ShouldBeNil)
configDigest := godigest.FromBytes(configBlob)
layerDigest := godigest.FromString("layer1")
layerblob := []byte("layer1")
layerDigest2 := godigest.FromString("layer2")
layerblob2 := []byte("layer2")
schemaVersion := 2
ispecManifest := ispec.Manifest{
Versioned: specs.Versioned{
SchemaVersion: schemaVersion,
},
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: configDigest,
Size: int64(len(configBlob)),
},
Layers: []ispec.Descriptor{ // just 1 layer in manifest
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: layerDigest,
Size: int64(len(layerblob)),
},
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: layerDigest2,
Size: int64(len(layerblob2)),
},
},
}
manifestLayersSize := ispecManifest.Layers[0].Size + ispecManifest.Layers[1].Size
manifestBlob, err := json.Marshal(ispecManifest)
So(err, ShouldBeNil)
manifestDigest := godigest.FromBytes(manifestBlob)
err = UploadImage(
Image{
Manifest: ispecManifest,
Config: config,
Layers: [][]byte{
layerblob,
layerblob2,
},
Tag: "0.0.1",
},
baseURL,
imageName,
)
So(err, ShouldBeNil)
imageConfig, err := olu.GetImageConfigInfo(imageName, manifestDigest)
So(err, ShouldBeNil)
isSigned := false
imageSummary := convert.BuildImageInfo(imageName, imageName, manifestDigest, ispecManifest,
imageConfig, isSigned)
So(len(imageSummary.Layers), ShouldEqual, len(ispecManifest.Layers))
imageSummaryLayerSize, err := strconv.Atoi(*imageSummary.Size)
So(err, ShouldBeNil)
So(imageSummaryLayerSize, ShouldEqual, manifestLayersSize)
})
Convey("Check image summary when layer count does not match history", t, func() {
imageName := "invalid"
config := ispec.Image{
Platform: ispec.Platform{
OS: "linux",
Architecture: "amd64",
},
RootFS: ispec.RootFS{
Type: "layers",
DiffIDs: []godigest.Digest{},
},
Author: "ZotUser",
History: []ispec.History{ // should contain 3 elements, 2 of which corresponding to layers
{
EmptyLayer: false,
},
{
EmptyLayer: false,
},
{
EmptyLayer: true,
},
},
}
configBlob, err := json.Marshal(config)
So(err, ShouldBeNil)
configDigest := godigest.FromBytes(configBlob)
layerDigest := godigest.FromString(imageName)
layerblob := []byte(imageName)
schemaVersion := 2
ispecManifest := ispec.Manifest{
Versioned: specs.Versioned{
SchemaVersion: schemaVersion,
},
Config: ispec.Descriptor{
MediaType: "application/vnd.oci.image.config.v1+json",
Digest: configDigest,
Size: int64(len(configBlob)),
},
Layers: []ispec.Descriptor{ // just 1 layer in manifest
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: layerDigest,
Size: int64(len(layerblob)),
},
},
}
manifestLayersSize := ispecManifest.Layers[0].Size
manifestBlob, err := json.Marshal(ispecManifest)
So(err, ShouldBeNil)
manifestDigest := godigest.FromBytes(manifestBlob)
err = UploadImage(
Image{
Manifest: ispecManifest,
Config: config,
Layers: [][]byte{
layerblob,
},
Tag: "0.0.1",
},
baseURL,
imageName,
)
So(err, ShouldBeNil)
imageConfig, err := olu.GetImageConfigInfo(imageName, manifestDigest)
So(err, ShouldBeNil)
isSigned := false
imageSummary := convert.BuildImageInfo(imageName, imageName, manifestDigest, ispecManifest,
imageConfig, isSigned)
So(len(imageSummary.Layers), ShouldEqual, len(ispecManifest.Layers))
imageSummaryLayerSize, err := strconv.Atoi(*imageSummary.Size)
So(err, ShouldBeNil)
So(imageSummaryLayerSize, ShouldEqual, manifestLayersSize)
})
}

21
pkg/extensions/search/gql_generated/generated.go
View file

@ -149,7 +149,7 @@ type ComplexityRoot struct {
ExpandedRepoInfo func(childComplexity int, repo string) int
GlobalSearch func(childComplexity int, query string, filter *Filter, requestedPage *PageInput) int
Image func(childComplexity int, image string) int
ImageList func(childComplexity int, repo string) int
ImageList func(childComplexity int, repo string, requestedPage *PageInput) int
ImageListForCve func(childComplexity int, id string, requestedPage *PageInput) int
ImageListForDigest func(childComplexity int, id string, requestedPage *PageInput) int
ImageListWithCVEFixed func(childComplexity int, id string, image string, requestedPage *PageInput) int
@ -191,7 +191,7 @@ type QueryResolver interface {
ImageListWithCVEFixed(ctx context.Context, id string, image string, requestedPage *PageInput) ([]*ImageSummary, error)
ImageListForDigest(ctx context.Context, id string, requestedPage *PageInput) ([]*ImageSummary, error)
RepoListWithNewestImage(ctx context.Context, requestedPage *PageInput) (*PaginatedReposResult, error)
ImageList(ctx context.Context, repo string) ([]*ImageSummary, error)
ImageList(ctx context.Context, repo string, requestedPage *PageInput) ([]*ImageSummary, error)
ExpandedRepoInfo(ctx context.Context, repo string) (*RepoInfo, error)
GlobalSearch(ctx context.Context, query string, filter *Filter, requestedPage *PageInput) (*GlobalSearchResult, error)
DerivedImageList(ctx context.Context, image string) ([]*ImageSummary, error)
@ -696,7 +696,7 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
return 0, false
}
return e.complexity.Query.ImageList(childComplexity, args["repo"].(string)), true
return e.complexity.Query.ImageList(childComplexity, args["repo"].(string), args["requestedPage"].(*PageInput)), true
case "Query.ImageListForCVE":
if e.complexity.Query.ImageListForCve == nil {
@ -1166,9 +1166,9 @@ type Query {
RepoListWithNewestImage(requestedPage: PageInput): PaginatedReposResult! # Newest based on created timestamp
"""
Returns all the images from the specified repo
Returns all the images from the specified repo | from all repos if specified repo is ""
"""
ImageList(repo: String!): [ImageSummary!]
ImageList(repo: String!, requestedPage: PageInput): [ImageSummary!]
"""
Returns information about the specified repo
@ -1395,6 +1395,15 @@ func (ec *executionContext) field_Query_ImageList_args(ctx context.Context, rawA
}
}
args["repo"] = arg0
var arg1 *PageInput
if tmp, ok := rawArgs["requestedPage"]; ok {
ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("requestedPage"))
arg1, err = ec.unmarshalOPageInput2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐPageInput(ctx, tmp)
if err != nil {
return nil, err
}
}
args["requestedPage"] = arg1
return args, nil
}
@ -4515,7 +4524,7 @@ func (ec *executionContext) _Query_ImageList(ctx context.Context, field graphql.
}()
resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
ctx = rctx // use context from middleware stack in children
return ec.resolvers.Query().ImageList(rctx, fc.Args["repo"].(string))
return ec.resolvers.Query().ImageList(rctx, fc.Args["repo"].(string), fc.Args["requestedPage"].(*PageInput))
})
if err != nil {
ec.Error(ctx, err)

95
pkg/extensions/search/resolver.go
View file

@ -701,74 +701,47 @@ func searchingForRepos(query string) bool {
return !strings.Contains(query, ":")
}
func (r *queryResolver) getImageList(store storage.ImageStore, imageName string) (
[]*gql_generated.ImageSummary, error,
) {
results := make([]*gql_generated.ImageSummary, 0)
func getImageList(ctx context.Context, repo string, repoDB repodb.RepoDB, cveInfo cveinfo.CveInfo,
requestedPage *gql_generated.PageInput, log log.Logger, //nolint:unparam
) ([]*gql_generated.ImageSummary, error) {
imageList := make([]*gql_generated.ImageSummary, 0)
repoList, err := store.GetRepositories()
if requestedPage == nil {
requestedPage = &gql_generated.PageInput{}
}
skip := convert.SkipQGLField{
Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Images.Vulnerabilities"),
}
pageInput := repodb.PageInput{
Limit: safeDerefferencing(requestedPage.Limit, 0),
Offset: safeDerefferencing(requestedPage.Offset, 0),
SortBy: repodb.SortCriteria(
safeDerefferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance),
),
}
// reposMeta, manifestMetaMap, err := repoDB.SearchRepos(ctx, repo, repodb.Filter{}, pageInput)
reposMeta, manifestMetaMap, err := repoDB.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool {
return true
},
pageInput)
if err != nil {
r.log.Error().Err(err).Msg("extension api: error extracting repositories list")
return results, err
return []*gql_generated.ImageSummary{}, err
}
layoutUtils := common.NewBaseOciLayoutUtils(r.storeController, r.log)
for _, repo := range repoList {
if (imageName != "" && repo == imageName) || imageName == "" {
tagsInfo, err := layoutUtils.GetImageTagsWithTimestamp(repo)
if err != nil {
r.log.Error().Err(err).Msg("extension api: error getting tag timestamp info")
return results, nil
}
if len(tagsInfo) == 0 {
r.log.Info().Str("no tagsinfo found for repo", repo).Msg(" continuing traversing")
continue
}
for i := range tagsInfo {
// using a loop variable called tag would be reassigned after each iteration, using the same memory address
// directly access the value at the current index in the slice as ImageInfo requires pointers to tag fields
tag := tagsInfo[i]
digest := tag.Digest
manifest, err := layoutUtils.GetImageBlobManifest(repo, digest)
if err != nil {
r.log.Error().Err(err).Msg("extension api: error reading manifest")
return results, err
}
imageConfig, err := layoutUtils.GetImageConfigInfo(repo, digest)
if err != nil {
return results, err
}
isSigned := layoutUtils.CheckManifestSignature(repo, digest)
tagPrefix := strings.HasPrefix(tag.Name, "sha256-")
tagSuffix := strings.HasSuffix(tag.Name, ".sig")
imageInfo := convert.BuildImageInfo(repo, tag.Name, digest, manifest,
imageConfig, isSigned)
// check if it's an image or a signature
if !tagPrefix && !tagSuffix {
results = append(results, imageInfo)
}
}
for _, repoMeta := range reposMeta {
if repoMeta.Name != repo && repo != "" {
continue
}
imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, skip, cveInfo)
imageList = append(imageList, imageSummaries...)
}
if len(results) == 0 {
r.log.Info().Msg("no repositories found")
}
return results, nil
return imageList, nil
}
func getReferrers(store storage.ImageStore, repoName string, digest string, artifactType string, log log.Logger) (

143
pkg/extensions/search/resolver_test.go
View file

@ -569,7 +569,6 @@ func TestImageListForDigest(t *testing.T) {
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, err := getImageListForDigest(responseContext, "invalid", mockSearchDB, mocks.CveInfoMock{}, nil)
So(err, ShouldNotBeNil)
})
@ -595,7 +594,6 @@ func TestImageListForDigest(t *testing.T) {
},
})
So(err, ShouldBeNil)
manifestBlob := []byte("invalid")
manifestMetaDatas := map[string]repodb.ManifestMetadata{
@ -648,7 +646,6 @@ func TestImageListForDigest(t *testing.T) {
DownloadCount: 0,
},
}
matchedTags := repos[0].Tags
for tag, descriptor := range repos[0].Tags {
if !filter(repos[0], manifestMetaDatas[descriptor.Digest]) {
@ -676,7 +673,6 @@ func TestImageListForDigest(t *testing.T) {
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
imageSummaries, err := getImageListForDigest(responseContext, manifestDigest,
mockSearchDB, mocks.CveInfoMock{}, &pageInput)
So(err, ShouldBeNil)
@ -989,6 +985,102 @@ func TestImageListForDigest(t *testing.T) {
})
}
func TestImageList(t *testing.T) {
Convey("getImageList", t, func() {
testLogger := log.NewLogger("debug", "")
Convey("no page requested, SearchRepoFn returns error", func() {
mockSearchDB := mocks.RepoDBMock{
FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, ErrTestError
},
}
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, err := getImageList(responseContext, "test", mockSearchDB, mocks.CveInfoMock{}, nil, testLogger)
So(err, ShouldNotBeNil)
})
Convey("valid repoList returned", func() {
mockSearchDB := mocks.RepoDBMock{
FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
repos := []repodb.RepoMetadata{
{
Name: "test",
Tags: map[string]repodb.Descriptor{
"1.0.1": {
Digest: "digestTag1.0.1",
MediaType: ispec.MediaTypeImageManifest,
},
},
Signatures: map[string]repodb.ManifestSignatures{
"digestTag1.0.1": {
"cosgin": []repodb.SignatureInfo{
{SignatureManifestDigest: "digestSignature1"},
},
},
},
Stars: 100,
},
}
configBlob, err := json.Marshal(ispec.Image{
Config: ispec.ImageConfig{
Labels: map[string]string{},
},
})
So(err, ShouldBeNil)
manifestBlob, err := json.Marshal(ispec.Manifest{})
So(err, ShouldBeNil)
manifestMetaDatas := map[string]repodb.ManifestMetadata{
"digestTag1.0.1": {
ManifestBlob: manifestBlob,
ConfigBlob: configBlob,
DownloadCount: 0,
Signatures: repodb.ManifestSignatures{
"cosgin": []repodb.SignatureInfo{
{SignatureManifestDigest: "digestSignature1"},
},
},
},
}
return repos, manifestMetaDatas, nil
},
}
limit := 1
ofset := 0
sortCriteria := gql_generated.SortCriteriaAlphabeticAsc
pageInput := gql_generated.PageInput{
Limit: &limit,
Offset: &ofset,
SortBy: &sortCriteria,
}
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
imageSummaries, err := getImageList(responseContext, "test", mockSearchDB,
mocks.CveInfoMock{}, &pageInput, testLogger)
So(err, ShouldBeNil)
So(len(imageSummaries), ShouldEqual, 1)
imageSummaries, err = getImageList(responseContext, "invalid", mockSearchDB,
mocks.CveInfoMock{}, &pageInput, testLogger)
So(err, ShouldBeNil)
So(len(imageSummaries), ShouldEqual, 0)
})
})
}
func TestGetReferrers(t *testing.T) {
Convey("getReferrers", t, func() {
Convey("GetReferrers returns error", func() {
@ -1302,14 +1394,14 @@ func TestQueryResolverErrors(t *testing.T) {
Convey("ImageList getImageList() errors", func() {
resolverConfig := NewResolver(
log,
storage.StoreController{
DefaultStore: mocks.MockedImageStore{
GetRepositoriesFn: func() ([]string, error) {
return nil, ErrTestError
},
storage.StoreController{},
mocks.RepoDBMock{
FilterTagsFn: func(ctx context.Context, filter repodb.FilterFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, ErrTestError
},
},
mocks.RepoDBMock{},
mocks.CveInfoMock{},
)
@ -1317,36 +1409,7 @@ func TestQueryResolverErrors(t *testing.T) {
resolverConfig,
}
_, err := qr.ImageList(ctx, "repo")
So(err, ShouldNotBeNil)
})
Convey("ImageList subpaths getImageList() errors", func() {
resolverConfig := NewResolver(
log,
storage.StoreController{
DefaultStore: mocks.MockedImageStore{
GetRepositoriesFn: func() ([]string, error) {
return []string{"sub1/repo"}, nil
},
},
SubStore: map[string]storage.ImageStore{
"/sub1": mocks.MockedImageStore{
GetRepositoriesFn: func() ([]string, error) {
return nil, ErrTestError
},
},
},
},
mocks.RepoDBMock{},
mocks.CveInfoMock{},
)
qr := queryResolver{
resolverConfig,
}
_, err := qr.ImageList(ctx, "repo")
_, err := qr.ImageList(ctx, "repo", &gql_generated.PageInput{})
So(err, ShouldNotBeNil)
})

4
pkg/extensions/search/schema.graphql
View file

@ -225,9 +225,9 @@ type Query {
RepoListWithNewestImage(requestedPage: PageInput): PaginatedReposResult! # Newest based on created timestamp
"""
Returns all the images from the specified repo
Returns all the images from the specified repo | from all repos if specified repo is ""
"""
ImageList(repo: String!): [ImageSummary!]
ImageList(repo: String!, requestedPage: PageInput): [ImageSummary!]
"""
Returns information about the specified repo

29
pkg/extensions/search/schema.resolvers.go
View file

@ -64,39 +64,16 @@ func (r *queryResolver) RepoListWithNewestImage(ctx context.Context, requestedPa
}
// ImageList is the resolver for the ImageList field.
func (r *queryResolver) ImageList(ctx context.Context, repo string) ([]*gql_generated.ImageSummary, error) {
func (r *queryResolver) ImageList(ctx context.Context, repo string, requestedPage *gql_generated.PageInput) ([]*gql_generated.ImageSummary, error) {
r.log.Info().Msg("extension api: getting a list of all images")
imageList := make([]*gql_generated.ImageSummary, 0)
defaultStore := r.storeController.DefaultStore
dsImageList, err := r.getImageList(defaultStore, repo)
imageList, err := getImageList(ctx, repo, r.repoDB, r.cveInfo, requestedPage, r.log)
if err != nil {
r.log.Error().Err(err).Msg("extension api: error extracting default store image list")
r.log.Error().Err(err).Msgf("unable to retrieve image list for repo: %s", repo)
return imageList, err
}
if len(dsImageList) != 0 {
imageList = append(imageList, dsImageList...)
}
subStore := r.storeController.SubStore
for _, store := range subStore {
ssImageList, err := r.getImageList(store, repo)
if err != nil {
r.log.Error().Err(err).Msg("extension api: error extracting substore image list")
return imageList, err
}
if len(ssImageList) != 0 {
imageList = append(imageList, ssImageList...)
}
}
return imageList, nil
}