diff --git a/pkg/cli/image_cmd.go b/pkg/cli/image_cmd.go index 0fbd9372..73c15e56 100644 --- a/pkg/cli/image_cmd.go +++ b/pkg/cli/image_cmd.go @@ -121,6 +121,8 @@ func setupImageFlags(imageCmd *cobra.Command, searchImageParams map[string]*stri searchImageParams["imageName"] = imageCmd.Flags().StringP("name", "n", "", "List image details by name") searchImageParams["digest"] = imageCmd.Flags().StringP("digest", "d", "", "List images containing a specific manifest, config, or layer digest") + searchImageParams["baseImage"] = imageCmd.Flags().StringP("base-images", "b", "", + "List images that are base for the given image") imageCmd.Flags().StringVar(servURL, "url", "", "Specify zot server URL if config-name is not mentioned") imageCmd.Flags().StringVarP(user, "user", "u", "", `User Credentials of zot server in "username:password" format`) diff --git a/pkg/cli/image_cmd_test.go b/pkg/cli/image_cmd_test.go index ba67c215..4ef36a49 100644 --- a/pkg/cli/image_cmd_test.go +++ b/pkg/cli/image_cmd_test.go @@ -250,6 +250,99 @@ func TestSearchImageCmd(t *testing.T) { }) } +func TestBaseImageList(t *testing.T) { + Convey("Test from real server", t, func() { + port := test.GetFreePort() + url := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + defaultVal := true + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{Enable: &defaultVal}, + } + ctlr := api.NewController(conf) + ctlr.Config.Storage.RootDirectory = t.TempDir() + + go func(controller *api.Controller) { + // this blocks + if err := controller.Run(context.Background()); err != nil { + return + } + }(ctlr) + // wait till ready + for { + _, err := resty.R().Get(url) + if err == nil { + break + } + + time.Sleep(100 * time.Millisecond) + } + defer func(controller *api.Controller) { + ctx := context.Background() + _ = controller.Server.Shutdown(ctx) + }(ctlr) + + err := uploadManifest(url) + So(err, ShouldBeNil) + t.Logf("rootDir: %s", ctlr.Config.Storage.RootDirectory) + + Convey("Test base images list working", func() { + t.Logf("%s", ctlr.Config.Storage.RootDirectory) + args := []string{"imagetest", "--base-images", "repo7:test:1.0"} + configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"imagetest","url":"%s","showspinner":false}]}`, url)) + defer os.Remove(configPath) + cmd := NewImageCommand(new(searchService)) + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err := cmd.Execute() + So(err, ShouldBeNil) + space := regexp.MustCompile(`\s+`) + str := space.ReplaceAllString(buff.String(), " ") + actual := strings.TrimSpace(str) + So(actual, ShouldContainSubstring, "IMAGE NAME TAG DIGEST SIZE") + So(actual, ShouldContainSubstring, "repo7 test:2.0 883fc0c5 492B") + }) + + Convey("Test base images fail", func() { + t.Logf("%s", ctlr.Config.Storage.RootDirectory) + err = os.Chmod(ctlr.Config.Storage.RootDirectory, 0o000) + So(err, ShouldBeNil) + + defer func() { + err := os.Chmod(ctlr.Config.Storage.RootDirectory, 0o755) + So(err, ShouldBeNil) + }() + args := []string{"imagetest", "--base-images", "repo7:test:1.0"} + configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"imagetest","url":"%s","showspinner":false}]}`, url)) + defer os.Remove(configPath) + cmd := NewImageCommand(new(searchService)) + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err = cmd.Execute() + So(err, ShouldNotBeNil) + }) + + Convey("Test base images list cannot print", func() { + t.Logf("%s", ctlr.Config.Storage.RootDirectory) + args := []string{"imagetest", "--base-images", "repo7:test:1.0", "-o", "random"} + configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"imagetest","url":"%s","showspinner":false}]}`, url)) + defer os.Remove(configPath) + cmd := NewImageCommand(new(searchService)) + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err := cmd.Execute() + So(err, ShouldNotBeNil) + }) + }) +} + func TestListRepos(t *testing.T) { Convey("Test listing repositories", t, func() { args := []string{"config-test"} @@ -308,7 +401,7 @@ func TestListRepos(t *testing.T) { }) Convey("Test unable to get config value", t, func() { - args := []string{"config-test-inexistent"} + args := []string{"config-test-nonexistent"} configPath := makeConfigFile(`{"configs":[{"_name":"config-test","url":"https://test-url.com","showspinner":false}]}`) defer os.Remove(configPath) cmd := NewRepoCommand(new(mockService)) @@ -1172,6 +1265,22 @@ func (service mockService) getRepos(ctx context.Context, config searchConfig, us channel <- stringResult{"", nil} } +func (service mockService) getBaseImageListGQL(ctx context.Context, config searchConfig, username, password string, + derivedImage string, +) (*imageListStructForBaseImagesGQL, error) { + imageListGQLResponse := &imageListStructForBaseImagesGQL{} + imageListGQLResponse.Data.ImageList = []imageStruct{ + { + RepoName: "dummyImageName", + Tag: "tag", + Digest: "DigestsAreReallyLong", + Size: "123445", + }, + } + + return imageListGQLResponse, nil +} + func (service mockService) getImagesGQL(ctx context.Context, config searchConfig, username, password string, imageName string, ) (*imageListStructGQL, error) { diff --git a/pkg/cli/searcher.go b/pkg/cli/searcher.go index cf08d086..6eed4aea 100644 --- a/pkg/cli/searcher.go +++ b/pkg/cli/searcher.go @@ -42,6 +42,7 @@ func getImageSearchersGQL() []searcher { new(allImagesSearcherGQL), new(imageByNameSearcherGQL), new(imagesByDigestSearcherGQL), + new(baseImageListSearcherGQL), } return searchers @@ -219,6 +220,31 @@ func (search imagesByDigestSearcher) search(config searchConfig) (bool, error) { } } +type baseImageListSearcherGQL struct{} + +func (search baseImageListSearcherGQL) search(config searchConfig) (bool, error) { + if !canSearch(config.params, newSet("baseImage")) { + return false, nil + } + + username, password := getUsernameAndPassword(*config.user) + ctx, cancel := context.WithCancel(context.Background()) + + defer cancel() + + imageList, err := config.searchService.getBaseImageListGQL(ctx, config, username, + password, *config.params["baseImage"]) + if err != nil { + return true, err + } + + if err := printResult(config, imageList.Data.ImageList); err != nil { + return true, err + } + + return true, nil +} + type imagesByDigestSearcherGQL struct{} func (search imagesByDigestSearcherGQL) search(config searchConfig) (bool, error) { diff --git a/pkg/cli/service.go b/pkg/cli/service.go index 0f97cca2..3367c7cb 100644 --- a/pkg/cli/service.go +++ b/pkg/cli/service.go @@ -34,6 +34,8 @@ type SearchService interface { cveID string) (*imagesForCve, error) getFixedTagsForCVEGQL(ctx context.Context, config searchConfig, username, password, imageName, cveID string) (*fixedTags, error) + getBaseImageListGQL(ctx context.Context, config searchConfig, username, password string, + baseImage string) (*imageListStructForBaseImagesGQL, error) getAllImages(ctx context.Context, config searchConfig, username, password string, channel chan stringResult, wtgrp *sync.WaitGroup) @@ -59,6 +61,32 @@ func NewSearchService() SearchService { return searchService{} } +func (service searchService) getBaseImageListGQL(ctx context.Context, config searchConfig, username, password string, + baseImage string, +) (*imageListStructForBaseImagesGQL, error) { + query := fmt.Sprintf(` + { + BaseImageList(image:"%s"){ + RepoName, + Tag, + Digest, + ConfigDigest, + LastUpdated, + IsSigned, + Size + } + }`, baseImage) + + result := &imageListStructForBaseImagesGQL{} + err := service.makeGraphQLQuery(ctx, config, username, password, query, result) + + if errResult := checkResultGraphQLQuery(ctx, err, result.Errors); errResult != nil { + return nil, errResult + } + + return result, nil +} + func (service searchService) getImagesGQL(ctx context.Context, config searchConfig, username, password string, imageName string, ) (*imageListStructGQL, error) { @@ -803,6 +831,13 @@ type imageListStructForDigestGQL struct { } `json:"data"` } +type imageListStructForBaseImagesGQL struct { + Errors []errorGraphQL `json:"errors"` + Data struct { + ImageList []imageStruct `json:"BaseImageList"` // nolint:tagliatelle + } `json:"data"` +} + type imagesForDigest struct { Errors []errorGraphQL `json:"errors"` Data struct { diff --git a/pkg/extensions/search/common/common_test.go b/pkg/extensions/search/common/common_test.go index a59c7eba..db3837ca 100644 --- a/pkg/extensions/search/common/common_test.go +++ b/pkg/extensions/search/common/common_test.go @@ -14,6 +14,7 @@ import ( "os/exec" "path" "strconv" + "strings" "testing" "time" @@ -708,7 +709,7 @@ func TestExpandedRepoInfo(t *testing.T) { So(err, ShouldBeNil) So(responseStruct.ExpandedRepoInfo.RepoInfo.Summary, ShouldNotBeEmpty) So(responseStruct.ExpandedRepoInfo.RepoInfo.Summary.Name, ShouldEqual, "test1") - So(len(responseStruct.ExpandedRepoInfo.RepoInfo.Images), ShouldEqual, 2) + So(len(responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries), ShouldEqual, 2) }) Convey("Test expanded repo info", t, func() { @@ -791,10 +792,10 @@ func TestExpandedRepoInfo(t *testing.T) { err = json.Unmarshal(resp.Body(), responseStruct) So(err, ShouldBeNil) - So(len(responseStruct.ExpandedRepoInfo.RepoInfo.Images), ShouldNotEqual, 0) - So(len(responseStruct.ExpandedRepoInfo.RepoInfo.Images[0].Layers), ShouldNotEqual, 0) + So(len(responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries), ShouldNotEqual, 0) + So(len(responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries[0].Layers), ShouldNotEqual, 0) found := false - for _, m := range responseStruct.ExpandedRepoInfo.RepoInfo.Images { + for _, m := range responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries { if m.Digest == "63a795ca90aa6e7cca60941e826810a4cd0a2e73ea02bf458241df2a5c973e29" { found = true So(m.IsSigned, ShouldEqual, false) @@ -812,10 +813,10 @@ func TestExpandedRepoInfo(t *testing.T) { err = json.Unmarshal(resp.Body(), responseStruct) So(err, ShouldBeNil) - So(len(responseStruct.ExpandedRepoInfo.RepoInfo.Images), ShouldNotEqual, 0) - So(len(responseStruct.ExpandedRepoInfo.RepoInfo.Images[0].Layers), ShouldNotEqual, 0) + So(len(responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries), ShouldNotEqual, 0) + So(len(responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries[0].Layers), ShouldNotEqual, 0) found = false - for _, m := range responseStruct.ExpandedRepoInfo.RepoInfo.Images { + for _, m := range responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries { if m.Digest == "63a795ca90aa6e7cca60941e826810a4cd0a2e73ea02bf458241df2a5c973e29" { found = true So(m.IsSigned, ShouldEqual, true) @@ -838,10 +839,10 @@ func TestExpandedRepoInfo(t *testing.T) { err = json.Unmarshal(resp.Body(), responseStruct) So(err, ShouldBeNil) - So(len(responseStruct.ExpandedRepoInfo.RepoInfo.Images), ShouldNotEqual, 0) - So(len(responseStruct.ExpandedRepoInfo.RepoInfo.Images[0].Layers), ShouldNotEqual, 0) + So(len(responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries), ShouldNotEqual, 0) + So(len(responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries[0].Layers), ShouldNotEqual, 0) found = false - for _, m := range responseStruct.ExpandedRepoInfo.RepoInfo.Images { + for _, m := range responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries { if m.Digest == "2bacca16b9df395fc855c14ccf50b12b58d35d468b8e7f25758aff90f89bf396" { found = true So(m.IsSigned, ShouldEqual, false) @@ -859,10 +860,10 @@ func TestExpandedRepoInfo(t *testing.T) { err = json.Unmarshal(resp.Body(), responseStruct) So(err, ShouldBeNil) - So(len(responseStruct.ExpandedRepoInfo.RepoInfo.Images), ShouldNotEqual, 0) - So(len(responseStruct.ExpandedRepoInfo.RepoInfo.Images[0].Layers), ShouldNotEqual, 0) + So(len(responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries), ShouldNotEqual, 0) + So(len(responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries[0].Layers), ShouldNotEqual, 0) found = false - for _, m := range responseStruct.ExpandedRepoInfo.RepoInfo.Images { + for _, m := range responseStruct.ExpandedRepoInfo.RepoInfo.ImageSummaries { if m.Digest == "2bacca16b9df395fc855c14ccf50b12b58d35d468b8e7f25758aff90f89bf396" { found = true So(m.IsSigned, ShouldEqual, true) @@ -1005,6 +1006,550 @@ func TestUtilsMethod(t *testing.T) { }) } +func TestGetImageManifest(t *testing.T) { + Convey("Test nonexistent image", t, func() { + mockImageStore := mocks.MockedImageStore{} + + storeController := storage.StoreController{ + DefaultStore: mockImageStore, + } + olu := common.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + + _, err := olu.GetImageManifest("nonexistent-repo", "latest") + So(err, ShouldNotBeNil) + }) + + Convey("Test nonexistent image", t, func() { + mockImageStore := mocks.MockedImageStore{ + GetImageManifestFn: func(repo string, reference string) ([]byte, string, string, error) { + return []byte{}, "", "", ErrTestError + }, + } + + storeController := storage.StoreController{ + DefaultStore: mockImageStore, + } + olu := common.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + + _, err := olu.GetImageManifest("test-repo", "latest") + So(err, ShouldNotBeNil) + }) +} + +func TestBaseImageList(t *testing.T) { + subpath := "/a" + + err := testSetup(t, subpath) + if err != nil { + panic(err) + } + + port := GetFreePort() + baseURL := GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + conf.Storage.RootDirectory = rootDir + conf.Storage.SubPaths = make(map[string]config.StorageConfig) + conf.Storage.SubPaths[subpath] = config.StorageConfig{RootDirectory: subRootDir} + defaultVal := true + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{Enable: &defaultVal}, + } + + conf.Extensions.Search.CVE = nil + + ctlr := api.NewController(conf) + + go func() { + // this blocks + if err := ctlr.Run(context.Background()); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(baseURL) + if err == nil { + break + } + + time.Sleep(100 * time.Millisecond) + } + + // shut down server + + defer func() { + ctx := context.Background() + _ = ctlr.Server.Shutdown(ctx) + }() + + Convey("Test base image list for image working", t, func() { + // create test images + config := ispec.Image{ + Architecture: "amd64", + OS: "linux", + RootFS: ispec.RootFS{ + Type: "layers", + DiffIDs: []digest.Digest{}, + }, + Author: "ZotUser", + } + + configBlob, err := json.Marshal(config) + So(err, ShouldBeNil) + + configDigest := digest.FromBytes(configBlob) + + layers := [][]byte{ + {10, 11, 10, 11}, + {11, 11, 11, 11}, + {10, 10, 10, 11}, + {10, 10, 10, 10}, + } + + manifest := ispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: configDigest, + Size: int64(len(configBlob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[0]), + Size: int64(len(layers[0])), + }, + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[1]), + Size: int64(len(layers[1])), + }, + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[2]), + Size: int64(len(layers[2])), + }, + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[3]), + Size: int64(len(layers[3])), + }, + }, + } + + repoName := "test-repo" + + err = UploadImage( + Image{ + Manifest: manifest, + Config: config, + Layers: layers, + Tag: "latest", + }, + baseURL, + repoName, + ) + So(err, ShouldBeNil) + + // create image with the same layers + manifest = ispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: configDigest, + Size: int64(len(configBlob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[0]), + Size: int64(len(layers[0])), + }, + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[1]), + Size: int64(len(layers[1])), + }, + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[2]), + Size: int64(len(layers[2])), + }, + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[3]), + Size: int64(len(layers[3])), + }, + }, + } + + repoName = "same-layers" + + err = UploadImage( + Image{ + Manifest: manifest, + Config: config, + Layers: layers, + Tag: "latest", + }, + baseURL, + repoName, + ) + So(err, ShouldBeNil) + + // create image with less layers than the given image, but which are in the given image + layers = [][]byte{ + {10, 11, 10, 11}, + {10, 10, 10, 11}, + } + + manifest = ispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: configDigest, + Size: int64(len(configBlob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[0]), + Size: int64(len(layers[0])), + }, + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[1]), + Size: int64(len(layers[1])), + }, + }, + } + + repoName = "less-layers" + + err = UploadImage( + Image{ + Manifest: manifest, + Config: config, + Layers: layers, + Tag: "latest", + }, + baseURL, + repoName, + ) + So(err, ShouldBeNil) + + // create image with less layers than the given image, but one layer isn't in the given image + layers = [][]byte{ + {10, 11, 10, 11}, + {11, 10, 10, 11}, + } + + manifest = ispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: configDigest, + Size: int64(len(configBlob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[0]), + Size: int64(len(layers[0])), + }, + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[1]), + Size: int64(len(layers[1])), + }, + }, + } + + repoName = "less-layers-false" + + err = UploadImage( + Image{ + Manifest: manifest, + Config: config, + Layers: layers, + Tag: "latest", + }, + baseURL, + repoName, + ) + So(err, ShouldBeNil) + + // create image with more layers than the original + layers = [][]byte{ + {10, 11, 10, 11}, + {11, 11, 11, 11}, + {10, 10, 10, 10}, + {10, 10, 10, 11}, + {11, 11, 10, 10}, + } + + manifest = ispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: configDigest, + Size: int64(len(configBlob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[0]), + Size: int64(len(layers[0])), + }, + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[1]), + Size: int64(len(layers[1])), + }, + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[2]), + Size: int64(len(layers[2])), + }, + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[3]), + Size: int64(len(layers[3])), + }, + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[4]), + Size: int64(len(layers[4])), + }, + }, + } + + repoName = "more-layers" + + err = UploadImage( + Image{ + Manifest: manifest, + Config: config, + Layers: layers, + Tag: "latest", + }, + baseURL, + repoName, + ) + So(err, ShouldBeNil) + + // create image with no shared layers with the given image + layers = [][]byte{ + {12, 12, 12, 12}, + {12, 10, 10, 12}, + } + + manifest = ispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: configDigest, + Size: int64(len(configBlob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[0]), + Size: int64(len(layers[0])), + }, + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest.FromBytes(layers[1]), + Size: int64(len(layers[1])), + }, + }, + } + + repoName = "diff-layers" + + err = UploadImage( + Image{ + Manifest: manifest, + Config: config, + Layers: layers, + Tag: "latest", + }, + baseURL, + repoName, + ) + So(err, ShouldBeNil) + + query := ` + { + BaseImageList(image:"test-repo"){ + RepoName, + Tag, + Digest, + ConfigDigest, + LastUpdated, + IsSigned, + Size + } + }` + + resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query)) + So(resp, ShouldNotBeNil) + So(strings.Contains(string(resp.Body()), "same-layers"), ShouldBeTrue) + So(strings.Contains(string(resp.Body()), "less-layers"), ShouldBeTrue) + So(strings.Contains(string(resp.Body()), "less-layers-false"), ShouldBeFalse) + So(strings.Contains(string(resp.Body()), "more-layers"), ShouldBeFalse) + So(strings.Contains(string(resp.Body()), "diff-layers"), ShouldBeFalse) + So(strings.Contains(string(resp.Body()), "test-repo"), ShouldBeTrue) // should not list given image + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + }) + + Convey("Nonexistent repository", t, func() { + query := ` + { + BaseImageList(image:"nonexistent-image"){ + RepoName, + Tag, + Digest, + ConfigDigest, + LastUpdated, + IsSigned, + Size + } + }` + + resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query)) + So(strings.Contains(string(resp.Body()), "repository: not found"), ShouldBeTrue) + So(err, ShouldBeNil) + }) + + Convey("Failed to get manifest", t, func() { + err := os.Mkdir(path.Join(rootDir, "fail-image"), 0o000) + So(err, ShouldBeNil) + + query := ` + { + BaseImageList(image:"fail-image"){ + RepoName, + Tag, + Digest, + ConfigDigest, + LastUpdated, + IsSigned, + Size + } + }` + + resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query)) + So(strings.Contains(string(resp.Body()), "permission denied"), ShouldBeTrue) + So(err, ShouldBeNil) + }) +} + +func TestBaseImageListNoRepos(t *testing.T) { + Convey("No repositories found", t, func() { + port := GetFreePort() + baseURL := GetBaseURL(port) + + conf := config.New() + conf.HTTP.Port = port + conf.Storage.RootDirectory = t.TempDir() + defaultVal := true + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{Enable: &defaultVal}, + } + + conf.Extensions.Search.CVE = nil + + ctlr := api.NewController(conf) + + go func() { + // this blocks + if err := ctlr.Run(context.Background()); err != nil { + return + } + }() + + // wait till ready + for { + _, err := resty.R().Get(baseURL) + if err == nil { + break + } + + time.Sleep(100 * time.Millisecond) + } + + // shut down server + + defer func() { + ctx := context.Background() + _ = ctlr.Server.Shutdown(ctx) + }() + + query := ` + { + BaseImageList(image:"test-image"){ + RepoName, + Tag, + Digest, + ConfigDigest, + LastUpdated, + IsSigned, + Size + } + }` + + resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query)) + So(strings.Contains(string(resp.Body()), "{\"data\":{\"BaseImageList\":[]}}"), ShouldBeTrue) + So(err, ShouldBeNil) + }) +} + +func TestGetRepositories(t *testing.T) { + Convey("Test getting the repositories list", t, func() { + mockImageStore := mocks.MockedImageStore{ + GetRepositoriesFn: func() ([]string, error) { + return []string{}, ErrTestError + }, + } + + storeController := storage.StoreController{ + DefaultStore: mockImageStore, + SubStore: map[string]storage.ImageStore{"test": mockImageStore}, + } + olu := common.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + + repoList, err := olu.GetRepositories() + So(repoList, ShouldBeEmpty) + So(err, ShouldNotBeNil) + + storeController = storage.StoreController{ + DefaultStore: mocks.MockedImageStore{}, + SubStore: map[string]storage.ImageStore{"test": mockImageStore}, + } + olu = common.NewBaseOciLayoutUtils(storeController, log.NewLogger("debug", "")) + + repoList, err = olu.GetRepositories() + So(repoList, ShouldBeEmpty) + So(err, ShouldNotBeNil) + }) +} + func TestGlobalSearch(t *testing.T) { Convey("Test utils", t, func() { subpath := "/a" diff --git a/pkg/extensions/search/common/oci_layout.go b/pkg/extensions/search/common/oci_layout.go index 4c330103..82015c10 100644 --- a/pkg/extensions/search/common/oci_layout.go +++ b/pkg/extensions/search/common/oci_layout.go @@ -33,6 +33,7 @@ type OciLayoutUtils interface { GetExpandedRepoInfo(name string) (RepoInfo, error) GetImageConfigInfo(repo string, manifestDigest godigest.Digest) (ispec.Image, error) CheckManifestSignature(name string, digest godigest.Digest) bool + GetRepositories() ([]string, error) } // OciLayoutInfo ... @@ -42,15 +43,8 @@ type BaseOciLayoutUtils struct { } type RepoInfo struct { - Summary RepoSummary - Images []Image `json:"images"` -} - -type Image struct { - Tag string `json:"tag"` - Digest string `json:"digest"` - IsSigned bool `json:"isSigned"` - Layers []Layer `json:"layers"` + Summary RepoSummary + ImageSummaries []ImageSummary `json:"images"` } type RepoSummary struct { @@ -81,6 +75,7 @@ type ImageSummary struct { Title string `json:"title"` Source string `json:"source"` Documentation string `json:"documentation"` + Layers []Layer `json:"layers"` } type OsArch struct { @@ -98,6 +93,49 @@ func NewBaseOciLayoutUtils(storeController storage.StoreController, log log.Logg return &BaseOciLayoutUtils{Log: log, StoreController: storeController} } +func (olu BaseOciLayoutUtils) GetImageManifest(repo string, reference string) (ispec.Manifest, error) { + imageStore := olu.StoreController.GetImageStore(repo) + + if reference == "" { + reference = "latest" + } + + buf, _, _, err := imageStore.GetImageManifest(repo, reference) + if err != nil { + return ispec.Manifest{}, err + } + + var manifest ispec.Manifest + + err = json.Unmarshal(buf, &manifest) + if err != nil { + return ispec.Manifest{}, err + } + + return manifest, nil +} + +func (olu BaseOciLayoutUtils) GetRepositories() ([]string, error) { + defaultStore := olu.StoreController.DefaultStore + substores := olu.StoreController.SubStore + + repoList, err := defaultStore.GetRepositories() + if err != nil { + return []string{}, err + } + + for _, sub := range substores { + repoListForSubstore, err := sub.GetRepositories() + if err != nil { + return []string{}, err + } + + repoList = append(repoList, repoListForSubstore...) + } + + return repoList, nil +} + // Below method will return image path including root dir, root dir is determined by splitting. func (olu BaseOciLayoutUtils) GetImageManifests(image string) ([]ispec.Descriptor, error) { imageStore := olu.StoreController.GetImageStore(image) @@ -374,7 +412,7 @@ func (olu BaseOciLayoutUtils) GetExpandedRepoInfo(name string) (RepoInfo, error) // made up of all manifests, configs and image layers repoSize := int64(0) - manifests := make([]Image, 0) + imageSummaries := make([]ImageSummary, 0) manifestList, err := olu.GetImageManifests(name) if err != nil { @@ -398,12 +436,6 @@ func (olu BaseOciLayoutUtils) GetExpandedRepoInfo(name string) (RepoInfo, error) for _, man := range manifestList { imageLayersSize := int64(0) - manifestInfo := Image{} - - manifestInfo.Digest = man.Digest.Encoded() - - manifestInfo.IsSigned = false - tag, ok := man.Annotations[ispec.AnnotationRefName] if !ok { olu.Log.Info().Msgf("skipping manifest with digest %s because it doesn't have a tag", string(man.Digest)) @@ -411,8 +443,6 @@ func (olu BaseOciLayoutUtils) GetExpandedRepoInfo(name string) (RepoInfo, error) continue } - manifestInfo.Tag = tag - manifest, err := olu.GetImageBlobManifest(name, man.Digest) if err != nil { olu.Log.Error().Err(err).Msg("error getting image manifest blob") @@ -421,7 +451,6 @@ func (olu BaseOciLayoutUtils) GetExpandedRepoInfo(name string) (RepoInfo, error) } isSigned := olu.CheckManifestSignature(name, man.Digest) - manifestInfo.IsSigned = isSigned manifestSize := olu.GetImageManifestSize(name, man.Digest) olu.Log.Debug().Msg(fmt.Sprintf("%v", man.Digest)) @@ -463,10 +492,6 @@ func (olu BaseOciLayoutUtils) GetExpandedRepoInfo(name string) (RepoInfo, error) imageSize := imageLayersSize + manifestSize + configSize - manifestInfo.Layers = layers - - manifests = append(manifests, manifestInfo) - // get image info from manifest annotation, if not found get from image config labels. annotations := GetAnnotations(manifest.Annotations, imageConfigInfo.Config.Labels) @@ -495,14 +520,17 @@ func (olu BaseOciLayoutUtils) GetExpandedRepoInfo(name string) (RepoInfo, error) Licenses: annotations.Licenses, Labels: annotations.Labels, Source: annotations.Source, + Layers: layers, } + imageSummaries = append(imageSummaries, imageSummary) + if man.Digest.String() == lastUpdatedTag.Digest { lastUpdatedImageSummary = imageSummary } } - repo.Images = manifests + repo.ImageSummaries = imageSummaries for blob := range repoBlob2Size { repoSize += repoBlob2Size[blob] @@ -531,9 +559,7 @@ func GetImageDirAndTag(imageName string) (string, string) { var imageTag string if strings.Contains(imageName, ":") { - splitImageName := strings.Split(imageName, ":") - imageDir = splitImageName[0] - imageTag = splitImageName[1] + imageDir, imageTag, _ = strings.Cut(imageName, ":") } else { imageDir = imageName } diff --git a/pkg/extensions/search/gql_generated/generated.go b/pkg/extensions/search/gql_generated/generated.go index cd63fa9e..2ea67b3e 100644 --- a/pkg/extensions/search/gql_generated/generated.go +++ b/pkg/extensions/search/gql_generated/generated.go @@ -115,6 +115,7 @@ type ComplexityRoot struct { } Query struct { + BaseImageList func(childComplexity int, image string) int CVEListForImage func(childComplexity int, image string) int ExpandedRepoInfo func(childComplexity int, repo string) int GlobalSearch func(childComplexity int, query string) int @@ -153,6 +154,7 @@ type QueryResolver interface { ImageList(ctx context.Context, repo string) ([]*ImageSummary, error) ExpandedRepoInfo(ctx context.Context, repo string) (*RepoInfo, error) GlobalSearch(ctx context.Context, query string) (*GlobalSearchResult, error) + BaseImageList(ctx context.Context, image string) ([]*ImageSummary, error) } type executableSchema struct { @@ -478,6 +480,18 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.PackageInfo.Name(childComplexity), true + case "Query.BaseImageList": + if e.complexity.Query.BaseImageList == nil { + break + } + + args, err := ec.field_Query_BaseImageList_args(context.TODO(), rawArgs) + if err != nil { + return 0, false + } + + return e.complexity.Query.BaseImageList(childComplexity, args["image"].(string)), true + case "Query.CVEListForImage": if e.complexity.Query.CVEListForImage == nil { break @@ -824,8 +838,8 @@ type Query { ImageList(repo: String!): [ImageSummary!] ExpandedRepoInfo(repo: String!): RepoInfo! GlobalSearch(query: String!): GlobalSearchResult! -} -`, BuiltIn: false}, + BaseImageList(image: String!): [ImageSummary!] +}`, BuiltIn: false}, } var parsedSchema = gqlparser.MustLoadSchema(sources...) @@ -833,6 +847,21 @@ var parsedSchema = gqlparser.MustLoadSchema(sources...) // region ***************************** args.gotpl ***************************** +func (ec *executionContext) field_Query_BaseImageList_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { + var err error + args := map[string]interface{}{} + var arg0 string + if tmp, ok := rawArgs["image"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("image")) + arg0, err = ec.unmarshalNString2string(ctx, tmp) + if err != nil { + return nil, err + } + } + args["image"] = arg0 + return args, nil +} + func (ec *executionContext) field_Query_CVEListForImage_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { var err error args := map[string]interface{}{} @@ -3553,6 +3582,98 @@ func (ec *executionContext) fieldContext_Query_GlobalSearch(ctx context.Context, return fc, nil } +func (ec *executionContext) _Query_BaseImageList(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Query_BaseImageList(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return ec.resolvers.Query().BaseImageList(rctx, fc.Args["image"].(string)) + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + return graphql.Null + } + res := resTmp.([]*ImageSummary) + fc.Result = res + return ec.marshalOImageSummary2ᚕᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐImageSummaryᚄ(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_Query_BaseImageList(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "Query", + Field: field, + IsMethod: true, + IsResolver: true, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "RepoName": + return ec.fieldContext_ImageSummary_RepoName(ctx, field) + case "Tag": + return ec.fieldContext_ImageSummary_Tag(ctx, field) + case "Digest": + return ec.fieldContext_ImageSummary_Digest(ctx, field) + case "ConfigDigest": + return ec.fieldContext_ImageSummary_ConfigDigest(ctx, field) + case "LastUpdated": + return ec.fieldContext_ImageSummary_LastUpdated(ctx, field) + case "IsSigned": + return ec.fieldContext_ImageSummary_IsSigned(ctx, field) + case "Size": + return ec.fieldContext_ImageSummary_Size(ctx, field) + case "Platform": + return ec.fieldContext_ImageSummary_Platform(ctx, field) + case "Vendor": + return ec.fieldContext_ImageSummary_Vendor(ctx, field) + case "Score": + return ec.fieldContext_ImageSummary_Score(ctx, field) + case "DownloadCount": + return ec.fieldContext_ImageSummary_DownloadCount(ctx, field) + case "Layers": + return ec.fieldContext_ImageSummary_Layers(ctx, field) + case "Description": + return ec.fieldContext_ImageSummary_Description(ctx, field) + case "Licenses": + return ec.fieldContext_ImageSummary_Licenses(ctx, field) + case "Labels": + return ec.fieldContext_ImageSummary_Labels(ctx, field) + case "Title": + return ec.fieldContext_ImageSummary_Title(ctx, field) + case "Source": + return ec.fieldContext_ImageSummary_Source(ctx, field) + case "Documentation": + return ec.fieldContext_ImageSummary_Documentation(ctx, field) + case "History": + return ec.fieldContext_ImageSummary_History(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type ImageSummary", field.Name) + }, + } + defer func() { + if r := recover(); r != nil { + err = ec.Recover(ctx, r) + ec.Error(ctx, err) + } + }() + ctx = graphql.WithFieldContext(ctx, fc) + if fc.Args, err = ec.field_Query_BaseImageList_args(ctx, field.ArgumentMap(ec.Variables)); err != nil { + ec.Error(ctx, err) + return + } + return fc, nil +} + func (ec *executionContext) _Query___type(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { fc, err := ec.fieldContext_Query___type(ctx, field) if err != nil { @@ -6616,6 +6737,26 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc) } + out.Concurrently(i, func() graphql.Marshaler { + return rrm(innerCtx) + }) + case "BaseImageList": + field := field + + innerFunc := func(ctx context.Context) (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Query_BaseImageList(ctx, field) + return res + } + + rrm := func(ctx context.Context) graphql.Marshaler { + return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc) + } + out.Concurrently(i, func() graphql.Marshaler { return rrm(innerCtx) }) diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index 7e4723f5..0bf170a4 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -146,7 +146,7 @@ func TestGlobalSearch(t *testing.T) { mockOlum := mocks.OciLayoutUtilsMock{ GetExpandedRepoInfoFn: func(name string) (common.RepoInfo, error) { return common.RepoInfo{ - Images: []common.Image{ + ImageSummaries: []common.ImageSummary{ { Tag: "latest", Layers: []common.Layer{ diff --git a/pkg/extensions/search/schema.graphql b/pkg/extensions/search/schema.graphql index 91dac0c1..e909607c 100644 --- a/pkg/extensions/search/schema.graphql +++ b/pkg/extensions/search/schema.graphql @@ -117,4 +117,5 @@ type Query { ImageList(repo: String!): [ImageSummary!] ExpandedRepoInfo(repo: String!): RepoInfo! GlobalSearch(query: String!): GlobalSearchResult! -} + BaseImageList(image: String!): [ImageSummary!] +} \ No newline at end of file diff --git a/pkg/extensions/search/schema.resolvers.go b/pkg/extensions/search/schema.resolvers.go index 2d7be997..07a961e1 100644 --- a/pkg/extensions/search/schema.resolvers.go +++ b/pkg/extensions/search/schema.resolvers.go @@ -435,12 +435,13 @@ func (r *queryResolver) ExpandedRepoInfo(ctx context.Context, repo string) (*gql score := -1 // score not relevant for this query summary.Score = &score - for _, image := range origRepoInfo.Images { + for _, image := range origRepoInfo.ImageSummaries { tag := image.Tag digest := image.Digest isSigned := image.IsSigned + size := image.Size - imageSummary := &gql_generated.ImageSummary{Tag: &tag, Digest: &digest, IsSigned: &isSigned} + imageSummary := &gql_generated.ImageSummary{Tag: &tag, Digest: &digest, IsSigned: &isSigned, RepoName: &repo} layers := make([]*gql_generated.LayerSummary, 0) @@ -454,7 +455,7 @@ func (r *queryResolver) ExpandedRepoInfo(ctx context.Context, repo string) (*gql } imageSummary.Layers = layers - + imageSummary.Size = &size images = append(images, imageSummary) } @@ -500,6 +501,84 @@ func (r *queryResolver) GlobalSearch(ctx context.Context, query string) (*gql_ge }, nil } +// BaseImageList is the resolver for the BaseImageList field. +func (r *queryResolver) BaseImageList(ctx context.Context, image string) ([]*gql_generated.ImageSummary, error) { + layoutUtils := common.NewBaseOciLayoutUtils(r.storeController, r.log) + imageList := make([]*gql_generated.ImageSummary, 0) + + repoList, err := layoutUtils.GetRepositories() + if err != nil { + r.log.Error().Err(err).Msg("unable to get repositories list") + + return nil, err + } + + if len(repoList) == 0 { + r.log.Info().Msg("no repositories found") + + return imageList, nil + } + + imageDir, imageTag := common.GetImageDirAndTag(image) + + imageManifest, err := layoutUtils.GetImageManifest(imageDir, imageTag) + if err != nil { + r.log.Info().Str("image", image).Msg("image not found") + + return imageList, err + } + + imageLayers := imageManifest.Layers + + // This logic may not scale well in the future as we need to read all the + // manifest files from the disk when the call is made, we should improve in a future PR + for _, repo := range repoList { + repoInfo, err := r.ExpandedRepoInfo(ctx, repo) + if err != nil { + r.log.Error().Err(err).Msg("unable to get image list") + + return nil, err + } + + imageSummaries := repoInfo.Images + + var addImageToList bool + // verify every image + for _, imageSummary := range imageSummaries { + if imageTag == *imageSummary.Tag && imageDir == repo { + continue + } + + addImageToList = true + layers := imageSummary.Layers + + for _, l := range layers { + foundLayer := false + + for _, k := range imageLayers { + if *l.Digest == k.Digest.Encoded() { + foundLayer = true + + break + } + } + + if !foundLayer { + addImageToList = false + + break + } + } + + if addImageToList { + imageList = append(imageList, imageSummary) + } + } + } + + return imageList, nil +} + // Query returns gql_generated.QueryResolver implementation. func (r *Resolver) Query() gql_generated.QueryResolver { return &queryResolver{r} } diff --git a/pkg/test/mocks/oci_mock.go b/pkg/test/mocks/oci_mock.go index 44d6bcf4..8b09b135 100644 --- a/pkg/test/mocks/oci_mock.go +++ b/pkg/test/mocks/oci_mock.go @@ -24,6 +24,15 @@ type OciLayoutUtilsMock struct { GetExpandedRepoInfoFn func(name string) (common.RepoInfo, error) GetImageConfigInfoFn func(repo string, manifestDigest godigest.Digest) (ispec.Image, error) CheckManifestSignatureFn func(name string, digest godigest.Digest) bool + GetRepositoriesFn func() ([]string, error) +} + +func (olum OciLayoutUtilsMock) GetRepositories() ([]string, error) { + if olum.GetImageManifestsFn != nil { + return olum.GetRepositoriesFn() + } + + return []string{}, nil } func (olum OciLayoutUtilsMock) GetImageManifests(image string) ([]ispec.Descriptor, error) {