From 620287c7a438a492fb78ad4ce2ea1ebc1ed73a38 Mon Sep 17 00:00:00 2001 From: LaurentiuNiculae Date: Thu, 22 Jun 2023 20:43:01 +0300 Subject: [PATCH] feat(cli): add referrers and search commands to cli (#1497) * feat(cli): add referrers command to cli Signed-off-by: Laurentiu Niculae * feat(cli): add global search command Signed-off-by: Laurentiu Niculae * feat(cli): fix comments Signed-off-by: Laurentiu Niculae --------- Signed-off-by: Laurentiu Niculae --- errors/errors.go | 1 + pkg/api/controller_test.go | 8 +- pkg/cli/cli.go | 1 + pkg/cli/image_cmd.go | 3 +- pkg/cli/image_cmd_test.go | 47 ++ pkg/cli/repo_cmd.go | 4 +- pkg/cli/search_cmd.go | 148 +++++++ pkg/cli/search_cmd_referrers_test.go | 619 +++++++++++++++++++++++++++ pkg/cli/search_cmd_test.go | 433 +++++++++++++++++++ pkg/cli/searcher.go | 304 ++++++++++++- pkg/cli/service.go | 355 ++++++++++++++- pkg/common/model.go | 19 +- pkg/common/oci.go | 29 ++ pkg/meta/repodb/storage_parsing.go | 84 ++-- pkg/test/common.go | 15 +- pkg/test/common_test.go | 4 +- 16 files changed, 1993 insertions(+), 81 deletions(-) create mode 100644 pkg/cli/search_cmd.go create mode 100644 pkg/cli/search_cmd_referrers_test.go create mode 100644 pkg/cli/search_cmd_test.go diff --git a/errors/errors.go b/errors/errors.go index b982006a..21d1ca36 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -32,6 +32,7 @@ var ( ErrInvalidArgs = errors.New("cli: Invalid Arguments") ErrInvalidFlagsCombination = errors.New("cli: Invalid combination of flags") ErrInvalidURL = errors.New("cli: invalid URL format") + ErrExtensionNotEnabled = errors.New("cli: functionality is not built in current version") ErrUnauthorizedAccess = errors.New("auth: unauthorized access. check credentials") ErrCannotResetConfigKey = errors.New("cli: cannot reset given config key") ErrConfigNotFound = errors.New("cli: config with the given name does not exist") diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 35135e68..a0410beb 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -5226,7 +5226,7 @@ func TestManifestImageIndex(t *testing.T) { Layers: layers, Manifest: manifest, }, baseURL, repoName) - So(err, ShouldNotBeNil) + So(err, ShouldBeNil) content, err = json.Marshal(manifest) So(err, ShouldBeNil) @@ -5251,7 +5251,7 @@ func TestManifestImageIndex(t *testing.T) { Layers: layers, Manifest: manifest, }, baseURL, repoName) - So(err, ShouldNotBeNil) + So(err, ShouldBeNil) content, err = json.Marshal(manifest) So(err, ShouldBeNil) @@ -5307,7 +5307,7 @@ func TestManifestImageIndex(t *testing.T) { Layers: layers, Manifest: manifest, }, baseURL, repoName) - So(err, ShouldNotBeNil) + So(err, ShouldBeNil) content, err = json.Marshal(manifest) So(err, ShouldBeNil) digest = godigest.FromBytes(content) @@ -5480,7 +5480,7 @@ func TestManifestImageIndex(t *testing.T) { Layers: layers, Manifest: manifest, }, baseURL, repoName) - So(err, ShouldNotBeNil) + So(err, ShouldBeNil) content, err = json.Marshal(manifest) So(err, ShouldBeNil) digest = godigest.FromBytes(content) diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 6c2d8b1a..47e7dcf9 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -10,4 +10,5 @@ func enableCli(rootCmd *cobra.Command) { rootCmd.AddCommand(NewImageCommand(NewSearchService())) rootCmd.AddCommand(NewCveCommand(NewSearchService())) rootCmd.AddCommand(NewRepoCommand(NewSearchService())) + rootCmd.AddCommand(NewSearchCommand(NewSearchService())) } diff --git a/pkg/cli/image_cmd.go b/pkg/cli/image_cmd.go index ace48e02..8661733c 100644 --- a/pkg/cli/image_cmd.go +++ b/pkg/cli/image_cmd.go @@ -15,6 +15,7 @@ import ( zotErrors "zotregistry.io/zot/errors" ) +//nolint:dupl func NewImageCommand(searchService SearchService) *cobra.Command { searchImageParams := make(map[string]*string) @@ -70,7 +71,7 @@ func NewImageCommand(searchService SearchService) *cobra.Command { } spin := spinner.New(spinner.CharSets[39], spinnerDuration, spinner.WithWriter(cmd.ErrOrStderr())) - spin.Prefix = "Searching... " + spin.Prefix = prefix searchConfig := searchConfig{ params: searchImageParams, diff --git a/pkg/cli/image_cmd_test.go b/pkg/cli/image_cmd_test.go index 8ac21ff4..715a942b 100644 --- a/pkg/cli/image_cmd_test.go +++ b/pkg/cli/image_cmd_test.go @@ -1710,6 +1710,53 @@ func (service mockService) getRepos(ctx context.Context, config searchConfig, us channel <- stringResult{"", nil} } +func (service mockService) getReferrers(ctx context.Context, config searchConfig, username, password string, + repo, digest string, +) (referrersResult, error) { + return referrersResult{}, nil +} + +func (service mockService) globalSearchGQL(ctx context.Context, config searchConfig, username, password string, + query string, +) (*common.GlobalSearch, error) { + return &common.GlobalSearch{ + Images: []common.ImageSummary{ + { + RepoName: "repo", + MediaType: ispec.MediaTypeImageManifest, + Manifests: []common.ManifestSummary{ + { + Digest: godigest.FromString("str").String(), + Size: "100", + }, + }, + }, + }, + Repos: []common.RepoSummary{ + { + Name: "repo", + }, + }, + }, nil +} + +func (service mockService) getReferrersGQL(ctx context.Context, config searchConfig, username, password string, + repo, digest string, +) (*common.ReferrersResp, error) { + return &common.ReferrersResp{ + ReferrersResult: common.ReferrersResult{ + Referrers: []common.Referrer{ + { + MediaType: "MediaType", + ArtifactType: "ArtifactType", + Size: 100, + Digest: "Digest", + }, + }, + }, + }, nil +} + func (service mockService) getDerivedImageListGQL(ctx context.Context, config searchConfig, username, password string, derivedImage string, ) (*common.DerivedImageListResponse, error) { diff --git a/pkg/cli/repo_cmd.go b/pkg/cli/repo_cmd.go index 1c7d578a..784de164 100644 --- a/pkg/cli/repo_cmd.go +++ b/pkg/cli/repo_cmd.go @@ -13,6 +13,8 @@ import ( zotErrors "zotregistry.io/zot/errors" ) +const prefix = "Searching... " + func NewRepoCommand(searchService SearchService) *cobra.Command { var servURL, user, outputFormat string @@ -66,7 +68,7 @@ func NewRepoCommand(searchService SearchService) *cobra.Command { } spin := spinner.New(spinner.CharSets[39], spinnerDuration, spinner.WithWriter(cmd.ErrOrStderr())) - spin.Prefix = "Searching... " + spin.Prefix = prefix searchConfig := searchConfig{ searchService: searchService, diff --git a/pkg/cli/search_cmd.go b/pkg/cli/search_cmd.go new file mode 100644 index 00000000..f963537a --- /dev/null +++ b/pkg/cli/search_cmd.go @@ -0,0 +1,148 @@ +//go:build search +// +build search + +package cli + +import ( + "os" + "path" + + "github.com/briandowns/spinner" + "github.com/spf13/cobra" + + zotErrors "zotregistry.io/zot/errors" +) + +//nolint:dupl +func NewSearchCommand(searchService SearchService) *cobra.Command { + searchImageParams := make(map[string]*string) + + var servURL, user, outputFormat string + + var isSpinner, verifyTLS, verbose, debug bool + + imageCmd := &cobra.Command{ + Use: "search [config-name]", + Short: "Search images and their tags", + Long: `Search repos or images +Example: + # For repo search specify a substring of the repo name without the tag + zli search --query test/repo + + # For image search specify the full repo name followed by the tag or a prefix of the tag. + zli search --query test/repo:2.1. + `, + RunE: func(cmd *cobra.Command, args []string) error { + home, err := os.UserHomeDir() + if err != nil { + panic(err) + } + + configPath := path.Join(home + "/.zot") + if servURL == "" { + if len(args) > 0 { + urlFromConfig, err := getConfigValue(configPath, args[0], "url") + if err != nil { + cmd.SilenceUsage = true + + return err + } + + if urlFromConfig == "" { + return zotErrors.ErrNoURLProvided + } + + servURL = urlFromConfig + } else { + return zotErrors.ErrNoURLProvided + } + } + + if len(args) > 0 { + var err error + isSpinner, err = parseBooleanConfig(configPath, args[0], showspinnerConfig) + if err != nil { + cmd.SilenceUsage = true + + return err + } + + verifyTLS, err = parseBooleanConfig(configPath, args[0], verifyTLSConfig) + if err != nil { + cmd.SilenceUsage = true + + return err + } + } + + spin := spinner.New(spinner.CharSets[39], spinnerDuration, spinner.WithWriter(cmd.ErrOrStderr())) + spin.Prefix = prefix + + searchConfig := searchConfig{ + params: searchImageParams, + searchService: searchService, + servURL: &servURL, + user: &user, + outputFormat: &outputFormat, + verbose: &verbose, + debug: &debug, + spinner: spinnerState{spin, isSpinner}, + verifyTLS: &verifyTLS, + resultWriter: cmd.OutOrStdout(), + } + + err = globalSearch(searchConfig) + + if err != nil { + cmd.SilenceUsage = true + + return err + } + + return nil + }, + } + + setupSearchFlags(imageCmd, searchImageParams, &servURL, &user, &outputFormat, &verbose, &debug) + imageCmd.SetUsageTemplate(imageCmd.UsageTemplate() + usageFooter) + + return imageCmd +} + +func setupSearchFlags(imageCmd *cobra.Command, searchImageParams map[string]*string, + servURL, user, outputFormat *string, verbose *bool, debug *bool, +) { + searchImageParams["query"] = imageCmd.Flags().StringP("query", "q", "", + "Specify what repo or image(repo:tag) to be searched") + + searchImageParams["subject"] = imageCmd.Flags().StringP("subject", "s", "", "List all referrers for this subject") + + imageCmd.Flags().StringVar(servURL, "url", "", "Specify zot server URL if config-name is not mentioned") + imageCmd.Flags().StringVarP(user, "user", "u", "", `User Credentials of zot server in "username:password" format`) + imageCmd.Flags().StringVarP(outputFormat, "output", "o", "", "Specify output format [text/json/yaml]") + imageCmd.Flags().BoolVar(verbose, "verbose", false, "Show verbose output") + imageCmd.Flags().BoolVar(debug, "debug", false, "Show debug output") +} + +func globalSearch(searchConfig searchConfig) error { + var searchers []searcher + + if checkExtEndPoint(searchConfig) { + searchers = getGlobalSearchersGQL() + } else { + searchers = getGlobalSearchersREST() + } + + for _, searcher := range searchers { + found, err := searcher.search(searchConfig) + if found { + if err != nil { + return err + } + + return nil + } + } + + return zotErrors.ErrInvalidFlagsCombination +} diff --git a/pkg/cli/search_cmd_referrers_test.go b/pkg/cli/search_cmd_referrers_test.go new file mode 100644 index 00000000..a8fe2367 --- /dev/null +++ b/pkg/cli/search_cmd_referrers_test.go @@ -0,0 +1,619 @@ +//go:build search +// +build search + +package cli //nolint:testpackage + +import ( + "bytes" + "fmt" + "net/http" + "os" + "regexp" + "strings" + "testing" + + ispec "github.com/opencontainers/image-spec/specs-go/v1" + . "github.com/smartystreets/goconvey/convey" + + "zotregistry.io/zot/pkg/api" + "zotregistry.io/zot/pkg/api/config" + extconf "zotregistry.io/zot/pkg/extensions/config" + "zotregistry.io/zot/pkg/test" +) + +func ref[T any](input T) *T { + obj := input + + return &obj +} + +const ( + customArtTypeV1 = "custom.art.type.v1" + customArtTypeV2 = "custom.art.type.v2" + repoName = "repo" +) + +func TestReferrersSearchers(t *testing.T) { + refSearcherGQL := referrerSearcherGQL{} + refSearcher := referrerSearcher{} + + Convey("GQL Searcher", t, func() { + Convey("Bad parameters", func() { + ok, err := refSearcherGQL.search(searchConfig{params: map[string]*string{ + "badParam": ref("badParam"), + }}) + + So(err, ShouldBeNil) + So(ok, ShouldBeFalse) + }) + + Convey("GetRepoRefference fails", func() { + conf := searchConfig{ + params: map[string]*string{ + "subject": ref("bad-subject"), + }, + user: ref("test:pass"), + } + + ok, err := refSearcherGQL.search(conf) + + So(err, ShouldNotBeNil) + So(ok, ShouldBeTrue) + }) + + Convey("fetchImageDigest for tags fails", func() { + conf := searchConfig{ + params: map[string]*string{ + "subject": ref("repo:tag"), + }, + user: ref("test:pass"), + servURL: ref("127.0.0.1:8080"), + } + + ok, err := refSearcherGQL.search(conf) + + So(err, ShouldNotBeNil) + So(ok, ShouldBeTrue) + }) + + Convey("search service fails", func() { + port := test.GetFreePort() + + conf := searchConfig{ + params: map[string]*string{ + "subject": ref("repo:tag"), + }, + searchService: NewSearchService(), + user: ref("test:pass"), + servURL: ref("http://127.0.0.1:" + port), + verifyTLS: ref(false), + debug: ref(false), + verbose: ref(false), + } + + server := test.StartTestHTTPServer(test.HTTPRoutes{ + test.RouteHandler{ + Route: "/v2/{repo}/manifests/{ref}", + HandlerFunc: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }, + AllowedMethods: []string{"HEAD"}, + }, + }, port) + + defer server.Close() + + ok, err := refSearcherGQL.search(conf) + + So(err, ShouldNotBeNil) + So(ok, ShouldBeTrue) + }) + }) + + Convey("REST searcher", t, func() { + Convey("Bad parameters", func() { + ok, err := refSearcher.search(searchConfig{params: map[string]*string{ + "badParam": ref("badParam"), + }}) + + So(err, ShouldBeNil) + So(ok, ShouldBeFalse) + }) + + Convey("GetRepoRefference fails", func() { + conf := searchConfig{ + params: map[string]*string{ + "subject": ref("bad-subject"), + }, + user: ref("test:pass"), + } + + ok, err := refSearcher.search(conf) + + So(err, ShouldNotBeNil) + So(ok, ShouldBeTrue) + }) + + Convey("fetchImageDigest for tags fails", func() { + conf := searchConfig{ + params: map[string]*string{ + "subject": ref("repo:tag"), + }, + user: ref("test:pass"), + servURL: ref("127.0.0.1:1000"), + } + + ok, err := refSearcher.search(conf) + + So(err, ShouldNotBeNil) + So(ok, ShouldBeTrue) + }) + + Convey("search service fails", func() { + port := test.GetFreePort() + + conf := searchConfig{ + params: map[string]*string{ + "subject": ref("repo:tag"), + }, + searchService: NewSearchService(), + user: ref("test:pass"), + servURL: ref("http://127.0.0.1:" + port), + verifyTLS: ref(false), + debug: ref(false), + verbose: ref(false), + fixedFlag: ref(false), + } + + server := test.StartTestHTTPServer(test.HTTPRoutes{ + test.RouteHandler{ + Route: "/v2/{repo}/manifests/{ref}", + HandlerFunc: func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + }, + AllowedMethods: []string{"HEAD"}, + }, + }, port) + + defer server.Close() + + ok, err := refSearcher.search(conf) + + So(err, ShouldNotBeNil) + So(ok, ShouldBeTrue) + }) + }) +} + +func TestReferrerCLI(t *testing.T) { + Convey("Test GQL", t, func() { + rootDir := t.TempDir() + + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + conf.Storage.GC = false + defaultVal := true + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, + } + ctlr := api.NewController(conf) + ctlr.Config.Storage.RootDirectory = rootDir + cm := test.NewControllerManager(ctlr) + cm.StartAndWait(conf.HTTP.Port) + defer cm.StopServer() + + repo := repoName + image, err := test.GetRandomImage("tag") + So(err, ShouldBeNil) + imgDigest, err := image.Digest() + So(err, ShouldBeNil) + + err = test.UploadImage(image, baseURL, repo) + So(err, ShouldBeNil) + + // add referrers + ref1, err := test.GetImageWithSubject(imgDigest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + ref1.Reference = "" + + ref1Digest, err := ref1.Digest() + So(err, ShouldBeNil) + + ref2, err := test.GetImageWithSubject(imgDigest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + ref2.Reference = "" + ref2.Manifest.Config.MediaType = customArtTypeV1 + ref2Digest, err := ref2.Digest() + So(err, ShouldBeNil) + + ref3, err := test.GetImageWithSubject(imgDigest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + ref3.Manifest.ArtifactType = customArtTypeV2 + ref3.Manifest.Config = ispec.DescriptorEmptyJSON + ref3.Reference = "" + ref3Digest, err := ref3.Digest() + So(err, ShouldBeNil) + + err = test.UploadImage(ref1, baseURL, repo) + So(err, ShouldBeNil) + + err = test.UploadImage(ref2, baseURL, repo) + So(err, ShouldBeNil) + + err = test.UploadImage(ref3, baseURL, repo) + So(err, ShouldBeNil) + + args := []string{"reftest", "--subject", repo + "@" + imgDigest.String()} + + configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"reftest","url":"%s","showspinner":false}]}`, + baseURL)) + defer os.Remove(configPath) + + cmd := NewSearchCommand(new(searchService)) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err = cmd.Execute() + So(err, ShouldBeNil) + space := regexp.MustCompile(`\s+`) + str := strings.TrimSpace(space.ReplaceAllString(buff.String(), " ")) + So(str, ShouldContainSubstring, "ARTIFACT TYPE SIZE DIGEST") + So(str, ShouldContainSubstring, "application/vnd.oci.image.config.v1+json 557 B "+ref1Digest.String()) + So(str, ShouldContainSubstring, "custom.art.type.v1 535 B "+ref2Digest.String()) + So(str, ShouldContainSubstring, "custom.art.type.v2 598 B "+ref3Digest.String()) + + fmt.Println(buff.String()) + + os.Remove(configPath) + + args = []string{"reftest", "--subject", repo + ":" + "tag"} + + configPath = makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"reftest","url":"%s","showspinner":false}]}`, + baseURL)) + defer os.Remove(configPath) + + cmd = NewSearchCommand(new(searchService)) + + buff = &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err = cmd.Execute() + So(err, ShouldBeNil) + str = strings.TrimSpace(space.ReplaceAllString(buff.String(), " ")) + So(str, ShouldContainSubstring, "ARTIFACT TYPE SIZE DIGEST") + So(str, ShouldContainSubstring, "application/vnd.oci.image.config.v1+json 557 B "+ref1Digest.String()) + So(str, ShouldContainSubstring, "custom.art.type.v1 535 B "+ref2Digest.String()) + So(str, ShouldContainSubstring, "custom.art.type.v2 598 B "+ref3Digest.String()) + + fmt.Println(buff.String()) + }) + + Convey("Test REST", t, func() { + rootDir := t.TempDir() + + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + conf.Storage.GC = false + defaultVal := false + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, + } + ctlr := api.NewController(conf) + ctlr.Config.Storage.RootDirectory = rootDir + cm := test.NewControllerManager(ctlr) + cm.StartAndWait(conf.HTTP.Port) + defer cm.StopServer() + + repo := repoName + image, err := test.GetRandomImage("tag") + So(err, ShouldBeNil) + imgDigest, err := image.Digest() + So(err, ShouldBeNil) + + err = test.UploadImage(image, baseURL, repo) + So(err, ShouldBeNil) + + // add referrers + ref1, err := test.GetImageWithSubject(imgDigest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + ref1Digest, err := ref1.Digest() + So(err, ShouldBeNil) + + ref2, err := test.GetImageWithSubject(imgDigest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + ref2.Manifest.Config.MediaType = customArtTypeV1 + ref2Digest, err := ref2.Digest() + So(err, ShouldBeNil) + + ref3, err := test.GetImageWithSubject(imgDigest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + ref3.Manifest.ArtifactType = customArtTypeV2 + ref3.Manifest.Config = ispec.DescriptorEmptyJSON + + ref3Digest, err := ref3.Digest() + So(err, ShouldBeNil) + + ref1.Reference = "" + err = test.UploadImage(ref1, baseURL, repo) + So(err, ShouldBeNil) + + ref2.Reference = "" + err = test.UploadImage(ref2, baseURL, repo) + So(err, ShouldBeNil) + + ref3.Reference = "" + err = test.UploadImage(ref3, baseURL, repo) + So(err, ShouldBeNil) + + // get referrers by digest + args := []string{"reftest", "--subject", repo + "@" + imgDigest.String()} + + configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"reftest","url":"%s","showspinner":false}]}`, + baseURL)) + + cmd := NewSearchCommand(new(searchService)) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err = cmd.Execute() + So(err, ShouldBeNil) + space := regexp.MustCompile(`\s+`) + str := strings.TrimSpace(space.ReplaceAllString(buff.String(), " ")) + So(str, ShouldContainSubstring, "ARTIFACT TYPE SIZE DIGEST") + So(str, ShouldContainSubstring, "application/vnd.oci.image.config.v1+json 557 B "+ref1Digest.String()) + So(str, ShouldContainSubstring, "custom.art.type.v1 535 B "+ref2Digest.String()) + So(str, ShouldContainSubstring, "custom.art.type.v2 598 B "+ref3Digest.String()) + fmt.Println(buff.String()) + + os.Remove(configPath) + + args = []string{"reftest", "--subject", repo + ":" + "tag"} + + configPath = makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"reftest","url":"%s","showspinner":false}]}`, + baseURL)) + defer os.Remove(configPath) + + buff = &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err = cmd.Execute() + So(err, ShouldBeNil) + str = strings.TrimSpace(space.ReplaceAllString(buff.String(), " ")) + So(str, ShouldContainSubstring, "ARTIFACT TYPE SIZE DIGEST") + So(str, ShouldContainSubstring, "application/vnd.oci.image.config.v1+json 557 B "+ref1Digest.String()) + So(str, ShouldContainSubstring, "custom.art.type.v1 535 B "+ref2Digest.String()) + So(str, ShouldContainSubstring, "custom.art.type.v2 598 B "+ref3Digest.String()) + fmt.Println(buff.String()) + }) +} + +func TestFormatsReferrersCLI(t *testing.T) { + Convey("Create server", t, func() { + rootDir := t.TempDir() + + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + conf.Storage.GC = false + defaultVal := false + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, + } + ctlr := api.NewController(conf) + ctlr.Config.Storage.RootDirectory = rootDir + cm := test.NewControllerManager(ctlr) + cm.StartAndWait(conf.HTTP.Port) + defer cm.StopServer() + + repo := repoName + image, err := test.GetRandomImage("tag") + So(err, ShouldBeNil) + imgDigest, err := image.Digest() + So(err, ShouldBeNil) + + err = test.UploadImage(image, baseURL, repo) + So(err, ShouldBeNil) + + // add referrers + ref1, err := test.GetImageWithSubject(imgDigest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + ref2, err := test.GetImageWithSubject(imgDigest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + ref2.Manifest.Config.MediaType = customArtTypeV1 + + ref3, err := test.GetImageWithSubject(imgDigest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + ref3.Manifest.ArtifactType = customArtTypeV2 + ref3.Manifest.Config = ispec.DescriptorEmptyJSON + + ref1.Reference = "" + err = test.UploadImage(ref1, baseURL, repo) + So(err, ShouldBeNil) + + ref2.Reference = "" + err = test.UploadImage(ref2, baseURL, repo) + So(err, ShouldBeNil) + + ref3.Reference = "" + err = test.UploadImage(ref3, baseURL, repo) + So(err, ShouldBeNil) + + Convey("JSON format", func() { + args := []string{"reftest", "--output", "json", "--subject", repo + "@" + imgDigest.String()} + + configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"reftest","url":"%s","showspinner":false}]}`, + baseURL)) + + defer os.Remove(configPath) + + cmd := NewSearchCommand(new(searchService)) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err = cmd.Execute() + So(err, ShouldBeNil) + fmt.Println(buff.String()) + }) + Convey("YAML format", func() { + args := []string{"reftest", "--output", "yaml", "--subject", repo + "@" + imgDigest.String()} + + configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"reftest","url":"%s","showspinner":false}]}`, + baseURL)) + + defer os.Remove(configPath) + + cmd := NewSearchCommand(new(searchService)) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err = cmd.Execute() + So(err, ShouldBeNil) + fmt.Println(buff.String()) + }) + Convey("Invalid format", func() { + args := []string{"reftest", "--output", "invalid_format", "--subject", repo + "@" + imgDigest.String()} + + configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"reftest","url":"%s","showspinner":false}]}`, + baseURL)) + + defer os.Remove(configPath) + + cmd := NewSearchCommand(new(searchService)) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err = cmd.Execute() + So(err, ShouldNotBeNil) + }) + }) +} + +func TestReferrersCLIErrors(t *testing.T) { + Convey("Errors", t, func() { + cmd := NewSearchCommand(new(searchService)) + + Convey("no url provided", func() { + args := []string{"reftest", "--output", "invalid", "--query", "repo/alpine"} + + configPath := makeConfigFile(`{"configs":[{"_name":"reftest","showspinner":false}]}`) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err := cmd.Execute() + So(err, ShouldNotBeNil) + }) + + Convey("getConfigValue", func() { + args := []string{"reftest", "--subject", "repo/alpine"} + + configPath := makeConfigFile(`bad-json`) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err := cmd.Execute() + So(err, ShouldNotBeNil) + }) + + Convey("bad showspinnerConfig ", func() { + args := []string{"reftest"} + + configPath := makeConfigFile(`{"configs":[{"_name":"reftest", "url":"http://127.0.0.1:8080", "showspinner":"bad"}]}`) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err := cmd.Execute() + So(err, ShouldNotBeNil) + }) + + Convey("bad verifyTLSConfig ", func() { + args := []string{"reftest"} + + configPath := makeConfigFile( + `{"configs":[{"_name":"reftest", "url":"http://127.0.0.1:8080", "showspinner":false, "verify-tls": "bad"}]}`) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err := cmd.Execute() + So(err, ShouldNotBeNil) + }) + + Convey("url from config is empty", func() { + args := []string{"reftest", "--subject", "repo/alpine"} + + configPath := makeConfigFile(`{"configs":[{"_name":"reftest", "url":"", "showspinner":false}]}`) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err := cmd.Execute() + So(err, ShouldNotBeNil) + }) + + Convey("bad params combination", func() { + args := []string{"reftest"} + + configPath := makeConfigFile(`{"configs":[{"_name":"reftest", "url":"http://127.0.0.1:8080", "showspinner":false}]}`) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err := cmd.Execute() + So(err, ShouldNotBeNil) + }) + + Convey("no url provided error", func() { + args := []string{} + + configPath := makeConfigFile(`bad-json`) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err := cmd.Execute() + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/pkg/cli/search_cmd_test.go b/pkg/cli/search_cmd_test.go new file mode 100644 index 00000000..2a331f8f --- /dev/null +++ b/pkg/cli/search_cmd_test.go @@ -0,0 +1,433 @@ +//go:build search +// +build search + +package cli //nolint:testpackage + +import ( + "bytes" + "fmt" + "io" + "os" + "regexp" + "strings" + "testing" + + ispec "github.com/opencontainers/image-spec/specs-go/v1" + . "github.com/smartystreets/goconvey/convey" + + "zotregistry.io/zot/pkg/api" + "zotregistry.io/zot/pkg/api/config" + extconf "zotregistry.io/zot/pkg/extensions/config" + "zotregistry.io/zot/pkg/test" +) + +func TestGlobalSearchers(t *testing.T) { + globalSearcher := globalSearcherGQL{} + + Convey("GQL Searcher", t, func() { + Convey("Bad parameters", func() { + ok, err := globalSearcher.search(searchConfig{params: map[string]*string{ + "badParam": ref("badParam"), + }}) + + So(err, ShouldBeNil) + So(ok, ShouldBeFalse) + }) + + Convey("global searcher service fail", func() { + conf := searchConfig{ + params: map[string]*string{ + "query": ref("repo"), + }, + searchService: NewSearchService(), + user: ref("test:pass"), + servURL: ref("127.0.0.1:8080"), + verifyTLS: ref(false), + debug: ref(false), + verbose: ref(false), + fixedFlag: ref(false), + } + ok, err := globalSearcher.search(conf) + + So(err, ShouldNotBeNil) + So(ok, ShouldBeTrue) + }) + + Convey("print images fail", func() { + conf := searchConfig{ + params: map[string]*string{ + "query": ref("repo"), + }, + user: ref("user:pass"), + outputFormat: ref("bad-format"), + searchService: mockService{}, + resultWriter: io.Discard, + verbose: ref(false), + } + ok, err := globalSearcher.search(conf) + + So(err, ShouldNotBeNil) + So(ok, ShouldBeTrue) + }) + }) +} + +func TestSearchCLI(t *testing.T) { + Convey("Test GQL", t, func() { + rootDir := t.TempDir() + + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + conf.Storage.GC = false + defaultVal := true + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, + } + ctlr := api.NewController(conf) + ctlr.Config.Storage.RootDirectory = rootDir + cm := test.NewControllerManager(ctlr) + cm.StartAndWait(conf.HTTP.Port) + defer cm.StopServer() + + const ( + repo1 = "repo" + r1tag1 = "repo1tag1" + r1tag2 = "repo1tag2" + + repo2 = "repo/alpine" + r2tag1 = "repo2tag1" + r2tag2 = "repo2tag2" + + repo3 = "repo/test/alpine" + r3tag1 = "repo3tag1" + r3tag2 = "repo3tag2" + ) + + image1, err := test.GetImageWithConfig(ispec.Image{ + Platform: ispec.Platform{ + OS: "Os", + Architecture: "Arch", + }, + }) + So(err, ShouldBeNil) + img1Digest, err := image1.Digest() + formatterDigest1 := img1Digest.Encoded()[:8] + So(err, ShouldBeNil) + + image2, err := test.GetRandomImage("") + So(err, ShouldBeNil) + img2Digest, err := image2.Digest() + formatterDigest2 := img2Digest.Encoded()[:8] + So(err, ShouldBeNil) + + // repo1 + image1.Reference = r1tag1 + err = test.UploadImage(image1, baseURL, repo1) + So(err, ShouldBeNil) + + image2.Reference = r1tag2 + err = test.UploadImage(image2, baseURL, repo1) + So(err, ShouldBeNil) + + // repo2 + image1.Reference = r2tag1 + err = test.UploadImage(image1, baseURL, repo2) + So(err, ShouldBeNil) + + image2.Reference = r2tag2 + err = test.UploadImage(image2, baseURL, repo2) + So(err, ShouldBeNil) + + // repo3 + image1.Reference = r3tag1 + err = test.UploadImage(image1, baseURL, repo3) + So(err, ShouldBeNil) + + image2.Reference = r3tag2 + err = test.UploadImage(image2, baseURL, repo3) + So(err, ShouldBeNil) + + // search by repos + + args := []string{"searchtest", "--query", "test/alpin", "--verbose"} + + configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"searchtest","url":"%s","showspinner":false}]}`, + baseURL)) + defer os.Remove(configPath) + + cmd := NewSearchCommand(new(searchService)) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err = cmd.Execute() + So(err, ShouldBeNil) + space := regexp.MustCompile(`\s+`) + str := strings.TrimSpace(space.ReplaceAllString(buff.String(), " ")) + So(str, ShouldContainSubstring, "NAME SIZE LAST UPDATED DOWNLOADS STARS PLATFORMS") + So(str, ShouldContainSubstring, "repo/test/alpine 1.1kB 0001-01-01 00:00:00 +0000 UTC 0 0") + So(str, ShouldContainSubstring, "Os/Arch") + So(str, ShouldContainSubstring, "linux/amd64") + + fmt.Println("\n", buff.String()) + + os.Remove(configPath) + + cmd = NewSearchCommand(new(searchService)) + + args = []string{"searchtest", "--query", "repo/alpine:"} + + configPath = makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"searchtest","url":"%s","showspinner":false}]}`, + baseURL)) + + defer os.Remove(configPath) + + buff = &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err = cmd.Execute() + So(err, ShouldBeNil) + str = strings.TrimSpace(space.ReplaceAllString(buff.String(), " ")) + So(str, ShouldContainSubstring, "IMAGE NAME TAG OS/ARCH DIGEST SIGNED SIZE") + So(str, ShouldContainSubstring, "repo/alpine repo2tag1 Os/Arch "+formatterDigest1+" false 577B") + So(str, ShouldContainSubstring, "repo/alpine repo2tag2 linux/amd64 "+formatterDigest2+" false 524B") + + fmt.Println("\n", buff.String()) + }) +} + +func TestFormatsSearchCLI(t *testing.T) { + Convey("", t, func() { + rootDir := t.TempDir() + + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + conf.Storage.GC = false + defaultVal := true + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, + } + ctlr := api.NewController(conf) + ctlr.Config.Storage.RootDirectory = rootDir + cm := test.NewControllerManager(ctlr) + cm.StartAndWait(conf.HTTP.Port) + defer cm.StopServer() + + const ( + repo1 = "repo" + r1tag1 = "repo1tag1" + r1tag2 = "repo1tag2" + + repo2 = "repo/alpine" + r2tag1 = "repo2tag1" + r2tag2 = "repo2tag2" + + repo3 = "repo/test/alpine" + r3tag1 = "repo3tag1" + r3tag2 = "repo3tag2" + ) + + image1, err := test.GetRandomImage("") + So(err, ShouldBeNil) + + image2, err := test.GetRandomImage("") + So(err, ShouldBeNil) + + // repo1 + image1.Reference = r1tag1 + err = test.UploadImage(image1, baseURL, repo1) + So(err, ShouldBeNil) + + image2.Reference = r1tag2 + err = test.UploadImage(image2, baseURL, repo1) + So(err, ShouldBeNil) + + // repo2 + image1.Reference = r2tag1 + err = test.UploadImage(image1, baseURL, repo2) + So(err, ShouldBeNil) + + image2.Reference = r2tag2 + err = test.UploadImage(image2, baseURL, repo2) + So(err, ShouldBeNil) + + // repo3 + image1.Reference = r3tag1 + err = test.UploadImage(image1, baseURL, repo3) + So(err, ShouldBeNil) + + image2.Reference = r3tag2 + err = test.UploadImage(image2, baseURL, repo3) + So(err, ShouldBeNil) + + cmd := NewSearchCommand(new(searchService)) + + Convey("JSON format", func() { + args := []string{"searchtest", "--output", "json", "--query", "repo/alpine"} + + configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"searchtest","url":"%s","showspinner":false}]}`, + baseURL)) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err = cmd.Execute() + So(err, ShouldBeNil) + fmt.Println(buff.String()) + }) + + Convey("YAML format", func() { + args := []string{"searchtest", "--output", "yaml", "--query", "repo/alpine"} + + configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"searchtest","url":"%s","showspinner":false}]}`, + baseURL)) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err = cmd.Execute() + So(err, ShouldBeNil) + fmt.Println(buff.String()) + }) + + Convey("Invalid format", func() { + args := []string{"searchtest", "--output", "invalid", "--query", "repo/alpine"} + + configPath := makeConfigFile(fmt.Sprintf(`{"configs":[{"_name":"searchtest","url":"%s","showspinner":false}]}`, + baseURL)) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err = cmd.Execute() + So(err, ShouldNotBeNil) + }) + }) +} + +func TestSearchCLIErrors(t *testing.T) { + Convey("Errors", t, func() { + cmd := NewSearchCommand(new(searchService)) + + Convey("no url provided", func() { + args := []string{"searchtest", "--output", "invalid", "--query", "repo/alpine"} + + configPath := makeConfigFile(`{"configs":[{"_name":"searchtest","showspinner":false}]}`) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err := cmd.Execute() + So(err, ShouldNotBeNil) + }) + + Convey("getConfigValue", func() { + args := []string{"searchtest", "--output", "invalid", "--query", "repo/alpine"} + + configPath := makeConfigFile(`bad-json`) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err := cmd.Execute() + So(err, ShouldNotBeNil) + }) + + Convey("bad showspinnerConfig ", func() { + args := []string{"searchtest"} + + configPath := makeConfigFile( + `{"configs":[{"_name":"searchtest", "url":"http://127.0.0.1:8080", "showspinner":"bad"}]}`) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err := cmd.Execute() + So(err, ShouldNotBeNil) + }) + + Convey("bad verifyTLSConfig ", func() { + args := []string{"searchtest"} + + configPath := makeConfigFile( + `{"configs":[{"_name":"searchtest", "url":"http://127.0.0.1:8080", "showspinner":false, "verify-tls": "bad"}]}`) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err := cmd.Execute() + So(err, ShouldNotBeNil) + }) + + Convey("url from config is empty", func() { + args := []string{"searchtest", "--output", "invalid", "--query", "repo/alpine"} + + configPath := makeConfigFile(`{"configs":[{"_name":"searchtest", "url":"", "showspinner":false}]}`) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err := cmd.Execute() + So(err, ShouldNotBeNil) + }) + + Convey("no url provided error", func() { + args := []string{} + + configPath := makeConfigFile(`bad-json`) + + defer os.Remove(configPath) + + buff := &bytes.Buffer{} + cmd.SetOut(buff) + cmd.SetErr(buff) + cmd.SetArgs(args) + err := cmd.Execute() + So(err, ShouldNotBeNil) + }) + + Convey("globalSearch without gql active", func() { + err := globalSearch(searchConfig{ + user: ref("t"), + servURL: ref("t"), + verifyTLS: ref(false), + debug: ref(false), + params: map[string]*string{ + "query": ref("t"), + }, + resultWriter: io.Discard, + }) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/pkg/cli/searcher.go b/pkg/cli/searcher.go index 5ae18fed..b8821e87 100644 --- a/pkg/cli/searcher.go +++ b/pkg/cli/searcher.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "io" + "math" "strings" "sync" "time" @@ -15,6 +16,8 @@ import ( "github.com/briandowns/spinner" zotErrors "zotregistry.io/zot/errors" + "zotregistry.io/zot/pkg/api/constants" + zcommon "zotregistry.io/zot/pkg/common" ) func getImageSearchers() []searcher { @@ -61,6 +64,24 @@ func getCveSearchersGQL() []searcher { return searchers } +func getGlobalSearchersGQL() []searcher { + searchers := []searcher{ + new(globalSearcherGQL), + new(referrerSearcherGQL), + } + + return searchers +} + +func getGlobalSearchersREST() []searcher { + searchers := []searcher{ + new(referrerSearcher), + new(globalSearcherREST), + } + + return searchers +} + type searcher interface { search(searchConfig searchConfig) (bool, error) } @@ -194,7 +215,7 @@ func getImages(config searchConfig) error { imageListData = append(imageListData, imageStruct(image)) } - return printResult(config, imageListData) + return printImageResult(config, imageListData) } type imagesByDigestSearcher struct{} @@ -253,7 +274,7 @@ func (search derivedImageListSearcherGQL) search(config searchConfig) (bool, err imageListData = append(imageListData, imageStruct(image)) } - if err := printResult(config, imageListData); err != nil { + if err := printImageResult(config, imageListData); err != nil { return true, err } @@ -284,7 +305,7 @@ func (search baseImageListSearcherGQL) search(config searchConfig) (bool, error) imageListData = append(imageListData, imageStruct(image)) } - if err := printResult(config, imageListData); err != nil { + if err := printImageResult(config, imageListData); err != nil { return true, err } @@ -316,7 +337,7 @@ func (search imagesByDigestSearcherGQL) search(config searchConfig) (bool, error imageListData = append(imageListData, imageStruct(image)) } - if err := printResult(config, imageListData); err != nil { + if err := printImageResult(config, imageListData); err != nil { return true, err } @@ -461,7 +482,7 @@ func (search imagesByCVEIDSearcherGQL) search(config searchConfig) (bool, error) imageListData = append(imageListData, imageStruct(image)) } - if err := printResult(config, imageListData); err != nil { + if err := printImageResult(config, imageListData); err != nil { return true, err } @@ -603,7 +624,153 @@ func getTagsByCVE(config searchConfig) error { } } - return printResult(config, imageList) + return printImageResult(config, imageList) +} + +type referrerSearcherGQL struct{} + +func (search referrerSearcherGQL) search(config searchConfig) (bool, error) { + if !canSearch(config.params, newSet("subject")) { + return false, nil + } + + username, password := getUsernameAndPassword(*config.user) + + repo, ref, refIsTag, err := zcommon.GetRepoRefference(*config.params["subject"]) + if err != nil { + return true, err + } + + digest := ref + + if refIsTag { + digest, err = fetchImageDigest(repo, ref, username, password, config) + if err != nil { + return true, err + } + } + + response, err := config.searchService.getReferrersGQL(context.Background(), config, username, password, repo, digest) + if err != nil { + return true, err + } + + referrersList := referrersResult(response.Referrers) + + maxArtifactTypeLen := math.MinInt + + for _, referrer := range referrersList { + if maxArtifactTypeLen < len(referrer.ArtifactType) { + maxArtifactTypeLen = len(referrer.ArtifactType) + } + } + + printReferrersTableHeader(config, config.resultWriter, maxArtifactTypeLen) + + return true, printReferrersResult(config, referrersList, maxArtifactTypeLen) +} + +func fetchImageDigest(repo, ref, username, password string, config searchConfig) (string, error) { + url, err := combineServerAndEndpointURL(*config.servURL, fmt.Sprintf("/v2/%s/manifests/%s", repo, ref)) + if err != nil { + return "", err + } + + res, err := makeHEADRequest(context.Background(), url, username, password, *config.verifyTLS, false) + + digestStr := res.Get(constants.DistContentDigestKey) + + return digestStr, err +} + +type referrerSearcher struct{} + +func (search referrerSearcher) search(config searchConfig) (bool, error) { + if !canSearch(config.params, newSet("subject")) { + return false, nil + } + + username, password := getUsernameAndPassword(*config.user) + + repo, ref, refIsTag, err := zcommon.GetRepoRefference(*config.params["subject"]) + if err != nil { + return true, err + } + + digest := ref + + if refIsTag { + digest, err = fetchImageDigest(repo, ref, username, password, config) + if err != nil { + return true, err + } + } + + referrersList, err := config.searchService.getReferrers(context.Background(), config, username, password, + repo, digest) + if err != nil { + return true, err + } + + maxArtifactTypeLen := math.MinInt + + for _, referrer := range referrersList { + if maxArtifactTypeLen < len(referrer.ArtifactType) { + maxArtifactTypeLen = len(referrer.ArtifactType) + } + } + + printReferrersTableHeader(config, config.resultWriter, maxArtifactTypeLen) + + return true, printReferrersResult(config, referrersList, maxArtifactTypeLen) +} + +type globalSearcherGQL struct{} + +func (search globalSearcherGQL) search(config searchConfig) (bool, error) { + if !canSearch(config.params, newSet("query")) { + return false, nil + } + + username, password := getUsernameAndPassword(*config.user) + ctx, cancel := context.WithCancel(context.Background()) + + defer cancel() + + query := *config.params["query"] + + globalSearchResult, err := config.searchService.globalSearchGQL(ctx, config, username, password, query) + if err != nil { + return true, err + } + + imagesList := []imageStruct{} + + for _, image := range globalSearchResult.Images { + imagesList = append(imagesList, imageStruct(image)) + } + + reposList := []repoStruct{} + + for _, repo := range globalSearchResult.Repos { + reposList = append(reposList, repoStruct(repo)) + } + + if err := printImageResult(config, imagesList); err != nil { + return true, err + } + + return true, printRepoResults(config, reposList) +} + +type globalSearcherREST struct{} + +func (search globalSearcherREST) search(config searchConfig) (bool, error) { + if !canSearch(config.params, newSet("query")) { + return false, nil + } + + return true, fmt.Errorf("search extension is not enabled: %w", zotErrors.ErrExtensionNotEnabled) } func collectResults(config searchConfig, wg *sync.WaitGroup, imageErr chan stringResult, @@ -779,7 +946,7 @@ func printImageTableHeader(writer io.Writer, verbose bool, maxImageNameLen, maxT } row[colDigestIndex] = "DIGEST" - row[colSizeIndex] = "SIZE" + row[colSizeIndex] = sizeColumn row[colIsSignedIndex] = "SIGNED" if verbose { @@ -802,7 +969,94 @@ func printCVETableHeader(writer io.Writer, verbose bool, maxImgLen, maxTagLen, m table.Render() } -func printResult(config searchConfig, imageList []imageStruct) error { +func printReferrersTableHeader(config searchConfig, writer io.Writer, maxArtifactTypeLen int) { + if *config.outputFormat != "" && *config.outputFormat != defaultOutoutFormat { + return + } + + table := getReferrersTableWriter(writer) + + table.SetColMinWidth(refArtifactTypeIndex, maxArtifactTypeLen) + table.SetColMinWidth(refDigestIndex, digestWidth) + table.SetColMinWidth(refSizeIndex, sizeWidth) + + row := make([]string, refRowWidth) + + // adding spaces so that image name and tag columns are aligned + // in case the name/tag are fully shown and too long + var offset string + + if maxArtifactTypeLen > len("ARTIFACT TYPE") { + offset = strings.Repeat(" ", maxArtifactTypeLen-len("ARTIFACT TYPE")) + row[refArtifactTypeIndex] = "ARTIFACT TYPE" + offset + } else { + row[refArtifactTypeIndex] = "ARTIFACT TYPE" + } + + row[refDigestIndex] = "DIGEST" + row[refSizeIndex] = sizeColumn + + table.Append(row) + table.Render() +} + +func printRepoTableHeader(writer io.Writer, repoMaxLen, maxTimeLen int, verbose bool) { + table := getRepoTableWriter(writer) + + table.SetColMinWidth(repoNameIndex, repoMaxLen) + table.SetColMinWidth(repoSizeIndex, sizeWidth) + table.SetColMinWidth(repoLastUpdatedIndex, maxTimeLen) + table.SetColMinWidth(repoDownloadsIndex, sizeWidth) + table.SetColMinWidth(repoStarsIndex, sizeWidth) + + if verbose { + table.SetColMinWidth(repoPlatformsIndex, platformWidth) + } + + row := make([]string, repoRowWidth) + + // adding spaces so that image name and tag columns are aligned + // in case the name/tag are fully shown and too long + var offset string + + if repoMaxLen > len("NAME") { + offset = strings.Repeat(" ", repoMaxLen-len("NAME")) + row[repoNameIndex] = "NAME" + offset + } else { + row[repoNameIndex] = "NAME" + } + + if repoMaxLen > len("LAST UPDATED") { + offset = strings.Repeat(" ", repoMaxLen-len("LAST UPDATED")) + row[repoLastUpdatedIndex] = "LAST UPDATED" + offset + } else { + row[repoLastUpdatedIndex] = "LAST UPDATED" + } + + row[repoSizeIndex] = sizeColumn + row[repoDownloadsIndex] = "DOWNLOADS" + row[repoStarsIndex] = "STARS" + + if verbose { + row[repoPlatformsIndex] = "PLATFORMS" + } + + table.Append(row) + table.Render() +} + +func printReferrersResult(config searchConfig, referrersList referrersResult, maxArtifactTypeLen int) error { + out, err := referrersList.string(*config.outputFormat, maxArtifactTypeLen) + if err != nil { + return err + } + + fmt.Fprint(config.resultWriter, out) + + return nil +} + +func printImageResult(config searchConfig, imageList []imageStruct) error { var builder strings.Builder maxImgNameLen := 0 maxTagLen := 0 @@ -846,6 +1100,36 @@ func printResult(config searchConfig, imageList []imageStruct) error { return nil } +func printRepoResults(config searchConfig, repoList []repoStruct) error { + maxRepoNameLen := 0 + maxTimeLen := 0 + + for _, repo := range repoList { + if maxRepoNameLen < len(repo.Name) { + maxRepoNameLen = len(repo.Name) + } + + if maxTimeLen < len(repo.LastUpdated.String()) { + maxTimeLen = len(repo.LastUpdated.String()) + } + } + + if len(repoList) > 0 { + printRepoTableHeader(config.resultWriter, maxRepoNameLen, maxTimeLen, *config.verbose) + } + + for _, repo := range repoList { + out, err := repo.string(*config.outputFormat, maxRepoNameLen, maxTimeLen, *config.verbose) + if err != nil { + return err + } + + fmt.Fprint(config.resultWriter, out) + } + + return nil +} + var ( errInvalidImageNameAndTag = errors.New("cli: Invalid input format. Expected IMAGENAME:TAG") errInvalidImageName = errors.New("cli: Invalid input format. Expected IMAGENAME without :TAG") @@ -876,3 +1160,7 @@ func (search repoSearcher) searchRepos(config searchConfig) error { return nil } } + +const ( + sizeColumn = "SIZE" +) diff --git a/pkg/cli/service.go b/pkg/cli/service.go index 7c0828a4..ca3e5bff 100644 --- a/pkg/cli/service.go +++ b/pkg/cli/service.go @@ -25,6 +25,12 @@ import ( "zotregistry.io/zot/pkg/common" ) +const ( + jsonFormat = "json" + yamlFormat = "yaml" + ymlFormat = "yml" +) + type SearchService interface { //nolint:interfacebloat getImagesGQL(ctx context.Context, config searchConfig, username, password string, imageName string) (*common.ImageListResponse, error) @@ -42,6 +48,10 @@ type SearchService interface { //nolint:interfacebloat derivedImage string) (*common.DerivedImageListResponse, error) getBaseImageListGQL(ctx context.Context, config searchConfig, username, password string, baseImage string) (*common.BaseImageListResponse, error) + getReferrersGQL(ctx context.Context, config searchConfig, username, password string, + repo, digest string) (*common.ReferrersResp, error) + globalSearchGQL(ctx context.Context, config searchConfig, username, password string, + query string) (*common.GlobalSearch, error) getAllImages(ctx context.Context, config searchConfig, username, password string, channel chan stringResult, wtgrp *sync.WaitGroup) @@ -59,6 +69,8 @@ type SearchService interface { //nolint:interfacebloat channel chan stringResult, wtgrp *sync.WaitGroup) getImageByNameAndCVEID(ctx context.Context, config searchConfig, username, password, imageName, cvid string, channel chan stringResult, wtgrp *sync.WaitGroup) + getReferrers(ctx context.Context, config searchConfig, username, password string, repo, digest string, + ) (referrersResult, error) } type searchService struct{} @@ -103,6 +115,75 @@ func (service searchService) getDerivedImageListGQL(ctx context.Context, config return result, nil } +func (service searchService) getReferrersGQL(ctx context.Context, config searchConfig, username, password string, + repo, digest string, +) (*common.ReferrersResp, error) { + query := fmt.Sprintf(` + { + Referrers( repo: "%s", digest: "%s" ){ + ArtifactType, + Digest, + MediaType, + Size, + Annotations{ + Key + Value + } + } + }`, repo, digest) + + result := &common.ReferrersResp{} + + err := service.makeGraphQLQuery(ctx, config, username, password, query, result) + if errResult := checkResultGraphQLQuery(ctx, err, result.Errors); errResult != nil { + return nil, errResult + } + + return result, nil +} + +func (service searchService) globalSearchGQL(ctx context.Context, config searchConfig, username, password string, + query string, +) (*common.GlobalSearch, error) { + GQLQuery := fmt.Sprintf(` + { + GlobalSearch(query:"%s"){ + Images { + RepoName + Tag + MediaType + Digest + Size + Manifests { + Digest + ConfigDigest + Platform {Os Arch} + Size + IsSigned + Layers {Digest Size} + } + } + Repos { + Name + Platforms { Os Arch } + LastUpdated + Size + DownloadCount + StarCount + } + } + }`, query) + + result := &common.GlobalSearchResultResp{} + + err := service.makeGraphQLQuery(ctx, config, username, password, GQLQuery, result) + if errResult := checkResultGraphQLQuery(ctx, err, result.Errors); errResult != nil { + return nil, errResult + } + + return &result.GlobalSearch, nil +} + func (service searchService) getBaseImageListGQL(ctx context.Context, config searchConfig, username, password string, baseImage string, ) (*common.BaseImageListResponse, error) { @@ -328,6 +409,44 @@ func (service searchService) getFixedTagsForCVEGQL(ctx context.Context, config s return result, nil } +func (service searchService) getReferrers(ctx context.Context, config searchConfig, username, password string, + repo, digest string, +) (referrersResult, error) { + referrersEndpoint, err := combineServerAndEndpointURL(*config.servURL, + fmt.Sprintf("/v2/%s/referrers/%s", repo, digest)) + if err != nil { + if isContextDone(ctx) { + return referrersResult{}, nil + } + + return referrersResult{}, err + } + + referrerResp := &ispec.Index{} + _, err = makeGETRequest(ctx, referrersEndpoint, username, password, *config.verifyTLS, + *config.debug, &referrerResp, config.resultWriter) + + if err != nil { + if isContextDone(ctx) { + return referrersResult{}, nil + } + + return referrersResult{}, err + } + + referrersList := referrersResult{} + + for _, referrer := range referrerResp.Manifests { + referrersList = append(referrersList, common.Referrer{ + ArtifactType: referrer.ArtifactType, + Digest: referrer.Digest.String(), + Size: int(referrer.Size), + }) + } + + return referrersList, nil +} + func (service searchService) getImageByName(ctx context.Context, config searchConfig, username, password, imageName string, rch chan stringResult, wtgrp *sync.WaitGroup, ) { @@ -940,9 +1059,9 @@ func (cve cveResult) string(format string) (string, error) { switch strings.ToLower(format) { case "", defaultOutoutFormat: return cve.stringPlainText() - case "json": + case jsonFormat: return cve.stringJSON() - case "yml", "yaml": + case ymlFormat, yamlFormat: return cve.stringYAML() default: return "", ErrInvalidOutputFormat @@ -991,15 +1110,167 @@ func (cve cveResult) stringYAML() (string, error) { return string(body), nil } +type referrersResult []common.Referrer + +func (ref referrersResult) string(format string, maxArtifactTypeLen int) (string, error) { + switch strings.ToLower(format) { + case "", defaultOutoutFormat: + return ref.stringPlainText(maxArtifactTypeLen) + case jsonFormat: + return ref.stringJSON() + case ymlFormat, yamlFormat: + return ref.stringYAML() + default: + return "", ErrInvalidOutputFormat + } +} + +func (ref referrersResult) stringPlainText(maxArtifactTypeLen int) (string, error) { + var builder strings.Builder + + table := getImageTableWriter(&builder) + + table.SetColMinWidth(refArtifactTypeIndex, maxArtifactTypeLen) + table.SetColMinWidth(refDigestIndex, digestWidth) + table.SetColMinWidth(refSizeIndex, sizeWidth) + + for _, referrer := range ref { + artifactType := ellipsize(referrer.ArtifactType, maxArtifactTypeLen, ellipsis) + // digest := ellipsize(godigest.Digest(referrer.Digest).Encoded(), digestWidth, "") + size := ellipsize(humanize.Bytes(uint64(referrer.Size)), sizeWidth, ellipsis) + + row := make([]string, refRowWidth) + row[refArtifactTypeIndex] = artifactType + row[refDigestIndex] = referrer.Digest + row[refSizeIndex] = size + + table.Append(row) + } + + table.Render() + + return builder.String(), nil +} + +func (ref referrersResult) stringJSON() (string, error) { + json := jsoniter.ConfigCompatibleWithStandardLibrary + + body, err := json.MarshalIndent(ref, "", " ") + if err != nil { + return "", err + } + + return string(body), nil +} + +func (ref referrersResult) stringYAML() (string, error) { + body, err := yaml.Marshal(ref) + if err != nil { + return "", err + } + + return string(body), nil +} + +type repoStruct common.RepoSummary + +func (repo repoStruct) string(format string, maxImgNameLen, maxTimeLen int, verbose bool) (string, error) { //nolint: lll + switch strings.ToLower(format) { + case "", defaultOutoutFormat: + return repo.stringPlainText(maxImgNameLen, maxTimeLen, verbose) + case jsonFormat: + return repo.stringJSON() + case ymlFormat, yamlFormat: + return repo.stringYAML() + default: + return "", ErrInvalidOutputFormat + } +} + +func (repo repoStruct) stringPlainText(repoMaxLen, maxTimeLen int, verbose bool) (string, error) { + var builder strings.Builder + + table := getImageTableWriter(&builder) + + table.SetColMinWidth(repoNameIndex, repoMaxLen) + table.SetColMinWidth(repoSizeIndex, sizeWidth) + table.SetColMinWidth(repoLastUpdatedIndex, maxTimeLen) + table.SetColMinWidth(repoDownloadsIndex, dounloadsWidth) + table.SetColMinWidth(repoStarsIndex, signedWidth) + + if verbose { + table.SetColMinWidth(repoPlatformsIndex, platformWidth) + } + + repoSize, err := strconv.Atoi(repo.Size) + if err != nil { + return "", err + } + + repoName := repo.Name + repoLastUpdated := repo.LastUpdated + repoDownloads := repo.DownloadCount + repoStars := repo.StarCount + repoPlatforms := repo.Platforms + + row := make([]string, repoRowWidth) + row[repoNameIndex] = repoName + row[repoSizeIndex] = ellipsize(strings.ReplaceAll(humanize.Bytes(uint64(repoSize)), " ", ""), sizeWidth, ellipsis) + row[repoLastUpdatedIndex] = repoLastUpdated.String() + row[repoDownloadsIndex] = strconv.Itoa(repoDownloads) + row[repoStarsIndex] = strconv.Itoa(repoStars) + + if verbose && len(repoPlatforms) > 0 { + row[repoPlatformsIndex] = getPlatformStr(repoPlatforms[0]) + repoPlatforms = repoPlatforms[1:] + } + + table.Append(row) + + if verbose { + for _, platform := range repoPlatforms { + row := make([]string, repoRowWidth) + + row[repoPlatformsIndex] = getPlatformStr(platform) + + table.Append(row) + } + } + + table.Render() + + return builder.String(), nil +} + +func (repo repoStruct) stringJSON() (string, error) { + json := jsoniter.ConfigCompatibleWithStandardLibrary + + body, err := json.MarshalIndent(repo, "", " ") + if err != nil { + return "", err + } + + return string(body), nil +} + +func (repo repoStruct) stringYAML() (string, error) { + body, err := yaml.Marshal(&repo) + if err != nil { + return "", err + } + + return string(body), nil +} + type imageStruct common.ImageSummary func (img imageStruct) string(format string, maxImgNameLen, maxTagLen, maxPlatformLen int, verbose bool) (string, error) { //nolint: lll switch strings.ToLower(format) { case "", defaultOutoutFormat: return img.stringPlainText(maxImgNameLen, maxTagLen, maxPlatformLen, verbose) - case "json": + case jsonFormat: return img.stringJSON() - case "yml", "yaml": + case ymlFormat, yamlFormat: return img.stringYAML() default: return "", ErrInvalidOutputFormat @@ -1283,6 +1554,42 @@ func getCVETableWriter(writer io.Writer) *tablewriter.Table { return table } +func getReferrersTableWriter(writer io.Writer) *tablewriter.Table { + table := tablewriter.NewWriter(writer) + + table.SetAutoWrapText(false) + table.SetAutoFormatHeaders(true) + table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) + table.SetAlignment(tablewriter.ALIGN_LEFT) + table.SetCenterSeparator("") + table.SetColumnSeparator("") + table.SetRowSeparator("") + table.SetHeaderLine(false) + table.SetBorder(false) + table.SetTablePadding(" ") + table.SetNoWhiteSpace(true) + + return table +} + +func getRepoTableWriter(writer io.Writer) *tablewriter.Table { + table := tablewriter.NewWriter(writer) + + table.SetAutoWrapText(false) + table.SetAutoFormatHeaders(true) + table.SetHeaderAlignment(tablewriter.ALIGN_LEFT) + table.SetAlignment(tablewriter.ALIGN_LEFT) + table.SetCenterSeparator("") + table.SetColumnSeparator("") + table.SetRowSeparator("") + table.SetHeaderLine(false) + table.SetBorder(false) + table.SetTablePadding(" ") + table.SetNoWhiteSpace(true) + + return table +} + func (service searchService) getRepos(ctx context.Context, config searchConfig, username, password string, rch chan stringResult, wtgrp *sync.WaitGroup, ) { @@ -1321,15 +1628,18 @@ func (service searchService) getRepos(ctx context.Context, config searchConfig, } const ( - imageNameWidth = 10 - tagWidth = 8 - digestWidth = 8 - platformWidth = 14 - sizeWidth = 8 - isSignedWidth = 8 - configWidth = 8 - layersWidth = 8 - ellipsis = "..." + imageNameWidth = 10 + tagWidth = 8 + digestWidth = 8 + platformWidth = 14 + sizeWidth = 10 + isSignedWidth = 8 + dounloadsWidth = 10 + signedWidth = 10 + lastUpdatedWidth = 14 + configWidth = 8 + layersWidth = 8 + ellipsis = "..." cveIDWidth = 16 cveSeverityWidth = 8 @@ -1354,3 +1664,22 @@ const ( rowWidth ) + +const ( + repoNameIndex = iota + repoSizeIndex + repoLastUpdatedIndex + repoDownloadsIndex + repoStarsIndex + repoPlatformsIndex + + repoRowWidth +) + +const ( + refArtifactTypeIndex = iota + refSizeIndex + refDigestIndex + + refRowWidth +) diff --git a/pkg/common/model.go b/pkg/common/model.go index 14f9adb8..09cec23e 100644 --- a/pkg/common/model.go +++ b/pkg/common/model.go @@ -15,15 +15,16 @@ type RepoInfo struct { } type RepoSummary struct { - Name string `json:"name"` - LastUpdated time.Time `json:"lastUpdated"` - Size string `json:"size"` - Platforms []Platform `json:"platforms"` - Vendors []string `json:"vendors"` - IsStarred bool `json:"isStarred"` - IsBookmarked bool `json:"isBookmarked"` - StarCount int `json:"starCount"` - NewestImage ImageSummary `json:"newestImage"` + Name string `json:"name"` + LastUpdated time.Time `json:"lastUpdated"` + Size string `json:"size"` + Platforms []Platform `json:"platforms"` + Vendors []string `json:"vendors"` + IsStarred bool `json:"isStarred"` + IsBookmarked bool `json:"isBookmarked"` + StarCount int `json:"starCount"` + DownloadCount int `json:"downloadCount"` + NewestImage ImageSummary `json:"newestImage"` } type PaginatedImagesResult struct { diff --git a/pkg/common/oci.go b/pkg/common/oci.go index 3758019c..620f05d3 100644 --- a/pkg/common/oci.go +++ b/pkg/common/oci.go @@ -5,6 +5,8 @@ import ( "time" ispec "github.com/opencontainers/image-spec/specs-go/v1" + + zerr "zotregistry.io/zot/errors" ) func GetImageDirAndTag(imageName string) (string, string) { @@ -76,3 +78,30 @@ func GetImageLastUpdated(imageInfo ispec.Image) time.Time { return *timeStamp } + +// GetRepoRefference returns the components of a repoName:tag or repoName@digest string. If the format is wrong +// an error is returned. +// The returned values have the following meaning: +// +// - string: repo name +// +// - string: reference (tag or digest) +// +// - bool: value for the statement: "the reference is a tag" +// +// - error: error value. +func GetRepoRefference(repo string) (string, string, bool, error) { + repoName, digest, found := strings.Cut(repo, "@") + + if !found { + repoName, tag, found := strings.Cut(repo, ":") + + if !found { + return "", "", false, zerr.ErrInvalidRepoTagFormat + } + + return repoName, tag, true, nil + } + + return repoName, digest, false, nil +} diff --git a/pkg/meta/repodb/storage_parsing.go b/pkg/meta/repodb/storage_parsing.go index 6f77997d..ba22e006 100644 --- a/pkg/meta/repodb/storage_parsing.go +++ b/pkg/meta/repodb/storage_parsing.go @@ -60,45 +60,25 @@ func ParseRepo(repo string, repoDB RepoDB, storeController storage.StoreControll return err } - err = resetRepoMetaTags(repo, repoDB, log) + err = resetRepoMeta(repo, repoDB, log) if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { log.Error().Err(err).Str("repository", repo).Msg("load-repo: failed to reset tag field in RepoMetadata for repo") return err } - for _, manifest := range indexContent.Manifests { - tag, hasTag := manifest.Annotations[ispec.AnnotationRefName] + for _, descriptor := range indexContent.Manifests { + tag := descriptor.Annotations[ispec.AnnotationRefName] - manifestMetaIsPresent, err := isManifestMetaPresent(repo, manifest, repoDB) + descriptorBlob, err := getCachedBlob(repo, descriptor, repoDB, imageStore, log) if err != nil { log.Error().Err(err).Msg("load-repo: error checking manifestMeta in RepoDB") return err } - // this check helps reduce unecesary reads from storage - if manifestMetaIsPresent && hasTag { - err = repoDB.SetRepoReference(repo, tag, manifest.Digest, manifest.MediaType) - if err != nil { - log.Error().Err(err).Str("repository", repo).Str("tag", tag).Msg("load-repo: failed to set repo tag") - - return err - } - - continue - } - - manifestBlob, digest, _, err := imageStore.GetImageManifest(repo, manifest.Digest.String()) - if err != nil { - log.Error().Err(err).Str("repository", repo).Str("tag", tag). - Msg("load-repo: failed to set repo tag for image") - - return err - } - isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(repo, - manifestBlob, tag) + descriptorBlob, tag) if err != nil { log.Error().Err(err).Str("repository", repo).Str("tag", tag). Msg("load-repo: failed checking if image is signature for specified image") @@ -107,8 +87,8 @@ func ParseRepo(repo string, repoDB RepoDB, storeController storage.StoreControll } if isSignature { - layers, err := GetSignatureLayersInfo(repo, tag, manifest.Digest.String(), signatureType, manifestBlob, - imageStore, log) + layers, err := GetSignatureLayersInfo(repo, tag, descriptor.Digest.String(), signatureType, + descriptorBlob, imageStore, log) if err != nil { return err } @@ -116,7 +96,7 @@ func ParseRepo(repo string, repoDB RepoDB, storeController storage.StoreControll err = repoDB.AddManifestSignature(repo, signedManifestDigest, SignatureMetadata{ SignatureType: signatureType, - SignatureDigest: digest.String(), + SignatureDigest: descriptor.Digest.String(), LayersInfo: layers, }) if err != nil { @@ -141,10 +121,10 @@ func ParseRepo(repo string, repoDB RepoDB, storeController storage.StoreControll reference := tag if tag == "" { - reference = manifest.Digest.String() + reference = descriptor.Digest.String() } - err = SetImageMetaFromInput(repo, reference, manifest.MediaType, manifest.Digest, manifestBlob, + err = SetImageMetaFromInput(repo, reference, descriptor.MediaType, descriptor.Digest, descriptorBlob, imageStore, repoDB, log) if err != nil { log.Error().Err(err).Str("repository", repo).Str("tag", tag). @@ -157,8 +137,9 @@ func ParseRepo(repo string, repoDB RepoDB, storeController storage.StoreControll return nil } -// resetRepoMetaTags will delete all tags from a repometadata. -func resetRepoMetaTags(repo string, repoDB RepoDB, log log.Logger) error { +// resetRepoMeta will delete all tags and non-user related information from a RepoMetadata. +// It is used to recalculate and keep RepoDB consistent with the layout in case of unexpected changes. +func resetRepoMeta(repo string, repoDB RepoDB, log log.Logger) error { repoMeta, err := repoDB.GetRepoMeta(repo) if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { log.Error().Err(err).Str("repository", repo).Msg("load-repo: failed to get RepoMeta for repo") @@ -202,18 +183,41 @@ func getAllRepos(storeController storage.StoreController) ([]string, error) { return allRepos, nil } -// isManifestMetaPresent checks if the manifest with a certain digest is present in a certain repo. -func isManifestMetaPresent(repo string, manifest ispec.Descriptor, repoDB RepoDB) (bool, error) { - _, err := repoDB.GetManifestMeta(repo, manifest.Digest) - if err != nil && !errors.Is(err, zerr.ErrManifestMetaNotFound) { - return false, err +func getCachedBlob(repo string, descriptor ispec.Descriptor, repoDB RepoDB, + imageStore storageTypes.ImageStore, log log.Logger, +) ([]byte, error) { + digest := descriptor.Digest + + descriptorBlob, err := getCachedBlobFromRepoDB(descriptor, repoDB) + + if err != nil || len(descriptorBlob) == 0 { + descriptorBlob, _, _, err = imageStore.GetImageManifest(repo, digest.String()) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", digest.String()). + Msg("load-repo: failed to get blob for image") + + return nil, err + } + + return descriptorBlob, nil } - if errors.Is(err, zerr.ErrManifestMetaNotFound) { - return false, nil + return descriptorBlob, nil +} + +func getCachedBlobFromRepoDB(descriptor ispec.Descriptor, repoDB RepoDB) ([]byte, error) { + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestData, err := repoDB.GetManifestData(descriptor.Digest) + + return manifestData.ManifestBlob, err + case ispec.MediaTypeImageIndex: + indexData, err := repoDB.GetIndexData(descriptor.Digest) + + return indexData.IndexBlob, err } - return true, nil + return nil, nil } func GetSignatureLayersInfo(repo, tag, manifestDigest, signatureType string, manifestBlob []byte, diff --git a/pkg/test/common.go b/pkg/test/common.go index a7b123c3..a4bab756 100644 --- a/pkg/test/common.go +++ b/pkg/test/common.go @@ -799,7 +799,7 @@ func GetImageWithSubject(subjectDigest godigest.Digest, mediaType string) (Image MediaType: mediaType, } - manifestBlob, err := json.Marshal(manifest) + blob, err := json.Marshal(manifest) if err != nil { return Image{}, err } @@ -808,7 +808,7 @@ func GetImageWithSubject(subjectDigest godigest.Digest, mediaType string) (Image Manifest: manifest, Config: conf, Layers: layers, - Reference: godigest.FromBytes(manifestBlob).String(), + Reference: godigest.FromBytes(blob).String(), }, nil } @@ -850,7 +850,8 @@ func UploadImage(img Image, baseURL, repo string) error { cdigest := godigest.FromBytes(cblob) - if img.Manifest.Config.MediaType == ispec.MediaTypeEmptyJSON { + if img.Manifest.Config.MediaType == ispec.MediaTypeEmptyJSON || + img.Manifest.Config.Digest == ispec.DescriptorEmptyJSON.Digest { cblob = ispec.DescriptorEmptyJSON.Data cdigest = ispec.DescriptorEmptyJSON.Digest } @@ -888,6 +889,10 @@ func UploadImage(img Image, baseURL, repo string) error { return err } + if img.Reference == "" { + img.Reference = godigest.FromBytes(manifestBlob).String() + } + resp, err = resty.R(). SetHeader("Content-type", ispec.MediaTypeImageManifest). SetBody(manifestBlob). @@ -1559,6 +1564,10 @@ func UploadImageWithBasicAuth(img Image, baseURL, repo, user, password string) e return err } + if img.Reference == "" { + img.Reference = godigest.FromBytes(manifestBlob).String() + } + _, err = resty.R(). SetBasicAuth(user, password). SetHeader("Content-type", "application/vnd.oci.image.manifest.v1+json"). diff --git a/pkg/test/common_test.go b/pkg/test/common_test.go index 3675a9ab..c519ce0e 100644 --- a/pkg/test/common_test.go +++ b/pkg/test/common_test.go @@ -464,12 +464,12 @@ func TestUploadImage(t *testing.T) { img := test.Image{ Layers: [][]byte{ layerBlob, - }, // invalid format that will result in an error + }, Config: ispec.Image{}, } err := test.UploadImage(img, baseURL, "test") - So(err, ShouldNotBeNil) + So(err, ShouldBeNil) }) Convey("Upload image with authentification", t, func() {