From 9cc990d7ca7f5e703fdd54083ca458c77ee8e47d Mon Sep 17 00:00:00 2001 From: LaurentiuNiculae Date: Mon, 24 Apr 2023 21:13:15 +0300 Subject: [PATCH] feat(repodb): add user related information to repodb (#1317) Initial code was contributed by Bogdan BIVOLARU <104334+bogdanbiv@users.noreply.github.com> Moved implementation from a separate db to repodb by Andrei Aaron Not done yet: - run/test dynamodb implementation, only boltdb was tested - add additional coverage for existing functionality - add web-based APIs to toggle the stars/bookmarks on/off Initially graphql mutation was discussed for the missing API but we decided REST endpoints would be better suited for configuration feat(userdb): complete functionality for userdb integration - dynamodb rollback changes to user starred repos in case increasing the total star count fails - dynamodb increment/decrement repostars in repometa when user stars/unstars a repo - dynamodb check anonymous user permissions are working as intendend - common test handle anonymous users - RepoMeta2RepoSummary set IsStarred and IsBookmarked feat(userdb): rest api calls for toggling stars/bookmarks on/off test(userdb): blackbox tests test(userdb): move preferences tests in a different file with specific build tags feat(repodb): add is-starred and is-bookmarked fields to repo-meta - removed duplicated logic for determining if a repo is starred/bookmarked Signed-off-by: Laurentiu Niculae Co-authored-by: Andrei Aaron --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/ecosystem-tools.yaml | 3 + .github/workflows/golangci-lint.yaml | 2 +- Makefile | 7 +- errors/errors.go | 167 ++-- examples/README.md | 1 + examples/config-dynamodb.json | 1 + pkg/api/authz.go | 17 +- pkg/api/constants/extensions.go | 11 +- pkg/api/controller_test.go | 6 + pkg/api/routes.go | 2 + pkg/common/common.go | 24 + pkg/common/common_test.go | 5 + pkg/common/model.go | 15 +- pkg/extensions/extension_userprefs.go | 143 +++ pkg/extensions/extension_userprefs_disable.go | 20 + pkg/extensions/extension_userprefs_test.go | 139 +++ pkg/extensions/search/convert/repodb.go | 22 +- pkg/extensions/search/digest_test.go | 5 +- .../search/gql_generated/generated.go | 242 +++++ pkg/extensions/search/resolver.go | 93 +- pkg/extensions/search/resolver_test.go | 172 +++- pkg/extensions/search/schema.graphql | 16 + pkg/extensions/search/schema.resolvers.go | 10 + pkg/extensions/search/search_test.go | 20 + pkg/extensions/search/userprefs_test.go | 632 +++++++++++++ pkg/meta/bolt/buckets.go | 4 +- pkg/meta/common/common.go | 96 ++ pkg/meta/common/common_test.go | 130 +++ pkg/meta/dynamo/parameters.go | 2 +- .../repodb/boltdb-wrapper/boltdb_wrapper.go | 857 ++++++++++++------ .../boltdb-wrapper/boltdb_wrapper_test.go | 196 +++- pkg/meta/repodb/common.go | 6 +- .../dynamodb-wrapper/dynamo_internal_test.go | 3 + .../repodb/dynamodb-wrapper/dynamo_test.go | 305 +++++++ .../repodb/dynamodb-wrapper/dynamo_wrapper.go | 787 +++++++++++----- pkg/meta/repodb/pagination.go | 14 +- pkg/meta/repodb/pagination_test.go | 20 +- pkg/meta/repodb/repodb.go | 39 +- pkg/meta/repodb/repodb_test.go | 559 ++++++++++++ .../repodb/repodbfactory/repodb_factory.go | 4 + .../repodbfactory/repodb_factory_test.go | 1 + pkg/meta/repodb/storage_parsing_test.go | 1 + pkg/meta/version/version_test.go | 1 + pkg/requestcontext/checkrepo.go | 8 + pkg/test/common.go | 1 + pkg/test/mocks/repo_db_mock.go | 54 ++ test/blackbox/cloud-only.bats | 1 + test/blackbox/metadata.bats | 138 +++ test/blackbox/referrers.bats | 1 - 50 files changed, 4357 insertions(+), 648 deletions(-) create mode 100644 pkg/extensions/extension_userprefs.go create mode 100644 pkg/extensions/extension_userprefs_disable.go create mode 100644 pkg/extensions/extension_userprefs_test.go create mode 100644 pkg/extensions/search/userprefs_test.go create mode 100644 test/blackbox/metadata.bats diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9ca7fa56..47e0fb37 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -39,7 +39,7 @@ jobs: # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed env: CGO_ENABLED: 0 - GOFLAGS: "-tags=sync,search,scrub,metrics,containers_image_openpgp" + GOFLAGS: "-tags=sync,search,scrub,metrics,userprefs,containers_image_openpgp" steps: - name: Checkout repository diff --git a/.github/workflows/ecosystem-tools.yaml b/.github/workflows/ecosystem-tools.yaml index 7582e7a2..fe826148 100644 --- a/.github/workflows/ecosystem-tools.yaml +++ b/.github/workflows/ecosystem-tools.yaml @@ -49,6 +49,9 @@ jobs: - name: Run referrers tests run: | make test-bats-referrers + - name: Run metadata tests + run: | + make test-bats-metadata - name: Run push-pull tests run: | make test-push-pull diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index 381f4731..25389287 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -32,7 +32,7 @@ jobs: # Optional: golangci-lint command line arguments. # args: --issues-exit-code=0 - args: --config ./golangcilint.yaml --enable-all --build-tags debug,needprivileges,sync,scrub,search,metrics,containers_image_openpgp,lint,mgmt ./cmd/... ./pkg/... + args: --config ./golangcilint.yaml --enable-all --build-tags debug,needprivileges,sync,scrub,search,userprefs,metrics,containers_image_openpgp,lint,mgmt ./cmd/... ./pkg/... # Optional: show only new issues if it's a pull request. The default value is `false`. # only-new-issues: true diff --git a/Makefile b/Makefile index 3548fae5..5d54d432 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ TESTDATA := $(TOP_LEVEL)/test/data OS ?= linux ARCH ?= amd64 BENCH_OUTPUT ?= stdout -EXTENSIONS ?= sync,search,scrub,metrics,lint,ui,mgmt +EXTENSIONS ?= sync,search,scrub,metrics,lint,ui,mgmt,userprefs comma:= , hyphen:= - extended-name:= @@ -338,6 +338,11 @@ test-bats-referrers: EXTENSIONS=search test-bats-referrers: binary check-skopeo $(BATS) $(ORAS) $(BATS) --trace --print-output-on-failure test/blackbox/referrers.bats +.PHONY: test-bats-metadata +test-bats-metadata: EXTENSIONS=search,userprefs +test-bats-metadata: binary check-skopeo $(BATS) + $(BATS) --trace --print-output-on-failure test/blackbox/metadata.bats + .PHONY: test-cloud-only test-cloud-only: binary check-skopeo $(BATS) $(BATS) --trace --print-output-on-failure test/blackbox/cloud-only.bats diff --git a/errors/errors.go b/errors/errors.go index ef6cd843..378c3b73 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -3,83 +3,92 @@ package errors import "errors" var ( - ErrBadConfig = errors.New("config: invalid config") - ErrCliBadConfig = errors.New("cli: bad config") - ErrRepoNotFound = errors.New("repository: not found") - ErrRepoIsNotDir = errors.New("repository: not a directory") - ErrRepoBadVersion = errors.New("repository: unsupported layout version") - ErrManifestNotFound = errors.New("manifest: not found") - ErrBadManifest = errors.New("manifest: invalid contents") - ErrBadIndex = errors.New("index: invalid contents") - ErrUploadNotFound = errors.New("uploads: not found") - ErrBadUploadRange = errors.New("uploads: bad range") - ErrBlobNotFound = errors.New("blob: not found") - ErrBadBlob = errors.New("blob: bad blob") - ErrBadBlobDigest = errors.New("blob: bad blob digest") - ErrUnknownCode = errors.New("error: unknown error code") - ErrBadCACert = errors.New("tls: invalid ca cert") - ErrBadUser = errors.New("auth: non-existent user") - ErrEntriesExceeded = errors.New("ldap: too many entries returned") - ErrLDAPEmptyPassphrase = errors.New("ldap: empty passphrase") - ErrLDAPBadConn = errors.New("ldap: bad connection") - ErrLDAPConfig = errors.New("config: invalid LDAP configuration") - ErrCacheRootBucket = errors.New("cache: unable to create/update root bucket") - ErrCacheNoBucket = errors.New("cache: unable to find bucket") - ErrCacheMiss = errors.New("cache: miss") - ErrRequireCred = errors.New("ldap: bind credentials required") - ErrInvalidCred = errors.New("ldap: invalid credentials") - ErrEmptyJSON = errors.New("cli: config json is empty") - ErrInvalidArgs = errors.New("cli: Invalid Arguments") - ErrInvalidFlagsCombination = errors.New("cli: Invalid combination of flags") - ErrInvalidURL = errors.New("cli: invalid URL format") - ErrUnauthorizedAccess = errors.New("auth: unauthorized access. check credentials") - ErrCannotResetConfigKey = errors.New("cli: cannot reset given config key") - ErrConfigNotFound = errors.New("cli: config with the given name does not exist") - ErrNoURLProvided = errors.New("cli: no URL provided in argument or via config") - ErrIllegalConfigKey = errors.New("cli: given config key is not allowed") - ErrScanNotSupported = errors.New("search: scanning of image media type not supported") - ErrCLITimeout = errors.New("cli: Query timed out while waiting for results") - ErrDuplicateConfigName = errors.New("cli: cli config name already added") - ErrInvalidRoute = errors.New("routes: invalid route prefix") - ErrImgStoreNotFound = errors.New("routes: image store not found corresponding to given route") - ErrEmptyValue = errors.New("cache: empty value") - ErrEmptyRepoList = errors.New("search: no repository found") - ErrCVESearchDisabled = errors.New("search: CVE search is disabled") - ErrInvalidRepositoryName = errors.New("repository: not a valid repository name") - ErrSyncMissingCatalog = errors.New("sync: couldn't fetch upstream registry's catalog") - ErrMethodNotSupported = errors.New("storage: method not supported") - ErrInvalidMetric = errors.New("metrics: invalid metric func") - ErrInjected = errors.New("test: injected failure") - ErrSyncInvalidUpstreamURL = errors.New("sync: upstream url not found in sync config") - ErrRegistryNoContent = errors.New("sync: could not find a Content that matches localRepo") - ErrSyncReferrerNotFound = errors.New("sync: couldn't find upstream referrer") - ErrSyncReferrer = errors.New("sync: failed to get upstream referrer") - ErrImageLintAnnotations = errors.New("routes: lint checks failed") - ErrParsingAuthHeader = errors.New("auth: failed parsing authorization header") - ErrBadType = errors.New("core: invalid type") - ErrParsingHTTPHeader = errors.New("routes: invalid HTTP header") - ErrBadRange = errors.New("storage: bad range") - ErrBadLayerCount = errors.New("manifest: layers count doesn't correspond to config history") - ErrManifestConflict = errors.New("manifest: multiple manifests found") - ErrManifestMetaNotFound = errors.New("repodb: image metadata not found for given manifest reference") - ErrManifestDataNotFound = errors.New("repodb: image data not found for given manifest digest") - ErrArtifactDataNotFound = errors.New("repodb: artifact data not found for given digest") - ErrIndexDataNotFount = errors.New("repodb: index data not found for given digest") - ErrRepoMetaNotFound = errors.New("repodb: repo metadata not found for given repo name") - ErrTagMetaNotFound = errors.New("repodb: tag metadata not found for given repo and tag names") - ErrTypeAssertionFailed = errors.New("storage: failed DatabaseDriver type assertion") - ErrInvalidRequestParams = errors.New("resolver: parameter sent has invalid value") - ErrOrphanSignature = errors.New("repodb: signature detected but signed image doesn't exit") - ErrBadCtxFormat = errors.New("type assertion failed") - ErrEmptyRepoName = errors.New("repodb: repo name can't be empty string") - ErrEmptyTag = errors.New("repodb: tag can't be empty string") - ErrEmptyDigest = errors.New("repodb: digest can't be empty string") - ErrInvalidRepoTagFormat = errors.New("invalid format for tag search, not following repo:tag") - ErrLimitIsNegative = errors.New("pageturner: limit has negative value") - ErrOffsetIsNegative = errors.New("pageturner: offset has negative value") - ErrSortCriteriaNotSupported = errors.New("pageturner: the sort criteria is not supported") - ErrMediaTypeNotSupported = errors.New("repodb: media type is not supported") - ErrTimeout = errors.New("operation timeout") - ErrNotImplemented = errors.New("not implemented") - ErrDedupeRebuild = errors.New("dedupe: couldn't rebuild dedupe index") + ErrBadConfig = errors.New("config: invalid config") + ErrCliBadConfig = errors.New("cli: bad config") + ErrRepoNotFound = errors.New("repository: not found") + ErrRepoIsNotDir = errors.New("repository: not a directory") + ErrRepoBadVersion = errors.New("repository: unsupported layout version") + ErrManifestNotFound = errors.New("manifest: not found") + ErrBadManifest = errors.New("manifest: invalid contents") + ErrBadIndex = errors.New("index: invalid contents") + ErrUploadNotFound = errors.New("uploads: not found") + ErrBadUploadRange = errors.New("uploads: bad range") + ErrBlobNotFound = errors.New("blob: not found") + ErrBadBlob = errors.New("blob: bad blob") + ErrBadBlobDigest = errors.New("blob: bad blob digest") + ErrUnknownCode = errors.New("error: unknown error code") + ErrBadCACert = errors.New("tls: invalid ca cert") + ErrBadUser = errors.New("auth: non-existent user") + ErrEntriesExceeded = errors.New("ldap: too many entries returned") + ErrLDAPEmptyPassphrase = errors.New("ldap: empty passphrase") + ErrLDAPBadConn = errors.New("ldap: bad connection") + ErrLDAPConfig = errors.New("config: invalid LDAP configuration") + ErrCacheRootBucket = errors.New("cache: unable to create/update root bucket") + ErrCacheNoBucket = errors.New("cache: unable to find bucket") + ErrCacheMiss = errors.New("cache: miss") + ErrRequireCred = errors.New("ldap: bind credentials required") + ErrInvalidCred = errors.New("ldap: invalid credentials") + ErrEmptyJSON = errors.New("cli: config json is empty") + ErrInvalidArgs = errors.New("cli: Invalid Arguments") + ErrInvalidFlagsCombination = errors.New("cli: Invalid combination of flags") + ErrInvalidURL = errors.New("cli: invalid URL format") + ErrUnauthorizedAccess = errors.New("auth: unauthorized access. check credentials") + ErrCannotResetConfigKey = errors.New("cli: cannot reset given config key") + ErrConfigNotFound = errors.New("cli: config with the given name does not exist") + ErrNoURLProvided = errors.New("cli: no URL provided in argument or via config") + ErrIllegalConfigKey = errors.New("cli: given config key is not allowed") + ErrScanNotSupported = errors.New("search: scanning of image media type not supported") + ErrCLITimeout = errors.New("cli: Query timed out while waiting for results") + ErrDuplicateConfigName = errors.New("cli: cli config name already added") + ErrInvalidRoute = errors.New("routes: invalid route prefix") + ErrImgStoreNotFound = errors.New("routes: image store not found corresponding to given route") + ErrEmptyValue = errors.New("cache: empty value") + ErrEmptyRepoList = errors.New("search: no repository found") + ErrCVESearchDisabled = errors.New("search: CVE search is disabled") + ErrInvalidRepositoryName = errors.New("repository: not a valid repository name") + ErrSyncMissingCatalog = errors.New("sync: couldn't fetch upstream registry's catalog") + ErrMethodNotSupported = errors.New("storage: method not supported") + ErrInvalidMetric = errors.New("metrics: invalid metric func") + ErrInjected = errors.New("test: injected failure") + ErrSyncInvalidUpstreamURL = errors.New("sync: upstream url not found in sync config") + ErrRegistryNoContent = errors.New("sync: could not find a Content that matches localRepo") + ErrSyncReferrerNotFound = errors.New("sync: couldn't find upstream referrer") + ErrSyncReferrer = errors.New("sync: failed to get upstream referrer") + ErrImageLintAnnotations = errors.New("routes: lint checks failed") + ErrParsingAuthHeader = errors.New("auth: failed parsing authorization header") + ErrBadType = errors.New("core: invalid type") + ErrParsingHTTPHeader = errors.New("routes: invalid HTTP header") + ErrBadRange = errors.New("storage: bad range") + ErrBadLayerCount = errors.New("manifest: layers count doesn't correspond to config history") + ErrManifestConflict = errors.New("manifest: multiple manifests found") + ErrManifestMetaNotFound = errors.New("repodb: image metadata not found for given manifest reference") + ErrManifestDataNotFound = errors.New("repodb: image data not found for given manifest digest") + ErrArtifactDataNotFound = errors.New("repodb: artifact data not found for given digest") + ErrIndexDataNotFount = errors.New("repodb: index data not found for given digest") + ErrRepoMetaNotFound = errors.New("repodb: repo metadata not found for given repo name") + ErrTagMetaNotFound = errors.New("repodb: tag metadata not found for given repo and tag names") + ErrTypeAssertionFailed = errors.New("storage: failed DatabaseDriver type assertion") + ErrInvalidRequestParams = errors.New("resolver: parameter sent has invalid value") + ErrOrphanSignature = errors.New("repodb: signature detected but signed image doesn't exit") + ErrBadCtxFormat = errors.New("type assertion failed") + ErrEmptyRepoName = errors.New("repodb: repo name can't be empty string") + ErrEmptyTag = errors.New("repodb: tag can't be empty string") + ErrEmptyDigest = errors.New("repodb: digest can't be empty string") + ErrInvalidRepoTagFormat = errors.New("invalid format for tag search, not following repo:tag") + ErrLimitIsNegative = errors.New("pageturner: limit has negative value") + ErrOffsetIsNegative = errors.New("pageturner: offset has negative value") + ErrSortCriteriaNotSupported = errors.New("pageturner: the sort criteria is not supported") + ErrMediaTypeNotSupported = errors.New("repodb: media type is not supported") + ErrTimeout = errors.New("operation timeout") + ErrNotImplemented = errors.New("not implemented") + ErrUnableToCreateUserBucket = errors.New("repodb: unable to create a user bucket for user") + ErrInvalidOldUserStarredRepos = errors.New("repodb: invalid old entry for user starred repos") + ErrUnmarshalledRepoListIsNil = errors.New("repodb: list of repos is still nil") + ErrCouldNotMarshalStarredRepos = errors.New("repodb: could not repack entry for user starred repos") + ErrInvalidOldUserBookmarkedRepos = errors.New("repodb: invalid old entry for user bookmarked repos") + ErrCouldNotMarshalBookmarkedRepos = errors.New("repodb: could not repack entry for user bookmarked repos") + ErrUserDataNotFound = errors.New("repodb: user data not found for given user identifier") + ErrUserDataNotAllowed = errors.New("repodb: user data operations are not allowed") + ErrCouldNotPersistData = errors.New("repodb: could not persist to db") + ErrDedupeRebuild = errors.New("dedupe: couldn't rebuild dedupe index") ) diff --git a/examples/README.md b/examples/README.md index 8ad1b96b..9b0daeaf 100644 --- a/examples/README.md +++ b/examples/README.md @@ -448,6 +448,7 @@ Additionally if search extension is enabled, additional parameters are needed: // used by search extensions "repoMetaTablename": "ZotRepoMetadataTable", "manifestDataTablename": "ZotManifestDataTable", + "userDataTablename": "ZotUserDataTable", "versionTablename": "ZotVersion" } ``` diff --git a/examples/config-dynamodb.json b/examples/config-dynamodb.json index b1ca64bc..51a2f598 100644 --- a/examples/config-dynamodb.json +++ b/examples/config-dynamodb.json @@ -21,6 +21,7 @@ "repoMetaTablename": "ZotRepoMetadataTable", "manifestDataTablename": "ZotManifestDataTable", "artifactDataTablename": "ZotArtifactDataTable", + "userDataTablename": "ZotUserDataTable", "versionTablename": "ZotVersion" } }, diff --git a/pkg/api/authz.go b/pkg/api/authz.go index 7725417b..081a975d 100644 --- a/pkg/api/authz.go +++ b/pkg/api/authz.go @@ -240,11 +240,6 @@ func AuthzHandler(ctlr *Controller) mux.MiddlewareFunc { return } - /* we want to bypass auth/authz for mgmt in case of authFail() authzFail() - unauthenticated users should have access to this route, but we also need to know if the user is an admin - */ - isMgmtRequested := request.RequestURI == constants.FullMgmtPrefix - acCtrlr := NewAccessController(ctlr.Config) var identity string @@ -279,10 +274,9 @@ func AuthzHandler(ctlr *Controller) mux.MiddlewareFunc { ctx := acCtrlr.getContext(identity, request) - // for extensions we only need to know if the user is admin and what repos he can read, so run next() - if request.RequestURI == fmt.Sprintf("%s%s", constants.RoutePrefix, constants.ExtCatalogPrefix) || - strings.Contains(request.RequestURI, constants.FullSearchPrefix) || - isMgmtRequested { + // for extensions, we only need to know the username, whether the user is an admin, and what repositories + // they can read. So, run next() + if isExtensionURI(request.RequestURI) { next.ServeHTTP(response, request.WithContext(ctx)) //nolint:contextcheck return @@ -321,6 +315,11 @@ func AuthzHandler(ctlr *Controller) mux.MiddlewareFunc { } } +func isExtensionURI(requestURI string) bool { + return strings.Contains(requestURI, constants.ExtPrefix) || + requestURI == fmt.Sprintf("%s%s", constants.RoutePrefix, constants.ExtCatalogPrefix) +} + func authzFail(w http.ResponseWriter, realm string, delay int) { time.Sleep(time.Duration(delay) * time.Second) w.Header().Set("WWW-Authenticate", realm) diff --git a/pkg/api/constants/extensions.go b/pkg/api/constants/extensions.go index 69cb2a80..72592f70 100644 --- a/pkg/api/constants/extensions.go +++ b/pkg/api/constants/extensions.go @@ -5,8 +5,11 @@ const ( ExtCatalogPrefix = "/_catalog" ExtOciDiscoverPrefix = "/_oci/ext/discover" // zot specific extensions. - ExtSearchPrefix = "/_zot/ext/search" - FullSearchPrefix = RoutePrefix + ExtSearchPrefix - ExtMgmtPrefix = "/_zot/ext/mgmt" - FullMgmtPrefix = RoutePrefix + ExtMgmtPrefix + ExtPrefix = "/_zot/ext" + ExtSearchPrefix = ExtPrefix + "/search" + FullSearchPrefix = RoutePrefix + ExtSearchPrefix + ExtMgmtPrefix = ExtPrefix + "/mgmt" + FullMgmtPrefix = RoutePrefix + ExtMgmtPrefix + ExtUserPreferencesPrefix = ExtPrefix + "/userprefs" + FullUserPreferencesPrefix = RoutePrefix + ExtUserPreferencesPrefix ) diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 618a5734..be46974b 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -158,6 +158,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { "repoMetaTablename": "RepoMetadataTable", "manifestDataTablename": "ManifestDataTable", "artifactDataTablename": "ArtifactDataTable", + "userDataTablename": "ZotUserDataTable", "versionTablename": "Version", } @@ -173,6 +174,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { "repoMetaTablename": "RepoMetadataTable", "manifestDataTablename": "ManifestDataTable", "artifactDataTablename": "ArtifactDataTable", + "userDataTablename": "ZotUserDataTable", "versionTablename": "Version", } @@ -187,6 +189,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { "repoMetaTablename": "RepoMetadataTable", "manifestDataTablename": "ManifestDataTable", "artifactDataTablename": "ArtifactDataTable", + "userDataTablename": "ZotUserDataTable", "versionTablename": "Version", } @@ -220,6 +223,7 @@ func TestCreateRepoDBDriver(t *testing.T) { "repometatablename": "RepoMetadataTable", "manifestdatatablename": "ManifestDataTable", "artifactDataTablename": "ArtifactDataTable", + "userdatatablename": "UserDatatable", } testFunc := func() { _, _ = repodbfactory.New(conf.Storage.StorageConfig, log) } @@ -233,6 +237,7 @@ func TestCreateRepoDBDriver(t *testing.T) { "repometatablename": "RepoMetadataTable", "manifestdatatablename": "ManifestDataTable", "artifactDataTablename": "ArtifactDataTable", + "userDataTablename": "ZotUserDataTable", "versiontablename": 1, } @@ -248,6 +253,7 @@ func TestCreateRepoDBDriver(t *testing.T) { "manifestdatatablename": "ManifestDataTable", "indexdatatablename": "IndexDataTable", "artifactdatatablename": "ArtifactDataTable", + "userdatatablename": "ZotUserDataTable", "versiontablename": "1", } diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 40119782..bddaffc7 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -127,6 +127,8 @@ func (rh *RouteHandler) SetupRoutes() { // extended build ext.SetupMetricsRoutes(rh.c.Config, rh.c.Router, rh.c.StoreController, AuthHandler(rh.c), rh.c.Log) ext.SetupSearchRoutes(rh.c.Config, prefixedRouter, rh.c.StoreController, rh.c.RepoDB, rh.c.CveInfo, rh.c.Log) + ext.SetupUserPreferencesRoutes(rh.c.Config, prefixedRouter, rh.c.StoreController, rh.c.RepoDB, rh.c.CveInfo, + rh.c.Log) ext.SetupUIRoutes(rh.c.Config, rh.c.Router, rh.c.StoreController, rh.c.Log) ext.SetupMgmtRoutes(rh.c.Config, prefixedRouter, rh.c.Log) gqlPlayground.SetupGQLPlaygroundRoutes(rh.c.Config, prefixedRouter, rh.c.StoreController, rh.c.Log) diff --git a/pkg/common/common.go b/pkg/common/common.go index 7ba87d6e..f8414d18 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -40,6 +40,30 @@ func Contains(slice []string, item string) bool { return false } +// first match of item in []. +func Index(slice []string, item string) int { + for k, v := range slice { + if item == v { + return k + } + } + + return -1 +} + +// remove matches of item in []. +func RemoveFrom(inputSlice []string, item string) []string { + var newSlice []string + + for _, v := range inputSlice { + if item != v { + newSlice = append(newSlice, v) + } + } + + return newSlice +} + func GetTLSConfig(certsPath string, caCertPool *x509.CertPool) (*tls.Config, error) { clientCert := filepath.Join(certsPath, clientCertFilename) clientKey := filepath.Join(certsPath, clientKeyFilename) diff --git a/pkg/common/common_test.go b/pkg/common/common_test.go index ea8248ab..87cb9f88 100644 --- a/pkg/common/common_test.go +++ b/pkg/common/common_test.go @@ -119,6 +119,11 @@ func TestCommon(t *testing.T) { resultPtr, baseURL+"/v2/", ispec.MediaTypeImageManifest, log.NewLogger("", "")) So(err, ShouldNotBeNil) }) + + Convey("Index func", t, func() { + So(common.Index([]string{"a", "b"}, "b"), ShouldEqual, 1) + So(common.Index([]string{"a", "b"}, "c"), ShouldEqual, -1) + }) Convey("Test image dir and digest", t, func() { repo, digest := common.GetImageDirAndDigest("image") So(repo, ShouldResemble, "image") diff --git a/pkg/common/model.go b/pkg/common/model.go index a83efaed..c31ed397 100644 --- a/pkg/common/model.go +++ b/pkg/common/model.go @@ -10,12 +10,15 @@ type RepoInfo struct { } type RepoSummary struct { - Name string `json:"name"` - LastUpdated time.Time `json:"lastUpdated"` - Size string `json:"size"` - Platforms []Platform `json:"platforms"` - Vendors []string `json:"vendors"` - NewestImage ImageSummary `json:"newestImage"` + Name string `json:"name"` + LastUpdated time.Time `json:"lastUpdated"` + Size string `json:"size"` + Platforms []Platform `json:"platforms"` + Vendors []string `json:"vendors"` + IsStarred bool `json:"isStarred"` + IsBookmarked bool `json:"isBookmarked"` + StarCount int `json:"starCount"` + NewestImage ImageSummary `json:"newestImage"` } type ImageSummary struct { diff --git a/pkg/extensions/extension_userprefs.go b/pkg/extensions/extension_userprefs.go new file mode 100644 index 00000000..ca4a27fd --- /dev/null +++ b/pkg/extensions/extension_userprefs.go @@ -0,0 +1,143 @@ +//go:build userprefs +// +build userprefs + +package extensions + +import ( + "errors" + "net/http" + "net/url" + + "github.com/gorilla/mux" + + zerr "zotregistry.io/zot/errors" + "zotregistry.io/zot/pkg/api/config" + "zotregistry.io/zot/pkg/api/constants" + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/repodb" + "zotregistry.io/zot/pkg/storage" +) + +const ( + ToggleRepoBookmarkAction = "toggleBookmark" + ToggleRepoStarAction = "toggleStar" +) + +func SetupUserPreferencesRoutes(config *config.Config, router *mux.Router, storeController storage.StoreController, + repoDB repodb.RepoDB, cveInfo CveInfo, log log.Logger, +) { + if config.Extensions.Search != nil && *config.Extensions.Search.Enable { + log.Info().Msg("setting up user preferences routes") + + userprefsRouter := router.PathPrefix(constants.ExtUserPreferencesPrefix).Subrouter() + + userprefsRouter.HandleFunc("", HandleUserPrefs(repoDB, log)).Methods(http.MethodPut) + } +} + +func HandleUserPrefs(repoDB repodb.RepoDB, log log.Logger) func(w http.ResponseWriter, r *http.Request) { + return func(rsp http.ResponseWriter, req *http.Request) { + if !queryHasParams(req.URL.Query(), []string{"action"}) { + rsp.WriteHeader(http.StatusBadRequest) + + return + } + + action := req.URL.Query().Get("action") + + switch action { + case ToggleRepoBookmarkAction: + PutBookmark(rsp, req, repoDB, log) //nolint:contextcheck + + return + case ToggleRepoStarAction: + PutStar(rsp, req, repoDB, log) //nolint:contextcheck + + return + default: + rsp.WriteHeader(http.StatusBadRequest) + + return + } + } +} + +func PutStar(rsp http.ResponseWriter, req *http.Request, repoDB repodb.RepoDB, log log.Logger) { + if !queryHasParams(req.URL.Query(), []string{"repo"}) { + rsp.WriteHeader(http.StatusBadRequest) + + return + } + + repo := req.URL.Query().Get("repo") + + if repo == "" { + rsp.WriteHeader(http.StatusNotFound) + + return + } + + _, err := repoDB.ToggleStarRepo(req.Context(), repo) + if err != nil { + if errors.Is(err, zerr.ErrRepoMetaNotFound) { + rsp.WriteHeader(http.StatusNotFound) + + return + } else if errors.Is(err, zerr.ErrUserDataNotAllowed) { + rsp.WriteHeader(http.StatusForbidden) + + return + } + + rsp.WriteHeader(http.StatusInternalServerError) + + return + } + + rsp.WriteHeader(http.StatusOK) +} + +func PutBookmark(rsp http.ResponseWriter, req *http.Request, repoDB repodb.RepoDB, log log.Logger) { + if !queryHasParams(req.URL.Query(), []string{"repo"}) { + rsp.WriteHeader(http.StatusBadRequest) + + return + } + + repo := req.URL.Query().Get("repo") + + if repo == "" { + rsp.WriteHeader(http.StatusNotFound) + + return + } + + _, err := repoDB.ToggleBookmarkRepo(req.Context(), repo) + if err != nil { + if errors.Is(err, zerr.ErrRepoMetaNotFound) { + rsp.WriteHeader(http.StatusNotFound) + + return + } else if errors.Is(err, zerr.ErrUserDataNotAllowed) { + rsp.WriteHeader(http.StatusForbidden) + + return + } + + rsp.WriteHeader(http.StatusInternalServerError) + + return + } + + rsp.WriteHeader(http.StatusOK) +} + +func queryHasParams(values url.Values, params []string) bool { + for _, param := range params { + if !values.Has(param) { + return false + } + } + + return true +} diff --git a/pkg/extensions/extension_userprefs_disable.go b/pkg/extensions/extension_userprefs_disable.go new file mode 100644 index 00000000..29b16a42 --- /dev/null +++ b/pkg/extensions/extension_userprefs_disable.go @@ -0,0 +1,20 @@ +//go:build !userprefs +// +build !userprefs + +package extensions + +import ( + "github.com/gorilla/mux" + + "zotregistry.io/zot/pkg/api/config" + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/repodb" + "zotregistry.io/zot/pkg/storage" +) + +func SetupUserPreferencesRoutes(config *config.Config, router *mux.Router, storeController storage.StoreController, + repoDB repodb.RepoDB, cveInfo CveInfo, log log.Logger, +) { + log.Warn().Msg("userprefs extension is disabled because given zot binary doesn't" + + "include this feature please build a binary that does so") +} diff --git a/pkg/extensions/extension_userprefs_test.go b/pkg/extensions/extension_userprefs_test.go new file mode 100644 index 00000000..37b7d113 --- /dev/null +++ b/pkg/extensions/extension_userprefs_test.go @@ -0,0 +1,139 @@ +//go:build userprefs +// +build userprefs + +package extensions_test + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gorilla/mux" + . "github.com/smartystreets/goconvey/convey" + + zerr "zotregistry.io/zot/errors" + "zotregistry.io/zot/pkg/extensions" + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/repodb" + "zotregistry.io/zot/pkg/test/mocks" +) + +var ErrTestError = errors.New("TestError") + +const UserprefsBaseURL = "http://127.0.0.1:8080/v2/_zot/ext/userprefs" + +func TestHandlers(t *testing.T) { + log := log.NewLogger("debug", "") + mockrepoDB := mocks.RepoDBMock{} + + Convey("No repo in request", t, func() { + request := httptest.NewRequest("GET", UserprefsBaseURL+"", strings.NewReader("My string")) + response := httptest.NewRecorder() + + extensions.PutStar(response, request, mockrepoDB, log) + res := response.Result() + So(res.StatusCode, ShouldEqual, http.StatusBadRequest) + defer res.Body.Close() + + extensions.PutBookmark(response, request, mockrepoDB, log) + res = response.Result() + So(res.StatusCode, ShouldEqual, http.StatusBadRequest) + defer res.Body.Close() + }) + + Convey("Empty repo in request", t, func() { + request := httptest.NewRequest("GET", UserprefsBaseURL+"?repo=", strings.NewReader("My string")) + response := httptest.NewRecorder() + + extensions.PutStar(response, request, mockrepoDB, log) + res := response.Result() + So(res.StatusCode, ShouldEqual, http.StatusNotFound) + defer res.Body.Close() + + extensions.PutBookmark(response, request, mockrepoDB, log) + res = response.Result() + So(res.StatusCode, ShouldEqual, http.StatusNotFound) + defer res.Body.Close() + }) + + Convey("ToggleStarRepo different errors", t, func() { + request := httptest.NewRequest("GET", UserprefsBaseURL+"?repo=test", + strings.NewReader("My string")) + + Convey("ErrRepoMetaNotFound", func() { + mockrepoDB.ToggleStarRepoFn = func(ctx context.Context, repo string) (repodb.ToggleState, error) { + return repodb.NotChanged, zerr.ErrRepoMetaNotFound + } + + mockrepoDB.ToggleBookmarkRepoFn = func(ctx context.Context, repo string) (repodb.ToggleState, error) { + return repodb.NotChanged, zerr.ErrRepoMetaNotFound + } + + response := httptest.NewRecorder() + extensions.PutBookmark(response, request, mockrepoDB, log) + res := response.Result() + So(res.StatusCode, ShouldEqual, http.StatusNotFound) + defer res.Body.Close() + + response = httptest.NewRecorder() + extensions.PutStar(response, request, mockrepoDB, log) + res = response.Result() + So(res.StatusCode, ShouldEqual, http.StatusNotFound) + defer res.Body.Close() + }) + + Convey("ErrUserDataNotAllowed", func() { + request = mux.SetURLVars(request, map[string]string{ + "name": "repo", + }) + + mockrepoDB.ToggleBookmarkRepoFn = func(ctx context.Context, repo string) (repodb.ToggleState, error) { + return repodb.NotChanged, zerr.ErrUserDataNotAllowed + } + + mockrepoDB.ToggleStarRepoFn = func(ctx context.Context, repo string) (repodb.ToggleState, error) { + return repodb.NotChanged, zerr.ErrUserDataNotAllowed + } + + response := httptest.NewRecorder() + extensions.PutBookmark(response, request, mockrepoDB, log) + res := response.Result() + So(res.StatusCode, ShouldEqual, http.StatusForbidden) + defer res.Body.Close() + + response = httptest.NewRecorder() + extensions.PutStar(response, request, mockrepoDB, log) + res = response.Result() + So(res.StatusCode, ShouldEqual, http.StatusForbidden) + defer res.Body.Close() + }) + + Convey("ErrUnexpectedError", func() { + request = mux.SetURLVars(request, map[string]string{ + "name": "repo", + }) + + mockrepoDB.ToggleBookmarkRepoFn = func(ctx context.Context, repo string) (repodb.ToggleState, error) { + return repodb.NotChanged, ErrTestError + } + + mockrepoDB.ToggleStarRepoFn = func(ctx context.Context, repo string) (repodb.ToggleState, error) { + return repodb.NotChanged, ErrTestError + } + response := httptest.NewRecorder() + extensions.PutBookmark(response, request, mockrepoDB, log) + res := response.Result() + So(res.StatusCode, ShouldEqual, http.StatusInternalServerError) + defer res.Body.Close() + + response = httptest.NewRecorder() + extensions.PutStar(response, request, mockrepoDB, log) + res = response.Result() + So(res.StatusCode, ShouldEqual, http.StatusInternalServerError) + defer res.Body.Close() + }) + }) +} diff --git a/pkg/extensions/search/convert/repodb.go b/pkg/extensions/search/convert/repodb.go index 3383e441..d42c7992 100644 --- a/pkg/extensions/search/convert/repodb.go +++ b/pkg/extensions/search/convert/repodb.go @@ -31,15 +31,15 @@ func RepoMeta2RepoSummary(ctx context.Context, repoMeta repodb.RepoMetadata, skip SkipQGLField, cveInfo cveinfo.CveInfo, ) *gql_generated.RepoSummary { var ( + repoName = repoMeta.Name repoLastUpdatedTimestamp = time.Time{} repoPlatformsSet = map[string]*gql_generated.Platform{} repoVendorsSet = map[string]bool{} lastUpdatedImageSummary *gql_generated.ImageSummary - repoStarCount = repoMeta.Stars - isBookmarked = false - isStarred = false repoDownloadCount = 0 - repoName = repoMeta.Name + repoStarCount = repoMeta.Stars // total number of stars + repoIsUserStarred = repoMeta.IsStarred // value specific to the current user + repoIsUserBookMarked = repoMeta.IsBookmarked // value specific to the current user // map used to keep track of all blobs of a repo without dublicates as // some images may have the same layers @@ -88,6 +88,7 @@ func RepoMeta2RepoSummary(ctx context.Context, repoMeta repodb.RepoMetadata, repoSize := strconv.FormatInt(size, 10) repoPlatforms := make([]*gql_generated.Platform, 0, len(repoPlatformsSet)) + for _, platform := range repoPlatformsSet { repoPlatforms = append(repoPlatforms, platform) } @@ -129,8 +130,8 @@ func RepoMeta2RepoSummary(ctx context.Context, repoMeta repodb.RepoMetadata, NewestImage: lastUpdatedImageSummary, DownloadCount: &repoDownloadCount, StarCount: &repoStarCount, - IsBookmarked: &isBookmarked, - IsStarred: &isStarred, + IsBookmarked: &repoIsUserBookMarked, + IsStarred: &repoIsUserStarred, } } @@ -574,15 +575,15 @@ func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta repodb.RepoMetadata skip SkipQGLField, cveInfo cveinfo.CveInfo, log log.Logger, ) (*gql_generated.RepoSummary, []*gql_generated.ImageSummary) { var ( + repoName = repoMeta.Name repoLastUpdatedTimestamp = time.Time{} repoPlatformsSet = map[string]*gql_generated.Platform{} repoVendorsSet = map[string]bool{} lastUpdatedImageSummary *gql_generated.ImageSummary - repoStarCount = repoMeta.Stars - isBookmarked = false - isStarred = false repoDownloadCount = 0 - repoName = repoMeta.Name + repoStarCount = repoMeta.Stars // total number of stars + isStarred = repoMeta.IsStarred // value specific to the current user + isBookmarked = repoMeta.IsBookmarked // value specific to the current user // map used to keep track of all blobs of a repo without dublicates as // some images may have the same layers @@ -632,6 +633,7 @@ func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta repodb.RepoMetadata repoSize := strconv.FormatInt(size, 10) repoPlatforms := make([]*gql_generated.Platform, 0, len(repoPlatformsSet)) + for _, platform := range repoPlatformsSet { repoPlatforms = append(repoPlatforms, platform) } diff --git a/pkg/extensions/search/digest_test.go b/pkg/extensions/search/digest_test.go index b5f04336..806a563a 100644 --- a/pkg/extensions/search/digest_test.go +++ b/pkg/extensions/search/digest_test.go @@ -86,7 +86,8 @@ func TestDigestSearchHTTP(t *testing.T) { ) So(err, ShouldBeNil) - image1.Reference = "0.0.1" + const ver001 = "0.0.1" + image1.Reference = ver001 err = UploadImage( image1, baseURL, @@ -109,7 +110,7 @@ func TestDigestSearchHTTP(t *testing.T) { ) So(err, ShouldBeNil) - image2.Reference = "0.0.1" + image2.Reference = ver001 manifestDigest, err := image2.Digest() So(err, ShouldBeNil) diff --git a/pkg/extensions/search/gql_generated/generated.go b/pkg/extensions/search/gql_generated/generated.go index 7c5ba28b..3f5892cd 100644 --- a/pkg/extensions/search/gql_generated/generated.go +++ b/pkg/extensions/search/gql_generated/generated.go @@ -156,6 +156,7 @@ type ComplexityRoot struct { Query struct { BaseImageList func(childComplexity int, image string, digest *string, requestedPage *PageInput) int + BookmarkedRepos func(childComplexity int, requestedPage *PageInput) int CVEListForImage func(childComplexity int, image string, requestedPage *PageInput, searchedCve *string) int DerivedImageList func(childComplexity int, image string, digest *string, requestedPage *PageInput) int ExpandedRepoInfo func(childComplexity int, repo string) int @@ -167,6 +168,7 @@ type ComplexityRoot struct { ImageListWithCVEFixed func(childComplexity int, id string, image string, requestedPage *PageInput) int Referrers func(childComplexity int, repo string, digest string, typeArg []string) int RepoListWithNewestImage func(childComplexity int, requestedPage *PageInput) int + StarredRepos func(childComplexity int, requestedPage *PageInput) int } Referrer struct { @@ -209,6 +211,8 @@ type QueryResolver interface { BaseImageList(ctx context.Context, image string, digest *string, requestedPage *PageInput) (*PaginatedImagesResult, error) Image(ctx context.Context, image string) (*ImageSummary, error) Referrers(ctx context.Context, repo string, digest string, typeArg []string) ([]*Referrer, error) + StarredRepos(ctx context.Context, requestedPage *PageInput) (*PaginatedReposResult, error) + BookmarkedRepos(ctx context.Context, requestedPage *PageInput) (*PaginatedReposResult, error) } type executableSchema struct { @@ -700,6 +704,18 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Query.BaseImageList(childComplexity, args["image"].(string), args["digest"].(*string), args["requestedPage"].(*PageInput)), true + case "Query.BookmarkedRepos": + if e.complexity.Query.BookmarkedRepos == nil { + break + } + + args, err := ec.field_Query_BookmarkedRepos_args(context.TODO(), rawArgs) + if err != nil { + return 0, false + } + + return e.complexity.Query.BookmarkedRepos(childComplexity, args["requestedPage"].(*PageInput)), true + case "Query.CVEListForImage": if e.complexity.Query.CVEListForImage == nil { break @@ -832,6 +848,18 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return e.complexity.Query.RepoListWithNewestImage(childComplexity, args["requestedPage"].(*PageInput)), true + case "Query.StarredRepos": + if e.complexity.Query.StarredRepos == nil { + break + } + + args, err := ec.field_Query_StarredRepos_args(context.TODO(), rawArgs) + if err != nil { + return 0, false + } + + return e.complexity.Query.StarredRepos(childComplexity, args["requestedPage"].(*PageInput)), true + case "Referrer.Annotations": if e.complexity.Referrer.Annotations == nil { break @@ -1688,6 +1716,22 @@ type Query { "Types of artifacts to return in the referrer list" type: [String!] ): [Referrer]! + + """ + Receive RepoSummaries of repos starred by current user + """ + StarredRepos( + "Sets the parameters of the requested page (how many to include and offset)" + requestedPage: PageInput + ): PaginatedReposResult! + + """ + Receive RepoSummaries of repos bookmarked by current user + """ + BookmarkedRepos( + "Sets the parameters of the requested page (how many to include and offset)" + requestedPage: PageInput + ): PaginatedReposResult! } `, BuiltIn: false}, } @@ -1730,6 +1774,21 @@ func (ec *executionContext) field_Query_BaseImageList_args(ctx context.Context, return args, nil } +func (ec *executionContext) field_Query_BookmarkedRepos_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { + var err error + args := map[string]interface{}{} + var arg0 *PageInput + if tmp, ok := rawArgs["requestedPage"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("requestedPage")) + arg0, err = ec.unmarshalOPageInput2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐPageInput(ctx, tmp) + if err != nil { + return nil, err + } + } + args["requestedPage"] = arg0 + return args, nil +} + func (ec *executionContext) field_Query_CVEListForImage_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { var err error args := map[string]interface{}{} @@ -2012,6 +2071,21 @@ func (ec *executionContext) field_Query_RepoListWithNewestImage_args(ctx context return args, nil } +func (ec *executionContext) field_Query_StarredRepos_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { + var err error + args := map[string]interface{}{} + var arg0 *PageInput + if tmp, ok := rawArgs["requestedPage"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("requestedPage")) + arg0, err = ec.unmarshalOPageInput2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐPageInput(ctx, tmp) + if err != nil { + return nil, err + } + } + args["requestedPage"] = arg0 + return args, nil +} + func (ec *executionContext) field_Query___type_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { var err error args := map[string]interface{}{} @@ -5831,6 +5905,128 @@ func (ec *executionContext) fieldContext_Query_Referrers(ctx context.Context, fi return fc, nil } +func (ec *executionContext) _Query_StarredRepos(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Query_StarredRepos(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return ec.resolvers.Query().StarredRepos(rctx, fc.Args["requestedPage"].(*PageInput)) + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(*PaginatedReposResult) + fc.Result = res + return ec.marshalNPaginatedReposResult2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐPaginatedReposResult(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_Query_StarredRepos(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "Query", + Field: field, + IsMethod: true, + IsResolver: true, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "Page": + return ec.fieldContext_PaginatedReposResult_Page(ctx, field) + case "Results": + return ec.fieldContext_PaginatedReposResult_Results(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type PaginatedReposResult", field.Name) + }, + } + defer func() { + if r := recover(); r != nil { + err = ec.Recover(ctx, r) + ec.Error(ctx, err) + } + }() + ctx = graphql.WithFieldContext(ctx, fc) + if fc.Args, err = ec.field_Query_StarredRepos_args(ctx, field.ArgumentMap(ec.Variables)); err != nil { + ec.Error(ctx, err) + return + } + return fc, nil +} + +func (ec *executionContext) _Query_BookmarkedRepos(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { + fc, err := ec.fieldContext_Query_BookmarkedRepos(ctx, field) + if err != nil { + return graphql.Null + } + ctx = graphql.WithFieldContext(ctx, fc) + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = graphql.Null + } + }() + resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { + ctx = rctx // use context from middleware stack in children + return ec.resolvers.Query().BookmarkedRepos(rctx, fc.Args["requestedPage"].(*PageInput)) + }) + if err != nil { + ec.Error(ctx, err) + return graphql.Null + } + if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } + return graphql.Null + } + res := resTmp.(*PaginatedReposResult) + fc.Result = res + return ec.marshalNPaginatedReposResult2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐPaginatedReposResult(ctx, field.Selections, res) +} + +func (ec *executionContext) fieldContext_Query_BookmarkedRepos(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { + fc = &graphql.FieldContext{ + Object: "Query", + Field: field, + IsMethod: true, + IsResolver: true, + Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) { + switch field.Name { + case "Page": + return ec.fieldContext_PaginatedReposResult_Page(ctx, field) + case "Results": + return ec.fieldContext_PaginatedReposResult_Results(ctx, field) + } + return nil, fmt.Errorf("no field named %q was found under type PaginatedReposResult", field.Name) + }, + } + defer func() { + if r := recover(); r != nil { + err = ec.Recover(ctx, r) + ec.Error(ctx, err) + } + }() + ctx = graphql.WithFieldContext(ctx, fc) + if fc.Args, err = ec.field_Query_BookmarkedRepos_args(ctx, field.ArgumentMap(ec.Variables)); err != nil { + ec.Error(ctx, err) + return + } + return fc, nil +} + func (ec *executionContext) _Query___type(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) { fc, err := ec.fieldContext_Query___type(ctx, field) if err != nil { @@ -9526,6 +9722,52 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc) } + out.Concurrently(i, func() graphql.Marshaler { + return rrm(innerCtx) + }) + case "StarredRepos": + field := field + + innerFunc := func(ctx context.Context) (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Query_StarredRepos(ctx, field) + if res == graphql.Null { + atomic.AddUint32(&invalids, 1) + } + return res + } + + rrm := func(ctx context.Context) graphql.Marshaler { + return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc) + } + + out.Concurrently(i, func() graphql.Marshaler { + return rrm(innerCtx) + }) + case "BookmarkedRepos": + field := field + + innerFunc := func(ctx context.Context) (res graphql.Marshaler) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + } + }() + res = ec._Query_BookmarkedRepos(ctx, field) + if res == graphql.Null { + atomic.AddUint32(&invalids, 1) + } + return res + } + + rrm := func(ctx context.Context) graphql.Marshaler { + return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc) + } + out.Concurrently(i, func() graphql.Marshaler { return rrm(innerCtx) }) diff --git a/pkg/extensions/search/resolver.go b/pkg/extensions/search/resolver.go index 581f4e35..402c0fbe 100644 --- a/pkg/extensions/search/resolver.go +++ b/pkg/extensions/search/resolver.go @@ -18,7 +18,7 @@ import ( "github.com/vektah/gqlparser/v2/gqlerror" zerr "zotregistry.io/zot/errors" - "zotregistry.io/zot/pkg/common" + zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/search/convert" cveinfo "zotregistry.io/zot/pkg/extensions/search/cve" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" @@ -300,7 +300,7 @@ func getCVEListForImage( ), } - repo, ref, isTag := common.GetImageDirAndReference(image) + repo, ref, isTag := zcommon.GetImageDirAndReference(image) if ref == "" { return &gql_generated.CVEResultForImage{}, gqlerror.Errorf("no reference provided") @@ -560,6 +560,91 @@ func repoListWithNewestImage( return paginatedRepos, nil } +func getBookmarkedRepos( + ctx context.Context, + cveInfo cveinfo.CveInfo, + log log.Logger, //nolint:unparam // may be used by devs for debugging + requestedPage *gql_generated.PageInput, + repoDB repodb.RepoDB, +) (*gql_generated.PaginatedReposResult, error) { + repoNames, err := repoDB.GetBookmarkedRepos(ctx) + if err != nil { + return &gql_generated.PaginatedReposResult{}, err + } + + filterFn := func(repoMeta repodb.RepoMetadata) bool { + return zcommon.Contains(repoNames, repoMeta.Name) + } + + return getFilteredPaginatedRepos(ctx, cveInfo, filterFn, log, requestedPage, repoDB) +} + +func getStarredRepos( + ctx context.Context, + cveInfo cveinfo.CveInfo, + log log.Logger, //nolint:unparam // may be used by devs for debugging + requestedPage *gql_generated.PageInput, + repoDB repodb.RepoDB, +) (*gql_generated.PaginatedReposResult, error) { + repoNames, err := repoDB.GetStarredRepos(ctx) + if err != nil { + return &gql_generated.PaginatedReposResult{}, err + } + + filterFn := func(repoMeta repodb.RepoMetadata) bool { + return zcommon.Contains(repoNames, repoMeta.Name) + } + + return getFilteredPaginatedRepos(ctx, cveInfo, filterFn, log, requestedPage, repoDB) +} + +func getFilteredPaginatedRepos( + ctx context.Context, + cveInfo cveinfo.CveInfo, + filterFn repodb.FilterRepoFunc, + log log.Logger, //nolint:unparam // may be used by devs for debugging + requestedPage *gql_generated.PageInput, + repoDB repodb.RepoDB, +) (*gql_generated.PaginatedReposResult, error) { + repos := []*gql_generated.RepoSummary{} + paginatedRepos := &gql_generated.PaginatedReposResult{} + + if requestedPage == nil { + requestedPage = &gql_generated.PageInput{} + } + + skip := convert.SkipQGLField{ + Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Results.NewestImage.Vulnerabilities"), + } + + pageInput := repodb.PageInput{ + Limit: safeDerefferencing(requestedPage.Limit, 0), + Offset: safeDerefferencing(requestedPage.Offset, 0), + SortBy: repodb.SortCriteria( + safeDerefferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), + ), + } + + reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := repoDB.FilterRepos(ctx, filterFn, pageInput) + if err != nil { + return paginatedRepos, err + } + + for _, repoMeta := range reposMeta { + repoSummary := convert.RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, indexDataMap, + skip, cveInfo) + repos = append(repos, repoSummary) + } + + paginatedRepos.Page = &gql_generated.PageInfo{ + TotalCount: pageInfo.TotalCount, + ItemCount: pageInfo.ItemCount, + } + paginatedRepos.Results = repos + + return paginatedRepos, nil +} + func globalSearch(ctx context.Context, query string, repoDB repodb.RepoDB, filter *gql_generated.Filter, requestedPage *gql_generated.PageInput, cveInfo cveinfo.CveInfo, log log.Logger, //nolint:unparam ) (*gql_generated.PaginatedReposResult, []*gql_generated.ImageSummary, []*gql_generated.LayerSummary, error, @@ -675,7 +760,7 @@ func derivedImageList(ctx context.Context, image string, digest *string, repoDB Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Vulnerabilities"), } - imageRepo, imageTag := common.GetImageDirAndTag(image) + imageRepo, imageTag := zcommon.GetImageDirAndTag(image) if imageTag == "" { return &gql_generated.PaginatedImagesResult{}, gqlerror.Errorf("no reference provided") } @@ -788,7 +873,7 @@ func baseImageList(ctx context.Context, image string, digest *string, repoDB rep Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Vulnerabilities"), } - imageRepo, imageTag := common.GetImageDirAndTag(image) + imageRepo, imageTag := zcommon.GetImageDirAndTag(image) if imageTag == "" { return &gql_generated.PaginatedImagesResult{}, gqlerror.Errorf("no reference provided") diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index 5da554b2..003cc965 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -22,6 +22,7 @@ import ( "zotregistry.io/zot/pkg/meta/bolt" "zotregistry.io/zot/pkg/meta/repodb" boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" + localCtx "zotregistry.io/zot/pkg/requestcontext" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/test/mocks" ) @@ -418,7 +419,7 @@ func TestRepoListWithNewestImage(t *testing.T) { So(repos.Results, ShouldBeEmpty) }) - Convey("RepoDB SearchRepo Bad manifest referenced", func() { + Convey("RepoDB SearchRepo bad manifest referenced", func() { mockRepoDB := mocks.RepoDBMock{ SearchReposFn: func(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) { @@ -546,8 +547,8 @@ func TestRepoListWithNewestImage(t *testing.T) { for _, repoMeta := range repos { pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repoMeta, - UpdateTime: createTime, + RepoMetadata: repoMeta, + UpdateTime: createTime, }) createTime = createTime.Add(time.Second) } @@ -622,6 +623,68 @@ func TestRepoListWithNewestImage(t *testing.T) { }) } +func TestGetBookmarkedRepos(t *testing.T) { + Convey("getBookmarkedRepos", t, func() { + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + _, err := getBookmarkedRepos( + responseContext, + mocks.CveInfoMock{}, + log.NewLogger("debug", ""), + nil, + mocks.RepoDBMock{ + GetBookmarkedReposFn: func(ctx context.Context) ([]string, error) { + return []string{}, ErrTestError + }, + }, + ) + So(err, ShouldNotBeNil) + }) +} + +func TestGetStarredRepos(t *testing.T) { + Convey("getStarredRepos", t, func() { + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + _, err := getStarredRepos( + responseContext, + mocks.CveInfoMock{}, + log.NewLogger("debug", ""), + nil, + mocks.RepoDBMock{ + GetStarredReposFn: func(ctx context.Context) ([]string, error) { + return []string{}, ErrTestError + }, + }, + ) + So(err, ShouldNotBeNil) + }) +} + +func TestGetFilteredPaginatedRepos(t *testing.T) { + Convey("getFilteredPaginatedRepos FilterRepos fails", t, func() { + responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + _, err := getFilteredPaginatedRepos( + responseContext, + mocks.CveInfoMock{}, + func(repoMeta repodb.RepoMetadata) bool { return true }, + log.NewLogger("debug", ""), + nil, + mocks.RepoDBMock{ + FilterReposFn: func(ctx context.Context, filter repodb.FilterRepoFunc, requestedPage repodb.PageInput, + ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, + error, + ) { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, + repodb.PageInfo{}, ErrTestError + }, + }, + ) + So(err, ShouldNotBeNil) + }) +} + func TestImageListForDigest(t *testing.T) { Convey("getImageList", t, func() { Convey("no page requested, FilterTagsFn returns error", func() { @@ -1028,7 +1091,7 @@ func TestImageListForDigest(t *testing.T) { repos[i].Tags = matchedTags pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repo, + RepoMetadata: repo, }) } @@ -2486,6 +2549,28 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo So(len(images.Results), ShouldEqual, 0) }) }) + + Convey("Errors for cve resolvers", t, func() { + _, err := getImageListForCVE( + context.Background(), + "id", + mocks.CveInfoMock{ + GetImageListForCVEFn: func(repo, cveID string) ([]cvemodel.TagInfo, error) { + return []cvemodel.TagInfo{}, ErrTestError + }, + }, + nil, + mocks.RepoDBMock{ + GetMultipleRepoMetaFn: func(ctx context.Context, filter func(repoMeta repodb.RepoMetadata) bool, + requestedPage repodb.PageInput, + ) ([]repodb.RepoMetadata, error) { + return []repodb.RepoMetadata{{}}, nil + }, + }, + log, + ) + So(err, ShouldNotBeNil) + }) } func getPageInput(limit int, offset int) *gql_generated.PageInput { @@ -2726,7 +2811,7 @@ func TestDerivedImageList(t *testing.T) { repos[i].Tags = matchedTags pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repo, + RepoMetadata: repo, }) } repos, pageInfo := pageFinder.Page() @@ -2989,7 +3074,7 @@ func TestBaseImageList(t *testing.T) { repos[i].Tags = matchedTags pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repo, + RepoMetadata: repo, }) } @@ -3163,7 +3248,7 @@ func TestBaseImageList(t *testing.T) { repos[i].Tags = matchedTags pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repo, + RepoMetadata: repo, }) } @@ -3182,6 +3267,8 @@ func TestBaseImageList(t *testing.T) { } func TestExpandedRepoInfo(t *testing.T) { + log := log.NewLogger("debug", "") + Convey("ExpandedRepoInfo Errors", t, func() { responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) @@ -3198,10 +3285,22 @@ func TestExpandedRepoInfo(t *testing.T) { Digest: "digestIndex", MediaType: ispec.MediaTypeImageIndex, }, + "tagGetIndexError": { + Digest: "errorIndexDigest", + MediaType: ispec.MediaTypeImageIndex, + }, "tagGoodIndexBadManifests": { Digest: "goodIndexBadManifests", MediaType: ispec.MediaTypeImageIndex, }, + "tagGoodIndex1GoodManfest": { + Digest: "goodIndexGoodManfest", + MediaType: ispec.MediaTypeImageIndex, + }, + "tagGoodIndex2GoodManfest": { + Digest: "goodIndexGoodManfest", + MediaType: ispec.MediaTypeImageIndex, + }, }, }, nil }, @@ -3227,6 +3326,16 @@ func TestExpandedRepoInfo(t *testing.T) { }) So(err, ShouldBeNil) + goodIndexGoodManfestBlob, err := json.Marshal(ispec.Index{ + Manifests: []ispec.Descriptor{ + { + Digest: "goodManifest", + MediaType: ispec.MediaTypeImageManifest, + }, + }, + }) + So(err, ShouldBeNil) + switch indexDigest { case "errorIndexDigest": return repodb.IndexData{}, ErrTestError @@ -3234,14 +3343,61 @@ func TestExpandedRepoInfo(t *testing.T) { return repodb.IndexData{ IndexBlob: goodIndexBadManifestsBlob, }, nil + case "goodIndexGoodManfest": + return repodb.IndexData{ + IndexBlob: goodIndexGoodManfestBlob, + }, nil default: return repodb.IndexData{}, nil } }, } - log := log.NewLogger("debug", "") _, err := expandedRepoInfo(responseContext, "repo", repoDB, mocks.CveInfoMock{}, log) So(err, ShouldBeNil) }) + + Convey("Access error", t, func() { + authzCtxKey := localCtx.GetContextKey() + acCtxUser := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": false, + }, + Username: "user", + } + + ctx := context.WithValue(context.Background(), authzCtxKey, acCtxUser) + + responseContext := graphql.WithResponseContext(ctx, graphql.DefaultErrorPresenter, + graphql.DefaultRecover) + + _, err := expandedRepoInfo(responseContext, "repo", mocks.RepoDBMock{}, mocks.CveInfoMock{}, log) + So(err, ShouldBeNil) + }) +} + +func TestFilterFunctions(t *testing.T) { + Convey("Filter Functions", t, func() { + Convey("FilterByDigest bad manifest blob", func() { + filterFunc := FilterByDigest("digest") + ok := filterFunc( + repodb.RepoMetadata{}, + repodb.ManifestMetadata{ + ManifestBlob: []byte("bad blob"), + }, + ) + So(ok, ShouldBeFalse) + }) + + Convey("filterDerivedImages bad manifest blob", func() { + filterFunc := filterDerivedImages(&gql_generated.ImageSummary{}) + ok := filterFunc( + repodb.RepoMetadata{}, + repodb.ManifestMetadata{ + ManifestBlob: []byte("bad blob"), + }, + ) + So(ok, ShouldBeFalse) + }) + }) } diff --git a/pkg/extensions/search/schema.graphql b/pkg/extensions/search/schema.graphql index 1212f930..8b33a64c 100644 --- a/pkg/extensions/search/schema.graphql +++ b/pkg/extensions/search/schema.graphql @@ -680,4 +680,20 @@ type Query { "Types of artifacts to return in the referrer list" type: [String!] ): [Referrer]! + + """ + Receive RepoSummaries of repos starred by current user + """ + StarredRepos( + "Sets the parameters of the requested page (how many to include and offset)" + requestedPage: PageInput + ): PaginatedReposResult! + + """ + Receive RepoSummaries of repos bookmarked by current user + """ + BookmarkedRepos( + "Sets the parameters of the requested page (how many to include and offset)" + requestedPage: PageInput + ): PaginatedReposResult! } diff --git a/pkg/extensions/search/schema.resolvers.go b/pkg/extensions/search/schema.resolvers.go index 6cd84905..93d8befd 100644 --- a/pkg/extensions/search/schema.resolvers.go +++ b/pkg/extensions/search/schema.resolvers.go @@ -144,6 +144,16 @@ func (r *queryResolver) Referrers(ctx context.Context, repo string, digest strin return referrers, nil } +// StarredRepos is the resolver for the StarredRepos field. +func (r *queryResolver) StarredRepos(ctx context.Context, requestedPage *gql_generated.PageInput) (*gql_generated.PaginatedReposResult, error) { + return getStarredRepos(ctx, r.cveInfo, r.log, requestedPage, r.repoDB) +} + +// BookmarkedRepos is the resolver for the BookmarkedRepos field. +func (r *queryResolver) BookmarkedRepos(ctx context.Context, requestedPage *gql_generated.PageInput) (*gql_generated.PaginatedReposResult, error) { + return getBookmarkedRepos(ctx, r.cveInfo, r.log, requestedPage, r.repoDB) +} + // Query returns gql_generated.QueryResolver implementation. func (r *Resolver) Query() gql_generated.QueryResolver { return &queryResolver{r} } diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index b5c25d89..edae8955 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -147,6 +147,26 @@ type ImageSummaryResult struct { Errors []ErrorGQL `json:"errors"` } +//nolint:tagliatelle // graphQL schema +type StarredRepos struct { + PaginatedReposResult `json:"StarredRepos"` +} + +//nolint:tagliatelle // graphQL schema +type BookmarkedRepos struct { + PaginatedReposResult `json:"BookmarkedRepos"` +} + +type StarredReposResponse struct { + StarredRepos `json:"data"` + Errors []ErrorGQL `json:"errors"` +} + +type BookmarkedReposResponse struct { + BookmarkedRepos `json:"data"` + Errors []ErrorGQL `json:"errors"` +} + func readFileAndSearchString(filePath string, stringToMatch string, timeout time.Duration) (bool, error) { ctx, cancelFunc := context.WithTimeout(context.Background(), timeout) defer cancelFunc() diff --git a/pkg/extensions/search/userprefs_test.go b/pkg/extensions/search/userprefs_test.go new file mode 100644 index 00000000..f1309fb5 --- /dev/null +++ b/pkg/extensions/search/userprefs_test.go @@ -0,0 +1,632 @@ +//go:build search && userprefs + +package search_test + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "testing" + + . "github.com/smartystreets/goconvey/convey" + "golang.org/x/crypto/bcrypt" + "gopkg.in/resty.v1" + + "zotregistry.io/zot/pkg/api" + "zotregistry.io/zot/pkg/api/config" + "zotregistry.io/zot/pkg/api/constants" + extconf "zotregistry.io/zot/pkg/extensions/config" + "zotregistry.io/zot/pkg/extensions/monitoring" + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/storage" + "zotregistry.io/zot/pkg/storage/local" + . "zotregistry.io/zot/pkg/test" +) + +//nolint:dupl +func TestUserData(t *testing.T) { + Convey("Test user stars and bookmarks", t, func(c C) { + port := GetFreePort() + baseURL := GetBaseURL(port) + defaultVal := true + + accessibleRepo := "accessible-repo" + forbiddenRepo := "forbidden-repo" + tag := "0.0.1" + + adminUser := "alice" + adminPassword := "deepGoesTheRabbitBurrow" + simpleUser := "test" + simpleUserPassword := "test123" + + twoCredTests := fmt.Sprintf("%s\n%s\n\n", getCredString(adminUser, adminPassword), + getCredString(simpleUser, simpleUserPassword)) + + htpasswdPath := MakeHtpasswdFileFromString(twoCredTests) + defer os.Remove(htpasswdPath) + + conf := config.New() + conf.Storage.RootDirectory = t.TempDir() + conf.HTTP.Port = port + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ + Path: htpasswdPath, + }, + } + conf.HTTP.AccessControl = &config.AccessControlConfig{ + Repositories: config.Repositories{ + "**": config.PolicyGroup{ + Policies: []config.Policy{ + { + Users: []string{simpleUser}, + Actions: []string{"read"}, + }, + }, + AnonymousPolicy: []string{"read"}, + DefaultPolicy: []string{}, + }, + forbiddenRepo: config.PolicyGroup{ + Policies: []config.Policy{ + { + Users: []string{}, + Actions: []string{}, + }, + }, + DefaultPolicy: []string{}, + }, + }, + AdminPolicy: config.Policy{ + Users: []string{adminUser}, + Actions: []string{"read", "create", "update"}, + }, + } + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, + } + + ctlr := api.NewController(conf) + + ctlrManager := NewControllerManager(ctlr) + ctlrManager.StartAndWait(port) + defer ctlrManager.StopServer() + + config, layers, manifest, err := GetImageComponents(100) + So(err, ShouldBeNil) + + err = UploadImageWithBasicAuth( + Image{ + Config: config, + Layers: layers, + Manifest: manifest, + Reference: tag, + }, baseURL, accessibleRepo, + adminUser, adminPassword, + ) + So(err, ShouldBeNil) + + err = UploadImageWithBasicAuth( + Image{ + Config: config, + Layers: layers, + Manifest: manifest, + Reference: tag, + }, baseURL, forbiddenRepo, + adminUser, adminPassword, + ) + So(err, ShouldBeNil) + + userStaredReposQuery := `{ + StarredRepos { + Results { + Name StarCount IsStarred + NewestImage { Tag } + } + } + }` + + userBookmarkedReposQuery := `{ + BookmarkedRepos { + Results { + Name IsBookmarked + NewestImage { Tag } + } + } + }` + + userprefsBaseURL := baseURL + constants.FullUserPreferencesPrefix + + Convey("Flip starred repo authorized", func(c C) { + clientHTTP := resty.R().SetBasicAuth(simpleUser, simpleUserPassword) + + resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userStaredReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct := StarredReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + + resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoStarURL(accessibleRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userStaredReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct = StarredReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 1) + So(responseStruct.Results[0].Name, ShouldEqual, accessibleRepo) + // need to update RepoSummary according to user settings + So(responseStruct.Results[0].IsStarred, ShouldEqual, true) + So(responseStruct.Results[0].StarCount, ShouldEqual, 1) + + resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoStarURL(accessibleRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userStaredReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct = StarredReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + }) + + Convey("Flip starred repo unauthenticated user", func(c C) { + clientHTTP := resty.R() + + resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userStaredReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct := StarredReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + + resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoStarURL(accessibleRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusForbidden) + + resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userStaredReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct = StarredReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + }) + + Convey("Flip starred repo unauthorized", func(c C) { + clientHTTP := resty.R().SetBasicAuth(simpleUser, simpleUserPassword) + + resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userStaredReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct := StarredReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + + resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoStarURL(forbiddenRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusForbidden) + + resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userStaredReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct = StarredReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + }) + + Convey("Flip starred repo with unauthorized repo and admin user", func(c C) { + clientHTTP := resty.R().SetBasicAuth(adminUser, adminPassword) + + resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userStaredReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct := StarredReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + + resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoStarURL(forbiddenRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userStaredReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct = StarredReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 1) + So(responseStruct.Results[0].Name, ShouldEqual, forbiddenRepo) + // need to update RepoSummary according to user settings + So(responseStruct.Results[0].IsStarred, ShouldEqual, true) + So(responseStruct.Results[0].StarCount, ShouldEqual, 1) + + resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoStarURL(forbiddenRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userStaredReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct = StarredReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + }) + + Convey("Flip bookmark repo authorized", func(c C) { + clientHTTP := resty.R().SetBasicAuth(simpleUser, simpleUserPassword) + + resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userBookmarkedReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct := BookmarkedReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + + resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoBookmarkURL(accessibleRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userBookmarkedReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct = BookmarkedReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 1) + So(responseStruct.Results[0].Name, ShouldEqual, accessibleRepo) + // need to update RepoSummary according to user settings + So(responseStruct.Results[0].IsBookmarked, ShouldEqual, true) + + resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoBookmarkURL(accessibleRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userBookmarkedReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct = BookmarkedReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + }) + + Convey("Flip bookmark repo unauthenticated user", func(c C) { + clientHTTP := resty.R() + + resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userBookmarkedReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct := BookmarkedReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + + resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoBookmarkURL(accessibleRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusForbidden) + + resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userBookmarkedReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct = BookmarkedReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + }) + + Convey("Flip bookmark repo unauthorized", func(c C) { + clientHTTP := resty.R().SetBasicAuth(simpleUser, simpleUserPassword) + + resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userBookmarkedReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct := BookmarkedReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + + resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoBookmarkURL(forbiddenRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusForbidden) + + resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userBookmarkedReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct = BookmarkedReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + }) + + Convey("Flip bookmarked unauthorized repo and admin user", func(c C) { + clientHTTP := resty.R().SetBasicAuth(adminUser, adminPassword) + + resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userBookmarkedReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct := BookmarkedReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + + resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoBookmarkURL(forbiddenRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userBookmarkedReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct = BookmarkedReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 1) + So(responseStruct.Results[0].Name, ShouldEqual, forbiddenRepo) + // need to update RepoSummary according to user settings + So(responseStruct.Results[0].IsBookmarked, ShouldEqual, true) + + resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoBookmarkURL(forbiddenRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix + + "?query=" + url.QueryEscape(userBookmarkedReposQuery)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + responseStruct = BookmarkedReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 0) + }) + }) +} + +func TestChangingRepoState(t *testing.T) { + port := GetFreePort() + baseURL := GetBaseURL(port) + defaultVal := true + + simpleUser := "test" + simpleUserPassword := "test123" + + forbiddenRepo := "forbidden" + accesibleRepo := "accesible" + + credTests := fmt.Sprintf("%s\n\n", getCredString(simpleUser, simpleUserPassword)) + + htpasswdPath := MakeHtpasswdFileFromString(credTests) + defer os.Remove(htpasswdPath) + + conf := config.New() + conf.Storage.RootDirectory = t.TempDir() + conf.HTTP.Port = port + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ + Path: htpasswdPath, + }, + } + conf.HTTP.AccessControl = &config.AccessControlConfig{ + Repositories: config.Repositories{ + "**": config.PolicyGroup{ + Policies: []config.Policy{ + { + Users: []string{simpleUser}, + Actions: []string{"read"}, + }, + }, + AnonymousPolicy: []string{"read"}, + DefaultPolicy: []string{}, + }, + forbiddenRepo: config.PolicyGroup{ + Policies: []config.Policy{ + { + Users: []string{}, + Actions: []string{}, + }, + }, + DefaultPolicy: []string{}, + }, + }, + } + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, + } + + gqlStarredRepos := ` + { + StarredRepos() { + Results { + Name + StarCount + IsBookmarked + IsStarred + } + } + } + ` + + gqlBookmarkedRepos := ` + { + BookmarkedRepos() { + Results { + Name + StarCount + IsBookmarked + IsStarred + } + } + } + ` + + ctlr := api.NewController(conf) + + img, err := GetRandomImage("tag") + if err != nil { + t.FailNow() + } + + // ------ Create the test repos + defaultStore := local.NewImageStore(conf.Storage.RootDirectory, false, 0, false, false, + log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) + + err = WriteImageToFileSystem(img, accesibleRepo, storage.StoreController{ + DefaultStore: defaultStore, + }) + if err != nil { + t.FailNow() + } + + err = WriteImageToFileSystem(img, forbiddenRepo, storage.StoreController{ + DefaultStore: defaultStore, + }) + if err != nil { + t.FailNow() + } + + ctlrManager := NewControllerManager(ctlr) + + ctlrManager.StartAndWait(port) + + defer ctlrManager.StopServer() + + simpleUserClient := resty.R().SetBasicAuth(simpleUser, simpleUserPassword) + anonynousClient := resty.R() + + userprefsBaseURL := baseURL + constants.FullUserPreferencesPrefix + + Convey("PutStars", t, func() { + resp, err := simpleUserClient.Put(userprefsBaseURL + PutRepoStarURL(accesibleRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = simpleUserClient.Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(gqlStarredRepos)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + responseStruct := StarredReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 1) + So(responseStruct.Results[0].IsStarred, ShouldBeTrue) + So(responseStruct.Results[0].Name, ShouldResemble, accesibleRepo) + + resp, err = anonynousClient.Put(userprefsBaseURL + PutRepoStarURL(accesibleRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusForbidden) + }) + // + Convey("PutBookmark", t, func() { + resp, err := simpleUserClient.Put(userprefsBaseURL + PutRepoBookmarkURL(accesibleRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + resp, err = simpleUserClient.Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(gqlBookmarkedRepos)) + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + responseStruct := BookmarkedReposResponse{} + err = json.Unmarshal(resp.Body(), &responseStruct) + So(err, ShouldBeNil) + So(err, ShouldBeNil) + So(len(responseStruct.Results), ShouldEqual, 1) + So(responseStruct.Results[0].IsBookmarked, ShouldBeTrue) + So(responseStruct.Results[0].Name, ShouldResemble, accesibleRepo) + + resp, err = anonynousClient.Put(userprefsBaseURL + PutRepoBookmarkURL(accesibleRepo)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusForbidden) + }) +} + +func PutRepoStarURL(repo string) string { + return fmt.Sprintf("?repo=%s&action=toggleStar", repo) +} + +func PutRepoBookmarkURL(repo string) string { + return fmt.Sprintf("?repo=%s&action=toggleBookmark", repo) +} + +func getCredString(username, password string) string { + hash, err := bcrypt.GenerateFromPassword([]byte(password), 10) + if err != nil { + panic(err) + } + + usernameAndHash := fmt.Sprintf("%s:%s", username, string(hash)) + + return usernameAndHash +} diff --git a/pkg/meta/bolt/buckets.go b/pkg/meta/bolt/buckets.go index 367e8b47..80237d06 100644 --- a/pkg/meta/bolt/buckets.go +++ b/pkg/meta/bolt/buckets.go @@ -5,7 +5,9 @@ const ( ManifestDataBucket = "ManifestData" IndexDataBucket = "IndexData" ArtifactDataBucket = "ArtifactData" - UserMetadataBucket = "UserMeta" RepoMetadataBucket = "RepoMetadata" + UserDataBucket = "UserData" VersionBucket = "Version" + StarredReposKey = "StarredReposKey" + BookmarkedReposKey = "BookmarkedReposKey" ) diff --git a/pkg/meta/common/common.go b/pkg/meta/common/common.go index c787cc8f..80b72789 100644 --- a/pkg/meta/common/common.go +++ b/pkg/meta/common/common.go @@ -2,6 +2,7 @@ package common import ( "encoding/json" + "fmt" "strings" "time" @@ -286,3 +287,98 @@ func CheckImageLastUpdated(repoLastUpdated time.Time, isSigned bool, noImageChec return repoLastUpdated, noImageChecked, isSigned } + +func FilterDataByRepo(foundRepos []repodb.RepoMetadata, manifestMetadataMap map[string]repodb.ManifestMetadata, + indexDataMap map[string]repodb.IndexData, +) (map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, error) { + var ( + foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata) + foundindexDataMap = make(map[string]repodb.IndexData) + ) + + // keep just the manifestMeta we need + for _, repoMeta := range foundRepos { + for _, descriptor := range repoMeta.Tags { + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest] + case ispec.MediaTypeImageIndex: + indexData := indexDataMap[descriptor.Digest] + + var indexContent ispec.Index + + err := json.Unmarshal(indexData.IndexBlob, &indexContent) + if err != nil { + return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, + fmt.Errorf("repodb: error while getting manifest data for digest %s %w", descriptor.Digest, err) + } + + for _, manifestDescriptor := range indexContent.Manifests { + manifestDigest := manifestDescriptor.Digest.String() + + foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest] + } + + foundindexDataMap[descriptor.Digest] = indexData + default: + } + } + } + + return foundManifestMetadataMap, foundindexDataMap, nil +} + +func FetchDataForRepos(repoDB repodb.RepoDB, foundRepos []repodb.RepoMetadata, +) (map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, error) { + foundManifestMetadataMap := map[string]repodb.ManifestMetadata{} + foundIndexDataMap := map[string]repodb.IndexData{} + + for idx := range foundRepos { + for _, descriptor := range foundRepos[idx].Tags { + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestData, err := repoDB.GetManifestData(godigest.Digest(descriptor.Digest)) + if err != nil { + return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, err + } + + foundManifestMetadataMap[descriptor.Digest] = repodb.ManifestMetadata{ + ManifestBlob: manifestData.ManifestBlob, + ConfigBlob: manifestData.ConfigBlob, + } + case ispec.MediaTypeImageIndex: + indexData, err := repoDB.GetIndexData(godigest.Digest(descriptor.Digest)) + if err != nil { + return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, err + } + + var indexContent ispec.Index + + err = json.Unmarshal(indexData.IndexBlob, &indexContent) + if err != nil { + return map[string]repodb.ManifestMetadata{}, + map[string]repodb.IndexData{}, + fmt.Errorf("repodb: error while getting index data for digest %s %w", descriptor.Digest, err) + } + + for _, manifestDescriptor := range indexContent.Manifests { + manifestDigest := manifestDescriptor.Digest.String() + + manifestData, err := repoDB.GetManifestData(manifestDescriptor.Digest) + if err != nil { + return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, err + } + + foundManifestMetadataMap[manifestDigest] = repodb.ManifestMetadata{ + ManifestBlob: manifestData.ManifestBlob, + ConfigBlob: manifestData.ConfigBlob, + } + } + + foundIndexDataMap[descriptor.Digest] = indexData + } + } + } + + return foundManifestMetadataMap, foundIndexDataMap, nil +} diff --git a/pkg/meta/common/common_test.go b/pkg/meta/common/common_test.go index 8f3e31e5..29dad10b 100644 --- a/pkg/meta/common/common_test.go +++ b/pkg/meta/common/common_test.go @@ -1,15 +1,21 @@ package common_test import ( + "errors" "testing" "time" + "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" "zotregistry.io/zot/pkg/meta/common" "zotregistry.io/zot/pkg/meta/repodb" + "zotregistry.io/zot/pkg/test/mocks" ) +var ErrTestError = errors.New("test error") + func TestUtils(t *testing.T) { Convey("GetReferredSubject", t, func() { _, err := common.GetReferredSubject([]byte("bad json")) @@ -94,4 +100,128 @@ func TestUtils(t *testing.T) { So(noImageChecked, ShouldEqual, false) }) }) + + Convey("SignatureAlreadyExists", t, func() { + res := common.SignatureAlreadyExists( + []repodb.SignatureInfo{{SignatureManifestDigest: "digest"}}, + repodb.SignatureMetadata{SignatureDigest: "digest"}, + ) + + So(res, ShouldEqual, true) + + res = common.SignatureAlreadyExists( + []repodb.SignatureInfo{{SignatureManifestDigest: "digest"}}, + repodb.SignatureMetadata{SignatureDigest: "digest2"}, + ) + + So(res, ShouldEqual, false) + }) + + Convey("FilterDataByRepo", t, func() { + Convey("Errors", func() { + // Unmarshal index data error + _, _, err := common.FilterDataByRepo( + []repodb.RepoMetadata{{ + Tags: map[string]repodb.Descriptor{ + "tag": { + Digest: "indexDigest", + MediaType: ispec.MediaTypeImageIndex, + }, + }, + }}, + map[string]repodb.ManifestMetadata{}, + map[string]repodb.IndexData{ + "indexDigest": { + IndexBlob: []byte("bad blob"), + }, + }, + ) + + So(err, ShouldNotBeNil) + }) + }) + + Convey("FetchDataForRepos", t, func() { + Convey("Errors", func() { + // Unmarshal index data error + _, _, err := common.FetchDataForRepos( + mocks.RepoDBMock{ + GetIndexDataFn: func(indexDigest digest.Digest) (repodb.IndexData, error) { + return repodb.IndexData{ + IndexBlob: []byte("bad blob"), + }, nil + }, + }, + []repodb.RepoMetadata{{ + Tags: map[string]repodb.Descriptor{ + "tag": { + Digest: "indexDigest", + MediaType: ispec.MediaTypeImageIndex, + }, + }, + }}, + ) + So(err, ShouldNotBeNil) + }) + }) +} + +func TestFetchDataForRepos(t *testing.T) { + Convey("GetReferredSubject", t, func() { + mockRepoDB := mocks.RepoDBMock{} + + Convey("GetManifestData errors", func() { + mockRepoDB.GetManifestDataFn = func(manifestDigest digest.Digest) (repodb.ManifestData, error) { + return repodb.ManifestData{}, ErrTestError + } + + _, _, err := common.FetchDataForRepos(mockRepoDB, []repodb.RepoMetadata{ + { + Tags: map[string]repodb.Descriptor{ + "tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest}, + }, + }, + }) + So(err, ShouldNotBeNil) + }) + + Convey("GetIndexData errors", func() { + mockRepoDB.GetIndexDataFn = func(indexDigest digest.Digest) (repodb.IndexData, error) { + return repodb.IndexData{}, ErrTestError + } + + _, _, err := common.FetchDataForRepos(mockRepoDB, []repodb.RepoMetadata{ + { + Tags: map[string]repodb.Descriptor{ + "tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageIndex}, + }, + }, + }) + So(err, ShouldNotBeNil) + }) + + Convey("GetIndexData ok, GetManifestData errors", func() { + mockRepoDB.GetIndexDataFn = func(indexDigest digest.Digest) (repodb.IndexData, error) { + return repodb.IndexData{ + IndexBlob: []byte(`{ + "manifests": [ + {"digest": "dig1"} + ] + }`), + }, nil + } + mockRepoDB.GetManifestDataFn = func(manifestDigest digest.Digest) (repodb.ManifestData, error) { + return repodb.ManifestData{}, ErrTestError + } + + _, _, err := common.FetchDataForRepos(mockRepoDB, []repodb.RepoMetadata{ + { + Tags: map[string]repodb.Descriptor{ + "tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageIndex}, + }, + }, + }) + So(err, ShouldNotBeNil) + }) + }) } diff --git a/pkg/meta/dynamo/parameters.go b/pkg/meta/dynamo/parameters.go index bd7d7d64..cc98debc 100644 --- a/pkg/meta/dynamo/parameters.go +++ b/pkg/meta/dynamo/parameters.go @@ -10,7 +10,7 @@ import ( type DBDriverParameters struct { Endpoint, Region, RepoMetaTablename, ManifestDataTablename, IndexDataTablename, - ArtifactDataTablename, VersionTablename string + ArtifactDataTablename, VersionTablename, UserDataTablename string } func GetDynamoClient(params DBDriverParameters) (*dynamodb.Client, error) { diff --git a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go index 287dd7ec..21e91421 100644 --- a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go +++ b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper.go @@ -12,6 +12,7 @@ import ( "go.etcd.io/bbolt" zerr "zotregistry.io/zot/errors" + zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/bolt" "zotregistry.io/zot/pkg/meta/common" @@ -58,6 +59,11 @@ func NewBoltDBWrapper(boltDB *bbolt.DB, log log.Logger) (*DBWrapper, error) { return err } + _, err = transaction.CreateBucketIfNotExists([]byte(bolt.UserDataBucket)) + if err != nil { + return err + } + return nil }) if err != nil { @@ -695,7 +701,7 @@ func (bdw *DBWrapper) GetMultipleRepoMeta(ctx context.Context, filter func(repoM if filter(repoMeta) { pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repoMeta, + RepoMetadata: repoMeta, }) } } @@ -889,6 +895,8 @@ func (bdw *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter repoBuck = transaction.Bucket([]byte(bolt.RepoMetadataBucket)) indexBuck = transaction.Bucket([]byte(bolt.IndexDataBucket)) manifestBuck = transaction.Bucket([]byte(bolt.ManifestDataBucket)) + userBookmarks = getUserBookmarks(ctx, transaction) + userStars = getUserStars(ctx, transaction) ) cursor := repoBuck.Cursor() @@ -905,147 +913,126 @@ func (bdw *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter return err } - if rank := common.RankRepoName(searchText, string(repoName)); rank != -1 { - var ( - // specific values used for sorting that need to be calculated based on all manifests from the repo - repoDownloads = 0 - repoLastUpdated = time.Time{} - noImageChecked = true - osSet = map[string]bool{} - archSet = map[string]bool{} - isSigned = false - ) + repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) + repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) - for tag, descriptor := range repoMeta.Tags { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDigest := descriptor.Digest + rank := common.RankRepoName(searchText, string(repoName)) + if rank == -1 { + continue + } - manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, - manifestMetadataMap, manifestBuck) - if err != nil { - return fmt.Errorf("repodb: error fetching manifest meta for manifest with digest %s %w", - manifestDigest, err) - } + var ( + repoDownloads = 0 + repoLastUpdated = time.Time{} + osSet = map[string]bool{} + archSet = map[string]bool{} + noImageChecked = true + isSigned = false + ) - manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) - if err != nil { - return fmt.Errorf("repodb: error collecting filter data for manifest with digest %s %w", - manifestDigest, err) - } + for tag, descriptor := range repoMeta.Tags { + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestDigest := descriptor.Digest - repoDownloads += manifestFilterData.DownloadCount - - for _, os := range manifestFilterData.OsList { - osSet[os] = true - } - for _, arch := range manifestFilterData.ArchList { - archSet[arch] = true - } - - repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned, - noImageChecked, manifestFilterData) - - manifestMetadataMap[descriptor.Digest] = manifestMeta - case ispec.MediaTypeImageIndex: - indexDigest := descriptor.Digest - - indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck) - if err != nil { - return fmt.Errorf("repodb: error fetching index data for index with digest %s %w", - indexDigest, err) - } - - var indexContent ispec.Index - - err = json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return fmt.Errorf("repodb: error while unmashaling index content for %s:%s %w", - repoName, tag, err) - } - - // this also updates manifestMetadataMap - indexFilterData, err := collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, manifestMetadataMap, - manifestBuck) - if err != nil { - return fmt.Errorf("repodb: error collecting filter data for index with digest %s %w", - indexDigest, err) - } - - for _, arch := range indexFilterData.ArchList { - archSet[arch] = true - } - - for _, os := range indexFilterData.OsList { - osSet[os] = true - } - - repoDownloads += indexFilterData.DownloadCount - - repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned, - noImageChecked, indexFilterData) - - indexDataMap[indexDigest] = indexData - default: - bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType) - - continue + manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, + manifestMetadataMap, manifestBuck) + if err != nil { + return fmt.Errorf("repodb: error fetching manifest meta for manifest with digest %s %w", + manifestDigest, err) } - } - repoFilterData := repodb.FilterData{ - OsList: common.GetMapKeys(osSet), - ArchList: common.GetMapKeys(archSet), - LastUpdated: repoLastUpdated, - DownloadCount: repoDownloads, - IsSigned: isSigned, - } + manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) + if err != nil { + return fmt.Errorf("repodb: error collecting filter data for manifest with digest %s %w", + manifestDigest, err) + } + + repoDownloads += manifestFilterData.DownloadCount + + for _, os := range manifestFilterData.OsList { + osSet[os] = true + } + for _, arch := range manifestFilterData.ArchList { + archSet[arch] = true + } + + repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned, + noImageChecked, manifestFilterData) + + manifestMetadataMap[descriptor.Digest] = manifestMeta + case ispec.MediaTypeImageIndex: + indexDigest := descriptor.Digest + + indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck) + if err != nil { + return fmt.Errorf("repodb: error fetching index data for index with digest %s %w", + indexDigest, err) + } + + var indexContent ispec.Index + + err = json.Unmarshal(indexData.IndexBlob, &indexContent) + if err != nil { + return fmt.Errorf("repodb: error while unmashaling index content for %s:%s %w", + repoName, tag, err) + } + + // this also updates manifestMetadataMap + indexFilterData, err := collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, manifestMetadataMap, + manifestBuck) + if err != nil { + return fmt.Errorf("repodb: error collecting filter data for index with digest %s %w", + indexDigest, err) + } + + for _, arch := range indexFilterData.ArchList { + archSet[arch] = true + } + + for _, os := range indexFilterData.OsList { + osSet[os] = true + } + + repoDownloads += indexFilterData.DownloadCount + + repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned, + noImageChecked, indexFilterData) + + indexDataMap[indexDigest] = indexData + default: + bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType) - if !common.AcceptedByFilter(filter, repoFilterData) { continue } - - pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repoMeta, - Rank: rank, - Downloads: repoDownloads, - UpdateTime: repoLastUpdated, - }) } + + repoFilterData := repodb.FilterData{ + OsList: common.GetMapKeys(osSet), + ArchList: common.GetMapKeys(archSet), + LastUpdated: repoLastUpdated, + DownloadCount: repoDownloads, + IsSigned: isSigned, + } + + if !common.AcceptedByFilter(filter, repoFilterData) { + continue + } + + pageFinder.Add(repodb.DetailedRepoMeta{ + RepoMetadata: repoMeta, + Rank: rank, + Downloads: repoDownloads, + UpdateTime: repoLastUpdated, + }) } foundRepos, pageInfo = pageFinder.Page() - // keep just the manifestMeta and indexData we need - for _, repoMeta := range foundRepos { - for _, descriptor := range repoMeta.Tags { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest] - case ispec.MediaTypeImageIndex: - indexData := indexDataMap[descriptor.Digest] + foundManifestMetadataMap, foundindexDataMap, err = common.FilterDataByRepo(foundRepos, manifestMetadataMap, + indexDataMap) - var indexContent ispec.Index - - err := json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return err - } - - for _, manifestDescriptor := range indexContent.Manifests { - manifestDigest := manifestDescriptor.Digest.String() - - foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest] - } - - foundindexDataMap[descriptor.Digest] = indexData - default: - bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType) - } - } - } - - return nil + return err }) return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err @@ -1230,12 +1217,14 @@ func (bdw *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, repodb.PageInfo{}, err } - err = bdw.DB.View(func(tx *bbolt.Tx) error { + err = bdw.DB.View(func(transaction *bbolt.Tx) error { var ( - repoBuck = tx.Bucket([]byte(bolt.RepoMetadataBucket)) - indexBuck = tx.Bucket([]byte(bolt.IndexDataBucket)) - manifestBuck = tx.Bucket([]byte(bolt.ManifestDataBucket)) - cursor = repoBuck.Cursor() + repoBuck = transaction.Bucket([]byte(bolt.RepoMetadataBucket)) + indexBuck = transaction.Bucket([]byte(bolt.IndexDataBucket)) + manifestBuck = transaction.Bucket([]byte(bolt.ManifestDataBucket)) + cursor = repoBuck.Cursor() + userBookmarks = getUserBookmarks(ctx, transaction) + userStars = getUserStars(ctx, transaction) ) repoName, repoMetaBlob := cursor.First() @@ -1252,6 +1241,9 @@ func (bdw *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, return err } + repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) + repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) + matchedTags := make(map[string]repodb.Descriptor) // take all manifestMetas for tag, descriptor := range repoMeta.Tags { @@ -1329,47 +1321,85 @@ func (bdw *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, repoMeta.Tags = matchedTags pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repoMeta, + RepoMetadata: repoMeta, }) } foundRepos, pageInfo = pageFinder.Page() - // keep just the manifestMeta and indexData we need - for _, repoMeta := range foundRepos { - for _, descriptor := range repoMeta.Tags { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest] - case ispec.MediaTypeImageIndex: - indexData := indexDataMap[descriptor.Digest] + foundManifestMetadataMap, foundindexDataMap, err = common.FilterDataByRepo(foundRepos, manifestMetadataMap, + indexDataMap) - var indexContent ispec.Index - - err := json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return err - } - - for _, manifestDescriptor := range indexContent.Manifests { - manifestDigest := manifestDescriptor.Digest.String() - - foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest] - } - - foundindexDataMap[descriptor.Digest] = indexData - default: - bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType) - } - } - } - - return nil + return err }) return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err } +func (bdw *DBWrapper) FilterRepos(ctx context.Context, + filter repodb.FilterRepoFunc, + requestedPage repodb.PageInput, +) ( + []repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error, +) { + var ( + foundRepos = make([]repodb.RepoMetadata, 0) + pageFinder repodb.PageFinder + pageInfo repodb.PageInfo + ) + + pageFinder, err := repodb.NewBaseRepoPageFinder( + requestedPage.Limit, + requestedPage.Offset, + requestedPage.SortBy, + ) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, err + } + + err = bdw.DB.View(func(tx *bbolt.Tx) error { + var ( + buck = tx.Bucket([]byte(bolt.RepoMetadataBucket)) + cursor = buck.Cursor() + userBookmarks = getUserBookmarks(ctx, tx) + userStars = getUserStars(ctx, tx) + ) + + for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { + if ok, err := localCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { + continue + } + + repoMeta := repodb.RepoMetadata{} + + err := json.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) + repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) + + if filter(repoMeta) { + pageFinder.Add(repodb.DetailedRepoMeta{ + RepoMetadata: repoMeta, + }) + } + } + + foundRepos, pageInfo = pageFinder.Page() + + return nil + }) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, err + } + + foundManifestMetadataMap, foundIndexDataMap, err := common.FetchDataForRepos(bdw, foundRepos) + + return foundRepos, foundManifestMetadataMap, foundIndexDataMap, pageInfo, err +} + func (bdw *DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) { @@ -1397,12 +1427,14 @@ func (bdw *DBWrapper) SearchTags(ctx context.Context, searchText string, filter fmt.Errorf("repodb: error while parsing search text, invalid format %w", err) } - err = bdw.DB.View(func(tx *bbolt.Tx) error { + err = bdw.DB.View(func(transaction *bbolt.Tx) error { var ( - repoBuck = tx.Bucket([]byte(bolt.RepoMetadataBucket)) - indexBuck = tx.Bucket([]byte(bolt.IndexDataBucket)) - manifestBuck = tx.Bucket([]byte(bolt.ManifestDataBucket)) - cursor = repoBuck.Cursor() + repoBuck = transaction.Bucket([]byte(bolt.RepoMetadataBucket)) + indexBuck = transaction.Bucket([]byte(bolt.IndexDataBucket)) + manifestBuck = transaction.Bucket([]byte(bolt.ManifestDataBucket)) + cursor = repoBuck.Cursor() + userBookmarks = getUserBookmarks(ctx, transaction) + userStars = getUserStars(ctx, transaction) ) repoName, repoMetaBlob := cursor.Seek([]byte(searchedRepo)) @@ -1419,140 +1451,119 @@ func (bdw *DBWrapper) SearchTags(ctx context.Context, searchText string, filter return err } - if string(repoName) == searchedRepo { - matchedTags := make(map[string]repodb.Descriptor) - // take all manifestMetas - for tag, descriptor := range repoMeta.Tags { - if !strings.HasPrefix(tag, searchedTag) { + repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) + repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) + + if string(repoName) != searchedRepo { + continue + } + + matchedTags := make(map[string]repodb.Descriptor) + + for tag, descriptor := range repoMeta.Tags { + if !strings.HasPrefix(tag, searchedTag) { + continue + } + + matchedTags[tag] = descriptor + + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestDigest := descriptor.Digest + + manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck) + if err != nil { + return fmt.Errorf("repodb: error fetching manifest meta for manifest with digest %s %w", + manifestDigest, err) + } + + imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) + if err != nil { + return fmt.Errorf("repodb: error collecting filter data for manifest with digest %s %w", + manifestDigest, err) + } + + if !common.AcceptedByFilter(filter, imageFilterData) { + delete(matchedTags, tag) + continue } - matchedTags[tag] = descriptor + manifestMetadataMap[descriptor.Digest] = manifestMeta + case ispec.MediaTypeImageIndex: + indexDigest := descriptor.Digest - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDigest := descriptor.Digest + indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck) + if err != nil { + return fmt.Errorf("repodb: error fetching index data for index with digest %s %w", + indexDigest, err) + } + + var indexContent ispec.Index + + err = json.Unmarshal(indexData.IndexBlob, &indexContent) + if err != nil { + return fmt.Errorf("repodb: error collecting filter data for index with digest %s %w", + indexDigest, err) + } + + manifestHasBeenMatched := false + + for _, manifest := range indexContent.Manifests { + manifestDigest := manifest.Digest.String() manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck) if err != nil { - return fmt.Errorf("repodb: error fetching manifest meta for manifest with digest %s %w", + return fmt.Errorf("repodb: error fetching from db manifest meta for manifest with digest %s %w", manifestDigest, err) } - imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) + manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) if err != nil { return fmt.Errorf("repodb: error collecting filter data for manifest with digest %s %w", manifestDigest, err) } - if !common.AcceptedByFilter(filter, imageFilterData) { - delete(matchedTags, tag) + manifestMetadataMap[manifestDigest] = manifestMeta - continue + if common.AcceptedByFilter(filter, manifestFilterData) { + manifestHasBeenMatched = true } + } - manifestMetadataMap[descriptor.Digest] = manifestMeta - case ispec.MediaTypeImageIndex: - indexDigest := descriptor.Digest - - indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck) - if err != nil { - return fmt.Errorf("repodb: error fetching index data for index with digest %s %w", - indexDigest, err) - } - - var indexContent ispec.Index - - err = json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return fmt.Errorf("repodb: error collecting filter data for index with digest %s %w", - indexDigest, err) - } - - manifestHasBeenMatched := false + if !manifestHasBeenMatched { + delete(matchedTags, tag) for _, manifest := range indexContent.Manifests { - manifestDigest := manifest.Digest.String() - - manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck) - if err != nil { - return fmt.Errorf("repodb: error fetching from db manifest meta for manifest with digest %s %w", - manifestDigest, err) - } - - manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) - if err != nil { - return fmt.Errorf("repodb: error collecting filter data for manifest with digest %s %w", - manifestDigest, err) - } - - manifestMetadataMap[manifestDigest] = manifestMeta - - if common.AcceptedByFilter(filter, manifestFilterData) { - manifestHasBeenMatched = true - } + delete(manifestMetadataMap, manifest.Digest.String()) } - if !manifestHasBeenMatched { - delete(matchedTags, tag) - - for _, manifest := range indexContent.Manifests { - delete(manifestMetadataMap, manifest.Digest.String()) - } - - continue - } - - indexDataMap[indexDigest] = indexData - default: - bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType) - continue } - } - if len(matchedTags) == 0 { + indexDataMap[indexDigest] = indexData + default: + bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType) + continue } - - repoMeta.Tags = matchedTags - - pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repoMeta, - }) } + + if len(matchedTags) == 0 { + continue + } + + repoMeta.Tags = matchedTags + + pageFinder.Add(repodb.DetailedRepoMeta{ + RepoMetadata: repoMeta, + }) } foundRepos, pageInfo = pageFinder.Page() - // keep just the manifestMeta and indexData we need - for _, repoMeta := range foundRepos { - for _, descriptor := range repoMeta.Tags { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest] - case ispec.MediaTypeImageIndex: - indexData := indexDataMap[descriptor.Digest] - - var indexContent ispec.Index - - err := json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return err - } - - for _, manifestDescriptor := range indexContent.Manifests { - manifestDigest := manifestDescriptor.Digest.String() - - foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest] - } - - foundindexDataMap[descriptor.Digest] = indexData - default: - bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType) - } - } - } + foundManifestMetadataMap, foundindexDataMap, err = common.FilterDataByRepo(foundRepos, manifestMetadataMap, + indexDataMap) return nil }) @@ -1560,6 +1571,248 @@ func (bdw *DBWrapper) SearchTags(ctx context.Context, searchText string, filter return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err } +func (bdw *DBWrapper) ToggleStarRepo(ctx context.Context, repo string) (repodb.ToggleState, error) { + acCtx, err := localCtx.GetAccessControlContext(ctx) + if err != nil { + return repodb.NotChanged, err + } + + userid := localCtx.GetUsernameFromContext(acCtx) + + if userid == "" { + // empty user is anonymous + return repodb.NotChanged, zerr.ErrUserDataNotAllowed + } + + if ok, err := localCtx.RepoIsUserAvailable(ctx, repo); !ok || err != nil { + return repodb.NotChanged, zerr.ErrUserDataNotAllowed + } + + var res repodb.ToggleState + + if err := bdw.DB.Update(func(tx *bbolt.Tx) error { //nolint:varnamelen + userdb := tx.Bucket([]byte(bolt.UserDataBucket)) + userBucket, err := userdb.CreateBucketIfNotExists([]byte(userid)) + if err != nil { + // this is a serious failure + return zerr.ErrUnableToCreateUserBucket + } + + mdata := userBucket.Get([]byte(bolt.StarredReposKey)) + unpacked := []string{} + if mdata != nil { + if err = json.Unmarshal(mdata, &unpacked); err != nil { + return zerr.ErrInvalidOldUserStarredRepos + } + } + + isRepoStarred := zcommon.Contains(unpacked, repo) + + if isRepoStarred { + res = repodb.Removed + unpacked = zcommon.RemoveFrom(unpacked, repo) + } else { + res = repodb.Added + unpacked = append(unpacked, repo) + } + + var repacked []byte + if repacked, err = json.Marshal(unpacked); err != nil { + return zerr.ErrCouldNotMarshalStarredRepos + } + + err = userBucket.Put([]byte(bolt.StarredReposKey), repacked) + if err != nil { + return zerr.ErrCouldNotPersistData + } + + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) + + repoMetaBlob := repoBuck.Get([]byte(repo)) + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound + } + + var repoMeta repodb.RepoMetadata + + err = json.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + switch res { + case repodb.Added: + repoMeta.Stars++ + case repodb.Removed: + repoMeta.Stars-- + } + + repoMetaBlob, err = json.Marshal(repoMeta) + if err != nil { + return err + } + + err = repoBuck.Put([]byte(repo), repoMetaBlob) + if err != nil { + return err + } + + return nil + }); err != nil { + return repodb.NotChanged, err + } + + return res, nil +} + +func (bdw *DBWrapper) GetStarredRepos(ctx context.Context) ([]string, error) { + starredRepos := make([]string, 0) + + acCtx, err := localCtx.GetAccessControlContext(ctx) + if err != nil { + return starredRepos, err + } + + userid := localCtx.GetUsernameFromContext(acCtx) + + err = bdw.DB.View(func(tx *bbolt.Tx) error { //nolint:dupl + if userid == "" { + return nil + } + + userdb := tx.Bucket([]byte(bolt.UserDataBucket)) + userBucket := userdb.Bucket([]byte(userid)) + + if userBucket == nil { + return nil + } + + mdata := userBucket.Get([]byte(bolt.StarredReposKey)) + if mdata == nil { + return nil + } + + if err := json.Unmarshal(mdata, &starredRepos); err != nil { + bdw.Log.Info().Str("user", userid).Err(err).Msg("unmarshal error") + + return zerr.ErrInvalidOldUserStarredRepos + } + + if starredRepos == nil { + starredRepos = make([]string, 0) + } + + return nil + }) + + return starredRepos, err +} + +func (bdw *DBWrapper) ToggleBookmarkRepo(ctx context.Context, repo string) (repodb.ToggleState, error) { + acCtx, err := localCtx.GetAccessControlContext(ctx) + if err != nil { + return repodb.NotChanged, err + } + + userid := localCtx.GetUsernameFromContext(acCtx) + if userid == "" { + // empty user is anonymous + return repodb.NotChanged, zerr.ErrUserDataNotAllowed + } + + if ok, err := localCtx.RepoIsUserAvailable(ctx, repo); !ok || err != nil { + return repodb.NotChanged, zerr.ErrUserDataNotAllowed + } + + var res repodb.ToggleState + + if err := bdw.DB.Update(func(tx *bbolt.Tx) error { //nolint:dupl + userdb := tx.Bucket([]byte(bolt.UserDataBucket)) + userBucket, err := userdb.CreateBucketIfNotExists([]byte(userid)) + if err != nil { + // this is a serious failure + return zerr.ErrUnableToCreateUserBucket + } + + mdata := userBucket.Get([]byte(bolt.BookmarkedReposKey)) + unpacked := []string{} + if mdata != nil { + if err = json.Unmarshal(mdata, &unpacked); err != nil { + return zerr.ErrInvalidOldUserBookmarkedRepos + } + } + + isRepoBookmarked := zcommon.Contains(unpacked, repo) + + if isRepoBookmarked { + res = repodb.Removed + unpacked = zcommon.RemoveFrom(unpacked, repo) + } else { + res = repodb.Added + unpacked = append(unpacked, repo) + } + + var repacked []byte + if repacked, err = json.Marshal(unpacked); err != nil { + return zerr.ErrCouldNotMarshalBookmarkedRepos + } + + err = userBucket.Put([]byte(bolt.BookmarkedReposKey), repacked) + if err != nil { + return zerr.ErrUnableToCreateUserBucket + } + + return nil + }); err != nil { + return repodb.NotChanged, err + } + + return res, nil +} + +func (bdw *DBWrapper) GetBookmarkedRepos(ctx context.Context) ([]string, error) { + bookmarkedRepos := []string{} + + acCtx, err := localCtx.GetAccessControlContext(ctx) + if err != nil { + return bookmarkedRepos, err + } + + userid := localCtx.GetUsernameFromContext(acCtx) + + err = bdw.DB.View(func(tx *bbolt.Tx) error { //nolint:dupl + if userid == "" { + return nil + } + + userdb := tx.Bucket([]byte(bolt.UserDataBucket)) + userBucket := userdb.Bucket([]byte(userid)) + + if userBucket == nil { + return nil + } + + mdata := userBucket.Get([]byte(bolt.BookmarkedReposKey)) + if mdata == nil { + return nil + } + + if err := json.Unmarshal(mdata, &bookmarkedRepos); err != nil { + bdw.Log.Info().Str("user", userid).Err(err).Msg("unmarshal error") + + return zerr.ErrInvalidOldUserBookmarkedRepos + } + + if bookmarkedRepos == nil { + bookmarkedRepos = make([]string, 0) + } + + return nil + }) + + return bookmarkedRepos, err +} + func (bdw *DBWrapper) PatchDB() error { var DBVersion string @@ -1590,3 +1843,69 @@ func (bdw *DBWrapper) PatchDB() error { return nil } + +func getUserStars(ctx context.Context, transaction *bbolt.Tx) []string { + acCtx, err := localCtx.GetAccessControlContext(ctx) + if err != nil { + return []string{} + } + + var ( + userid = localCtx.GetUsernameFromContext(acCtx) + starredRepos = []string{} + userdb = transaction.Bucket([]byte(bolt.UserDataBucket)) + userBucket = userdb.Bucket([]byte(userid)) + ) + + if userid == "" { + return []string{} + } + + if userBucket == nil { + return []string{} + } + + mdata := userBucket.Get([]byte(bolt.StarredReposKey)) + if mdata == nil { + return []string{} + } + + if err := json.Unmarshal(mdata, &starredRepos); err != nil { + return []string{} + } + + return starredRepos +} + +func getUserBookmarks(ctx context.Context, transaction *bbolt.Tx) []string { + acCtx, err := localCtx.GetAccessControlContext(ctx) + if err != nil { + return []string{} + } + + var ( + userid = localCtx.GetUsernameFromContext(acCtx) + bookmarkedRepos = []string{} + userdb = transaction.Bucket([]byte(bolt.UserDataBucket)) + userBucket = userdb.Bucket([]byte(userid)) + ) + + if userid == "" { + return []string{} + } + + if userBucket == nil { + return []string{} + } + + mdata := userBucket.Get([]byte(bolt.BookmarkedReposKey)) + if mdata == nil { + return []string{} + } + + if err := json.Unmarshal(mdata, &bookmarkedRepos); err != nil { + return []string{} + } + + return bookmarkedRepos +} diff --git a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go index a0e9bfda..5f3227ab 100644 --- a/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go +++ b/pkg/meta/repodb/boltdb-wrapper/boltdb_wrapper_test.go @@ -3,7 +3,6 @@ package bolt_test import ( "context" "encoding/json" - "os" "testing" "github.com/opencontainers/go-digest" @@ -15,6 +14,7 @@ import ( "zotregistry.io/zot/pkg/meta/bolt" "zotregistry.io/zot/pkg/meta/repodb" boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper" + localCtx "zotregistry.io/zot/pkg/requestcontext" "zotregistry.io/zot/pkg/test" ) @@ -29,7 +29,6 @@ func TestWrapperErrors(t *testing.T) { log := log.NewLogger("debug", "") boltdbWrapper, err := boltdb_wrapper.NewBoltDBWrapper(boltDriver, log) - defer os.Remove("repo.db") So(boltdbWrapper, ShouldNotBeNil) So(err, ShouldBeNil) @@ -77,6 +76,21 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("FilterRepos", func() { + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) + err := buck.Put([]byte("badRepo"), []byte("bad repo")) + So(err, ShouldBeNil) + + return nil + }) + So(err, ShouldBeNil) + + _, _, _, _, err = boltdbWrapper.FilterRepos(context.Background(), + func(repoMeta repodb.RepoMetadata) bool { return true }, repodb.PageInput{}) + So(err, ShouldNotBeNil) + }) + Convey("GetArtifactData", func() { err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { artifactBuck := tx.Bucket([]byte(bolt.ArtifactDataBucket)) @@ -705,6 +719,184 @@ func TestWrapperErrors(t *testing.T) { }) }) + Convey("ToggleStarRepo bad context errors", func() { + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, "bad context") + + _, err := boltdbWrapper.ToggleStarRepo(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("ToggleStarRepo, getting StarredRepoKey from bucket fails", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": true, + }, + Username: "username", + } + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + userdb, err := tx.CreateBucketIfNotExists([]byte(bolt.UserDataBucket)) + So(err, ShouldBeNil) + userBucket, err := userdb.CreateBucketIfNotExists([]byte(acCtx.Username)) + So(err, ShouldBeNil) + + err = userBucket.Put([]byte(bolt.StarredReposKey), []byte("bad array")) + So(err, ShouldBeNil) + + return nil + }) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.ToggleStarRepo(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("ToggleBookmarkRepo, unmarshal error", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": true, + }, + Username: "username", + } + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + userdb, err := tx.CreateBucketIfNotExists([]byte(bolt.UserDataBucket)) + So(err, ShouldBeNil) + userBucket, err := userdb.CreateBucketIfNotExists([]byte(acCtx.Username)) + So(err, ShouldBeNil) + + err = userBucket.Put([]byte(bolt.BookmarkedReposKey), []byte("bad array")) + So(err, ShouldBeNil) + + return nil + }) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.ToggleBookmarkRepo(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("ToggleStarRepo, no repoMeta found", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": true, + }, + Username: "username", + } + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket)) + + err := repoBuck.Put([]byte("repo"), []byte("bad repo")) + So(err, ShouldBeNil) + + return nil + }) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.ToggleStarRepo(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("ToggleStarRepo, bad repoMeta found", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": true, + }, + Username: "username", + } + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + _, err = boltdbWrapper.ToggleStarRepo(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("ToggleBookmarkRepo bad context errors", func() { + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, "bad context") + + _, err := boltdbWrapper.ToggleBookmarkRepo(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("GetStarredRepos bad context errors", func() { + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, "bad context") + + _, err := boltdbWrapper.GetStarredRepos(ctx) + So(err, ShouldNotBeNil) + }) + + Convey("GetStarredRepos user data unmarshal error", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": true, + }, + Username: "username", + } + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + userdb, err := tx.CreateBucketIfNotExists([]byte(bolt.UserDataBucket)) + So(err, ShouldBeNil) + userBucket, err := userdb.CreateBucketIfNotExists([]byte(acCtx.Username)) + So(err, ShouldBeNil) + + err = userBucket.Put([]byte(bolt.StarredReposKey), []byte("bad array")) + So(err, ShouldBeNil) + + return nil + }) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.GetStarredRepos(ctx) + So(err, ShouldNotBeNil) + }) + + Convey("GetBookmarkedRepos user data unmarshal error", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": true, + }, + Username: "username", + } + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { + userdb, err := tx.CreateBucketIfNotExists([]byte(bolt.UserDataBucket)) + So(err, ShouldBeNil) + userBucket, err := userdb.CreateBucketIfNotExists([]byte(acCtx.Username)) + So(err, ShouldBeNil) + + err = userBucket.Put([]byte(bolt.BookmarkedReposKey), []byte("bad array")) + So(err, ShouldBeNil) + + return nil + }) + So(err, ShouldBeNil) + + _, err = boltdbWrapper.GetBookmarkedRepos(ctx) + So(err, ShouldNotBeNil) + }) + + Convey("GetBookmarkedRepos bad context errors", func() { + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, "bad context") + + _, err := boltdbWrapper.GetBookmarkedRepos(ctx) + So(err, ShouldNotBeNil) + }) + Convey("Unsuported type", func() { digest := digest.FromString("digest") diff --git a/pkg/meta/repodb/common.go b/pkg/meta/repodb/common.go index db81896a..3fdf4917 100644 --- a/pkg/meta/repodb/common.go +++ b/pkg/meta/repodb/common.go @@ -8,7 +8,7 @@ import ( // that's not directly available in the RepoMetadata structure (ex. that needs to be calculated // by iterating the manifests, etc.) type DetailedRepoMeta struct { - RepoMeta RepoMetadata + RepoMetadata Rank int Downloads int UpdateTime time.Time @@ -26,13 +26,13 @@ func SortFunctions() map[SortCriteria]func(pageBuffer []DetailedRepoMeta) func(i func SortByAlphabeticAsc(pageBuffer []DetailedRepoMeta) func(i, j int) bool { return func(i, j int) bool { - return pageBuffer[i].RepoMeta.Name < pageBuffer[j].RepoMeta.Name + return pageBuffer[i].Name < pageBuffer[j].Name } } func SortByAlphabeticDsc(pageBuffer []DetailedRepoMeta) func(i, j int) bool { return func(i, j int) bool { - return pageBuffer[i].RepoMeta.Name > pageBuffer[j].RepoMeta.Name + return pageBuffer[i].Name > pageBuffer[j].Name } } diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go index 521b5c1d..8721d1e2 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_internal_test.go @@ -31,6 +31,7 @@ func TestWrapperErrors(t *testing.T) { manifestDataTablename := "ManifestDataTable" + uuid.String() indexDataTablename := "IndexDataTable" + uuid.String() artifactDataTablename := "ArtifactDataTable" + uuid.String() + userDataTablename := "UserDataTable" + uuid.String() versionTablename := "Version" + uuid.String() @@ -58,6 +59,7 @@ func TestWrapperErrors(t *testing.T) { IndexDataTablename: indexDataTablename, ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, + UserDataTablename: userDataTablename, Patches: version.GetDynamoDBPatches(), Log: log.Logger{Logger: zerolog.New(os.Stdout)}, } @@ -98,6 +100,7 @@ func TestWrapperErrors(t *testing.T) { VersionTablename: versionTablename, IndexDataTablename: indexDataTablename, ArtifactDataTablename: artifactDataTablename, + UserDataTablename: userDataTablename, Patches: version.GetDynamoDBPatches(), Log: log.Logger{Logger: zerolog.New(os.Stdout)}, } diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go index ec46964b..bfbe1b51 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_test.go @@ -21,6 +21,7 @@ import ( "zotregistry.io/zot/pkg/meta/dynamo" "zotregistry.io/zot/pkg/meta/repodb" dynamoWrapper "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper" + localCtx "zotregistry.io/zot/pkg/requestcontext" "zotregistry.io/zot/pkg/test" ) @@ -42,6 +43,7 @@ func TestIterator(t *testing.T) { versionTablename := "Version" + uuid.String() indexDataTablename := "IndexDataTable" + uuid.String() artifactDataTablename := "ArtifactDataTable" + uuid.String() + userDataTablename := "UserDataTable" + uuid.String() log := log.NewLogger("debug", "") @@ -54,6 +56,7 @@ func TestIterator(t *testing.T) { IndexDataTablename: indexDataTablename, ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, + UserDataTablename: userDataTablename, } client, err := dynamo.GetDynamoClient(params) So(err, ShouldBeNil) @@ -143,6 +146,7 @@ func TestWrapperErrors(t *testing.T) { versionTablename := "Version" + uuid.String() indexDataTablename := "IndexDataTable" + uuid.String() artifactDataTablename := "ArtifactData" + uuid.String() + userDataTablename := "UserDataTable" + uuid.String() ctx := context.Background() @@ -156,6 +160,7 @@ func TestWrapperErrors(t *testing.T) { ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, ArtifactDataTablename: artifactDataTablename, + UserDataTablename: userDataTablename, VersionTablename: versionTablename, } client, err := dynamo.GetDynamoClient(params) //nolint:contextcheck @@ -167,6 +172,179 @@ func TestWrapperErrors(t *testing.T) { So(dynamoWrapper.ResetManifestDataTable(), ShouldBeNil) //nolint:contextcheck So(dynamoWrapper.ResetRepoMetaTable(), ShouldBeNil) //nolint:contextcheck + Convey("ToggleBookmarkRepo no access", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": false, + }, + Username: "username", + } + + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + _, err := dynamoWrapper.ToggleBookmarkRepo(ctx, "unaccesible") + So(err, ShouldNotBeNil) + }) + + Convey("ToggleBookmarkRepo GetUserMeta no user data", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": true, + }, + Username: "username", + } + + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + status, err := dynamoWrapper.ToggleBookmarkRepo(ctx, "repo") + So(err, ShouldBeNil) + So(status, ShouldEqual, repodb.NotChanged) + }) + + Convey("ToggleBookmarkRepo GetUserMeta client error", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": true, + }, + Username: "username", + } + + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + dynamoWrapper.UserDataTablename = badTablename + + status, err := dynamoWrapper.ToggleBookmarkRepo(ctx, "repo") + So(err, ShouldNotBeNil) + So(status, ShouldEqual, repodb.NotChanged) + }) + + Convey("GetBookmarkedRepos", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": true, + }, + Username: "username", + } + + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + repos, err := dynamoWrapper.GetBookmarkedRepos(ctx) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + }) + + Convey("ToggleStarRepo GetUserMeta bad context", func() { + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, "some bad context") + + _, err := dynamoWrapper.ToggleStarRepo(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("ToggleStarRepo GetUserMeta no access", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": false, + }, + Username: "username", + } + + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + _, err := dynamoWrapper.ToggleStarRepo(ctx, "unaccesible") + So(err, ShouldNotBeNil) + }) + + Convey("ToggleStarRepo GetUserMeta error", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": false, + }, + Username: "username", + } + + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + dynamoWrapper.UserDataTablename = badTablename + + _, err := dynamoWrapper.ToggleStarRepo(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("ToggleStarRepo GetRepoMeta error", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": true, + }, + Username: "username", + } + + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + dynamoWrapper.RepoMetaTablename = badTablename + + _, err := dynamoWrapper.ToggleStarRepo(ctx, "repo") + So(err, ShouldNotBeNil) + }) + + Convey("GetUserMeta bad context", func() { + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, "bad context") + + userData, err := dynamoWrapper.GetUserMeta(ctx) + So(err, ShouldNotBeNil) + So(userData.BookmarkedRepos, ShouldBeEmpty) + So(userData.StarredRepos, ShouldBeEmpty) + }) + + Convey("GetUserMeta client error", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": true, + }, + Username: "username", + } + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + dynamoWrapper.UserDataTablename = badTablename + + _, err := dynamoWrapper.GetUserMeta(ctx) + So(err, ShouldNotBeNil) + }) + + Convey("GetUserMeta unmarshal error, bad user data", func() { + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + "repo": true, + }, + Username: "username", + } + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + err := setBadUserData(dynamoWrapper.Client, userDataTablename, acCtx.Username) + So(err, ShouldBeNil) + + _, err = dynamoWrapper.GetUserMeta(ctx) + So(err, ShouldNotBeNil) + }) + + Convey("SetUserMeta bad context", func() { + authzCtxKey := localCtx.GetContextKey() + ctx := context.WithValue(context.Background(), authzCtxKey, "bad context") + + err := dynamoWrapper.SetUserMeta(ctx, repodb.UserData{}) + So(err, ShouldNotBeNil) + }) + Convey("SetManifestData", func() { dynamoWrapper.ManifestDataTablename = "WRONG tables" @@ -253,6 +431,13 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("SetRepoReference client error", func() { + dynamoWrapper.RepoMetaTablename = badTablename + digest := digest.FromString("str") + err := dynamoWrapper.SetRepoReference("repo", digest.String(), digest, ispec.MediaTypeImageManifest) + So(err, ShouldNotBeNil) + }) + Convey("SetReferrer client error", func() { dynamoWrapper.RepoMetaTablename = badTablename err := dynamoWrapper.SetReferrer("repo", "", repodb.ReferrerInfo{}) @@ -683,6 +868,19 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("FilterRepos NewBaseRepoPageFinder errors", func() { + _, _, _, _, err := dynamoWrapper.SearchRepos(ctx, "text", repodb.Filter{}, repodb.PageInput{Offset: -2, Limit: -2}) + So(err, ShouldNotBeNil) + }) + + Convey("FilterRepos attributevalue.Unmarshal(repoMetaAttribute) errors", func() { + err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck + So(err, ShouldBeNil) + + _, _, _, _, err := dynamoWrapper.SearchRepos(ctx, "repo", repodb.Filter{}, repodb.PageInput{}) + So(err, ShouldNotBeNil) + }) + Convey("FilterTags repoMeta unmarshal error", func() { err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck So(err, ShouldBeNil) @@ -804,6 +1002,41 @@ func TestWrapperErrors(t *testing.T) { ) So(err, ShouldBeNil) }) + + Convey("PatchDB dwr.getDBVersion errors", func() { + dynamoWrapper.VersionTablename = badTablename + + err := dynamoWrapper.PatchDB() + So(err, ShouldNotBeNil) + }) + + Convey("PatchDB patchIndex < version.GetVersionIndex", func() { + err := setVersion(dynamoWrapper.Client, versionTablename, "V2") + So(err, ShouldBeNil) + + dynamoWrapper.Patches = []func(client *dynamodb.Client, tableNames map[string]string) error{ + func(client *dynamodb.Client, tableNames map[string]string) error { return nil }, + func(client *dynamodb.Client, tableNames map[string]string) error { return nil }, + func(client *dynamodb.Client, tableNames map[string]string) error { return nil }, + } + + err = dynamoWrapper.PatchDB() + So(err, ShouldBeNil) + }) + + Convey("ResetRepoMetaTable client errors", func() { + dynamoWrapper.RepoMetaTablename = badTablename + + err := dynamoWrapper.ResetRepoMetaTable() + So(err, ShouldNotBeNil) + }) + + Convey("getDBVersion client errors", func() { + dynamoWrapper.VersionTablename = badTablename + + err := dynamoWrapper.PatchDB() + So(err, ShouldNotBeNil) + }) }) Convey("NewDynamoDBWrapper errors", t, func() { @@ -814,6 +1047,7 @@ func TestWrapperErrors(t *testing.T) { ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, ArtifactDataTablename: artifactDataTablename, + UserDataTablename: userDataTablename, VersionTablename: versionTablename, } client, err := dynamo.GetDynamoClient(params) @@ -829,6 +1063,7 @@ func TestWrapperErrors(t *testing.T) { ManifestDataTablename: "", IndexDataTablename: indexDataTablename, ArtifactDataTablename: artifactDataTablename, + UserDataTablename: userDataTablename, VersionTablename: versionTablename, } client, err = dynamo.GetDynamoClient(params) @@ -844,6 +1079,7 @@ func TestWrapperErrors(t *testing.T) { ManifestDataTablename: manifestDataTablename, IndexDataTablename: "", ArtifactDataTablename: artifactDataTablename, + UserDataTablename: userDataTablename, VersionTablename: versionTablename, } client, err = dynamo.GetDynamoClient(params) @@ -859,6 +1095,7 @@ func TestWrapperErrors(t *testing.T) { ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, ArtifactDataTablename: artifactDataTablename, + UserDataTablename: userDataTablename, VersionTablename: "", } client, err = dynamo.GetDynamoClient(params) @@ -874,6 +1111,7 @@ func TestWrapperErrors(t *testing.T) { ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, ArtifactDataTablename: "", + UserDataTablename: userDataTablename, VersionTablename: versionTablename, } client, err = dynamo.GetDynamoClient(params) @@ -889,6 +1127,7 @@ func TestWrapperErrors(t *testing.T) { ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, VersionTablename: versionTablename, + UserDataTablename: userDataTablename, ArtifactDataTablename: artifactDataTablename, } client, err = dynamo.GetDynamoClient(params) @@ -896,6 +1135,22 @@ func TestWrapperErrors(t *testing.T) { _, err = dynamoWrapper.NewDynamoDBWrapper(client, params, log) So(err, ShouldBeNil) + + params = dynamo.DBDriverParameters{ //nolint:contextcheck + Endpoint: endpoint, + Region: region, + RepoMetaTablename: repoMetaTablename, + ManifestDataTablename: manifestDataTablename, + IndexDataTablename: indexDataTablename, + VersionTablename: versionTablename, + UserDataTablename: "", + ArtifactDataTablename: artifactDataTablename, + } + client, err = dynamo.GetDynamoClient(params) + So(err, ShouldBeNil) + + _, err = dynamoWrapper.NewDynamoDBWrapper(client, params, log) + So(err, ShouldNotBeNil) }) } @@ -999,6 +1254,56 @@ func setBadRepoMeta(client *dynamodb.Client, repoMetadataTableName, repoName str return err } +func setBadUserData(client *dynamodb.Client, userDataTablename, userID string) error { + userAttributeValue, err := attributevalue.Marshal("string") + if err != nil { + return err + } + + _, err = client.UpdateItem(context.Background(), &dynamodb.UpdateItemInput{ + ExpressionAttributeNames: map[string]string{ + "#UM": "UserData", + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":UserData": userAttributeValue, + }, + Key: map[string]types.AttributeValue{ + "UserID": &types.AttributeValueMemberS{ + Value: userID, + }, + }, + TableName: aws.String(userDataTablename), + UpdateExpression: aws.String("SET #UM = :UserData"), + }) + + return err +} + +func setVersion(client *dynamodb.Client, versionTablename string, version string) error { + mdAttributeValue, err := attributevalue.Marshal(version) + if err != nil { + return err + } + + _, err = client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ + ExpressionAttributeNames: map[string]string{ + "#V": "Version", + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":Version": mdAttributeValue, + }, + Key: map[string]types.AttributeValue{ + "VersionKey": &types.AttributeValueMemberS{ + Value: "DBVersion", + }, + }, + TableName: aws.String(versionTablename), + UpdateExpression: aws.String("SET #V = :Version"), + }) + + return err +} + func setRepoMeta(client *dynamodb.Client, repoMetadataTableName string, repoMeta repodb.RepoMetadata) error { repoAttributeValue, err := attributevalue.Marshal(repoMeta) if err != nil { diff --git a/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go b/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go index 46585acd..d1309a59 100644 --- a/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go +++ b/pkg/meta/repodb/dynamodb-wrapper/dynamo_wrapper.go @@ -16,6 +16,7 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" zerr "zotregistry.io/zot/errors" + zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/common" "zotregistry.io/zot/pkg/meta/dynamo" @@ -32,6 +33,7 @@ type DBWrapper struct { IndexDataTablename string ManifestDataTablename string ArtifactDataTablename string + UserDataTablename string VersionTablename string Patches []func(client *dynamodb.Client, tableNames map[string]string) error Log log.Logger @@ -45,6 +47,7 @@ func NewDynamoDBWrapper(client *dynamodb.Client, params dynamo.DBDriverParameter IndexDataTablename: params.IndexDataTablename, ArtifactDataTablename: params.ArtifactDataTablename, VersionTablename: params.VersionTablename, + UserDataTablename: params.UserDataTablename, Patches: version.GetDynamoDBPatches(), Log: log, } @@ -74,6 +77,11 @@ func NewDynamoDBWrapper(client *dynamodb.Client, params dynamo.DBDriverParameter return nil, err } + err = dynamoWrapper.createUserDataTable() + if err != nil { + return nil, err + } + // Using the Config value, create the DynamoDB client return &dynamoWrapper, nil } @@ -751,7 +759,7 @@ func (dwr *DBWrapper) GetMultipleRepoMeta(ctx context.Context, if filter(repoMeta) { pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repoMeta, + RepoMetadata: repoMeta, }) } } @@ -770,6 +778,9 @@ func (dwr *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter repoMetaAttributeIterator dynamo.AttributesIterator pageFinder repodb.PageFinder pageInfo repodb.PageInfo + + userBookmarks = getUserBookmarks(ctx, dwr) + userStars = getUserStars(ctx, dwr) ) repoMetaAttributeIterator = dynamo.NewBaseDynamoAttributesIterator( @@ -786,7 +797,6 @@ func (dwr *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { if err != nil { - // log return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, err } @@ -803,119 +813,143 @@ func (dwr *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter continue } - if rank := common.RankRepoName(searchText, repoMeta.Name); rank != -1 { - var ( - // specific values used for sorting that need to be calculated based on all manifests from the repo - repoDownloads = 0 - repoLastUpdated = time.Time{} - noImageChecked = true - osSet = map[string]bool{} - archSet = map[string]bool{} - isSigned = false - ) + rank := common.RankRepoName(searchText, repoMeta.Name) + if rank == -1 { + continue + } - for _, descriptor := range repoMeta.Tags { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDigest := descriptor.Digest + repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) + repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) - manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck - manifestMetadataMap) - if err != nil { - return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, - pageInfo, - fmt.Errorf("%w", err) - } + var ( + repoDownloads = 0 + repoLastUpdated = time.Time{} + osSet = map[string]bool{} + archSet = map[string]bool{} + noImageChecked = true + isSigned = false + ) - manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) - if err != nil { - return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, - pageInfo, - fmt.Errorf("%w", err) - } + for _, descriptor := range repoMeta.Tags { + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestDigest := descriptor.Digest - repoDownloads += manifestFilterData.DownloadCount - - for _, os := range manifestFilterData.OsList { - osSet[os] = true - } - - for _, arch := range manifestFilterData.ArchList { - archSet[arch] = true - } - - repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned, - noImageChecked, manifestFilterData) - - manifestMetadataMap[descriptor.Digest] = manifestMeta - case ispec.MediaTypeImageIndex: - indexDigest := descriptor.Digest - - indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck - if err != nil { - return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, - pageInfo, - fmt.Errorf("%w", err) - } - - // this also updates manifestMetadataMap - indexFilterData, err := dwr.collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, //nolint:contextcheck - manifestMetadataMap) - if err != nil { - return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, - pageInfo, - fmt.Errorf("%w", err) - } - - for _, arch := range indexFilterData.ArchList { - archSet[arch] = true - } - - for _, os := range indexFilterData.OsList { - osSet[os] = true - } - - repoDownloads += indexFilterData.DownloadCount - - repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned, - noImageChecked, indexFilterData) - - indexDataMap[indexDigest] = indexData - default: - dwr.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType) - - continue + manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck + manifestMetadataMap) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, + pageInfo, + fmt.Errorf("%w", err) } - } - repoFilterData := repodb.FilterData{ - OsList: common.GetMapKeys(osSet), - ArchList: common.GetMapKeys(archSet), - LastUpdated: repoLastUpdated, - DownloadCount: repoDownloads, - IsSigned: isSigned, - } + manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, + pageInfo, + fmt.Errorf("%w", err) + } + + repoDownloads += manifestFilterData.DownloadCount + + for _, os := range manifestFilterData.OsList { + osSet[os] = true + } + + for _, arch := range manifestFilterData.ArchList { + archSet[arch] = true + } + + repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned, + noImageChecked, manifestFilterData) + + manifestMetadataMap[descriptor.Digest] = manifestMeta + case ispec.MediaTypeImageIndex: + indexDigest := descriptor.Digest + + indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, + pageInfo, + fmt.Errorf("%w", err) + } + + // this also updates manifestMetadataMap + indexFilterData, err := dwr.collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, //nolint:contextcheck + manifestMetadataMap) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, + pageInfo, + fmt.Errorf("%w", err) + } + + for _, arch := range indexFilterData.ArchList { + archSet[arch] = true + } + + for _, os := range indexFilterData.OsList { + osSet[os] = true + } + + repoDownloads += indexFilterData.DownloadCount + + repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned, + noImageChecked, indexFilterData) + + indexDataMap[indexDigest] = indexData + default: + dwr.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType) - if !common.AcceptedByFilter(filter, repoFilterData) { continue } - - pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repoMeta, - Rank: rank, - Downloads: repoDownloads, - UpdateTime: repoLastUpdated, - }) } + + repoFilterData := repodb.FilterData{ + OsList: common.GetMapKeys(osSet), + ArchList: common.GetMapKeys(archSet), + LastUpdated: repoLastUpdated, + DownloadCount: repoDownloads, + IsSigned: isSigned, + } + + if !common.AcceptedByFilter(filter, repoFilterData) { + continue + } + + pageFinder.Add(repodb.DetailedRepoMeta{ + RepoMetadata: repoMeta, + Rank: rank, + Downloads: repoDownloads, + UpdateTime: repoLastUpdated, + }) } foundRepos, pageInfo := pageFinder.Page() - foundManifestMetadataMap, foundindexDataMap, err := filterFoundData(foundRepos, manifestMetadataMap, indexDataMap) + foundManifestMetadataMap, foundindexDataMap, err := common.FilterDataByRepo(foundRepos, manifestMetadataMap, + indexDataMap) return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err } +func getUserStars(ctx context.Context, dwr *DBWrapper) []string { + starredRepos, err := dwr.GetStarredRepos(ctx) + if err != nil { + return []string{} + } + + return starredRepos +} + +func getUserBookmarks(ctx context.Context, dwr *DBWrapper) []string { + bookmarkedRepos, err := dwr.GetBookmarkedRepos(ctx) + if err != nil { + return []string{} + } + + return bookmarkedRepos +} + func (dwr *DBWrapper) fetchManifestMetaWithCheck(repoName string, manifestDigest string, manifestMetadataMap map[string]repodb.ManifestMetadata, ) (repodb.ManifestMetadata, error) { @@ -1049,9 +1083,11 @@ func (dwr *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, var ( manifestMetadataMap = make(map[string]repodb.ManifestMetadata) indexDataMap = make(map[string]repodb.IndexData) - pageFinder repodb.PageFinder repoMetaAttributeIterator dynamo.AttributesIterator + pageFinder repodb.PageFinder pageInfo repodb.PageInfo + userBookmarks = getUserBookmarks(ctx, dwr) + userStars = getUserStars(ctx, dwr) ) repoMetaAttributeIterator = dynamo.NewBaseDynamoAttributesIterator( @@ -1068,7 +1104,6 @@ func (dwr *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { if err != nil { - // log return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, err } @@ -1084,8 +1119,11 @@ func (dwr *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil { continue } + + repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) + repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) + matchedTags := make(map[string]repodb.Descriptor) - // take all manifestMetas for tag, descriptor := range repoMeta.Tags { matchedTags[tag] = descriptor @@ -1172,29 +1210,89 @@ func (dwr *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc, repoMeta.Tags = matchedTags pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repoMeta, + RepoMetadata: repoMeta, }) } foundRepos, pageInfo := pageFinder.Page() - foundManifestMetadataMap, foundindexDataMap, err := filterFoundData(foundRepos, manifestMetadataMap, indexDataMap) + foundManifestMetadataMap, foundindexDataMap, err := common.FilterDataByRepo(foundRepos, manifestMetadataMap, + indexDataMap) return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err } +func (dwr *DBWrapper) FilterRepos(ctx context.Context, + filter repodb.FilterRepoFunc, + requestedPage repodb.PageInput, +) ( + []repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error, +) { + var ( + repoMetaAttributeIterator dynamo.AttributesIterator + pageInfo repodb.PageInfo + userBookmarks = getUserBookmarks(ctx, dwr) + userStars = getUserStars(ctx, dwr) + ) + + repoMetaAttributeIterator = dynamo.NewBaseDynamoAttributesIterator( + dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, + ) + + pageFinder, err := repodb.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, err + } + + repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, err + } + + for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, err + } + + var repoMeta repodb.RepoMetadata + + err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, err + } + + if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil { + continue + } + + repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) + repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) + + if filter(repoMeta) { + pageFinder.Add(repodb.DetailedRepoMeta{ + RepoMetadata: repoMeta, + }) + } + } + + foundRepos, pageInfo := pageFinder.Page() + + foundManifestMetadataMap, foundIndexDataMap, err := common.FetchDataForRepos(dwr, foundRepos) + + return foundRepos, foundManifestMetadataMap, foundIndexDataMap, pageInfo, err +} + func (dwr *DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) { var ( manifestMetadataMap = make(map[string]repodb.ManifestMetadata) indexDataMap = make(map[string]repodb.IndexData) - repoMetaAttributeIterator = dynamo.NewBaseDynamoAttributesIterator( - dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, - ) - - pageFinder repodb.PageFinder - pageInfo repodb.PageInfo + repoMetaAttributeIterator dynamo.AttributesIterator + pageFinder repodb.PageFinder + pageInfo repodb.PageInfo + userBookmarks = getUserBookmarks(ctx, dwr) + userStars = getUserStars(ctx, dwr) ) pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy) @@ -1203,6 +1301,10 @@ func (dwr *DBWrapper) SearchTags(ctx context.Context, searchText string, filter pageInfo, err } + repoMetaAttributeIterator = dynamo.NewBaseDynamoAttributesIterator( + dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, + ) + searchedRepo, searchedTag, err := common.GetRepoTag(searchText) if err != nil { return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, @@ -1231,165 +1333,131 @@ func (dwr *DBWrapper) SearchTags(ctx context.Context, searchText string, filter continue } - if repoMeta.Name == searchedRepo { - matchedTags := make(map[string]repodb.Descriptor) - // take all manifestMetas - for tag, descriptor := range repoMeta.Tags { - if !strings.HasPrefix(tag, searchedTag) { + if repoMeta.Name != searchedRepo { + continue + } + + repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) + repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) + + matchedTags := make(map[string]repodb.Descriptor) + + for tag, descriptor := range repoMeta.Tags { + if !strings.HasPrefix(tag, searchedTag) { + continue + } + + matchedTags[tag] = descriptor + + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestDigest := descriptor.Digest + + manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck + manifestMetadataMap) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, + pageInfo, + fmt.Errorf("repodb: error while unmashaling manifest metadata for digest %s %w", descriptor.Digest, err) + } + + imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, + pageInfo, + fmt.Errorf("%w", err) + } + + if !common.AcceptedByFilter(filter, imageFilterData) { + delete(matchedTags, tag) + continue } - matchedTags[tag] = descriptor + manifestMetadataMap[descriptor.Digest] = manifestMeta + case ispec.MediaTypeImageIndex: + indexDigest := descriptor.Digest - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDigest := descriptor.Digest + indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, + pageInfo, + fmt.Errorf("%w", err) + } + + var indexContent ispec.Index + + err = json.Unmarshal(indexData.IndexBlob, &indexContent) + if err != nil { + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, + pageInfo, + fmt.Errorf("repodb: error while unmashaling index content for digest %s %w", indexDigest, err) + } + + manifestHasBeenMatched := false + + for _, manifest := range indexContent.Manifests { + manifestDigest := manifest.Digest.String() manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck manifestMetadataMap) if err != nil { return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, - fmt.Errorf("repodb: error while unmashaling manifest metadata for digest %s %w", descriptor.Digest, err) + fmt.Errorf("%w", err) } - imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) + manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) if err != nil { return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, fmt.Errorf("%w", err) } - if !common.AcceptedByFilter(filter, imageFilterData) { - delete(matchedTags, tag) + manifestMetadataMap[manifestDigest] = manifestMeta - continue + if common.AcceptedByFilter(filter, manifestFilterData) { + manifestHasBeenMatched = true } + } - manifestMetadataMap[descriptor.Digest] = manifestMeta - case ispec.MediaTypeImageIndex: - indexDigest := descriptor.Digest - - indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck - if err != nil { - return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, - pageInfo, - fmt.Errorf("%w", err) - } - - var indexContent ispec.Index - - err = json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, - pageInfo, - fmt.Errorf("repodb: error while unmashaling index content for digest %s %w", indexDigest, err) - } - - manifestHasBeenMatched := false + if !manifestHasBeenMatched { + delete(matchedTags, tag) for _, manifest := range indexContent.Manifests { - manifestDigest := manifest.Digest.String() - - manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck - manifestMetadataMap) - if err != nil { - return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, - pageInfo, - fmt.Errorf("%w", err) - } - - manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta) - if err != nil { - return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, - pageInfo, - fmt.Errorf("%w", err) - } - - manifestMetadataMap[manifestDigest] = manifestMeta - - if common.AcceptedByFilter(filter, manifestFilterData) { - manifestHasBeenMatched = true - } + delete(manifestMetadataMap, manifest.Digest.String()) } - if !manifestHasBeenMatched { - delete(matchedTags, tag) - - for _, manifest := range indexContent.Manifests { - delete(manifestMetadataMap, manifest.Digest.String()) - } - - continue - } - - indexDataMap[indexDigest] = indexData - default: - dwr.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType) - continue } - } - if len(matchedTags) == 0 { + indexDataMap[indexDigest] = indexData + default: + dwr.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType) + continue } - - repoMeta.Tags = matchedTags - - pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repoMeta, - }) } + + if len(matchedTags) == 0 { + continue + } + + repoMeta.Tags = matchedTags + + pageFinder.Add(repodb.DetailedRepoMeta{ + RepoMetadata: repoMeta, + }) } foundRepos, pageInfo := pageFinder.Page() - foundManifestMetadataMap, foundindexDataMap, err := filterFoundData(foundRepos, manifestMetadataMap, indexDataMap) + foundManifestMetadataMap, foundindexDataMap, err := common.FilterDataByRepo(foundRepos, manifestMetadataMap, + indexDataMap) return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err } -func filterFoundData(foundRepos []repodb.RepoMetadata, manifestMetadataMap map[string]repodb.ManifestMetadata, - indexDataMap map[string]repodb.IndexData, -) (map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, error) { - var ( - foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata) - foundindexDataMap = make(map[string]repodb.IndexData) - ) - - // keep just the manifestMeta we need - for _, repoMeta := range foundRepos { - for _, descriptor := range repoMeta.Tags { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest] - case ispec.MediaTypeImageIndex: - indexData := indexDataMap[descriptor.Digest] - - var indexContent ispec.Index - - err := json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, - fmt.Errorf("repodb: error while getting manifest data for digest %s %w", descriptor.Digest, err) - } - - for _, manifestDescriptor := range indexContent.Manifests { - manifestDigest := manifestDescriptor.Digest.String() - - foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest] - } - - foundindexDataMap[descriptor.Digest] = indexData - default: - } - } - } - - return foundManifestMetadataMap, foundindexDataMap, nil -} - func (dwr *DBWrapper) PatchDB() error { DBVersion, err := dwr.getDBVersion() if err != nil { @@ -1693,3 +1761,266 @@ func (dwr *DBWrapper) ResetManifestDataTable() error { return dwr.createManifestDataTable() } + +func (dwr *DBWrapper) ToggleBookmarkRepo(ctx context.Context, repo string) ( + repodb.ToggleState, error, +) { + res := repodb.NotChanged + + if ok, err := localCtx.RepoIsUserAvailable(ctx, repo); !ok || err != nil { + return res, zerr.ErrUserDataNotAllowed + } + + userMeta, err := dwr.GetUserMeta(ctx) + if err != nil { + if errors.Is(err, zerr.ErrUserDataNotFound) { + return repodb.NotChanged, nil + } + + return res, err + } + + if !zcommon.Contains(userMeta.BookmarkedRepos, repo) { + userMeta.BookmarkedRepos = append(userMeta.BookmarkedRepos, repo) + res = repodb.Added + } else { + userMeta.BookmarkedRepos = zcommon.RemoveFrom(userMeta.BookmarkedRepos, repo) + res = repodb.Removed + } + + if res != repodb.NotChanged { + err = dwr.SetUserMeta(ctx, userMeta) + } + + if err != nil { + res = repodb.NotChanged + + return res, err + } + + return res, nil +} + +func (dwr *DBWrapper) GetBookmarkedRepos(ctx context.Context) ([]string, error) { + userMeta, err := dwr.GetUserMeta(ctx) + + if errors.Is(err, zerr.ErrUserDataNotFound) { + return []string{}, nil + } + + return userMeta.BookmarkedRepos, err +} + +func (dwr *DBWrapper) ToggleStarRepo(ctx context.Context, repo string) ( + repodb.ToggleState, error, +) { + res := repodb.NotChanged + + acCtx, err := localCtx.GetAccessControlContext(ctx) + if err != nil { + return res, err + } + + userid := localCtx.GetUsernameFromContext(acCtx) + + if userid == "" { + // empty user is anonymous, it has no data + return res, zerr.ErrUserDataNotAllowed + } + + if ok, err := localCtx.RepoIsUserAvailable(ctx, repo); !ok || err != nil { + return res, zerr.ErrUserDataNotAllowed + } + + userData, err := dwr.GetUserMeta(ctx) + if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { + return res, err + } + + if !zcommon.Contains(userData.StarredRepos, repo) { + userData.StarredRepos = append(userData.StarredRepos, repo) + res = repodb.Added + } else { + userData.StarredRepos = zcommon.RemoveFrom(userData.StarredRepos, repo) + res = repodb.Removed + } + + if res != repodb.NotChanged { + repoMeta, err := dwr.GetRepoMeta(repo) //nolint:contextcheck + if err != nil { + return repodb.NotChanged, err + } + + switch res { + case repodb.Added: + repoMeta.Stars++ + case repodb.Removed: + repoMeta.Stars-- + } + + repoAttributeValue, err := attributevalue.Marshal(repoMeta) + if err != nil { + return repodb.NotChanged, err + } + + userAttributeValue, err := attributevalue.Marshal(userData) + if err != nil { + return repodb.NotChanged, err + } + + _, err = dwr.Client.TransactWriteItems(ctx, &dynamodb.TransactWriteItemsInput{ + TransactItems: []types.TransactWriteItem{ + { + // Update User Meta + Update: &types.Update{ + ExpressionAttributeNames: map[string]string{ + "#UM": "UserData", + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":UserData": userAttributeValue, + }, + Key: map[string]types.AttributeValue{ + "UserID": &types.AttributeValueMemberS{ + Value: userid, + }, + }, + TableName: aws.String(dwr.UserDataTablename), + UpdateExpression: aws.String("SET #UM = :UserData"), + }, + }, + { + // Update Repo Meta with updated repo stars + Update: &types.Update{ + ExpressionAttributeNames: map[string]string{ + "#RM": "RepoMetadata", + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":RepoMetadata": repoAttributeValue, + }, + Key: map[string]types.AttributeValue{ + "RepoName": &types.AttributeValueMemberS{ + Value: repo, + }, + }, + TableName: aws.String(dwr.RepoMetaTablename), + UpdateExpression: aws.String("SET #RM = :RepoMetadata"), + }, + }, + }, + }) + if err != nil { + return repodb.NotChanged, err + } + } + + return res, nil +} + +func (dwr *DBWrapper) GetStarredRepos(ctx context.Context) ([]string, error) { + userMeta, err := dwr.GetUserMeta(ctx) + + if errors.Is(err, zerr.ErrUserDataNotFound) { + return []string{}, nil + } + + return userMeta.StarredRepos, err +} + +func (dwr DBWrapper) GetUserMeta(ctx context.Context) (repodb.UserData, error) { + acCtx, err := localCtx.GetAccessControlContext(ctx) + if err != nil { + return repodb.UserData{}, err + } + + userid := localCtx.GetUsernameFromContext(acCtx) + + if userid == "" { + // empty user is anonymous, it has no data + return repodb.UserData{}, nil + } + + resp, err := dwr.Client.GetItem(ctx, &dynamodb.GetItemInput{ + TableName: aws.String(dwr.UserDataTablename), + Key: map[string]types.AttributeValue{ + "UserID": &types.AttributeValueMemberS{Value: userid}, + }, + }) + if err != nil { + return repodb.UserData{}, err + } + + if resp.Item == nil { + return repodb.UserData{}, zerr.ErrUserDataNotFound + } + + var userMeta repodb.UserData + + err = attributevalue.Unmarshal(resp.Item["UserData"], &userMeta) + if err != nil { + return repodb.UserData{}, err + } + + return userMeta, nil +} + +func (dwr DBWrapper) createUserDataTable() error { + _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ + TableName: aws.String(dwr.UserDataTablename), + AttributeDefinitions: []types.AttributeDefinition{ + { + AttributeName: aws.String("UserID"), + AttributeType: types.ScalarAttributeTypeS, + }, + }, + KeySchema: []types.KeySchemaElement{ + { + AttributeName: aws.String("UserID"), + KeyType: types.KeyTypeHash, + }, + }, + BillingMode: types.BillingModePayPerRequest, + }) + + if err != nil && !strings.Contains(err.Error(), "Table already exists") { + return err + } + + return dwr.waitTableToBeCreated(dwr.UserDataTablename) +} + +func (dwr DBWrapper) SetUserMeta(ctx context.Context, userMeta repodb.UserData) error { + acCtx, err := localCtx.GetAccessControlContext(ctx) + if err != nil { + return err + } + + userid := localCtx.GetUsernameFromContext(acCtx) + + if userid == "" { + // empty user is anonymous, it has no data + return zerr.ErrUserDataNotAllowed + } + + userAttributeValue, err := attributevalue.Marshal(userMeta) + if err != nil { + return err + } + + _, err = dwr.Client.UpdateItem(ctx, &dynamodb.UpdateItemInput{ + ExpressionAttributeNames: map[string]string{ + "#UM": "UserData", + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":UserData": userAttributeValue, + }, + Key: map[string]types.AttributeValue{ + "UserID": &types.AttributeValueMemberS{ + Value: userid, + }, + }, + TableName: aws.String(dwr.UserDataTablename), + UpdateExpression: aws.String("SET #UM = :UserData"), + }) + + return err +} diff --git a/pkg/meta/repodb/pagination.go b/pkg/meta/repodb/pagination.go index 30c2fd9e..1b4af9cb 100644 --- a/pkg/meta/repodb/pagination.go +++ b/pkg/meta/repodb/pagination.go @@ -93,7 +93,7 @@ func (bpt *RepoPageFinder) Page() ([]RepoMetadata, PageInfo) { repos := make([]RepoMetadata, 0, len(detailedReposPage)) for _, drm := range detailedReposPage { - repos = append(repos, drm.RepoMeta) + repos = append(repos, drm.RepoMetadata) } pageInfo.TotalCount = len(bpt.pageBuffer) @@ -150,7 +150,7 @@ func (bpt *ImagePageFinder) Page() ([]RepoMetadata, PageInfo) { pageInfo := PageInfo{} for _, drm := range bpt.pageBuffer { - repo := drm.RepoMeta + repo := drm.RepoMetadata pageInfo.TotalCount += len(repo.Tags) } @@ -167,7 +167,7 @@ func (bpt *ImagePageFinder) Page() ([]RepoMetadata, PageInfo) { if remainingOffset == 0 && remainingLimit == 0 { for _, drm := range bpt.pageBuffer { - repo := drm.RepoMeta + repo := drm.RepoMetadata repos = append(repos, repo) pageInfo.ItemCount += len(repo.Tags) @@ -178,13 +178,13 @@ func (bpt *ImagePageFinder) Page() ([]RepoMetadata, PageInfo) { // bring cursor to position in RepoMeta array for _, drm := range bpt.pageBuffer { - if remainingOffset < len(drm.RepoMeta.Tags) { + if remainingOffset < len(drm.Tags) { tagStartIndex = remainingOffset break } - remainingOffset -= len(drm.RepoMeta.Tags) + remainingOffset -= len(drm.Tags) repoStartIndex++ } @@ -195,7 +195,7 @@ func (bpt *ImagePageFinder) Page() ([]RepoMetadata, PageInfo) { // finish counting remaining tags inside the first repo meta partialTags := map[string]Descriptor{} - firstRepoMeta := bpt.pageBuffer[repoStartIndex].RepoMeta + firstRepoMeta := bpt.pageBuffer[repoStartIndex].RepoMetadata tags := make([]string, 0, len(firstRepoMeta.Tags)) for k := range firstRepoMeta.Tags { @@ -226,7 +226,7 @@ func (bpt *ImagePageFinder) Page() ([]RepoMetadata, PageInfo) { // continue with the remaining repos for i := repoStartIndex; i < len(bpt.pageBuffer); i++ { - repoMeta := bpt.pageBuffer[i].RepoMeta + repoMeta := bpt.pageBuffer[i].RepoMetadata if len(repoMeta.Tags) > remainingLimit { partialTags := map[string]Descriptor{} diff --git a/pkg/meta/repodb/pagination_test.go b/pkg/meta/repodb/pagination_test.go index a3c2f19d..809bc3b5 100644 --- a/pkg/meta/repodb/pagination_test.go +++ b/pkg/meta/repodb/pagination_test.go @@ -64,7 +64,7 @@ func TestPagination(t *testing.T) { So(pageFinder, ShouldNotBeNil) pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repodb.RepoMetadata{ + RepoMetadata: repodb.RepoMetadata{ Name: "repo1", Tags: map[string]repodb.Descriptor{ "tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest}, @@ -73,7 +73,7 @@ func TestPagination(t *testing.T) { }) pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repodb.RepoMetadata{ + RepoMetadata: repodb.RepoMetadata{ Name: "repo2", Tags: map[string]repodb.Descriptor{ "Tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest}, @@ -93,7 +93,7 @@ func TestPagination(t *testing.T) { So(pageFinder, ShouldNotBeNil) pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repodb.RepoMetadata{ + RepoMetadata: repodb.RepoMetadata{ Name: "repo1", Tags: map[string]repodb.Descriptor{ "tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest}, @@ -102,7 +102,7 @@ func TestPagination(t *testing.T) { }) pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repodb.RepoMetadata{ + RepoMetadata: repodb.RepoMetadata{ Name: "repo2", Tags: map[string]repodb.Descriptor{ "Tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest}, @@ -122,7 +122,7 @@ func TestPagination(t *testing.T) { So(pageFinder, ShouldNotBeNil) pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repodb.RepoMetadata{ + RepoMetadata: repodb.RepoMetadata{ Name: "repo1", Tags: map[string]repodb.Descriptor{ "tag1": { @@ -134,7 +134,7 @@ func TestPagination(t *testing.T) { }) pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repodb.RepoMetadata{ + RepoMetadata: repodb.RepoMetadata{ Name: "repo2", Tags: map[string]repodb.Descriptor{ "Tag1": { @@ -157,7 +157,7 @@ func TestPagination(t *testing.T) { }, }) pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repodb.RepoMetadata{ + RepoMetadata: repodb.RepoMetadata{ Name: "repo3", Tags: map[string]repodb.Descriptor{ "Tag11": { @@ -196,7 +196,7 @@ func TestPagination(t *testing.T) { So(pageFinder, ShouldNotBeNil) pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repodb.RepoMetadata{ + RepoMetadata: repodb.RepoMetadata{ Name: "repo1", Tags: map[string]repodb.Descriptor{ "tag1": { @@ -208,7 +208,7 @@ func TestPagination(t *testing.T) { }) pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repodb.RepoMetadata{ + RepoMetadata: repodb.RepoMetadata{ Name: "repo2", Tags: map[string]repodb.Descriptor{ "Tag1": { @@ -219,7 +219,7 @@ func TestPagination(t *testing.T) { }, }) pageFinder.Add(repodb.DetailedRepoMeta{ - RepoMeta: repodb.RepoMetadata{ + RepoMetadata: repodb.RepoMetadata{ Name: "repo3", Tags: map[string]repodb.Descriptor{ "Tag11": { diff --git a/pkg/meta/repodb/repodb.go b/pkg/meta/repodb/repodb.go index 54d4ba8d..c74f5c5e 100644 --- a/pkg/meta/repodb/repodb.go +++ b/pkg/meta/repodb/repodb.go @@ -14,7 +14,19 @@ const ( CosignType = "cosign" ) -type FilterFunc func(repoMeta RepoMetadata, manifestMeta ManifestMetadata) bool +// Used to model changes to an object after a call to the the DB. +type ToggleState int + +const ( + NotChanged ToggleState = iota + Added + Removed +) + +type ( + FilterFunc func(repoMeta RepoMetadata, manifestMeta ManifestMetadata) bool + FilterRepoFunc func(repoMeta RepoMetadata) bool +) type RepoDB interface { //nolint:interfacebloat // IncrementRepoStars adds 1 to the star count of an image @@ -95,10 +107,26 @@ type RepoDB interface { //nolint:interfacebloat SearchTags(ctx context.Context, searchText string, filter Filter, requestedPage PageInput) ( []RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, PageInfo, error) + // FilterRepos filters for repos given a filter function + FilterRepos(ctx context.Context, filter FilterRepoFunc, requestedPage PageInput) ( + []RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, PageInfo, error) + // FilterTags filters for images given a filter function FilterTags(ctx context.Context, filter FilterFunc, requestedPage PageInput) ([]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, PageInfo, error) + // GetStarredRepos returns starred repos and takes current user in consideration + GetStarredRepos(ctx context.Context) ([]string, error) + + // GetBookmarkedRepos returns bookmarked repos and takes current user in consideration + GetBookmarkedRepos(ctx context.Context) ([]string, error) + + // ToggleStarRepo adds/removes stars on repos + ToggleStarRepo(ctx context.Context, reponame string) (ToggleState, error) + + // ToggleBookmarkRepo adds/removes bookmarks on repos + ToggleBookmarkRepo(ctx context.Context, reponame string) (ToggleState, error) + PatchDB() error } @@ -150,6 +178,9 @@ type RepoMetadata struct { Signatures map[string]ManifestSignatures Referrers map[string][]ReferrerInfo + IsStarred bool + IsBookmarked bool + Stars int } @@ -171,6 +202,12 @@ type SignatureMetadata struct { LayersInfo []LayerInfo } +type UserData struct { + // data for each user. + StarredRepos []string + BookmarkedRepos []string +} + type SortCriteria string const ( diff --git a/pkg/meta/repodb/repodb_test.go b/pkg/meta/repodb/repodb_test.go index d50fe853..efe94f69 100644 --- a/pkg/meta/repodb/repodb_test.go +++ b/pkg/meta/repodb/repodb_test.go @@ -88,6 +88,7 @@ func TestDynamoDBWrapper(t *testing.T) { versionTablename := "Version" + uuid.String() indexDataTablename := "IndexDataTable" + uuid.String() artifactDataTablename := "ArtifactDataTable" + uuid.String() + userDataTablename := "UserDataTable" + uuid.String() Convey("DynamoDB Wrapper", t, func() { dynamoDBDriverParams := dynamo.DBDriverParameters{ @@ -97,6 +98,7 @@ func TestDynamoDBWrapper(t *testing.T) { IndexDataTablename: indexDataTablename, ArtifactDataTablename: artifactDataTablename, VersionTablename: versionTablename, + UserDataTablename: userDataTablename, Region: "us-east-2", } @@ -115,6 +117,8 @@ func TestDynamoDBWrapper(t *testing.T) { return err } + // Note: Tests are very slow if we reset the UserData table every new convey. We'll reset it as needed + err = dynamoDriver.ResetManifestDataTable() return err @@ -531,6 +535,399 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { So(err, ShouldNotBeNil) }) + Convey("Test repo stars for user", func() { + var ( + repo1 = "repo1" + tag1 = "0.0.1" + manifestDigest1 = godigest.FromString("fake-manifest1") + repo2 = "repo2" + ) + + authzCtxKey := localCtx.GetContextKey() + + acCtx1 := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + repo1: true, + repo2: true, + }, + Username: "user1", + } + + // "user1" + ctx1 := context.WithValue(context.Background(), authzCtxKey, acCtx1) + + acCtx2 := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + repo1: true, + repo2: true, + }, + Username: "user2", + } + + // "user2" + ctx2 := context.WithValue(context.Background(), authzCtxKey, acCtx2) + + acCtx3 := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + repo1: true, + repo2: true, + }, + Username: "", + } + + // anonymous + ctx3 := context.WithValue(context.Background(), authzCtxKey, acCtx3) + + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + err = repoDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + starCount, err := repoDB.GetRepoStars(repo1) + So(err, ShouldBeNil) + So(starCount, ShouldEqual, 0) + + starCount, err = repoDB.GetRepoStars(repo2) + So(err, ShouldBeNil) + So(starCount, ShouldEqual, 0) + + repos, err := repoDB.GetStarredRepos(ctx1) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + repos, err = repoDB.GetStarredRepos(ctx2) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + repos, err = repoDB.GetStarredRepos(ctx3) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + // User 1 bookmarks repo 1, User 2 has no stars + toggleState, err := repoDB.ToggleStarRepo(ctx1, repo1) + So(err, ShouldBeNil) + So(toggleState, ShouldEqual, repodb.Added) + + repoMeta, err := repoDB.GetRepoMeta(repo1) + So(err, ShouldBeNil) + So(repoMeta.Stars, ShouldEqual, 1) + + starCount, err = repoDB.GetRepoStars(repo1) + So(err, ShouldBeNil) + So(starCount, ShouldEqual, 1) + + repos, err = repoDB.GetStarredRepos(ctx1) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos, ShouldContain, repo1) + + repos, err = repoDB.GetStarredRepos(ctx2) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + repos, err = repoDB.GetStarredRepos(ctx3) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + // User 1 and User 2 star only repo 1 + toggleState, err = repoDB.ToggleStarRepo(ctx2, repo1) + So(err, ShouldBeNil) + So(toggleState, ShouldEqual, repodb.Added) + + repoMeta, err = repoDB.GetRepoMeta(repo1) + So(err, ShouldBeNil) + So(repoMeta.Stars, ShouldEqual, 2) + + starCount, err = repoDB.GetRepoStars(repo1) + So(err, ShouldBeNil) + So(starCount, ShouldEqual, 2) + + repos, err = repoDB.GetStarredRepos(ctx1) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos, ShouldContain, repo1) + + repos, err = repoDB.GetStarredRepos(ctx2) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos, ShouldContain, repo1) + + repos, err = repoDB.GetStarredRepos(ctx3) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + // User 1 stars repos 1 and 2, and User 2 stars only repo 1 + toggleState, err = repoDB.ToggleStarRepo(ctx1, repo2) + So(err, ShouldBeNil) + So(toggleState, ShouldEqual, repodb.Added) + + repoMeta, err = repoDB.GetRepoMeta(repo2) + So(err, ShouldBeNil) + So(repoMeta.Stars, ShouldEqual, 1) + + starCount, err = repoDB.GetRepoStars(repo2) + So(err, ShouldBeNil) + So(starCount, ShouldEqual, 1) + + repos, err = repoDB.GetStarredRepos(ctx1) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 2) + So(repos, ShouldContain, repo1) + So(repos, ShouldContain, repo2) + + repos, err = repoDB.GetStarredRepos(ctx2) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos, ShouldContain, repo1) + + repos, err = repoDB.GetStarredRepos(ctx3) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + // User 1 stars only repo 2, and User 2 stars only repo 1 + toggleState, err = repoDB.ToggleStarRepo(ctx1, repo1) + So(err, ShouldBeNil) + So(toggleState, ShouldEqual, repodb.Removed) + + repoMeta, err = repoDB.GetRepoMeta(repo1) + So(err, ShouldBeNil) + So(repoMeta.Stars, ShouldEqual, 1) + + starCount, err = repoDB.GetRepoStars(repo1) + So(err, ShouldBeNil) + So(starCount, ShouldEqual, 1) + + repos, err = repoDB.GetStarredRepos(ctx1) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos, ShouldContain, repo2) + + repos, err = repoDB.GetStarredRepos(ctx2) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos, ShouldContain, repo1) + + repos, err = repoDB.GetStarredRepos(ctx3) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + // User 1 stars both repos 1 and 2, and User 2 removes all stars + toggleState, err = repoDB.ToggleStarRepo(ctx1, repo1) + So(err, ShouldBeNil) + So(toggleState, ShouldEqual, repodb.Added) + + toggleState, err = repoDB.ToggleStarRepo(ctx2, repo1) + So(err, ShouldBeNil) + So(toggleState, ShouldEqual, repodb.Removed) + + repoMeta, err = repoDB.GetRepoMeta(repo1) + So(err, ShouldBeNil) + So(repoMeta.Stars, ShouldEqual, 1) + + repoMeta, err = repoDB.GetRepoMeta(repo2) + So(err, ShouldBeNil) + So(repoMeta.Stars, ShouldEqual, 1) + + starCount, err = repoDB.GetRepoStars(repo1) + So(err, ShouldBeNil) + So(starCount, ShouldEqual, 1) + + starCount, err = repoDB.GetRepoStars(repo2) + So(err, ShouldBeNil) + So(starCount, ShouldEqual, 1) + + repos, err = repoDB.GetStarredRepos(ctx1) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 2) + So(repos, ShouldContain, repo1) + So(repos, ShouldContain, repo2) + + repos, err = repoDB.GetStarredRepos(ctx2) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + repos, err = repoDB.GetStarredRepos(ctx3) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + // Anonyous user attempts to toggle a star + toggleState, err = repoDB.ToggleStarRepo(ctx3, repo1) + So(err, ShouldNotBeNil) + So(toggleState, ShouldEqual, repodb.NotChanged) + + starCount, err = repoDB.GetRepoStars(repo1) + So(err, ShouldBeNil) + So(starCount, ShouldEqual, 1) + + repos, err = repoDB.GetStarredRepos(ctx3) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + // User 1 stars just repo 1 + toggleState, err = repoDB.ToggleStarRepo(ctx1, repo2) + So(err, ShouldBeNil) + So(toggleState, ShouldEqual, repodb.Removed) + + starCount, err = repoDB.GetRepoStars(repo2) + So(err, ShouldBeNil) + So(starCount, ShouldEqual, 0) + + repos, err = repoDB.GetStarredRepos(ctx3) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + }) + + Convey("Test repo bookmarks for user", func() { + var ( + repo1 = "repo1" + tag1 = "0.0.1" + manifestDigest1 = godigest.FromString("fake-manifest1") + repo2 = "repo2" + ) + + authzCtxKey := localCtx.GetContextKey() + + acCtx1 := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + repo1: true, + repo2: true, + }, + Username: "user1", + } + + // "user1" + ctx1 := context.WithValue(context.Background(), authzCtxKey, acCtx1) + + acCtx2 := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + repo1: true, + repo2: true, + }, + Username: "user2", + } + + // "user2" + ctx2 := context.WithValue(context.Background(), authzCtxKey, acCtx2) + + acCtx3 := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + repo1: true, + repo2: true, + }, + Username: "", + } + + // anonymous + ctx3 := context.WithValue(context.Background(), authzCtxKey, acCtx3) + + err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + err = repoDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + repos, err := repoDB.GetBookmarkedRepos(ctx1) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + repos, err = repoDB.GetBookmarkedRepos(ctx2) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + // anonymous cannot use bookmarks + repos, err = repoDB.GetBookmarkedRepos(ctx3) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + toggleState, err := repoDB.ToggleBookmarkRepo(ctx3, repo1) + So(err, ShouldNotBeNil) + So(toggleState, ShouldEqual, repodb.NotChanged) + + repos, err = repoDB.GetBookmarkedRepos(ctx3) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + // User 1 bookmarks repo 1, User 2 has no bookmarks + toggleState, err = repoDB.ToggleBookmarkRepo(ctx1, repo1) + So(err, ShouldBeNil) + So(toggleState, ShouldEqual, repodb.Added) + + repos, err = repoDB.GetBookmarkedRepos(ctx1) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos, ShouldContain, repo1) + + repos, err = repoDB.GetBookmarkedRepos(ctx2) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + + // User 1 and User 2 bookmark only repo 1 + toggleState, err = repoDB.ToggleBookmarkRepo(ctx2, repo1) + So(err, ShouldBeNil) + So(toggleState, ShouldEqual, repodb.Added) + + repos, err = repoDB.GetBookmarkedRepos(ctx1) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos, ShouldContain, repo1) + + repos, err = repoDB.GetBookmarkedRepos(ctx2) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos, ShouldContain, repo1) + + // User 1 bookmarks repos 1 and 2, and User 2 bookmarks only repo 1 + toggleState, err = repoDB.ToggleBookmarkRepo(ctx1, repo2) + So(err, ShouldBeNil) + So(toggleState, ShouldEqual, repodb.Added) + + repos, err = repoDB.GetBookmarkedRepos(ctx1) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 2) + So(repos, ShouldContain, repo1) + So(repos, ShouldContain, repo2) + + repos, err = repoDB.GetBookmarkedRepos(ctx2) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos, ShouldContain, repo1) + + // User 1 bookmarks only repo 2, and User 2 bookmarks only repo 1 + toggleState, err = repoDB.ToggleBookmarkRepo(ctx1, repo1) + So(err, ShouldBeNil) + So(toggleState, ShouldEqual, repodb.Removed) + + repos, err = repoDB.GetBookmarkedRepos(ctx1) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos, ShouldContain, repo2) + + repos, err = repoDB.GetBookmarkedRepos(ctx2) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos, ShouldContain, repo1) + + // User 1 bookmarks both repos 1 and 2, and User 2 removes all bookmarks + toggleState, err = repoDB.ToggleBookmarkRepo(ctx1, repo1) + So(err, ShouldBeNil) + So(toggleState, ShouldEqual, repodb.Added) + + toggleState, err = repoDB.ToggleBookmarkRepo(ctx2, repo1) + So(err, ShouldBeNil) + So(toggleState, ShouldEqual, repodb.Removed) + + repos, err = repoDB.GetBookmarkedRepos(ctx1) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 2) + So(repos, ShouldContain, repo1) + So(repos, ShouldContain, repo2) + + repos, err = repoDB.GetBookmarkedRepos(ctx2) + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 0) + }) + Convey("Test IncrementImageDownloads", func() { var ( repo1 = "repo1" @@ -1974,9 +2371,171 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) { So(referrerInfo[0].ArtifactType, ShouldResemble, "wantedType") So(referrerInfo[0].Digest, ShouldResemble, "goodArtifact") }) + + Convey("FilterRepos", func() { + img, err := test.GetRandomImage("img1") + So(err, ShouldBeNil) + imgDigest, err := img.Digest() + So(err, ShouldBeNil) + + manifestData, err := NewManifestData(img.Manifest, img.Config) + So(err, ShouldBeNil) + + err = repoDB.SetManifestData(imgDigest, manifestData) + So(err, ShouldBeNil) + + multiarch, err := test.GetRandomMultiarchImage("multi") + So(err, ShouldBeNil) + multiarchDigest, err := multiarch.Digest() + So(err, ShouldBeNil) + + indexData, err := NewIndexData(multiarch.Index) + So(err, ShouldBeNil) + + err = repoDB.SetIndexData(multiarchDigest, indexData) + So(err, ShouldBeNil) + + for _, img := range multiarch.Images { + digest, err := img.Digest() + So(err, ShouldBeNil) + + indManData1, err := NewManifestData(multiarch.Images[0].Manifest, multiarch.Images[0].Config) + So(err, ShouldBeNil) + + err = repoDB.SetManifestData(digest, indManData1) + So(err, ShouldBeNil) + } + + err = repoDB.SetRepoReference("repo", img.Reference, imgDigest, img.Manifest.MediaType) + So(err, ShouldBeNil) + + err = repoDB.SetRepoReference("repo", multiarch.Reference, multiarchDigest, ispec.MediaTypeImageIndex) + So(err, ShouldBeNil) + + repoMetas, _, _, _, err := repoDB.FilterRepos(context.Background(), + func(repoMeta repodb.RepoMetadata) bool { return true }, repodb.PageInput{}) + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 1) + + _, _, _, _, err = repoDB.FilterRepos(context.Background(), + func(repoMeta repodb.RepoMetadata) bool { return true }, repodb.PageInput{ + Limit: -1, + Offset: -1, + }) + So(err, ShouldNotBeNil) + }) + + Convey("Test bookmarked/starred field present in returned RepoMeta", func() { + repo99 := "repo99" + authzCtxKey := localCtx.GetContextKey() + + acCtx := localCtx.AccessControlContext{ + ReadGlobPatterns: map[string]bool{ + repo99: true, + }, + Username: "user1", + } + ctx := context.WithValue(context.Background(), authzCtxKey, acCtx) + + manifestDigest := godigest.FromString("dig") + err := repoDB.SetManifestData(manifestDigest, repodb.ManifestData{ + ManifestBlob: []byte("{}"), + ConfigBlob: []byte("{}"), + }) + So(err, ShouldBeNil) + + err = repoDB.SetRepoReference(repo99, "tag", manifestDigest, ispec.MediaTypeImageManifest) + So(err, ShouldBeNil) + + repoMetas, _, _, _, err := repoDB.SearchRepos(ctx, repo99, repodb.Filter{}, repodb.PageInput{}) + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 1) + So(repoMetas[0].IsBookmarked, ShouldBeFalse) + So(repoMetas[0].IsStarred, ShouldBeFalse) + + repoMetas, _, _, _, err = repoDB.SearchTags(ctx, repo99+":", repodb.Filter{}, repodb.PageInput{}) + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 1) + So(repoMetas[0].IsBookmarked, ShouldBeFalse) + So(repoMetas[0].IsStarred, ShouldBeFalse) + + repoMetas, _, _, _, err = repoDB.FilterRepos(ctx, func(repoMeta repodb.RepoMetadata) bool { + return true + }, repodb.PageInput{}) + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 1) + So(repoMetas[0].IsBookmarked, ShouldBeFalse) + So(repoMetas[0].IsStarred, ShouldBeFalse) + + repoMetas, _, _, _, err = repoDB.FilterTags(ctx, + func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true }, + repodb.PageInput{}, + ) + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 1) + So(repoMetas[0].IsBookmarked, ShouldBeFalse) + So(repoMetas[0].IsStarred, ShouldBeFalse) + + _, err = repoDB.ToggleBookmarkRepo(ctx, repo99) + So(err, ShouldBeNil) + + _, err = repoDB.ToggleStarRepo(ctx, repo99) + So(err, ShouldBeNil) + + repoMetas, _, _, _, err = repoDB.SearchRepos(ctx, repo99, repodb.Filter{}, repodb.PageInput{}) + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 1) + So(repoMetas[0].IsBookmarked, ShouldBeTrue) + So(repoMetas[0].IsStarred, ShouldBeTrue) + + repoMetas, _, _, _, err = repoDB.SearchTags(ctx, repo99+":", repodb.Filter{}, repodb.PageInput{}) + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 1) + So(repoMetas[0].IsBookmarked, ShouldBeTrue) + So(repoMetas[0].IsStarred, ShouldBeTrue) + + repoMetas, _, _, _, err = repoDB.FilterRepos(ctx, func(repoMeta repodb.RepoMetadata) bool { + return true + }, repodb.PageInput{}) + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 1) + So(repoMetas[0].IsBookmarked, ShouldBeTrue) + So(repoMetas[0].IsStarred, ShouldBeTrue) + + repoMetas, _, _, _, err = repoDB.FilterTags(ctx, + func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true }, + repodb.PageInput{}, + ) + So(err, ShouldBeNil) + So(len(repoMetas), ShouldEqual, 1) + So(repoMetas[0].IsBookmarked, ShouldBeTrue) + So(repoMetas[0].IsStarred, ShouldBeTrue) + }) }) } +func NewManifestData(manifest ispec.Manifest, config ispec.Image) (repodb.ManifestData, error) { + configBlob, err := json.Marshal(config) + if err != nil { + return repodb.ManifestData{}, err + } + + manifest.Config.Digest = godigest.FromBytes(configBlob) + + manifestBlob, err := json.Marshal(manifest) + if err != nil { + return repodb.ManifestData{}, err + } + + return repodb.ManifestData{ManifestBlob: manifestBlob, ConfigBlob: configBlob}, nil +} + +func NewIndexData(index ispec.Index) (repodb.IndexData, error) { + indexBlob, err := json.Marshal(index) + + return repodb.IndexData{IndexBlob: indexBlob}, err +} + func TestRelevanceSorting(t *testing.T) { Convey("Test Relevance Sorting", t, func() { So(common.RankRepoName("alpine", "alpine"), ShouldEqual, 0) diff --git a/pkg/meta/repodb/repodbfactory/repodb_factory.go b/pkg/meta/repodb/repodbfactory/repodb_factory.go index 33658274..d7d6f093 100644 --- a/pkg/meta/repodb/repodbfactory/repodb_factory.go +++ b/pkg/meta/repodb/repodbfactory/repodb_factory.go @@ -95,6 +95,9 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d versionTablename, ok := toStringIfOk(cacheDriverConfig, "versiontablename", log) allParametersOk = allParametersOk && ok + userDataTablename, ok := toStringIfOk(cacheDriverConfig, "userdatatablename", log) + allParametersOk = allParametersOk && ok + if !allParametersOk { panic("dynamo parameters are not specified correctly, can't proceede") } @@ -106,6 +109,7 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d ManifestDataTablename: manifestDataTablename, IndexDataTablename: indexDataTablename, ArtifactDataTablename: artifactDataTablename, + UserDataTablename: userDataTablename, VersionTablename: versionTablename, } } diff --git a/pkg/meta/repodb/repodbfactory/repodb_factory_test.go b/pkg/meta/repodb/repodbfactory/repodb_factory_test.go index 01ba76e3..28f73a7d 100644 --- a/pkg/meta/repodb/repodbfactory/repodb_factory_test.go +++ b/pkg/meta/repodb/repodbfactory/repodb_factory_test.go @@ -23,6 +23,7 @@ func TestCreateDynamo(t *testing.T) { ManifestDataTablename: "ManifestDataTable", IndexDataTablename: "IndexDataTable", ArtifactDataTablename: "ArtifactDataTable", + UserDataTablename: "UserDataTable", VersionTablename: "Version", Region: "us-east-2", } diff --git a/pkg/meta/repodb/storage_parsing_test.go b/pkg/meta/repodb/storage_parsing_test.go index a6a1b21b..985d6d00 100644 --- a/pkg/meta/repodb/storage_parsing_test.go +++ b/pkg/meta/repodb/storage_parsing_test.go @@ -293,6 +293,7 @@ func TestParseStorageDynamoWrapper(t *testing.T) { ManifestDataTablename: "ManifestDataTable", IndexDataTablename: "IndexDataTable", ArtifactDataTablename: "ArtifactDataTable", + UserDataTablename: "UserDataTable", VersionTablename: "Version", } diff --git a/pkg/meta/version/version_test.go b/pkg/meta/version/version_test.go index ad961b92..2ffbed00 100644 --- a/pkg/meta/version/version_test.go +++ b/pkg/meta/version/version_test.go @@ -127,6 +127,7 @@ func TestVersioningDynamoDB(t *testing.T) { ManifestDataTablename: "ManifestDataTable", ArtifactDataTablename: "ArtifactDataTable", IndexDataTablename: "IndexDataTable", + UserDataTablename: "UserDataTable", VersionTablename: "Version", } diff --git a/pkg/requestcontext/checkrepo.go b/pkg/requestcontext/checkrepo.go index cfc1b8cf..de388109 100644 --- a/pkg/requestcontext/checkrepo.go +++ b/pkg/requestcontext/checkrepo.go @@ -26,3 +26,11 @@ func RepoIsUserAvailable(ctx context.Context, repoName string) (bool, error) { return true, nil } + +func GetUsernameFromContext(ctx *AccessControlContext) string { + if ctx == nil { + return "" + } + + return ctx.Username +} diff --git a/pkg/test/common.go b/pkg/test/common.go index f0b6345d..aa58cbfe 100644 --- a/pkg/test/common.go +++ b/pkg/test/common.go @@ -657,6 +657,7 @@ func GetRandomImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manif Size: int64(len(layers[0])), }, }, + MediaType: ispec.MediaTypeImageManifest, } return config, layers, manifest, nil diff --git a/pkg/test/mocks/repo_db_mock.go b/pkg/test/mocks/repo_db_mock.go index ce957b24..bb2e08a5 100644 --- a/pkg/test/mocks/repo_db_mock.go +++ b/pkg/test/mocks/repo_db_mock.go @@ -67,6 +67,9 @@ type RepoDBMock struct { SearchTagsFn func(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput) ( []repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) + FilterReposFn func(ctx context.Context, filter repodb.FilterRepoFunc, requestedPage repodb.PageInput) ( + []repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) + FilterTagsFn func(ctx context.Context, filter repodb.FilterFunc, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) @@ -83,6 +86,14 @@ type RepoDBMock struct { SearchForDescendantImagesFn func(ctx context.Context, searchText string, requestedPage repodb.PageInput) ( []repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error) + GetStarredReposFn func(ctx context.Context) ([]string, error) + + GetBookmarkedReposFn func(ctx context.Context) ([]string, error) + + ToggleStarRepoFn func(ctx context.Context, repo string) (repodb.ToggleState, error) + + ToggleBookmarkRepoFn func(ctx context.Context, repo string) (repodb.ToggleState, error) + PatchDBFn func() error } @@ -244,6 +255,17 @@ func (sdm RepoDBMock) SearchTags(ctx context.Context, searchText string, filter map[string]repodb.IndexData{}, repodb.PageInfo{}, nil } +func (sdm RepoDBMock) FilterRepos(ctx context.Context, filter repodb.FilterRepoFunc, + requestedPage repodb.PageInput, +) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) { + if sdm.FilterReposFn != nil { + return sdm.FilterReposFn(ctx, filter, requestedPage) + } + + return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, + map[string]repodb.IndexData{}, repodb.PageInfo{}, nil +} + func (sdm RepoDBMock) FilterTags(ctx context.Context, filter repodb.FilterFunc, requestedPage repodb.PageInput, ) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) { @@ -359,3 +381,35 @@ func (sdm RepoDBMock) GetReferrersInfo(repo string, referredDigest godigest.Dige return []repodb.ReferrerInfo{}, nil } + +func (sdm RepoDBMock) GetStarredRepos(ctx context.Context) ([]string, error) { + if sdm.GetStarredReposFn != nil { + return sdm.GetStarredReposFn(ctx) + } + + return []string{}, nil +} + +func (sdm RepoDBMock) GetBookmarkedRepos(ctx context.Context) ([]string, error) { + if sdm.GetBookmarkedReposFn != nil { + return sdm.GetBookmarkedReposFn(ctx) + } + + return []string{}, nil +} + +func (sdm RepoDBMock) ToggleStarRepo(ctx context.Context, repo string) (repodb.ToggleState, error) { + if sdm.ToggleStarRepoFn != nil { + return sdm.ToggleStarRepoFn(ctx, repo) + } + + return repodb.NotChanged, nil +} + +func (sdm RepoDBMock) ToggleBookmarkRepo(ctx context.Context, repo string) (repodb.ToggleState, error) { + if sdm.ToggleBookmarkRepoFn != nil { + return sdm.ToggleBookmarkRepoFn(ctx, repo) + } + + return repodb.NotChanged, nil +} diff --git a/test/blackbox/cloud-only.bats b/test/blackbox/cloud-only.bats index 4535f539..acb8b156 100644 --- a/test/blackbox/cloud-only.bats +++ b/test/blackbox/cloud-only.bats @@ -39,6 +39,7 @@ function setup() { "manifestDataTablename": "ManifestDataTable", "artifactDataTablename": "ArtifactDataTable", "indexDataTablename": "IndexDataTable", + "userDataTablename": "UserDataTable", "versionTablename": "Version" } }, diff --git a/test/blackbox/metadata.bats b/test/blackbox/metadata.bats new file mode 100644 index 00000000..b18690d8 --- /dev/null +++ b/test/blackbox/metadata.bats @@ -0,0 +1,138 @@ +load helpers_pushpull + +function setup_file() { + # Verify prerequisites are available + if ! verify_prerequisites; then + exit 1 + fi + + # Download test data to folder common for the entire suite, not just this file + skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.18 oci:${TEST_DATA_DIR}/golang:1.18 + # Setup zot server + local zot_root_dir=${BATS_FILE_TMPDIR}/zot + local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json + local oci_data_dir=${BATS_FILE_TMPDIR}/oci + local htpasswordFile=${BATS_FILE_TMPDIR}/htpasswd + mkdir -p ${zot_root_dir} + mkdir -p ${oci_data_dir} + echo 'test:$2a$10$EIIoeCnvsIDAJeDL4T1sEOnL2fWOvsq7ACZbs3RT40BBBXg.Ih7V.' >> ${htpasswordFile} + cat > ${zot_config_file}<