0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00

feat(repodb): add user related information to repodb (#1317)

Initial code was contributed by Bogdan BIVOLARU <104334+bogdanbiv@users.noreply.github.com>
Moved implementation from a separate db to repodb by Andrei Aaron <aaaron@luxoft.com>

Not done yet:
- run/test dynamodb implementation, only boltdb was tested
- add additional coverage for existing functionality
- add web-based APIs to toggle the stars/bookmarks on/off

Initially graphql mutation was discussed for the missing API but
we decided REST endpoints would be better suited for configuration



feat(userdb): complete functionality for userdb integration

- dynamodb rollback changes to user starred repos in case increasing the total star count fails
- dynamodb increment/decrement repostars in repometa when user stars/unstars a repo
- dynamodb check anonymous user permissions are working as intendend
- common test handle anonymous users
- RepoMeta2RepoSummary set IsStarred and IsBookmarked



feat(userdb): rest api calls for toggling stars/bookmarks on/off



test(userdb): blackbox tests



test(userdb): move preferences tests in a different file with specific build tags



feat(repodb): add is-starred and is-bookmarked fields to repo-meta

- removed duplicated logic for determining if a repo is starred/bookmarked

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
Co-authored-by: Andrei Aaron <aaaron@luxoft.com>
This commit is contained in:
LaurentiuNiculae 2023-04-24 21:13:15 +03:00 committed by GitHub
parent ef51fd692d
commit 9cc990d7ca
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
50 changed files with 4357 additions and 648 deletions

View file

@ -39,7 +39,7 @@ jobs:
# https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
env:
CGO_ENABLED: 0
GOFLAGS: "-tags=sync,search,scrub,metrics,containers_image_openpgp"
GOFLAGS: "-tags=sync,search,scrub,metrics,userprefs,containers_image_openpgp"
steps:
- name: Checkout repository

View file

@ -49,6 +49,9 @@ jobs:
- name: Run referrers tests
run: |
make test-bats-referrers
- name: Run metadata tests
run: |
make test-bats-metadata
- name: Run push-pull tests
run: |
make test-push-pull

View file

@ -32,7 +32,7 @@ jobs:
# Optional: golangci-lint command line arguments.
# args: --issues-exit-code=0
args: --config ./golangcilint.yaml --enable-all --build-tags debug,needprivileges,sync,scrub,search,metrics,containers_image_openpgp,lint,mgmt ./cmd/... ./pkg/...
args: --config ./golangcilint.yaml --enable-all --build-tags debug,needprivileges,sync,scrub,search,userprefs,metrics,containers_image_openpgp,lint,mgmt ./cmd/... ./pkg/...
# Optional: show only new issues if it's a pull request. The default value is `false`.
# only-new-issues: true

View file

@ -30,7 +30,7 @@ TESTDATA := $(TOP_LEVEL)/test/data
OS ?= linux
ARCH ?= amd64
BENCH_OUTPUT ?= stdout
EXTENSIONS ?= sync,search,scrub,metrics,lint,ui,mgmt
EXTENSIONS ?= sync,search,scrub,metrics,lint,ui,mgmt,userprefs
comma:= ,
hyphen:= -
extended-name:=
@ -338,6 +338,11 @@ test-bats-referrers: EXTENSIONS=search
test-bats-referrers: binary check-skopeo $(BATS) $(ORAS)
$(BATS) --trace --print-output-on-failure test/blackbox/referrers.bats
.PHONY: test-bats-metadata
test-bats-metadata: EXTENSIONS=search,userprefs
test-bats-metadata: binary check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox/metadata.bats
.PHONY: test-cloud-only
test-cloud-only: binary check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox/cloud-only.bats

View file

@ -3,83 +3,92 @@ package errors
import "errors"
var (
ErrBadConfig = errors.New("config: invalid config")
ErrCliBadConfig = errors.New("cli: bad config")
ErrRepoNotFound = errors.New("repository: not found")
ErrRepoIsNotDir = errors.New("repository: not a directory")
ErrRepoBadVersion = errors.New("repository: unsupported layout version")
ErrManifestNotFound = errors.New("manifest: not found")
ErrBadManifest = errors.New("manifest: invalid contents")
ErrBadIndex = errors.New("index: invalid contents")
ErrUploadNotFound = errors.New("uploads: not found")
ErrBadUploadRange = errors.New("uploads: bad range")
ErrBlobNotFound = errors.New("blob: not found")
ErrBadBlob = errors.New("blob: bad blob")
ErrBadBlobDigest = errors.New("blob: bad blob digest")
ErrUnknownCode = errors.New("error: unknown error code")
ErrBadCACert = errors.New("tls: invalid ca cert")
ErrBadUser = errors.New("auth: non-existent user")
ErrEntriesExceeded = errors.New("ldap: too many entries returned")
ErrLDAPEmptyPassphrase = errors.New("ldap: empty passphrase")
ErrLDAPBadConn = errors.New("ldap: bad connection")
ErrLDAPConfig = errors.New("config: invalid LDAP configuration")
ErrCacheRootBucket = errors.New("cache: unable to create/update root bucket")
ErrCacheNoBucket = errors.New("cache: unable to find bucket")
ErrCacheMiss = errors.New("cache: miss")
ErrRequireCred = errors.New("ldap: bind credentials required")
ErrInvalidCred = errors.New("ldap: invalid credentials")
ErrEmptyJSON = errors.New("cli: config json is empty")
ErrInvalidArgs = errors.New("cli: Invalid Arguments")
ErrInvalidFlagsCombination = errors.New("cli: Invalid combination of flags")
ErrInvalidURL = errors.New("cli: invalid URL format")
ErrUnauthorizedAccess = errors.New("auth: unauthorized access. check credentials")
ErrCannotResetConfigKey = errors.New("cli: cannot reset given config key")
ErrConfigNotFound = errors.New("cli: config with the given name does not exist")
ErrNoURLProvided = errors.New("cli: no URL provided in argument or via config")
ErrIllegalConfigKey = errors.New("cli: given config key is not allowed")
ErrScanNotSupported = errors.New("search: scanning of image media type not supported")
ErrCLITimeout = errors.New("cli: Query timed out while waiting for results")
ErrDuplicateConfigName = errors.New("cli: cli config name already added")
ErrInvalidRoute = errors.New("routes: invalid route prefix")
ErrImgStoreNotFound = errors.New("routes: image store not found corresponding to given route")
ErrEmptyValue = errors.New("cache: empty value")
ErrEmptyRepoList = errors.New("search: no repository found")
ErrCVESearchDisabled = errors.New("search: CVE search is disabled")
ErrInvalidRepositoryName = errors.New("repository: not a valid repository name")
ErrSyncMissingCatalog = errors.New("sync: couldn't fetch upstream registry's catalog")
ErrMethodNotSupported = errors.New("storage: method not supported")
ErrInvalidMetric = errors.New("metrics: invalid metric func")
ErrInjected = errors.New("test: injected failure")
ErrSyncInvalidUpstreamURL = errors.New("sync: upstream url not found in sync config")
ErrRegistryNoContent = errors.New("sync: could not find a Content that matches localRepo")
ErrSyncReferrerNotFound = errors.New("sync: couldn't find upstream referrer")
ErrSyncReferrer = errors.New("sync: failed to get upstream referrer")
ErrImageLintAnnotations = errors.New("routes: lint checks failed")
ErrParsingAuthHeader = errors.New("auth: failed parsing authorization header")
ErrBadType = errors.New("core: invalid type")
ErrParsingHTTPHeader = errors.New("routes: invalid HTTP header")
ErrBadRange = errors.New("storage: bad range")
ErrBadLayerCount = errors.New("manifest: layers count doesn't correspond to config history")
ErrManifestConflict = errors.New("manifest: multiple manifests found")
ErrManifestMetaNotFound = errors.New("repodb: image metadata not found for given manifest reference")
ErrManifestDataNotFound = errors.New("repodb: image data not found for given manifest digest")
ErrArtifactDataNotFound = errors.New("repodb: artifact data not found for given digest")
ErrIndexDataNotFount = errors.New("repodb: index data not found for given digest")
ErrRepoMetaNotFound = errors.New("repodb: repo metadata not found for given repo name")
ErrTagMetaNotFound = errors.New("repodb: tag metadata not found for given repo and tag names")
ErrTypeAssertionFailed = errors.New("storage: failed DatabaseDriver type assertion")
ErrInvalidRequestParams = errors.New("resolver: parameter sent has invalid value")
ErrOrphanSignature = errors.New("repodb: signature detected but signed image doesn't exit")
ErrBadCtxFormat = errors.New("type assertion failed")
ErrEmptyRepoName = errors.New("repodb: repo name can't be empty string")
ErrEmptyTag = errors.New("repodb: tag can't be empty string")
ErrEmptyDigest = errors.New("repodb: digest can't be empty string")
ErrInvalidRepoTagFormat = errors.New("invalid format for tag search, not following repo:tag")
ErrLimitIsNegative = errors.New("pageturner: limit has negative value")
ErrOffsetIsNegative = errors.New("pageturner: offset has negative value")
ErrSortCriteriaNotSupported = errors.New("pageturner: the sort criteria is not supported")
ErrMediaTypeNotSupported = errors.New("repodb: media type is not supported")
ErrTimeout = errors.New("operation timeout")
ErrNotImplemented = errors.New("not implemented")
ErrDedupeRebuild = errors.New("dedupe: couldn't rebuild dedupe index")
ErrBadConfig = errors.New("config: invalid config")
ErrCliBadConfig = errors.New("cli: bad config")
ErrRepoNotFound = errors.New("repository: not found")
ErrRepoIsNotDir = errors.New("repository: not a directory")
ErrRepoBadVersion = errors.New("repository: unsupported layout version")
ErrManifestNotFound = errors.New("manifest: not found")
ErrBadManifest = errors.New("manifest: invalid contents")
ErrBadIndex = errors.New("index: invalid contents")
ErrUploadNotFound = errors.New("uploads: not found")
ErrBadUploadRange = errors.New("uploads: bad range")
ErrBlobNotFound = errors.New("blob: not found")
ErrBadBlob = errors.New("blob: bad blob")
ErrBadBlobDigest = errors.New("blob: bad blob digest")
ErrUnknownCode = errors.New("error: unknown error code")
ErrBadCACert = errors.New("tls: invalid ca cert")
ErrBadUser = errors.New("auth: non-existent user")
ErrEntriesExceeded = errors.New("ldap: too many entries returned")
ErrLDAPEmptyPassphrase = errors.New("ldap: empty passphrase")
ErrLDAPBadConn = errors.New("ldap: bad connection")
ErrLDAPConfig = errors.New("config: invalid LDAP configuration")
ErrCacheRootBucket = errors.New("cache: unable to create/update root bucket")
ErrCacheNoBucket = errors.New("cache: unable to find bucket")
ErrCacheMiss = errors.New("cache: miss")
ErrRequireCred = errors.New("ldap: bind credentials required")
ErrInvalidCred = errors.New("ldap: invalid credentials")
ErrEmptyJSON = errors.New("cli: config json is empty")
ErrInvalidArgs = errors.New("cli: Invalid Arguments")
ErrInvalidFlagsCombination = errors.New("cli: Invalid combination of flags")
ErrInvalidURL = errors.New("cli: invalid URL format")
ErrUnauthorizedAccess = errors.New("auth: unauthorized access. check credentials")
ErrCannotResetConfigKey = errors.New("cli: cannot reset given config key")
ErrConfigNotFound = errors.New("cli: config with the given name does not exist")
ErrNoURLProvided = errors.New("cli: no URL provided in argument or via config")
ErrIllegalConfigKey = errors.New("cli: given config key is not allowed")
ErrScanNotSupported = errors.New("search: scanning of image media type not supported")
ErrCLITimeout = errors.New("cli: Query timed out while waiting for results")
ErrDuplicateConfigName = errors.New("cli: cli config name already added")
ErrInvalidRoute = errors.New("routes: invalid route prefix")
ErrImgStoreNotFound = errors.New("routes: image store not found corresponding to given route")
ErrEmptyValue = errors.New("cache: empty value")
ErrEmptyRepoList = errors.New("search: no repository found")
ErrCVESearchDisabled = errors.New("search: CVE search is disabled")
ErrInvalidRepositoryName = errors.New("repository: not a valid repository name")
ErrSyncMissingCatalog = errors.New("sync: couldn't fetch upstream registry's catalog")
ErrMethodNotSupported = errors.New("storage: method not supported")
ErrInvalidMetric = errors.New("metrics: invalid metric func")
ErrInjected = errors.New("test: injected failure")
ErrSyncInvalidUpstreamURL = errors.New("sync: upstream url not found in sync config")
ErrRegistryNoContent = errors.New("sync: could not find a Content that matches localRepo")
ErrSyncReferrerNotFound = errors.New("sync: couldn't find upstream referrer")
ErrSyncReferrer = errors.New("sync: failed to get upstream referrer")
ErrImageLintAnnotations = errors.New("routes: lint checks failed")
ErrParsingAuthHeader = errors.New("auth: failed parsing authorization header")
ErrBadType = errors.New("core: invalid type")
ErrParsingHTTPHeader = errors.New("routes: invalid HTTP header")
ErrBadRange = errors.New("storage: bad range")
ErrBadLayerCount = errors.New("manifest: layers count doesn't correspond to config history")
ErrManifestConflict = errors.New("manifest: multiple manifests found")
ErrManifestMetaNotFound = errors.New("repodb: image metadata not found for given manifest reference")
ErrManifestDataNotFound = errors.New("repodb: image data not found for given manifest digest")
ErrArtifactDataNotFound = errors.New("repodb: artifact data not found for given digest")
ErrIndexDataNotFount = errors.New("repodb: index data not found for given digest")
ErrRepoMetaNotFound = errors.New("repodb: repo metadata not found for given repo name")
ErrTagMetaNotFound = errors.New("repodb: tag metadata not found for given repo and tag names")
ErrTypeAssertionFailed = errors.New("storage: failed DatabaseDriver type assertion")
ErrInvalidRequestParams = errors.New("resolver: parameter sent has invalid value")
ErrOrphanSignature = errors.New("repodb: signature detected but signed image doesn't exit")
ErrBadCtxFormat = errors.New("type assertion failed")
ErrEmptyRepoName = errors.New("repodb: repo name can't be empty string")
ErrEmptyTag = errors.New("repodb: tag can't be empty string")
ErrEmptyDigest = errors.New("repodb: digest can't be empty string")
ErrInvalidRepoTagFormat = errors.New("invalid format for tag search, not following repo:tag")
ErrLimitIsNegative = errors.New("pageturner: limit has negative value")
ErrOffsetIsNegative = errors.New("pageturner: offset has negative value")
ErrSortCriteriaNotSupported = errors.New("pageturner: the sort criteria is not supported")
ErrMediaTypeNotSupported = errors.New("repodb: media type is not supported")
ErrTimeout = errors.New("operation timeout")
ErrNotImplemented = errors.New("not implemented")
ErrUnableToCreateUserBucket = errors.New("repodb: unable to create a user bucket for user")
ErrInvalidOldUserStarredRepos = errors.New("repodb: invalid old entry for user starred repos")
ErrUnmarshalledRepoListIsNil = errors.New("repodb: list of repos is still nil")
ErrCouldNotMarshalStarredRepos = errors.New("repodb: could not repack entry for user starred repos")
ErrInvalidOldUserBookmarkedRepos = errors.New("repodb: invalid old entry for user bookmarked repos")
ErrCouldNotMarshalBookmarkedRepos = errors.New("repodb: could not repack entry for user bookmarked repos")
ErrUserDataNotFound = errors.New("repodb: user data not found for given user identifier")
ErrUserDataNotAllowed = errors.New("repodb: user data operations are not allowed")
ErrCouldNotPersistData = errors.New("repodb: could not persist to db")
ErrDedupeRebuild = errors.New("dedupe: couldn't rebuild dedupe index")
)

View file

@ -448,6 +448,7 @@ Additionally if search extension is enabled, additional parameters are needed:
// used by search extensions
"repoMetaTablename": "ZotRepoMetadataTable",
"manifestDataTablename": "ZotManifestDataTable",
"userDataTablename": "ZotUserDataTable",
"versionTablename": "ZotVersion"
}
```

View file

@ -21,6 +21,7 @@
"repoMetaTablename": "ZotRepoMetadataTable",
"manifestDataTablename": "ZotManifestDataTable",
"artifactDataTablename": "ZotArtifactDataTable",
"userDataTablename": "ZotUserDataTable",
"versionTablename": "ZotVersion"
}
},

View file

@ -240,11 +240,6 @@ func AuthzHandler(ctlr *Controller) mux.MiddlewareFunc {
return
}
/* we want to bypass auth/authz for mgmt in case of authFail() authzFail()
unauthenticated users should have access to this route, but we also need to know if the user is an admin
*/
isMgmtRequested := request.RequestURI == constants.FullMgmtPrefix
acCtrlr := NewAccessController(ctlr.Config)
var identity string
@ -279,10 +274,9 @@ func AuthzHandler(ctlr *Controller) mux.MiddlewareFunc {
ctx := acCtrlr.getContext(identity, request)
// for extensions we only need to know if the user is admin and what repos he can read, so run next()
if request.RequestURI == fmt.Sprintf("%s%s", constants.RoutePrefix, constants.ExtCatalogPrefix) ||
strings.Contains(request.RequestURI, constants.FullSearchPrefix) ||
isMgmtRequested {
// for extensions, we only need to know the username, whether the user is an admin, and what repositories
// they can read. So, run next()
if isExtensionURI(request.RequestURI) {
next.ServeHTTP(response, request.WithContext(ctx)) //nolint:contextcheck
return
@ -321,6 +315,11 @@ func AuthzHandler(ctlr *Controller) mux.MiddlewareFunc {
}
}
func isExtensionURI(requestURI string) bool {
return strings.Contains(requestURI, constants.ExtPrefix) ||
requestURI == fmt.Sprintf("%s%s", constants.RoutePrefix, constants.ExtCatalogPrefix)
}
func authzFail(w http.ResponseWriter, realm string, delay int) {
time.Sleep(time.Duration(delay) * time.Second)
w.Header().Set("WWW-Authenticate", realm)

View file

@ -5,8 +5,11 @@ const (
ExtCatalogPrefix = "/_catalog"
ExtOciDiscoverPrefix = "/_oci/ext/discover"
// zot specific extensions.
ExtSearchPrefix = "/_zot/ext/search"
FullSearchPrefix = RoutePrefix + ExtSearchPrefix
ExtMgmtPrefix = "/_zot/ext/mgmt"
FullMgmtPrefix = RoutePrefix + ExtMgmtPrefix
ExtPrefix = "/_zot/ext"
ExtSearchPrefix = ExtPrefix + "/search"
FullSearchPrefix = RoutePrefix + ExtSearchPrefix
ExtMgmtPrefix = ExtPrefix + "/mgmt"
FullMgmtPrefix = RoutePrefix + ExtMgmtPrefix
ExtUserPreferencesPrefix = ExtPrefix + "/userprefs"
FullUserPreferencesPrefix = RoutePrefix + ExtUserPreferencesPrefix
)

View file

@ -158,6 +158,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
"repoMetaTablename": "RepoMetadataTable",
"manifestDataTablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"userDataTablename": "ZotUserDataTable",
"versionTablename": "Version",
}
@ -173,6 +174,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
"repoMetaTablename": "RepoMetadataTable",
"manifestDataTablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"userDataTablename": "ZotUserDataTable",
"versionTablename": "Version",
}
@ -187,6 +189,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
"repoMetaTablename": "RepoMetadataTable",
"manifestDataTablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"userDataTablename": "ZotUserDataTable",
"versionTablename": "Version",
}
@ -220,6 +223,7 @@ func TestCreateRepoDBDriver(t *testing.T) {
"repometatablename": "RepoMetadataTable",
"manifestdatatablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"userdatatablename": "UserDatatable",
}
testFunc := func() { _, _ = repodbfactory.New(conf.Storage.StorageConfig, log) }
@ -233,6 +237,7 @@ func TestCreateRepoDBDriver(t *testing.T) {
"repometatablename": "RepoMetadataTable",
"manifestdatatablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"userDataTablename": "ZotUserDataTable",
"versiontablename": 1,
}
@ -248,6 +253,7 @@ func TestCreateRepoDBDriver(t *testing.T) {
"manifestdatatablename": "ManifestDataTable",
"indexdatatablename": "IndexDataTable",
"artifactdatatablename": "ArtifactDataTable",
"userdatatablename": "ZotUserDataTable",
"versiontablename": "1",
}

View file

@ -127,6 +127,8 @@ func (rh *RouteHandler) SetupRoutes() {
// extended build
ext.SetupMetricsRoutes(rh.c.Config, rh.c.Router, rh.c.StoreController, AuthHandler(rh.c), rh.c.Log)
ext.SetupSearchRoutes(rh.c.Config, prefixedRouter, rh.c.StoreController, rh.c.RepoDB, rh.c.CveInfo, rh.c.Log)
ext.SetupUserPreferencesRoutes(rh.c.Config, prefixedRouter, rh.c.StoreController, rh.c.RepoDB, rh.c.CveInfo,
rh.c.Log)
ext.SetupUIRoutes(rh.c.Config, rh.c.Router, rh.c.StoreController, rh.c.Log)
ext.SetupMgmtRoutes(rh.c.Config, prefixedRouter, rh.c.Log)
gqlPlayground.SetupGQLPlaygroundRoutes(rh.c.Config, prefixedRouter, rh.c.StoreController, rh.c.Log)

View file

@ -40,6 +40,30 @@ func Contains(slice []string, item string) bool {
return false
}
// first match of item in [].
func Index(slice []string, item string) int {
for k, v := range slice {
if item == v {
return k
}
}
return -1
}
// remove matches of item in [].
func RemoveFrom(inputSlice []string, item string) []string {
var newSlice []string
for _, v := range inputSlice {
if item != v {
newSlice = append(newSlice, v)
}
}
return newSlice
}
func GetTLSConfig(certsPath string, caCertPool *x509.CertPool) (*tls.Config, error) {
clientCert := filepath.Join(certsPath, clientCertFilename)
clientKey := filepath.Join(certsPath, clientKeyFilename)

View file

@ -119,6 +119,11 @@ func TestCommon(t *testing.T) {
resultPtr, baseURL+"/v2/", ispec.MediaTypeImageManifest, log.NewLogger("", ""))
So(err, ShouldNotBeNil)
})
Convey("Index func", t, func() {
So(common.Index([]string{"a", "b"}, "b"), ShouldEqual, 1)
So(common.Index([]string{"a", "b"}, "c"), ShouldEqual, -1)
})
Convey("Test image dir and digest", t, func() {
repo, digest := common.GetImageDirAndDigest("image")
So(repo, ShouldResemble, "image")

View file

@ -10,12 +10,15 @@ type RepoInfo struct {
}
type RepoSummary struct {
Name string `json:"name"`
LastUpdated time.Time `json:"lastUpdated"`
Size string `json:"size"`
Platforms []Platform `json:"platforms"`
Vendors []string `json:"vendors"`
NewestImage ImageSummary `json:"newestImage"`
Name string `json:"name"`
LastUpdated time.Time `json:"lastUpdated"`
Size string `json:"size"`
Platforms []Platform `json:"platforms"`
Vendors []string `json:"vendors"`
IsStarred bool `json:"isStarred"`
IsBookmarked bool `json:"isBookmarked"`
StarCount int `json:"starCount"`
NewestImage ImageSummary `json:"newestImage"`
}
type ImageSummary struct {

View file

@ -0,0 +1,143 @@
//go:build userprefs
// +build userprefs
package extensions
import (
"errors"
"net/http"
"net/url"
"github.com/gorilla/mux"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/api/constants"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
)
const (
ToggleRepoBookmarkAction = "toggleBookmark"
ToggleRepoStarAction = "toggleStar"
)
func SetupUserPreferencesRoutes(config *config.Config, router *mux.Router, storeController storage.StoreController,
repoDB repodb.RepoDB, cveInfo CveInfo, log log.Logger,
) {
if config.Extensions.Search != nil && *config.Extensions.Search.Enable {
log.Info().Msg("setting up user preferences routes")
userprefsRouter := router.PathPrefix(constants.ExtUserPreferencesPrefix).Subrouter()
userprefsRouter.HandleFunc("", HandleUserPrefs(repoDB, log)).Methods(http.MethodPut)
}
}
func HandleUserPrefs(repoDB repodb.RepoDB, log log.Logger) func(w http.ResponseWriter, r *http.Request) {
return func(rsp http.ResponseWriter, req *http.Request) {
if !queryHasParams(req.URL.Query(), []string{"action"}) {
rsp.WriteHeader(http.StatusBadRequest)
return
}
action := req.URL.Query().Get("action")
switch action {
case ToggleRepoBookmarkAction:
PutBookmark(rsp, req, repoDB, log) //nolint:contextcheck
return
case ToggleRepoStarAction:
PutStar(rsp, req, repoDB, log) //nolint:contextcheck
return
default:
rsp.WriteHeader(http.StatusBadRequest)
return
}
}
}
func PutStar(rsp http.ResponseWriter, req *http.Request, repoDB repodb.RepoDB, log log.Logger) {
if !queryHasParams(req.URL.Query(), []string{"repo"}) {
rsp.WriteHeader(http.StatusBadRequest)
return
}
repo := req.URL.Query().Get("repo")
if repo == "" {
rsp.WriteHeader(http.StatusNotFound)
return
}
_, err := repoDB.ToggleStarRepo(req.Context(), repo)
if err != nil {
if errors.Is(err, zerr.ErrRepoMetaNotFound) {
rsp.WriteHeader(http.StatusNotFound)
return
} else if errors.Is(err, zerr.ErrUserDataNotAllowed) {
rsp.WriteHeader(http.StatusForbidden)
return
}
rsp.WriteHeader(http.StatusInternalServerError)
return
}
rsp.WriteHeader(http.StatusOK)
}
func PutBookmark(rsp http.ResponseWriter, req *http.Request, repoDB repodb.RepoDB, log log.Logger) {
if !queryHasParams(req.URL.Query(), []string{"repo"}) {
rsp.WriteHeader(http.StatusBadRequest)
return
}
repo := req.URL.Query().Get("repo")
if repo == "" {
rsp.WriteHeader(http.StatusNotFound)
return
}
_, err := repoDB.ToggleBookmarkRepo(req.Context(), repo)
if err != nil {
if errors.Is(err, zerr.ErrRepoMetaNotFound) {
rsp.WriteHeader(http.StatusNotFound)
return
} else if errors.Is(err, zerr.ErrUserDataNotAllowed) {
rsp.WriteHeader(http.StatusForbidden)
return
}
rsp.WriteHeader(http.StatusInternalServerError)
return
}
rsp.WriteHeader(http.StatusOK)
}
func queryHasParams(values url.Values, params []string) bool {
for _, param := range params {
if !values.Has(param) {
return false
}
}
return true
}

View file

@ -0,0 +1,20 @@
//go:build !userprefs
// +build !userprefs
package extensions
import (
"github.com/gorilla/mux"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/storage"
)
func SetupUserPreferencesRoutes(config *config.Config, router *mux.Router, storeController storage.StoreController,
repoDB repodb.RepoDB, cveInfo CveInfo, log log.Logger,
) {
log.Warn().Msg("userprefs extension is disabled because given zot binary doesn't" +
"include this feature please build a binary that does so")
}

View file

@ -0,0 +1,139 @@
//go:build userprefs
// +build userprefs
package extensions_test
import (
"context"
"errors"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gorilla/mux"
. "github.com/smartystreets/goconvey/convey"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/extensions"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/test/mocks"
)
var ErrTestError = errors.New("TestError")
const UserprefsBaseURL = "http://127.0.0.1:8080/v2/_zot/ext/userprefs"
func TestHandlers(t *testing.T) {
log := log.NewLogger("debug", "")
mockrepoDB := mocks.RepoDBMock{}
Convey("No repo in request", t, func() {
request := httptest.NewRequest("GET", UserprefsBaseURL+"", strings.NewReader("My string"))
response := httptest.NewRecorder()
extensions.PutStar(response, request, mockrepoDB, log)
res := response.Result()
So(res.StatusCode, ShouldEqual, http.StatusBadRequest)
defer res.Body.Close()
extensions.PutBookmark(response, request, mockrepoDB, log)
res = response.Result()
So(res.StatusCode, ShouldEqual, http.StatusBadRequest)
defer res.Body.Close()
})
Convey("Empty repo in request", t, func() {
request := httptest.NewRequest("GET", UserprefsBaseURL+"?repo=", strings.NewReader("My string"))
response := httptest.NewRecorder()
extensions.PutStar(response, request, mockrepoDB, log)
res := response.Result()
So(res.StatusCode, ShouldEqual, http.StatusNotFound)
defer res.Body.Close()
extensions.PutBookmark(response, request, mockrepoDB, log)
res = response.Result()
So(res.StatusCode, ShouldEqual, http.StatusNotFound)
defer res.Body.Close()
})
Convey("ToggleStarRepo different errors", t, func() {
request := httptest.NewRequest("GET", UserprefsBaseURL+"?repo=test",
strings.NewReader("My string"))
Convey("ErrRepoMetaNotFound", func() {
mockrepoDB.ToggleStarRepoFn = func(ctx context.Context, repo string) (repodb.ToggleState, error) {
return repodb.NotChanged, zerr.ErrRepoMetaNotFound
}
mockrepoDB.ToggleBookmarkRepoFn = func(ctx context.Context, repo string) (repodb.ToggleState, error) {
return repodb.NotChanged, zerr.ErrRepoMetaNotFound
}
response := httptest.NewRecorder()
extensions.PutBookmark(response, request, mockrepoDB, log)
res := response.Result()
So(res.StatusCode, ShouldEqual, http.StatusNotFound)
defer res.Body.Close()
response = httptest.NewRecorder()
extensions.PutStar(response, request, mockrepoDB, log)
res = response.Result()
So(res.StatusCode, ShouldEqual, http.StatusNotFound)
defer res.Body.Close()
})
Convey("ErrUserDataNotAllowed", func() {
request = mux.SetURLVars(request, map[string]string{
"name": "repo",
})
mockrepoDB.ToggleBookmarkRepoFn = func(ctx context.Context, repo string) (repodb.ToggleState, error) {
return repodb.NotChanged, zerr.ErrUserDataNotAllowed
}
mockrepoDB.ToggleStarRepoFn = func(ctx context.Context, repo string) (repodb.ToggleState, error) {
return repodb.NotChanged, zerr.ErrUserDataNotAllowed
}
response := httptest.NewRecorder()
extensions.PutBookmark(response, request, mockrepoDB, log)
res := response.Result()
So(res.StatusCode, ShouldEqual, http.StatusForbidden)
defer res.Body.Close()
response = httptest.NewRecorder()
extensions.PutStar(response, request, mockrepoDB, log)
res = response.Result()
So(res.StatusCode, ShouldEqual, http.StatusForbidden)
defer res.Body.Close()
})
Convey("ErrUnexpectedError", func() {
request = mux.SetURLVars(request, map[string]string{
"name": "repo",
})
mockrepoDB.ToggleBookmarkRepoFn = func(ctx context.Context, repo string) (repodb.ToggleState, error) {
return repodb.NotChanged, ErrTestError
}
mockrepoDB.ToggleStarRepoFn = func(ctx context.Context, repo string) (repodb.ToggleState, error) {
return repodb.NotChanged, ErrTestError
}
response := httptest.NewRecorder()
extensions.PutBookmark(response, request, mockrepoDB, log)
res := response.Result()
So(res.StatusCode, ShouldEqual, http.StatusInternalServerError)
defer res.Body.Close()
response = httptest.NewRecorder()
extensions.PutStar(response, request, mockrepoDB, log)
res = response.Result()
So(res.StatusCode, ShouldEqual, http.StatusInternalServerError)
defer res.Body.Close()
})
})
}

View file

@ -31,15 +31,15 @@ func RepoMeta2RepoSummary(ctx context.Context, repoMeta repodb.RepoMetadata,
skip SkipQGLField, cveInfo cveinfo.CveInfo,
) *gql_generated.RepoSummary {
var (
repoName = repoMeta.Name
repoLastUpdatedTimestamp = time.Time{}
repoPlatformsSet = map[string]*gql_generated.Platform{}
repoVendorsSet = map[string]bool{}
lastUpdatedImageSummary *gql_generated.ImageSummary
repoStarCount = repoMeta.Stars
isBookmarked = false
isStarred = false
repoDownloadCount = 0
repoName = repoMeta.Name
repoStarCount = repoMeta.Stars // total number of stars
repoIsUserStarred = repoMeta.IsStarred // value specific to the current user
repoIsUserBookMarked = repoMeta.IsBookmarked // value specific to the current user
// map used to keep track of all blobs of a repo without dublicates as
// some images may have the same layers
@ -88,6 +88,7 @@ func RepoMeta2RepoSummary(ctx context.Context, repoMeta repodb.RepoMetadata,
repoSize := strconv.FormatInt(size, 10)
repoPlatforms := make([]*gql_generated.Platform, 0, len(repoPlatformsSet))
for _, platform := range repoPlatformsSet {
repoPlatforms = append(repoPlatforms, platform)
}
@ -129,8 +130,8 @@ func RepoMeta2RepoSummary(ctx context.Context, repoMeta repodb.RepoMetadata,
NewestImage: lastUpdatedImageSummary,
DownloadCount: &repoDownloadCount,
StarCount: &repoStarCount,
IsBookmarked: &isBookmarked,
IsStarred: &isStarred,
IsBookmarked: &repoIsUserBookMarked,
IsStarred: &repoIsUserStarred,
}
}
@ -574,15 +575,15 @@ func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta repodb.RepoMetadata
skip SkipQGLField, cveInfo cveinfo.CveInfo, log log.Logger,
) (*gql_generated.RepoSummary, []*gql_generated.ImageSummary) {
var (
repoName = repoMeta.Name
repoLastUpdatedTimestamp = time.Time{}
repoPlatformsSet = map[string]*gql_generated.Platform{}
repoVendorsSet = map[string]bool{}
lastUpdatedImageSummary *gql_generated.ImageSummary
repoStarCount = repoMeta.Stars
isBookmarked = false
isStarred = false
repoDownloadCount = 0
repoName = repoMeta.Name
repoStarCount = repoMeta.Stars // total number of stars
isStarred = repoMeta.IsStarred // value specific to the current user
isBookmarked = repoMeta.IsBookmarked // value specific to the current user
// map used to keep track of all blobs of a repo without dublicates as
// some images may have the same layers
@ -632,6 +633,7 @@ func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta repodb.RepoMetadata
repoSize := strconv.FormatInt(size, 10)
repoPlatforms := make([]*gql_generated.Platform, 0, len(repoPlatformsSet))
for _, platform := range repoPlatformsSet {
repoPlatforms = append(repoPlatforms, platform)
}

View file

@ -86,7 +86,8 @@ func TestDigestSearchHTTP(t *testing.T) {
)
So(err, ShouldBeNil)
image1.Reference = "0.0.1"
const ver001 = "0.0.1"
image1.Reference = ver001
err = UploadImage(
image1,
baseURL,
@ -109,7 +110,7 @@ func TestDigestSearchHTTP(t *testing.T) {
)
So(err, ShouldBeNil)
image2.Reference = "0.0.1"
image2.Reference = ver001
manifestDigest, err := image2.Digest()
So(err, ShouldBeNil)

View file

@ -156,6 +156,7 @@ type ComplexityRoot struct {
Query struct {
BaseImageList func(childComplexity int, image string, digest *string, requestedPage *PageInput) int
BookmarkedRepos func(childComplexity int, requestedPage *PageInput) int
CVEListForImage func(childComplexity int, image string, requestedPage *PageInput, searchedCve *string) int
DerivedImageList func(childComplexity int, image string, digest *string, requestedPage *PageInput) int
ExpandedRepoInfo func(childComplexity int, repo string) int
@ -167,6 +168,7 @@ type ComplexityRoot struct {
ImageListWithCVEFixed func(childComplexity int, id string, image string, requestedPage *PageInput) int
Referrers func(childComplexity int, repo string, digest string, typeArg []string) int
RepoListWithNewestImage func(childComplexity int, requestedPage *PageInput) int
StarredRepos func(childComplexity int, requestedPage *PageInput) int
}
Referrer struct {
@ -209,6 +211,8 @@ type QueryResolver interface {
BaseImageList(ctx context.Context, image string, digest *string, requestedPage *PageInput) (*PaginatedImagesResult, error)
Image(ctx context.Context, image string) (*ImageSummary, error)
Referrers(ctx context.Context, repo string, digest string, typeArg []string) ([]*Referrer, error)
StarredRepos(ctx context.Context, requestedPage *PageInput) (*PaginatedReposResult, error)
BookmarkedRepos(ctx context.Context, requestedPage *PageInput) (*PaginatedReposResult, error)
}
type executableSchema struct {
@ -700,6 +704,18 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
return e.complexity.Query.BaseImageList(childComplexity, args["image"].(string), args["digest"].(*string), args["requestedPage"].(*PageInput)), true
case "Query.BookmarkedRepos":
if e.complexity.Query.BookmarkedRepos == nil {
break
}
args, err := ec.field_Query_BookmarkedRepos_args(context.TODO(), rawArgs)
if err != nil {
return 0, false
}
return e.complexity.Query.BookmarkedRepos(childComplexity, args["requestedPage"].(*PageInput)), true
case "Query.CVEListForImage":
if e.complexity.Query.CVEListForImage == nil {
break
@ -832,6 +848,18 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in
return e.complexity.Query.RepoListWithNewestImage(childComplexity, args["requestedPage"].(*PageInput)), true
case "Query.StarredRepos":
if e.complexity.Query.StarredRepos == nil {
break
}
args, err := ec.field_Query_StarredRepos_args(context.TODO(), rawArgs)
if err != nil {
return 0, false
}
return e.complexity.Query.StarredRepos(childComplexity, args["requestedPage"].(*PageInput)), true
case "Referrer.Annotations":
if e.complexity.Referrer.Annotations == nil {
break
@ -1688,6 +1716,22 @@ type Query {
"Types of artifacts to return in the referrer list"
type: [String!]
): [Referrer]!
"""
Receive RepoSummaries of repos starred by current user
"""
StarredRepos(
"Sets the parameters of the requested page (how many to include and offset)"
requestedPage: PageInput
): PaginatedReposResult!
"""
Receive RepoSummaries of repos bookmarked by current user
"""
BookmarkedRepos(
"Sets the parameters of the requested page (how many to include and offset)"
requestedPage: PageInput
): PaginatedReposResult!
}
`, BuiltIn: false},
}
@ -1730,6 +1774,21 @@ func (ec *executionContext) field_Query_BaseImageList_args(ctx context.Context,
return args, nil
}
func (ec *executionContext) field_Query_BookmarkedRepos_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) {
var err error
args := map[string]interface{}{}
var arg0 *PageInput
if tmp, ok := rawArgs["requestedPage"]; ok {
ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("requestedPage"))
arg0, err = ec.unmarshalOPageInput2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐPageInput(ctx, tmp)
if err != nil {
return nil, err
}
}
args["requestedPage"] = arg0
return args, nil
}
func (ec *executionContext) field_Query_CVEListForImage_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) {
var err error
args := map[string]interface{}{}
@ -2012,6 +2071,21 @@ func (ec *executionContext) field_Query_RepoListWithNewestImage_args(ctx context
return args, nil
}
func (ec *executionContext) field_Query_StarredRepos_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) {
var err error
args := map[string]interface{}{}
var arg0 *PageInput
if tmp, ok := rawArgs["requestedPage"]; ok {
ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("requestedPage"))
arg0, err = ec.unmarshalOPageInput2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐPageInput(ctx, tmp)
if err != nil {
return nil, err
}
}
args["requestedPage"] = arg0
return args, nil
}
func (ec *executionContext) field_Query___type_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) {
var err error
args := map[string]interface{}{}
@ -5831,6 +5905,128 @@ func (ec *executionContext) fieldContext_Query_Referrers(ctx context.Context, fi
return fc, nil
}
func (ec *executionContext) _Query_StarredRepos(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) {
fc, err := ec.fieldContext_Query_StarredRepos(ctx, field)
if err != nil {
return graphql.Null
}
ctx = graphql.WithFieldContext(ctx, fc)
defer func() {
if r := recover(); r != nil {
ec.Error(ctx, ec.Recover(ctx, r))
ret = graphql.Null
}
}()
resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
ctx = rctx // use context from middleware stack in children
return ec.resolvers.Query().StarredRepos(rctx, fc.Args["requestedPage"].(*PageInput))
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
ec.Errorf(ctx, "must not be null")
}
return graphql.Null
}
res := resTmp.(*PaginatedReposResult)
fc.Result = res
return ec.marshalNPaginatedReposResult2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐPaginatedReposResult(ctx, field.Selections, res)
}
func (ec *executionContext) fieldContext_Query_StarredRepos(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) {
fc = &graphql.FieldContext{
Object: "Query",
Field: field,
IsMethod: true,
IsResolver: true,
Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) {
switch field.Name {
case "Page":
return ec.fieldContext_PaginatedReposResult_Page(ctx, field)
case "Results":
return ec.fieldContext_PaginatedReposResult_Results(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type PaginatedReposResult", field.Name)
},
}
defer func() {
if r := recover(); r != nil {
err = ec.Recover(ctx, r)
ec.Error(ctx, err)
}
}()
ctx = graphql.WithFieldContext(ctx, fc)
if fc.Args, err = ec.field_Query_StarredRepos_args(ctx, field.ArgumentMap(ec.Variables)); err != nil {
ec.Error(ctx, err)
return
}
return fc, nil
}
func (ec *executionContext) _Query_BookmarkedRepos(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) {
fc, err := ec.fieldContext_Query_BookmarkedRepos(ctx, field)
if err != nil {
return graphql.Null
}
ctx = graphql.WithFieldContext(ctx, fc)
defer func() {
if r := recover(); r != nil {
ec.Error(ctx, ec.Recover(ctx, r))
ret = graphql.Null
}
}()
resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) {
ctx = rctx // use context from middleware stack in children
return ec.resolvers.Query().BookmarkedRepos(rctx, fc.Args["requestedPage"].(*PageInput))
})
if err != nil {
ec.Error(ctx, err)
return graphql.Null
}
if resTmp == nil {
if !graphql.HasFieldError(ctx, fc) {
ec.Errorf(ctx, "must not be null")
}
return graphql.Null
}
res := resTmp.(*PaginatedReposResult)
fc.Result = res
return ec.marshalNPaginatedReposResult2ᚖzotregistryᚗioᚋzotᚋpkgᚋextensionsᚋsearchᚋgql_generatedᚐPaginatedReposResult(ctx, field.Selections, res)
}
func (ec *executionContext) fieldContext_Query_BookmarkedRepos(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) {
fc = &graphql.FieldContext{
Object: "Query",
Field: field,
IsMethod: true,
IsResolver: true,
Child: func(ctx context.Context, field graphql.CollectedField) (*graphql.FieldContext, error) {
switch field.Name {
case "Page":
return ec.fieldContext_PaginatedReposResult_Page(ctx, field)
case "Results":
return ec.fieldContext_PaginatedReposResult_Results(ctx, field)
}
return nil, fmt.Errorf("no field named %q was found under type PaginatedReposResult", field.Name)
},
}
defer func() {
if r := recover(); r != nil {
err = ec.Recover(ctx, r)
ec.Error(ctx, err)
}
}()
ctx = graphql.WithFieldContext(ctx, fc)
if fc.Args, err = ec.field_Query_BookmarkedRepos_args(ctx, field.ArgumentMap(ec.Variables)); err != nil {
ec.Error(ctx, err)
return
}
return fc, nil
}
func (ec *executionContext) _Query___type(ctx context.Context, field graphql.CollectedField) (ret graphql.Marshaler) {
fc, err := ec.fieldContext_Query___type(ctx, field)
if err != nil {
@ -9526,6 +9722,52 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr
return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc)
}
out.Concurrently(i, func() graphql.Marshaler {
return rrm(innerCtx)
})
case "StarredRepos":
field := field
innerFunc := func(ctx context.Context) (res graphql.Marshaler) {
defer func() {
if r := recover(); r != nil {
ec.Error(ctx, ec.Recover(ctx, r))
}
}()
res = ec._Query_StarredRepos(ctx, field)
if res == graphql.Null {
atomic.AddUint32(&invalids, 1)
}
return res
}
rrm := func(ctx context.Context) graphql.Marshaler {
return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc)
}
out.Concurrently(i, func() graphql.Marshaler {
return rrm(innerCtx)
})
case "BookmarkedRepos":
field := field
innerFunc := func(ctx context.Context) (res graphql.Marshaler) {
defer func() {
if r := recover(); r != nil {
ec.Error(ctx, ec.Recover(ctx, r))
}
}()
res = ec._Query_BookmarkedRepos(ctx, field)
if res == graphql.Null {
atomic.AddUint32(&invalids, 1)
}
return res
}
rrm := func(ctx context.Context) graphql.Marshaler {
return ec.OperationContext.RootResolverMiddleware(ctx, innerFunc)
}
out.Concurrently(i, func() graphql.Marshaler {
return rrm(innerCtx)
})

View file

@ -18,7 +18,7 @@ import (
"github.com/vektah/gqlparser/v2/gqlerror"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/common"
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/extensions/search/convert"
cveinfo "zotregistry.io/zot/pkg/extensions/search/cve"
cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model"
@ -300,7 +300,7 @@ func getCVEListForImage(
),
}
repo, ref, isTag := common.GetImageDirAndReference(image)
repo, ref, isTag := zcommon.GetImageDirAndReference(image)
if ref == "" {
return &gql_generated.CVEResultForImage{}, gqlerror.Errorf("no reference provided")
@ -560,6 +560,91 @@ func repoListWithNewestImage(
return paginatedRepos, nil
}
func getBookmarkedRepos(
ctx context.Context,
cveInfo cveinfo.CveInfo,
log log.Logger, //nolint:unparam // may be used by devs for debugging
requestedPage *gql_generated.PageInput,
repoDB repodb.RepoDB,
) (*gql_generated.PaginatedReposResult, error) {
repoNames, err := repoDB.GetBookmarkedRepos(ctx)
if err != nil {
return &gql_generated.PaginatedReposResult{}, err
}
filterFn := func(repoMeta repodb.RepoMetadata) bool {
return zcommon.Contains(repoNames, repoMeta.Name)
}
return getFilteredPaginatedRepos(ctx, cveInfo, filterFn, log, requestedPage, repoDB)
}
func getStarredRepos(
ctx context.Context,
cveInfo cveinfo.CveInfo,
log log.Logger, //nolint:unparam // may be used by devs for debugging
requestedPage *gql_generated.PageInput,
repoDB repodb.RepoDB,
) (*gql_generated.PaginatedReposResult, error) {
repoNames, err := repoDB.GetStarredRepos(ctx)
if err != nil {
return &gql_generated.PaginatedReposResult{}, err
}
filterFn := func(repoMeta repodb.RepoMetadata) bool {
return zcommon.Contains(repoNames, repoMeta.Name)
}
return getFilteredPaginatedRepos(ctx, cveInfo, filterFn, log, requestedPage, repoDB)
}
func getFilteredPaginatedRepos(
ctx context.Context,
cveInfo cveinfo.CveInfo,
filterFn repodb.FilterRepoFunc,
log log.Logger, //nolint:unparam // may be used by devs for debugging
requestedPage *gql_generated.PageInput,
repoDB repodb.RepoDB,
) (*gql_generated.PaginatedReposResult, error) {
repos := []*gql_generated.RepoSummary{}
paginatedRepos := &gql_generated.PaginatedReposResult{}
if requestedPage == nil {
requestedPage = &gql_generated.PageInput{}
}
skip := convert.SkipQGLField{
Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Results.NewestImage.Vulnerabilities"),
}
pageInput := repodb.PageInput{
Limit: safeDerefferencing(requestedPage.Limit, 0),
Offset: safeDerefferencing(requestedPage.Offset, 0),
SortBy: repodb.SortCriteria(
safeDerefferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime),
),
}
reposMeta, manifestMetaMap, indexDataMap, pageInfo, err := repoDB.FilterRepos(ctx, filterFn, pageInput)
if err != nil {
return paginatedRepos, err
}
for _, repoMeta := range reposMeta {
repoSummary := convert.RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, indexDataMap,
skip, cveInfo)
repos = append(repos, repoSummary)
}
paginatedRepos.Page = &gql_generated.PageInfo{
TotalCount: pageInfo.TotalCount,
ItemCount: pageInfo.ItemCount,
}
paginatedRepos.Results = repos
return paginatedRepos, nil
}
func globalSearch(ctx context.Context, query string, repoDB repodb.RepoDB, filter *gql_generated.Filter,
requestedPage *gql_generated.PageInput, cveInfo cveinfo.CveInfo, log log.Logger, //nolint:unparam
) (*gql_generated.PaginatedReposResult, []*gql_generated.ImageSummary, []*gql_generated.LayerSummary, error,
@ -675,7 +760,7 @@ func derivedImageList(ctx context.Context, image string, digest *string, repoDB
Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Vulnerabilities"),
}
imageRepo, imageTag := common.GetImageDirAndTag(image)
imageRepo, imageTag := zcommon.GetImageDirAndTag(image)
if imageTag == "" {
return &gql_generated.PaginatedImagesResult{}, gqlerror.Errorf("no reference provided")
}
@ -788,7 +873,7 @@ func baseImageList(ctx context.Context, image string, digest *string, repoDB rep
Vulnerabilities: canSkipField(convert.GetPreloads(ctx), "Vulnerabilities"),
}
imageRepo, imageTag := common.GetImageDirAndTag(image)
imageRepo, imageTag := zcommon.GetImageDirAndTag(image)
if imageTag == "" {
return &gql_generated.PaginatedImagesResult{}, gqlerror.Errorf("no reference provided")

View file

@ -22,6 +22,7 @@ import (
"zotregistry.io/zot/pkg/meta/bolt"
"zotregistry.io/zot/pkg/meta/repodb"
boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper"
localCtx "zotregistry.io/zot/pkg/requestcontext"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/test/mocks"
)
@ -418,7 +419,7 @@ func TestRepoListWithNewestImage(t *testing.T) {
So(repos.Results, ShouldBeEmpty)
})
Convey("RepoDB SearchRepo Bad manifest referenced", func() {
Convey("RepoDB SearchRepo bad manifest referenced", func() {
mockRepoDB := mocks.RepoDBMock{
SearchReposFn: func(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
@ -546,8 +547,8 @@ func TestRepoListWithNewestImage(t *testing.T) {
for _, repoMeta := range repos {
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repoMeta,
UpdateTime: createTime,
RepoMetadata: repoMeta,
UpdateTime: createTime,
})
createTime = createTime.Add(time.Second)
}
@ -622,6 +623,68 @@ func TestRepoListWithNewestImage(t *testing.T) {
})
}
func TestGetBookmarkedRepos(t *testing.T) {
Convey("getBookmarkedRepos", t, func() {
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, err := getBookmarkedRepos(
responseContext,
mocks.CveInfoMock{},
log.NewLogger("debug", ""),
nil,
mocks.RepoDBMock{
GetBookmarkedReposFn: func(ctx context.Context) ([]string, error) {
return []string{}, ErrTestError
},
},
)
So(err, ShouldNotBeNil)
})
}
func TestGetStarredRepos(t *testing.T) {
Convey("getStarredRepos", t, func() {
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, err := getStarredRepos(
responseContext,
mocks.CveInfoMock{},
log.NewLogger("debug", ""),
nil,
mocks.RepoDBMock{
GetStarredReposFn: func(ctx context.Context) ([]string, error) {
return []string{}, ErrTestError
},
},
)
So(err, ShouldNotBeNil)
})
}
func TestGetFilteredPaginatedRepos(t *testing.T) {
Convey("getFilteredPaginatedRepos FilterRepos fails", t, func() {
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, err := getFilteredPaginatedRepos(
responseContext,
mocks.CveInfoMock{},
func(repoMeta repodb.RepoMetadata) bool { return true },
log.NewLogger("debug", ""),
nil,
mocks.RepoDBMock{
FilterReposFn: func(ctx context.Context, filter repodb.FilterRepoFunc, requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo,
error,
) {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
repodb.PageInfo{}, ErrTestError
},
},
)
So(err, ShouldNotBeNil)
})
}
func TestImageListForDigest(t *testing.T) {
Convey("getImageList", t, func() {
Convey("no page requested, FilterTagsFn returns error", func() {
@ -1028,7 +1091,7 @@ func TestImageListForDigest(t *testing.T) {
repos[i].Tags = matchedTags
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repo,
RepoMetadata: repo,
})
}
@ -2486,6 +2549,28 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo
So(len(images.Results), ShouldEqual, 0)
})
})
Convey("Errors for cve resolvers", t, func() {
_, err := getImageListForCVE(
context.Background(),
"id",
mocks.CveInfoMock{
GetImageListForCVEFn: func(repo, cveID string) ([]cvemodel.TagInfo, error) {
return []cvemodel.TagInfo{}, ErrTestError
},
},
nil,
mocks.RepoDBMock{
GetMultipleRepoMetaFn: func(ctx context.Context, filter func(repoMeta repodb.RepoMetadata) bool,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, error) {
return []repodb.RepoMetadata{{}}, nil
},
},
log,
)
So(err, ShouldNotBeNil)
})
}
func getPageInput(limit int, offset int) *gql_generated.PageInput {
@ -2726,7 +2811,7 @@ func TestDerivedImageList(t *testing.T) {
repos[i].Tags = matchedTags
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repo,
RepoMetadata: repo,
})
}
repos, pageInfo := pageFinder.Page()
@ -2989,7 +3074,7 @@ func TestBaseImageList(t *testing.T) {
repos[i].Tags = matchedTags
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repo,
RepoMetadata: repo,
})
}
@ -3163,7 +3248,7 @@ func TestBaseImageList(t *testing.T) {
repos[i].Tags = matchedTags
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repo,
RepoMetadata: repo,
})
}
@ -3182,6 +3267,8 @@ func TestBaseImageList(t *testing.T) {
}
func TestExpandedRepoInfo(t *testing.T) {
log := log.NewLogger("debug", "")
Convey("ExpandedRepoInfo Errors", t, func() {
responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
@ -3198,10 +3285,22 @@ func TestExpandedRepoInfo(t *testing.T) {
Digest: "digestIndex",
MediaType: ispec.MediaTypeImageIndex,
},
"tagGetIndexError": {
Digest: "errorIndexDigest",
MediaType: ispec.MediaTypeImageIndex,
},
"tagGoodIndexBadManifests": {
Digest: "goodIndexBadManifests",
MediaType: ispec.MediaTypeImageIndex,
},
"tagGoodIndex1GoodManfest": {
Digest: "goodIndexGoodManfest",
MediaType: ispec.MediaTypeImageIndex,
},
"tagGoodIndex2GoodManfest": {
Digest: "goodIndexGoodManfest",
MediaType: ispec.MediaTypeImageIndex,
},
},
}, nil
},
@ -3227,6 +3326,16 @@ func TestExpandedRepoInfo(t *testing.T) {
})
So(err, ShouldBeNil)
goodIndexGoodManfestBlob, err := json.Marshal(ispec.Index{
Manifests: []ispec.Descriptor{
{
Digest: "goodManifest",
MediaType: ispec.MediaTypeImageManifest,
},
},
})
So(err, ShouldBeNil)
switch indexDigest {
case "errorIndexDigest":
return repodb.IndexData{}, ErrTestError
@ -3234,14 +3343,61 @@ func TestExpandedRepoInfo(t *testing.T) {
return repodb.IndexData{
IndexBlob: goodIndexBadManifestsBlob,
}, nil
case "goodIndexGoodManfest":
return repodb.IndexData{
IndexBlob: goodIndexGoodManfestBlob,
}, nil
default:
return repodb.IndexData{}, nil
}
},
}
log := log.NewLogger("debug", "")
_, err := expandedRepoInfo(responseContext, "repo", repoDB, mocks.CveInfoMock{}, log)
So(err, ShouldBeNil)
})
Convey("Access error", t, func() {
authzCtxKey := localCtx.GetContextKey()
acCtxUser := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": false,
},
Username: "user",
}
ctx := context.WithValue(context.Background(), authzCtxKey, acCtxUser)
responseContext := graphql.WithResponseContext(ctx, graphql.DefaultErrorPresenter,
graphql.DefaultRecover)
_, err := expandedRepoInfo(responseContext, "repo", mocks.RepoDBMock{}, mocks.CveInfoMock{}, log)
So(err, ShouldBeNil)
})
}
func TestFilterFunctions(t *testing.T) {
Convey("Filter Functions", t, func() {
Convey("FilterByDigest bad manifest blob", func() {
filterFunc := FilterByDigest("digest")
ok := filterFunc(
repodb.RepoMetadata{},
repodb.ManifestMetadata{
ManifestBlob: []byte("bad blob"),
},
)
So(ok, ShouldBeFalse)
})
Convey("filterDerivedImages bad manifest blob", func() {
filterFunc := filterDerivedImages(&gql_generated.ImageSummary{})
ok := filterFunc(
repodb.RepoMetadata{},
repodb.ManifestMetadata{
ManifestBlob: []byte("bad blob"),
},
)
So(ok, ShouldBeFalse)
})
})
}

View file

@ -680,4 +680,20 @@ type Query {
"Types of artifacts to return in the referrer list"
type: [String!]
): [Referrer]!
"""
Receive RepoSummaries of repos starred by current user
"""
StarredRepos(
"Sets the parameters of the requested page (how many to include and offset)"
requestedPage: PageInput
): PaginatedReposResult!
"""
Receive RepoSummaries of repos bookmarked by current user
"""
BookmarkedRepos(
"Sets the parameters of the requested page (how many to include and offset)"
requestedPage: PageInput
): PaginatedReposResult!
}

View file

@ -144,6 +144,16 @@ func (r *queryResolver) Referrers(ctx context.Context, repo string, digest strin
return referrers, nil
}
// StarredRepos is the resolver for the StarredRepos field.
func (r *queryResolver) StarredRepos(ctx context.Context, requestedPage *gql_generated.PageInput) (*gql_generated.PaginatedReposResult, error) {
return getStarredRepos(ctx, r.cveInfo, r.log, requestedPage, r.repoDB)
}
// BookmarkedRepos is the resolver for the BookmarkedRepos field.
func (r *queryResolver) BookmarkedRepos(ctx context.Context, requestedPage *gql_generated.PageInput) (*gql_generated.PaginatedReposResult, error) {
return getBookmarkedRepos(ctx, r.cveInfo, r.log, requestedPage, r.repoDB)
}
// Query returns gql_generated.QueryResolver implementation.
func (r *Resolver) Query() gql_generated.QueryResolver { return &queryResolver{r} }

View file

@ -147,6 +147,26 @@ type ImageSummaryResult struct {
Errors []ErrorGQL `json:"errors"`
}
//nolint:tagliatelle // graphQL schema
type StarredRepos struct {
PaginatedReposResult `json:"StarredRepos"`
}
//nolint:tagliatelle // graphQL schema
type BookmarkedRepos struct {
PaginatedReposResult `json:"BookmarkedRepos"`
}
type StarredReposResponse struct {
StarredRepos `json:"data"`
Errors []ErrorGQL `json:"errors"`
}
type BookmarkedReposResponse struct {
BookmarkedRepos `json:"data"`
Errors []ErrorGQL `json:"errors"`
}
func readFileAndSearchString(filePath string, stringToMatch string, timeout time.Duration) (bool, error) {
ctx, cancelFunc := context.WithTimeout(context.Background(), timeout)
defer cancelFunc()

View file

@ -0,0 +1,632 @@
//go:build search && userprefs
package search_test
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"os"
"testing"
. "github.com/smartystreets/goconvey/convey"
"golang.org/x/crypto/bcrypt"
"gopkg.in/resty.v1"
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/api/constants"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/local"
. "zotregistry.io/zot/pkg/test"
)
//nolint:dupl
func TestUserData(t *testing.T) {
Convey("Test user stars and bookmarks", t, func(c C) {
port := GetFreePort()
baseURL := GetBaseURL(port)
defaultVal := true
accessibleRepo := "accessible-repo"
forbiddenRepo := "forbidden-repo"
tag := "0.0.1"
adminUser := "alice"
adminPassword := "deepGoesTheRabbitBurrow"
simpleUser := "test"
simpleUserPassword := "test123"
twoCredTests := fmt.Sprintf("%s\n%s\n\n", getCredString(adminUser, adminPassword),
getCredString(simpleUser, simpleUserPassword))
htpasswdPath := MakeHtpasswdFileFromString(twoCredTests)
defer os.Remove(htpasswdPath)
conf := config.New()
conf.Storage.RootDirectory = t.TempDir()
conf.HTTP.Port = port
conf.HTTP.Auth = &config.AuthConfig{
HTPasswd: config.AuthHTPasswd{
Path: htpasswdPath,
},
}
conf.HTTP.AccessControl = &config.AccessControlConfig{
Repositories: config.Repositories{
"**": config.PolicyGroup{
Policies: []config.Policy{
{
Users: []string{simpleUser},
Actions: []string{"read"},
},
},
AnonymousPolicy: []string{"read"},
DefaultPolicy: []string{},
},
forbiddenRepo: config.PolicyGroup{
Policies: []config.Policy{
{
Users: []string{},
Actions: []string{},
},
},
DefaultPolicy: []string{},
},
},
AdminPolicy: config.Policy{
Users: []string{adminUser},
Actions: []string{"read", "create", "update"},
},
}
conf.Extensions = &extconf.ExtensionConfig{
Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}},
}
ctlr := api.NewController(conf)
ctlrManager := NewControllerManager(ctlr)
ctlrManager.StartAndWait(port)
defer ctlrManager.StopServer()
config, layers, manifest, err := GetImageComponents(100)
So(err, ShouldBeNil)
err = UploadImageWithBasicAuth(
Image{
Config: config,
Layers: layers,
Manifest: manifest,
Reference: tag,
}, baseURL, accessibleRepo,
adminUser, adminPassword,
)
So(err, ShouldBeNil)
err = UploadImageWithBasicAuth(
Image{
Config: config,
Layers: layers,
Manifest: manifest,
Reference: tag,
}, baseURL, forbiddenRepo,
adminUser, adminPassword,
)
So(err, ShouldBeNil)
userStaredReposQuery := `{
StarredRepos {
Results {
Name StarCount IsStarred
NewestImage { Tag }
}
}
}`
userBookmarkedReposQuery := `{
BookmarkedRepos {
Results {
Name IsBookmarked
NewestImage { Tag }
}
}
}`
userprefsBaseURL := baseURL + constants.FullUserPreferencesPrefix
Convey("Flip starred repo authorized", func(c C) {
clientHTTP := resty.R().SetBasicAuth(simpleUser, simpleUserPassword)
resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userStaredReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct := StarredReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoStarURL(accessibleRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userStaredReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct = StarredReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 1)
So(responseStruct.Results[0].Name, ShouldEqual, accessibleRepo)
// need to update RepoSummary according to user settings
So(responseStruct.Results[0].IsStarred, ShouldEqual, true)
So(responseStruct.Results[0].StarCount, ShouldEqual, 1)
resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoStarURL(accessibleRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userStaredReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct = StarredReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
})
Convey("Flip starred repo unauthenticated user", func(c C) {
clientHTTP := resty.R()
resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userStaredReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct := StarredReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoStarURL(accessibleRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusForbidden)
resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userStaredReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct = StarredReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
})
Convey("Flip starred repo unauthorized", func(c C) {
clientHTTP := resty.R().SetBasicAuth(simpleUser, simpleUserPassword)
resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userStaredReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct := StarredReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoStarURL(forbiddenRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusForbidden)
resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userStaredReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct = StarredReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
})
Convey("Flip starred repo with unauthorized repo and admin user", func(c C) {
clientHTTP := resty.R().SetBasicAuth(adminUser, adminPassword)
resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userStaredReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct := StarredReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoStarURL(forbiddenRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userStaredReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct = StarredReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 1)
So(responseStruct.Results[0].Name, ShouldEqual, forbiddenRepo)
// need to update RepoSummary according to user settings
So(responseStruct.Results[0].IsStarred, ShouldEqual, true)
So(responseStruct.Results[0].StarCount, ShouldEqual, 1)
resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoStarURL(forbiddenRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userStaredReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct = StarredReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
})
Convey("Flip bookmark repo authorized", func(c C) {
clientHTTP := resty.R().SetBasicAuth(simpleUser, simpleUserPassword)
resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userBookmarkedReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct := BookmarkedReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoBookmarkURL(accessibleRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userBookmarkedReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct = BookmarkedReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 1)
So(responseStruct.Results[0].Name, ShouldEqual, accessibleRepo)
// need to update RepoSummary according to user settings
So(responseStruct.Results[0].IsBookmarked, ShouldEqual, true)
resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoBookmarkURL(accessibleRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userBookmarkedReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct = BookmarkedReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
})
Convey("Flip bookmark repo unauthenticated user", func(c C) {
clientHTTP := resty.R()
resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userBookmarkedReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct := BookmarkedReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoBookmarkURL(accessibleRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusForbidden)
resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userBookmarkedReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct = BookmarkedReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
})
Convey("Flip bookmark repo unauthorized", func(c C) {
clientHTTP := resty.R().SetBasicAuth(simpleUser, simpleUserPassword)
resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userBookmarkedReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct := BookmarkedReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoBookmarkURL(forbiddenRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusForbidden)
resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userBookmarkedReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct = BookmarkedReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
})
Convey("Flip bookmarked unauthorized repo and admin user", func(c C) {
clientHTTP := resty.R().SetBasicAuth(adminUser, adminPassword)
resp, err := clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userBookmarkedReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct := BookmarkedReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoBookmarkURL(forbiddenRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userBookmarkedReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct = BookmarkedReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 1)
So(responseStruct.Results[0].Name, ShouldEqual, forbiddenRepo)
// need to update RepoSummary according to user settings
So(responseStruct.Results[0].IsBookmarked, ShouldEqual, true)
resp, err = clientHTTP.Put(userprefsBaseURL + PutRepoBookmarkURL(forbiddenRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
resp, err = clientHTTP.Get(baseURL + constants.FullSearchPrefix +
"?query=" + url.QueryEscape(userBookmarkedReposQuery))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct = BookmarkedReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 0)
})
})
}
func TestChangingRepoState(t *testing.T) {
port := GetFreePort()
baseURL := GetBaseURL(port)
defaultVal := true
simpleUser := "test"
simpleUserPassword := "test123"
forbiddenRepo := "forbidden"
accesibleRepo := "accesible"
credTests := fmt.Sprintf("%s\n\n", getCredString(simpleUser, simpleUserPassword))
htpasswdPath := MakeHtpasswdFileFromString(credTests)
defer os.Remove(htpasswdPath)
conf := config.New()
conf.Storage.RootDirectory = t.TempDir()
conf.HTTP.Port = port
conf.HTTP.Auth = &config.AuthConfig{
HTPasswd: config.AuthHTPasswd{
Path: htpasswdPath,
},
}
conf.HTTP.AccessControl = &config.AccessControlConfig{
Repositories: config.Repositories{
"**": config.PolicyGroup{
Policies: []config.Policy{
{
Users: []string{simpleUser},
Actions: []string{"read"},
},
},
AnonymousPolicy: []string{"read"},
DefaultPolicy: []string{},
},
forbiddenRepo: config.PolicyGroup{
Policies: []config.Policy{
{
Users: []string{},
Actions: []string{},
},
},
DefaultPolicy: []string{},
},
},
}
conf.Extensions = &extconf.ExtensionConfig{
Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}},
}
gqlStarredRepos := `
{
StarredRepos() {
Results {
Name
StarCount
IsBookmarked
IsStarred
}
}
}
`
gqlBookmarkedRepos := `
{
BookmarkedRepos() {
Results {
Name
StarCount
IsBookmarked
IsStarred
}
}
}
`
ctlr := api.NewController(conf)
img, err := GetRandomImage("tag")
if err != nil {
t.FailNow()
}
// ------ Create the test repos
defaultStore := local.NewImageStore(conf.Storage.RootDirectory, false, 0, false, false,
log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)
err = WriteImageToFileSystem(img, accesibleRepo, storage.StoreController{
DefaultStore: defaultStore,
})
if err != nil {
t.FailNow()
}
err = WriteImageToFileSystem(img, forbiddenRepo, storage.StoreController{
DefaultStore: defaultStore,
})
if err != nil {
t.FailNow()
}
ctlrManager := NewControllerManager(ctlr)
ctlrManager.StartAndWait(port)
defer ctlrManager.StopServer()
simpleUserClient := resty.R().SetBasicAuth(simpleUser, simpleUserPassword)
anonynousClient := resty.R()
userprefsBaseURL := baseURL + constants.FullUserPreferencesPrefix
Convey("PutStars", t, func() {
resp, err := simpleUserClient.Put(userprefsBaseURL + PutRepoStarURL(accesibleRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
resp, err = simpleUserClient.Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(gqlStarredRepos))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
responseStruct := StarredReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 1)
So(responseStruct.Results[0].IsStarred, ShouldBeTrue)
So(responseStruct.Results[0].Name, ShouldResemble, accesibleRepo)
resp, err = anonynousClient.Put(userprefsBaseURL + PutRepoStarURL(accesibleRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusForbidden)
})
//
Convey("PutBookmark", t, func() {
resp, err := simpleUserClient.Put(userprefsBaseURL + PutRepoBookmarkURL(accesibleRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
resp, err = simpleUserClient.Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(gqlBookmarkedRepos))
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
responseStruct := BookmarkedReposResponse{}
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(err, ShouldBeNil)
So(len(responseStruct.Results), ShouldEqual, 1)
So(responseStruct.Results[0].IsBookmarked, ShouldBeTrue)
So(responseStruct.Results[0].Name, ShouldResemble, accesibleRepo)
resp, err = anonynousClient.Put(userprefsBaseURL + PutRepoBookmarkURL(accesibleRepo))
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusForbidden)
})
}
func PutRepoStarURL(repo string) string {
return fmt.Sprintf("?repo=%s&action=toggleStar", repo)
}
func PutRepoBookmarkURL(repo string) string {
return fmt.Sprintf("?repo=%s&action=toggleBookmark", repo)
}
func getCredString(username, password string) string {
hash, err := bcrypt.GenerateFromPassword([]byte(password), 10)
if err != nil {
panic(err)
}
usernameAndHash := fmt.Sprintf("%s:%s", username, string(hash))
return usernameAndHash
}

View file

@ -5,7 +5,9 @@ const (
ManifestDataBucket = "ManifestData"
IndexDataBucket = "IndexData"
ArtifactDataBucket = "ArtifactData"
UserMetadataBucket = "UserMeta"
RepoMetadataBucket = "RepoMetadata"
UserDataBucket = "UserData"
VersionBucket = "Version"
StarredReposKey = "StarredReposKey"
BookmarkedReposKey = "BookmarkedReposKey"
)

View file

@ -2,6 +2,7 @@ package common
import (
"encoding/json"
"fmt"
"strings"
"time"
@ -286,3 +287,98 @@ func CheckImageLastUpdated(repoLastUpdated time.Time, isSigned bool, noImageChec
return repoLastUpdated, noImageChecked, isSigned
}
func FilterDataByRepo(foundRepos []repodb.RepoMetadata, manifestMetadataMap map[string]repodb.ManifestMetadata,
indexDataMap map[string]repodb.IndexData,
) (map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, error) {
var (
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
foundindexDataMap = make(map[string]repodb.IndexData)
)
// keep just the manifestMeta we need
for _, repoMeta := range foundRepos {
for _, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
case ispec.MediaTypeImageIndex:
indexData := indexDataMap[descriptor.Digest]
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
fmt.Errorf("repodb: error while getting manifest data for digest %s %w", descriptor.Digest, err)
}
for _, manifestDescriptor := range indexContent.Manifests {
manifestDigest := manifestDescriptor.Digest.String()
foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest]
}
foundindexDataMap[descriptor.Digest] = indexData
default:
}
}
}
return foundManifestMetadataMap, foundindexDataMap, nil
}
func FetchDataForRepos(repoDB repodb.RepoDB, foundRepos []repodb.RepoMetadata,
) (map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, error) {
foundManifestMetadataMap := map[string]repodb.ManifestMetadata{}
foundIndexDataMap := map[string]repodb.IndexData{}
for idx := range foundRepos {
for _, descriptor := range foundRepos[idx].Tags {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestData, err := repoDB.GetManifestData(godigest.Digest(descriptor.Digest))
if err != nil {
return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, err
}
foundManifestMetadataMap[descriptor.Digest] = repodb.ManifestMetadata{
ManifestBlob: manifestData.ManifestBlob,
ConfigBlob: manifestData.ConfigBlob,
}
case ispec.MediaTypeImageIndex:
indexData, err := repoDB.GetIndexData(godigest.Digest(descriptor.Digest))
if err != nil {
return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, err
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return map[string]repodb.ManifestMetadata{},
map[string]repodb.IndexData{},
fmt.Errorf("repodb: error while getting index data for digest %s %w", descriptor.Digest, err)
}
for _, manifestDescriptor := range indexContent.Manifests {
manifestDigest := manifestDescriptor.Digest.String()
manifestData, err := repoDB.GetManifestData(manifestDescriptor.Digest)
if err != nil {
return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, err
}
foundManifestMetadataMap[manifestDigest] = repodb.ManifestMetadata{
ManifestBlob: manifestData.ManifestBlob,
ConfigBlob: manifestData.ConfigBlob,
}
}
foundIndexDataMap[descriptor.Digest] = indexData
}
}
}
return foundManifestMetadataMap, foundIndexDataMap, nil
}

View file

@ -1,15 +1,21 @@
package common_test
import (
"errors"
"testing"
"time"
"github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/pkg/meta/common"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/test/mocks"
)
var ErrTestError = errors.New("test error")
func TestUtils(t *testing.T) {
Convey("GetReferredSubject", t, func() {
_, err := common.GetReferredSubject([]byte("bad json"))
@ -94,4 +100,128 @@ func TestUtils(t *testing.T) {
So(noImageChecked, ShouldEqual, false)
})
})
Convey("SignatureAlreadyExists", t, func() {
res := common.SignatureAlreadyExists(
[]repodb.SignatureInfo{{SignatureManifestDigest: "digest"}},
repodb.SignatureMetadata{SignatureDigest: "digest"},
)
So(res, ShouldEqual, true)
res = common.SignatureAlreadyExists(
[]repodb.SignatureInfo{{SignatureManifestDigest: "digest"}},
repodb.SignatureMetadata{SignatureDigest: "digest2"},
)
So(res, ShouldEqual, false)
})
Convey("FilterDataByRepo", t, func() {
Convey("Errors", func() {
// Unmarshal index data error
_, _, err := common.FilterDataByRepo(
[]repodb.RepoMetadata{{
Tags: map[string]repodb.Descriptor{
"tag": {
Digest: "indexDigest",
MediaType: ispec.MediaTypeImageIndex,
},
},
}},
map[string]repodb.ManifestMetadata{},
map[string]repodb.IndexData{
"indexDigest": {
IndexBlob: []byte("bad blob"),
},
},
)
So(err, ShouldNotBeNil)
})
})
Convey("FetchDataForRepos", t, func() {
Convey("Errors", func() {
// Unmarshal index data error
_, _, err := common.FetchDataForRepos(
mocks.RepoDBMock{
GetIndexDataFn: func(indexDigest digest.Digest) (repodb.IndexData, error) {
return repodb.IndexData{
IndexBlob: []byte("bad blob"),
}, nil
},
},
[]repodb.RepoMetadata{{
Tags: map[string]repodb.Descriptor{
"tag": {
Digest: "indexDigest",
MediaType: ispec.MediaTypeImageIndex,
},
},
}},
)
So(err, ShouldNotBeNil)
})
})
}
func TestFetchDataForRepos(t *testing.T) {
Convey("GetReferredSubject", t, func() {
mockRepoDB := mocks.RepoDBMock{}
Convey("GetManifestData errors", func() {
mockRepoDB.GetManifestDataFn = func(manifestDigest digest.Digest) (repodb.ManifestData, error) {
return repodb.ManifestData{}, ErrTestError
}
_, _, err := common.FetchDataForRepos(mockRepoDB, []repodb.RepoMetadata{
{
Tags: map[string]repodb.Descriptor{
"tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
},
},
})
So(err, ShouldNotBeNil)
})
Convey("GetIndexData errors", func() {
mockRepoDB.GetIndexDataFn = func(indexDigest digest.Digest) (repodb.IndexData, error) {
return repodb.IndexData{}, ErrTestError
}
_, _, err := common.FetchDataForRepos(mockRepoDB, []repodb.RepoMetadata{
{
Tags: map[string]repodb.Descriptor{
"tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageIndex},
},
},
})
So(err, ShouldNotBeNil)
})
Convey("GetIndexData ok, GetManifestData errors", func() {
mockRepoDB.GetIndexDataFn = func(indexDigest digest.Digest) (repodb.IndexData, error) {
return repodb.IndexData{
IndexBlob: []byte(`{
"manifests": [
{"digest": "dig1"}
]
}`),
}, nil
}
mockRepoDB.GetManifestDataFn = func(manifestDigest digest.Digest) (repodb.ManifestData, error) {
return repodb.ManifestData{}, ErrTestError
}
_, _, err := common.FetchDataForRepos(mockRepoDB, []repodb.RepoMetadata{
{
Tags: map[string]repodb.Descriptor{
"tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageIndex},
},
},
})
So(err, ShouldNotBeNil)
})
})
}

View file

@ -10,7 +10,7 @@ import (
type DBDriverParameters struct {
Endpoint, Region, RepoMetaTablename, ManifestDataTablename, IndexDataTablename,
ArtifactDataTablename, VersionTablename string
ArtifactDataTablename, VersionTablename, UserDataTablename string
}
func GetDynamoClient(params DBDriverParameters) (*dynamodb.Client, error) {

View file

@ -12,6 +12,7 @@ import (
"go.etcd.io/bbolt"
zerr "zotregistry.io/zot/errors"
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/bolt"
"zotregistry.io/zot/pkg/meta/common"
@ -58,6 +59,11 @@ func NewBoltDBWrapper(boltDB *bbolt.DB, log log.Logger) (*DBWrapper, error) {
return err
}
_, err = transaction.CreateBucketIfNotExists([]byte(bolt.UserDataBucket))
if err != nil {
return err
}
return nil
})
if err != nil {
@ -695,7 +701,7 @@ func (bdw *DBWrapper) GetMultipleRepoMeta(ctx context.Context, filter func(repoM
if filter(repoMeta) {
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repoMeta,
RepoMetadata: repoMeta,
})
}
}
@ -889,6 +895,8 @@ func (bdw *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
repoBuck = transaction.Bucket([]byte(bolt.RepoMetadataBucket))
indexBuck = transaction.Bucket([]byte(bolt.IndexDataBucket))
manifestBuck = transaction.Bucket([]byte(bolt.ManifestDataBucket))
userBookmarks = getUserBookmarks(ctx, transaction)
userStars = getUserStars(ctx, transaction)
)
cursor := repoBuck.Cursor()
@ -905,147 +913,126 @@ func (bdw *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
return err
}
if rank := common.RankRepoName(searchText, string(repoName)); rank != -1 {
var (
// specific values used for sorting that need to be calculated based on all manifests from the repo
repoDownloads = 0
repoLastUpdated = time.Time{}
noImageChecked = true
osSet = map[string]bool{}
archSet = map[string]bool{}
isSigned = false
)
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
for tag, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
rank := common.RankRepoName(searchText, string(repoName))
if rank == -1 {
continue
}
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest,
manifestMetadataMap, manifestBuck)
if err != nil {
return fmt.Errorf("repodb: error fetching manifest meta for manifest with digest %s %w",
manifestDigest, err)
}
var (
repoDownloads = 0
repoLastUpdated = time.Time{}
osSet = map[string]bool{}
archSet = map[string]bool{}
noImageChecked = true
isSigned = false
)
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return fmt.Errorf("repodb: error collecting filter data for manifest with digest %s %w",
manifestDigest, err)
}
for tag, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
repoDownloads += manifestFilterData.DownloadCount
for _, os := range manifestFilterData.OsList {
osSet[os] = true
}
for _, arch := range manifestFilterData.ArchList {
archSet[arch] = true
}
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, manifestFilterData)
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck)
if err != nil {
return fmt.Errorf("repodb: error fetching index data for index with digest %s %w",
indexDigest, err)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return fmt.Errorf("repodb: error while unmashaling index content for %s:%s %w",
repoName, tag, err)
}
// this also updates manifestMetadataMap
indexFilterData, err := collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, manifestMetadataMap,
manifestBuck)
if err != nil {
return fmt.Errorf("repodb: error collecting filter data for index with digest %s %w",
indexDigest, err)
}
for _, arch := range indexFilterData.ArchList {
archSet[arch] = true
}
for _, os := range indexFilterData.OsList {
osSet[os] = true
}
repoDownloads += indexFilterData.DownloadCount
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, indexFilterData)
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
continue
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest,
manifestMetadataMap, manifestBuck)
if err != nil {
return fmt.Errorf("repodb: error fetching manifest meta for manifest with digest %s %w",
manifestDigest, err)
}
}
repoFilterData := repodb.FilterData{
OsList: common.GetMapKeys(osSet),
ArchList: common.GetMapKeys(archSet),
LastUpdated: repoLastUpdated,
DownloadCount: repoDownloads,
IsSigned: isSigned,
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return fmt.Errorf("repodb: error collecting filter data for manifest with digest %s %w",
manifestDigest, err)
}
repoDownloads += manifestFilterData.DownloadCount
for _, os := range manifestFilterData.OsList {
osSet[os] = true
}
for _, arch := range manifestFilterData.ArchList {
archSet[arch] = true
}
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, manifestFilterData)
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck)
if err != nil {
return fmt.Errorf("repodb: error fetching index data for index with digest %s %w",
indexDigest, err)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return fmt.Errorf("repodb: error while unmashaling index content for %s:%s %w",
repoName, tag, err)
}
// this also updates manifestMetadataMap
indexFilterData, err := collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, manifestMetadataMap,
manifestBuck)
if err != nil {
return fmt.Errorf("repodb: error collecting filter data for index with digest %s %w",
indexDigest, err)
}
for _, arch := range indexFilterData.ArchList {
archSet[arch] = true
}
for _, os := range indexFilterData.OsList {
osSet[os] = true
}
repoDownloads += indexFilterData.DownloadCount
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, indexFilterData)
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
if !common.AcceptedByFilter(filter, repoFilterData) {
continue
}
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repoMeta,
Rank: rank,
Downloads: repoDownloads,
UpdateTime: repoLastUpdated,
})
}
repoFilterData := repodb.FilterData{
OsList: common.GetMapKeys(osSet),
ArchList: common.GetMapKeys(archSet),
LastUpdated: repoLastUpdated,
DownloadCount: repoDownloads,
IsSigned: isSigned,
}
if !common.AcceptedByFilter(filter, repoFilterData) {
continue
}
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMetadata: repoMeta,
Rank: rank,
Downloads: repoDownloads,
UpdateTime: repoLastUpdated,
})
}
foundRepos, pageInfo = pageFinder.Page()
// keep just the manifestMeta and indexData we need
for _, repoMeta := range foundRepos {
for _, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
case ispec.MediaTypeImageIndex:
indexData := indexDataMap[descriptor.Digest]
foundManifestMetadataMap, foundindexDataMap, err = common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return err
}
for _, manifestDescriptor := range indexContent.Manifests {
manifestDigest := manifestDescriptor.Digest.String()
foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest]
}
foundindexDataMap[descriptor.Digest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
}
}
return nil
return err
})
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
@ -1230,12 +1217,14 @@ func (bdw *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
repodb.PageInfo{}, err
}
err = bdw.DB.View(func(tx *bbolt.Tx) error {
err = bdw.DB.View(func(transaction *bbolt.Tx) error {
var (
repoBuck = tx.Bucket([]byte(bolt.RepoMetadataBucket))
indexBuck = tx.Bucket([]byte(bolt.IndexDataBucket))
manifestBuck = tx.Bucket([]byte(bolt.ManifestDataBucket))
cursor = repoBuck.Cursor()
repoBuck = transaction.Bucket([]byte(bolt.RepoMetadataBucket))
indexBuck = transaction.Bucket([]byte(bolt.IndexDataBucket))
manifestBuck = transaction.Bucket([]byte(bolt.ManifestDataBucket))
cursor = repoBuck.Cursor()
userBookmarks = getUserBookmarks(ctx, transaction)
userStars = getUserStars(ctx, transaction)
)
repoName, repoMetaBlob := cursor.First()
@ -1252,6 +1241,9 @@ func (bdw *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
return err
}
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
matchedTags := make(map[string]repodb.Descriptor)
// take all manifestMetas
for tag, descriptor := range repoMeta.Tags {
@ -1329,47 +1321,85 @@ func (bdw *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
repoMeta.Tags = matchedTags
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repoMeta,
RepoMetadata: repoMeta,
})
}
foundRepos, pageInfo = pageFinder.Page()
// keep just the manifestMeta and indexData we need
for _, repoMeta := range foundRepos {
for _, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
case ispec.MediaTypeImageIndex:
indexData := indexDataMap[descriptor.Digest]
foundManifestMetadataMap, foundindexDataMap, err = common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return err
}
for _, manifestDescriptor := range indexContent.Manifests {
manifestDigest := manifestDescriptor.Digest.String()
foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest]
}
foundindexDataMap[descriptor.Digest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
}
}
return nil
return err
})
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func (bdw *DBWrapper) FilterRepos(ctx context.Context,
filter repodb.FilterRepoFunc,
requestedPage repodb.PageInput,
) (
[]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error,
) {
var (
foundRepos = make([]repodb.RepoMetadata, 0)
pageFinder repodb.PageFinder
pageInfo repodb.PageInfo
)
pageFinder, err := repodb.NewBaseRepoPageFinder(
requestedPage.Limit,
requestedPage.Offset,
requestedPage.SortBy,
)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, err
}
err = bdw.DB.View(func(tx *bbolt.Tx) error {
var (
buck = tx.Bucket([]byte(bolt.RepoMetadataBucket))
cursor = buck.Cursor()
userBookmarks = getUserBookmarks(ctx, tx)
userStars = getUserStars(ctx, tx)
)
for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() {
if ok, err := localCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil {
continue
}
repoMeta := repodb.RepoMetadata{}
err := json.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil {
return err
}
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
if filter(repoMeta) {
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
}
}
foundRepos, pageInfo = pageFinder.Page()
return nil
})
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, err
}
foundManifestMetadataMap, foundIndexDataMap, err := common.FetchDataForRepos(bdw, foundRepos)
return foundRepos, foundManifestMetadataMap, foundIndexDataMap, pageInfo, err
}
func (bdw *DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
@ -1397,12 +1427,14 @@ func (bdw *DBWrapper) SearchTags(ctx context.Context, searchText string, filter
fmt.Errorf("repodb: error while parsing search text, invalid format %w", err)
}
err = bdw.DB.View(func(tx *bbolt.Tx) error {
err = bdw.DB.View(func(transaction *bbolt.Tx) error {
var (
repoBuck = tx.Bucket([]byte(bolt.RepoMetadataBucket))
indexBuck = tx.Bucket([]byte(bolt.IndexDataBucket))
manifestBuck = tx.Bucket([]byte(bolt.ManifestDataBucket))
cursor = repoBuck.Cursor()
repoBuck = transaction.Bucket([]byte(bolt.RepoMetadataBucket))
indexBuck = transaction.Bucket([]byte(bolt.IndexDataBucket))
manifestBuck = transaction.Bucket([]byte(bolt.ManifestDataBucket))
cursor = repoBuck.Cursor()
userBookmarks = getUserBookmarks(ctx, transaction)
userStars = getUserStars(ctx, transaction)
)
repoName, repoMetaBlob := cursor.Seek([]byte(searchedRepo))
@ -1419,140 +1451,119 @@ func (bdw *DBWrapper) SearchTags(ctx context.Context, searchText string, filter
return err
}
if string(repoName) == searchedRepo {
matchedTags := make(map[string]repodb.Descriptor)
// take all manifestMetas
for tag, descriptor := range repoMeta.Tags {
if !strings.HasPrefix(tag, searchedTag) {
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
if string(repoName) != searchedRepo {
continue
}
matchedTags := make(map[string]repodb.Descriptor)
for tag, descriptor := range repoMeta.Tags {
if !strings.HasPrefix(tag, searchedTag) {
continue
}
matchedTags[tag] = descriptor
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return fmt.Errorf("repodb: error fetching manifest meta for manifest with digest %s %w",
manifestDigest, err)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return fmt.Errorf("repodb: error collecting filter data for manifest with digest %s %w",
manifestDigest, err)
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
continue
}
matchedTags[tag] = descriptor
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck)
if err != nil {
return fmt.Errorf("repodb: error fetching index data for index with digest %s %w",
indexDigest, err)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return fmt.Errorf("repodb: error collecting filter data for index with digest %s %w",
indexDigest, err)
}
manifestHasBeenMatched := false
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return fmt.Errorf("repodb: error fetching manifest meta for manifest with digest %s %w",
return fmt.Errorf("repodb: error fetching from db manifest meta for manifest with digest %s %w",
manifestDigest, err)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return fmt.Errorf("repodb: error collecting filter data for manifest with digest %s %w",
manifestDigest, err)
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
manifestMetadataMap[manifestDigest] = manifestMeta
continue
if common.AcceptedByFilter(filter, manifestFilterData) {
manifestHasBeenMatched = true
}
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck)
if err != nil {
return fmt.Errorf("repodb: error fetching index data for index with digest %s %w",
indexDigest, err)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return fmt.Errorf("repodb: error collecting filter data for index with digest %s %w",
indexDigest, err)
}
manifestHasBeenMatched := false
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck)
if err != nil {
return fmt.Errorf("repodb: error fetching from db manifest meta for manifest with digest %s %w",
manifestDigest, err)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return fmt.Errorf("repodb: error collecting filter data for manifest with digest %s %w",
manifestDigest, err)
}
manifestMetadataMap[manifestDigest] = manifestMeta
if common.AcceptedByFilter(filter, manifestFilterData) {
manifestHasBeenMatched = true
}
delete(manifestMetadataMap, manifest.Digest.String())
}
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
delete(manifestMetadataMap, manifest.Digest.String())
}
continue
}
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
continue
}
}
if len(matchedTags) == 0 {
indexDataMap[indexDigest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
continue
}
repoMeta.Tags = matchedTags
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repoMeta,
})
}
if len(matchedTags) == 0 {
continue
}
repoMeta.Tags = matchedTags
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
}
foundRepos, pageInfo = pageFinder.Page()
// keep just the manifestMeta and indexData we need
for _, repoMeta := range foundRepos {
for _, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
case ispec.MediaTypeImageIndex:
indexData := indexDataMap[descriptor.Digest]
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return err
}
for _, manifestDescriptor := range indexContent.Manifests {
manifestDigest := manifestDescriptor.Digest.String()
foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest]
}
foundindexDataMap[descriptor.Digest] = indexData
default:
bdw.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
}
}
}
foundManifestMetadataMap, foundindexDataMap, err = common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
return nil
})
@ -1560,6 +1571,248 @@ func (bdw *DBWrapper) SearchTags(ctx context.Context, searchText string, filter
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func (bdw *DBWrapper) ToggleStarRepo(ctx context.Context, repo string) (repodb.ToggleState, error) {
acCtx, err := localCtx.GetAccessControlContext(ctx)
if err != nil {
return repodb.NotChanged, err
}
userid := localCtx.GetUsernameFromContext(acCtx)
if userid == "" {
// empty user is anonymous
return repodb.NotChanged, zerr.ErrUserDataNotAllowed
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repo); !ok || err != nil {
return repodb.NotChanged, zerr.ErrUserDataNotAllowed
}
var res repodb.ToggleState
if err := bdw.DB.Update(func(tx *bbolt.Tx) error { //nolint:varnamelen
userdb := tx.Bucket([]byte(bolt.UserDataBucket))
userBucket, err := userdb.CreateBucketIfNotExists([]byte(userid))
if err != nil {
// this is a serious failure
return zerr.ErrUnableToCreateUserBucket
}
mdata := userBucket.Get([]byte(bolt.StarredReposKey))
unpacked := []string{}
if mdata != nil {
if err = json.Unmarshal(mdata, &unpacked); err != nil {
return zerr.ErrInvalidOldUserStarredRepos
}
}
isRepoStarred := zcommon.Contains(unpacked, repo)
if isRepoStarred {
res = repodb.Removed
unpacked = zcommon.RemoveFrom(unpacked, repo)
} else {
res = repodb.Added
unpacked = append(unpacked, repo)
}
var repacked []byte
if repacked, err = json.Marshal(unpacked); err != nil {
return zerr.ErrCouldNotMarshalStarredRepos
}
err = userBucket.Put([]byte(bolt.StarredReposKey), repacked)
if err != nil {
return zerr.ErrCouldNotPersistData
}
repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket))
repoMetaBlob := repoBuck.Get([]byte(repo))
if repoMetaBlob == nil {
return zerr.ErrRepoMetaNotFound
}
var repoMeta repodb.RepoMetadata
err = json.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil {
return err
}
switch res {
case repodb.Added:
repoMeta.Stars++
case repodb.Removed:
repoMeta.Stars--
}
repoMetaBlob, err = json.Marshal(repoMeta)
if err != nil {
return err
}
err = repoBuck.Put([]byte(repo), repoMetaBlob)
if err != nil {
return err
}
return nil
}); err != nil {
return repodb.NotChanged, err
}
return res, nil
}
func (bdw *DBWrapper) GetStarredRepos(ctx context.Context) ([]string, error) {
starredRepos := make([]string, 0)
acCtx, err := localCtx.GetAccessControlContext(ctx)
if err != nil {
return starredRepos, err
}
userid := localCtx.GetUsernameFromContext(acCtx)
err = bdw.DB.View(func(tx *bbolt.Tx) error { //nolint:dupl
if userid == "" {
return nil
}
userdb := tx.Bucket([]byte(bolt.UserDataBucket))
userBucket := userdb.Bucket([]byte(userid))
if userBucket == nil {
return nil
}
mdata := userBucket.Get([]byte(bolt.StarredReposKey))
if mdata == nil {
return nil
}
if err := json.Unmarshal(mdata, &starredRepos); err != nil {
bdw.Log.Info().Str("user", userid).Err(err).Msg("unmarshal error")
return zerr.ErrInvalidOldUserStarredRepos
}
if starredRepos == nil {
starredRepos = make([]string, 0)
}
return nil
})
return starredRepos, err
}
func (bdw *DBWrapper) ToggleBookmarkRepo(ctx context.Context, repo string) (repodb.ToggleState, error) {
acCtx, err := localCtx.GetAccessControlContext(ctx)
if err != nil {
return repodb.NotChanged, err
}
userid := localCtx.GetUsernameFromContext(acCtx)
if userid == "" {
// empty user is anonymous
return repodb.NotChanged, zerr.ErrUserDataNotAllowed
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repo); !ok || err != nil {
return repodb.NotChanged, zerr.ErrUserDataNotAllowed
}
var res repodb.ToggleState
if err := bdw.DB.Update(func(tx *bbolt.Tx) error { //nolint:dupl
userdb := tx.Bucket([]byte(bolt.UserDataBucket))
userBucket, err := userdb.CreateBucketIfNotExists([]byte(userid))
if err != nil {
// this is a serious failure
return zerr.ErrUnableToCreateUserBucket
}
mdata := userBucket.Get([]byte(bolt.BookmarkedReposKey))
unpacked := []string{}
if mdata != nil {
if err = json.Unmarshal(mdata, &unpacked); err != nil {
return zerr.ErrInvalidOldUserBookmarkedRepos
}
}
isRepoBookmarked := zcommon.Contains(unpacked, repo)
if isRepoBookmarked {
res = repodb.Removed
unpacked = zcommon.RemoveFrom(unpacked, repo)
} else {
res = repodb.Added
unpacked = append(unpacked, repo)
}
var repacked []byte
if repacked, err = json.Marshal(unpacked); err != nil {
return zerr.ErrCouldNotMarshalBookmarkedRepos
}
err = userBucket.Put([]byte(bolt.BookmarkedReposKey), repacked)
if err != nil {
return zerr.ErrUnableToCreateUserBucket
}
return nil
}); err != nil {
return repodb.NotChanged, err
}
return res, nil
}
func (bdw *DBWrapper) GetBookmarkedRepos(ctx context.Context) ([]string, error) {
bookmarkedRepos := []string{}
acCtx, err := localCtx.GetAccessControlContext(ctx)
if err != nil {
return bookmarkedRepos, err
}
userid := localCtx.GetUsernameFromContext(acCtx)
err = bdw.DB.View(func(tx *bbolt.Tx) error { //nolint:dupl
if userid == "" {
return nil
}
userdb := tx.Bucket([]byte(bolt.UserDataBucket))
userBucket := userdb.Bucket([]byte(userid))
if userBucket == nil {
return nil
}
mdata := userBucket.Get([]byte(bolt.BookmarkedReposKey))
if mdata == nil {
return nil
}
if err := json.Unmarshal(mdata, &bookmarkedRepos); err != nil {
bdw.Log.Info().Str("user", userid).Err(err).Msg("unmarshal error")
return zerr.ErrInvalidOldUserBookmarkedRepos
}
if bookmarkedRepos == nil {
bookmarkedRepos = make([]string, 0)
}
return nil
})
return bookmarkedRepos, err
}
func (bdw *DBWrapper) PatchDB() error {
var DBVersion string
@ -1590,3 +1843,69 @@ func (bdw *DBWrapper) PatchDB() error {
return nil
}
func getUserStars(ctx context.Context, transaction *bbolt.Tx) []string {
acCtx, err := localCtx.GetAccessControlContext(ctx)
if err != nil {
return []string{}
}
var (
userid = localCtx.GetUsernameFromContext(acCtx)
starredRepos = []string{}
userdb = transaction.Bucket([]byte(bolt.UserDataBucket))
userBucket = userdb.Bucket([]byte(userid))
)
if userid == "" {
return []string{}
}
if userBucket == nil {
return []string{}
}
mdata := userBucket.Get([]byte(bolt.StarredReposKey))
if mdata == nil {
return []string{}
}
if err := json.Unmarshal(mdata, &starredRepos); err != nil {
return []string{}
}
return starredRepos
}
func getUserBookmarks(ctx context.Context, transaction *bbolt.Tx) []string {
acCtx, err := localCtx.GetAccessControlContext(ctx)
if err != nil {
return []string{}
}
var (
userid = localCtx.GetUsernameFromContext(acCtx)
bookmarkedRepos = []string{}
userdb = transaction.Bucket([]byte(bolt.UserDataBucket))
userBucket = userdb.Bucket([]byte(userid))
)
if userid == "" {
return []string{}
}
if userBucket == nil {
return []string{}
}
mdata := userBucket.Get([]byte(bolt.BookmarkedReposKey))
if mdata == nil {
return []string{}
}
if err := json.Unmarshal(mdata, &bookmarkedRepos); err != nil {
return []string{}
}
return bookmarkedRepos
}

View file

@ -3,7 +3,6 @@ package bolt_test
import (
"context"
"encoding/json"
"os"
"testing"
"github.com/opencontainers/go-digest"
@ -15,6 +14,7 @@ import (
"zotregistry.io/zot/pkg/meta/bolt"
"zotregistry.io/zot/pkg/meta/repodb"
boltdb_wrapper "zotregistry.io/zot/pkg/meta/repodb/boltdb-wrapper"
localCtx "zotregistry.io/zot/pkg/requestcontext"
"zotregistry.io/zot/pkg/test"
)
@ -29,7 +29,6 @@ func TestWrapperErrors(t *testing.T) {
log := log.NewLogger("debug", "")
boltdbWrapper, err := boltdb_wrapper.NewBoltDBWrapper(boltDriver, log)
defer os.Remove("repo.db")
So(boltdbWrapper, ShouldNotBeNil)
So(err, ShouldBeNil)
@ -77,6 +76,21 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("FilterRepos", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(bolt.RepoMetadataBucket))
err := buck.Put([]byte("badRepo"), []byte("bad repo"))
So(err, ShouldBeNil)
return nil
})
So(err, ShouldBeNil)
_, _, _, _, err = boltdbWrapper.FilterRepos(context.Background(),
func(repoMeta repodb.RepoMetadata) bool { return true }, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("GetArtifactData", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
artifactBuck := tx.Bucket([]byte(bolt.ArtifactDataBucket))
@ -705,6 +719,184 @@ func TestWrapperErrors(t *testing.T) {
})
})
Convey("ToggleStarRepo bad context errors", func() {
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, "bad context")
_, err := boltdbWrapper.ToggleStarRepo(ctx, "repo")
So(err, ShouldNotBeNil)
})
Convey("ToggleStarRepo, getting StarredRepoKey from bucket fails", func() {
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": true,
},
Username: "username",
}
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
userdb, err := tx.CreateBucketIfNotExists([]byte(bolt.UserDataBucket))
So(err, ShouldBeNil)
userBucket, err := userdb.CreateBucketIfNotExists([]byte(acCtx.Username))
So(err, ShouldBeNil)
err = userBucket.Put([]byte(bolt.StarredReposKey), []byte("bad array"))
So(err, ShouldBeNil)
return nil
})
So(err, ShouldBeNil)
_, err = boltdbWrapper.ToggleStarRepo(ctx, "repo")
So(err, ShouldNotBeNil)
})
Convey("ToggleBookmarkRepo, unmarshal error", func() {
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": true,
},
Username: "username",
}
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
userdb, err := tx.CreateBucketIfNotExists([]byte(bolt.UserDataBucket))
So(err, ShouldBeNil)
userBucket, err := userdb.CreateBucketIfNotExists([]byte(acCtx.Username))
So(err, ShouldBeNil)
err = userBucket.Put([]byte(bolt.BookmarkedReposKey), []byte("bad array"))
So(err, ShouldBeNil)
return nil
})
So(err, ShouldBeNil)
_, err = boltdbWrapper.ToggleBookmarkRepo(ctx, "repo")
So(err, ShouldNotBeNil)
})
Convey("ToggleStarRepo, no repoMeta found", func() {
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": true,
},
Username: "username",
}
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(bolt.RepoMetadataBucket))
err := repoBuck.Put([]byte("repo"), []byte("bad repo"))
So(err, ShouldBeNil)
return nil
})
So(err, ShouldBeNil)
_, err = boltdbWrapper.ToggleStarRepo(ctx, "repo")
So(err, ShouldNotBeNil)
})
Convey("ToggleStarRepo, bad repoMeta found", func() {
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": true,
},
Username: "username",
}
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
_, err = boltdbWrapper.ToggleStarRepo(ctx, "repo")
So(err, ShouldNotBeNil)
})
Convey("ToggleBookmarkRepo bad context errors", func() {
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, "bad context")
_, err := boltdbWrapper.ToggleBookmarkRepo(ctx, "repo")
So(err, ShouldNotBeNil)
})
Convey("GetStarredRepos bad context errors", func() {
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, "bad context")
_, err := boltdbWrapper.GetStarredRepos(ctx)
So(err, ShouldNotBeNil)
})
Convey("GetStarredRepos user data unmarshal error", func() {
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": true,
},
Username: "username",
}
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
userdb, err := tx.CreateBucketIfNotExists([]byte(bolt.UserDataBucket))
So(err, ShouldBeNil)
userBucket, err := userdb.CreateBucketIfNotExists([]byte(acCtx.Username))
So(err, ShouldBeNil)
err = userBucket.Put([]byte(bolt.StarredReposKey), []byte("bad array"))
So(err, ShouldBeNil)
return nil
})
So(err, ShouldBeNil)
_, err = boltdbWrapper.GetStarredRepos(ctx)
So(err, ShouldNotBeNil)
})
Convey("GetBookmarkedRepos user data unmarshal error", func() {
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": true,
},
Username: "username",
}
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
userdb, err := tx.CreateBucketIfNotExists([]byte(bolt.UserDataBucket))
So(err, ShouldBeNil)
userBucket, err := userdb.CreateBucketIfNotExists([]byte(acCtx.Username))
So(err, ShouldBeNil)
err = userBucket.Put([]byte(bolt.BookmarkedReposKey), []byte("bad array"))
So(err, ShouldBeNil)
return nil
})
So(err, ShouldBeNil)
_, err = boltdbWrapper.GetBookmarkedRepos(ctx)
So(err, ShouldNotBeNil)
})
Convey("GetBookmarkedRepos bad context errors", func() {
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, "bad context")
_, err := boltdbWrapper.GetBookmarkedRepos(ctx)
So(err, ShouldNotBeNil)
})
Convey("Unsuported type", func() {
digest := digest.FromString("digest")

View file

@ -8,7 +8,7 @@ import (
// that's not directly available in the RepoMetadata structure (ex. that needs to be calculated
// by iterating the manifests, etc.)
type DetailedRepoMeta struct {
RepoMeta RepoMetadata
RepoMetadata
Rank int
Downloads int
UpdateTime time.Time
@ -26,13 +26,13 @@ func SortFunctions() map[SortCriteria]func(pageBuffer []DetailedRepoMeta) func(i
func SortByAlphabeticAsc(pageBuffer []DetailedRepoMeta) func(i, j int) bool {
return func(i, j int) bool {
return pageBuffer[i].RepoMeta.Name < pageBuffer[j].RepoMeta.Name
return pageBuffer[i].Name < pageBuffer[j].Name
}
}
func SortByAlphabeticDsc(pageBuffer []DetailedRepoMeta) func(i, j int) bool {
return func(i, j int) bool {
return pageBuffer[i].RepoMeta.Name > pageBuffer[j].RepoMeta.Name
return pageBuffer[i].Name > pageBuffer[j].Name
}
}

View file

@ -31,6 +31,7 @@ func TestWrapperErrors(t *testing.T) {
manifestDataTablename := "ManifestDataTable" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
artifactDataTablename := "ArtifactDataTable" + uuid.String()
userDataTablename := "UserDataTable" + uuid.String()
versionTablename := "Version" + uuid.String()
@ -58,6 +59,7 @@ func TestWrapperErrors(t *testing.T) {
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
VersionTablename: versionTablename,
UserDataTablename: userDataTablename,
Patches: version.GetDynamoDBPatches(),
Log: log.Logger{Logger: zerolog.New(os.Stdout)},
}
@ -98,6 +100,7 @@ func TestWrapperErrors(t *testing.T) {
VersionTablename: versionTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
UserDataTablename: userDataTablename,
Patches: version.GetDynamoDBPatches(),
Log: log.Logger{Logger: zerolog.New(os.Stdout)},
}

View file

@ -21,6 +21,7 @@ import (
"zotregistry.io/zot/pkg/meta/dynamo"
"zotregistry.io/zot/pkg/meta/repodb"
dynamoWrapper "zotregistry.io/zot/pkg/meta/repodb/dynamodb-wrapper"
localCtx "zotregistry.io/zot/pkg/requestcontext"
"zotregistry.io/zot/pkg/test"
)
@ -42,6 +43,7 @@ func TestIterator(t *testing.T) {
versionTablename := "Version" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
artifactDataTablename := "ArtifactDataTable" + uuid.String()
userDataTablename := "UserDataTable" + uuid.String()
log := log.NewLogger("debug", "")
@ -54,6 +56,7 @@ func TestIterator(t *testing.T) {
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
VersionTablename: versionTablename,
UserDataTablename: userDataTablename,
}
client, err := dynamo.GetDynamoClient(params)
So(err, ShouldBeNil)
@ -143,6 +146,7 @@ func TestWrapperErrors(t *testing.T) {
versionTablename := "Version" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
artifactDataTablename := "ArtifactData" + uuid.String()
userDataTablename := "UserDataTable" + uuid.String()
ctx := context.Background()
@ -156,6 +160,7 @@ func TestWrapperErrors(t *testing.T) {
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
UserDataTablename: userDataTablename,
VersionTablename: versionTablename,
}
client, err := dynamo.GetDynamoClient(params) //nolint:contextcheck
@ -167,6 +172,179 @@ func TestWrapperErrors(t *testing.T) {
So(dynamoWrapper.ResetManifestDataTable(), ShouldBeNil) //nolint:contextcheck
So(dynamoWrapper.ResetRepoMetaTable(), ShouldBeNil) //nolint:contextcheck
Convey("ToggleBookmarkRepo no access", func() {
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": false,
},
Username: "username",
}
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
_, err := dynamoWrapper.ToggleBookmarkRepo(ctx, "unaccesible")
So(err, ShouldNotBeNil)
})
Convey("ToggleBookmarkRepo GetUserMeta no user data", func() {
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": true,
},
Username: "username",
}
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
status, err := dynamoWrapper.ToggleBookmarkRepo(ctx, "repo")
So(err, ShouldBeNil)
So(status, ShouldEqual, repodb.NotChanged)
})
Convey("ToggleBookmarkRepo GetUserMeta client error", func() {
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": true,
},
Username: "username",
}
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
dynamoWrapper.UserDataTablename = badTablename
status, err := dynamoWrapper.ToggleBookmarkRepo(ctx, "repo")
So(err, ShouldNotBeNil)
So(status, ShouldEqual, repodb.NotChanged)
})
Convey("GetBookmarkedRepos", func() {
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": true,
},
Username: "username",
}
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
repos, err := dynamoWrapper.GetBookmarkedRepos(ctx)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
})
Convey("ToggleStarRepo GetUserMeta bad context", func() {
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, "some bad context")
_, err := dynamoWrapper.ToggleStarRepo(ctx, "repo")
So(err, ShouldNotBeNil)
})
Convey("ToggleStarRepo GetUserMeta no access", func() {
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": false,
},
Username: "username",
}
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
_, err := dynamoWrapper.ToggleStarRepo(ctx, "unaccesible")
So(err, ShouldNotBeNil)
})
Convey("ToggleStarRepo GetUserMeta error", func() {
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": false,
},
Username: "username",
}
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
dynamoWrapper.UserDataTablename = badTablename
_, err := dynamoWrapper.ToggleStarRepo(ctx, "repo")
So(err, ShouldNotBeNil)
})
Convey("ToggleStarRepo GetRepoMeta error", func() {
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": true,
},
Username: "username",
}
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
dynamoWrapper.RepoMetaTablename = badTablename
_, err := dynamoWrapper.ToggleStarRepo(ctx, "repo")
So(err, ShouldNotBeNil)
})
Convey("GetUserMeta bad context", func() {
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, "bad context")
userData, err := dynamoWrapper.GetUserMeta(ctx)
So(err, ShouldNotBeNil)
So(userData.BookmarkedRepos, ShouldBeEmpty)
So(userData.StarredRepos, ShouldBeEmpty)
})
Convey("GetUserMeta client error", func() {
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": true,
},
Username: "username",
}
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
dynamoWrapper.UserDataTablename = badTablename
_, err := dynamoWrapper.GetUserMeta(ctx)
So(err, ShouldNotBeNil)
})
Convey("GetUserMeta unmarshal error, bad user data", func() {
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
"repo": true,
},
Username: "username",
}
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
err := setBadUserData(dynamoWrapper.Client, userDataTablename, acCtx.Username)
So(err, ShouldBeNil)
_, err = dynamoWrapper.GetUserMeta(ctx)
So(err, ShouldNotBeNil)
})
Convey("SetUserMeta bad context", func() {
authzCtxKey := localCtx.GetContextKey()
ctx := context.WithValue(context.Background(), authzCtxKey, "bad context")
err := dynamoWrapper.SetUserMeta(ctx, repodb.UserData{})
So(err, ShouldNotBeNil)
})
Convey("SetManifestData", func() {
dynamoWrapper.ManifestDataTablename = "WRONG tables"
@ -253,6 +431,13 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("SetRepoReference client error", func() {
dynamoWrapper.RepoMetaTablename = badTablename
digest := digest.FromString("str")
err := dynamoWrapper.SetRepoReference("repo", digest.String(), digest, ispec.MediaTypeImageManifest)
So(err, ShouldNotBeNil)
})
Convey("SetReferrer client error", func() {
dynamoWrapper.RepoMetaTablename = badTablename
err := dynamoWrapper.SetReferrer("repo", "", repodb.ReferrerInfo{})
@ -683,6 +868,19 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("FilterRepos NewBaseRepoPageFinder errors", func() {
_, _, _, _, err := dynamoWrapper.SearchRepos(ctx, "text", repodb.Filter{}, repodb.PageInput{Offset: -2, Limit: -2})
So(err, ShouldNotBeNil)
})
Convey("FilterRepos attributevalue.Unmarshal(repoMetaAttribute) errors", func() {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
_, _, _, _, err := dynamoWrapper.SearchRepos(ctx, "repo", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldNotBeNil)
})
Convey("FilterTags repoMeta unmarshal error", func() {
err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck
So(err, ShouldBeNil)
@ -804,6 +1002,41 @@ func TestWrapperErrors(t *testing.T) {
)
So(err, ShouldBeNil)
})
Convey("PatchDB dwr.getDBVersion errors", func() {
dynamoWrapper.VersionTablename = badTablename
err := dynamoWrapper.PatchDB()
So(err, ShouldNotBeNil)
})
Convey("PatchDB patchIndex < version.GetVersionIndex", func() {
err := setVersion(dynamoWrapper.Client, versionTablename, "V2")
So(err, ShouldBeNil)
dynamoWrapper.Patches = []func(client *dynamodb.Client, tableNames map[string]string) error{
func(client *dynamodb.Client, tableNames map[string]string) error { return nil },
func(client *dynamodb.Client, tableNames map[string]string) error { return nil },
func(client *dynamodb.Client, tableNames map[string]string) error { return nil },
}
err = dynamoWrapper.PatchDB()
So(err, ShouldBeNil)
})
Convey("ResetRepoMetaTable client errors", func() {
dynamoWrapper.RepoMetaTablename = badTablename
err := dynamoWrapper.ResetRepoMetaTable()
So(err, ShouldNotBeNil)
})
Convey("getDBVersion client errors", func() {
dynamoWrapper.VersionTablename = badTablename
err := dynamoWrapper.PatchDB()
So(err, ShouldNotBeNil)
})
})
Convey("NewDynamoDBWrapper errors", t, func() {
@ -814,6 +1047,7 @@ func TestWrapperErrors(t *testing.T) {
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
UserDataTablename: userDataTablename,
VersionTablename: versionTablename,
}
client, err := dynamo.GetDynamoClient(params)
@ -829,6 +1063,7 @@ func TestWrapperErrors(t *testing.T) {
ManifestDataTablename: "",
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
UserDataTablename: userDataTablename,
VersionTablename: versionTablename,
}
client, err = dynamo.GetDynamoClient(params)
@ -844,6 +1079,7 @@ func TestWrapperErrors(t *testing.T) {
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: "",
ArtifactDataTablename: artifactDataTablename,
UserDataTablename: userDataTablename,
VersionTablename: versionTablename,
}
client, err = dynamo.GetDynamoClient(params)
@ -859,6 +1095,7 @@ func TestWrapperErrors(t *testing.T) {
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
UserDataTablename: userDataTablename,
VersionTablename: "",
}
client, err = dynamo.GetDynamoClient(params)
@ -874,6 +1111,7 @@ func TestWrapperErrors(t *testing.T) {
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: "",
UserDataTablename: userDataTablename,
VersionTablename: versionTablename,
}
client, err = dynamo.GetDynamoClient(params)
@ -889,6 +1127,7 @@ func TestWrapperErrors(t *testing.T) {
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
UserDataTablename: userDataTablename,
ArtifactDataTablename: artifactDataTablename,
}
client, err = dynamo.GetDynamoClient(params)
@ -896,6 +1135,22 @@ func TestWrapperErrors(t *testing.T) {
_, err = dynamoWrapper.NewDynamoDBWrapper(client, params, log)
So(err, ShouldBeNil)
params = dynamo.DBDriverParameters{ //nolint:contextcheck
Endpoint: endpoint,
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
UserDataTablename: "",
ArtifactDataTablename: artifactDataTablename,
}
client, err = dynamo.GetDynamoClient(params)
So(err, ShouldBeNil)
_, err = dynamoWrapper.NewDynamoDBWrapper(client, params, log)
So(err, ShouldNotBeNil)
})
}
@ -999,6 +1254,56 @@ func setBadRepoMeta(client *dynamodb.Client, repoMetadataTableName, repoName str
return err
}
func setBadUserData(client *dynamodb.Client, userDataTablename, userID string) error {
userAttributeValue, err := attributevalue.Marshal("string")
if err != nil {
return err
}
_, err = client.UpdateItem(context.Background(), &dynamodb.UpdateItemInput{
ExpressionAttributeNames: map[string]string{
"#UM": "UserData",
},
ExpressionAttributeValues: map[string]types.AttributeValue{
":UserData": userAttributeValue,
},
Key: map[string]types.AttributeValue{
"UserID": &types.AttributeValueMemberS{
Value: userID,
},
},
TableName: aws.String(userDataTablename),
UpdateExpression: aws.String("SET #UM = :UserData"),
})
return err
}
func setVersion(client *dynamodb.Client, versionTablename string, version string) error {
mdAttributeValue, err := attributevalue.Marshal(version)
if err != nil {
return err
}
_, err = client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
ExpressionAttributeNames: map[string]string{
"#V": "Version",
},
ExpressionAttributeValues: map[string]types.AttributeValue{
":Version": mdAttributeValue,
},
Key: map[string]types.AttributeValue{
"VersionKey": &types.AttributeValueMemberS{
Value: "DBVersion",
},
},
TableName: aws.String(versionTablename),
UpdateExpression: aws.String("SET #V = :Version"),
})
return err
}
func setRepoMeta(client *dynamodb.Client, repoMetadataTableName string, repoMeta repodb.RepoMetadata) error {
repoAttributeValue, err := attributevalue.Marshal(repoMeta)
if err != nil {

View file

@ -16,6 +16,7 @@ import (
ispec "github.com/opencontainers/image-spec/specs-go/v1"
zerr "zotregistry.io/zot/errors"
zcommon "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/common"
"zotregistry.io/zot/pkg/meta/dynamo"
@ -32,6 +33,7 @@ type DBWrapper struct {
IndexDataTablename string
ManifestDataTablename string
ArtifactDataTablename string
UserDataTablename string
VersionTablename string
Patches []func(client *dynamodb.Client, tableNames map[string]string) error
Log log.Logger
@ -45,6 +47,7 @@ func NewDynamoDBWrapper(client *dynamodb.Client, params dynamo.DBDriverParameter
IndexDataTablename: params.IndexDataTablename,
ArtifactDataTablename: params.ArtifactDataTablename,
VersionTablename: params.VersionTablename,
UserDataTablename: params.UserDataTablename,
Patches: version.GetDynamoDBPatches(),
Log: log,
}
@ -74,6 +77,11 @@ func NewDynamoDBWrapper(client *dynamodb.Client, params dynamo.DBDriverParameter
return nil, err
}
err = dynamoWrapper.createUserDataTable()
if err != nil {
return nil, err
}
// Using the Config value, create the DynamoDB client
return &dynamoWrapper, nil
}
@ -751,7 +759,7 @@ func (dwr *DBWrapper) GetMultipleRepoMeta(ctx context.Context,
if filter(repoMeta) {
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repoMeta,
RepoMetadata: repoMeta,
})
}
}
@ -770,6 +778,9 @@ func (dwr *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
repoMetaAttributeIterator dynamo.AttributesIterator
pageFinder repodb.PageFinder
pageInfo repodb.PageInfo
userBookmarks = getUserBookmarks(ctx, dwr)
userStars = getUserStars(ctx, dwr)
)
repoMetaAttributeIterator = dynamo.NewBaseDynamoAttributesIterator(
@ -786,7 +797,6 @@ func (dwr *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
// log
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
@ -803,119 +813,143 @@ func (dwr *DBWrapper) SearchRepos(ctx context.Context, searchText string, filter
continue
}
if rank := common.RankRepoName(searchText, repoMeta.Name); rank != -1 {
var (
// specific values used for sorting that need to be calculated based on all manifests from the repo
repoDownloads = 0
repoLastUpdated = time.Time{}
noImageChecked = true
osSet = map[string]bool{}
archSet = map[string]bool{}
isSigned = false
)
rank := common.RankRepoName(searchText, repoMeta.Name)
if rank == -1 {
continue
}
for _, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
var (
repoDownloads = 0
repoLastUpdated = time.Time{}
osSet = map[string]bool{}
archSet = map[string]bool{}
noImageChecked = true
isSigned = false
)
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
for _, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
repoDownloads += manifestFilterData.DownloadCount
for _, os := range manifestFilterData.OsList {
osSet[os] = true
}
for _, arch := range manifestFilterData.ArchList {
archSet[arch] = true
}
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, manifestFilterData)
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
// this also updates manifestMetadataMap
indexFilterData, err := dwr.collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
for _, arch := range indexFilterData.ArchList {
archSet[arch] = true
}
for _, os := range indexFilterData.OsList {
osSet[os] = true
}
repoDownloads += indexFilterData.DownloadCount
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, indexFilterData)
indexDataMap[indexDigest] = indexData
default:
dwr.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
continue
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
}
repoFilterData := repodb.FilterData{
OsList: common.GetMapKeys(osSet),
ArchList: common.GetMapKeys(archSet),
LastUpdated: repoLastUpdated,
DownloadCount: repoDownloads,
IsSigned: isSigned,
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
repoDownloads += manifestFilterData.DownloadCount
for _, os := range manifestFilterData.OsList {
osSet[os] = true
}
for _, arch := range manifestFilterData.ArchList {
archSet[arch] = true
}
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, manifestFilterData)
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
// this also updates manifestMetadataMap
indexFilterData, err := dwr.collectImageIndexFilterInfo(indexDigest, repoMeta, indexData, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
for _, arch := range indexFilterData.ArchList {
archSet[arch] = true
}
for _, os := range indexFilterData.OsList {
osSet[os] = true
}
repoDownloads += indexFilterData.DownloadCount
repoLastUpdated, noImageChecked, isSigned = common.CheckImageLastUpdated(repoLastUpdated, isSigned,
noImageChecked, indexFilterData)
indexDataMap[indexDigest] = indexData
default:
dwr.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
if !common.AcceptedByFilter(filter, repoFilterData) {
continue
}
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repoMeta,
Rank: rank,
Downloads: repoDownloads,
UpdateTime: repoLastUpdated,
})
}
repoFilterData := repodb.FilterData{
OsList: common.GetMapKeys(osSet),
ArchList: common.GetMapKeys(archSet),
LastUpdated: repoLastUpdated,
DownloadCount: repoDownloads,
IsSigned: isSigned,
}
if !common.AcceptedByFilter(filter, repoFilterData) {
continue
}
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMetadata: repoMeta,
Rank: rank,
Downloads: repoDownloads,
UpdateTime: repoLastUpdated,
})
}
foundRepos, pageInfo := pageFinder.Page()
foundManifestMetadataMap, foundindexDataMap, err := filterFoundData(foundRepos, manifestMetadataMap, indexDataMap)
foundManifestMetadataMap, foundindexDataMap, err := common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func getUserStars(ctx context.Context, dwr *DBWrapper) []string {
starredRepos, err := dwr.GetStarredRepos(ctx)
if err != nil {
return []string{}
}
return starredRepos
}
func getUserBookmarks(ctx context.Context, dwr *DBWrapper) []string {
bookmarkedRepos, err := dwr.GetBookmarkedRepos(ctx)
if err != nil {
return []string{}
}
return bookmarkedRepos
}
func (dwr *DBWrapper) fetchManifestMetaWithCheck(repoName string, manifestDigest string,
manifestMetadataMap map[string]repodb.ManifestMetadata,
) (repodb.ManifestMetadata, error) {
@ -1049,9 +1083,11 @@ func (dwr *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
var (
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
indexDataMap = make(map[string]repodb.IndexData)
pageFinder repodb.PageFinder
repoMetaAttributeIterator dynamo.AttributesIterator
pageFinder repodb.PageFinder
pageInfo repodb.PageInfo
userBookmarks = getUserBookmarks(ctx, dwr)
userStars = getUserStars(ctx, dwr)
)
repoMetaAttributeIterator = dynamo.NewBaseDynamoAttributesIterator(
@ -1068,7 +1104,6 @@ func (dwr *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
// log
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo, err
}
@ -1084,8 +1119,11 @@ func (dwr *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
continue
}
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
matchedTags := make(map[string]repodb.Descriptor)
// take all manifestMetas
for tag, descriptor := range repoMeta.Tags {
matchedTags[tag] = descriptor
@ -1172,29 +1210,89 @@ func (dwr *DBWrapper) FilterTags(ctx context.Context, filter repodb.FilterFunc,
repoMeta.Tags = matchedTags
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repoMeta,
RepoMetadata: repoMeta,
})
}
foundRepos, pageInfo := pageFinder.Page()
foundManifestMetadataMap, foundindexDataMap, err := filterFoundData(foundRepos, manifestMetadataMap, indexDataMap)
foundManifestMetadataMap, foundindexDataMap, err := common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func (dwr *DBWrapper) FilterRepos(ctx context.Context,
filter repodb.FilterRepoFunc,
requestedPage repodb.PageInput,
) (
[]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error,
) {
var (
repoMetaAttributeIterator dynamo.AttributesIterator
pageInfo repodb.PageInfo
userBookmarks = getUserBookmarks(ctx, dwr)
userStars = getUserStars(ctx, dwr)
)
repoMetaAttributeIterator = dynamo.NewBaseDynamoAttributesIterator(
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
)
pageFinder, err := repodb.NewBaseRepoPageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, err
}
repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, err
}
for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) {
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, err
}
var repoMeta repodb.RepoMetadata
err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{}, pageInfo, err
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil {
continue
}
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
if filter(repoMeta) {
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
}
}
foundRepos, pageInfo := pageFinder.Page()
foundManifestMetadataMap, foundIndexDataMap, err := common.FetchDataForRepos(dwr, foundRepos)
return foundRepos, foundManifestMetadataMap, foundIndexDataMap, pageInfo, err
}
func (dwr *DBWrapper) SearchTags(ctx context.Context, searchText string, filter repodb.Filter,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
var (
manifestMetadataMap = make(map[string]repodb.ManifestMetadata)
indexDataMap = make(map[string]repodb.IndexData)
repoMetaAttributeIterator = dynamo.NewBaseDynamoAttributesIterator(
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
)
pageFinder repodb.PageFinder
pageInfo repodb.PageInfo
repoMetaAttributeIterator dynamo.AttributesIterator
pageFinder repodb.PageFinder
pageInfo repodb.PageInfo
userBookmarks = getUserBookmarks(ctx, dwr)
userStars = getUserStars(ctx, dwr)
)
pageFinder, err := repodb.NewBaseImagePageFinder(requestedPage.Limit, requestedPage.Offset, requestedPage.SortBy)
@ -1203,6 +1301,10 @@ func (dwr *DBWrapper) SearchTags(ctx context.Context, searchText string, filter
pageInfo, err
}
repoMetaAttributeIterator = dynamo.NewBaseDynamoAttributesIterator(
dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log,
)
searchedRepo, searchedTag, err := common.GetRepoTag(searchText)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
@ -1231,165 +1333,131 @@ func (dwr *DBWrapper) SearchTags(ctx context.Context, searchText string, filter
continue
}
if repoMeta.Name == searchedRepo {
matchedTags := make(map[string]repodb.Descriptor)
// take all manifestMetas
for tag, descriptor := range repoMeta.Tags {
if !strings.HasPrefix(tag, searchedTag) {
if repoMeta.Name != searchedRepo {
continue
}
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
matchedTags := make(map[string]repodb.Descriptor)
for tag, descriptor := range repoMeta.Tags {
if !strings.HasPrefix(tag, searchedTag) {
continue
}
matchedTags[tag] = descriptor
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("repodb: error while unmashaling manifest metadata for digest %s %w", descriptor.Digest, err)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
continue
}
matchedTags[tag] = descriptor
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("repodb: error while unmashaling index content for digest %s %w", indexDigest, err)
}
manifestHasBeenMatched := false
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("repodb: error while unmashaling manifest metadata for digest %s %w", descriptor.Digest, err)
fmt.Errorf("%w", err)
}
imageFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
if !common.AcceptedByFilter(filter, imageFilterData) {
delete(matchedTags, tag)
manifestMetadataMap[manifestDigest] = manifestMeta
continue
if common.AcceptedByFilter(filter, manifestFilterData) {
manifestHasBeenMatched = true
}
}
manifestMetadataMap[descriptor.Digest] = manifestMeta
case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest
indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
var indexContent ispec.Index
err = json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("repodb: error while unmashaling index content for digest %s %w", indexDigest, err)
}
manifestHasBeenMatched := false
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
manifestDigest := manifest.Digest.String()
manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck
manifestMetadataMap)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
manifestFilterData, err := collectImageManifestFilterData(manifestDigest, repoMeta, manifestMeta)
if err != nil {
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
pageInfo,
fmt.Errorf("%w", err)
}
manifestMetadataMap[manifestDigest] = manifestMeta
if common.AcceptedByFilter(filter, manifestFilterData) {
manifestHasBeenMatched = true
}
delete(manifestMetadataMap, manifest.Digest.String())
}
if !manifestHasBeenMatched {
delete(matchedTags, tag)
for _, manifest := range indexContent.Manifests {
delete(manifestMetadataMap, manifest.Digest.String())
}
continue
}
indexDataMap[indexDigest] = indexData
default:
dwr.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
continue
}
}
if len(matchedTags) == 0 {
indexDataMap[indexDigest] = indexData
default:
dwr.Log.Error().Msgf("Unsupported type: %s", descriptor.MediaType)
continue
}
repoMeta.Tags = matchedTags
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repoMeta,
})
}
if len(matchedTags) == 0 {
continue
}
repoMeta.Tags = matchedTags
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMetadata: repoMeta,
})
}
foundRepos, pageInfo := pageFinder.Page()
foundManifestMetadataMap, foundindexDataMap, err := filterFoundData(foundRepos, manifestMetadataMap, indexDataMap)
foundManifestMetadataMap, foundindexDataMap, err := common.FilterDataByRepo(foundRepos, manifestMetadataMap,
indexDataMap)
return foundRepos, foundManifestMetadataMap, foundindexDataMap, pageInfo, err
}
func filterFoundData(foundRepos []repodb.RepoMetadata, manifestMetadataMap map[string]repodb.ManifestMetadata,
indexDataMap map[string]repodb.IndexData,
) (map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, error) {
var (
foundManifestMetadataMap = make(map[string]repodb.ManifestMetadata)
foundindexDataMap = make(map[string]repodb.IndexData)
)
// keep just the manifestMeta we need
for _, repoMeta := range foundRepos {
for _, descriptor := range repoMeta.Tags {
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest]
case ispec.MediaTypeImageIndex:
indexData := indexDataMap[descriptor.Digest]
var indexContent ispec.Index
err := json.Unmarshal(indexData.IndexBlob, &indexContent)
if err != nil {
return map[string]repodb.ManifestMetadata{}, map[string]repodb.IndexData{},
fmt.Errorf("repodb: error while getting manifest data for digest %s %w", descriptor.Digest, err)
}
for _, manifestDescriptor := range indexContent.Manifests {
manifestDigest := manifestDescriptor.Digest.String()
foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest]
}
foundindexDataMap[descriptor.Digest] = indexData
default:
}
}
}
return foundManifestMetadataMap, foundindexDataMap, nil
}
func (dwr *DBWrapper) PatchDB() error {
DBVersion, err := dwr.getDBVersion()
if err != nil {
@ -1693,3 +1761,266 @@ func (dwr *DBWrapper) ResetManifestDataTable() error {
return dwr.createManifestDataTable()
}
func (dwr *DBWrapper) ToggleBookmarkRepo(ctx context.Context, repo string) (
repodb.ToggleState, error,
) {
res := repodb.NotChanged
if ok, err := localCtx.RepoIsUserAvailable(ctx, repo); !ok || err != nil {
return res, zerr.ErrUserDataNotAllowed
}
userMeta, err := dwr.GetUserMeta(ctx)
if err != nil {
if errors.Is(err, zerr.ErrUserDataNotFound) {
return repodb.NotChanged, nil
}
return res, err
}
if !zcommon.Contains(userMeta.BookmarkedRepos, repo) {
userMeta.BookmarkedRepos = append(userMeta.BookmarkedRepos, repo)
res = repodb.Added
} else {
userMeta.BookmarkedRepos = zcommon.RemoveFrom(userMeta.BookmarkedRepos, repo)
res = repodb.Removed
}
if res != repodb.NotChanged {
err = dwr.SetUserMeta(ctx, userMeta)
}
if err != nil {
res = repodb.NotChanged
return res, err
}
return res, nil
}
func (dwr *DBWrapper) GetBookmarkedRepos(ctx context.Context) ([]string, error) {
userMeta, err := dwr.GetUserMeta(ctx)
if errors.Is(err, zerr.ErrUserDataNotFound) {
return []string{}, nil
}
return userMeta.BookmarkedRepos, err
}
func (dwr *DBWrapper) ToggleStarRepo(ctx context.Context, repo string) (
repodb.ToggleState, error,
) {
res := repodb.NotChanged
acCtx, err := localCtx.GetAccessControlContext(ctx)
if err != nil {
return res, err
}
userid := localCtx.GetUsernameFromContext(acCtx)
if userid == "" {
// empty user is anonymous, it has no data
return res, zerr.ErrUserDataNotAllowed
}
if ok, err := localCtx.RepoIsUserAvailable(ctx, repo); !ok || err != nil {
return res, zerr.ErrUserDataNotAllowed
}
userData, err := dwr.GetUserMeta(ctx)
if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) {
return res, err
}
if !zcommon.Contains(userData.StarredRepos, repo) {
userData.StarredRepos = append(userData.StarredRepos, repo)
res = repodb.Added
} else {
userData.StarredRepos = zcommon.RemoveFrom(userData.StarredRepos, repo)
res = repodb.Removed
}
if res != repodb.NotChanged {
repoMeta, err := dwr.GetRepoMeta(repo) //nolint:contextcheck
if err != nil {
return repodb.NotChanged, err
}
switch res {
case repodb.Added:
repoMeta.Stars++
case repodb.Removed:
repoMeta.Stars--
}
repoAttributeValue, err := attributevalue.Marshal(repoMeta)
if err != nil {
return repodb.NotChanged, err
}
userAttributeValue, err := attributevalue.Marshal(userData)
if err != nil {
return repodb.NotChanged, err
}
_, err = dwr.Client.TransactWriteItems(ctx, &dynamodb.TransactWriteItemsInput{
TransactItems: []types.TransactWriteItem{
{
// Update User Meta
Update: &types.Update{
ExpressionAttributeNames: map[string]string{
"#UM": "UserData",
},
ExpressionAttributeValues: map[string]types.AttributeValue{
":UserData": userAttributeValue,
},
Key: map[string]types.AttributeValue{
"UserID": &types.AttributeValueMemberS{
Value: userid,
},
},
TableName: aws.String(dwr.UserDataTablename),
UpdateExpression: aws.String("SET #UM = :UserData"),
},
},
{
// Update Repo Meta with updated repo stars
Update: &types.Update{
ExpressionAttributeNames: map[string]string{
"#RM": "RepoMetadata",
},
ExpressionAttributeValues: map[string]types.AttributeValue{
":RepoMetadata": repoAttributeValue,
},
Key: map[string]types.AttributeValue{
"RepoName": &types.AttributeValueMemberS{
Value: repo,
},
},
TableName: aws.String(dwr.RepoMetaTablename),
UpdateExpression: aws.String("SET #RM = :RepoMetadata"),
},
},
},
})
if err != nil {
return repodb.NotChanged, err
}
}
return res, nil
}
func (dwr *DBWrapper) GetStarredRepos(ctx context.Context) ([]string, error) {
userMeta, err := dwr.GetUserMeta(ctx)
if errors.Is(err, zerr.ErrUserDataNotFound) {
return []string{}, nil
}
return userMeta.StarredRepos, err
}
func (dwr DBWrapper) GetUserMeta(ctx context.Context) (repodb.UserData, error) {
acCtx, err := localCtx.GetAccessControlContext(ctx)
if err != nil {
return repodb.UserData{}, err
}
userid := localCtx.GetUsernameFromContext(acCtx)
if userid == "" {
// empty user is anonymous, it has no data
return repodb.UserData{}, nil
}
resp, err := dwr.Client.GetItem(ctx, &dynamodb.GetItemInput{
TableName: aws.String(dwr.UserDataTablename),
Key: map[string]types.AttributeValue{
"UserID": &types.AttributeValueMemberS{Value: userid},
},
})
if err != nil {
return repodb.UserData{}, err
}
if resp.Item == nil {
return repodb.UserData{}, zerr.ErrUserDataNotFound
}
var userMeta repodb.UserData
err = attributevalue.Unmarshal(resp.Item["UserData"], &userMeta)
if err != nil {
return repodb.UserData{}, err
}
return userMeta, nil
}
func (dwr DBWrapper) createUserDataTable() error {
_, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{
TableName: aws.String(dwr.UserDataTablename),
AttributeDefinitions: []types.AttributeDefinition{
{
AttributeName: aws.String("UserID"),
AttributeType: types.ScalarAttributeTypeS,
},
},
KeySchema: []types.KeySchemaElement{
{
AttributeName: aws.String("UserID"),
KeyType: types.KeyTypeHash,
},
},
BillingMode: types.BillingModePayPerRequest,
})
if err != nil && !strings.Contains(err.Error(), "Table already exists") {
return err
}
return dwr.waitTableToBeCreated(dwr.UserDataTablename)
}
func (dwr DBWrapper) SetUserMeta(ctx context.Context, userMeta repodb.UserData) error {
acCtx, err := localCtx.GetAccessControlContext(ctx)
if err != nil {
return err
}
userid := localCtx.GetUsernameFromContext(acCtx)
if userid == "" {
// empty user is anonymous, it has no data
return zerr.ErrUserDataNotAllowed
}
userAttributeValue, err := attributevalue.Marshal(userMeta)
if err != nil {
return err
}
_, err = dwr.Client.UpdateItem(ctx, &dynamodb.UpdateItemInput{
ExpressionAttributeNames: map[string]string{
"#UM": "UserData",
},
ExpressionAttributeValues: map[string]types.AttributeValue{
":UserData": userAttributeValue,
},
Key: map[string]types.AttributeValue{
"UserID": &types.AttributeValueMemberS{
Value: userid,
},
},
TableName: aws.String(dwr.UserDataTablename),
UpdateExpression: aws.String("SET #UM = :UserData"),
})
return err
}

View file

@ -93,7 +93,7 @@ func (bpt *RepoPageFinder) Page() ([]RepoMetadata, PageInfo) {
repos := make([]RepoMetadata, 0, len(detailedReposPage))
for _, drm := range detailedReposPage {
repos = append(repos, drm.RepoMeta)
repos = append(repos, drm.RepoMetadata)
}
pageInfo.TotalCount = len(bpt.pageBuffer)
@ -150,7 +150,7 @@ func (bpt *ImagePageFinder) Page() ([]RepoMetadata, PageInfo) {
pageInfo := PageInfo{}
for _, drm := range bpt.pageBuffer {
repo := drm.RepoMeta
repo := drm.RepoMetadata
pageInfo.TotalCount += len(repo.Tags)
}
@ -167,7 +167,7 @@ func (bpt *ImagePageFinder) Page() ([]RepoMetadata, PageInfo) {
if remainingOffset == 0 && remainingLimit == 0 {
for _, drm := range bpt.pageBuffer {
repo := drm.RepoMeta
repo := drm.RepoMetadata
repos = append(repos, repo)
pageInfo.ItemCount += len(repo.Tags)
@ -178,13 +178,13 @@ func (bpt *ImagePageFinder) Page() ([]RepoMetadata, PageInfo) {
// bring cursor to position in RepoMeta array
for _, drm := range bpt.pageBuffer {
if remainingOffset < len(drm.RepoMeta.Tags) {
if remainingOffset < len(drm.Tags) {
tagStartIndex = remainingOffset
break
}
remainingOffset -= len(drm.RepoMeta.Tags)
remainingOffset -= len(drm.Tags)
repoStartIndex++
}
@ -195,7 +195,7 @@ func (bpt *ImagePageFinder) Page() ([]RepoMetadata, PageInfo) {
// finish counting remaining tags inside the first repo meta
partialTags := map[string]Descriptor{}
firstRepoMeta := bpt.pageBuffer[repoStartIndex].RepoMeta
firstRepoMeta := bpt.pageBuffer[repoStartIndex].RepoMetadata
tags := make([]string, 0, len(firstRepoMeta.Tags))
for k := range firstRepoMeta.Tags {
@ -226,7 +226,7 @@ func (bpt *ImagePageFinder) Page() ([]RepoMetadata, PageInfo) {
// continue with the remaining repos
for i := repoStartIndex; i < len(bpt.pageBuffer); i++ {
repoMeta := bpt.pageBuffer[i].RepoMeta
repoMeta := bpt.pageBuffer[i].RepoMetadata
if len(repoMeta.Tags) > remainingLimit {
partialTags := map[string]Descriptor{}

View file

@ -64,7 +64,7 @@ func TestPagination(t *testing.T) {
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repodb.RepoMetadata{
RepoMetadata: repodb.RepoMetadata{
Name: "repo1",
Tags: map[string]repodb.Descriptor{
"tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
@ -73,7 +73,7 @@ func TestPagination(t *testing.T) {
})
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repodb.RepoMetadata{
RepoMetadata: repodb.RepoMetadata{
Name: "repo2",
Tags: map[string]repodb.Descriptor{
"Tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
@ -93,7 +93,7 @@ func TestPagination(t *testing.T) {
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repodb.RepoMetadata{
RepoMetadata: repodb.RepoMetadata{
Name: "repo1",
Tags: map[string]repodb.Descriptor{
"tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
@ -102,7 +102,7 @@ func TestPagination(t *testing.T) {
})
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repodb.RepoMetadata{
RepoMetadata: repodb.RepoMetadata{
Name: "repo2",
Tags: map[string]repodb.Descriptor{
"Tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest},
@ -122,7 +122,7 @@ func TestPagination(t *testing.T) {
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repodb.RepoMetadata{
RepoMetadata: repodb.RepoMetadata{
Name: "repo1",
Tags: map[string]repodb.Descriptor{
"tag1": {
@ -134,7 +134,7 @@ func TestPagination(t *testing.T) {
})
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repodb.RepoMetadata{
RepoMetadata: repodb.RepoMetadata{
Name: "repo2",
Tags: map[string]repodb.Descriptor{
"Tag1": {
@ -157,7 +157,7 @@ func TestPagination(t *testing.T) {
},
})
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repodb.RepoMetadata{
RepoMetadata: repodb.RepoMetadata{
Name: "repo3",
Tags: map[string]repodb.Descriptor{
"Tag11": {
@ -196,7 +196,7 @@ func TestPagination(t *testing.T) {
So(pageFinder, ShouldNotBeNil)
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repodb.RepoMetadata{
RepoMetadata: repodb.RepoMetadata{
Name: "repo1",
Tags: map[string]repodb.Descriptor{
"tag1": {
@ -208,7 +208,7 @@ func TestPagination(t *testing.T) {
})
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repodb.RepoMetadata{
RepoMetadata: repodb.RepoMetadata{
Name: "repo2",
Tags: map[string]repodb.Descriptor{
"Tag1": {
@ -219,7 +219,7 @@ func TestPagination(t *testing.T) {
},
})
pageFinder.Add(repodb.DetailedRepoMeta{
RepoMeta: repodb.RepoMetadata{
RepoMetadata: repodb.RepoMetadata{
Name: "repo3",
Tags: map[string]repodb.Descriptor{
"Tag11": {

View file

@ -14,7 +14,19 @@ const (
CosignType = "cosign"
)
type FilterFunc func(repoMeta RepoMetadata, manifestMeta ManifestMetadata) bool
// Used to model changes to an object after a call to the the DB.
type ToggleState int
const (
NotChanged ToggleState = iota
Added
Removed
)
type (
FilterFunc func(repoMeta RepoMetadata, manifestMeta ManifestMetadata) bool
FilterRepoFunc func(repoMeta RepoMetadata) bool
)
type RepoDB interface { //nolint:interfacebloat
// IncrementRepoStars adds 1 to the star count of an image
@ -95,10 +107,26 @@ type RepoDB interface { //nolint:interfacebloat
SearchTags(ctx context.Context, searchText string, filter Filter, requestedPage PageInput) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, PageInfo, error)
// FilterRepos filters for repos given a filter function
FilterRepos(ctx context.Context, filter FilterRepoFunc, requestedPage PageInput) (
[]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, PageInfo, error)
// FilterTags filters for images given a filter function
FilterTags(ctx context.Context, filter FilterFunc,
requestedPage PageInput) ([]RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, PageInfo, error)
// GetStarredRepos returns starred repos and takes current user in consideration
GetStarredRepos(ctx context.Context) ([]string, error)
// GetBookmarkedRepos returns bookmarked repos and takes current user in consideration
GetBookmarkedRepos(ctx context.Context) ([]string, error)
// ToggleStarRepo adds/removes stars on repos
ToggleStarRepo(ctx context.Context, reponame string) (ToggleState, error)
// ToggleBookmarkRepo adds/removes bookmarks on repos
ToggleBookmarkRepo(ctx context.Context, reponame string) (ToggleState, error)
PatchDB() error
}
@ -150,6 +178,9 @@ type RepoMetadata struct {
Signatures map[string]ManifestSignatures
Referrers map[string][]ReferrerInfo
IsStarred bool
IsBookmarked bool
Stars int
}
@ -171,6 +202,12 @@ type SignatureMetadata struct {
LayersInfo []LayerInfo
}
type UserData struct {
// data for each user.
StarredRepos []string
BookmarkedRepos []string
}
type SortCriteria string
const (

View file

@ -88,6 +88,7 @@ func TestDynamoDBWrapper(t *testing.T) {
versionTablename := "Version" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
artifactDataTablename := "ArtifactDataTable" + uuid.String()
userDataTablename := "UserDataTable" + uuid.String()
Convey("DynamoDB Wrapper", t, func() {
dynamoDBDriverParams := dynamo.DBDriverParameters{
@ -97,6 +98,7 @@ func TestDynamoDBWrapper(t *testing.T) {
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
VersionTablename: versionTablename,
UserDataTablename: userDataTablename,
Region: "us-east-2",
}
@ -115,6 +117,8 @@ func TestDynamoDBWrapper(t *testing.T) {
return err
}
// Note: Tests are very slow if we reset the UserData table every new convey. We'll reset it as needed
err = dynamoDriver.ResetManifestDataTable()
return err
@ -531,6 +535,399 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
So(err, ShouldNotBeNil)
})
Convey("Test repo stars for user", func() {
var (
repo1 = "repo1"
tag1 = "0.0.1"
manifestDigest1 = godigest.FromString("fake-manifest1")
repo2 = "repo2"
)
authzCtxKey := localCtx.GetContextKey()
acCtx1 := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
repo1: true,
repo2: true,
},
Username: "user1",
}
// "user1"
ctx1 := context.WithValue(context.Background(), authzCtxKey, acCtx1)
acCtx2 := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
repo1: true,
repo2: true,
},
Username: "user2",
}
// "user2"
ctx2 := context.WithValue(context.Background(), authzCtxKey, acCtx2)
acCtx3 := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
repo1: true,
repo2: true,
},
Username: "",
}
// anonymous
ctx3 := context.WithValue(context.Background(), authzCtxKey, acCtx3)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
starCount, err := repoDB.GetRepoStars(repo1)
So(err, ShouldBeNil)
So(starCount, ShouldEqual, 0)
starCount, err = repoDB.GetRepoStars(repo2)
So(err, ShouldBeNil)
So(starCount, ShouldEqual, 0)
repos, err := repoDB.GetStarredRepos(ctx1)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
repos, err = repoDB.GetStarredRepos(ctx2)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
repos, err = repoDB.GetStarredRepos(ctx3)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
// User 1 bookmarks repo 1, User 2 has no stars
toggleState, err := repoDB.ToggleStarRepo(ctx1, repo1)
So(err, ShouldBeNil)
So(toggleState, ShouldEqual, repodb.Added)
repoMeta, err := repoDB.GetRepoMeta(repo1)
So(err, ShouldBeNil)
So(repoMeta.Stars, ShouldEqual, 1)
starCount, err = repoDB.GetRepoStars(repo1)
So(err, ShouldBeNil)
So(starCount, ShouldEqual, 1)
repos, err = repoDB.GetStarredRepos(ctx1)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(repos, ShouldContain, repo1)
repos, err = repoDB.GetStarredRepos(ctx2)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
repos, err = repoDB.GetStarredRepos(ctx3)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
// User 1 and User 2 star only repo 1
toggleState, err = repoDB.ToggleStarRepo(ctx2, repo1)
So(err, ShouldBeNil)
So(toggleState, ShouldEqual, repodb.Added)
repoMeta, err = repoDB.GetRepoMeta(repo1)
So(err, ShouldBeNil)
So(repoMeta.Stars, ShouldEqual, 2)
starCount, err = repoDB.GetRepoStars(repo1)
So(err, ShouldBeNil)
So(starCount, ShouldEqual, 2)
repos, err = repoDB.GetStarredRepos(ctx1)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(repos, ShouldContain, repo1)
repos, err = repoDB.GetStarredRepos(ctx2)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(repos, ShouldContain, repo1)
repos, err = repoDB.GetStarredRepos(ctx3)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
// User 1 stars repos 1 and 2, and User 2 stars only repo 1
toggleState, err = repoDB.ToggleStarRepo(ctx1, repo2)
So(err, ShouldBeNil)
So(toggleState, ShouldEqual, repodb.Added)
repoMeta, err = repoDB.GetRepoMeta(repo2)
So(err, ShouldBeNil)
So(repoMeta.Stars, ShouldEqual, 1)
starCount, err = repoDB.GetRepoStars(repo2)
So(err, ShouldBeNil)
So(starCount, ShouldEqual, 1)
repos, err = repoDB.GetStarredRepos(ctx1)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 2)
So(repos, ShouldContain, repo1)
So(repos, ShouldContain, repo2)
repos, err = repoDB.GetStarredRepos(ctx2)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(repos, ShouldContain, repo1)
repos, err = repoDB.GetStarredRepos(ctx3)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
// User 1 stars only repo 2, and User 2 stars only repo 1
toggleState, err = repoDB.ToggleStarRepo(ctx1, repo1)
So(err, ShouldBeNil)
So(toggleState, ShouldEqual, repodb.Removed)
repoMeta, err = repoDB.GetRepoMeta(repo1)
So(err, ShouldBeNil)
So(repoMeta.Stars, ShouldEqual, 1)
starCount, err = repoDB.GetRepoStars(repo1)
So(err, ShouldBeNil)
So(starCount, ShouldEqual, 1)
repos, err = repoDB.GetStarredRepos(ctx1)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(repos, ShouldContain, repo2)
repos, err = repoDB.GetStarredRepos(ctx2)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(repos, ShouldContain, repo1)
repos, err = repoDB.GetStarredRepos(ctx3)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
// User 1 stars both repos 1 and 2, and User 2 removes all stars
toggleState, err = repoDB.ToggleStarRepo(ctx1, repo1)
So(err, ShouldBeNil)
So(toggleState, ShouldEqual, repodb.Added)
toggleState, err = repoDB.ToggleStarRepo(ctx2, repo1)
So(err, ShouldBeNil)
So(toggleState, ShouldEqual, repodb.Removed)
repoMeta, err = repoDB.GetRepoMeta(repo1)
So(err, ShouldBeNil)
So(repoMeta.Stars, ShouldEqual, 1)
repoMeta, err = repoDB.GetRepoMeta(repo2)
So(err, ShouldBeNil)
So(repoMeta.Stars, ShouldEqual, 1)
starCount, err = repoDB.GetRepoStars(repo1)
So(err, ShouldBeNil)
So(starCount, ShouldEqual, 1)
starCount, err = repoDB.GetRepoStars(repo2)
So(err, ShouldBeNil)
So(starCount, ShouldEqual, 1)
repos, err = repoDB.GetStarredRepos(ctx1)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 2)
So(repos, ShouldContain, repo1)
So(repos, ShouldContain, repo2)
repos, err = repoDB.GetStarredRepos(ctx2)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
repos, err = repoDB.GetStarredRepos(ctx3)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
// Anonyous user attempts to toggle a star
toggleState, err = repoDB.ToggleStarRepo(ctx3, repo1)
So(err, ShouldNotBeNil)
So(toggleState, ShouldEqual, repodb.NotChanged)
starCount, err = repoDB.GetRepoStars(repo1)
So(err, ShouldBeNil)
So(starCount, ShouldEqual, 1)
repos, err = repoDB.GetStarredRepos(ctx3)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
// User 1 stars just repo 1
toggleState, err = repoDB.ToggleStarRepo(ctx1, repo2)
So(err, ShouldBeNil)
So(toggleState, ShouldEqual, repodb.Removed)
starCount, err = repoDB.GetRepoStars(repo2)
So(err, ShouldBeNil)
So(starCount, ShouldEqual, 0)
repos, err = repoDB.GetStarredRepos(ctx3)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
})
Convey("Test repo bookmarks for user", func() {
var (
repo1 = "repo1"
tag1 = "0.0.1"
manifestDigest1 = godigest.FromString("fake-manifest1")
repo2 = "repo2"
)
authzCtxKey := localCtx.GetContextKey()
acCtx1 := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
repo1: true,
repo2: true,
},
Username: "user1",
}
// "user1"
ctx1 := context.WithValue(context.Background(), authzCtxKey, acCtx1)
acCtx2 := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
repo1: true,
repo2: true,
},
Username: "user2",
}
// "user2"
ctx2 := context.WithValue(context.Background(), authzCtxKey, acCtx2)
acCtx3 := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
repo1: true,
repo2: true,
},
Username: "",
}
// anonymous
ctx3 := context.WithValue(context.Background(), authzCtxKey, acCtx3)
err := repoDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = repoDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
repos, err := repoDB.GetBookmarkedRepos(ctx1)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
repos, err = repoDB.GetBookmarkedRepos(ctx2)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
// anonymous cannot use bookmarks
repos, err = repoDB.GetBookmarkedRepos(ctx3)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
toggleState, err := repoDB.ToggleBookmarkRepo(ctx3, repo1)
So(err, ShouldNotBeNil)
So(toggleState, ShouldEqual, repodb.NotChanged)
repos, err = repoDB.GetBookmarkedRepos(ctx3)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
// User 1 bookmarks repo 1, User 2 has no bookmarks
toggleState, err = repoDB.ToggleBookmarkRepo(ctx1, repo1)
So(err, ShouldBeNil)
So(toggleState, ShouldEqual, repodb.Added)
repos, err = repoDB.GetBookmarkedRepos(ctx1)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(repos, ShouldContain, repo1)
repos, err = repoDB.GetBookmarkedRepos(ctx2)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
// User 1 and User 2 bookmark only repo 1
toggleState, err = repoDB.ToggleBookmarkRepo(ctx2, repo1)
So(err, ShouldBeNil)
So(toggleState, ShouldEqual, repodb.Added)
repos, err = repoDB.GetBookmarkedRepos(ctx1)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(repos, ShouldContain, repo1)
repos, err = repoDB.GetBookmarkedRepos(ctx2)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(repos, ShouldContain, repo1)
// User 1 bookmarks repos 1 and 2, and User 2 bookmarks only repo 1
toggleState, err = repoDB.ToggleBookmarkRepo(ctx1, repo2)
So(err, ShouldBeNil)
So(toggleState, ShouldEqual, repodb.Added)
repos, err = repoDB.GetBookmarkedRepos(ctx1)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 2)
So(repos, ShouldContain, repo1)
So(repos, ShouldContain, repo2)
repos, err = repoDB.GetBookmarkedRepos(ctx2)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(repos, ShouldContain, repo1)
// User 1 bookmarks only repo 2, and User 2 bookmarks only repo 1
toggleState, err = repoDB.ToggleBookmarkRepo(ctx1, repo1)
So(err, ShouldBeNil)
So(toggleState, ShouldEqual, repodb.Removed)
repos, err = repoDB.GetBookmarkedRepos(ctx1)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(repos, ShouldContain, repo2)
repos, err = repoDB.GetBookmarkedRepos(ctx2)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 1)
So(repos, ShouldContain, repo1)
// User 1 bookmarks both repos 1 and 2, and User 2 removes all bookmarks
toggleState, err = repoDB.ToggleBookmarkRepo(ctx1, repo1)
So(err, ShouldBeNil)
So(toggleState, ShouldEqual, repodb.Added)
toggleState, err = repoDB.ToggleBookmarkRepo(ctx2, repo1)
So(err, ShouldBeNil)
So(toggleState, ShouldEqual, repodb.Removed)
repos, err = repoDB.GetBookmarkedRepos(ctx1)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 2)
So(repos, ShouldContain, repo1)
So(repos, ShouldContain, repo2)
repos, err = repoDB.GetBookmarkedRepos(ctx2)
So(err, ShouldBeNil)
So(len(repos), ShouldEqual, 0)
})
Convey("Test IncrementImageDownloads", func() {
var (
repo1 = "repo1"
@ -1974,9 +2371,171 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
So(referrerInfo[0].ArtifactType, ShouldResemble, "wantedType")
So(referrerInfo[0].Digest, ShouldResemble, "goodArtifact")
})
Convey("FilterRepos", func() {
img, err := test.GetRandomImage("img1")
So(err, ShouldBeNil)
imgDigest, err := img.Digest()
So(err, ShouldBeNil)
manifestData, err := NewManifestData(img.Manifest, img.Config)
So(err, ShouldBeNil)
err = repoDB.SetManifestData(imgDigest, manifestData)
So(err, ShouldBeNil)
multiarch, err := test.GetRandomMultiarchImage("multi")
So(err, ShouldBeNil)
multiarchDigest, err := multiarch.Digest()
So(err, ShouldBeNil)
indexData, err := NewIndexData(multiarch.Index)
So(err, ShouldBeNil)
err = repoDB.SetIndexData(multiarchDigest, indexData)
So(err, ShouldBeNil)
for _, img := range multiarch.Images {
digest, err := img.Digest()
So(err, ShouldBeNil)
indManData1, err := NewManifestData(multiarch.Images[0].Manifest, multiarch.Images[0].Config)
So(err, ShouldBeNil)
err = repoDB.SetManifestData(digest, indManData1)
So(err, ShouldBeNil)
}
err = repoDB.SetRepoReference("repo", img.Reference, imgDigest, img.Manifest.MediaType)
So(err, ShouldBeNil)
err = repoDB.SetRepoReference("repo", multiarch.Reference, multiarchDigest, ispec.MediaTypeImageIndex)
So(err, ShouldBeNil)
repoMetas, _, _, _, err := repoDB.FilterRepos(context.Background(),
func(repoMeta repodb.RepoMetadata) bool { return true }, repodb.PageInput{})
So(err, ShouldBeNil)
So(len(repoMetas), ShouldEqual, 1)
_, _, _, _, err = repoDB.FilterRepos(context.Background(),
func(repoMeta repodb.RepoMetadata) bool { return true }, repodb.PageInput{
Limit: -1,
Offset: -1,
})
So(err, ShouldNotBeNil)
})
Convey("Test bookmarked/starred field present in returned RepoMeta", func() {
repo99 := "repo99"
authzCtxKey := localCtx.GetContextKey()
acCtx := localCtx.AccessControlContext{
ReadGlobPatterns: map[string]bool{
repo99: true,
},
Username: "user1",
}
ctx := context.WithValue(context.Background(), authzCtxKey, acCtx)
manifestDigest := godigest.FromString("dig")
err := repoDB.SetManifestData(manifestDigest, repodb.ManifestData{
ManifestBlob: []byte("{}"),
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
err = repoDB.SetRepoReference(repo99, "tag", manifestDigest, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
repoMetas, _, _, _, err := repoDB.SearchRepos(ctx, repo99, repodb.Filter{}, repodb.PageInput{})
So(err, ShouldBeNil)
So(len(repoMetas), ShouldEqual, 1)
So(repoMetas[0].IsBookmarked, ShouldBeFalse)
So(repoMetas[0].IsStarred, ShouldBeFalse)
repoMetas, _, _, _, err = repoDB.SearchTags(ctx, repo99+":", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldBeNil)
So(len(repoMetas), ShouldEqual, 1)
So(repoMetas[0].IsBookmarked, ShouldBeFalse)
So(repoMetas[0].IsStarred, ShouldBeFalse)
repoMetas, _, _, _, err = repoDB.FilterRepos(ctx, func(repoMeta repodb.RepoMetadata) bool {
return true
}, repodb.PageInput{})
So(err, ShouldBeNil)
So(len(repoMetas), ShouldEqual, 1)
So(repoMetas[0].IsBookmarked, ShouldBeFalse)
So(repoMetas[0].IsStarred, ShouldBeFalse)
repoMetas, _, _, _, err = repoDB.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true },
repodb.PageInput{},
)
So(err, ShouldBeNil)
So(len(repoMetas), ShouldEqual, 1)
So(repoMetas[0].IsBookmarked, ShouldBeFalse)
So(repoMetas[0].IsStarred, ShouldBeFalse)
_, err = repoDB.ToggleBookmarkRepo(ctx, repo99)
So(err, ShouldBeNil)
_, err = repoDB.ToggleStarRepo(ctx, repo99)
So(err, ShouldBeNil)
repoMetas, _, _, _, err = repoDB.SearchRepos(ctx, repo99, repodb.Filter{}, repodb.PageInput{})
So(err, ShouldBeNil)
So(len(repoMetas), ShouldEqual, 1)
So(repoMetas[0].IsBookmarked, ShouldBeTrue)
So(repoMetas[0].IsStarred, ShouldBeTrue)
repoMetas, _, _, _, err = repoDB.SearchTags(ctx, repo99+":", repodb.Filter{}, repodb.PageInput{})
So(err, ShouldBeNil)
So(len(repoMetas), ShouldEqual, 1)
So(repoMetas[0].IsBookmarked, ShouldBeTrue)
So(repoMetas[0].IsStarred, ShouldBeTrue)
repoMetas, _, _, _, err = repoDB.FilterRepos(ctx, func(repoMeta repodb.RepoMetadata) bool {
return true
}, repodb.PageInput{})
So(err, ShouldBeNil)
So(len(repoMetas), ShouldEqual, 1)
So(repoMetas[0].IsBookmarked, ShouldBeTrue)
So(repoMetas[0].IsStarred, ShouldBeTrue)
repoMetas, _, _, _, err = repoDB.FilterTags(ctx,
func(repoMeta repodb.RepoMetadata, manifestMeta repodb.ManifestMetadata) bool { return true },
repodb.PageInput{},
)
So(err, ShouldBeNil)
So(len(repoMetas), ShouldEqual, 1)
So(repoMetas[0].IsBookmarked, ShouldBeTrue)
So(repoMetas[0].IsStarred, ShouldBeTrue)
})
})
}
func NewManifestData(manifest ispec.Manifest, config ispec.Image) (repodb.ManifestData, error) {
configBlob, err := json.Marshal(config)
if err != nil {
return repodb.ManifestData{}, err
}
manifest.Config.Digest = godigest.FromBytes(configBlob)
manifestBlob, err := json.Marshal(manifest)
if err != nil {
return repodb.ManifestData{}, err
}
return repodb.ManifestData{ManifestBlob: manifestBlob, ConfigBlob: configBlob}, nil
}
func NewIndexData(index ispec.Index) (repodb.IndexData, error) {
indexBlob, err := json.Marshal(index)
return repodb.IndexData{IndexBlob: indexBlob}, err
}
func TestRelevanceSorting(t *testing.T) {
Convey("Test Relevance Sorting", t, func() {
So(common.RankRepoName("alpine", "alpine"), ShouldEqual, 0)

View file

@ -95,6 +95,9 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d
versionTablename, ok := toStringIfOk(cacheDriverConfig, "versiontablename", log)
allParametersOk = allParametersOk && ok
userDataTablename, ok := toStringIfOk(cacheDriverConfig, "userdatatablename", log)
allParametersOk = allParametersOk && ok
if !allParametersOk {
panic("dynamo parameters are not specified correctly, can't proceede")
}
@ -106,6 +109,7 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
UserDataTablename: userDataTablename,
VersionTablename: versionTablename,
}
}

View file

@ -23,6 +23,7 @@ func TestCreateDynamo(t *testing.T) {
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
ArtifactDataTablename: "ArtifactDataTable",
UserDataTablename: "UserDataTable",
VersionTablename: "Version",
Region: "us-east-2",
}

View file

@ -293,6 +293,7 @@ func TestParseStorageDynamoWrapper(t *testing.T) {
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
ArtifactDataTablename: "ArtifactDataTable",
UserDataTablename: "UserDataTable",
VersionTablename: "Version",
}

View file

@ -127,6 +127,7 @@ func TestVersioningDynamoDB(t *testing.T) {
ManifestDataTablename: "ManifestDataTable",
ArtifactDataTablename: "ArtifactDataTable",
IndexDataTablename: "IndexDataTable",
UserDataTablename: "UserDataTable",
VersionTablename: "Version",
}

View file

@ -26,3 +26,11 @@ func RepoIsUserAvailable(ctx context.Context, repoName string) (bool, error) {
return true, nil
}
func GetUsernameFromContext(ctx *AccessControlContext) string {
if ctx == nil {
return ""
}
return ctx.Username
}

View file

@ -657,6 +657,7 @@ func GetRandomImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manif
Size: int64(len(layers[0])),
},
},
MediaType: ispec.MediaTypeImageManifest,
}
return config, layers, manifest, nil

View file

@ -67,6 +67,9 @@ type RepoDBMock struct {
SearchTagsFn func(ctx context.Context, searchText string, filter repodb.Filter, requestedPage repodb.PageInput) (
[]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error)
FilterReposFn func(ctx context.Context, filter repodb.FilterRepoFunc, requestedPage repodb.PageInput) (
[]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error)
FilterTagsFn func(ctx context.Context, filter repodb.FilterFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error)
@ -83,6 +86,14 @@ type RepoDBMock struct {
SearchForDescendantImagesFn func(ctx context.Context, searchText string, requestedPage repodb.PageInput) (
[]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, error)
GetStarredReposFn func(ctx context.Context) ([]string, error)
GetBookmarkedReposFn func(ctx context.Context) ([]string, error)
ToggleStarRepoFn func(ctx context.Context, repo string) (repodb.ToggleState, error)
ToggleBookmarkRepoFn func(ctx context.Context, repo string) (repodb.ToggleState, error)
PatchDBFn func() error
}
@ -244,6 +255,17 @@ func (sdm RepoDBMock) SearchTags(ctx context.Context, searchText string, filter
map[string]repodb.IndexData{}, repodb.PageInfo{}, nil
}
func (sdm RepoDBMock) FilterRepos(ctx context.Context, filter repodb.FilterRepoFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
if sdm.FilterReposFn != nil {
return sdm.FilterReposFn(ctx, filter, requestedPage)
}
return []repodb.RepoMetadata{}, map[string]repodb.ManifestMetadata{},
map[string]repodb.IndexData{}, repodb.PageInfo{}, nil
}
func (sdm RepoDBMock) FilterTags(ctx context.Context, filter repodb.FilterFunc,
requestedPage repodb.PageInput,
) ([]repodb.RepoMetadata, map[string]repodb.ManifestMetadata, map[string]repodb.IndexData, repodb.PageInfo, error) {
@ -359,3 +381,35 @@ func (sdm RepoDBMock) GetReferrersInfo(repo string, referredDigest godigest.Dige
return []repodb.ReferrerInfo{}, nil
}
func (sdm RepoDBMock) GetStarredRepos(ctx context.Context) ([]string, error) {
if sdm.GetStarredReposFn != nil {
return sdm.GetStarredReposFn(ctx)
}
return []string{}, nil
}
func (sdm RepoDBMock) GetBookmarkedRepos(ctx context.Context) ([]string, error) {
if sdm.GetBookmarkedReposFn != nil {
return sdm.GetBookmarkedReposFn(ctx)
}
return []string{}, nil
}
func (sdm RepoDBMock) ToggleStarRepo(ctx context.Context, repo string) (repodb.ToggleState, error) {
if sdm.ToggleStarRepoFn != nil {
return sdm.ToggleStarRepoFn(ctx, repo)
}
return repodb.NotChanged, nil
}
func (sdm RepoDBMock) ToggleBookmarkRepo(ctx context.Context, repo string) (repodb.ToggleState, error) {
if sdm.ToggleBookmarkRepoFn != nil {
return sdm.ToggleBookmarkRepoFn(ctx, repo)
}
return repodb.NotChanged, nil
}

View file

@ -39,6 +39,7 @@ function setup() {
"manifestDataTablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"indexDataTablename": "IndexDataTable",
"userDataTablename": "UserDataTable",
"versionTablename": "Version"
}
},

138
test/blackbox/metadata.bats Normal file
View file

@ -0,0 +1,138 @@
load helpers_pushpull
function setup_file() {
# Verify prerequisites are available
if ! verify_prerequisites; then
exit 1
fi
# Download test data to folder common for the entire suite, not just this file
skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.18 oci:${TEST_DATA_DIR}/golang:1.18
# Setup zot server
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json
local oci_data_dir=${BATS_FILE_TMPDIR}/oci
local htpasswordFile=${BATS_FILE_TMPDIR}/htpasswd
mkdir -p ${zot_root_dir}
mkdir -p ${oci_data_dir}
echo 'test:$2a$10$EIIoeCnvsIDAJeDL4T1sEOnL2fWOvsq7ACZbs3RT40BBBXg.Ih7V.' >> ${htpasswordFile}
cat > ${zot_config_file}<<EOF
{
"distSpecVersion": "1.1.0",
"storage": {
"rootDirectory": "${zot_root_dir}"
},
"extensions": {
"search": {
"enable": true
}
},
"http": {
"address": "0.0.0.0",
"port": "8080",
"auth": {
"htpasswd": {
"path": "${htpasswordFile}"
}
},
"accessControl": {
"repositories": {
"**": {
"anonymousPolicy": ["read"],
"policies": [
{
"users": [
"test"
],
"actions": [
"read",
"create",
"update"
]
}
]
}
}
}
},
"log": {
"level": "debug"
}
}
EOF
git -C ${BATS_FILE_TMPDIR} clone https://github.com/project-zot/helm-charts.git
setup_zot_file_level ${zot_config_file}
wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
}
function teardown_file() {
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
local oci_data_dir=${BATS_FILE_TMPDIR}/oci
teardown_zot_file_level
rm -rf ${zot_root_dir}
rm -rf ${oci_data_dir}
}
@test "push image user policy" {
run skopeo --insecure-policy copy --dest-creds test:test --dest-tls-verify=false \
oci:${TEST_DATA_DIR}/golang:1.18 \
docker://127.0.0.1:8080/golang:1.18
[ "$status" -eq 0 ]
}
@test "User metadata starredRepos" {
run skopeo --insecure-policy copy --dest-creds test:test --dest-tls-verify=false \
oci:${TEST_DATA_DIR}/golang:1.18 \
docker://127.0.0.1:8080/golang:1.18
[ "$status" -eq 0 ]
USER_STAR_REPOS_QUERY='{ "query": "{ StarredRepos { Results { Name } } }"}'
run curl --user "test:test" -X POST -H "Content-Type: application/json" --data "${USER_STAR_REPOS_QUERY}" http://localhost:8080/v2/_zot/ext/search
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.data.StarredRepos.Results') = '[]' ]
run curl --user "test:test" -X PUT "http://127.0.0.1:8080/v2/_zot/ext/userprefs?repo=golang&action=toggleStar"
[ "$status" -eq 0 ]
run curl --user "test:test" -X POST -H "Content-Type: application/json" --data "${USER_STAR_REPOS_QUERY}" http://localhost:8080/v2/_zot/ext/search
[ "$status" -eq 0 ]
echo $(echo "${lines[-1]}" | jq '.data.StarredRepos.Results[0].Name')
[ $(echo "${lines[-1]}" | jq -r '.data.StarredRepos.Results[0].Name') = 'golang' ]
run curl --user "test:test" -X PUT "http://127.0.0.1:8080/v2/_zot/ext/userprefs?repo=golang&action=toggleStar"
[ "$status" -eq 0 ]
run curl --user "test:test" -X POST -H "Content-Type: application/json" --data "${USER_STAR_REPOS_QUERY}" http://localhost:8080/v2/_zot/ext/search
[ "$status" -eq 0 ]
echo $(echo "${lines[-1]}" | jq '.data.StarredRepos.Results')
[ $(echo "${lines[-1]}" | jq -r '.data.StarredRepos.Results') = '[]' ]
}
@test "User metadata bookmarkedRepos" {
run skopeo --insecure-policy copy --dest-creds test:test --dest-tls-verify=false \
oci:${TEST_DATA_DIR}/golang:1.18 \
docker://127.0.0.1:8080/golang:1.18
[ "$status" -eq 0 ]
USER_BOOKMARK_REPOS_QUERY='{ "query": "{ BookmarkedRepos { Results { Name } } }"}'
run curl --user "test:test" -X POST -H "Content-Type: application/json" --data "${USER_BOOKMARK_REPOS_QUERY}" http://localhost:8080/v2/_zot/ext/search
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.data.BookmarkedRepos.Results') = '[]' ]
run curl --user "test:test" -X PUT "http://127.0.0.1:8080/v2/_zot/ext/userprefs?repo=golang&action=toggleBookmark"
[ "$status" -eq 0 ]
run curl --user "test:test" -X POST -H "Content-Type: application/json" --data "${USER_BOOKMARK_REPOS_QUERY}" http://localhost:8080/v2/_zot/ext/search
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq -r '.data.BookmarkedRepos.Results[0].Name') = 'golang' ]
run curl --user "test:test" -X PUT "http://127.0.0.1:8080/v2/_zot/ext/userprefs?repo=golang&action=toggleBookmark"
[ "$status" -eq 0 ]
run curl --user "test:test" -X POST -H "Content-Type: application/json" --data "${USER_BOOKMARK_REPOS_QUERY}" http://localhost:8080/v2/_zot/ext/search
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq -r '.data.BookmarkedRepos.Results') = '[]' ]
}

View file

@ -56,7 +56,6 @@ EOF
EOF
setup_zot_file_level ${ZOT_CONFIG_FILE}
echo "yes"
wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
run skopeo --insecure-policy copy --dest-tls-verify=false \