Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00
Code Issues Activity

fix(sync): use pagination when querying remote catalog

Browse source 
feat(api): added /v2/_catalog pagination, fixes #2715

Signed-off-by: Eusebiu Petu <petu.eusebiu@gmail.com>
This commit is contained in:
Eusebiu Petu 2024-11-04 20:22:32 +02:00
parent 5e30fec65c
commit 7afee3d679
16 changed files with 768 additions and 68 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
pkg/api/authz.go
View file

@ -12,6 +12,7 @@ import (
"zotregistry.dev/zot/pkg/common"
"zotregistry.dev/zot/pkg/log"
reqCtx "zotregistry.dev/zot/pkg/requestcontext"
storageTypes "zotregistry.dev/zot/pkg/storage/types"
)
const (
@ -20,6 +21,20 @@ const (
OPENID = "OpenID"
)
func AuthzFilterFunc(userAc *reqCtx.UserAccessControl) storageTypes.FilterRepoFunc {
return func(repo string) (bool, error) {
if userAc == nil {
return true, nil
}
if userAc.Can(constants.ReadPermission, repo) {
return true, nil
}
return false, nil
}
}
// AccessController authorizes users to act on resources.
type AccessController struct {
Config *config.AccessControlConfig

433
pkg/api/controller_test.go
View file

@ -8145,6 +8145,439 @@ func TestRouteFailures(t *testing.T) {
})
}
func TestPagedRepositoriesWithAuthorization(t *testing.T) {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
username, _ := test.GenerateRandomString()
password, _ := test.GenerateRandomString()
htpasswdPath := test.MakeHtpasswdFileFromString(test.GetCredString(username, password))
defer os.Remove(htpasswdPath)
conf.HTTP.Auth = &config.AuthConfig{
HTPasswd: config.AuthHTPasswd{
Path: htpasswdPath,
},
}
readPolicyGroup := config.PolicyGroup{
Policies: []config.Policy{
{
Users: []string{username},
Actions: []string{
constants.ReadPermission,
},
},
},
DefaultPolicy: []string{},
}
conf.HTTP.AccessControl = &config.AccessControlConfig{
Repositories: config.Repositories{
test.AuthorizationAllRepos: config.PolicyGroup{
Policies: []config.Policy{
{
Users: []string{username},
Actions: []string{
constants.ReadPermission,
constants.CreatePermission,
},
},
},
DefaultPolicy: []string{},
},
},
AdminPolicy: config.Policy{
Users: []string{},
Actions: []string{},
},
}
ctlr := api.NewController(conf)
ctlr.Config.Storage.RootDirectory = t.TempDir()
cm := test.NewControllerManager(ctlr)
cm.StartAndWait(port)
defer cm.StopServer()
client := resty.New()
client.SetBasicAuth(username, password)
img := CreateRandomImage()
repoNames := []string{
"alpine1", "alpine2", "alpine3",
"alpine4", "alpine5", "alpine6",
"alpine7", "alpine8", "alpine9",
}
for _, repo := range repoNames {
err := UploadImageWithBasicAuth(img, baseURL, repo, "0.1", username, password)
if err != nil {
panic(err)
}
}
conf.HTTP.AccessControl.Repositories = config.Repositories{
"alpine[13579]": readPolicyGroup,
"alpine[2468]": config.PolicyGroup{},
}
// Note empty strings signify the query parameter is not set
// There are separate tests for passing the empty string as query parameter
testCases := []struct {
testCaseName string
pageSize string
last string
expectedRepos []string
}{
{
testCaseName: "no parameters",
pageSize: "",
last: "",
expectedRepos: []string{"alpine1", "alpine3", "alpine5", "alpine7", "alpine9"},
},
{
testCaseName: "first 3",
pageSize: "3",
last: "",
expectedRepos: []string{"alpine1", "alpine3", "alpine5"},
},
{
testCaseName: "next 2",
pageSize: "3",
last: "alpine5",
expectedRepos: []string{"alpine7", "alpine9"},
},
{
testCaseName: "0",
pageSize: "0",
last: "",
expectedRepos: []string{},
},
{
testCaseName: "Test the parameter 'last' without parameter 'n'",
pageSize: "",
last: "alpine3",
expectedRepos: []string{"alpine5", "alpine7", "alpine9"},
},
{
testCaseName: "Test the parameter 'last' with the final repo as value",
pageSize: "",
last: "alpine9",
expectedRepos: []string{},
},
}
for _, testCase := range testCases {
Convey(testCase.testCaseName, t, func() {
testHTTPPagedRepositories(t, client, baseURL, testCase.testCaseName, testCase.pageSize,
testCase.last, testCase.expectedRepos, repoNames[len(repoNames)-1])
})
}
}
func TestPagedRepositoriesWithSubpaths(t *testing.T) {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
dir := t.TempDir()
firstSubDir := t.TempDir()
secondSubDir := t.TempDir()
subPaths := make(map[string]config.StorageConfig)
subPaths["/a"] = config.StorageConfig{RootDirectory: firstSubDir}
subPaths["/b"] = config.StorageConfig{RootDirectory: secondSubDir}
ctlr := makeController(conf, dir)
ctlr.Config.Storage.SubPaths = subPaths
ctlr.Config.Storage.Commit = true
cm := test.NewControllerManager(ctlr)
cm.StartAndWait(port)
defer cm.StopServer()
rthdlr := api.NewRouteHandler(ctlr)
img := CreateRandomImage()
repoNames := []string{
"alpine1", "alpine2", "alpine3",
"a/alpine4", "a/alpine5", "a/alpine6",
"b/alpine7", "b/alpine8", "b/alpine9",
}
for _, repo := range repoNames {
err := UploadImage(img, baseURL, repo, "0.1")
if err != nil {
panic(err)
}
}
// Note empty strings signify the query parameter is not set
// There are separate tests for passing the empty string as query parameter
testCases := []struct {
testCaseName string
pageSize string
last string
expectedRepos []string
}{
{
testCaseName: "no parameters",
pageSize: "",
last: "",
expectedRepos: repoNames,
},
{
testCaseName: "first 5",
pageSize: "5",
last: "",
expectedRepos: repoNames[:5],
},
{
testCaseName: "next 5",
pageSize: "5",
last: "a/alpine5",
expectedRepos: repoNames[5:9],
},
{
testCaseName: "0",
pageSize: "0",
last: "",
expectedRepos: []string{},
},
{
testCaseName: "Test the parameter 'last' without parameter 'n'",
pageSize: "",
last: "alpine2",
expectedRepos: repoNames[2:9],
},
{
testCaseName: "Test the parameter 'last' with the final repo as value",
pageSize: "",
last: repoNames[len(repoNames)-1],
expectedRepos: []string{},
},
}
for _, testCase := range testCases {
Convey(testCase.testCaseName, t, func() {
testPagedRepositories(t, rthdlr, baseURL, testCase.testCaseName, testCase.pageSize,
testCase.last, testCase.expectedRepos, repoNames[len(repoNames)-1])
})
}
}
func TestPagedRepositories(t *testing.T) {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
ctlr := makeController(conf, t.TempDir())
ctlr.Config.Storage.Commit = true
cm := test.NewControllerManager(ctlr)
cm.StartAndWait(port)
defer cm.StopServer()
rthdlr := api.NewRouteHandler(ctlr)
img := CreateRandomImage()
repoName := "alpine"
repoNames := []string{
"alpine1", "alpine2", "alpine3",
"alpine4", "alpine5", "alpine6",
"alpine7", "alpine8", "alpine9",
}
for _, repo := range repoNames {
err := UploadImage(img, baseURL, repo, "0.1")
if err != nil {
panic(err)
}
}
// Note empty strings signify the query parameter is not set
// There are separate tests for passing the empty string as query parameter
testCases := []struct {
testCaseName string
pageSize string
last string
expectedRepos []string
}{
{
testCaseName: "no parameters",
pageSize: "",
last: "",
expectedRepos: repoNames,
},
{
testCaseName: "first 5",
pageSize: "5",
last: "",
expectedRepos: repoNames[:5],
},
{
testCaseName: "next 5",
pageSize: "5",
last: repoName + "5",
expectedRepos: repoNames[5:9],
},
{
testCaseName: "0",
pageSize: "0",
last: "",
expectedRepos: []string{},
},
{
testCaseName: "Test the parameter 'last' without parameter 'n'",
pageSize: "",
last: repoName + "2",
expectedRepos: repoNames[2:9],
},
{
testCaseName: "Test the parameter 'last' with the final repo as value",
pageSize: "",
last: repoName + "9",
expectedRepos: []string{},
},
}
for _, testCase := range testCases {
Convey(testCase.testCaseName, t, func() {
testPagedRepositories(t, rthdlr, baseURL, testCase.testCaseName, testCase.pageSize,
testCase.last, testCase.expectedRepos, repoNames[len(repoNames)-1])
})
}
}
func testHTTPPagedRepositories(t *testing.T, client *resty.Client, baseURL string, testCaseName string,
pageSize string,
last string,
expectedRepos []string, lastRepoInStorage string,
) {
t.Helper()
Convey(testCaseName, func() {
t.Log("Running " + testCaseName)
params := make(map[string]string)
if pageSize != "" || last != "" {
if pageSize != "" {
params["n"] = pageSize
}
if last != "" {
params["last"] = last
}
}
resp, err := client.R().SetQueryParams(params).Get(baseURL + "/v2/_catalog")
So(err, ShouldBeNil)
So(resp, ShouldNotBeNil)
So(resp.StatusCode(), ShouldEqual, http.StatusOK)
catalog := struct {
Repositories []string `json:"repositories"`
}{}
err = json.Unmarshal(resp.Body(), &catalog)
So(err, ShouldBeNil)
So(catalog.Repositories, ShouldEqual, expectedRepos)
actualLinkValue := resp.Header().Get("Link")
if pageSize == "0" || pageSize == "" { //nolint:gocritic
So(actualLinkValue, ShouldEqual, "")
} else if expectedRepos[len(expectedRepos)-1] == lastRepoInStorage {
So(actualLinkValue, ShouldEqual, "")
} else {
expectedLinkValue := fmt.Sprintf("</v2/_catalog?n=%s&last=%s>; rel=\"next\"",
pageSize, catalog.Repositories[len(catalog.Repositories)-1],
)
So(actualLinkValue, ShouldEqual, expectedLinkValue)
}
t.Log("Finished " + testCaseName)
})
}
func testPagedRepositories(t *testing.T, rthdlr *api.RouteHandler, baseURL string, testCaseName string,
pageSize string,
last string,
expectedRepos []string, lastRepoInStorage string,
) {
t.Helper()
Convey(testCaseName, func() {
t.Log("Running " + testCaseName)
request, _ := http.NewRequestWithContext(context.TODO(), http.MethodGet,
baseURL+constants.RoutePrefix+constants.ExtCatalogPrefix, nil)
if pageSize != "" || last != "" {
qparm := request.URL.Query()
if pageSize != "" {
qparm.Add("n", pageSize)
}
if last != "" {
qparm.Add("last", last)
}
request.URL.RawQuery = qparm.Encode()
}
response := httptest.NewRecorder()
rthdlr.ListRepositories(response, request)
resp := response.Result()
defer resp.Body.Close()
So(resp, ShouldNotBeNil)
So(resp.StatusCode, ShouldEqual, http.StatusOK)
catalog := struct {
Repositories []string `json:"repositories"`
}{}
body, err := io.ReadAll(resp.Body)
So(err, ShouldBeNil)
err = json.Unmarshal(body, &catalog)
So(err, ShouldBeNil)
So(catalog.Repositories, ShouldEqual, expectedRepos)
actualLinkValue := resp.Header.Get("Link")
if pageSize == "0" || pageSize == "" { //nolint:gocritic
So(actualLinkValue, ShouldEqual, "")
} else if expectedRepos[len(expectedRepos)-1] == lastRepoInStorage {
So(actualLinkValue, ShouldEqual, "")
} else {
expectedLinkValue := fmt.Sprintf("</v2/_catalog?n=%s&last=%s>; rel=\"next\"",
pageSize, catalog.Repositories[len(catalog.Repositories)-1],
)
So(actualLinkValue, ShouldEqual, expectedLinkValue)
}
t.Log("Finished " + testCaseName)
})
}
func TestListingTags(t *testing.T) {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)

130
pkg/api/routes.go
View file

@ -47,6 +47,7 @@ import (
mTypes "zotregistry.dev/zot/pkg/meta/types"
zreg "zotregistry.dev/zot/pkg/regexp"
reqCtx "zotregistry.dev/zot/pkg/requestcontext"
"zotregistry.dev/zot/pkg/storage"
storageCommon "zotregistry.dev/zot/pkg/storage/common"
storageTypes "zotregistry.dev/zot/pkg/storage/types"
"zotregistry.dev/zot/pkg/test/inject"
@ -1763,6 +1764,81 @@ type RepositoryList struct {
Repositories []string `json:"repositories"`
}
func (rh *RouteHandler) listStorageRepositories(lastEntry string, maxEntries int,
userAc *reqCtx.UserAccessControl,
) ([]string, bool, error) {
var moreEntries bool
var err error
var repos []string
remainder := maxEntries
combineRepoList := make([]string, 0)
subStore := rh.c.StoreController.SubStore
subPaths := make([]string, 0)
for subPath := range subStore {
subPaths = append(subPaths, subPath)
}
sort.Strings(subPaths)
storePath := rh.c.StoreController.GetStorePath(lastEntry)
if storePath == storage.DefaultStorePath {
singleStore := rh.c.StoreController.DefaultStore
repos, moreEntries, err = singleStore.GetNextRepositories(lastEntry, remainder, AuthzFilterFunc(userAc))
if err != nil {
return repos, false, err
}
remainder = maxEntries - len(repos)
if moreEntries && remainder <= 0 && len(repos) > 0 {
// maxEntries has been hit
lastEntry = repos[len(repos)-1]
} else {
// reset for the next substores
lastEntry = ""
}
combineRepoList = append(combineRepoList, repos...)
}
for _, subPath := range subPaths {
imgStore := subStore[subPath]
if lastEntry != "" && subPath != storePath {
continue
}
if remainder > 0 || maxEntries == -1 {
repos, moreEntries, err = imgStore.GetNextRepositories(lastEntry, remainder, AuthzFilterFunc(userAc))
if err != nil {
return combineRepoList, false, err
}
// compute remainder
remainder -= len(repos)
if moreEntries && remainder <= 0 && len(repos) > 0 {
// maxEntries has been hit
lastEntry = repos[len(repos)-1]
} else {
// reset for the next substores
lastEntry = ""
}
combineRepoList = append(combineRepoList, repos...)
}
}
return combineRepoList, moreEntries, nil
}
// ListRepositories godoc
// @Summary List image repositories
// @Description List all image repositories
@ -1776,34 +1852,15 @@ func (rh *RouteHandler) ListRepositories(response http.ResponseWriter, request *
return
}
combineRepoList := make([]string, 0)
q := request.URL.Query()
subStore := rh.c.StoreController.SubStore
lastEntry := q.Get("last")
for _, imgStore := range subStore {
repos, err := imgStore.GetRepositories()
if err != nil {
response.WriteHeader(http.StatusInternalServerError)
return
}
combineRepoList = append(combineRepoList, repos...)
maxEntries, err := strconv.Atoi(q.Get("n"))
if err != nil {
maxEntries = -1
}
singleStore := rh.c.StoreController.DefaultStore
if singleStore != nil {
repos, err := singleStore.GetRepositories()
if err != nil {
response.WriteHeader(http.StatusInternalServerError)
return
}
combineRepoList = append(combineRepoList, repos...)
}
repos := make([]string, 0)
// authz context
userAc, err := reqCtx.UserAcFromContext(request.Context())
if err != nil {
@ -1812,14 +1869,23 @@ func (rh *RouteHandler) ListRepositories(response http.ResponseWriter, request *
return
}
if userAc != nil {
for _, r := range combineRepoList {
if userAc.Can(constants.ReadPermission, r) {
repos = append(repos, r)
}
}
} else {
repos = combineRepoList
repos, moreEntries, err := rh.listStorageRepositories(lastEntry, maxEntries, userAc)
if err != nil {
response.WriteHeader(http.StatusInternalServerError)
return
}
if moreEntries && len(repos) > 0 {
lastRepo := repos[len(repos)-1]
response.Header().Set(
"Link",
fmt.Sprintf("</v2/_catalog?n=%d&last=%s>; rel=\"next\"",
maxEntries,
lastRepo,
),
)
}
is := RepositoryList{Repositories: repos}

30
pkg/api/routes_test.go
View file

@ -1359,8 +1359,10 @@ func TestRoutes(t *testing.T) {
"session_id": "test",
},
&mocks.MockedImageStore{
GetRepositoriesFn: func() ([]string, error) {
return []string{}, ErrUnexpectedError
GetNextRepositoriesFn: func(lastRepo string, maxEntries int,
fn storageTypes.FilterRepoFunc,
) ([]string, bool, error) {
return []string{}, false, ErrUnexpectedError
},
},
)
@ -1374,8 +1376,10 @@ func TestRoutes(t *testing.T) {
"session_id": "test",
},
&mocks.MockedImageStore{
GetRepositoriesFn: func() ([]string, error) {
return []string{}, ErrUnexpectedError
GetNextRepositoriesFn: func(lastRepo string, maxEntries int,
fn storageTypes.FilterRepoFunc,
) ([]string, bool, error) {
return []string{}, false, ErrUnexpectedError
},
},
)
@ -1384,19 +1388,25 @@ func TestRoutes(t *testing.T) {
Convey("ListRepositories with Authz", func() {
ctlr.StoreController.DefaultStore = &mocks.MockedImageStore{
GetRepositoriesFn: func() ([]string, error) {
return []string{"repo"}, nil
GetNextRepositoriesFn: func(lastRepo string, maxEntries int,
fn storageTypes.FilterRepoFunc,
) ([]string, bool, error) {
return []string{"repo"}, false, nil
},
}
ctlr.StoreController.SubStore = map[string]storageTypes.ImageStore{
"test1": &mocks.MockedImageStore{
GetRepositoriesFn: func() ([]string, error) {
return []string{"repo1"}, nil
GetNextRepositoriesFn: func(lastRepo string, maxEntries int,
fn storageTypes.FilterRepoFunc,
) ([]string, bool, error) {
return []string{"repo1"}, false, nil
},
},
"test2": &mocks.MockedImageStore{
GetRepositoriesFn: func() ([]string, error) {
return []string{"repo2"}, nil
GetNextRepositoriesFn: func(lastRepo string, maxEntries int,
fn storageTypes.FilterRepoFunc,
) ([]string, bool, error) {
return []string{"repo2"}, false, nil
},
},
}

15
pkg/extensions/sync/httpclient/client.go
View file

@ -177,9 +177,9 @@ func (httpClient *Client) Ping() bool {
return false
}
func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interface{}, mediaType string,
func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interface{}, mediaType string, rawQuery string,
route ...string,
) ([]byte, string, int, error) {
) ([]byte, http.Header, int, error) {
httpClient.lock.RLock()
defer httpClient.lock.RUnlock()
@ -192,11 +192,12 @@ func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interfac
// we know that the second route argument is always the repo name.
// need it for caching tokens, it's not used in requests made to authz server.
if idx == 1 {
namespace = path
namespace = strings.Trim(path, "/")
}
}
url.RawQuery = url.Query().Encode()
url.RawQuery = rawQuery
//nolint: bodyclose,contextcheck
resp, body, err := httpClient.makeAndDoRequest(http.MethodGet, mediaType, namespace, url.String())
if err != nil {
@ -204,11 +205,11 @@ func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interfac
Str("errorType", common.TypeOf(err)).
Msg("failed to make request")
return nil, "", -1, err
return nil, nil, -1, err
}
if resp.StatusCode != http.StatusOK {
return nil, "", resp.StatusCode, errors.New(string(body)) //nolint:goerr113
return nil, nil, resp.StatusCode, errors.New(string(body)) //nolint:goerr113
}
// read blob
@ -216,7 +217,7 @@ func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interfac
err = json.Unmarshal(body, &resultPtr)
}
return body, resp.Header.Get("Content-Type"), resp.StatusCode, err
return body, resp.Header, resp.StatusCode, err
}
func (httpClient *Client) getAuthType(resp *http.Response) {

2
pkg/extensions/sync/references/cosign.go
View file

@ -172,7 +172,7 @@ func (ref CosignReference) SyncReferences(ctx context.Context, localRepo, remote
func (ref CosignReference) getManifest(ctx context.Context, repo, cosignTag string) (*ispec.Manifest, []byte, error) {
var cosignManifest ispec.Manifest
body, _, statusCode, err := ref.client.MakeGetRequest(ctx, &cosignManifest, ispec.MediaTypeImageManifest,
body, _, statusCode, err := ref.client.MakeGetRequest(ctx, &cosignManifest, ispec.MediaTypeImageManifest, "",
"v2", repo, "manifests", cosignTag)
if err != nil {
if statusCode == http.StatusNotFound {

4
pkg/extensions/sync/references/oci.go
View file

@ -159,7 +159,7 @@ func (ref OciReferences) SyncReferences(ctx context.Context, localRepo, remoteRe
func (ref OciReferences) getIndex(ctx context.Context, repo, subjectDigestStr string) (ispec.Index, error) {
var index ispec.Index
_, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex,
_, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex, "",
"v2", repo, "referrers", subjectDigestStr)
if err != nil {
if statusCode == http.StatusNotFound {
@ -182,7 +182,7 @@ func syncManifest(ctx context.Context, client *client.Client, imageStore storage
var refDigest godigest.Digest
OCIRefBuf, _, statusCode, err := client.MakeGetRequest(ctx, &manifest, ispec.MediaTypeImageManifest,
OCIRefBuf, _, statusCode, err := client.MakeGetRequest(ctx, &manifest, ispec.MediaTypeImageManifest, "",
"v2", remoteRepo, "manifests", desc.Digest.String())
if err != nil {
if statusCode == http.StatusNotFound {

2
pkg/extensions/sync/references/references.go
View file

@ -152,7 +152,7 @@ func syncBlob(ctx context.Context, client *client.Client, imageStore storageType
) error {
var resultPtr interface{}
body, _, statusCode, err := client.MakeGetRequest(ctx, resultPtr, "", "v2", remoteRepo, "blobs", digest.String())
body, _, statusCode, err := client.MakeGetRequest(ctx, resultPtr, "", "", "v2", remoteRepo, "blobs", digest.String())
if err != nil {
if statusCode != http.StatusOK {
log.Info().Str("repo", remoteRepo).Str("digest", digest.String()).Msg("couldn't get remote blob")

2
pkg/extensions/sync/references/referrers_tag.go
View file

@ -151,7 +151,7 @@ func (ref TagReferences) getIndex(
) (ispec.Index, []byte, error) {
var index ispec.Index
content, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex,
content, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex, "",
"v2", repo, "manifests", getReferrersTagFromSubjectDigest(subjectDigestStr))
if err != nil {
if statusCode == http.StatusNotFound {

29
pkg/extensions/sync/remote.go
View file

@ -6,6 +6,7 @@ package sync
import (
"context"
"fmt"
"net/url"
"strings"
"github.com/containers/image/v5/docker"
@ -50,13 +51,37 @@ func (registry *RemoteRegistry) GetContext() *types.SystemContext {
func (registry *RemoteRegistry) GetRepositories(ctx context.Context) ([]string, error) {
var catalog catalog
_, _, _, err := registry.client.MakeGetRequest(ctx, &catalog, "application/json", //nolint: dogsled
_, header, _, err := registry.client.MakeGetRequest(ctx, &catalog, "application/json", "", //nolint: dogsled
constants.RoutePrefix, constants.ExtCatalogPrefix)
if err != nil {
return []string{}, err
}
return catalog.Repositories, nil
var repos []string
repos = append(repos, catalog.Repositories...)
link := header.Get("Link")
for link != "" {
linkURLPart, _, _ := strings.Cut(link, ";")
linkURL, err := url.Parse(strings.Trim(linkURLPart, "<>"))
if err != nil {
return catalog.Repositories, err
}
_, header, _, err := registry.client.MakeGetRequest(ctx, &catalog, "application/json",
linkURL.RawQuery, constants.RoutePrefix, constants.ExtCatalogPrefix) //nolint: dogsled
if err != nil {
return repos, err
}
repos = append(repos, catalog.Repositories...)
link = header.Get("Link")
}
return repos, nil
}
func (registry *RemoteRegistry) GetDockerRemoteRepo(repo string) string {

83
pkg/storage/imagestore/imagestore.go
View file

@ -265,6 +265,89 @@ func (is *ImageStore) ValidateRepo(name string) (bool, error) {
return true, nil
}
func (is *ImageStore) GetNextRepositories(lastRepo string, maxEntries int, filterFn storageTypes.FilterRepoFunc,
) ([]string, bool, error) {
var lockLatency time.Time
dir := is.rootDir
is.RLock(&lockLatency)
defer is.RUnlock(&lockLatency)
stores := make([]string, 0)
moreEntries := false
entries := 0
found := false
err := is.storeDriver.Walk(dir, func(fileInfo driver.FileInfo) error {
if entries == maxEntries {
moreEntries = true
return io.EOF
}
if !fileInfo.IsDir() {
return nil
}
// skip .sync and .uploads dirs no need to try to validate them
if strings.HasSuffix(fileInfo.Path(), syncConstants.SyncBlobUploadDir) ||
strings.HasSuffix(fileInfo.Path(), ispec.ImageBlobsDir) ||
strings.HasSuffix(fileInfo.Path(), storageConstants.BlobUploadDir) {
return driver.ErrSkipDir
}
rel, err := filepath.Rel(is.rootDir, fileInfo.Path())
if err != nil {
return nil //nolint:nilerr // ignore paths that are not under root dir
}
if ok, err := is.ValidateRepo(rel); !ok || err != nil {
return nil //nolint:nilerr // ignore invalid repos
}
if lastRepo == rel {
found = true
return nil
}
if lastRepo == "" {
found = true
}
ok, err := filterFn(rel)
if err != nil {
return err
}
if found && ok {
entries++
stores = append(stores, rel)
}
return nil
})
// if the root directory is not yet created then return an empty slice of repositories
driverErr := &driver.Error{}
if errors.As(err, &driver.PathNotFoundError{}) {
is.log.Debug().Msg("empty rootDir")
return stores, false, nil
}
if errors.Is(err, io.EOF) ||
(errors.As(err, driverErr) && errors.Is(driverErr.Detail, io.EOF)) {
return stores, moreEntries, nil
}
return stores, moreEntries, err
}
// GetRepositories returns a list of all the repositories under this store.
func (is *ImageStore) GetRepositories() ([]string, error) {
var lockLatency time.Time

20
pkg/storage/storage_controller.go
View file

@ -7,8 +7,9 @@ import (
)
const (
CosignType = "cosign"
NotationType = "notation"
CosignType = "cosign"
NotationType = "notation"
DefaultStorePath = "/"
)
type StoreController struct {
@ -29,6 +30,21 @@ func GetRoutePrefix(name string) string {
return "/" + names[0]
}
func (sc StoreController) GetStorePath(name string) string {
if sc.SubStore != nil && name != "" {
subStorePath := GetRoutePrefix(name)
_, ok := sc.SubStore[subStorePath]
if !ok {
return DefaultStorePath
}
return subStorePath
}
return DefaultStorePath
}
func (sc StoreController) GetImageStore(name string) storageTypes.ImageStore {
if sc.SubStore != nil {
// SubStore is being provided, now we need to find equivalent image store and this will be found by splitting name

23
pkg/storage/storage_test.go
View file

@ -283,6 +283,14 @@ func TestStorageAPIs(t *testing.T) {
repos, err := imgStore.GetRepositories()
So(err, ShouldBeNil)
So(repos, ShouldNotBeEmpty)
repos, more, err := imgStore.GetNextRepositories("", -1, func(repo string) (bool, error) {
return true, nil
})
So(more, ShouldBeFalse)
So(err, ShouldBeNil)
So(repos, ShouldNotBeEmpty)
})
Convey("Get image tags", func() {
@ -564,6 +572,21 @@ func TestStorageAPIs(t *testing.T) {
So(len(repos), ShouldEqual, 1)
So(repos[0], ShouldEqual, "test")
repos, more, err := imgStore.GetNextRepositories("", -1, func(repo string) (bool, error) {
return true, nil
})
So(err, ShouldBeNil)
So(more, ShouldBeFalse)
So(len(repos), ShouldEqual, 1)
So(repos[0], ShouldEqual, "test")
repos, more, err = imgStore.GetNextRepositories("", -1, func(repo string) (bool, error) {
return false, nil
})
So(err, ShouldBeNil)
So(more, ShouldBeFalse)
So(len(repos), ShouldEqual, 0)
// We deleted only one tag, make sure blob should not be removed.
hasBlob, _, err = imgStore.CheckBlob("test", digest)
So(err, ShouldBeNil)

3
pkg/storage/types/types.go
View file

@ -12,6 +12,8 @@ import (
"zotregistry.dev/zot/pkg/scheduler"
)
type FilterRepoFunc func(repo string) (bool, error)
type StoreController interface {
GetImageStore(name string) ImageStore
GetDefaultImageStore() ImageStore
@ -30,6 +32,7 @@ type ImageStore interface { //nolint:interfacebloat
ValidateRepo(name string) (bool, error)
GetRepositories() ([]string, error)
GetNextRepository(repo string) (string, error)
GetNextRepositories(repo string, maxEntries int, fn FilterRepoFunc) ([]string, bool, error)
GetImageTags(repo string) ([]string, error)
GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error)
PutImageManifest(repo, reference, mediaType string, body []byte) (godigest.Digest, godigest.Digest, error)

32
pkg/test/mocks/image_store_mock.go
View file

@ -9,19 +9,21 @@ import (
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"zotregistry.dev/zot/pkg/scheduler"
storageTypes "zotregistry.dev/zot/pkg/storage/types"
)
type MockedImageStore struct {
NameFn func() string
DirExistsFn func(d string) bool
RootDirFn func() string
InitRepoFn func(name string) error
ValidateRepoFn func(name string) (bool, error)
GetRepositoriesFn func() ([]string, error)
GetNextRepositoryFn func(repo string) (string, error)
GetImageTagsFn func(repo string) ([]string, error)
GetImageManifestFn func(repo string, reference string) ([]byte, godigest.Digest, string, error)
PutImageManifestFn func(repo string, reference string, mediaType string, body []byte) (godigest.Digest,
NameFn func() string
DirExistsFn func(d string) bool
RootDirFn func() string
InitRepoFn func(name string) error
ValidateRepoFn func(name string) (bool, error)
GetRepositoriesFn func() ([]string, error)
GetNextRepositoryFn func(repo string) (string, error)
GetNextRepositoriesFn func(lastRepo string, maxEntries int, fn storageTypes.FilterRepoFunc) ([]string, bool, error)
GetImageTagsFn func(repo string) ([]string, error)
GetImageManifestFn func(repo string, reference string) ([]byte, godigest.Digest, string, error)
PutImageManifestFn func(repo string, reference string, mediaType string, body []byte) (godigest.Digest,
godigest.Digest, error)
DeleteImageManifestFn func(repo string, reference string, detectCollision bool) error
BlobUploadPathFn func(repo string, uuid string) string
@ -138,6 +140,16 @@ func (is MockedImageStore) GetNextRepository(repo string) (string, error) {
return "", nil
}
func (is MockedImageStore) GetNextRepositories(lastRepo string, maxEntries int,
fn storageTypes.FilterRepoFunc,
) ([]string, bool, error) {
if is.GetNextRepositoriesFn != nil {
return is.GetNextRepositoriesFn(lastRepo, maxEntries, fn)
}
return []string{}, false, nil
}
func (is MockedImageStore) GetImageManifest(repo string, reference string) ([]byte, godigest.Digest, string, error) {
if is.GetImageManifestFn != nil {
return is.GetImageManifestFn(repo, reference)

13
test/blackbox/pushpull.bats
View file

@ -248,6 +248,19 @@ function teardown_file() {
fi
done
[ "$found" -eq 1 ]
run regctl repo ls --limit 2 localhost:${zot_port}
[ "$status" -eq 0 ]
echo "$output"
[ $(echo "$output" | wc -l) -eq 2 ]
[ "${lines[-2]}" == "busybox" ]
[ "${lines[-1]}" == "golang" ]
run regctl repo ls --last busybox --limit 1 localhost:${zot_port}
[ "$status" -eq 0 ]
echo "$output"
[ $(echo "$output" | wc -l) -eq 1 ]
[ "${lines[-1]}" == "golang" ]
}
@test "list image tags with regclient" {