Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2025-01-06 22:40:28 -05:00
Code Issues Activity

refactor: Centralise extensions config entries (#1177)

Browse source 
Except for registry sync config

Signed-off-by: Bogdan BIVOLARU <104334+bogdanbiv@users.noreply.github.com>
This commit is contained in:
Bogdan Bivolaru 2023-02-16 08:20:28 +02:00 committed by GitHub
parent 4aa0106b0a
commit 7c3bf86a6b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 360 additions and 344 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
pkg/extensions/config/config.go
View file

@ -3,7 +3,7 @@ package config
import (
"time"
"zotregistry.io/zot/pkg/extensions/sync"
"zotregistry.io/zot/pkg/extensions/config/sync"
)
// BaseConfig has params applicable to all extensions.

43
pkg/extensions/config/sync/config.go Normal file
View file

@ -0,0 +1,43 @@
package sync
import (
"time"
)
// key is registry address.
type CredentialsFile map[string]Credentials
type Credentials struct {
Username string
Password string
}
type Config struct {
Enable *bool
CredentialsFile string
Registries []RegistryConfig
}
type RegistryConfig struct {
URLs []string
PollInterval time.Duration
Content []Content
TLSVerify *bool
OnDemand bool
CertDir string
MaxRetries *int
RetryDelay *time.Duration
OnlySigned *bool
}
type Content struct {
Prefix string
Tags *Tags
Destination string `mapstructure:",omitempty"`
StripPrefix bool
}
type Tags struct {
Regex *string
Semver *bool
}

6
pkg/extensions/extensions_test.go
View file

@ -12,7 +12,7 @@ import (
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/extensions/sync"
syncconf "zotregistry.io/zot/pkg/extensions/config/sync"
"zotregistry.io/zot/pkg/test"
)
@ -23,9 +23,9 @@ func TestEnableExtension(t *testing.T) {
conf := config.New()
falseValue := false
syncConfig := &sync.Config{
syncConfig := &syncconf.Config{
Enable: &falseValue,
Registries: []sync.RegistryConfig{},
Registries: []syncconf.RegistryConfig{},
}
// conf.Extensions.Sync.Enable = &falseValue

10
pkg/extensions/sync/on_demand.go
View file

@ -16,6 +16,7 @@ import (
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"zotregistry.io/zot/pkg/common"
syncconf "zotregistry.io/zot/pkg/extensions/config/sync"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
@ -58,7 +59,7 @@ func (di *demandedImages) delete(key string) {
di.syncedMap.Delete(key)
}
func OneImage(ctx context.Context, cfg Config, storeController storage.StoreController,
func OneImage(ctx context.Context, cfg syncconf.Config, storeController storage.StoreController,
repo, reference string, artifactType string, log log.Logger,
) error {
// guard against multiple parallel requests
@ -91,10 +92,11 @@ func OneImage(ctx context.Context, cfg Config, storeController storage.StoreCont
return err
}
func syncOneImage(ctx context.Context, imageChannel chan error, cfg Config, storeController storage.StoreController,
func syncOneImage(ctx context.Context, imageChannel chan error,
cfg syncconf.Config, storeController storage.StoreController,
localRepo, reference string, artifactType string, log log.Logger,
) {
var credentialsFile CredentialsFile
var credentialsFile syncconf.CredentialsFile
if cfg.CredentialsFile != "" {
var err error
@ -258,7 +260,7 @@ func syncOneImage(ctx context.Context, imageChannel chan error, cfg Config, stor
imageChannel <- nil
}
func syncRun(regCfg RegistryConfig,
func syncRun(regCfg syncconf.RegistryConfig,
localRepo, upstreamRepo, reference string, utils syncContextUtils, sig *signaturesCopier,
log log.Logger,
) (bool, error) {

6
pkg/extensions/sync/signatures.go
View file

@ -18,6 +18,7 @@ import (
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/api/constants"
"zotregistry.io/zot/pkg/common"
syncconf "zotregistry.io/zot/pkg/extensions/config/sync"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
@ -26,11 +27,12 @@ type signaturesCopier struct {
client *http.Client
upstreamURL url.URL
storeController storage.StoreController
credentials Credentials
credentials syncconf.Credentials
log log.Logger
}
func newSignaturesCopier(httpClient *http.Client, credentials Credentials, upstreamURL url.URL,
func newSignaturesCopier(httpClient *http.Client, credentials syncconf.Credentials,
upstreamURL url.URL,
storeController storage.StoreController, log log.Logger,
) *signaturesCopier {
return &signaturesCopier{

59
pkg/extensions/sync/sync.go
View file

@ -23,6 +23,7 @@ import (
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/api/constants"
"zotregistry.io/zot/pkg/common"
syncconf "zotregistry.io/zot/pkg/extensions/config/sync"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/test"
@ -38,44 +39,6 @@ type catalog struct {
Repositories []string `json:"repositories"`
}
// key is registry address.
type CredentialsFile map[string]Credentials
type Credentials struct {
Username string
Password string
}
type Config struct {
Enable *bool
CredentialsFile string
Registries []RegistryConfig
}
type RegistryConfig struct {
URLs []string
PollInterval time.Duration
Content []Content
TLSVerify *bool
OnDemand bool
CertDir string
MaxRetries *int
RetryDelay *time.Duration
OnlySigned *bool
}
type Content struct {
Prefix string
Tags *Tags
Destination string `mapstructure:",omitempty"`
StripPrefix bool
}
type Tags struct {
Regex *string
Semver *bool
}
type RepoReferences struct {
contentID int // matched registry config content
name string // repo name
@ -103,7 +66,7 @@ func GetUpstreamCatalog(client *http.Client, upstreamURL, username, password str
// imagesToCopyFromRepos lists all images given a registry name and its repos.
func imagesToCopyFromUpstream(ctx context.Context, registryName string, repoName string,
upstreamCtx *types.SystemContext, content Content, log log.Logger,
upstreamCtx *types.SystemContext, content syncconf.Content, log log.Logger,
) ([]types.ImageReference, error) {
imageRefs := []types.ImageReference{}
@ -179,7 +142,7 @@ func getCopyOptions(upstreamCtx, localCtx *types.SystemContext) copy.Options {
return options
}
func getUpstreamContext(regCfg *RegistryConfig, credentials Credentials) *types.SystemContext {
func getUpstreamContext(regCfg *syncconf.RegistryConfig, credentials syncconf.Credentials) *types.SystemContext {
upstreamCtx := &types.SystemContext{}
upstreamCtx.DockerCertPath = regCfg.CertDir
upstreamCtx.DockerDaemonCertPath = regCfg.CertDir
@ -192,7 +155,7 @@ func getUpstreamContext(regCfg *RegistryConfig, credentials Credentials) *types.
upstreamCtx.DockerInsecureSkipTLSVerify = types.NewOptionalBool(true)
}
if credentials != (Credentials{}) {
if credentials != (syncconf.Credentials{}) {
upstreamCtx.DockerAuthConfig = &types.DockerAuthConfig{
Username: credentials.Username,
Password: credentials.Password,
@ -203,10 +166,10 @@ func getUpstreamContext(regCfg *RegistryConfig, credentials Credentials) *types.
}
//nolint:gocyclo // offloading some of the functionalities from here would make the code harder to follow
func syncRegistry(ctx context.Context, regCfg RegistryConfig,
func syncRegistry(ctx context.Context, regCfg syncconf.RegistryConfig,
upstreamURL string,
storeController storage.StoreController, localCtx *types.SystemContext,
policyCtx *signature.PolicyContext, credentials Credentials,
policyCtx *signature.PolicyContext, credentials syncconf.Credentials,
retryOptions *retry.RetryOptions, log log.Logger,
) error {
log.Info().Msgf("syncing registry: %s", upstreamURL)
@ -266,8 +229,8 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
var imageReferences []types.ImageReference
if err = retry.RetryIfNecessary(ctx, func() error {
imageReferences, err = imagesToCopyFromUpstream(ctx, upstreamAddr, repoName, upstreamCtx,
regCfg.Content[contentID], log)
imageReferences, err = imagesToCopyFromUpstream(ctx, upstreamAddr,
repoName, upstreamCtx, regCfg.Content[contentID], log)
return err
}, retryOptions); err != nil {
@ -458,11 +421,11 @@ func getLocalContexts(log log.Logger) (*types.SystemContext, *signature.PolicyCo
return localCtx, policyContext, nil
}
func Run(ctx context.Context, cfg Config,
func Run(ctx context.Context, cfg syncconf.Config,
storeController storage.StoreController,
wtgrp *goSync.WaitGroup, logger log.Logger,
) error {
var credentialsFile CredentialsFile
var credentialsFile syncconf.CredentialsFile
var err error
@ -509,7 +472,7 @@ func Run(ctx context.Context, cfg Config,
}
// schedule each registry sync
go func(ctx context.Context, regCfg RegistryConfig, logger log.Logger) {
go func(ctx context.Context, regCfg syncconf.RegistryConfig, logger log.Logger) {
for {
// increment reference since will be busy, so shutdown has to wait
wtgrp.Add(1)

4
pkg/extensions/sync/sync_disabled_test.go
View file

@ -13,7 +13,7 @@ import (
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/extensions/sync"
syncconf "zotregistry.io/zot/pkg/extensions/config/sync"
"zotregistry.io/zot/pkg/test"
)
@ -34,7 +34,7 @@ func TestSyncExtension(t *testing.T) {
conf.Storage.RootDirectory = globalDir
conf.Storage.Commit = true
conf.Extensions = &extconf.ExtensionConfig{}
conf.Extensions.Sync = &sync.Config{
conf.Extensions.Sync = &syncconf.Config{
Enable: &defaultValue,
}
conf.Log.Level = "warn"

76
pkg/extensions/sync/sync_internal_test.go
View file

@ -24,6 +24,7 @@ import (
"zotregistry.io/zot/errors"
. "zotregistry.io/zot/pkg/common"
syncconf "zotregistry.io/zot/pkg/extensions/config/sync"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
@ -133,8 +134,8 @@ func TestSyncInternal(t *testing.T) {
updateDuration := time.Microsecond
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
syncRegistryConfig := RegistryConfig{
Content: []Content{
syncRegistryConfig := syncconf.RegistryConfig{
Content: []syncconf.Content{
{
Prefix: testImage,
},
@ -146,8 +147,8 @@ func TestSyncInternal(t *testing.T) {
}
defaultValue := true
cfg := Config{
Registries: []RegistryConfig{syncRegistryConfig},
cfg := syncconf.Config{
Registries: []syncconf.RegistryConfig{syncRegistryConfig},
Enable: &defaultValue,
CredentialsFile: "/invalid/path/to/file",
}
@ -165,8 +166,8 @@ func TestSyncInternal(t *testing.T) {
updateDuration := time.Microsecond
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
syncRegistryConfig := RegistryConfig{
Content: []Content{
syncRegistryConfig := syncconf.RegistryConfig{
Content: []syncconf.Content{
{
Prefix: testImage,
},
@ -190,7 +191,8 @@ func TestSyncInternal(t *testing.T) {
So(err, ShouldBeNil)
err = syncRegistry(ctx, syncRegistryConfig, "randomUpstreamURL",
storage.StoreController{DefaultStore: imageStore}, localCtx, policyCtx, Credentials{}, nil, log)
storage.StoreController{DefaultStore: imageStore}, localCtx, policyCtx,
syncconf.Credentials{}, nil, log)
So(err, ShouldNotBeNil)
})
@ -240,8 +242,8 @@ func TestSyncInternal(t *testing.T) {
Convey("Test getUpstreamCatalog() with missing certs", t, func() {
var tlsVerify bool
updateDuration := time.Microsecond
syncRegistryConfig := RegistryConfig{
Content: []Content{
syncRegistryConfig := syncconf.RegistryConfig{
Content: []syncconf.Content{
{
Prefix: testImage,
},
@ -276,8 +278,8 @@ func TestSyncInternal(t *testing.T) {
baseURL := test.GetBaseURL(port)
baseSecureURL := test.GetSecureBaseURL(port)
syncRegistryConfig := RegistryConfig{
Content: []Content{
syncRegistryConfig := syncconf.RegistryConfig{
Content: []syncconf.Content{
{
Prefix: testImage,
},
@ -340,12 +342,12 @@ func TestSyncInternal(t *testing.T) {
Convey("Test imagesToCopyFromUpstream()", t, func() {
upstreamCtx := &types.SystemContext{}
_, err := imagesToCopyFromUpstream(context.Background(), "localhost:4566", "repo1", upstreamCtx,
Content{}, log.NewLogger("debug", ""))
_, err := imagesToCopyFromUpstream(context.Background(), "localhost:4566", "repo1",
upstreamCtx, syncconf.Content{}, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
_, err = imagesToCopyFromUpstream(context.Background(), "docker://localhost:4566", "repo1", upstreamCtx,
Content{}, log.NewLogger("debug", ""))
_, err = imagesToCopyFromUpstream(context.Background(), "docker://localhost:4566", "repo1",
upstreamCtx, syncconf.Content{}, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
})
@ -377,7 +379,8 @@ func TestSyncInternal(t *testing.T) {
false, false, log, metrics, nil, nil,
)
sig := newSignaturesCopier(client, Credentials{}, *regURL, storage.StoreController{DefaultStore: imageStore}, log)
sig := newSignaturesCopier(client, syncconf.Credentials{},
*regURL, storage.StoreController{DefaultStore: imageStore}, log)
err = sig.syncCosignSignature(testImage, testImage, testImageTag, &ispec.Manifest{})
So(err, ShouldNotBeNil)
@ -427,7 +430,8 @@ func TestSyncInternal(t *testing.T) {
So(regURL, ShouldNotBeNil)
client := &http.Client{}
sig := newSignaturesCopier(client, Credentials{}, *regURL, storage.StoreController{DefaultStore: imageStore}, log)
sig := newSignaturesCopier(client, syncconf.Credentials{},
*regURL, storage.StoreController{DefaultStore: imageStore}, log)
canBeSkipped, err = sig.canSkipOCIRefs(testImage, testImageManifestDigest.String(), refs)
So(err, ShouldBeNil)
@ -544,7 +548,7 @@ func TestSyncInternal(t *testing.T) {
Convey("Test filterRepos()", t, func() {
repos := []string{"repo", "repo1", "repo2", "repo/repo2", "repo/repo2/repo3/repo4"}
contents := []Content{
contents := []syncconf.Content{
{
Prefix: "repo",
},
@ -560,7 +564,7 @@ func TestSyncInternal(t *testing.T) {
So(filteredRepos[1], ShouldResemble, []string{"repo/repo2", "repo/repo2/repo3/repo4"})
So(filteredRepos[2], ShouldResemble, []string{"repo1", "repo2"})
contents = []Content{
contents = []syncconf.Content{
{
Prefix: "[repo%#@",
},
@ -833,52 +837,52 @@ func TestSyncInternal(t *testing.T) {
func TestURLHelperFunctions(t *testing.T) {
testCases := []struct {
repo string
content Content
content syncconf.Content
expected string
}{
{
repo: "alpine/zot-fold/alpine",
content: Content{Prefix: "zot-fold/alpine", Destination: "/alpine", StripPrefix: false},
content: syncconf.Content{Prefix: "zot-fold/alpine", Destination: "/alpine", StripPrefix: false},
expected: "zot-fold/alpine",
},
{
repo: "zot-fold/alpine",
content: Content{Prefix: "zot-fold/alpine", Destination: "/", StripPrefix: false},
content: syncconf.Content{Prefix: "zot-fold/alpine", Destination: "/", StripPrefix: false},
expected: "zot-fold/alpine",
},
{
repo: "alpine",
content: Content{Prefix: "zot-fold/alpine", Destination: "/alpine", StripPrefix: true},
content: syncconf.Content{Prefix: "zot-fold/alpine", Destination: "/alpine", StripPrefix: true},
expected: "zot-fold/alpine",
},
{
repo: "/",
content: Content{Prefix: "zot-fold/alpine", Destination: "/", StripPrefix: true},
content: syncconf.Content{Prefix: "zot-fold/alpine", Destination: "/", StripPrefix: true},
expected: "zot-fold/alpine",
},
{
repo: "/",
content: Content{Prefix: "/", Destination: "/", StripPrefix: true},
content: syncconf.Content{Prefix: "/", Destination: "/", StripPrefix: true},
expected: "/",
},
{
repo: "alpine",
content: Content{Prefix: "zot-fold/alpine", Destination: "/alpine", StripPrefix: true},
content: syncconf.Content{Prefix: "zot-fold/alpine", Destination: "/alpine", StripPrefix: true},
expected: "zot-fold/alpine",
},
{
repo: "alpine",
content: Content{Prefix: "zot-fold/*", Destination: "/", StripPrefix: true},
content: syncconf.Content{Prefix: "zot-fold/*", Destination: "/", StripPrefix: true},
expected: "zot-fold/alpine",
},
{
repo: "alpine",
content: Content{Prefix: "zot-fold/**", Destination: "/", StripPrefix: true},
content: syncconf.Content{Prefix: "zot-fold/**", Destination: "/", StripPrefix: true},
expected: "zot-fold/alpine",
},
{
repo: "zot-fold/alpine",
content: Content{Prefix: "zot-fold/**", Destination: "/", StripPrefix: false},
content: syncconf.Content{Prefix: "zot-fold/**", Destination: "/", StripPrefix: false},
expected: "zot-fold/alpine",
},
}
@ -902,7 +906,7 @@ func TestURLHelperFunctions(t *testing.T) {
func TestFindRepoMatchingContentID(t *testing.T) {
testCases := []struct {
repo string
content []Content
content []syncconf.Content
expected struct {
contentID int
err error
@ -910,7 +914,7 @@ func TestFindRepoMatchingContentID(t *testing.T) {
}{
{
repo: "alpine/zot-fold/alpine",
content: []Content{
content: []syncconf.Content{
{Prefix: "zot-fold/alpine/", Destination: "/alpine", StripPrefix: true},
{Prefix: "zot-fold/alpine", Destination: "/alpine", StripPrefix: false},
},
@ -921,7 +925,7 @@ func TestFindRepoMatchingContentID(t *testing.T) {
},
{
repo: "alpine/zot-fold/alpine",
content: []Content{
content: []syncconf.Content{
{Prefix: "zot-fold/*", Destination: "/alpine", StripPrefix: false},
{Prefix: "zot-fold/alpine", Destination: "/alpine", StripPrefix: true},
},
@ -932,7 +936,7 @@ func TestFindRepoMatchingContentID(t *testing.T) {
},
{
repo: "myFold/zot-fold/internal/alpine",
content: []Content{
content: []syncconf.Content{
{Prefix: "zot-fold/alpine", Destination: "/alpine", StripPrefix: true},
{Prefix: "zot-fold/**", Destination: "/myFold", StripPrefix: false},
},
@ -943,7 +947,7 @@ func TestFindRepoMatchingContentID(t *testing.T) {
},
{
repo: "alpine",
content: []Content{
content: []syncconf.Content{
{Prefix: "zot-fold/*", Destination: "/alpine", StripPrefix: true},
{Prefix: "zot-fold/alpine", Destination: "/", StripPrefix: true},
},
@ -954,7 +958,7 @@ func TestFindRepoMatchingContentID(t *testing.T) {
},
{
repo: "alpine",
content: []Content{
content: []syncconf.Content{
{Prefix: "zot-fold/*", Destination: "/alpine", StripPrefix: true},
{Prefix: "zot-fold/*", Destination: "/", StripPrefix: true},
},
@ -965,7 +969,7 @@ func TestFindRepoMatchingContentID(t *testing.T) {
},
{
repo: "alpine/alpine",
content: []Content{
content: []syncconf.Content{
{Prefix: "zot-fold/*", Destination: "/alpine", StripPrefix: true},
{Prefix: "zot-fold/*", Destination: "/", StripPrefix: true},
},

485
pkg/extensions/sync/sync_test.go
View file

File diff suppressed because it is too large Load diff

13
pkg/extensions/sync/utils.go
View file

@ -26,6 +26,7 @@ import (
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/common"
syncconf "zotregistry.io/zot/pkg/extensions/config/sync"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
@ -121,7 +122,7 @@ func parseRepositoryReference(input string) (reference.Named, error) {
}
// filterRepos filters repos based on prefix given in the config.
func filterRepos(repos []string, contentList []Content, log log.Logger) map[int][]string {
func filterRepos(repos []string, contentList []syncconf.Content, log log.Logger) map[int][]string {
filtered := make(map[int][]string)
for _, repo := range repos {
@ -155,7 +156,7 @@ func filterRepos(repos []string, contentList []Content, log log.Logger) map[int]
}
// findRepoContentID return the contentID that maches the localRepo path for a given RegistryConfig in the config file.
func findRepoMatchingContentID(localRepo string, contentList []Content) (int, error) {
func findRepoMatchingContentID(localRepo string, contentList []syncconf.Content) (int, error) {
contentID := -1
localRepo = strings.Trim(localRepo, "/")
@ -194,7 +195,7 @@ func findRepoMatchingContentID(localRepo string, contentList []Content) (int, er
return contentID, nil
}
func getRepoSource(localRepo string, content Content) string {
func getRepoSource(localRepo string, content syncconf.Content) string {
localRepo = strings.Trim(localRepo, "/")
destination := strings.Trim(content.Destination, "/")
prefix := strings.Trim(content.Prefix, "/*")
@ -219,7 +220,7 @@ func getRepoSource(localRepo string, content Content) string {
}
// getRepoDestination returns the local storage path of the synced repo based on the specified destination.
func getRepoDestination(remoteRepo string, content Content) string {
func getRepoDestination(remoteRepo string, content syncconf.Content) string {
remoteRepo = strings.Trim(remoteRepo, "/")
destination := strings.Trim(content.Destination, "/")
prefix := strings.Trim(content.Prefix, "/*")
@ -244,13 +245,13 @@ func getRepoDestination(remoteRepo string, content Content) string {
}
// Get sync.FileCredentials from file.
func getFileCredentials(filepath string) (CredentialsFile, error) {
func getFileCredentials(filepath string) (syncconf.CredentialsFile, error) {
credsFile, err := os.ReadFile(filepath)
if err != nil {
return nil, err
}
var creds CredentialsFile
var creds syncconf.CredentialsFile
err = json.Unmarshal(credsFile, &creds)
if err != nil {