Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00
Code Issues Activity

fix: removed resty calls from sync (#1016)

Browse source 
Signed-off-by: Ana-Roberta Lisca <ana.kagome@yahoo.com>
This commit is contained in:
Lisca Ana-Roberta 2022-12-22 20:19:42 +02:00 committed by GitHub
parent 50bdc2f402
commit 14238d4a8d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 820 additions and 568 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
pkg/api/controller_test.go
View file

@ -49,6 +49,7 @@ import (
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/api/constants"
"zotregistry.io/zot/pkg/common"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
@ -6668,7 +6669,7 @@ func TestDistSpecExtensions(t *testing.T) {
func getAllBlobs(imagePath string) []string {
blobList := make([]string, 0)
if !local.DirExists(imagePath) {
if !common.DirExists(imagePath) {
return []string{}
}
@ -6713,7 +6714,7 @@ func getAllBlobs(imagePath string) []string {
func getAllManifests(imagePath string) []string {
manifestList := make([]string, 0)
if !local.DirExists(imagePath) {
if !common.DirExists(imagePath) {
return []string{}
}

90
pkg/cli/client.go
View file

@ -6,16 +6,12 @@ package cli
import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
@ -26,7 +22,7 @@ import (
zotErrors "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/common"
)
var (
@ -43,33 +39,6 @@ const (
caCertFilename = "ca.crt"
)
func createHTTPClient(verifyTLS bool, host string) *http.Client {
htr := http.DefaultTransport.(*http.Transport).Clone() //nolint: forcetypeassert
if !verifyTLS {
htr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint: gosec
return &http.Client{
Timeout: httpTimeout,
Transport: htr,
}
}
// Add a copy of the system cert pool
caCertPool, _ := x509.SystemCertPool()
tlsConfig := loadPerHostCerts(caCertPool, host)
if tlsConfig == nil {
tlsConfig = &tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12}
}
htr.TLSClientConfig = tlsConfig
return &http.Client{
Timeout: httpTimeout,
Transport: htr,
}
}
func makeGETRequest(ctx context.Context, url, username, password string,
verifyTLS bool, debug bool, resultsPtr interface{}, configWriter io.Writer,
) (http.Header, error) {
@ -112,12 +81,17 @@ func doHTTPRequest(req *http.Request, verifyTLS bool, debug bool,
) (http.Header, error) {
var httpClient *http.Client
var err error
host := req.Host
httpClientLock.Lock()
if httpClientsMap[host] == nil {
httpClient = createHTTPClient(verifyTLS, host)
httpClient, err = common.CreateHTTPClient(verifyTLS, host, "")
if err != nil {
return nil, err
}
httpClientsMap[host] = httpClient
} else {
@ -159,56 +133,6 @@ func doHTTPRequest(req *http.Request, verifyTLS bool, debug bool,
return resp.Header, nil
}
func loadPerHostCerts(caCertPool *x509.CertPool, host string) *tls.Config {
// Check if the /home/user/.config/containers/certs.d/$IP:$PORT dir exists
home := os.Getenv("HOME")
clientCertsDir := filepath.Join(home, homeCertsDir, host)
if local.DirExists(clientCertsDir) {
tlsConfig, err := getTLSConfig(clientCertsDir, caCertPool)
if err == nil {
return tlsConfig
}
}
// Check if the /etc/containers/certs.d/$IP:$PORT dir exists
clientCertsDir = filepath.Join(certsPath, host)
if local.DirExists(clientCertsDir) {
tlsConfig, err := getTLSConfig(clientCertsDir, caCertPool)
if err == nil {
return tlsConfig
}
}
return nil
}
func getTLSConfig(certsPath string, caCertPool *x509.CertPool) (*tls.Config, error) {
clientCert := filepath.Join(certsPath, clientCertFilename)
clientKey := filepath.Join(certsPath, clientKeyFilename)
caCertFile := filepath.Join(certsPath, caCertFilename)
cert, err := tls.LoadX509KeyPair(clientCert, clientKey)
if err != nil {
return nil, err
}
caCert, err := os.ReadFile(caCertFile)
if err != nil {
return nil, err
}
caCertPool.AppendCertsFromPEM(caCert)
return &tls.Config{
Certificates: []tls.Certificate{cert},
RootCAs: caCertPool,
MinVersion: tls.VersionTLS12,
}, nil
}
func isURL(str string) bool {
u, err := url.Parse(str)

202
pkg/common/common.go
View file

@ -1,5 +1,33 @@
package common
import (
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"net/http"
"os"
"path"
"path/filepath"
"syscall"
"time"
"unicode/utf8"
"zotregistry.io/zot/pkg/log"
)
const (
httpTimeout = 5 * time.Minute
certsPath = "/etc/containers/certs.d"
homeCertsDir = ".config/containers/certs.d"
clientCertFilename = "client.cert"
clientKeyFilename = "client.key"
caCertFilename = "ca.crt"
)
func Contains(slice []string, item string) bool {
for _, v := range slice {
if item == v {
@ -9,3 +37,177 @@ func Contains(slice []string, item string) bool {
return false
}
func GetTLSConfig(certsPath string, caCertPool *x509.CertPool) (*tls.Config, error) {
clientCert := filepath.Join(certsPath, clientCertFilename)
clientKey := filepath.Join(certsPath, clientKeyFilename)
caCertFile := filepath.Join(certsPath, caCertFilename)
cert, err := tls.LoadX509KeyPair(clientCert, clientKey)
if err != nil {
return nil, err
}
caCert, err := os.ReadFile(caCertFile)
if err != nil {
return nil, err
}
caCertPool.AppendCertsFromPEM(caCert)
return &tls.Config{
Certificates: []tls.Certificate{cert},
RootCAs: caCertPool,
MinVersion: tls.VersionTLS12,
}, nil
}
func loadPerHostCerts(caCertPool *x509.CertPool, host string) *tls.Config {
// Check if the /home/user/.config/containers/certs.d/$IP:$PORT dir exists
home := os.Getenv("HOME")
clientCertsDir := filepath.Join(home, homeCertsDir, host)
if DirExists(clientCertsDir) {
tlsConfig, err := GetTLSConfig(clientCertsDir, caCertPool)
if err == nil {
return tlsConfig
}
}
// Check if the /etc/containers/certs.d/$IP:$PORT dir exists
clientCertsDir = filepath.Join(certsPath, host)
if DirExists(clientCertsDir) {
tlsConfig, err := GetTLSConfig(clientCertsDir, caCertPool)
if err == nil {
return tlsConfig
}
}
return nil
}
func CreateHTTPClient(verifyTLS bool, host string, certDir string) (*http.Client, error) {
htr := http.DefaultTransport.(*http.Transport).Clone() //nolint: forcetypeassert
if !verifyTLS {
htr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint: gosec
return &http.Client{
Timeout: httpTimeout,
Transport: htr,
}, nil
}
// Add a copy of the system cert pool
caCertPool, _ := x509.SystemCertPool()
tlsConfig := loadPerHostCerts(caCertPool, host)
if tlsConfig == nil {
tlsConfig = &tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12}
}
htr.TLSClientConfig = tlsConfig
if certDir != "" {
clientCert := path.Join(certDir, "client.cert")
clientKey := path.Join(certDir, "client.key")
caCertPath := path.Join(certDir, "ca.crt")
caCert, err := os.ReadFile(caCertPath)
if err != nil {
return nil, err
}
caCertPool.AppendCertsFromPEM(caCert)
cert, err := tls.LoadX509KeyPair(clientCert, clientKey)
if err != nil {
return nil, err
}
htr.TLSClientConfig.Certificates = append(htr.TLSClientConfig.Certificates, cert)
}
return &http.Client{
Timeout: httpTimeout,
Transport: htr,
}, nil
}
func TypeOf(v interface{}) string {
return fmt.Sprintf("%T", v)
}
func MakeHTTPGetRequest(httpClient *http.Client, username string, password string, resultPtr interface{},
blobURL string, mediaType string, log log.Logger,
) ([]byte, int, error) {
req, err := http.NewRequest(http.MethodGet, blobURL, nil) //nolint
if err != nil {
return nil, 0, err
}
req.Header.Set("Content-Type", mediaType)
req.SetBasicAuth(username, password)
resp, err := httpClient.Do(req)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get blob: %s", blobURL)
return nil, -1, err
}
body, err := io.ReadAll(resp.Body)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get blob: %s", blobURL)
return nil, resp.StatusCode, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Error().Str("status code", fmt.Sprint(resp.StatusCode)).Err(err).Msgf("couldn't get blob: %s", blobURL)
return nil, resp.StatusCode, errors.New(string(body)) //nolint:goerr113
}
// read blob
err = json.Unmarshal(body, &resultPtr)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't unmarshal blob: %s", blobURL)
return body, resp.StatusCode, err
}
return body, resp.StatusCode, err
}
func DirExists(d string) bool {
if !utf8.ValidString(d) {
return false
}
fileInfo, err := os.Stat(d)
if err != nil {
if e, ok := err.(*fs.PathError); ok && errors.Is(e.Err, syscall.ENAMETOOLONG) || //nolint: errorlint
errors.Is(e.Err, syscall.EINVAL) {
return false
}
}
if err != nil && os.IsNotExist(err) {
return false
}
if !fileInfo.IsDir() {
return false
}
return true
}

94
pkg/common/common_test.go
View file

@ -1,11 +1,20 @@
package common_test
import (
"context"
"crypto/x509"
"os"
"path"
"testing"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/test"
)
func TestCommon(t *testing.T) {
@ -15,4 +24,89 @@ func TestCommon(t *testing.T) {
So(common.Contains(first, "peach"), ShouldBeFalse)
So(common.Contains([]string{}, "apple"), ShouldBeFalse)
})
Convey("test getTLSConfig()", t, func() {
caCertPool, _ := x509.SystemCertPool()
tlsConfig, err := common.GetTLSConfig("wrongPath", caCertPool)
So(tlsConfig, ShouldBeNil)
So(err, ShouldNotBeNil)
tempDir := t.TempDir()
err = test.CopyFiles("../../test/data", tempDir)
So(err, ShouldBeNil)
err = os.Chmod(path.Join(tempDir, "ca.crt"), 0o000)
So(err, ShouldBeNil)
_, err = common.GetTLSConfig(tempDir, caCertPool)
So(err, ShouldNotBeNil)
})
Convey("test dirExists()", t, func() {
exists := common.DirExists("testdir")
So(exists, ShouldBeFalse)
file, err := os.Create("file.txt")
So(err, ShouldBeNil)
isDir := common.DirExists(file.Name())
So(isDir, ShouldBeFalse)
})
Convey("test CreateHTTPClient() no permissions on certificate", t, func() {
tempDir := t.TempDir()
err := test.CopyFiles("../../test/data", tempDir)
So(err, ShouldBeNil)
err = os.Chmod(path.Join(tempDir, "ca.crt"), 0o000)
So(err, ShouldBeNil)
_, err = common.CreateHTTPClient(true, "localhost", tempDir)
So(err, ShouldNotBeNil)
})
Convey("test CreateHTTPClient() no permissions on key", t, func() {
tempDir := t.TempDir()
err := test.CopyFiles("../../test/data", tempDir)
So(err, ShouldBeNil)
err = os.Chmod(path.Join(tempDir, "client.key"), 0o000)
So(err, ShouldBeNil)
_, err = common.CreateHTTPClient(true, "localhost", tempDir)
So(err, ShouldNotBeNil)
})
Convey("test MakeHTTPGetRequest() no permissions on key", t, func() {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
ctlr := api.NewController(conf)
tempDir := t.TempDir()
err := test.CopyFiles("../../test/data", tempDir)
So(err, ShouldBeNil)
ctlr.Config.Storage.RootDirectory = tempDir
go startServer(ctlr)
defer stopServer(ctlr)
test.WaitTillServerReady(baseURL)
var resultPtr interface{}
httpClient, err := common.CreateHTTPClient(true, "localhost", tempDir)
So(err, ShouldBeNil)
_, _, err = common.MakeHTTPGetRequest(httpClient, "", "",
resultPtr, baseURL+"/v2/", ispec.MediaTypeImageManifest, log.NewLogger("", ""))
So(err, ShouldNotBeNil)
})
}
func startServer(c *api.Controller) {
// this blocks
ctx := context.Background()
if err := c.Run(ctx); err != nil {
return
}
}
func stopServer(c *api.Controller) {
ctx := context.Background()
_ = c.Server.Shutdown(ctx)
}

55
pkg/extensions/sync/on_demand.go
View file

@ -3,6 +3,7 @@ package sync
import (
"context"
"fmt"
"net/url"
"os"
"sync"
"time"
@ -14,6 +15,7 @@ import (
"github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
@ -99,7 +101,7 @@ func syncOneImage(ctx context.Context, imageChannel chan error, cfg Config, stor
credentialsFile, err = getFileCredentials(cfg.CredentialsFile)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get registry credentials from %s", cfg.CredentialsFile)
imageChannel <- err
@ -154,14 +156,29 @@ func syncOneImage(ctx context.Context, imageChannel chan error, cfg Config, stor
upstreamAddr := StripRegistryTransport(upstreamURL)
httpClient, registryURL, err := getHTTPClient(&regCfg, upstreamURL, credentialsFile[upstreamAddr], log)
var TLSverify bool
if regCfg.TLSVerify != nil && *regCfg.TLSVerify {
TLSverify = true
}
registryURL, err := url.Parse(upstreamURL)
if err != nil {
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Str("url", upstreamURL).Msg("couldn't parse url")
imageChannel <- err
return
}
httpClient, err := common.CreateHTTPClient(TLSverify, registryURL.Host, regCfg.CertDir)
if err != nil {
imageChannel <- err
return
}
sig := newSignaturesCopier(httpClient, *registryURL, storeController, log)
sig := newSignaturesCopier(httpClient, credentialsFile[upstreamAddr], *registryURL, storeController, log)
upstreamCtx := getUpstreamContext(&regCfg, credentialsFile[upstreamAddr])
options := getCopyOptions(upstreamCtx, localCtx)
@ -226,7 +243,7 @@ func syncOneImage(ctx context.Context, imageChannel chan error, cfg Config, stor
return err
}, retryOptions); err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("sync routine: error while copying image %s", demandedImageRef)
}
}()
@ -249,7 +266,7 @@ func syncRun(regCfg RegistryConfig,
upstreamImageRef, err := getImageRef(utils.upstreamAddr, upstreamRepo, reference)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("error creating docker reference for repository %s/%s:%s",
utils.upstreamAddr, upstreamRepo, reference)
@ -279,13 +296,13 @@ func syncRun(regCfg RegistryConfig,
// get upstream signatures
cosignManifest, err := sig.getCosignManifest(upstreamRepo, upstreamImageDigest.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get upstream image %s cosign manifest", upstreamImageRef.DockerReference())
}
refs, err := sig.getNotaryRefs(upstreamRepo, upstreamImageDigest.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get upstream image %s notary references", upstreamImageRef.DockerReference())
}
@ -309,7 +326,7 @@ func syncRun(regCfg RegistryConfig,
localImageRef, err := getLocalImageRef(localCachePath, localRepo, reference)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't obtain a valid image reference for reference %s/%s:%s",
localCachePath, localRepo, reference)
@ -322,7 +339,7 @@ func syncRun(regCfg RegistryConfig,
_, err = copy.Image(context.Background(), utils.policyCtx, localImageRef, upstreamImageRef, &utils.copyOptions)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("error encountered while syncing on demand %s to %s",
upstreamImageRef.DockerReference(), localCachePath)
@ -331,7 +348,7 @@ func syncRun(regCfg RegistryConfig,
err = pushSyncedLocalImage(localRepo, reference, localCachePath, imageStore, log)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("error while pushing synced cached image %s",
fmt.Sprintf("%s/%s:%s", localCachePath, localRepo, reference))
@ -340,7 +357,7 @@ func syncRun(regCfg RegistryConfig,
index, err := sig.getOCIRefs(upstreamRepo, upstreamImageDigest.String())
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get upstream image %s oci references", upstreamImageRef.DockerReference())
}
@ -351,7 +368,7 @@ func syncRun(regCfg RegistryConfig,
err = sig.syncCosignSignature(localRepo, upstreamRepo, upstreamImageDigest.String(), cosignManifest)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't copy image cosign signature %s/%s:%s", utils.upstreamAddr, upstreamRepo, reference)
return false, err
@ -359,7 +376,7 @@ func syncRun(regCfg RegistryConfig,
err = sig.syncNotaryRefs(localRepo, upstreamRepo, upstreamImageDigest.String(), refs)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't copy image notary signature %s/%s:%s", utils.upstreamAddr, upstreamRepo, reference)
return false, err
@ -378,7 +395,7 @@ func syncSignaturesArtifacts(sig *signaturesCopier, localRepo, upstreamRepo, ref
// is cosign signature
cosignManifest, err := sig.getCosignManifest(upstreamRepo, reference)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get upstream image %s:%s:%s cosign manifest", upstreamURL, upstreamRepo, reference)
return err
@ -386,7 +403,7 @@ func syncSignaturesArtifacts(sig *signaturesCopier, localRepo, upstreamRepo, ref
err = sig.syncCosignSignature(localRepo, upstreamRepo, reference, cosignManifest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't copy upstream image cosign signature %s/%s:%s", upstreamURL, upstreamRepo, reference)
return err
@ -395,7 +412,7 @@ func syncSignaturesArtifacts(sig *signaturesCopier, localRepo, upstreamRepo, ref
// is notary signature
refs, err := sig.getNotaryRefs(upstreamRepo, reference)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get upstream image %s/%s:%s notary references", upstreamURL, upstreamRepo, reference)
return err
@ -403,7 +420,7 @@ func syncSignaturesArtifacts(sig *signaturesCopier, localRepo, upstreamRepo, ref
err = sig.syncNotaryRefs(localRepo, upstreamRepo, reference, refs)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't copy image signature %s/%s:%s", upstreamURL, upstreamRepo, reference)
return err
@ -411,7 +428,7 @@ func syncSignaturesArtifacts(sig *signaturesCopier, localRepo, upstreamRepo, ref
case artifactType == OCIReference:
index, err := sig.getOCIRefs(upstreamRepo, reference)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get oci references %s/%s:%s", upstreamURL, upstreamRepo, reference)
return err
@ -419,7 +436,7 @@ func syncSignaturesArtifacts(sig *signaturesCopier, localRepo, upstreamRepo, ref
err = sig.syncOCIRefs(localRepo, upstreamRepo, reference, index)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't copy oci references %s/%s:%s", upstreamURL, upstreamRepo, reference)
return err

480
pkg/extensions/sync/signatures.go
View file

@ -1,6 +1,7 @@
package sync
import (
"context"
"encoding/json"
"errors"
"net/http"
@ -13,26 +14,28 @@ import (
ispec "github.com/opencontainers/image-spec/specs-go/v1"
oras "github.com/oras-project/artifacts-spec/specs-go/v1"
"github.com/sigstore/cosign/pkg/oci/remote"
"gopkg.in/resty.v1"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/api/constants"
"zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
)
type signaturesCopier struct {
client *resty.Client
client *http.Client
upstreamURL url.URL
storeController storage.StoreController
credentials Credentials
log log.Logger
}
func newSignaturesCopier(httpClient *resty.Client, upstreamURL url.URL,
func newSignaturesCopier(httpClient *http.Client, credentials Credentials, upstreamURL url.URL,
storeController storage.StoreController, log log.Logger,
) *signaturesCopier {
return &signaturesCopier{
client: httpClient,
credentials: credentials,
upstreamURL: upstreamURL,
storeController: storeController,
log: log,
@ -50,35 +53,19 @@ func (sig *signaturesCopier) getCosignManifest(repo, digestStr string) (*ispec.M
getCosignManifestURL.RawQuery = getCosignManifestURL.Query().Encode()
resp, err := sig.client.R().
SetHeader("Content-Type", ispec.MediaTypeImageManifest).
Get(getCosignManifestURL.String())
_, statusCode, err := common.MakeHTTPGetRequest(sig.client, sig.credentials.Username,
sig.credentials.Password, &cosignManifest,
getCosignManifestURL.String(), ispec.MediaTypeImageManifest, sig.log)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("url", getCosignManifestURL.String()).
Msgf("couldn't get cosign manifest: %s", cosignTag)
if statusCode == http.StatusNotFound {
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't find any cosign manifest: %s", getCosignManifestURL.String())
return nil, err
}
return nil, zerr.ErrSyncReferrerNotFound
}
if resp.StatusCode() == http.StatusNotFound {
sig.log.Info().Msgf("couldn't find any cosign signature from %s, status code: %d skipping",
getCosignManifestURL.String(), resp.StatusCode())
return nil, zerr.ErrSyncReferrerNotFound
} else if resp.IsError() {
sig.log.Error().Str("errorType", TypeOf(zerr.ErrSyncReferrer)).
Err(zerr.ErrSyncReferrer).Msgf("couldn't get cosign signature from %s, status code: %d skipping",
getCosignManifestURL.String(), resp.StatusCode())
return nil, zerr.ErrSyncReferrer
}
err = json.Unmarshal(resp.Body(), &cosignManifest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("url", getCosignManifestURL.String()).
Msgf("couldn't unmarshal cosign manifest %s", cosignTag)
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get cosign manifest: %s", getCosignManifestURL.String())
return nil, err
}
@ -97,34 +84,17 @@ func (sig *signaturesCopier) getNotaryRefs(repo, digestStr string) (ReferenceLis
getReferrersURL.RawQuery = getReferrersURL.Query().Encode()
resp, err := sig.client.R().
SetHeader("Content-Type", "application/json").
Get(getReferrersURL.String())
_, statusCode, err := common.MakeHTTPGetRequest(sig.client, sig.credentials.Username,
sig.credentials.Password, &referrers,
getReferrersURL.String(), "application/json", sig.log)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("url", getReferrersURL.String()).Msg("couldn't get referrers")
if statusCode == http.StatusNotFound {
sig.log.Info().Err(err).Msg("couldn't find any notary signatures/oras artifacts")
return referrers, err
}
return referrers, zerr.ErrSyncReferrerNotFound
}
if resp.StatusCode() == http.StatusNotFound || resp.StatusCode() == http.StatusBadRequest {
sig.log.Info().Msgf("couldn't find any notary signature from %s, status code: %d, skipping",
getReferrersURL.String(), resp.StatusCode())
return ReferenceList{}, zerr.ErrSyncReferrerNotFound
} else if resp.IsError() {
sig.log.Error().Str("errorType", TypeOf(zerr.ErrSyncReferrer)).
Err(zerr.ErrSyncReferrer).Msgf("couldn't get notary signature from %s, status code: %d skipping",
getReferrersURL.String(), resp.StatusCode())
return ReferenceList{}, zerr.ErrSyncReferrer
}
err = json.Unmarshal(resp.Body(), &referrers)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("url", getReferrersURL.String()).
Msgf("couldn't unmarshal notary signature")
sig.log.Error().Err(err).Msg("couldn't get notary signatures/oras artifacts")
return referrers, err
}
@ -141,38 +111,24 @@ func (sig *signaturesCopier) getOCIRefs(repo, digestStr string) (ispec.Index, er
getReferrersURL.RawQuery = getReferrersURL.Query().Encode()
resp, err := sig.client.R().
SetHeader("Content-Type", "application/json").
Get(getReferrersURL.String())
_, statusCode, err := common.MakeHTTPGetRequest(sig.client, sig.credentials.Username,
sig.credentials.Password, &index,
getReferrersURL.String(), "application/json", sig.log)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("url", getReferrersURL.String()).Msg("couldn't get referrers")
if statusCode == http.StatusNotFound {
sig.log.Info().Msgf("couldn't find any oci reference from %s, status code: %d, skipping",
getReferrersURL.String(), statusCode)
return index, err
}
return index, zerr.ErrSyncReferrerNotFound
}
if resp.StatusCode() == http.StatusNotFound {
sig.log.Info().Msgf("couldn't find any oci reference from %s, status code: %d, skipping",
getReferrersURL.String(), resp.StatusCode())
return index, zerr.ErrSyncReferrerNotFound
} else if resp.IsError() {
sig.log.Error().Str("errorType", TypeOf(zerr.ErrSyncReferrer)).
sig.log.Error().Str("errorType", common.TypeOf(zerr.ErrSyncReferrer)).
Err(zerr.ErrSyncReferrer).Msgf("couldn't get oci reference from %s, status code: %d skipping",
getReferrersURL.String(), resp.StatusCode())
getReferrersURL.String(), statusCode)
return index, zerr.ErrSyncReferrer
}
err = json.Unmarshal(resp.Body(), &index)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("url", getReferrersURL.String()).
Msgf("couldn't unmarshal oci reference")
return index, err
}
return index, nil
}
@ -213,17 +169,15 @@ func (sig *signaturesCopier) syncCosignSignature(localRepo, remoteRepo, digestSt
cosignManifestBuf, err := json.Marshal(cosignManifest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't marshal cosign manifest")
return err
}
// push manifest
_, err = imageStore.PutImageManifest(localRepo, cosignTag,
ispec.MediaTypeImageManifest, cosignManifestBuf)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't upload cosign manifest")
return err
@ -234,6 +188,172 @@ func (sig *signaturesCopier) syncCosignSignature(localRepo, remoteRepo, digestSt
return nil
}
func (sig *signaturesCopier) syncNotaryRefs(localRepo, remoteRepo, digestStr string, referrers ReferenceList,
) error {
if len(referrers.References) == 0 {
return nil
}
skipNotarySig, err := sig.canSkipNotaryRefs(localRepo, digestStr, referrers)
if err != nil {
sig.log.Error().Err(err).Msgf("couldn't check if the upstream image %s:%s notary signature can be skipped",
remoteRepo, digestStr)
}
if skipNotarySig {
return nil
}
imageStore := sig.storeController.GetImageStore(localRepo)
sig.log.Info().Msg("syncing notary signatures")
for _, ref := range referrers.References {
// get referrer manifest
getRefManifestURL := sig.upstreamURL
getRefManifestURL.Path = path.Join(getRefManifestURL.Path, "v2", remoteRepo, "manifests", ref.Digest.String())
getRefManifestURL.RawQuery = getRefManifestURL.Query().Encode()
var artifactManifest oras.Manifest
body, statusCode, err := common.MakeHTTPGetRequest(sig.client, sig.credentials.Username,
sig.credentials.Password, &artifactManifest,
getRefManifestURL.String(), ref.MediaType, sig.log)
if err != nil {
if statusCode == http.StatusNotFound {
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't find any notary manifest: %s", getRefManifestURL.String())
return zerr.ErrSyncReferrerNotFound
}
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get notary manifest: %s", getRefManifestURL.String())
return err
}
for _, blob := range artifactManifest.Blobs {
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, blob.Digest); err != nil {
return err
}
}
_, err = imageStore.PutImageManifest(localRepo, ref.Digest.String(),
oras.MediaTypeArtifactManifest, body)
if err != nil {
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't upload notary sig manifest")
return err
}
}
sig.log.Info().Msgf("successfully synced notary signature for repo %s digest %s", localRepo, digestStr)
return nil
}
func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, digestStr string, index ispec.Index,
) error {
if len(index.Manifests) == 0 {
return nil
}
skipOCIRefs, err := sig.canSkipOCIRefs(localRepo, digestStr, index)
if err != nil {
sig.log.Error().Err(err).Msgf("couldn't check if the upstream image %s:%s oci references can be skipped",
remoteRepo, digestStr)
}
if skipOCIRefs {
return nil
}
imageStore := sig.storeController.GetImageStore(localRepo)
sig.log.Info().Msg("syncing oci references")
for _, ref := range index.Manifests {
getRefManifestURL := sig.upstreamURL
getRefManifestURL.Path = path.Join(getRefManifestURL.Path, "v2", remoteRepo, "manifests", ref.Digest.String())
getRefManifestURL.RawQuery = getRefManifestURL.Query().Encode()
var artifactManifest oras.Manifest
body, statusCode, err := common.MakeHTTPGetRequest(sig.client, sig.credentials.Username,
sig.credentials.Password, &artifactManifest,
getRefManifestURL.String(), ref.MediaType, sig.log)
if err != nil {
if statusCode == http.StatusNotFound {
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't find any oci reference manifest: %s", getRefManifestURL.String())
return zerr.ErrSyncReferrerNotFound
}
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get oci reference manifest: %s", getRefManifestURL.String())
return err
}
if ref.MediaType == ispec.MediaTypeImageManifest {
// read manifest
var manifest ispec.Manifest
err = json.Unmarshal(body, &manifest)
if err != nil {
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't unmarshal oci reference manifest: %s", getRefManifestURL.String())
return err
}
for _, layer := range manifest.Layers {
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, layer.Digest); err != nil {
return err
}
}
// sync config blob
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, manifest.Config.Digest); err != nil {
return err
}
} else if ref.MediaType == ispec.MediaTypeArtifactManifest {
// read manifest
var manifest ispec.Artifact
err = json.Unmarshal(body, &manifest)
if err != nil {
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't unmarshal oci reference manifest: %s", getRefManifestURL.String())
return err
}
for _, layer := range manifest.Blobs {
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, layer.Digest); err != nil {
return err
}
}
}
_, err = imageStore.PutImageManifest(localRepo, ref.Digest.String(),
ref.MediaType, body)
if err != nil {
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't upload oci reference manifest")
return err
}
}
sig.log.Info().Msgf("successfully synced oci references for repo %s digest %s", localRepo, digestStr)
return nil
}
func (sig *signaturesCopier) syncOCIArtifact(localRepo, remoteRepo, reference string,
ociArtifactBuf []byte,
) error {
@ -268,7 +388,7 @@ func (sig *signaturesCopier) syncOCIArtifact(localRepo, remoteRepo, reference st
artifactManifestBuf, err := json.Marshal(ociArtifact)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't marshal OCI artifact")
return err
@ -278,7 +398,7 @@ func (sig *signaturesCopier) syncOCIArtifact(localRepo, remoteRepo, reference st
_, err = imageStore.PutImageManifest(localRepo, reference,
ispec.MediaTypeArtifactManifest, artifactManifestBuf)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't upload OCI artifact manifest")
return err
@ -289,165 +409,6 @@ func (sig *signaturesCopier) syncOCIArtifact(localRepo, remoteRepo, reference st
return nil
}
func (sig *signaturesCopier) syncNotaryRefs(localRepo, remoteRepo, digestStr string, referrers ReferenceList,
) error {
if len(referrers.References) == 0 {
return nil
}
skipNotarySig, err := sig.canSkipNotaryRefs(localRepo, digestStr, referrers)
if err != nil {
sig.log.Error().Err(err).Msgf("couldn't check if the upstream image %s:%s notary signature can be skipped",
remoteRepo, digestStr)
}
if skipNotarySig {
return nil
}
imageStore := sig.storeController.GetImageStore(localRepo)
sig.log.Info().Msg("syncing notary signatures")
for _, ref := range referrers.References {
// get referrer manifest
getRefManifestURL := sig.upstreamURL
getRefManifestURL.Path = path.Join(getRefManifestURL.Path, "v2", remoteRepo, "manifests", ref.Digest.String())
getRefManifestURL.RawQuery = getRefManifestURL.Query().Encode()
resp, err := sig.client.R().
SetHeader("Content-Type", ref.MediaType).
Get(getRefManifestURL.String())
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get notary manifest: %s", getRefManifestURL.String())
return err
}
// read manifest
var artifactManifest oras.Manifest
err = json.Unmarshal(resp.Body(), &artifactManifest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't unmarshal notary manifest: %s", getRefManifestURL.String())
return err
}
for _, blob := range artifactManifest.Blobs {
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, blob.Digest); err != nil {
return err
}
}
_, err = imageStore.PutImageManifest(localRepo, ref.Digest.String(),
oras.MediaTypeArtifactManifest, resp.Body())
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload notary sig manifest")
return err
}
}
sig.log.Info().Msgf("successfully synced notary signature for repo %s digest %s", localRepo, digestStr)
return nil
}
func (sig *signaturesCopier) syncOCIRefs(localRepo, remoteRepo, digestStr string, index ispec.Index,
) error {
if len(index.Manifests) == 0 {
return nil
}
skipOCIRefs, err := sig.canSkipOCIRefs(localRepo, digestStr, index)
if err != nil {
sig.log.Error().Err(err).Msgf("couldn't check if the upstream image %s:%s oci references can be skipped",
remoteRepo, digestStr)
}
if skipOCIRefs {
return nil
}
imageStore := sig.storeController.GetImageStore(localRepo)
sig.log.Info().Msg("syncing OCI references")
for _, ref := range index.Manifests {
getRefManifestURL := sig.upstreamURL
getRefManifestURL.Path = path.Join(getRefManifestURL.Path, "v2", remoteRepo, "manifests", ref.Digest.String())
getRefManifestURL.RawQuery = getRefManifestURL.Query().Encode()
resp, err := sig.client.R().
SetHeader("Content-Type", ref.MediaType).
Get(getRefManifestURL.String())
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get oci reference manifest: %s", getRefManifestURL.String())
return err
}
if ref.MediaType == ispec.MediaTypeImageManifest {
// read manifest
var manifest ispec.Manifest
err = json.Unmarshal(resp.Body(), &manifest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't unmarshal oci reference manifest: %s", getRefManifestURL.String())
return err
}
for _, layer := range manifest.Layers {
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, layer.Digest); err != nil {
return err
}
}
// sync config blob
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, manifest.Config.Digest); err != nil {
return err
}
} else if ref.MediaType == ispec.MediaTypeArtifactManifest {
// read manifest
var manifest ispec.Artifact
err = json.Unmarshal(resp.Body(), &manifest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't unmarshal oci reference manifest: %s", getRefManifestURL.String())
return err
}
for _, layer := range manifest.Blobs {
if err := syncBlob(sig, imageStore, localRepo, remoteRepo, layer.Digest); err != nil {
return err
}
}
}
_, err = imageStore.PutImageManifest(localRepo, ref.Digest.String(),
ref.MediaType, resp.Body())
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't upload oci reference manifest")
return err
}
}
sig.log.Info().Msgf("successfully synced oci references for repo %s digest %s", localRepo, digestStr)
return nil
}
func (sig *signaturesCopier) canSkipNotaryRefs(localRepo, digestStr string, refs ReferenceList,
) (bool, error) {
imageStore := sig.storeController.GetImageStore(localRepo)
@ -461,7 +422,7 @@ func (sig *signaturesCopier) canSkipNotaryRefs(localRepo, digestStr string, refs
return false, nil
}
sig.log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get local notary signature %s:%s manifest", localRepo, digestStr)
return false, err
@ -491,7 +452,7 @@ func (sig *signaturesCopier) canSkipOCIArtifact(localRepo, reference string, art
return false, nil
}
sig.log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get local OCI artifact %s:%s manifest", localRepo, reference)
return false, err
@ -499,7 +460,7 @@ func (sig *signaturesCopier) canSkipOCIArtifact(localRepo, reference string, art
err = json.Unmarshal(localArtifactBuf, &localArtifactManifest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't unmarshal local OCI artifact %s:%s manifest", localRepo, reference)
return false, err
@ -533,7 +494,7 @@ func (sig *signaturesCopier) canSkipCosignSignature(localRepo, digestStr string,
return false, nil
}
sig.log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get local cosign %s:%s manifest", localRepo, digestStr)
return false, err
@ -541,7 +502,7 @@ func (sig *signaturesCopier) canSkipCosignSignature(localRepo, digestStr string,
err = json.Unmarshal(localCosignManifestBuf, &localCosignManifest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't unmarshal local cosign signature %s:%s manifest", localRepo, digestStr)
return false, err
@ -572,8 +533,8 @@ func (sig *signaturesCopier) canSkipOCIRefs(localRepo, digestStr string, index i
return false, nil
}
sig.log.Error().Str("errorType", TypeOf(err)).
Err(err).Msgf("couldn't get local oci references for %s:%s manifest", localRepo, digestStr)
sig.log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get local ocireferences for %s:%s manifest", localRepo, digestStr)
return false, err
}
@ -597,26 +558,31 @@ func syncBlob(sig *signaturesCopier, imageStore storage.ImageStore, remoteRepo,
getBlobURL.Path = path.Join(getBlobURL.Path, "v2", remoteRepo, "blobs", digest.String())
getBlobURL.RawQuery = getBlobURL.Query().Encode()
resp, err := sig.client.R().SetDoNotParseResponse(true).Get(getBlobURL.String())
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, getBlobURL.String(), nil)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).Str("url", getBlobURL.String()).
return err
}
resp, err := sig.client.Do(req)
if err != nil {
sig.log.Error().Str("errorType", common.TypeOf(err)).Str("url", getBlobURL.String()).
Err(err).Msgf("couldn't get blob: %s", getBlobURL.String())
return err
}
defer resp.RawBody().Close()
defer resp.Body.Close()
if resp.IsError() {
if resp.StatusCode != http.StatusOK {
sig.log.Info().Str("url", getBlobURL.String()).Msgf("couldn't find blob from %s, status code: %d",
getBlobURL.String(), resp.StatusCode())
getBlobURL.String(), resp.StatusCode)
return zerr.ErrSyncReferrer
}
_, _, err = imageStore.FullBlobUpload(localRepo, resp.RawBody(), digest)
_, _, err = imageStore.FullBlobUpload(localRepo, resp.Body, digest)
if err != nil {
sig.log.Error().Str("errorType", TypeOf(err)).Str("digest", digest.String()).
sig.log.Error().Str("errorType", common.TypeOf(err)).Str("digest", digest.String()).
Err(err).Msg("couldn't upload blob")
return err

73
pkg/extensions/sync/sync.go
View file

@ -2,10 +2,11 @@ package sync
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"os"
goSync "sync"
"time"
@ -18,10 +19,10 @@ import (
"github.com/containers/image/v5/types"
"github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"gopkg.in/resty.v1"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/api/constants"
"zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/test"
@ -82,28 +83,17 @@ type RepoReferences struct {
}
// getUpstreamCatalog gets all repos from a registry.
func getUpstreamCatalog(client *resty.Client, upstreamURL string, log log.Logger) (catalog, error) {
func GetUpstreamCatalog(client *http.Client, upstreamURL, username, password string, log log.Logger) (catalog, error) { //nolint
var catalog catalog
registryCatalogURL := fmt.Sprintf("%s%s%s", upstreamURL, constants.RoutePrefix, constants.ExtCatalogPrefix)
resp, err := client.R().SetHeader("Content-Type", "application/json").Get(registryCatalogURL)
body, statusCode, err := common.MakeHTTPGetRequest(client, username,
password, &catalog,
registryCatalogURL, "application/json", log)
if err != nil {
log.Err(err).Msgf("couldn't query %s", registryCatalogURL)
return catalog, err
}
if resp.IsError() {
log.Error().Msgf("couldn't query %s, status code: %d, body: %s", registryCatalogURL,
resp.StatusCode(), resp.Body())
return catalog, zerr.ErrSyncMissingCatalog
}
err = json.Unmarshal(resp.Body(), &catalog)
if err != nil {
log.Err(err).Str("body", string(resp.Body())).Msg("couldn't unmarshal registry's catalog")
statusCode, body)
return catalog, err
}
@ -119,7 +109,7 @@ func imagesToCopyFromUpstream(ctx context.Context, registryName string, repoName
repoRef, err := parseRepositoryReference(fmt.Sprintf("%s/%s", registryName, repoName))
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't parse repository reference: %s", repoRef)
return imageRefs, err
@ -127,7 +117,7 @@ func imagesToCopyFromUpstream(ctx context.Context, registryName string, repoName
tags, err := getImageTags(ctx, upstreamCtx, repoRef)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't fetch tags for %s", repoRef)
return imageRefs, err
@ -228,19 +218,34 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
upstreamCtx := getUpstreamContext(&regCfg, credentials)
options := getCopyOptions(upstreamCtx, localCtx)
httpClient, registryURL, err := getHTTPClient(&regCfg, upstreamURL, credentials, log)
if !common.Contains(regCfg.URLs, upstreamURL) {
return zerr.ErrSyncInvalidUpstreamURL
}
registryURL, err := url.Parse(upstreamURL)
if err != nil {
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Str("url", upstreamURL).Msg("couldn't parse url")
return err
}
httpClient, err := common.CreateHTTPClient(*regCfg.TLSVerify, registryURL.Host, regCfg.CertDir)
if err != nil {
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("error while creating http client")
return err
}
var catalog catalog
if err = retry.RetryIfNecessary(ctx, func() error {
catalog, err = getUpstreamCatalog(httpClient, upstreamURL, log)
catalog, err = GetUpstreamCatalog(httpClient, upstreamURL, credentials.Username, credentials.Password, log)
return err
}, retryOptions); err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("error while getting upstream catalog, retrying...")
return err
@ -266,7 +271,7 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
return err
}, retryOptions); err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("error while getting images references from upstream, retrying...")
return err
@ -280,7 +285,7 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
}
}
sig := newSignaturesCopier(httpClient, *registryURL, storeController, log)
sig := newSignaturesCopier(httpClient, credentials, *registryURL, storeController, log)
for _, repoReference := range reposReferences {
upstreamRepo := repoReference.name
@ -292,7 +297,7 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
localCachePath, err := getLocalCachePath(imageStore, localRepo)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get localCachePath for %s", localRepo)
return err
@ -312,7 +317,7 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
if !isSupportedMediaType(mediaType) {
if mediaType == ispec.MediaTypeArtifactManifest {
err = sig.syncOCIArtifact(localRepo, upstreamRepo, tag, manifestBuf)
err = sig.syncOCIArtifact(localRepo, upstreamRepo, tag, manifestBuf) //nolint
if err != nil {
return err
}
@ -359,7 +364,7 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
// sync image
localImageRef, err := getLocalImageRef(localCachePath, localRepo, tag)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't obtain a valid image reference for reference %s/%s:%s",
localCachePath, localRepo, tag)
@ -373,7 +378,7 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
return err
}, retryOptions); err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("error while copying image %s to %s",
upstreamImageRef.DockerReference(), localCachePath)
@ -382,7 +387,7 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
// push from cache to repo
err = pushSyncedLocalImage(localRepo, tag, localCachePath, imageStore, log)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("error while pushing synced cached image %s",
fmt.Sprintf("%s/%s:%s", localCachePath, localRepo, tag))
@ -416,7 +421,7 @@ func syncRegistry(ctx context.Context, regCfg RegistryConfig,
return nil
}, retryOptions); err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't copy referrer for %s", upstreamImageRef.DockerReference())
}
}
@ -443,7 +448,7 @@ func getLocalContexts(log log.Logger) (*types.SystemContext, *signature.PolicyCo
policyContext, err := signature.NewPolicyContext(policy)
if err := test.Error(err); err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't create policy context")
return &types.SystemContext{}, &signature.PolicyContext{}, err
@ -463,7 +468,7 @@ func Run(ctx context.Context, cfg Config,
if cfg.CredentialsFile != "" {
credentialsFile, err = getFileCredentials(cfg.CredentialsFile)
if err != nil {
logger.Error().Str("errortype", TypeOf(err)).
logger.Error().Str("errortype", common.TypeOf(err)).
Err(err).Msgf("couldn't get registry credentials from %s", cfg.CredentialsFile)
return err
@ -513,7 +518,7 @@ func Run(ctx context.Context, cfg Config,
// first try syncing main registry
if err := syncRegistry(ctx, regCfg, upstreamURL, storeController, localCtx, policyCtx,
credentialsFile[upstreamAddr], retryOptions, logger); err != nil {
logger.Error().Str("errortype", TypeOf(err)).
logger.Error().Str("errortype", common.TypeOf(err)).
Err(err).Str("registry", upstreamURL).
Msg("sync exited with error, falling back to auxiliary registries if any")
} else {

132
pkg/extensions/sync/sync_internal_test.go
View file

@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/url"
"os"
"path"
@ -20,9 +21,9 @@ import (
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"github.com/rs/zerolog"
. "github.com/smartystreets/goconvey/convey"
"gopkg.in/resty.v1"
"zotregistry.io/zot/errors"
. "zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
@ -159,6 +160,40 @@ func TestSyncInternal(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("Verify syncRegistry func with wrong upstreamURL", t, func() {
tlsVerify := false
updateDuration := time.Microsecond
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
syncRegistryConfig := RegistryConfig{
Content: []Content{
{
Prefix: testImage,
},
},
URLs: []string{baseURL},
PollInterval: updateDuration,
TLSVerify: &tlsVerify,
CertDir: "",
}
ctx := context.Background()
log := log.NewLogger("debug", "")
metrics := monitoring.NewMetricsServer(false, log)
imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay,
false, false, log, metrics, nil, nil,
)
localCtx, policyCtx, err := getLocalContexts(log)
So(err, ShouldBeNil)
err = syncRegistry(ctx, syncRegistryConfig, "randomUpstreamURL",
storage.StoreController{DefaultStore: imageStore}, localCtx, policyCtx, Credentials{}, nil, log)
So(err, ShouldNotBeNil)
})
Convey("Verify getLocalImageRef() and getLocalCachePath()", t, func() {
log := log.Logger{Logger: zerolog.New(os.Stdout)}
metrics := monitoring.NewMetricsServer(false, log)
@ -220,12 +255,12 @@ func TestSyncInternal(t *testing.T) {
port := test.GetFreePort()
baseURL := test.GetBaseURL(port)
httpClient, registryURL, err := getHTTPClient(&syncRegistryConfig, baseURL, Credentials{}, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
So(httpClient, ShouldBeNil)
So(registryURL, ShouldBeNil)
// _, err = getUpstreamCatalog(httpClient, baseURL, log.NewLogger("debug", ""))
// So(err, ShouldNotBeNil)
httpClient, err := CreateHTTPClient(*syncRegistryConfig.TLSVerify, baseURL, "")
So(httpClient, ShouldNotBeNil)
So(err, ShouldBeNil)
registryURL, err := url.Parse(baseURL)
So(registryURL, ShouldNotBeNil)
So(err, ShouldBeNil)
})
Convey("Test getHttpClient() with bad certs", t, func() {
@ -253,40 +288,53 @@ func TestSyncInternal(t *testing.T) {
CertDir: badCertsDir,
}
httpClient, _, err := getHTTPClient(&syncRegistryConfig, baseURL, Credentials{}, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
So(httpClient, ShouldBeNil)
httpClient, err := CreateHTTPClient(*syncRegistryConfig.TLSVerify, baseURL, "")
So(httpClient, ShouldNotBeNil)
So(err, ShouldBeNil)
registryURL, err := url.Parse(baseURL)
So(registryURL, ShouldNotBeNil)
So(err, ShouldBeNil)
syncRegistryConfig.CertDir = "/path/to/invalid/cert"
httpClient, _, err = getHTTPClient(&syncRegistryConfig, baseURL, Credentials{}, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
So(httpClient, ShouldBeNil)
httpClient, err = CreateHTTPClient(*syncRegistryConfig.TLSVerify, baseURL, "")
So(httpClient, ShouldNotBeNil)
So(err, ShouldBeNil)
registryURL, err = url.Parse(baseURL)
So(registryURL, ShouldNotBeNil)
So(err, ShouldBeNil)
syncRegistryConfig.CertDir = ""
syncRegistryConfig.URLs = []string{baseSecureURL}
httpClient, registryURL, err := getHTTPClient(&syncRegistryConfig, baseSecureURL,
Credentials{}, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
httpClient, err = CreateHTTPClient(*syncRegistryConfig.TLSVerify, baseSecureURL, "")
So(httpClient, ShouldNotBeNil)
So(registryURL.String(), ShouldEqual, baseSecureURL)
So(err, ShouldBeNil)
registryURL, err = url.Parse(baseSecureURL)
So(registryURL, ShouldNotBeNil)
So(err, ShouldBeNil)
_, err = getUpstreamCatalog(httpClient, baseURL, log.NewLogger("debug", ""))
_, err = GetUpstreamCatalog(httpClient, baseURL, "", "", log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
_, err = getUpstreamCatalog(httpClient, "http://invalid:5000", log.NewLogger("debug", ""))
_, err = GetUpstreamCatalog(httpClient, "http://invalid:5000", "", "", log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
syncRegistryConfig.URLs = []string{test.BaseURL}
httpClient, _, err = getHTTPClient(&syncRegistryConfig, baseSecureURL, Credentials{}, log.NewLogger("debug", ""))
httpClient, err = CreateHTTPClient(*syncRegistryConfig.TLSVerify, test.BaseURL, "")
So(httpClient, ShouldNotBeNil)
So(err, ShouldBeNil)
registryURL, err = url.Parse(test.BaseURL) //nolint
So(registryURL, ShouldBeNil)
So(err, ShouldNotBeNil)
So(httpClient, ShouldBeNil)
syncRegistryConfig.URLs = []string{"%"}
httpClient, _, err = getHTTPClient(&syncRegistryConfig, "%", Credentials{}, log.NewLogger("debug", ""))
httpClient, err = CreateHTTPClient(*syncRegistryConfig.TLSVerify, test.BaseURL, "")
So(httpClient, ShouldNotBeNil)
So(err, ShouldBeNil)
registryURL, err = url.Parse(test.BaseURL) //nolint
So(registryURL, ShouldBeNil)
So(err, ShouldNotBeNil)
So(httpClient, ShouldBeNil)
})
Convey("Test imagesToCopyFromUpstream()", t, func() {
@ -304,7 +352,7 @@ func TestSyncInternal(t *testing.T) {
Convey("Test signatures", t, func() {
log := log.NewLogger("debug", "")
client := resty.New()
client := &http.Client{}
regURL, err := url.Parse("http://zot")
So(err, ShouldBeNil)
@ -327,7 +375,7 @@ func TestSyncInternal(t *testing.T) {
false, false, log, metrics, nil, nil,
)
sig := newSignaturesCopier(client, *regURL, storage.StoreController{DefaultStore: imageStore}, log)
sig := newSignaturesCopier(client, Credentials{}, *regURL, storage.StoreController{DefaultStore: imageStore}, log)
err = sig.syncCosignSignature(testImage, testImage, testImageTag, &ispec.Manifest{})
So(err, ShouldNotBeNil)
@ -335,39 +383,8 @@ func TestSyncInternal(t *testing.T) {
err = sig.syncCosignSignature(testImage, testImage, testImageTag, &manifest)
So(err, ShouldNotBeNil)
err = sig.syncOCIArtifact(testImage, testImage, testImageTag, nil)
So(err, ShouldNotBeNil)
ociArtifactBuf, err := json.Marshal(ispec.Artifact{})
So(err, ShouldBeNil)
err = sig.syncOCIArtifact(testImage, testImage, testImageTag, ociArtifactBuf)
So(err, ShouldBeNil)
ociArtifactBuf, err = json.Marshal(
ispec.Artifact{Blobs: []ispec.Descriptor{{Digest: "fakeDigest"}}})
So(err, ShouldBeNil)
err = sig.syncOCIArtifact(testImage, testImage, testImageTag, ociArtifactBuf)
So(err, ShouldNotBeNil)
err = sig.syncNotaryRefs(testImage, testImage, "invalidDigest", ReferenceList{[]artifactspec.Descriptor{ref}})
So(err, ShouldNotBeNil)
Convey("Trigger unmarshal error on canSkipOCIArtifact", func() {
sig := newSignaturesCopier(client, *regURL, storage.StoreController{DefaultStore: mocks.MockedImageStore{
GetImageManifestFn: func(repo, reference string) ([]byte, godigest.Digest, string, error) {
result := []byte{}
digest := godigest.FromBytes(result)
return result, digest, "", nil
},
}}, log)
skip, err := sig.canSkipOCIArtifact(testImage, testImageTag, ispec.Artifact{})
So(skip, ShouldBeFalse)
So(err, ShouldNotBeNil)
})
})
Convey("Test canSkipImage()", t, func() {
@ -408,7 +425,8 @@ func TestSyncInternal(t *testing.T) {
So(err, ShouldBeNil)
So(regURL, ShouldNotBeNil)
sig := newSignaturesCopier(resty.New(), *regURL, storage.StoreController{DefaultStore: imageStore}, log)
client := &http.Client{}
sig := newSignaturesCopier(client, Credentials{}, *regURL, storage.StoreController{DefaultStore: imageStore}, log)
canBeSkipped, err = sig.canSkipNotaryRefs(testImage, testImageManifestDigest.String(), refs)
So(err, ShouldBeNil)

126
pkg/extensions/sync/sync_test.go
View file

@ -40,8 +40,10 @@ import (
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/api/constants"
"zotregistry.io/zot/pkg/cli"
"zotregistry.io/zot/pkg/common"
extconf "zotregistry.io/zot/pkg/extensions/config"
"zotregistry.io/zot/pkg/extensions/sync"
logger "zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/storage/local"
"zotregistry.io/zot/pkg/test"
@ -947,7 +949,7 @@ func TestMandatoryAnnotations(t *testing.T) {
}()
// give it time to set up sync
time.Sleep(3 * time.Second)
time.Sleep(5 * time.Second)
resp, err := destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/0.0.1")
So(err, ShouldBeNil)
@ -1661,6 +1663,57 @@ func TestNotSemver(t *testing.T) {
})
}
func TestErrorOnCatalog(t *testing.T) {
Convey("Verify error on catalog", t, func() {
updateDuration, _ := time.ParseDuration("1h")
sctlr, srcBaseURL, destDir, _, _ := startUpstreamServer(t, true, false)
defer func() {
sctlr.Shutdown()
}()
err := os.Chmod(destDir, 0o000)
So(err, ShouldBeNil)
tlsVerify := false
syncRegistryConfig := sync.RegistryConfig{
Content: []sync.Content{
{
Prefix: testImage,
},
},
URLs: []string{srcBaseURL},
PollInterval: updateDuration,
TLSVerify: &tlsVerify,
OnDemand: true,
}
defaultVal := true
syncConfig := &sync.Config{
Enable: &defaultVal,
Registries: []sync.RegistryConfig{syncRegistryConfig},
}
dctlr, _, _, _ := startDownstreamServer(t, false, syncConfig)
httpClient, err := common.CreateHTTPClient(*syncRegistryConfig.TLSVerify, "localhost", "")
So(httpClient, ShouldNotBeNil)
So(err, ShouldBeNil)
_, err = sync.GetUpstreamCatalog(httpClient, srcBaseURL, "", "", logger.NewLogger("", ""))
So(err, ShouldNotBeNil)
err = os.Chmod(destDir, 0o755)
So(err, ShouldBeNil)
defer func() {
dctlr.Shutdown()
}()
})
}
func TestInvalidCerts(t *testing.T) {
Convey("Verify sync with bad certs", t, func() {
updateDuration, _ := time.ParseDuration("1h")
@ -1703,12 +1756,79 @@ func TestInvalidCerts(t *testing.T) {
panic(err)
}
var tlsVerify bool
tlsVerify := true
syncRegistryConfig := sync.RegistryConfig{
Content: []sync.Content{
{
Prefix: "",
Prefix: testImage,
},
},
URLs: []string{srcBaseURL},
PollInterval: updateDuration,
TLSVerify: &tlsVerify,
CertDir: clientCertDir,
OnDemand: true,
}
defaultVal := true
syncConfig := &sync.Config{
Enable: &defaultVal,
Registries: []sync.RegistryConfig{syncRegistryConfig},
}
dctlr, destBaseURL, _, destClient := startDownstreamServer(t, false, syncConfig)
resp, err := destClient.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 404)
defer func() {
dctlr.Shutdown()
}()
})
}
func TestCertsWithWrongPerms(t *testing.T) {
Convey("Verify sync with wrong permissions on certs", t, func() {
updateDuration, _ := time.ParseDuration("1h")
sctlr, srcBaseURL, _, _, _ := startUpstreamServer(t, true, false)
defer func() {
sctlr.Shutdown()
}()
// copy client certs, use them in sync config
clientCertDir := t.TempDir()
destFilePath := path.Join(clientCertDir, "ca.crt")
err := copyFile(CACert, destFilePath)
if err != nil {
panic(err)
}
err = os.Chmod(destFilePath, 0o000)
So(err, ShouldBeNil)
destFilePath = path.Join(clientCertDir, "client.cert")
err = copyFile(ClientCert, destFilePath)
if err != nil {
panic(err)
}
destFilePath = path.Join(clientCertDir, "client.key")
err = copyFile(ClientKey, destFilePath)
if err != nil {
panic(err)
}
tlsVerify := true
syncRegistryConfig := sync.RegistryConfig{
Content: []sync.Content{
{
Prefix: testImage,
},
},
URLs: []string{srcBaseURL},

94
pkg/extensions/sync/utils.go
View file

@ -2,12 +2,9 @@ package sync
import (
"context"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"path"
"regexp"
@ -26,7 +23,6 @@ import (
ispec "github.com/opencontainers/image-spec/specs-go/v1"
artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1"
"github.com/sigstore/cosign/pkg/oci/static"
"gopkg.in/resty.v1"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/common"
@ -41,10 +37,6 @@ type ReferenceList struct {
References []artifactspec.Descriptor `json:"references"`
}
func TypeOf(v interface{}) string {
return fmt.Sprintf("%T", v)
}
// getTagFromRef returns a tagged reference from an image reference.
func getTagFromRef(ref types.ImageReference, log log.Logger) reference.Tagged {
tagged, isTagged := ref.DockerReference().(reference.Tagged)
@ -144,7 +136,7 @@ func filterRepos(repos []string, contentList []Content, log log.Logger) map[int]
matched, err := glob.Match(prefix, repo)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Str("pattern",
prefix).Msg("error while parsing glob pattern, skipping it...")
@ -268,68 +260,6 @@ func getFileCredentials(filepath string) (CredentialsFile, error) {
return creds, nil
}
func getHTTPClient(regCfg *RegistryConfig, upstreamURL string, credentials Credentials,
log log.Logger,
) (*resty.Client, *url.URL, error) {
client := resty.New()
if !common.Contains(regCfg.URLs, upstreamURL) {
return nil, nil, zerr.ErrSyncInvalidUpstreamURL
}
registryURL, err := url.Parse(upstreamURL)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Str("url", upstreamURL).Msg("couldn't parse url")
return nil, nil, err
}
if regCfg.CertDir != "" {
log.Debug().Msgf("sync: using certs directory: %s", regCfg.CertDir)
clientCert := path.Join(regCfg.CertDir, "client.cert")
clientKey := path.Join(regCfg.CertDir, "client.key")
caCertPath := path.Join(regCfg.CertDir, "ca.crt")
caCert, err := os.ReadFile(caCertPath)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't read CA certificate")
return nil, nil, err
}
caCertPool := x509.NewCertPool()
caCertPool.AppendCertsFromPEM(caCert)
client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12})
cert, err := tls.LoadX509KeyPair(clientCert, clientKey)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
Err(err).Msg("couldn't read certificates key pairs")
return nil, nil, err
}
client.SetCertificates(cert)
}
//nolint: gosec
if regCfg.TLSVerify != nil && !*regCfg.TLSVerify && registryURL.Scheme == "https" {
client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
}
if credentials.Username != "" && credentials.Password != "" {
log.Debug().Msgf("sync: using basic auth")
client.SetBasicAuth(credentials.Username, credentials.Password)
}
client.SetRedirectPolicy(resty.FlexibleRedirectPolicy(httpMaxRedirectsCount))
return client, registryURL, nil
}
func pushSyncedLocalImage(localRepo, reference, localCachePath string,
imageStore storage.ImageStore, log log.Logger,
) error {
@ -344,7 +274,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
manifestContent, _, mediaType, err := cacheImageStore.GetImageManifest(localRepo, reference)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Str("dir", path.Join(cacheImageStore.RootDir(), localRepo)).
Msgf("couldn't find %s manifest", reference)
@ -356,7 +286,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
case ispec.MediaTypeImageManifest:
if err := copyManifest(localRepo, manifestContent, reference, cacheImageStore, imageStore, log); err != nil {
if errors.Is(err, zerr.ErrImageLintAnnotations) {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't upload manifest because of missing annotations")
return nil
@ -369,7 +299,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
var indexManifest ispec.Index
if err := json.Unmarshal(manifestContent, &indexManifest); err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Str("dir", path.Join(cacheImageStore.RootDir(), localRepo)).
Msg("invalid JSON")
@ -382,7 +312,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
cacheImageStore.RUnlock(&lockLatency)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Str("dir", path.Join(cacheImageStore.RootDir(), localRepo)).Str("digest", manifest.Digest.String()).
Msg("couldn't find manifest which is part of an image index")
@ -392,7 +322,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
if err := copyManifest(localRepo, manifestBuf, manifest.Digest.String(),
cacheImageStore, imageStore, log); err != nil {
if errors.Is(err, zerr.ErrImageLintAnnotations) {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't upload manifest because of missing annotations")
return nil
@ -404,7 +334,7 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string,
_, err = imageStore.PutImageManifest(localRepo, reference, mediaType, manifestContent)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't upload manifest")
return err
@ -422,7 +352,7 @@ func copyManifest(localRepo string, manifestContent []byte, reference string,
var err error
if err := json.Unmarshal(manifestContent, &manifest); err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Str("dir", path.Join(cacheImageStore.RootDir(), localRepo)).
Msg("invalid JSON")
@ -446,7 +376,7 @@ func copyManifest(localRepo string, manifestContent []byte, reference string,
_, err = imageStore.PutImageManifest(localRepo, reference,
ispec.MediaTypeImageManifest, manifestContent)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msg("couldn't upload manifest")
return err
@ -466,7 +396,7 @@ func copyBlob(localRepo string, blobDigest godigest.Digest, blobMediaType string
blobReadCloser, _, err := souceImageStore.GetBlob(localRepo, blobDigest, blobMediaType)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).Err(err).
log.Error().Str("errorType", common.TypeOf(err)).Err(err).
Str("dir", path.Join(souceImageStore.RootDir(), localRepo)).
Str("blob digest", blobDigest.String()).Str("media type", blobMediaType).
Msg("couldn't read blob")
@ -477,7 +407,7 @@ func copyBlob(localRepo string, blobDigest godigest.Digest, blobMediaType string
_, _, err = destinationImageStore.FullBlobUpload(localRepo, blobReadCloser, blobDigest)
if err != nil {
log.Error().Str("errorType", TypeOf(err)).Err(err).
log.Error().Str("errorType", common.TypeOf(err)).Err(err).
Str("blob digest", blobDigest.String()).Str("media type", blobMediaType).
Msg("couldn't upload blob")
}
@ -585,7 +515,7 @@ func canSkipImage(repo, tag string, digest godigest.Digest, imageStore storage.I
return false, nil
}
log.Error().Str("errorType", TypeOf(err)).
log.Error().Str("errorType", common.TypeOf(err)).
Err(err).Msgf("couldn't get local image %s:%s manifest", repo, tag)
return false, err

28
pkg/storage/local/local.go
View file

@ -6,13 +6,11 @@ import (
"errors"
"fmt"
"io"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
"sync"
"syscall"
"time"
"unicode/utf8"
@ -66,7 +64,7 @@ func (is *ImageStoreLocal) RootDir() string {
}
func (is *ImageStoreLocal) DirExists(d string) bool {
return DirExists(d)
return common.DirExists(d)
}
// NewImageStore returns a new image store backed by a file storage.
@ -1670,30 +1668,6 @@ func isBlobOlderThan(imgStore *ImageStoreLocal, repo string, digest godigest.Dig
return true, nil
}
func DirExists(d string) bool {
if !utf8.ValidString(d) {
return false
}
fileInfo, err := os.Stat(d)
if err != nil {
if e, ok := err.(*fs.PathError); ok && errors.Is(e.Err, syscall.ENAMETOOLONG) || //nolint: errorlint
errors.Is(e.Err, syscall.EINVAL) {
return false
}
}
if err != nil && os.IsNotExist(err) {
return false
}
if !fileInfo.IsDir() {
return false
}
return true
}
func (is *ImageStoreLocal) gcRepo(repo string) error {
dir := path.Join(is.RootDir(), repo)

9
pkg/storage/local/local_test.go
View file

@ -25,6 +25,7 @@ import (
. "github.com/smartystreets/goconvey/convey"
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/common"
"zotregistry.io/zot/pkg/extensions/monitoring"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/storage"
@ -482,7 +483,7 @@ func FuzzTestDeleteImageManifest(f *testing.F) {
func FuzzDirExists(f *testing.F) {
f.Fuzz(func(t *testing.T, data string) {
_ = local.DirExists(data)
_ = common.DirExists(data)
})
}
@ -1602,7 +1603,7 @@ func TestNegativeCases(t *testing.T) {
panic(err)
}
ok := local.DirExists(filePath)
ok := common.DirExists(filePath)
So(ok, ShouldBeFalse)
})
@ -1610,7 +1611,7 @@ func TestNegativeCases(t *testing.T) {
dir := t.TempDir()
filePath := path.Join(dir, "hi \255")
ok := local.DirExists(filePath)
ok := common.DirExists(filePath)
So(ok, ShouldBeFalse)
})
@ -1623,7 +1624,7 @@ func TestNegativeCases(t *testing.T) {
}
}
path := builder.String()
ok := local.DirExists(path)
ok := common.DirExists(path)
So(ok, ShouldBeFalse)
})
}