diff --git a/go.mod b/go.mod index d9d118e7..97043934 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,6 @@ require ( github.com/bmatcuk/doublestar/v4 v4.6.1 github.com/briandowns/spinner v1.23.1 github.com/chartmuseum/auth v0.5.0 - github.com/containers/common v0.60.1 github.com/containers/image/v5 v5.32.1 github.com/dchest/siphash v1.2.3 github.com/didip/tollbooth/v7 v7.0.2 @@ -48,6 +47,7 @@ require ( github.com/project-zot/mockoidc v0.0.0-20240610203808-d69d9e02020a github.com/prometheus/client_golang v1.20.0 github.com/prometheus/client_model v0.6.1 + github.com/regclient/regclient v0.7.1 github.com/rs/zerolog v1.33.0 github.com/sigstore/cosign/v2 v2.4.0 github.com/sigstore/sigstore v1.8.8 @@ -213,6 +213,7 @@ require ( github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect github.com/emicklei/proto v1.12.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect @@ -262,6 +263,7 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/licenseclassifier/v2 v2.0.0 // indirect + github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect github.com/google/s2a-go v0.1.8 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/wire v0.6.0 // indirect @@ -294,7 +296,6 @@ require ( github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.17.9 // indirect - github.com/klauspost/pgzip v1.2.6 // indirect github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f // indirect github.com/knqyf263/go-deb-version v0.0.0-20230223133812-3ed183d23422 // indirect github.com/knqyf263/go-rpm-version v0.0.0-20220614171824-631e686d1075 // indirect @@ -322,7 +323,6 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mattn/go-shellwords v1.0.12 // indirect - github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/microsoft/go-rustaudit v0.0.0-20220808201409-204dfee52032 // indirect github.com/miekg/pkcs11 v1.1.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect @@ -353,6 +353,8 @@ require ( github.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/oleiade/reflections v1.0.1 // indirect + github.com/onsi/ginkgo/v2 v2.20.0 // indirect + github.com/onsi/gomega v1.34.1 // indirect github.com/open-policy-agent/opa v0.67.0 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect github.com/opencontainers/selinux v1.11.0 // indirect @@ -368,7 +370,6 @@ require ( github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/proglottis/gpgme v0.1.3 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/protocolbuffers/txtpbfmt v0.0.0-20231025115547-084445ff1adf // indirect @@ -404,11 +405,9 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spiffe/go-spiffe/v2 v2.3.0 // indirect - github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/swaggo/files v1.0.1 // indirect - github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/tchap/go-patricia/v2 v2.3.1 // indirect github.com/tetratelabs/wazero v1.7.3 // indirect @@ -422,7 +421,6 @@ require ( github.com/ulikunitz/xz v0.5.12 // indirect github.com/urfave/cli/v2 v2.27.3 // indirect github.com/vbatts/tar-split v0.11.5 // indirect - github.com/vbauerster/mpb/v8 v8.7.5 // indirect github.com/xanzy/go-gitlab v0.107.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect @@ -437,7 +435,6 @@ require ( github.com/zitadel/logging v0.6.0 // indirect github.com/zitadel/schema v1.3.0 // indirect go.mongodb.org/mongo-driver v1.16.0 // indirect - go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/bridges/prometheus v0.53.0 // indirect go.opentelemetry.io/contrib/exporters/autoexport v0.53.0 // indirect diff --git a/go.sum b/go.sum index af1a9ab7..7164a753 100644 --- a/go.sum +++ b/go.sum @@ -549,8 +549,6 @@ github.com/containerd/ttrpc v1.2.5 h1:IFckT1EFQoFBMG4c3sMdT8EP3/aKfumK1msY+Ze4oL github.com/containerd/ttrpc v1.2.5/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= github.com/containerd/typeurl/v2 v2.2.0 h1:6NBDbQzr7I5LHgp34xAXYF5DOTQDn05X58lsPEmzLso= github.com/containerd/typeurl/v2 v2.2.0/go.mod h1:8XOOxnyatxSWuG8OfsZXVnAF4iZfedjS/8UHSPJnX4g= -github.com/containers/common v0.60.1 h1:hMJNKfDxfXY91zD7mr4t/Ybe8JbAsTq5nkrUaCqTKsA= -github.com/containers/common v0.60.1/go.mod h1:tB0DRxznmHviECVHnqgWbl+8AVCSMZLA8qe7+U7KD6k= github.com/containers/image/v5 v5.32.1 h1:fVa7GxRC4BCPGsfSRs4JY12WyeY26SUYQ0NuANaCFrI= github.com/containers/image/v5 v5.32.1/go.mod h1:v1l73VeMugfj/QtKI+jhYbwnwFCFnNGckvbST3rQ5Hk= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= @@ -1304,6 +1302,8 @@ github.com/redis/go-redis/extra/redisotel/v9 v9.0.5 h1:EfpWLLCyXw8PSM2/XNJLjI3Pb github.com/redis/go-redis/extra/redisotel/v9 v9.0.5/go.mod h1:WZjPDy7VNzn77AAfnAfVjZNvfJTYfPetfZk5yoSTLaQ= github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8= github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= +github.com/regclient/regclient v0.7.1 h1:qEsJrTmZd98fZKjueAbrZCSNGU+ifnr6xjlSAs3WOPs= +github.com/regclient/regclient v0.7.1/go.mod h1:+w/BFtJuw0h0nzIw/z2+1FuA2/dVXBzDq4rYmziJpMc= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -1419,8 +1419,6 @@ github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= github.com/spiffe/go-spiffe/v2 v2.3.0 h1:g2jYNb/PDMB8I7mBGL2Zuq/Ur6hUhoroxGQFyD6tTj8= github.com/spiffe/go-spiffe/v2 v2.3.0/go.mod h1:Oxsaio7DBgSNqhAO9i/9tLClaVlfRok7zvJnTV8ZyIY= -github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 h1:pnnLyeX7o/5aX8qUQ69P/mLojDqwda8hFOCBTmP/6hw= -github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6/go.mod h1:39R/xuhNgVhi+K0/zst4TLrJrVmbm6LVgl4A0+ZFS5M= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -1450,6 +1448,8 @@ github.com/swaggo/http-swagger v1.3.4 h1:q7t/XLx0n15H1Q9/tk3Y9L4n210XzJF5WtnDX64 github.com/swaggo/http-swagger v1.3.4/go.mod h1:9dAh0unqMBAlbp1uE2Uc2mQTxNMU/ha4UbucIg1MFkQ= github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/tchap/go-patricia/v2 v2.3.1 h1:6rQp39lgIYZ+MHmdEq4xzuk1t7OdC35z/xm0BGhTkes= @@ -1489,8 +1489,6 @@ github.com/urfave/cli/v2 v2.27.3 h1:/POWahRmdh7uztQ3CYnaDddk0Rm90PyOgIxgW2rr41M= github.com/urfave/cli/v2 v2.27.3/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ= github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= -github.com/vbauerster/mpb/v8 v8.7.5 h1:hUF3zaNsuaBBwzEFoCvfuX3cpesQXZC0Phm/JcHZQ+c= -github.com/vbauerster/mpb/v8 v8.7.5/go.mod h1:bRCnR7K+mj5WXKsy0NWB6Or+wctYGvVwKn6huwvxKa0= github.com/vektah/gqlparser/v2 v2.5.16 h1:1gcmLTvs3JLKXckwCwlUagVn/IlV2bwqle0vJ0vy5p8= github.com/vektah/gqlparser/v2 v2.5.16/go.mod h1:1lz1OeCqgQbQepsGxPVywrjdBHW2T08PUS3pJqepRww= github.com/veraison/go-cose v1.2.1 h1:Gj4x20D0YP79J2+cK3anjGEMwIkg2xX+TKVVGUXwNAc= @@ -1558,8 +1556,6 @@ go.etcd.io/bbolt v1.3.10 h1:+BqfJTcCzTItrop8mq/lbzL8wSGtj94UO/3U31shqG0= go.etcd.io/bbolt v1.3.10/go.mod h1:bK3UQLPJZly7IlNmV7uVHJDxfe5aK9Ll93e/74Y9oEQ= go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4= go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= -go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak= -go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= diff --git a/pkg/extensions/sync/destination.go b/pkg/extensions/sync/destination.go index 1b871459..ec06af62 100644 --- a/pkg/extensions/sync/destination.go +++ b/pkg/extensions/sync/destination.go @@ -89,7 +89,7 @@ func (registry *DestinationRegistry) GetImageReference(repo, reference string) ( func (registry *DestinationRegistry) CommitAll(repo string, imageReference ref.Ref) error { imageStore := registry.storeController.GetImageStore(repo) - tempImageStore := getImageStoreFromImageReference(imageReference, repo, reference, registry.log) + tempImageStore := getImageStoreFromImageReference(repo, imageReference, registry.log) defer os.RemoveAll(tempImageStore.RootDir()) @@ -288,23 +288,10 @@ func (registry *DestinationRegistry) copyBlob(repo string, blobDigest godigest.D } // use only with local imageReferences. -func getImageStoreFromImageReference(imageReference types.ImageReference, repo, reference string, log log.Logger, -) storageTypes.ImageStore { - tmpRootDir := getTempRootDirFromImageReference(imageReference, repo, reference) +func getImageStoreFromImageReference(repo string, imageReference ref.Ref, log log.Logger) storageTypes.ImageStore { + sessionRootDir := strings.TrimSuffix(imageReference.Path, repo) - return getImageStore(tmpRootDir, log) -} - -func getTempRootDirFromImageReference(imageReference types.ImageReference, repo, reference string) string { - var tmpRootDir string - - if strings.HasSuffix(imageReference.StringWithinTransport(), reference) { - tmpRootDir = strings.ReplaceAll(imageReference.StringWithinTransport(), fmt.Sprintf("%s:%s", repo, reference), "") - } else { - tmpRootDir = strings.ReplaceAll(imageReference.StringWithinTransport(), repo+":", "") - } - - return tmpRootDir + return getImageStore(sessionRootDir, log) } func getImageStore(rootDir string, log log.Logger) storageTypes.ImageStore { diff --git a/pkg/extensions/sync/httpclient/client.go b/pkg/extensions/sync/httpclient/client.go deleted file mode 100644 index ea248257..00000000 --- a/pkg/extensions/sync/httpclient/client.go +++ /dev/null @@ -1,482 +0,0 @@ -package client - -import ( - "context" - "encoding/json" - "errors" - "io" - "net/http" - "net/url" - "path/filepath" - "strings" - "sync" - "time" - - zerr "zotregistry.dev/zot/errors" - "zotregistry.dev/zot/pkg/common" - "zotregistry.dev/zot/pkg/log" -) - -const ( - minimumTokenLifetimeSeconds = 60 // in seconds - pingTimeout = 5 * time.Second - // tokenBuffer is used to renew a token before it actually expires - // to account for the time to process requests on the server. - tokenBuffer = 5 * time.Second -) - -type authType int - -const ( - noneAuth authType = iota - basicAuth - tokenAuth -) - -type challengeParams struct { - realm string - service string - scope string - err string -} - -type bearerToken struct { - Token string `json:"token"` //nolint: tagliatelle - AccessToken string `json:"access_token"` //nolint: tagliatelle - ExpiresIn int `json:"expires_in"` //nolint: tagliatelle - IssuedAt time.Time `json:"issued_at"` //nolint: tagliatelle - expirationTime time.Time -} - -func (token *bearerToken) isExpired() bool { - // use tokenBuffer to expire it a bit earlier - return time.Now().After(token.expirationTime.Add(-1 * tokenBuffer)) -} - -type Config struct { - URL string - Username string - Password string - CertDir string - TLSVerify bool -} - -type Client struct { - config *Config - client *http.Client - url *url.URL - authType authType - cache *TokenCache - lock *sync.RWMutex - log log.Logger -} - -func New(config Config, log log.Logger) (*Client, error) { - client := &Client{log: log, lock: new(sync.RWMutex)} - - client.cache = NewTokenCache() - - if err := client.SetConfig(config); err != nil { - return nil, err - } - - return client, nil -} - -func (httpClient *Client) GetConfig() *Config { - httpClient.lock.RLock() - defer httpClient.lock.RUnlock() - - return httpClient.config -} - -func (httpClient *Client) GetHostname() string { - httpClient.lock.RLock() - defer httpClient.lock.RUnlock() - - return httpClient.url.Host -} - -func (httpClient *Client) GetBaseURL() string { - httpClient.lock.RLock() - defer httpClient.lock.RUnlock() - - return httpClient.url.String() -} - -func (httpClient *Client) SetConfig(config Config) error { - httpClient.lock.Lock() - defer httpClient.lock.Unlock() - - clientURL, err := url.Parse(config.URL) - if err != nil { - return err - } - - httpClient.url = clientURL - - clientOpts := common.HTTPClientOptions{ - // we want TLS enabled when verifyTLS is true. - TLSEnabled: config.TLSVerify, - VerifyTLS: config.TLSVerify, - Host: clientURL.Host, - } - - if config.CertDir != "" { - // only configure the default cert file names if the CertDir was specified. - clientOpts.CertOptions = common.HTTPClientCertOptions{ - // filepath is the recommended library to use for joining paths - // taking into account the underlying OS. - // ref: https://stackoverflow.com/a/39182128 - ClientCertFile: filepath.Join(config.CertDir, common.ClientCertFilename), - ClientKeyFile: filepath.Join(config.CertDir, common.ClientKeyFilename), - RootCaCertFile: filepath.Join(config.CertDir, common.CaCertFilename), - } - } - - client, err := common.CreateHTTPClient(&clientOpts) - if err != nil { - return err - } - - httpClient.client = client - httpClient.config = &config - - return nil -} - -func (httpClient *Client) Ping() bool { - httpClient.lock.Lock() - defer httpClient.lock.Unlock() - - pingURL := *httpClient.url - - pingURL = *pingURL.JoinPath("/v2/") - - // for the ping function we want to timeout fast - ctx, cancel := context.WithTimeout(context.Background(), pingTimeout) - defer cancel() - - //nolint: bodyclose - resp, _, err := httpClient.get(ctx, pingURL.String(), false) - if err != nil { - return false - } - - httpClient.getAuthType(resp) - - if resp.StatusCode >= http.StatusOK && resp.StatusCode <= http.StatusForbidden { - return true - } - - httpClient.log.Error().Str("url", pingURL.String()).Int("statusCode", resp.StatusCode). - Str("component", "sync").Msg("failed to ping registry") - - return false -} - -func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interface{}, mediaType string, - route ...string, -) ([]byte, string, int, error) { - httpClient.lock.RLock() - defer httpClient.lock.RUnlock() - - var namespace string - - url := *httpClient.url - for idx, path := range route { - url = *url.JoinPath(path) - - // we know that the second route argument is always the repo name. - // need it for caching tokens, it's not used in requests made to authz server. - if idx == 1 { - namespace = path - } - } - - url.RawQuery = url.Query().Encode() - //nolint: bodyclose,contextcheck - resp, body, err := httpClient.makeAndDoRequest(http.MethodGet, mediaType, namespace, url.String()) - if err != nil { - httpClient.log.Error().Err(err).Str("url", url.String()).Str("component", "sync"). - Str("errorType", common.TypeOf(err)). - Msg("failed to make request") - - return nil, "", -1, err - } - - if resp.StatusCode != http.StatusOK { - return nil, "", resp.StatusCode, errors.New(string(body)) //nolint:goerr113 - } - - // read blob - if len(body) > 0 { - err = json.Unmarshal(body, &resultPtr) - } - - return body, resp.Header.Get("Content-Type"), resp.StatusCode, err -} - -func (httpClient *Client) getAuthType(resp *http.Response) { - authHeader := resp.Header.Get("www-authenticate") - - authHeaderLower := strings.ToLower(authHeader) - - //nolint: gocritic - if strings.Contains(authHeaderLower, "bearer") { - httpClient.authType = tokenAuth - } else if strings.Contains(authHeaderLower, "basic") { - httpClient.authType = basicAuth - } else { - httpClient.authType = noneAuth - } -} - -func (httpClient *Client) setupAuth(req *http.Request, namespace string) error { - if httpClient.authType == tokenAuth { - token, err := httpClient.getToken(req.URL.String(), namespace) - if err != nil { - httpClient.log.Error().Err(err).Str("url", req.URL.String()).Str("component", "sync"). - Str("errorType", common.TypeOf(err)). - Msg("failed to get token from authorization realm") - - return err - } - - req.Header.Set("Authorization", "Bearer "+token.Token) - } else if httpClient.authType == basicAuth { - req.SetBasicAuth(httpClient.config.Username, httpClient.config.Password) - } - - return nil -} - -func (httpClient *Client) get(ctx context.Context, url string, setAuth bool) (*http.Response, []byte, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) //nolint - if err != nil { - return nil, nil, err - } - - if setAuth && httpClient.config.Username != "" && httpClient.config.Password != "" { - req.SetBasicAuth(httpClient.config.Username, httpClient.config.Password) - } - - return httpClient.doRequest(req) -} - -func (httpClient *Client) doRequest(req *http.Request) (*http.Response, []byte, error) { - resp, err := httpClient.client.Do(req) - if err != nil { - httpClient.log.Error().Err(err).Str("url", req.URL.String()).Str("component", "sync"). - Str("errorType", common.TypeOf(err)). - Msg("failed to make request") - - return nil, nil, err - } - - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - httpClient.log.Error().Err(err).Str("url", req.URL.String()). - Str("errorType", common.TypeOf(err)). - Msg("failed to read body") - - return nil, nil, err - } - - return resp, body, nil -} - -func (httpClient *Client) makeAndDoRequest(method, mediaType, namespace, urlStr string, -) (*http.Response, []byte, error) { - req, err := http.NewRequest(method, urlStr, nil) //nolint - if err != nil { - return nil, nil, err - } - - if err := httpClient.setupAuth(req, namespace); err != nil { - return nil, nil, err - } - - if mediaType != "" { - req.Header.Set("Accept", mediaType) - } - - resp, body, err := httpClient.doRequest(req) - if err != nil { - return nil, nil, err - } - - // let's retry one time if we get an insufficient_scope error - if ok, challengeParams := needsRetryWithUpdatedScope(err, resp); ok { - var tokenURL *url.URL - - var token *bearerToken - - tokenURL, err = getTokenURLFromChallengeParams(challengeParams, httpClient.config.Username) - if err != nil { - return nil, nil, err - } - - token, err = httpClient.getTokenFromURL(tokenURL.String(), namespace) - if err != nil { - return nil, nil, err - } - - req.Header.Set("Authorization", "Bearer "+token.Token) - - resp, body, err = httpClient.doRequest(req) - } - - return resp, body, err -} - -func (httpClient *Client) getTokenFromURL(urlStr, namespace string) (*bearerToken, error) { - //nolint: bodyclose - resp, body, err := httpClient.get(context.Background(), urlStr, true) - if err != nil { - return nil, err - } - - if resp.StatusCode != http.StatusOK { - return nil, zerr.ErrUnauthorizedAccess - } - - token, err := newBearerToken(body) - if err != nil { - return nil, err - } - - // cache it - httpClient.cache.Set(namespace, token) - - return token, nil -} - -// Gets bearer token from Authorization realm. -func (httpClient *Client) getToken(urlStr, namespace string) (*bearerToken, error) { - // first check cache - token := httpClient.cache.Get(namespace) - if token != nil && !token.isExpired() { - return token, nil - } - - //nolint: bodyclose - resp, _, err := httpClient.get(context.Background(), urlStr, false) - if err != nil { - return nil, err - } - - challengeParams, err := parseAuthHeader(resp) - if err != nil { - return nil, err - } - - tokenURL, err := getTokenURLFromChallengeParams(challengeParams, httpClient.config.Username) - if err != nil { - return nil, err - } - - return httpClient.getTokenFromURL(tokenURL.String(), namespace) -} - -func newBearerToken(blob []byte) (*bearerToken, error) { - token := new(bearerToken) - if err := json.Unmarshal(blob, &token); err != nil { - return nil, err - } - - if token.Token == "" { - token.Token = token.AccessToken - } - - if token.ExpiresIn < minimumTokenLifetimeSeconds { - token.ExpiresIn = minimumTokenLifetimeSeconds - } - - if token.IssuedAt.IsZero() { - token.IssuedAt = time.Now().UTC() - } - - token.expirationTime = token.IssuedAt.Add(time.Duration(token.ExpiresIn) * time.Second) - - return token, nil -} - -func getTokenURLFromChallengeParams(params challengeParams, account string) (*url.URL, error) { - parsedRealm, err := url.Parse(params.realm) - if err != nil { - return nil, err - } - - query := parsedRealm.Query() - query.Set("service", params.service) - query.Set("scope", params.scope) - - if account != "" { - query.Set("account", account) - } - - parsedRealm.RawQuery = query.Encode() - - return parsedRealm, nil -} - -func parseAuthHeader(resp *http.Response) (challengeParams, error) { - authHeader := resp.Header.Get("www-authenticate") - - authHeaderSlice := strings.Split(authHeader, ",") - - params := challengeParams{} - - for _, elem := range authHeaderSlice { - if strings.Contains(strings.ToLower(elem), "bearer") { - elem = strings.Split(elem, " ")[1] - } - - elem := strings.ReplaceAll(elem, "\"", "") - - elemSplit := strings.Split(elem, "=") - if len(elemSplit) != 2 { //nolint:mnd - return params, zerr.ErrParsingAuthHeader - } - - authKey := elemSplit[0] - - authValue := elemSplit[1] - - switch authKey { - case "realm": - params.realm = authValue - case "service": - params.service = authValue - case "scope": - params.scope = authValue - case "error": - params.err = authValue - } - } - - return params, nil -} - -// Checks if the auth headers in the response contain an indication of a failed -// authorization because of an "insufficient_scope" error. -func needsRetryWithUpdatedScope(err error, resp *http.Response) (bool, challengeParams) { - params := challengeParams{} - if err == nil && resp.StatusCode == http.StatusUnauthorized { - params, err = parseAuthHeader(resp) - if err != nil { - return false, params - } - - if params.err == "insufficient_scope" { - if params.scope != "" { - return true, params - } - } - } - - return false, params -} diff --git a/pkg/extensions/sync/oci_layout.go b/pkg/extensions/sync/oci_layout.go index 75a8a148..0e4850b3 100644 --- a/pkg/extensions/sync/oci_layout.go +++ b/pkg/extensions/sync/oci_layout.go @@ -29,8 +29,9 @@ func NewOciLayoutStorage(storeController storage.StoreController) OciLayoutStora func (oci OciLayoutStorageImpl) GetImageReference(repo string, reference string) (ref.Ref, error) { localImageStore := oci.storeController.GetImageStore(repo) if localImageStore == nil { - return nil, zerr.ErrLocalImgStoreNotFound + return ref.Ref{}, zerr.ErrLocalImgStoreNotFound } + tempSyncPath := path.Join(localImageStore.RootDir(), repo, constants.SyncBlobUploadDir) // create session folder diff --git a/pkg/extensions/sync/references/cosign.go b/pkg/extensions/sync/references/cosign.go deleted file mode 100644 index eaa6008c..00000000 --- a/pkg/extensions/sync/references/cosign.go +++ /dev/null @@ -1,219 +0,0 @@ -//go:build sync -// +build sync - -package references - -import ( - "context" - "errors" - "fmt" - "net/http" - "strings" - - godigest "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/sigstore/cosign/v2/pkg/oci/remote" - - zerr "zotregistry.dev/zot/errors" - "zotregistry.dev/zot/pkg/common" - "zotregistry.dev/zot/pkg/extensions/sync/constants" - client "zotregistry.dev/zot/pkg/extensions/sync/httpclient" - "zotregistry.dev/zot/pkg/log" - "zotregistry.dev/zot/pkg/meta" - mTypes "zotregistry.dev/zot/pkg/meta/types" - "zotregistry.dev/zot/pkg/storage" -) - -type CosignReference struct { - client *client.Client - storeController storage.StoreController - metaDB mTypes.MetaDB - log log.Logger -} - -func NewCosignReference(httpClient *client.Client, storeController storage.StoreController, - metaDB mTypes.MetaDB, log log.Logger, -) CosignReference { - return CosignReference{ - client: httpClient, - storeController: storeController, - metaDB: metaDB, - log: log, - } -} - -func (ref CosignReference) Name() string { - return constants.Cosign -} - -func (ref CosignReference) IsSigned(ctx context.Context, upstreamRepo, subjectDigestStr string) bool { - cosignSignatureTag := getCosignSignatureTagFromSubjectDigest(subjectDigestStr) - _, _, err := ref.getManifest(ctx, upstreamRepo, cosignSignatureTag) - - return err == nil -} - -func (ref CosignReference) canSkipReferences(localRepo, digest string, manifest *ispec.Manifest) ( - bool, error, -) { - if manifest == nil { - return true, nil - } - - imageStore := ref.storeController.GetImageStore(localRepo) - - // check cosign signature already synced - _, localDigest, _, err := imageStore.GetImageManifest(localRepo, digest) - if err != nil { - if errors.Is(err, zerr.ErrManifestNotFound) { - return false, nil - } - - ref.log.Error().Str("errorType", common.TypeOf(err)).Err(err). - Str("repository", localRepo).Str("reference", digest). - Msg("couldn't get local cosign manifest") - - return false, err - } - - if localDigest.String() != digest { - return false, nil - } - - ref.log.Info().Str("repository", localRepo).Str("reference", digest). - Msg("skipping syncing cosign reference, already synced") - - return true, nil -} - -func (ref CosignReference) SyncReferences(ctx context.Context, localRepo, remoteRepo, subjectDigestStr string) ( - []godigest.Digest, error, -) { - cosignTags := getCosignTagsFromSubjectDigest(subjectDigestStr) - - refsDigests := make([]godigest.Digest, 0, len(cosignTags)) - - for _, cosignTag := range cosignTags { - manifest, manifestBuf, err := ref.getManifest(ctx, remoteRepo, cosignTag) - if err != nil { - if errors.Is(err, zerr.ErrSyncReferrerNotFound) { - continue - } - - return refsDigests, err - } - - digest := godigest.FromBytes(manifestBuf) - - skip, err := ref.canSkipReferences(localRepo, digest.String(), manifest) - if err != nil { - ref.log.Error().Err(err).Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("couldn't check if the remote image cosign reference can be skipped") - } - - if skip { - refsDigests = append(refsDigests, digest) - - continue - } - - imageStore := ref.storeController.GetImageStore(localRepo) - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("syncing cosign reference for image") - - for _, blob := range manifest.Layers { - if err := syncBlob(ctx, ref.client, imageStore, localRepo, remoteRepo, blob.Digest, ref.log); err != nil { - return refsDigests, err - } - } - - // sync config blob - if err := syncBlob(ctx, ref.client, imageStore, localRepo, remoteRepo, manifest.Config.Digest, ref.log); err != nil { - return refsDigests, err - } - - // push manifest - referenceDigest, _, err := imageStore.PutImageManifest(localRepo, cosignTag, - ispec.MediaTypeImageManifest, manifestBuf) - if err != nil { - ref.log.Error().Str("errorType", common.TypeOf(err)). - Str("repository", localRepo).Str("subject", subjectDigestStr). - Err(err).Msg("couldn't upload cosign reference manifest for image") - - return refsDigests, err - } - - refsDigests = append(refsDigests, digest) - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr). - Msg("successfully synced cosign reference for image") - - if ref.metaDB != nil { - ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr).Str("component", "metadb"). - Msg("trying to sync cosign reference for image") - - err = meta.SetImageMetaFromInput(ctx, localRepo, cosignTag, ispec.MediaTypeImageManifest, - referenceDigest, manifestBuf, ref.storeController.GetImageStore(localRepo), - ref.metaDB, ref.log) - if err != nil { - return refsDigests, fmt.Errorf("failed to set metadata for cosign reference in '%s@%s': %w", - localRepo, subjectDigestStr, err) - } - - ref.log.Info().Str("repository", localRepo).Str("subject", subjectDigestStr).Str("component", "metadb"). - Msg("successfully added cosign reference for image") - } - } - - return refsDigests, nil -} - -func (ref CosignReference) getManifest(ctx context.Context, repo, cosignTag string) (*ispec.Manifest, []byte, error) { - var cosignManifest ispec.Manifest - - body, _, statusCode, err := ref.client.MakeGetRequest(ctx, &cosignManifest, ispec.MediaTypeImageManifest, - "v2", repo, "manifests", cosignTag) - if err != nil { - if statusCode == http.StatusNotFound { - ref.log.Debug().Str("errorType", common.TypeOf(err)). - Str("repository", repo).Str("tag", cosignTag). - Err(err).Msg("couldn't find any cosign manifest for image") - - return nil, nil, zerr.ErrSyncReferrerNotFound - } - - return nil, nil, err - } - - return &cosignManifest, body, nil -} - -func getCosignSignatureTagFromSubjectDigest(digestStr string) string { - return strings.Replace(digestStr, ":", "-", 1) + "." + remote.SignatureTagSuffix -} - -func getCosignSBOMTagFromSubjectDigest(digestStr string) string { - return strings.Replace(digestStr, ":", "-", 1) + "." + remote.SBOMTagSuffix -} - -func getCosignTagsFromSubjectDigest(digestStr string) []string { - var cosignTags []string - - // signature tag - cosignTags = append(cosignTags, getCosignSignatureTagFromSubjectDigest(digestStr)) - // sbom tag - cosignTags = append(cosignTags, getCosignSBOMTagFromSubjectDigest(digestStr)) - - return cosignTags -} - -// this function will check if tag is a cosign tag (signature or sbom). -func IsCosignTag(tag string) bool { - if strings.HasPrefix(tag, "sha256-") && - (strings.HasSuffix(tag, remote.SignatureTagSuffix) || strings.HasSuffix(tag, remote.SBOMTagSuffix)) { - return true - } - - return false -} diff --git a/pkg/extensions/sync/service.go b/pkg/extensions/sync/service.go index 2ac1c290..7d52a3c0 100644 --- a/pkg/extensions/sync/service.go +++ b/pkg/extensions/sync/service.go @@ -322,9 +322,9 @@ func (service *BaseService) SyncRepo(ctx context.Context, repo string) error { return ctx.Err() } - // if isCosignTag(tag) || common.IsReferrersTag(tag) { - // continue - // } + if isCosignTag(tag) || common.IsReferrersTag(tag) { + continue + } err = service.syncTagAndReferrers(ctx, localRepo, repo, tag) if err != nil { diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 0bf83829..9ba420b5 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -4605,7 +4605,7 @@ func TestPeriodicallySignaturesErr(t *testing.T) { func TestSignatures(t *testing.T) { Convey("Verify sync signatures", t, func() { - // updateDuration, _ := time.ParseDuration("30m") + updateDuration, _ := time.ParseDuration("10s") sctlr, srcBaseURL, srcDir, _, _ := makeUpstreamServer(t, false, false) @@ -4714,12 +4714,12 @@ func TestSignatures(t *testing.T) { }, }, }, - URLs: []string{srcBaseURL}, - // PollInterval: updateDuration, - TLSVerify: &tlsVerify, - CertDir: "", - OnlySigned: &onlySigned, - OnDemand: true, + URLs: []string{srcBaseURL}, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: "", + OnlySigned: &onlySigned, + OnDemand: true, } defaultVal := true