From 1109bb4dde45317fd997ef882a954d896e59bc04 Mon Sep 17 00:00:00 2001 From: Petu Eusebiu Date: Tue, 7 Dec 2021 20:26:26 +0200 Subject: [PATCH] sync: Added support for syncing notary/cosign signatures, closes #261 Signed-off-by: Petu Eusebiu --- pkg/cli/root_test.go | 22 + pkg/extensions/sync/on_demand.go | 30 +- pkg/extensions/sync/sync.go | 79 ++- pkg/extensions/sync/sync_internal_test.go | 31 +- pkg/extensions/sync/sync_test.go | 593 ++++++++++++++++++++++ pkg/extensions/sync/utils.go | 325 ++++++++++++ pkg/storage/cache_test.go | 3 + 7 files changed, 1026 insertions(+), 57 deletions(-) diff --git a/pkg/cli/root_test.go b/pkg/cli/root_test.go index bf90a9a6..dfcf4318 100644 --- a/pkg/cli/root_test.go +++ b/pkg/cli/root_test.go @@ -188,6 +188,22 @@ func TestVerify(t *testing.T) { So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic) }) + Convey("Test verify w/ sync and w/ filesystem storage", t, func(c C) { + tmpfile, err := ioutil.TempFile("", "zot-test*.json") + So(err, ShouldBeNil) + defer os.Remove(tmpfile.Name()) // clean up + content := []byte(`{"storage":{"rootDirectory":"/tmp/zot"}, + "http":{"address":"127.0.0.1","port":"8080","realm":"zot", + "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}, + "extensions":{"sync": {"registries": [{"url":"localhost:9999"}]}}}`) + _, err = tmpfile.Write(content) + So(err, ShouldBeNil) + err = tmpfile.Close() + So(err, ShouldBeNil) + os.Args = []string{"cli_test", "verify", tmpfile.Name()} + So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldNotPanic) + }) + Convey("Test verify with bad sync prefixes", t, func(c C) { tmpfile, err := ioutil.TempFile("", "zot-test*.json") So(err, ShouldBeNil) @@ -274,6 +290,12 @@ func TestScrub(t *testing.T) { So(err, ShouldBeNil) }) + Convey("Test scrub no args", t, func(c C) { + os.Args = []string{"cli_test", "scrub"} + err := cli.NewServerRootCmd().Execute() + So(err, ShouldBeNil) + }) + Convey("Test scrub config", t, func(c C) { Convey("non-existent config", func(c C) { os.Args = []string{"cli_test", "scrub", path.Join(os.TempDir(), "/x.yaml")} diff --git a/pkg/extensions/sync/on_demand.go b/pkg/extensions/sync/on_demand.go index ca263d89..73ea8019 100644 --- a/pkg/extensions/sync/on_demand.go +++ b/pkg/extensions/sync/on_demand.go @@ -21,6 +21,12 @@ func OneImage(cfg Config, storeController storage.StoreController, repo, tag string, log log.Logger) error { var credentialsFile CredentialsFile + /* don't copy cosign signature, containers/image doesn't support it + we will copy it manually later */ + if isCosignTag(tag) { + return nil + } + if cfg.CredentialsFile != "" { var err error @@ -46,7 +52,8 @@ func OneImage(cfg Config, storeController storage.StoreController, return err } - for _, regCfg := range cfg.Registries { + for _, registryCfg := range cfg.Registries { + regCfg := registryCfg if !regCfg.OnDemand { log.Info().Msgf("skipping syncing on demand from %s, onDemand flag is false", regCfg.URL) @@ -127,9 +134,9 @@ func OneImage(cfg Config, storeController storage.StoreController, if err = retry.RetryIfNecessary(context.Background(), func() error { _, copyErr = copy.Image(context.Background(), policyCtx, localRef, upstreamRef, &options) - return err - }, retryOptions); copyErr != nil { - log.Error().Err(copyErr).Msgf("error while copying image %s to %s", + return copyErr + }, retryOptions); err != nil { + log.Error().Err(err).Msgf("error while copying image %s to %s", upstreamRef.DockerReference().Name(), localTaggedRepo) } else { log.Info().Msgf("successfully synced %s", upstreamRef.DockerReference().Name()) @@ -142,6 +149,21 @@ func OneImage(cfg Config, storeController storage.StoreController, return err } + httpClient, err := getHTTPClient(®Cfg, credentialsFile[upstreamRegistryName], log) + if err != nil { + return err + } + + if err = retry.RetryIfNecessary(context.Background(), func() error { + err = syncSignatures(httpClient, storeController, regCfg.URL, imageName, upstreamTaggedRef.Tag(), log) + + return err + }, retryOptions); err != nil { + log.Error().Err(err).Msgf("Couldn't copy image signature %s", upstreamRef.DockerReference().Name()) + + return err + } + return nil } } diff --git a/pkg/extensions/sync/sync.go b/pkg/extensions/sync/sync.go index 0bf274e3..33d79dd8 100644 --- a/pkg/extensions/sync/sync.go +++ b/pkg/extensions/sync/sync.go @@ -2,12 +2,9 @@ package sync import ( "context" - "crypto/tls" - "crypto/x509" "encoding/json" "fmt" "io" - "io/ioutil" "os" "path" "regexp" @@ -75,49 +72,10 @@ type Tags struct { } // getUpstreamCatalog gets all repos from a registry. -func getUpstreamCatalog(regCfg *RegistryConfig, credentials Credentials, log log.Logger) (catalog, error) { +func getUpstreamCatalog(client *resty.Client, regCfg *RegistryConfig, log log.Logger) (catalog, error) { var c catalog registryCatalogURL := fmt.Sprintf("%s%s", regCfg.URL, "/v2/_catalog") - client := resty.New() - - if regCfg.CertDir != "" { - log.Debug().Msgf("sync: using certs directory: %s", regCfg.CertDir) - clientCert := path.Join(regCfg.CertDir, "client.cert") - clientKey := path.Join(regCfg.CertDir, "client.key") - caCertPath := path.Join(regCfg.CertDir, "ca.crt") - - caCert, err := ioutil.ReadFile(caCertPath) - if err != nil { - log.Error().Err(err).Msg("couldn't read CA certificate") - - return c, err - } - - caCertPool := x509.NewCertPool() - caCertPool.AppendCertsFromPEM(caCert) - - client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12}) - - cert, err := tls.LoadX509KeyPair(clientCert, clientKey) - if err != nil { - log.Error().Err(err).Msg("couldn't read certificates key pairs") - - return c, err - } - - client.SetCertificates(cert) - } - - // nolint: gosec - if regCfg.TLSVerify != nil && !*regCfg.TLSVerify { - client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) - } - - if credentials.Username != "" && credentials.Password != "" { - log.Debug().Msgf("sync: using basic auth") - client.SetBasicAuth(credentials.Username, credentials.Password) - } resp, err := client.R().SetHeader("Content-Type", "application/json").Get(registryCatalogURL) if err != nil { @@ -247,6 +205,12 @@ func imagesToCopyFromUpstream(registryName string, repos []string, upstreamCtx * } for _, tag := range tags { + // don't copy cosign signature, containers/image doesn't support it + // we will copy it manually later + if isCosignTag(tag) { + continue + } + taggedRef, err := reference.WithTag(repoRef, tag) if err != nil { log.Err(err).Msgf("error creating a reference for repository %s and tag %q", repoRef.Name(), tag) @@ -283,11 +247,10 @@ func imagesToCopyFromUpstream(registryName string, repos []string, upstreamCtx * func getCopyOptions(upstreamCtx, localCtx *types.SystemContext) copy.Options { options := copy.Options{ - DestinationCtx: localCtx, - SourceCtx: upstreamCtx, - ReportWriter: io.Discard, - // force only oci manifest MIME type - ForceManifestMIMEType: ispec.MediaTypeImageManifest, + DestinationCtx: localCtx, + SourceCtx: upstreamCtx, + ReportWriter: io.Discard, + ForceManifestMIMEType: ispec.MediaTypeImageManifest, // force only oci manifest MIME type } return options @@ -295,7 +258,6 @@ func getCopyOptions(upstreamCtx, localCtx *types.SystemContext) copy.Options { func getUpstreamContext(regCfg *RegistryConfig, credentials Credentials) *types.SystemContext { upstreamCtx := &types.SystemContext{} - upstreamCtx.DockerCertPath = regCfg.CertDir upstreamCtx.DockerDaemonCertPath = regCfg.CertDir @@ -336,8 +298,13 @@ func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController var catalog catalog + httpClient, err := getHTTPClient(®Cfg, credentials, log) + if err != nil { + return err + } + if err = retry.RetryIfNecessary(context.Background(), func() error { - catalog, err = getUpstreamCatalog(®Cfg, credentials, log) + catalog, err = getUpstreamCatalog(httpClient, ®Cfg, log) return err }, retryOptions); err != nil { @@ -430,6 +397,16 @@ func syncRegistry(regCfg RegistryConfig, storeController storage.StoreController return err } + + if err = retry.RetryIfNecessary(context.Background(), func() error { + err = syncSignatures(httpClient, storeController, regCfg.URL, imageName, upstreamTaggedRef.Tag(), log) + + return err + }, retryOptions); err != nil { + log.Error().Err(err).Msgf("Couldn't copy image signature %s", upstreamRef.DockerReference().Name()) + + return err + } } log.Info().Msgf("finished syncing %s", regCfg.URL) @@ -445,6 +422,8 @@ func getLocalContexts(log log.Logger) (*types.SystemContext, *signature.PolicyCo var err error localCtx := &types.SystemContext{} + // preserve compression + localCtx.OCIAcceptUncompressedLayers = true // accept any image with or without signature policy = &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}} diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index ca604f20..678bd307 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io/ioutil" + "net/url" "os" "path" goSync "sync" @@ -18,6 +19,7 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/rs/zerolog" . "github.com/smartystreets/goconvey/convey" + "gopkg.in/resty.v1" "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/log" @@ -123,11 +125,11 @@ func TestSyncInternal(t *testing.T) { CertDir: "/tmp/missing_certs/a/b/c/d/z", } - _, err := getUpstreamCatalog(&syncRegistryConfig, Credentials{}, log.NewLogger("debug", "")) + _, err := getUpstreamCatalog(resty.New(), &syncRegistryConfig, log.NewLogger("debug", "")) So(err, ShouldNotBeNil) }) - Convey("Test getUpstreamCatalog() with bad certs", t, func() { + Convey("Test getHttpClient() with bad certs", t, func() { badCertsDir, err := ioutil.TempDir("", "bad_certs*") if err != nil { panic(err) @@ -153,7 +155,10 @@ func TestSyncInternal(t *testing.T) { CertDir: badCertsDir, } - _, err = getUpstreamCatalog(&syncRegistryConfig, Credentials{}, log.NewLogger("debug", "")) + _, err = getHTTPClient(&syncRegistryConfig, Credentials{}, log.NewLogger("debug", "")) + So(err, ShouldNotBeNil) + syncRegistryConfig.CertDir = "/path/to/invalid/cert" + _, err = getHTTPClient(&syncRegistryConfig, Credentials{}, log.NewLogger("debug", "")) So(err, ShouldNotBeNil) }) @@ -169,6 +174,26 @@ func TestSyncInternal(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("Test OneImage() skips cosign signatures", t, func() { + err := OneImage(Config{}, storage.StoreController{}, "repo", "sha256-.sig", log.NewLogger("", "")) + So(err, ShouldBeNil) + }) + + Convey("Test syncSignatures()", t, func() { + log := log.NewLogger("", "") + err := syncSignatures(resty.New(), storage.StoreController{}, "%", "repo", "tag", log) + So(err, ShouldNotBeNil) + err = syncSignatures(resty.New(), storage.StoreController{}, "http://zot", "repo", "tag", log) + So(err, ShouldNotBeNil) + err = syncSignatures(resty.New(), storage.StoreController{}, "https://google.com", "repo", "tag", log) + So(err, ShouldNotBeNil) + url, _ := url.Parse("invalid") + err = syncCosignSignature(resty.New(), storage.StoreController{}, *url, "repo", "tag", log) + So(err, ShouldNotBeNil) + err = syncNotarySignature(resty.New(), storage.StoreController{}, *url, "repo", "tag", log) + So(err, ShouldNotBeNil) + }) + Convey("Test filterRepos()", t, func() { repos := []string{"repo", "repo1", "repo2", "repo/repo2", "repo/repo2/repo3/repo4"} contents := []Content{ diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 183c15c4..dd656887 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -4,6 +4,7 @@ package sync_test import ( + "context" "crypto/tls" "crypto/x509" "encoding/json" @@ -12,13 +13,21 @@ import ( "io" "io/ioutil" "os" + "os/exec" "path" "reflect" "strings" "testing" "time" + "github.com/notaryproject/notation-go-lib" + godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" + artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" + "github.com/sigstore/cosign/cmd/cosign/cli/generate" + "github.com/sigstore/cosign/cmd/cosign/cli/options" + "github.com/sigstore/cosign/cmd/cosign/cli/sign" + "github.com/sigstore/cosign/cmd/cosign/cli/verify" . "github.com/smartystreets/goconvey/convey" "gopkg.in/resty.v1" "zotregistry.io/zot/pkg/api" @@ -47,6 +56,10 @@ type TagsList struct { Tags []string } +type ReferenceList struct { + References []notation.Descriptor `json:"references"` +} + type catalog struct { Repositories []string `json:"repositories"` } @@ -199,6 +212,8 @@ func startDownstreamServer(secure bool, syncConfig *sync.Config) (*api.Controlle } destConfig.Storage.RootDirectory = destDir + destConfig.Storage.Dedupe = false + destConfig.Storage.GC = false destConfig.Extensions = &extconf.ExtensionConfig{} destConfig.Extensions.Search = nil @@ -1632,6 +1647,37 @@ func TestSyncSubPaths(t *testing.T) { }) } +func TestSyncOnDemandRepoErr(t *testing.T) { + Convey("Verify sync on demand parseRepositoryReference error", t, func() { + tlsVerify := false + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + // will sync on demand, should not be filtered out + Prefix: testImage, + }, + }, + URL: "docker://invalid", + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, + } + + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) + + defer func() { + dc.Shutdown() + }() + + resp, err := resty.R().Get(destBaseURL + "/v2/" + testImage + "/manifests/" + testImageTag) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 404) + }) +} + func TestSyncOnDemandContentFiltering(t *testing.T) { Convey("Verify sync on demand feature", t, func() { sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false) @@ -1821,3 +1867,550 @@ func TestSyncConfigRules(t *testing.T) { }) }) } + +func TestSyncSignatures(t *testing.T) { + Convey("Verify sync signatures", t, func() { + updateDuration, _ := time.ParseDuration("30m") + + sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false) + defer os.RemoveAll(srcDir) + + defer func() { + sc.Shutdown() + }() + + // create repo, push and sign it + repoName := "signed-repo" + var digest godigest.Digest + So(func() { digest = pushRepo(srcBaseURL, repoName) }, ShouldNotPanic) + + splittedURL := strings.SplitAfter(srcBaseURL, ":") + srcPort := splittedURL[len(splittedURL)-1] + + cwd, err := os.Getwd() + So(err, ShouldBeNil) + + defer func() { _ = os.Chdir(cwd) }() + tdir, err := ioutil.TempDir("", "sigs") + So(err, ShouldBeNil) + defer os.RemoveAll(tdir) + _ = os.Chdir(tdir) + + So(func() { signImage(tdir, srcPort, repoName, digest) }, ShouldNotPanic) + + regex := ".*" + var semver bool + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + // won't match any image on source registry, we will sync on demand + Prefix: repoName, + Tags: &sync.Tags{ + Regex: ®ex, + Semver: &semver, + }, + }, + }, + URL: srcBaseURL, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, + } + + syncConfig := &sync.Config{ + Registries: []sync.RegistryConfig{syncRegistryConfig}, + } + + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) + + defer func() { + dc.Shutdown() + }() + + // wait for sync + var destTagsList TagsList + + for { + resp, err := destClient.R().Get(destBaseURL + "/v2/" + repoName + "/tags/list") + if err != nil { + panic(err) + } + + err = json.Unmarshal(resp.Body(), &destTagsList) + if err != nil { + panic(err) + } + + if len(destTagsList.Tags) > 0 { + break + } + + time.Sleep(500 * time.Millisecond) + } + + splittedURL = strings.SplitAfter(destBaseURL, ":") + destPort := splittedURL[len(splittedURL)-1] + + a := &options.AnnotationOptions{Annotations: []string{"tag=1.0"}} + amap, err := a.AnnotationsMap() + if err != nil { + panic(err) + } + + // notation verify the image + image := fmt.Sprintf("localhost:%s/%s:%s", destPort, repoName, "1.0") + cmd := exec.Command("notation", "verify", "--cert", "good", "--plain-http", image) + out, err := cmd.CombinedOutput() + So(err, ShouldBeNil) + msg := string(out) + So(msg, ShouldNotBeEmpty) + So(strings.Contains(msg, "verification failure"), ShouldBeFalse) + + // cosign verify the image + vrfy := verify.VerifyCommand{ + RegistryOptions: options.RegistryOptions{AllowInsecure: true}, + CheckClaims: true, + KeyRef: path.Join(tdir, "cosign.pub"), + Annotations: amap, + } + err = vrfy.Exec(context.TODO(), []string{fmt.Sprintf("localhost:%s/%s:%s", destPort, repoName, "1.0")}) + So(err, ShouldBeNil) + + // test negative cases (trigger errors) + // test notary signatures errors + + // based on manifest digest get referrers + getReferrersURL := srcBaseURL + path.Join("/oras/artifacts/v1/", repoName, "manifests", digest.String(), "referrers") + + resp, err := resty.R(). + SetHeader("Content-Type", "application/json"). + SetQueryParam("artifactType", "application/vnd.cncf.notary.v2.signature"). + Get(getReferrersURL) + + So(err, ShouldBeNil) + So(resp, ShouldNotBeEmpty) + + var referrers ReferenceList + + err = json.Unmarshal(resp.Body(), &referrers) + So(err, ShouldBeNil) + + // read manifest + var nm artifactspec.Manifest + for _, ref := range referrers.References { + refPath := path.Join(srcDir, repoName, "blobs", string(ref.Digest.Algorithm()), ref.Digest.Hex()) + body, err := ioutil.ReadFile(refPath) + So(err, ShouldBeNil) + + err = json.Unmarshal(body, &nm) + So(err, ShouldBeNil) + + // triggers perm denied on sig blobs + for _, blob := range nm.Blobs { + blobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex()) + err := os.Chmod(blobPath, 0o000) + So(err, ShouldBeNil) + } + } + + // err = os.Chmod(path.Join(srcDir, repoName, "index.json"), 0000) + // if err != nil { + // panic(err) + // } + + resp, _ = resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 500) + + for _, blob := range nm.Blobs { + srcBlobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex()) + err := os.Chmod(srcBlobPath, 0o755) + So(err, ShouldBeNil) + + destBlobPath := path.Join(destDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex()) + err = os.Remove(destBlobPath) + So(err, ShouldBeNil) + err = os.MkdirAll(destBlobPath, 0o000) + So(err, ShouldBeNil) + } + + resp, _ = resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 500) + + // clean + for _, blob := range nm.Blobs { + destBlobPath := path.Join(destDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex()) + err = os.Chmod(destBlobPath, 0o755) + So(err, ShouldBeNil) + err = os.Remove(destBlobPath) + So(err, ShouldBeNil) + } + + // test cosign signatures errors + // based on manifest digest get cosign manifest + cosignEncodedDigest := strings.Replace(digest.String(), ":", "-", 1) + ".sig" + getCosignManifestURL := srcBaseURL + path.Join("/v2", repoName, "manifests", cosignEncodedDigest) + + mResp, err := resty.R().Get(getCosignManifestURL) + So(err, ShouldBeNil) + + var cm ispec.Manifest + + err = json.Unmarshal(mResp.Body(), &cm) + So(err, ShouldBeNil) + + for _, blob := range cm.Layers { + blobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex()) + err := os.Chmod(blobPath, 0o000) + So(err, ShouldBeNil) + } + + resp, _ = resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 500) + + for _, blob := range cm.Layers { + srcBlobPath := path.Join(srcDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex()) + err := os.Chmod(srcBlobPath, 0o755) + So(err, ShouldBeNil) + + destBlobPath := path.Join(destDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex()) + err = os.Remove(destBlobPath) + So(err, ShouldBeNil) + err = os.MkdirAll(destBlobPath, 0o000) + So(err, ShouldBeNil) + } + + resp, _ = resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 500) + + for _, blob := range cm.Layers { + destBlobPath := path.Join(destDir, repoName, "blobs", string(blob.Digest.Algorithm()), blob.Digest.Hex()) + err = os.Chmod(destBlobPath, 0o755) + So(err, ShouldBeNil) + err = os.Remove(destBlobPath) + So(err, ShouldBeNil) + } + + // trigger error on config blob + srcConfigBlobPath := path.Join(srcDir, repoName, "blobs", string(cm.Config.Digest.Algorithm()), + cm.Config.Digest.Hex()) + err = os.Chmod(srcConfigBlobPath, 0o000) + So(err, ShouldBeNil) + + resp, _ = resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 500) + + err = os.Chmod(srcConfigBlobPath, 0o755) + So(err, ShouldBeNil) + + destConfigBlobPath := path.Join(destDir, repoName, "blobs", string(cm.Config.Digest.Algorithm()), + cm.Config.Digest.Hex()) + err = os.Remove(destConfigBlobPath) + So(err, ShouldBeNil) + err = os.MkdirAll(destConfigBlobPath, 0o000) + So(err, ShouldBeNil) + + resp, _ = resty.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 500) + }) +} + +func TestSyncError(t *testing.T) { + Convey("Verify periodically sync pushSyncedLocalImage() error", t, func() { + updateDuration, _ := time.ParseDuration("30m") + + sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false) + defer os.RemoveAll(srcDir) + + defer func() { + sc.Shutdown() + }() + + regex := ".*" + semver := true + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + Content: []sync.Content{ + { + Prefix: testImage, + Tags: &sync.Tags{ + Regex: ®ex, + Semver: &semver, + }, + }, + }, + URL: srcBaseURL, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: "", + } + + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc, destBaseURL, destDir, destClient := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) + + defer func() { + dc.Shutdown() + }() + + // give permission denied on pushSyncedLocalImage() + localRepoPath := path.Join(destDir, testImage, "blobs") + err := os.MkdirAll(localRepoPath, 0o000) + So(err, ShouldBeNil) + + resp, _ := destClient.R().Post(destBaseURL + "/sync") + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, 500) + }) +} + +func TestSyncSignaturesOnDemand(t *testing.T) { + Convey("Verify sync signatures on demand feature", t, func() { + sc, srcBaseURL, srcDir, _, _ := startUpstreamServer(false, false) + defer os.RemoveAll(srcDir) + + defer func() { + sc.Shutdown() + }() + + // create repo, push and sign it + repoName := "signed-repo" + var digest godigest.Digest + So(func() { digest = pushRepo(srcBaseURL, repoName) }, ShouldNotPanic) + + splittedURL := strings.SplitAfter(srcBaseURL, ":") + srcPort := splittedURL[len(splittedURL)-1] + + cwd, err := os.Getwd() + So(err, ShouldBeNil) + + defer func() { _ = os.Chdir(cwd) }() + tdir, err := ioutil.TempDir("", "sigs") + So(err, ShouldBeNil) + defer os.RemoveAll(tdir) + _ = os.Chdir(tdir) + + So(func() { signImage(tdir, srcPort, repoName, digest) }, ShouldNotPanic) + + var tlsVerify bool + + syncRegistryConfig := sync.RegistryConfig{ + URL: srcBaseURL, + TLSVerify: &tlsVerify, + CertDir: "", + OnDemand: true, + } + + syncConfig := &sync.Config{Registries: []sync.RegistryConfig{syncRegistryConfig}} + + dc, destBaseURL, destDir, _ := startDownstreamServer(false, syncConfig) + defer os.RemoveAll(destDir) + + defer func() { + dc.Shutdown() + }() + + // sync on demand + resp, err := resty.R().Get(destBaseURL + "/v2/" + repoName + "/manifests/1.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 200) + + splittedURL = strings.SplitAfter(destBaseURL, ":") + destPort := splittedURL[len(splittedURL)-1] + + a := &options.AnnotationOptions{Annotations: []string{"tag=1.0"}} + amap, err := a.AnnotationsMap() + if err != nil { + panic(err) + } + + // notation verify the synced image + image := fmt.Sprintf("localhost:%s/%s:%s", destPort, repoName, "1.0") + cmd := exec.Command("notation", "verify", "--cert", "good", "--plain-http", image) + out, err := cmd.CombinedOutput() + So(err, ShouldBeNil) + msg := string(out) + So(msg, ShouldNotBeEmpty) + So(strings.Contains(msg, "verification failure"), ShouldBeFalse) + + // cosign verify the synced image + vrfy := verify.VerifyCommand{ + RegistryOptions: options.RegistryOptions{AllowInsecure: true}, + CheckClaims: true, + KeyRef: path.Join(tdir, "cosign.pub"), + Annotations: amap, + } + err = vrfy.Exec(context.TODO(), []string{fmt.Sprintf("localhost:%s/%s:%s", destPort, repoName, "1.0")}) + So(err, ShouldBeNil) + + // test negative case + cosignEncodedDigest := strings.Replace(digest.String(), ":", "-", 1) + ".sig" + getCosignManifestURL := srcBaseURL + path.Join("/v2", repoName, "manifests", cosignEncodedDigest) + + mResp, err := resty.R().Get(getCosignManifestURL) + So(err, ShouldBeNil) + + var cm ispec.Manifest + + err = json.Unmarshal(mResp.Body(), &cm) + So(err, ShouldBeNil) + + // trigger error on config blob + srcConfigBlobPath := path.Join(srcDir, repoName, "blobs", string(cm.Config.Digest.Algorithm()), + cm.Config.Digest.Hex()) + err = os.Chmod(srcConfigBlobPath, 0o000) + So(err, ShouldBeNil) + + // remove already synced image + err = os.RemoveAll(path.Join(destDir, repoName)) + So(err, ShouldBeNil) + + // sync on demand + resp, err = resty.R().Get(destBaseURL + "/v2/" + repoName + "/manifests/1.0") + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, 404) + }) +} + +func signImage(tdir, port, repoName string, digest godigest.Digest) { + // push signatures to upstream server so that we can sync them later + // generate a keypair + os.Setenv("COSIGN_PASSWORD", "") + + err := generate.GenerateKeyPairCmd(context.TODO(), "", nil) + if err != nil { + panic(err) + } + + // sign the image + err = sign.SignCmd(context.TODO(), + sign.KeyOpts{KeyRef: path.Join(tdir, "cosign.key"), PassFunc: generate.GetPass}, + options.RegistryOptions{AllowInsecure: true}, + map[string]interface{}{"tag": "1.0"}, + []string{fmt.Sprintf("localhost:%s/%s@%s", port, repoName, digest.String())}, + "", true, "", false, false, "") + if err != nil { + panic(err) + } + + // verify the image + a := &options.AnnotationOptions{Annotations: []string{"tag=1.0"}} + + amap, err := a.AnnotationsMap() + if err != nil { + panic(err) + } + + vrfy := verify.VerifyCommand{ + RegistryOptions: options.RegistryOptions{AllowInsecure: true}, + CheckClaims: true, + KeyRef: path.Join(tdir, "cosign.pub"), + Annotations: amap, + } + + err = vrfy.Exec(context.TODO(), []string{fmt.Sprintf("localhost:%s/%s:%s", port, repoName, "1.0")}) + if err != nil { + panic(err) + } + + // "notation" (notaryv2) doesn't yet support exported apis, so use the binary instead + _, err = exec.LookPath("notation") + if err != nil { + panic(err) + } + + os.Setenv("XDG_CONFIG_HOME", tdir) + + // generate a keypair + cmd := exec.Command("notation", "cert", "generate-test", "--trust", "good") + + err = cmd.Run() + if err != nil { + panic(err) + } + + // sign the image + image := fmt.Sprintf("localhost:%s/%s:%s", port, repoName, "1.0") + cmd = exec.Command("notation", "sign", "--key", "good", "--plain-http", image) + + err = cmd.Run() + if err != nil { + panic(err) + } + + // verify the image + cmd = exec.Command("notation", "verify", "--cert", "good", "--plain-http", image) + out, err := cmd.CombinedOutput() + So(err, ShouldBeNil) + + msg := string(out) + So(msg, ShouldNotBeEmpty) + So(strings.Contains(msg, "verification failure"), ShouldBeFalse) +} + +func pushRepo(url, repoName string) godigest.Digest { + // create a blob/layer + resp, err := resty.R().Post(url + fmt.Sprintf("/v2/%s/blobs/uploads/", repoName)) + if err != nil { + panic(err) + } + + loc := Location(url, resp) + + _, err = resty.R().Get(loc) + if err != nil { + panic(err) + } + + content := []byte("this is a blob") + digest := godigest.FromBytes(content) + + _, err = resty.R().SetQueryParam("digest", digest.String()). + SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(loc) + if err != nil { + panic(err) + } + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + Digest: digest, + Size: int64(len(content)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest, + Size: int64(len(content)), + }, + }, + } + + manifest.SchemaVersion = 2 + + content, err = json.Marshal(manifest) + if err != nil { + panic(err) + } + + digest = godigest.FromBytes(content) + + _, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). + SetBody(content).Put(url + fmt.Sprintf("/v2/%s/manifests/1.0", repoName)) + if err != nil { + panic(err) + } + + return digest +} diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go index f5f3a4c2..b00db63d 100644 --- a/pkg/extensions/sync/utils.go +++ b/pkg/extensions/sync/utils.go @@ -1,8 +1,11 @@ package sync import ( + "crypto/tls" + "crypto/x509" "encoding/json" "io/ioutil" + "net/url" "os" "path" "strings" @@ -10,13 +13,20 @@ import ( glob "github.com/bmatcuk/doublestar/v4" "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/types" + "github.com/notaryproject/notation-go-lib" ispec "github.com/opencontainers/image-spec/specs-go/v1" + artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" + "gopkg.in/resty.v1" "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/storage" ) +type ReferenceList struct { + References []notation.Descriptor `json:"references"` +} + // getTagFromRef returns a tagged reference from an image reference. func getTagFromRef(ref types.ImageReference, log log.Logger) reference.Tagged { tagged, isTagged := ref.DockerReference().(reference.Tagged) @@ -91,6 +101,311 @@ func getFileCredentials(filepath string) (CredentialsFile, error) { return creds, nil } +func getHTTPClient(regCfg *RegistryConfig, credentials Credentials, log log.Logger) (*resty.Client, error) { + client := resty.New() + + if regCfg.CertDir != "" { + log.Debug().Msgf("sync: using certs directory: %s", regCfg.CertDir) + clientCert := path.Join(regCfg.CertDir, "client.cert") + clientKey := path.Join(regCfg.CertDir, "client.key") + caCertPath := path.Join(regCfg.CertDir, "ca.crt") + + caCert, err := ioutil.ReadFile(caCertPath) + if err != nil { + log.Error().Err(err).Msg("couldn't read CA certificate") + + return nil, err + } + + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + + client.SetTLSClientConfig(&tls.Config{RootCAs: caCertPool, MinVersion: tls.VersionTLS12}) + + cert, err := tls.LoadX509KeyPair(clientCert, clientKey) + if err != nil { + log.Error().Err(err).Msg("couldn't read certificates key pairs") + + return nil, err + } + + client.SetCertificates(cert) + } + + // nolint: gosec + if regCfg.TLSVerify != nil && !*regCfg.TLSVerify { + client.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) + } + + if credentials.Username != "" && credentials.Password != "" { + log.Debug().Msgf("sync: using basic auth") + client.SetBasicAuth(credentials.Username, credentials.Password) + } + + return client, nil +} + +func syncCosignSignature(client *resty.Client, storeController storage.StoreController, + regURL url.URL, repo, digest string, log log.Logger) error { + log.Info().Msg("syncing cosign signatures") + + getCosignManifestURL := regURL + + cosignEncodedDigest := strings.Replace(digest, ":", "-", 1) + ".sig" + getCosignManifestURL.Path = path.Join(getCosignManifestURL.Path, "v2", repo, "manifests", cosignEncodedDigest) + getCosignManifestURL.RawQuery = getCosignManifestURL.Query().Encode() + + mResp, err := client.R().Get(getCosignManifestURL.String()) + if err != nil { + log.Error().Err(err).Str("url", getCosignManifestURL.String()). + Msgf("couldn't get cosign manifest: %s", cosignEncodedDigest) + + return err + } + + if mResp.IsError() { + log.Info().Msgf("couldn't find any cosign signature from %s, status code: %d skipping", + getCosignManifestURL.String(), mResp.StatusCode()) + + return nil + } + + var m ispec.Manifest + + err = json.Unmarshal(mResp.Body(), &m) + if err != nil { + log.Error().Err(err).Str("url", getCosignManifestURL.String()). + Msgf("couldn't unmarshal cosign manifest %s", cosignEncodedDigest) + + return err + } + + imageStore := storeController.GetImageStore(repo) + + for _, blob := range m.Layers { + // get blob + getBlobURL := regURL + getBlobURL.Path = path.Join(getBlobURL.Path, "v2", repo, "blobs", blob.Digest.String()) + getBlobURL.RawQuery = getBlobURL.Query().Encode() + + resp, err := client.R().SetDoNotParseResponse(true).Get(getBlobURL.String()) + if err != nil { + log.Error().Err(err).Msgf("couldn't get cosign blob: %s", blob.Digest.String()) + + return err + } + + if resp.IsError() { + log.Info().Msgf("couldn't find cosign blob from %s, status code: %d", getBlobURL.String(), resp.StatusCode()) + + return errors.ErrBadBlobDigest + } + + defer resp.RawBody().Close() + + // push blob + _, _, err = imageStore.FullBlobUpload(repo, resp.RawBody(), blob.Digest.String()) + if err != nil { + log.Error().Err(err).Msg("couldn't upload cosign blob") + + return err + } + } + + // get config blob + getBlobURL := regURL + getBlobURL.Path = path.Join(getBlobURL.Path, "v2", repo, "blobs", m.Config.Digest.String()) + getBlobURL.RawQuery = getBlobURL.Query().Encode() + + resp, err := client.R().SetDoNotParseResponse(true).Get(getBlobURL.String()) + if err != nil { + log.Error().Err(err).Msgf("couldn't get cosign config blob: %s", getBlobURL.String()) + + return err + } + + if resp.IsError() { + log.Info().Msgf("couldn't find cosign config blob from %s, status code: %d", getBlobURL.String(), resp.StatusCode()) + + return errors.ErrBadBlobDigest + } + + defer resp.RawBody().Close() + + // push config blob + _, _, err = imageStore.FullBlobUpload(repo, resp.RawBody(), m.Config.Digest.String()) + if err != nil { + log.Error().Err(err).Msg("couldn't upload cosign blob") + + return err + } + + // push manifest + _, err = imageStore.PutImageManifest(repo, cosignEncodedDigest, ispec.MediaTypeImageManifest, mResp.Body()) + if err != nil { + log.Error().Err(err).Msg("couldn't upload cosing manifest") + + return err + } + + return nil +} + +func syncNotarySignature(client *resty.Client, storeController storage.StoreController, + regURL url.URL, repo, digest string, log log.Logger) error { + log.Info().Msg("syncing notary signatures") + + getReferrersURL := regURL + getRefManifestURL := regURL + + // based on manifest digest get referrers + getReferrersURL.Path = path.Join(getReferrersURL.Path, "oras/artifacts/v1/", repo, "manifests", digest, "referrers") + getReferrersURL.RawQuery = getReferrersURL.Query().Encode() + + resp, err := client.R(). + SetHeader("Content-Type", "application/json"). + SetQueryParam("artifactType", "application/vnd.cncf.notary.v2.signature"). + Get(getReferrersURL.String()) + if err != nil { + log.Error().Err(err).Msgf("couldn't get referrers from %s", getReferrersURL.String()) + + return err + } + + if resp.IsError() { + log.Info().Msgf("couldn't find any notary signature from %s, status code: %d, skipping", + getReferrersURL.String(), resp.StatusCode()) + + return nil + } + + var referrers ReferenceList + + err = json.Unmarshal(resp.Body(), &referrers) + if err != nil { + log.Error().Err(err).Msgf("couldn't unmarshal notary signature from %s", getReferrersURL.String()) + + return err + } + + imageStore := storeController.GetImageStore(repo) + + for _, ref := range referrers.References { + // get referrer manifest + getRefManifestURL.Path = path.Join(getRefManifestURL.Path, "v2", repo, "manifests", ref.Digest.String()) + getRefManifestURL.RawQuery = getRefManifestURL.Query().Encode() + + resp, err := client.R(). + Get(getRefManifestURL.String()) + if err != nil { + log.Error().Err(err).Msgf("couldn't get notary manifest: %s", getRefManifestURL.String()) + + return err + } + + // read manifest + var m artifactspec.Manifest + + err = json.Unmarshal(resp.Body(), &m) + if err != nil { + log.Error().Err(err).Msgf("couldn't unmarshal notary manifest: %s", getRefManifestURL.String()) + + return err + } + + for _, blob := range m.Blobs { + getBlobURL := regURL + getBlobURL.Path = path.Join(getBlobURL.Path, "v2", repo, "blobs", blob.Digest.String()) + getBlobURL.RawQuery = getBlobURL.Query().Encode() + + resp, err := client.R().SetDoNotParseResponse(true).Get(getBlobURL.String()) + if err != nil { + log.Error().Err(err).Msgf("couldn't get notary blob: %s", getBlobURL.String()) + + return err + } + + defer resp.RawBody().Close() + + if resp.IsError() { + log.Info().Msgf("couldn't find notary blob from %s, status code: %d", + getBlobURL.String(), resp.StatusCode()) + + return errors.ErrBadBlobDigest + } + + _, _, err = imageStore.FullBlobUpload(repo, resp.RawBody(), blob.Digest.String()) + if err != nil { + log.Error().Err(err).Msg("couldn't upload notary sig blob") + + return err + } + } + + _, err = imageStore.PutImageManifest(repo, ref.Digest.String(), artifactspec.MediaTypeArtifactManifest, resp.Body()) + if err != nil { + log.Error().Err(err).Msg("couldn't upload notary sig manifest") + + return err + } + } + + return nil +} + +func syncSignatures(client *resty.Client, storeController storage.StoreController, + registryURL, repo, tag string, log log.Logger) error { + log.Info().Msg("syncing signatures") + // get manifest and find out its digest + regURL, err := url.Parse(registryURL) + if err != nil { + log.Error().Err(err).Msgf("couldn't parse registry URL: %s", registryURL) + + return err + } + + getManifestURL := *regURL + + getManifestURL.Path = path.Join(getManifestURL.Path, "v2", repo, "manifests", tag) + + resp, err := client.R().SetHeader("Content-Type", "application/json").Head(getManifestURL.String()) + if err != nil { + log.Error().Err(err).Str("url", getManifestURL.String()). + Msgf("couldn't query %s", registryURL) + + return err + } + + digests, ok := resp.Header()["Docker-Content-Digest"] + if !ok { + log.Error().Err(errors.ErrBadBlobDigest).Str("url", getManifestURL.String()). + Msgf("couldn't get digest for manifest: %s:%s", repo, tag) + + return errors.ErrBadBlobDigest + } + + if len(digests) != 1 { + log.Error().Err(errors.ErrBadBlobDigest).Str("url", getManifestURL.String()). + Msgf("multiple digests found for: %s:%s", repo, tag) + + return errors.ErrBadBlobDigest + } + + err = syncNotarySignature(client, storeController, *regURL, repo, digests[0], log) + if err != nil { + return err + } + + err = syncCosignSignature(client, storeController, *regURL, repo, digests[0], log) + if err != nil { + return err + } + + log.Info().Msg("successfully synced signatures") + + return nil +} + func pushSyncedLocalImage(repo, tag, uuid string, storeController storage.StoreController, log log.Logger) error { log.Info().Msgf("pushing synced local image %s:%s to local registry", repo, tag) @@ -165,3 +480,13 @@ func pushSyncedLocalImage(repo, tag, uuid string, return nil } + +// sync feature will try to pull cosign signature because for sync cosign signature is just an image +// this function will check if tag is a cosign tag. +func isCosignTag(tag string) bool { + if strings.HasPrefix(tag, "sha256-") && strings.HasSuffix(tag, ".sig") { + return true + } + + return false +} diff --git a/pkg/storage/cache_test.go b/pkg/storage/cache_test.go index 27befdf1..188331b6 100644 --- a/pkg/storage/cache_test.go +++ b/pkg/storage/cache_test.go @@ -37,6 +37,9 @@ func TestCache(t *testing.T) { err = cache.PutBlob("key", path.Join(dir, "value")) So(err, ShouldBeNil) + err = cache.PutBlob("key", "value") + So(err, ShouldNotBeNil) + exists = cache.HasBlob("key", "value") So(exists, ShouldBeTrue)