From d5065513f5bc080a3366e972edb34bf0aa95bfd7 Mon Sep 17 00:00:00 2001 From: Andreea Lupu <58118008+Andreea-Lupu@users.noreply.github.com> Date: Tue, 7 Nov 2023 00:09:39 +0200 Subject: [PATCH] feat: add support for oci1.1 cosign signatures(using referrers) (#1963) - Cosign supports 2 types of signature formats: 1. Using tag -> each new signature of the same manifest is added as a new layer of the signature manifest having that specific tag("{alghoritm}-{digest_of_signed_manifest}.sig") 2. Using referrers -> each new signature of the same manifest is added as a new manifest - For adding these cosign signature to metadb, we reserved index 0 of the list of cosign signatures for tag-based signatures. When a new tag-based signature is added for the same manifest, the element on first position in its list of cosign signatures(in metadb) will be updated/overwritten. When a new cosign signature(using referrers) will be added for the same manifest this new signature will be appended to the list of cosign signatures. Signed-off-by: Andreea-Lupu --- pkg/cli/client/client.go | 22 +++- pkg/cli/client/image_cmd_test.go | 72 ++++++++++-- pkg/common/common.go | 1 + pkg/extensions/search/search_test.go | 16 +-- pkg/extensions/sync/references/oci.go | 2 +- pkg/extensions/sync/references/references.go | 21 ++-- .../references/references_internal_test.go | 11 -- pkg/extensions/sync/sync_test.go | 103 +++++++++++++++++- pkg/meta/boltdb/boltdb.go | 48 +++++--- pkg/meta/dynamodb/dynamodb.go | 48 +++++--- pkg/meta/meta_test.go | 18 ++- pkg/meta/parse.go | 6 + pkg/meta/types/types.go | 1 + pkg/storage/common/common.go | 5 + pkg/storage/gc/gc.go | 4 +- pkg/storage/local/local_test.go | 9 ++ pkg/storage/storage.go | 5 + pkg/test/oci-utils/oci_layout.go | 14 +++ pkg/test/oci-utils/oci_layout_test.go | 94 ++++++++++++++++ pkg/test/signature/cosign.go | 20 +++- test/blackbox/annotations.bats | 76 ++++++++++++- 21 files changed, 511 insertions(+), 85 deletions(-) diff --git a/pkg/cli/client/client.go b/pkg/cli/client/client.go index d2dcf7a6..9388ea42 100644 --- a/pkg/cli/client/client.go +++ b/pkg/cli/client/client.go @@ -509,7 +509,27 @@ func isCosignSigned(ctx context.Context, repo, digestStr string, searchConf Sear _, err := makeGETRequest(ctx, URL, username, password, searchConf.VerifyTLS, searchConf.Debug, &result, searchConf.ResultWriter) - return err == nil + if err == nil { + return true + } + + var referrers ispec.Index + + artifactType := url.QueryEscape(common.ArtifactTypeCosign) + URL = fmt.Sprintf("%s/v2/%s/referrers/%s?artifactType=%s", + searchConf.ServURL, repo, digestStr, artifactType) + + _, err = makeGETRequest(ctx, URL, username, password, searchConf.VerifyTLS, + searchConf.Debug, &referrers, searchConf.ResultWriter) + if err != nil { + return false + } + + if len(referrers.Manifests) == 0 { + return false + } + + return true } func (p *requestsPool) submitJob(job *httpJob) { diff --git a/pkg/cli/client/image_cmd_test.go b/pkg/cli/client/image_cmd_test.go index 00561818..307d6c54 100644 --- a/pkg/cli/client/image_cmd_test.go +++ b/pkg/cli/client/image_cmd_test.go @@ -34,13 +34,17 @@ import ( "zotregistry.io/zot/pkg/test/signature" ) +//nolint:dupl func TestSignature(t *testing.T) { space := regexp.MustCompile(`\s+`) + repoName := "repo7" - Convey("Test from real server", t, func() { + Convey("Test with cosign signature(tag)", t, func() { currentWorkingDir, err := os.Getwd() So(err, ShouldBeNil) + defer func() { _ = os.Chdir(currentWorkingDir) }() + currentDir := t.TempDir() err = os.Chdir(currentDir) So(err, ShouldBeNil) @@ -59,7 +63,6 @@ func TestSignature(t *testing.T) { cm.StartAndWait(conf.HTTP.Port) defer cm.StopServer() - repoName := "repo7" image := CreateDefaultImage() err = UploadImage(image, url, repoName, "1.0") So(err, ShouldBeNil) @@ -108,15 +111,68 @@ func TestSignature(t *testing.T) { actual = strings.TrimSpace(space.ReplaceAllString(buff.String(), " ")) So(actual, ShouldContainSubstring, "REPOSITORY TAG OS/ARCH DIGEST SIGNED SIZE") So(actual, ShouldContainSubstring, "repo7 1.0 linux/amd64 db573b01 true 854B") - - err = os.Chdir(currentWorkingDir) - So(err, ShouldBeNil) }) - Convey("Test with notation signature", t, func() { + Convey("Test with cosign signature(withReferrers)", t, func() { currentWorkingDir, err := os.Getwd() So(err, ShouldBeNil) + defer func() { _ = os.Chdir(currentWorkingDir) }() + + currentDir := t.TempDir() + err = os.Chdir(currentDir) + So(err, ShouldBeNil) + + port := test.GetFreePort() + url := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + defaultVal := true + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, + } + ctlr := api.NewController(conf) + ctlr.Config.Storage.RootDirectory = currentDir + cm := test.NewControllerManager(ctlr) + cm.StartAndWait(conf.HTTP.Port) + defer cm.StopServer() + + err = UploadImage(CreateDefaultImage(), url, repoName, "0.0.1") + So(err, ShouldBeNil) + + err = signature.SignImageUsingCosign("repo7:0.0.1", port, true) + So(err, ShouldBeNil) + + searchConfig := getTestSearchConfig(url, client.NewSearchService()) + + t.Logf("%s", ctlr.Config.Storage.RootDirectory) + + buff := &bytes.Buffer{} + searchConfig.ResultWriter = buff + err = client.SearchAllImagesGQL(searchConfig) + So(err, ShouldBeNil) + + actual := strings.TrimSpace(space.ReplaceAllString(buff.String(), " ")) + So(actual, ShouldContainSubstring, "REPOSITORY TAG OS/ARCH DIGEST SIGNED SIZE") + So(actual, ShouldContainSubstring, "repo7 0.0.1 linux/amd64 db573b01 true 854B") + + t.Log("Test getting all images using rest calls to get catalog and individual manifests") + buff = &bytes.Buffer{} + searchConfig.ResultWriter = buff + err = client.SearchAllImages(searchConfig) + So(err, ShouldBeNil) + + actual = strings.TrimSpace(space.ReplaceAllString(buff.String(), " ")) + So(actual, ShouldContainSubstring, "REPOSITORY TAG OS/ARCH DIGEST SIGNED SIZE") + So(actual, ShouldContainSubstring, "repo7 0.0.1 linux/amd64 db573b01 true 854B") + }) + + Convey("Test with notation signature", t, func() { + currentWorkingDir, err := os.Getwd() + So(err, ShouldBeNil) + + defer func() { _ = os.Chdir(currentWorkingDir) }() + currentDir := t.TempDir() err = os.Chdir(currentDir) So(err, ShouldBeNil) @@ -135,7 +191,6 @@ func TestSignature(t *testing.T) { cm.StartAndWait(conf.HTTP.Port) defer cm.StopServer() - repoName := "repo7" err = UploadImage(CreateDefaultImage(), url, repoName, "0.0.1") So(err, ShouldBeNil) @@ -164,9 +219,6 @@ func TestSignature(t *testing.T) { actual = strings.TrimSpace(space.ReplaceAllString(buff.String(), " ")) So(actual, ShouldContainSubstring, "REPOSITORY TAG OS/ARCH DIGEST SIGNED SIZE") So(actual, ShouldContainSubstring, "repo7 0.0.1 linux/amd64 db573b01 true 854B") - - err = os.Chdir(currentWorkingDir) - So(err, ShouldBeNil) }) } diff --git a/pkg/common/common.go b/pkg/common/common.go index e374e21f..846b7e0c 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -27,6 +27,7 @@ const ( // same value as github.com/notaryproject/notation-go/registry.ArtifactTypeNotation (assert by internal test). // reason used: to reduce zot minimal binary size (otherwise adds oras.land/oras-go/v2 deps). ArtifactTypeNotation = "application/vnd.cncf.notary.signature" + ArtifactTypeCosign = "application/vnd.dev.cosign.artifact.sig.v1+json" ) var cosignTagRule = regexp.MustCompile(`sha256\-.+\.sig`) diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index 95d6dccc..e1d11639 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -1349,7 +1349,7 @@ func TestExpandedRepoInfo(t *testing.T) { } So(found, ShouldEqual, true) - err = signature.SignImageUsingCosign("zot-cve-test:0.0.1", port) + err = signature.SignImageUsingCosign("zot-cve-test:0.0.1", port, false) So(err, ShouldBeNil) resp, err = resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query)) @@ -1421,7 +1421,7 @@ func TestExpandedRepoInfo(t *testing.T) { } So(found, ShouldEqual, true) - err = signature.SignImageUsingCosign("zot-test@"+testManifestDigest.String(), port) + err = signature.SignImageUsingCosign("zot-test@"+testManifestDigest.String(), port, false) So(err, ShouldBeNil) resp, err = resty.R().Get(baseURL + graphqlQueryPrefix + "/query?query=" + url.QueryEscape(query)) @@ -3759,7 +3759,7 @@ func TestGlobalSearchFiltering(t *testing.T) { ) So(err, ShouldBeNil) - err = signature.SignImageUsingCosign("signed-repo:test", port) + err = signature.SignImageUsingCosign("signed-repo:test", port, false) So(err, ShouldBeNil) query := `{ @@ -4323,7 +4323,7 @@ func TestMetaDBWhenSigningImages(t *testing.T) { ` Convey("Sign with cosign", func() { - err = signature.SignImageUsingCosign("repo1:1.0.1", port) + err = signature.SignImageUsingCosign("repo1:1.0.1", port, false) So(err, ShouldBeNil) resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(queryImage1)) @@ -4403,7 +4403,7 @@ func TestMetaDBWhenSigningImages(t *testing.T) { }, } - err := signature.SignImageUsingCosign("repo1:1.0.1", port) + err := signature.SignImageUsingCosign("repo1:1.0.1", port, false) So(err, ShouldNotBeNil) }) }) @@ -4443,7 +4443,7 @@ func TestMetaDBWhenSigningImages(t *testing.T) { }) Convey("Sign with cosign index", func() { - err = signature.SignImageUsingCosign("repo1:index", port) + err = signature.SignImageUsingCosign("repo1:index", port, false) So(err, ShouldBeNil) resp, err := resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(queryIndex)) @@ -4572,7 +4572,7 @@ func RunMetaDBIndexTests(baseURL, port string) { responseImage := responseImages[0] So(len(responseImage.Manifests), ShouldEqual, 3) - err = signature.SignImageUsingCosign(fmt.Sprintf("repo@%s", multiarchImage.DigestStr()), port) + err = signature.SignImageUsingCosign(fmt.Sprintf("repo@%s", multiarchImage.DigestStr()), port, false) So(err, ShouldBeNil) resp, err = resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query)) @@ -5301,7 +5301,7 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { Convey("Delete a cosign signature", func() { repo := "repo1" - err := signature.SignImageUsingCosign("repo1:1.0.1", port) + err := signature.SignImageUsingCosign("repo1:1.0.1", port, false) So(err, ShouldBeNil) query := ` diff --git a/pkg/extensions/sync/references/oci.go b/pkg/extensions/sync/references/oci.go index 60abe254..de1c3a91 100644 --- a/pkg/extensions/sync/references/oci.go +++ b/pkg/extensions/sync/references/oci.go @@ -53,7 +53,7 @@ func (ref OciReferences) IsSigned(ctx context.Context, remoteRepo, subjectDigest return false } - if len(getNotationManifestsFromOCIRefs(index)) > 0 { + if len(getNotationManifestsFromOCIRefs(index)) > 0 || len(getCosignManifestsFromOCIRefs(index)) > 0 { return true } diff --git a/pkg/extensions/sync/references/references.go b/pkg/extensions/sync/references/references.go index 9e448100..c1e5bcb0 100644 --- a/pkg/extensions/sync/references/references.go +++ b/pkg/extensions/sync/references/references.go @@ -17,7 +17,6 @@ import ( "zotregistry.io/zot/pkg/common" client "zotregistry.io/zot/pkg/extensions/sync/httpclient" "zotregistry.io/zot/pkg/log" - "zotregistry.io/zot/pkg/meta" mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/storage" storageTypes "zotregistry.io/zot/pkg/storage/types" @@ -218,20 +217,14 @@ func getNotationManifestsFromOCIRefs(ociRefs ispec.Index) []ispec.Descriptor { return notaryManifests } -func addSigToMeta( - metaDB mTypes.MetaDB, repo, sigType, tag string, signedManifestDig, referenceDigest godigest.Digest, - referenceBuf []byte, imageStore storageTypes.ImageStore, log log.Logger, -) error { - layersInfo, errGetLayers := meta.GetSignatureLayersInfo(repo, tag, referenceDigest.String(), - sigType, referenceBuf, imageStore, log) +func getCosignManifestsFromOCIRefs(ociRefs ispec.Index) []ispec.Descriptor { + cosignManifests := []ispec.Descriptor{} - if errGetLayers != nil { - return errGetLayers + for _, ref := range ociRefs.Manifests { + if ref.ArtifactType == common.ArtifactTypeCosign { + cosignManifests = append(cosignManifests, ref) + } } - return metaDB.AddManifestSignature(repo, signedManifestDig, mTypes.SignatureMetadata{ - SignatureType: sigType, - SignatureDigest: referenceDigest.String(), - LayersInfo: layersInfo, - }) + return cosignManifests } diff --git a/pkg/extensions/sync/references/references_internal_test.go b/pkg/extensions/sync/references/references_internal_test.go index 7a85c2fd..38511683 100644 --- a/pkg/extensions/sync/references/references_internal_test.go +++ b/pkg/extensions/sync/references/references_internal_test.go @@ -440,14 +440,3 @@ func TestCompareArtifactRefs(t *testing.T) { } }) } - -func TestAddSigToMeta(t *testing.T) { - Convey("Test addSigToMeta", t, func() { - imageStore := mocks.MockedImageStore{} - metaDB := mocks.MetaDBMock{} - - err := addSigToMeta(metaDB, "repo", "cosign", "tag", godigest.FromString("signedmanifest"), - godigest.FromString("reference"), []byte("bad"), imageStore, log.Logger{}) - So(err, ShouldNotBeNil) - }) -} diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 71ba125d..35901ae2 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -746,7 +746,7 @@ func TestOnDemand(t *testing.T) { So(err, ShouldBeNil) // sign using cosign - err = signature.SignImageUsingCosign(fmt.Sprintf("remote-repo@%s", manifestDigest.String()), port) + err = signature.SignImageUsingCosign(fmt.Sprintf("remote-repo@%s", manifestDigest.String()), port, false) So(err, ShouldBeNil) // add cosign sbom @@ -4593,6 +4593,100 @@ func TestSignatures(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusOK) }) + + Convey("Verify sync oci1.1 cosign signatures", t, func() { + updateDuration, _ := time.ParseDuration("30m") + + sctlr, srcBaseURL, _, _, _ := makeUpstreamServer(t, false, false) + + scm := test.NewControllerManager(sctlr) + scm.StartAndWait(sctlr.Config.HTTP.Port) + defer scm.StopServer() + + // create repo, push and sign it + repoName := testSignedImage + var digest godigest.Digest + So(func() { digest = pushRepo(srcBaseURL, repoName) }, ShouldNotPanic) + + splittedURL := strings.SplitAfter(srcBaseURL, ":") + srcPort := splittedURL[len(splittedURL)-1] + t.Logf(srcPort) + + err := signature.SignImageUsingCosign(fmt.Sprintf("%s@%s", repoName, digest.String()), srcPort, true) + So(err, ShouldBeNil) + + regex := ".*" + var semver bool + var tlsVerify bool + onlySigned := true + + syncRegistryConfig := syncconf.RegistryConfig{ + Content: []syncconf.Content{ + { + Prefix: "**", + Tags: &syncconf.Tags{ + Regex: ®ex, + Semver: &semver, + }, + }, + }, + URLs: []string{srcBaseURL}, + PollInterval: updateDuration, + TLSVerify: &tlsVerify, + CertDir: "", + OnlySigned: &onlySigned, + OnDemand: true, + } + + defaultVal := true + syncConfig := &syncconf.Config{ + Enable: &defaultVal, + Registries: []syncconf.RegistryConfig{syncRegistryConfig}, + } + + dctlr, destBaseURL, _, destClient := makeDownstreamServer(t, false, syncConfig) + + dcm := test.NewControllerManager(dctlr) + dcm.StartAndWait(dctlr.Config.HTTP.Port) + defer dcm.StopServer() + + // wait for sync + var destTagsList TagsList + + for { + resp, err := destClient.R().Get(destBaseURL + "/v2/" + repoName + "/tags/list") + if err != nil { + panic(err) + } + + err = json.Unmarshal(resp.Body(), &destTagsList) + if err != nil { + panic(err) + } + + if len(destTagsList.Tags) > 0 { + break + } + + time.Sleep(500 * time.Millisecond) + } + + time.Sleep(1 * time.Second) + + // get oci references from downstream, should be synced + getOCIReferrersURL := destBaseURL + path.Join("/v2", repoName, "referrers", digest.String()) + resp, err := resty.R().Get(getOCIReferrersURL) + So(err, ShouldBeNil) + So(resp, ShouldNotBeEmpty) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + var index ispec.Index + + err = json.Unmarshal(resp.Body(), &index) + So(err, ShouldBeNil) + + So(len(index.Manifests), ShouldEqual, 3) + }) } func getPortFromBaseURL(baseURL string) string { @@ -4626,7 +4720,10 @@ func TestSyncedSignaturesMetaDB(t *testing.T) { err = signature.SignImageUsingNotary(repoName+":"+tag, srcPort, true) So(err, ShouldBeNil) - err = signature.SignImageUsingCosign(repoName+":"+tag, srcPort) + err = signature.SignImageUsingCosign(repoName+":"+tag, srcPort, true) + So(err, ShouldBeNil) + + err = signature.SignImageUsingCosign(repoName+":"+tag, srcPort, false) So(err, ShouldBeNil) // Create destination registry @@ -4676,7 +4773,7 @@ func TestSyncedSignaturesMetaDB(t *testing.T) { imageSignatures := repoMeta.Signatures[signedImage.DigestStr()] So(imageSignatures, ShouldContainKey, zcommon.CosignSignature) - So(len(imageSignatures[zcommon.CosignSignature]), ShouldEqual, 1) + So(len(imageSignatures[zcommon.CosignSignature]), ShouldEqual, 2) So(imageSignatures, ShouldContainKey, zcommon.NotationSignature) So(len(imageSignatures[zcommon.NotationSignature]), ShouldEqual, 1) }) diff --git a/pkg/meta/boltdb/boltdb.go b/pkg/meta/boltdb/boltdb.go index 45e9d9bb..904f5e3b 100644 --- a/pkg/meta/boltdb/boltdb.go +++ b/pkg/meta/boltdb/boltdb.go @@ -813,7 +813,7 @@ func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta } func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, - sygMeta mTypes.SignatureMetadata, + sigMeta mTypes.SignatureMetadata, ) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error { repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) @@ -829,11 +829,11 @@ func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godige Signatures: map[string]*proto_go.ManifestSignatures{ signedManifestDigest.String(): { Map: map[string]*proto_go.SignaturesInfo{ - sygMeta.SignatureType: { + sigMeta.SignatureType: { List: []*proto_go.SignatureInfo{ { - SignatureManifestDigest: sygMeta.SignatureDigest, - LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), + SignatureManifestDigest: sigMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sigMeta.LayersInfo), }, }, }, @@ -867,26 +867,46 @@ func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godige } signatureSlice := &proto_go.SignaturesInfo{List: []*proto_go.SignatureInfo{}} - if sigSlice, found := manifestSignatures.Map[sygMeta.SignatureType]; found { + if sigSlice, found := manifestSignatures.Map[sigMeta.SignatureType]; found { signatureSlice = sigSlice } - if !common.ProtoSignatureAlreadyExists(signatureSlice.List, sygMeta) { - switch sygMeta.SignatureType { + if !common.ProtoSignatureAlreadyExists(signatureSlice.List, sigMeta) { + switch sigMeta.SignatureType { case zcommon.NotationSignature: signatureSlice.List = append(signatureSlice.List, &proto_go.SignatureInfo{ - SignatureManifestDigest: sygMeta.SignatureDigest, - LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), + SignatureManifestDigest: sigMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sigMeta.LayersInfo), }) case zcommon.CosignSignature: - signatureSlice.List = []*proto_go.SignatureInfo{{ - SignatureManifestDigest: sygMeta.SignatureDigest, - LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), - }} + newCosignSig := &proto_go.SignatureInfo{ + SignatureManifestDigest: sigMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sigMeta.LayersInfo), + } + + if zcommon.IsCosignTag(sigMeta.SignatureTag) { + // the entry for "sha256-{digest}.sig" signatures should be overwritten if + // it exists or added on the first position if it doesn't exist + if len(signatureSlice.GetList()) == 0 { + signatureSlice.List = []*proto_go.SignatureInfo{newCosignSig} + } else { + signatureSlice.List[0] = newCosignSig + } + } else { + // the first position should be reserved for "sha256-{digest}.sig" signatures + if len(signatureSlice.GetList()) == 0 { + signatureSlice.List = []*proto_go.SignatureInfo{{ + SignatureManifestDigest: "", + LayersInfo: []*proto_go.LayersInfo{}, + }} + } + + signatureSlice.List = append(signatureSlice.List, newCosignSig) + } } } - manifestSignatures.Map[sygMeta.SignatureType] = signatureSlice + manifestSignatures.Map[sigMeta.SignatureType] = signatureSlice protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures return setProtoRepoMeta(protoRepoMeta, repoMetaBuck) diff --git a/pkg/meta/dynamodb/dynamodb.go b/pkg/meta/dynamodb/dynamodb.go index 5d3c2fcb..92ec2271 100644 --- a/pkg/meta/dynamodb/dynamodb.go +++ b/pkg/meta/dynamodb/dynamodb.go @@ -1041,7 +1041,7 @@ func (dwr *DynamoDB) UpdateSignaturesValidity(repo string, manifestDigest godige } func (dwr *DynamoDB) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, - sygMeta mTypes.SignatureMetadata, + sigMeta mTypes.SignatureMetadata, ) error { protoRepoMeta, err := dwr.getProtoRepoMeta(context.Background(), repo) if err != nil { @@ -1054,11 +1054,11 @@ func (dwr *DynamoDB) AddManifestSignature(repo string, signedManifestDigest godi Signatures: map[string]*proto_go.ManifestSignatures{ signedManifestDigest.String(): { Map: map[string]*proto_go.SignaturesInfo{ - sygMeta.SignatureType: { + sigMeta.SignatureType: { List: []*proto_go.SignatureInfo{ { - SignatureManifestDigest: sygMeta.SignatureDigest, - LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), + SignatureManifestDigest: sigMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sigMeta.LayersInfo), }, }, }, @@ -1083,26 +1083,46 @@ func (dwr *DynamoDB) AddManifestSignature(repo string, signedManifestDigest godi } signatureSlice := &proto_go.SignaturesInfo{List: []*proto_go.SignatureInfo{}} - if sigSlice, found := manifestSignatures.Map[sygMeta.SignatureType]; found { + if sigSlice, found := manifestSignatures.Map[sigMeta.SignatureType]; found { signatureSlice = sigSlice } - if !common.ProtoSignatureAlreadyExists(signatureSlice.List, sygMeta) { - switch sygMeta.SignatureType { + if !common.ProtoSignatureAlreadyExists(signatureSlice.List, sigMeta) { + switch sigMeta.SignatureType { case zcommon.NotationSignature: signatureSlice.List = append(signatureSlice.List, &proto_go.SignatureInfo{ - SignatureManifestDigest: sygMeta.SignatureDigest, - LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), + SignatureManifestDigest: sigMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sigMeta.LayersInfo), }) case zcommon.CosignSignature: - signatureSlice.List = []*proto_go.SignatureInfo{{ - SignatureManifestDigest: sygMeta.SignatureDigest, - LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), - }} + newCosignSig := &proto_go.SignatureInfo{ + SignatureManifestDigest: sigMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sigMeta.LayersInfo), + } + + if zcommon.IsCosignTag(sigMeta.SignatureTag) { + // the entry for "sha256-{digest}.sig" signatures should be overwritten if + // it exists or added on the first position if it doesn't exist + if len(signatureSlice.GetList()) == 0 { + signatureSlice.List = []*proto_go.SignatureInfo{newCosignSig} + } else { + signatureSlice.List[0] = newCosignSig + } + } else { + // the first position should be reserved for "sha256-{digest}.sig" signatures + if len(signatureSlice.GetList()) == 0 { + signatureSlice.List = []*proto_go.SignatureInfo{{ + SignatureManifestDigest: "", + LayersInfo: []*proto_go.LayersInfo{}, + }} + } + + signatureSlice.List = append(signatureSlice.List, newCosignSig) + } } } - manifestSignatures.Map[sygMeta.SignatureType] = signatureSlice + manifestSignatures.Map[sigMeta.SignatureType] = signatureSlice protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures return dwr.setProtoRepoMeta(protoRepoMeta.Name, protoRepoMeta) diff --git a/pkg/meta/meta_test.go b/pkg/meta/meta_test.go index c4e79872..14fb30fc 100644 --- a/pkg/meta/meta_test.go +++ b/pkg/meta/meta_test.go @@ -1290,18 +1290,31 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) So(err, ShouldBeNil) + err = metaDB.AddManifestSignature(repo1, image1.Digest, mTypes.SignatureMetadata{ + SignatureType: "cosign", + SignatureTag: fmt.Sprintf("sha256-%s.sig", image1.Digest.Encoded()), + SignatureDigest: "digesttag", + LayersInfo: []mTypes.LayerInfo{{LayerDigest: "layer-digest", LayerContent: []byte{10}}}, + }) + So(err, ShouldBeNil) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) So(repoMeta.Signatures[image1.Digest.String()]["cosign"][0].SignatureManifestDigest, + ShouldResemble, "digesttag") + So(repoMeta.Signatures[image1.Digest.String()]["cosign"][1].SignatureManifestDigest, ShouldResemble, "digest") imageMeta, err := metaDB.GetImageMeta(image1.Digest) fullImageMeta := convert.GetFullImageMeta(tag1, repoMeta, imageMeta) So(err, ShouldBeNil) - So(fullImageMeta.Signatures["cosign"][0].SignatureManifestDigest, ShouldResemble, "digest") + So(fullImageMeta.Signatures["cosign"][0].SignatureManifestDigest, ShouldResemble, "digesttag") So(fullImageMeta.Signatures["cosign"][0].LayersInfo[0].LayerDigest, ShouldResemble, "layer-digest") So(fullImageMeta.Signatures["cosign"][0].LayersInfo[0].LayerContent[0], ShouldEqual, 10) + So(fullImageMeta.Signatures["cosign"][1].SignatureManifestDigest, ShouldResemble, "digest") + So(fullImageMeta.Signatures["cosign"][1].LayersInfo[0].LayerDigest, ShouldResemble, "layer-digest") + So(fullImageMeta.Signatures["cosign"][1].LayersInfo[0].LayerContent[0], ShouldEqual, 10) }) Convey("Test UpdateSignaturesValidity", func() { @@ -1320,6 +1333,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.AddManifestSignature(repo1, image1.Digest(), mTypes.SignatureMetadata{ SignatureType: "cosign", SignatureDigest: image1.DigestStr(), + SignatureTag: fmt.Sprintf("sha256-%s.sig", image1.Digest().Encoded()), LayersInfo: []mTypes.LayerInfo{layerInfo}, }) So(err, ShouldBeNil) @@ -1442,6 +1456,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err := metaDB.AddManifestSignature(repo1, image1.Digest(), mTypes.SignatureMetadata{ SignatureType: "cosign", + SignatureTag: fmt.Sprintf("sha256-%s.sig", image1.Digest().Encoded()), SignatureDigest: "digest", }) So(err, ShouldBeNil) @@ -1467,6 +1482,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = metaDB.AddManifestSignature(repo1, image1.Digest(), mTypes.SignatureMetadata{ SignatureType: "cosign", + SignatureTag: fmt.Sprintf("sha256-%s.sig", image1.Digest().Encoded()), SignatureDigest: "digest", }) So(err, ShouldBeNil) diff --git a/pkg/meta/parse.go b/pkg/meta/parse.go index a69933aa..f6979fbd 100644 --- a/pkg/meta/parse.go +++ b/pkg/meta/parse.go @@ -289,6 +289,7 @@ func SetImageMetaFromInput(ctx context.Context, repo, reference, mediaType strin mTypes.SignatureMetadata{ SignatureType: sigType, SignatureDigest: digest.String(), + SignatureTag: reference, LayersInfo: layers, }) if err != nil { @@ -342,6 +343,11 @@ func isSignature(reference string, manifestContent ispec.Manifest) (bool, string return true, NotationType, manifestContent.Subject.Digest } + // check cosign signature + if manifestArtifactType == zcommon.ArtifactTypeCosign && manifestContent.Subject != nil { + return true, CosignType, manifestContent.Subject.Digest + } + if tag := reference; zcommon.IsCosignTag(reference) { prefixLen := len("sha256-") digestLen := 64 diff --git a/pkg/meta/types/types.go b/pkg/meta/types/types.go index 0936ae38..c7be5307 100644 --- a/pkg/meta/types/types.go +++ b/pkg/meta/types/types.go @@ -298,6 +298,7 @@ type SignatureInfo struct { type SignatureMetadata struct { SignatureType string SignatureDigest string + SignatureTag string LayersInfo []LayerInfo } diff --git a/pkg/storage/common/common.go b/pkg/storage/common/common.go index 1a3fb35e..ab610588 100644 --- a/pkg/storage/common/common.go +++ b/pkg/storage/common/common.go @@ -613,6 +613,11 @@ func IsSignature(descriptor ispec.Descriptor) bool { return true } + // is cosign signature (OCI 1.1 support) + if descriptor.ArtifactType == zcommon.ArtifactTypeCosign { + return true + } + // is notation signature if descriptor.ArtifactType == zcommon.ArtifactTypeNotation { return true diff --git a/pkg/storage/gc/gc.go b/pkg/storage/gc/gc.go index 556bf57e..0dace3d7 100644 --- a/pkg/storage/gc/gc.go +++ b/pkg/storage/gc/gc.go @@ -277,9 +277,11 @@ func (gc GarbageCollect) removeReferrer(repo string, index *ispec.Index, manifes referenced := isManifestReferencedInIndex(index, subject.Digest) var signatureType string - // check if its notation signature + // check if its notation or cosign signature if artifactType == zcommon.ArtifactTypeNotation { signatureType = storage.NotationType + } else if artifactType == zcommon.ArtifactTypeCosign { + signatureType = storage.CosignType } if !referenced { diff --git a/pkg/storage/local/local_test.go b/pkg/storage/local/local_test.go index ebcfedd4..60452b45 100644 --- a/pkg/storage/local/local_test.go +++ b/pkg/storage/local/local_test.go @@ -2154,6 +2154,15 @@ func TestGarbageCollectForImageStore(t *testing.T) { err = WriteImageToFileSystem(notationSig, repoName, "notation", storeController) So(err, ShouldBeNil) + // add fake signature for tag1 + cosignWithReferrersSig := CreateImageWith(). + RandomLayers(1, 10). + ArtifactConfig(common.ArtifactTypeCosign). + Subject(img.DescriptorRef()).Build() + + err = WriteImageToFileSystem(cosignWithReferrersSig, repoName, "cosign", storeController) + So(err, ShouldBeNil) + err = gc.CleanRepo(repoName) So(err, ShouldBeNil) }) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 7d3cf4cd..e9f5e37e 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -227,6 +227,11 @@ func CheckIsImageSignature(repoName string, manifestBlob []byte, reference strin return true, NotationType, manifestContent.Subject.Digest, nil } + // check cosign signature (OCI 1.1 support) + if manifestArtifactType == zcommon.ArtifactTypeCosign && manifestContent.Subject != nil { + return true, CosignType, manifestContent.Subject.Digest, nil + } + if tag := reference; zcommon.IsCosignTag(reference) { prefixLen := len("sha256-") digestLen := 64 diff --git a/pkg/test/oci-utils/oci_layout.go b/pkg/test/oci-utils/oci_layout.go index 7d3528a5..a8c5d54c 100644 --- a/pkg/test/oci-utils/oci_layout.go +++ b/pkg/test/oci-utils/oci_layout.go @@ -262,7 +262,21 @@ func (olu BaseOciLayoutUtils) checkCosignSignature(name string, digest godigest. reference := fmt.Sprintf("sha256-%s.sig", digest.Encoded()) _, _, _, err := imageStore.GetImageManifest(name, reference) //nolint: dogsled + if err == nil { + return true + } + + mediaType := common.ArtifactTypeCosign + + referrers, err := imageStore.GetReferrers(name, digest, []string{mediaType}) if err != nil { + olu.Log.Info().Err(err).Str("repository", name).Str("digest", + digest.String()).Str("mediatype", mediaType).Msg("invalid cosign signature") + + return false + } + + if len(referrers.Manifests) == 0 { olu.Log.Info().Err(err).Str("repository", name).Str("digest", digest.String()).Msg("invalid cosign signature") diff --git a/pkg/test/oci-utils/oci_layout_test.go b/pkg/test/oci-utils/oci_layout_test.go index 31319a71..e9cd6a4e 100644 --- a/pkg/test/oci-utils/oci_layout_test.go +++ b/pkg/test/oci-utils/oci_layout_test.go @@ -331,6 +331,100 @@ func TestBaseOciLayoutUtils(t *testing.T) { isSigned = olu.CheckManifestSignature(repo, manifestList[0].Digest) So(isSigned, ShouldBeTrue) }) + + //nolint: dupl + Convey("CheckManifestSignature: cosign(tag)", t, func() { + // checkCosignSignature -> true (tag) + dir := t.TempDir() + + port := tcommon.GetFreePort() + baseURL := tcommon.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + conf.Storage.RootDirectory = dir + defaultVal := true + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, + } + + conf.Extensions.Search.CVE = nil + + ctlr := api.NewController(conf) + + ctlrManager := tcommon.NewControllerManager(ctlr) + ctlrManager.StartAndWait(port) + defer ctlrManager.StopServer() + + // push test image to repo + image := CreateRandomImage() + + repo := "repo2" + tag := "1.0.2" + err := UploadImage(image, baseURL, repo, tag) + So(err, ShouldBeNil) + + olu := ociutils.NewBaseOciLayoutUtils(ctlr.StoreController, log.NewLogger("debug", "")) + manifestList, err := olu.GetImageManifests(repo) + So(err, ShouldBeNil) + So(len(manifestList), ShouldEqual, 1) + + isSigned := olu.CheckManifestSignature(repo, manifestList[0].Digest) + So(isSigned, ShouldBeFalse) + + // checkCosignSignature -> true (tag) + err = signature.SignImageUsingCosign(fmt.Sprintf("%s:%s", repo, tag), port, false) + So(err, ShouldBeNil) + + isSigned = olu.CheckManifestSignature(repo, manifestList[0].Digest) + So(isSigned, ShouldBeTrue) + }) + + //nolint: dupl + Convey("CheckManifestSignature: cosign(with referrers)", t, func() { + // checkCosignSignature -> true (referrers) + dir := t.TempDir() + + port := tcommon.GetFreePort() + baseURL := tcommon.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + conf.Storage.RootDirectory = dir + defaultVal := true + conf.Extensions = &extconf.ExtensionConfig{ + Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}}, + } + + conf.Extensions.Search.CVE = nil + + ctlr := api.NewController(conf) + + ctlrManager := tcommon.NewControllerManager(ctlr) + ctlrManager.StartAndWait(port) + defer ctlrManager.StopServer() + + // push test image to repo + image := CreateRandomImage() + + repo := "repo3" + tag := "1.0.3" + err := UploadImage(image, baseURL, repo, tag) + So(err, ShouldBeNil) + + olu := ociutils.NewBaseOciLayoutUtils(ctlr.StoreController, log.NewLogger("debug", "")) + manifestList, err := olu.GetImageManifests(repo) + So(err, ShouldBeNil) + So(len(manifestList), ShouldEqual, 1) + + isSigned := olu.CheckManifestSignature(repo, manifestList[0].Digest) + So(isSigned, ShouldBeFalse) + + // checkCosignSignature -> true (referrers) + err = signature.SignImageUsingCosign(fmt.Sprintf("%s:%s", repo, tag), port, true) + So(err, ShouldBeNil) + + isSigned = olu.CheckManifestSignature(repo, manifestList[0].Digest) + So(isSigned, ShouldBeTrue) + }) } func TestExtractImageDetails(t *testing.T) { diff --git a/pkg/test/signature/cosign.go b/pkg/test/signature/cosign.go index 7ef0a899..52d6d646 100644 --- a/pkg/test/signature/cosign.go +++ b/pkg/test/signature/cosign.go @@ -30,7 +30,7 @@ func GetCosignSignatureTagForDigest(manifestDigest godigest.Digest) string { return manifestDigest.Algorithm().String() + "-" + manifestDigest.Encoded() + ".sig" } -func SignImageUsingCosign(repoTag, port string) error { +func SignImageUsingCosign(repoTag, port string, withReferrers bool) error { cwd, err := os.Getwd() if err != nil { return err @@ -59,13 +59,21 @@ func SignImageUsingCosign(repoTag, port string) error { const timeoutPeriod = 5 + signOpts := options.SignOptions{ + Registry: options.RegistryOptions{AllowInsecure: true}, + AnnotationOptions: options.AnnotationOptions{Annotations: []string{"tag=1.0"}}, + Upload: true, + } + + if withReferrers { + signOpts.RegistryExperimental = options.RegistryExperimentalOptions{ + RegistryReferrersMode: options.RegistryReferrersModeOCI11, + } + } + // sign the image return sign.SignCmd(&options.RootOptions{Verbose: true, Timeout: timeoutPeriod * time.Minute}, options.KeyOpts{KeyRef: path.Join(tdir, "cosign.key"), PassFunc: generate.GetPass}, - options.SignOptions{ - Registry: options.RegistryOptions{AllowInsecure: true}, - AnnotationOptions: options.AnnotationOptions{Annotations: []string{"tag=1.0"}}, - Upload: true, - }, + signOpts, []string{imageURL}) } diff --git a/test/blackbox/annotations.bats b/test/blackbox/annotations.bats index bfada64b..2cdd9019 100644 --- a/test/blackbox/annotations.bats +++ b/test/blackbox/annotations.bats @@ -115,7 +115,7 @@ function teardown_file() { [ $(echo "${lines[-1]}" | jq '.data.ImageList.Results[0].Licenses') = '"GPLv2"' ] } -@test "sign/verify with cosign" { +@test "sign/verify with cosign (only tag-based signatures)" { run curl -X POST -H "Content-Type: application/json" --data '{ "query": "{ ImageList(repo: \"annotations\") { Results { RepoName Tag Manifests {Digest ConfigDigest Size Layers { Size Digest }} Vendor Licenses }}}"}' http://localhost:8080/v2/_zot/ext/search [ "$status" -eq 0 ] [ $(echo "${lines[-1]}" | jq '.data.ImageList.Results[0].RepoName') = '"annotations"' ] @@ -133,6 +133,80 @@ function teardown_file() { [[ "$sigName" == *"${digest}"* ]] } +@test "sign/verify with cosign (only referrers)" { + run curl -X POST -H "Content-Type: application/json" --data '{ "query": "{ ImageList(repo: \"annotations\") { Results { RepoName Tag Manifests {Digest ConfigDigest Size Layers { Size Digest }} Vendor Licenses }}}"}' http://localhost:8080/v2/_zot/ext/search + [ "$status" -eq 0 ] + [ $(echo "${lines[-1]}" | jq '.data.ImageList.Results[0].RepoName') = '"annotations"' ] + local digest=$(echo "${lines[-1]}" | jq -r '.data.ImageList.Results[0].Manifests[0].Digest') + + export COSIGN_OCI_EXPERIMENTAL=1 + export COSIGN_EXPERIMENTAL=1 + run cosign initialize + [ "$status" -eq 0 ] + run cosign generate-key-pair --output-key-prefix "${BATS_FILE_TMPDIR}/cosign-sign-test-experimental" + [ "$status" -eq 0 ] + run cosign sign --registry-referrers-mode=oci-1-1 --key ${BATS_FILE_TMPDIR}/cosign-sign-test-experimental.key localhost:8080/annotations:latest --yes + [ "$status" -eq 0 ] + run cosign verify --key ${BATS_FILE_TMPDIR}/cosign-sign-test-experimental.pub localhost:8080/annotations:latest + [ "$status" -eq 0 ] + local sigName=$(echo "${lines[-1]}" | jq '.[].critical.image."docker-manifest-digest"') + [[ "$sigName" == *"${digest}"* ]] + unset COSIGN_OCI_EXPERIMENTAL + unset COSIGN_EXPERIMENTAL +} + +@test "sign/verify with cosign (tag and referrers)" { + run curl -X POST -H "Content-Type: application/json" --data '{ "query": "{ ImageList(repo: \"annotations\") { Results { RepoName Tag Manifests {Digest ConfigDigest Size Layers { Size Digest }} Vendor Licenses }}}"}' http://localhost:8080/v2/_zot/ext/search + [ "$status" -eq 0 ] + [ $(echo "${lines[-1]}" | jq '.data.ImageList.Results[0].RepoName') = '"annotations"' ] + local digest=$(echo "${lines[-1]}" | jq -r '.data.ImageList.Results[0].Manifests[0].Digest') + + export COSIGN_OCI_EXPERIMENTAL=1 + export COSIGN_EXPERIMENTAL=1 + run cosign initialize + [ "$status" -eq 0 ] + + run cosign generate-key-pair --output-key-prefix "${BATS_FILE_TMPDIR}/cosign-sign-test-tag-1" + [ "$status" -eq 0 ] + run cosign sign --key ${BATS_FILE_TMPDIR}/cosign-sign-test-tag-1.key localhost:8080/annotations:latest --yes + [ "$status" -eq 0 ] + + run cosign generate-key-pair --output-key-prefix "${BATS_FILE_TMPDIR}/cosign-sign-test-referrers-1" + [ "$status" -eq 0 ] + run cosign sign --registry-referrers-mode=oci-1-1 --key ${BATS_FILE_TMPDIR}/cosign-sign-test-referrers-1.key localhost:8080/annotations:latest --yes + [ "$status" -eq 0 ] + + run cosign generate-key-pair --output-key-prefix "${BATS_FILE_TMPDIR}/cosign-sign-test-tag-2" + [ "$status" -eq 0 ] + run cosign sign --key ${BATS_FILE_TMPDIR}/cosign-sign-test-tag-2.key localhost:8080/annotations:latest --yes + [ "$status" -eq 0 ] + + run cosign verify --key ${BATS_FILE_TMPDIR}/cosign-sign-test-tag-1.pub localhost:8080/annotations:latest + [ "$status" -eq 0 ] + local sigName=$(echo "${lines[-1]}" | jq '.[].critical.image."docker-manifest-digest"') + [[ "$sigName" == *"${digest}"* ]] + run cosign verify --key ${BATS_FILE_TMPDIR}/cosign-sign-test-tag-2.pub localhost:8080/annotations:latest + [ "$status" -eq 0 ] + local sigName=$(echo "${lines[-1]}" | jq '.[].critical.image."docker-manifest-digest"') + [[ "$sigName" == *"${digest}"* ]] + run cosign verify --key ${BATS_FILE_TMPDIR}/cosign-sign-test-referrers-1.pub localhost:8080/annotations:latest + [ "$status" -eq 0 ] + local sigName=$(echo "${lines[-1]}" | jq '.[].critical.image."docker-manifest-digest"') + [[ "$sigName" == *"${digest}"* ]] + + run cosign generate-key-pair --output-key-prefix "${BATS_FILE_TMPDIR}/cosign-sign-test-referrers-2" + [ "$status" -eq 0 ] + run cosign sign --registry-referrers-mode=oci-1-1 --key ${BATS_FILE_TMPDIR}/cosign-sign-test-referrers-2.key localhost:8080/annotations:latest --yes + [ "$status" -eq 0 ] + run cosign verify --key ${BATS_FILE_TMPDIR}/cosign-sign-test-referrers-2.pub localhost:8080/annotations:latest + [ "$status" -eq 0 ] + local sigName=$(echo "${lines[-1]}" | jq '.[].critical.image."docker-manifest-digest"') + [[ "$sigName" == *"${digest}"* ]] + + unset COSIGN_OCI_EXPERIMENTAL + unset COSIGN_EXPERIMENTAL +} + @test "sign/verify with notation" { run curl -X POST -H "Content-Type: application/json" --data '{ "query": "{ ImageList(repo: \"annotations\") { Results { RepoName Tag Manifests {Digest ConfigDigest Size Layers { Size Digest }} Vendor Licenses }}}"}' http://localhost:8080/v2/_zot/ext/search [ "$status" -eq 0 ]