From 56ad9e6707e16be94198814f33d7556355705d3b Mon Sep 17 00:00:00 2001 From: LaurentiuNiculae Date: Mon, 30 Oct 2023 22:06:04 +0200 Subject: [PATCH] refactor(metadb): improve UX by speeding up metadb serialize/deserialize (#1842) Use protocol buffers and update the metadb interface to better suit our search needs Signed-off-by: Ramkumar Chinchani Signed-off-by: Laurentiu Niculae Co-authored-by: Ramkumar Chinchani --- .github/workflows/golangci-lint.yaml | 1 + .gitignore | 1 + Makefile | 74 + codecov.yml | 1 + errors/errors.go | 5 +- examples/config-all-remote.json | 3 +- examples/config-dynamodb.json | 3 +- go.mod | 2 +- pkg/api/controller_test.go | 137 +- pkg/api/routes.go | 2 +- pkg/cli/client/cve_cmd_test.go | 14 +- pkg/common/common.go | 6 + pkg/extensions/extension_image_trust_test.go | 24 +- pkg/extensions/imagetrust/image_trust.go | 26 +- .../imagetrust/image_trust_disabled.go | 4 +- .../imagetrust/image_trust_disabled_test.go | 8 +- pkg/extensions/imagetrust/image_trust_test.go | 130 +- .../search/convert/convert_internal_test.go | 49 +- pkg/extensions/search/convert/convert_test.go | 561 ++-- pkg/extensions/search/convert/metadb.go | 918 +++--- pkg/extensions/search/convert/oci.go | 13 - pkg/extensions/search/cve/cve.go | 30 +- pkg/extensions/search/cve/cve_test.go | 397 +-- pkg/extensions/search/cve/pagination_test.go | 76 +- pkg/extensions/search/cve/scan.go | 33 +- pkg/extensions/search/cve/scan_test.go | 176 +- pkg/extensions/search/cve/trivy/scanner.go | 101 +- .../search/cve/trivy/scanner_internal_test.go | 242 +- .../search/cve/trivy/scanner_test.go | 28 - pkg/extensions/search/cve/update_test.go | 4 +- pkg/extensions/search/resolver.go | 442 +-- pkg/extensions/search/resolver_test.go | 2584 ++++------------ pkg/extensions/search/search_test.go | 119 +- pkg/extensions/search/userprefs_test.go | 24 +- pkg/extensions/sync/references/cosign.go | 18 +- pkg/extensions/sync/references/oci.go | 19 +- .../sync/references/referrers_tag.go | 19 +- pkg/extensions/sync/sync_internal_test.go | 7 +- pkg/extensions/sync/sync_test.go | 10 +- pkg/meta/boltdb/boltdb.go | 2322 +++++++-------- pkg/meta/boltdb/boltdb_test.go | 705 +---- pkg/meta/boltdb/buckets.go | 12 +- pkg/meta/common/common.go | 308 +- pkg/meta/common/common_test.go | 149 - pkg/meta/convert/convert.go | 611 ++++ pkg/meta/convert/convert_proto.go | 392 +++ pkg/meta/dynamodb/dynamodb.go | 2589 ++++++++--------- pkg/meta/dynamodb/dynamodb_internal_test.go | 47 +- pkg/meta/dynamodb/dynamodb_test.go | 925 +----- pkg/meta/dynamodb/parameters.go | 2 +- pkg/meta/hooks.go | 83 +- pkg/meta/hooks_test.go | 262 +- pkg/meta/meta.go | 20 +- pkg/meta/meta_test.go | 2122 ++++++-------- pkg/meta/parse.go | 316 +- pkg/meta/parse_test.go | 327 +-- pkg/meta/proto/gen/config.pb.go | 628 ++++ pkg/meta/proto/gen/descriptor.pb.go | 328 +++ pkg/meta/proto/gen/index.pb.go | 217 ++ pkg/meta/proto/gen/manifest.pb.go | 229 ++ pkg/meta/proto/gen/meta.pb.go | 1521 ++++++++++ pkg/meta/proto/gen/versioned.pb.go | 142 + pkg/meta/proto/meta/meta.proto | 115 + pkg/meta/proto/oci/config.proto | 45 + pkg/meta/proto/oci/descriptor.proto | 23 + pkg/meta/proto/oci/index.proto | 16 + pkg/meta/proto/oci/manifest.proto | 17 + pkg/meta/proto/oci/timestamp.proto | 144 + pkg/meta/proto/oci/versioned.proto | 8 + pkg/meta/types/types.go | 281 +- pkg/meta/version/version_test.go | 21 +- pkg/storage/storage.go | 6 +- pkg/test/common/utils.go | 32 + pkg/test/deprecated/deprecated.go | 2 +- pkg/test/image-utils/images.go | 48 +- pkg/test/image-utils/images_test.go | 10 - pkg/test/image-utils/multiarch.go | 44 +- pkg/test/image-utils/upload.go | 12 +- pkg/test/image-utils/utils.go | 18 - pkg/test/mocks/repo_db_mock.go | 555 ++-- pkg/test/oci-utils/repo.go | 162 +- test/blackbox/cloud_only.bats | 4 +- 82 files changed, 10455 insertions(+), 11676 deletions(-) create mode 100644 pkg/meta/convert/convert.go create mode 100644 pkg/meta/convert/convert_proto.go create mode 100644 pkg/meta/proto/gen/config.pb.go create mode 100644 pkg/meta/proto/gen/descriptor.pb.go create mode 100644 pkg/meta/proto/gen/index.pb.go create mode 100644 pkg/meta/proto/gen/manifest.pb.go create mode 100644 pkg/meta/proto/gen/meta.pb.go create mode 100644 pkg/meta/proto/gen/versioned.pb.go create mode 100644 pkg/meta/proto/meta/meta.proto create mode 100644 pkg/meta/proto/oci/config.proto create mode 100644 pkg/meta/proto/oci/descriptor.proto create mode 100644 pkg/meta/proto/oci/index.proto create mode 100644 pkg/meta/proto/oci/manifest.proto create mode 100644 pkg/meta/proto/oci/timestamp.proto create mode 100644 pkg/meta/proto/oci/versioned.proto diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index 6e278a56..10bf0d45 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -38,6 +38,7 @@ jobs: # Optional: if set to true then the action don't cache or restore ~/go/pkg. # skip-pkg-cache: true + # skip-pkg-cache: false # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. # skip-build-cache: true diff --git a/.gitignore b/.gitignore index 6724673e..4e0ce816 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ pkg/extensions/build/ hack/ .stacker/ oci/ +!pkg/meta/proto/oci roots/ bin/ bazel-* diff --git a/Makefile b/Makefile index 598e6897..87295743 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,22 @@ TESTDATA := $(TOP_LEVEL)/test/data OS ?= $(shell go env GOOS) ARCH ?= $(shell go env GOARCH) +PROTOC := $(TOOLSDIR)/bin/protoc +PROTOC_VERSION := 24.4 +GO_PROTOC_VERSION := 1.31.0 +HOST_OS := $(shell go env GOOS) +HOST_ARCH := $(shell go env GOARCH) +ifeq ($(HOST_OS),linux) + PROTOC_OS := linux +else ifeq ($(HOST_OS),darwin) + PROTOC_OS := osx +endif +ifeq ($(HOST_ARCH),amd64) + PROTOC_ARCH := x86_64 +else ifeq ($(HOST_ARCH),arm64) + PROTOC_ARCH := aarch_64 +endif + BENCH_OUTPUT ?= stdout ALL_EXTENSIONS = debug,imagetrust,lint,metrics,mgmt,profile,scrub,search,sync,ui,userprefs EXTENSIONS ?= sync,search,scrub,metrics,lint,ui,mgmt,profile,userprefs,imagetrust @@ -97,6 +113,51 @@ build-metadata: $(if $(findstring ui,$(BUILD_LABELS)), ui) echo "\n Files: \n" go list -tags $(BUILD_TAGS) -f '{{ join .GoFiles "\n" }}' ./... | sort -u +.PHONY: gen-protobuf +gen-protobuf: check-not-freebds $(PROTOC) + $(PROTOC) --experimental_allow_proto3_optional \ + --proto_path=$(TOP_LEVEL)/pkg/meta/proto \ + --go_out=$(TOP_LEVEL)/pkg/meta/proto \ + --go_opt='Moci/oci.proto=./gen' \ + --go_opt='Mmeta/meta.proto=./gen' \ + --go_opt='Moci/config.proto=./gen' \ + --go_opt='Moci/manifest.proto=./gen' \ + --go_opt='Moci/index.proto=./gen' \ + --go_opt='Moci/descriptor.proto=./gen' \ + --go_opt='Moci/versioned.proto=./gen' \ + $(TOP_LEVEL)/pkg/meta/proto/meta/meta.proto + $(PROTOC) --experimental_allow_proto3_optional \ + --proto_path=$(TOP_LEVEL)/pkg/meta/proto \ + --go_out=$(TOP_LEVEL)/pkg/meta/proto \ + --go_opt='Moci/versioned.proto=./gen' \ + $(TOP_LEVEL)/pkg/meta/proto/oci/versioned.proto + $(PROTOC) --experimental_allow_proto3_optional \ + --proto_path=$(TOP_LEVEL)/pkg/meta/proto \ + --go_out=$(TOP_LEVEL)/pkg/meta/proto \ + --go_opt='Moci/descriptor.proto=./gen' \ + $(TOP_LEVEL)/pkg/meta/proto/oci/descriptor.proto + $(PROTOC) --experimental_allow_proto3_optional \ + --proto_path=$(TOP_LEVEL)/pkg/meta/proto \ + --go_out=$(TOP_LEVEL)/pkg/meta/proto \ + --go_opt='Moci/descriptor.proto=./gen' \ + --go_opt='Moci/versioned.proto=./gen' \ + --go_opt='Moci/index.proto=./gen' \ + $(TOP_LEVEL)/pkg/meta/proto/oci/index.proto + $(PROTOC) --experimental_allow_proto3_optional \ + --proto_path=$(TOP_LEVEL)/pkg/meta/proto \ + --go_out=$(TOP_LEVEL)/pkg/meta/proto \ + --go_opt='Moci/oci.proto=./gen' \ + --go_opt='Moci/descriptor.proto=./gen' \ + --go_opt='Moci/config.proto=./gen' \ + $(TOP_LEVEL)/pkg/meta/proto/oci/config.proto + $(PROTOC) --experimental_allow_proto3_optional \ + --proto_path=$(TOP_LEVEL)/pkg/meta/proto \ + --go_out=$(TOP_LEVEL)/pkg/meta/proto \ + --go_opt='Moci/versioned.proto=./gen' \ + --go_opt='Moci/descriptor.proto=./gen' \ + --go_opt='Moci/manifest.proto=./gen' \ + $(TOP_LEVEL)/pkg/meta/proto/oci/manifest.proto + .PHONY: binary-minimal binary-minimal: EXTENSIONS= binary-minimal: modcheck build-metadata @@ -218,6 +279,13 @@ $(CRICTL): mv crictl $(TOOLSDIR)/bin/crictl chmod +x $(TOOLSDIR)/bin/crictl +$(PROTOC): + mkdir -p $(TOOLSDIR)/bin + curl -Lo protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VERSION)/protoc-$(PROTOC_VERSION)-$(PROTOC_OS)-$(PROTOC_ARCH).zip + unzip -o -d $(TOOLSDIR) protoc.zip bin/protoc + rm protoc.zip + chmod +x $(PROTOC) + go install google.golang.org/protobuf/cmd/protoc-gen-go@v$(GO_PROTOC_VERSION) $(ACTION_VALIDATOR): mkdir -p $(TOOLSDIR)/bin @@ -515,6 +583,12 @@ ifneq ($(shell go env GOOS),linux) $(error makefile target can be run only on linux) endif +.PHONY: check-not-freebds +check-not-freebds: +ifneq ($(shell go env GOOS),freebsd) + $(error makefile target can't be run on freebsd) +endif + .PHONY: check-compatibility check-compatibility: ifeq ($(OS),freebsd) diff --git a/codecov.yml b/codecov.yml index 913bdf3a..2ca1cf4c 100644 --- a/codecov.yml +++ b/codecov.yml @@ -6,3 +6,4 @@ ignore: - "./pkg/test/mocks/*.go" - "./swagger/*.go" - "./pkg/test/test_http_server.go" + - "./pkg/meta/proto/gen/*.go" diff --git a/errors/errors.go b/errors/errors.go index ed2dcf42..15065303 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -107,8 +107,8 @@ var ( ErrBadLayerCount = errors.New("manifest: layers count doesn't correspond to config history") ErrManifestConflict = errors.New("manifest: multiple manifests found") ErrManifestMetaNotFound = errors.New("metadb: image metadata not found for given manifest reference") - ErrManifestDataNotFound = errors.New("metadb: image data not found for given manifest digest") - ErrIndexDataNotFount = errors.New("metadb: index data not found for given digest") + ErrImageMetaNotFound = errors.New("metadb: image meta not found") + ErrUnexpectedMediaType = errors.New("metadb: got unexpected media type") ErrRepoMetaNotFound = errors.New("metadb: repo metadata not found for given repo name") ErrTagMetaNotFound = errors.New("metadb: tag metadata not found for given repo and tag names") ErrTypeAssertionFailed = errors.New("storage: failed DatabaseDriver type assertion") @@ -163,4 +163,5 @@ var ( ErrInvalidOutputFormat = errors.New("cli: invalid output format") ErrFlagValueUnsupported = errors.New("supported values ") ErrUnknownSubcommand = errors.New("cli: unknown subcommand") + ErrMultipleReposSameName = errors.New("test: can't have multiple repos with the same name") ) diff --git a/examples/config-all-remote.json b/examples/config-all-remote.json index a6bca614..89b734a9 100644 --- a/examples/config-all-remote.json +++ b/examples/config-all-remote.json @@ -18,7 +18,8 @@ "region": "us-east-2", "cacheTablename": "ZotBlobTable", "repoMetaTablename": "ZotRepoMetadataTable", - "manifestDataTablename": "ZotManifestDataTable", + "imageMetaTablename": "ZotImageMetaTable", + "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable", "versionTablename": "ZotVersion" } }, diff --git a/examples/config-dynamodb.json b/examples/config-dynamodb.json index d154dc65..b269a047 100644 --- a/examples/config-dynamodb.json +++ b/examples/config-dynamodb.json @@ -19,7 +19,8 @@ "region": "us-east-2", "cacheTablename": "ZotBlobTable", "repoMetaTablename": "ZotRepoMetadataTable", - "manifestDataTablename": "ZotManifestDataTable", + "imageMetaTablename": "ZotImageMetaTable", + "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable", "userDataTablename": "ZotUserDataTable", "versionTablename": "ZotVersion" } diff --git a/go.mod b/go.mod index d1c497ed..1481432a 100644 --- a/go.mod +++ b/go.mod @@ -500,7 +500,7 @@ require ( google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect google.golang.org/grpc v1.58.2 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/protobuf v1.31.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index faeb9482..e3396a04 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -143,14 +143,15 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { endpoint := os.Getenv("DYNAMODBMOCK_ENDPOINT") conf.Storage.CacheDriver = map[string]interface{}{ - "name": "dynamodb", - "endpoint": endpoint, - "region": "us-east-2", - "cacheTablename": "BlobTable", - "repoMetaTablename": "RepoMetadataTable", - "manifestDataTablename": "ManifestDataTable", - "userDataTablename": "ZotUserDataTable", - "versionTablename": "Version", + "name": "dynamodb", + "endpoint": endpoint, + "region": "us-east-2", + "cacheTablename": "BlobTable", + "repoMetaTablename": "RepoMetadataTable", + "imageMetaTablename": "ZotImageMetaTable", + "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable", + "userDataTablename": "ZotUserDataTable", + "versionTablename": "Version", } driver := storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) @@ -159,27 +160,29 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { // negative test cases conf.Storage.CacheDriver = map[string]interface{}{ - "endpoint": endpoint, - "region": "us-east-2", - "cacheTablename": "BlobTable", - "repoMetaTablename": "RepoMetadataTable", - "manifestDataTablename": "ManifestDataTable", - "userDataTablename": "ZotUserDataTable", - "versionTablename": "Version", + "endpoint": endpoint, + "region": "us-east-2", + "cacheTablename": "BlobTable", + "repoMetaTablename": "RepoMetadataTable", + "imageMetaTablename": "ZotImageMetaTable", + "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable", + "userDataTablename": "ZotUserDataTable", + "versionTablename": "Version", } driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) So(driver, ShouldBeNil) conf.Storage.CacheDriver = map[string]interface{}{ - "name": "dummy", - "endpoint": endpoint, - "region": "us-east-2", - "cacheTablename": "BlobTable", - "repoMetaTablename": "RepoMetadataTable", - "manifestDataTablename": "ManifestDataTable", - "userDataTablename": "ZotUserDataTable", - "versionTablename": "Version", + "name": "dummy", + "endpoint": endpoint, + "region": "us-east-2", + "cacheTablename": "BlobTable", + "repoMetaTablename": "RepoMetadataTable", + "imageMetaTablename": "ZotImageMetaTable", + "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable", + "userDataTablename": "ZotUserDataTable", + "versionTablename": "Version", } driver = storage.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) @@ -205,43 +208,45 @@ func TestCreateMetaDBDriver(t *testing.T) { } conf.Storage.CacheDriver = map[string]interface{}{ - "name": "dummy", - "endpoint": "http://localhost:4566", - "region": "us-east-2", - "cachetablename": "BlobTable", - "repometatablename": "RepoMetadataTable", - "manifestdatatablename": "ManifestDataTable", - "userdatatablename": "UserDatatable", + "name": "dummy", + "endpoint": "http://localhost:4566", + "region": "us-east-2", + "cachetablename": "BlobTable", + "repometatablename": "RepoMetadataTable", + "imageMetaTablename": "ZotImageMetaTable", + "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable", + "userdatatablename": "UserDatatable", } testFunc := func() { _, _ = meta.New(conf.Storage.StorageConfig, log) } So(testFunc, ShouldPanic) conf.Storage.CacheDriver = map[string]interface{}{ - "name": "dummy", - "endpoint": "http://localhost:4566", - "region": "us-east-2", - "cachetablename": "", - "repometatablename": "RepoMetadataTable", - "manifestdatatablename": "ManifestDataTable", - "userDataTablename": "ZotUserDataTable", - "versiontablename": 1, + "name": "dummy", + "endpoint": "http://localhost:4566", + "region": "us-east-2", + "cachetablename": "", + "repometatablename": "RepoMetadataTable", + "imageMetaTablename": "ZotImageMetaTable", + "repoBlobsInfoTablename": "ZotRepoBlobsInfoTable", + "userDataTablename": "ZotUserDataTable", + "versiontablename": 1, } testFunc = func() { _, _ = meta.New(conf.Storage.StorageConfig, log) } So(testFunc, ShouldPanic) conf.Storage.CacheDriver = map[string]interface{}{ - "name": "dummy", - "endpoint": "http://localhost:4566", - "region": "us-east-2", - "cachetablename": "test", - "repometatablename": "RepoMetadataTable", - "manifestdatatablename": "ManifestDataTable", - "indexdatatablename": "IndexDataTable", - "userdatatablename": "ZotUserDataTable", - "apikeytablename": "APIKeyTable", - "versiontablename": "1", + "name": "dummy", + "endpoint": "http://localhost:4566", + "region": "us-east-2", + "cachetablename": "test", + "repometatablename": "RepoMetadataTable", + "imagemetatablename": "ZotImageMetaTable", + "repoblobsinfotablename": "ZotRepoBlobsInfoTable", + "userdatatablename": "ZotUserDataTable", + "apikeytablename": "APIKeyTable", + "versiontablename": "1", } testFunc = func() { _, _ = meta.New(conf.Storage.StorageConfig, log) } @@ -414,16 +419,16 @@ func TestObjectStorageController(t *testing.T) { conf.Storage.StorageDriver = storageDriverParams conf.Storage.CacheDriver = map[string]interface{}{ - "name": "dynamodb", - "endpoint": os.Getenv("DYNAMODBMOCK_ENDPOINT"), - "region": "us-east-2", - "cachetablename": "test", - "repometatablename": "RepoMetadataTable", - "manifestdatatablename": "ManifestDataTable", - "indexdatatablename": "IndexDataTable", - "userdatatablename": "ZotUserDataTable", - "apikeytablename": "APIKeyTable1", - "versiontablename": "Version", + "name": "dynamodb", + "endpoint": os.Getenv("DYNAMODBMOCK_ENDPOINT"), + "region": "us-east-2", + "cachetablename": "test", + "repometatablename": "RepoMetadataTable", + "imagemetatablename": "ZotImageMetaTable", + "repoblobsinfotablename": "ZotRepoBlobsInfoTable", + "userdatatablename": "ZotUserDataTable", + "apikeytablename": "APIKeyTable1", + "versiontablename": "Version", } mockOIDCServer, err := authutils.MockOIDCRun() @@ -7773,6 +7778,8 @@ func TestInjectTooManyOpenFiles(t *testing.T) { } func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { + ctx := context.Background() + Convey("Make controller", t, func() { Convey("Garbage collect signatures without subject and manifests without tags", func(c C) { repoName := "testrepo" //nolint:goconst @@ -7804,7 +7811,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { ctlr.Config.Storage.Dedupe = false cm := test.NewControllerManager(ctlr) - cm.StartServer() + cm.StartServer() //nolint: contextcheck cm.WaitServerToBeReady(port) defer cm.StopServer() @@ -7835,7 +7842,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { // generate a keypair os.Setenv("COSIGN_PASSWORD", "") - err = generate.GenerateKeyPairCmd(context.TODO(), "", "cosign", nil) + err = generate.GenerateKeyPairCmd(ctx, "", "cosign", nil) So(err, ShouldBeNil) image := fmt.Sprintf("localhost:%s/%s@%s", port, repoName, digest.String()) @@ -7864,7 +7871,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { So(err, ShouldBeNil) // sign the image - err = signature.SignWithNotation("good", image, tdir, true) + err = signature.SignWithNotation("good", image, tdir, true) //nolint: contextcheck So(err, ShouldBeNil) // get cosign signature manifest @@ -7894,7 +7901,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { So(err, ShouldBeNil) // make sure both signatures are stored in repodb - repoMeta, err := ctlr.MetaDB.GetRepoMeta(repoName) + repoMeta, err := ctlr.MetaDB.GetRepoMeta(ctx, repoName) So(err, ShouldBeNil) sigMeta := repoMeta.Signatures[digest.String()] @@ -7955,7 +7962,7 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { So(err, ShouldBeNil) // make sure repoDB reference was added - repoMeta, err := ctlr.MetaDB.GetRepoMeta(repoName) + repoMeta, err := ctlr.MetaDB.GetRepoMeta(ctx, repoName) So(err, ShouldBeNil) _, ok := repoMeta.Referrers[untaggedManifestDigest.String()] @@ -7984,8 +7991,8 @@ func TestGCSignaturesAndUntaggedManifestsWithMetaDB(t *testing.T) { err = gc.CleanRepo(repoName) So(err, ShouldBeNil) - // make sure both signatures are removed from repodb and repo reference for untagged is removed - repoMeta, err = ctlr.MetaDB.GetRepoMeta(repoName) + // make sure both signatures are removed from metaDB and repo reference for untagged is removed + repoMeta, err = ctlr.MetaDB.GetRepoMeta(ctx, repoName) So(err, ShouldBeNil) sigMeta := repoMeta.Signatures[digest.String()] diff --git a/pkg/api/routes.go b/pkg/api/routes.go index ec93ffdc..ef6516ec 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -516,7 +516,7 @@ func (rh *RouteHandler) GetManifest(response http.ResponseWriter, request *http. } if rh.c.MetaDB != nil { - err := meta.OnGetManifest(name, reference, content, rh.c.StoreController, rh.c.MetaDB, rh.c.Log) + err := meta.OnGetManifest(name, reference, mediaType, content, rh.c.StoreController, rh.c.MetaDB, rh.c.Log) if err != nil { response.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/cli/client/cve_cmd_test.go b/pkg/cli/client/cve_cmd_test.go index ebb0c817..6c46968d 100644 --- a/pkg/cli/client/cve_cmd_test.go +++ b/pkg/cli/client/cve_cmd_test.go @@ -6,7 +6,6 @@ package client_test import ( "bytes" "context" - "encoding/json" "errors" "fmt" "io" @@ -733,7 +732,7 @@ func getMockCveScanner(metaDB mTypes.MetaDB) cveinfo.Scanner { imageDir := repo inputTag := reference - repoMeta, err := metaDB.GetRepoMeta(imageDir) + repoMeta, err := metaDB.GetRepoMeta(context.Background(), imageDir) if err != nil { return false, err } @@ -756,19 +755,12 @@ func getMockCveScanner(metaDB mTypes.MetaDB) cveinfo.Scanner { return false, err } - manifestData, err := metaDB.GetManifestData(manifestDigest) + manifestData, err := metaDB.GetImageMeta(manifestDigest) if err != nil { return false, err } - var manifestContent ispec.Manifest - - err = json.Unmarshal(manifestData.ManifestBlob, &manifestContent) - if err != nil { - return false, zerr.ErrScanNotSupported - } - - for _, imageLayer := range manifestContent.Layers { + for _, imageLayer := range manifestData.Manifests[0].Manifest.Layers { switch imageLayer.MediaType { case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer): diff --git a/pkg/common/common.go b/pkg/common/common.go index 6081ab7d..e374e21f 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -29,6 +29,12 @@ const ( ArtifactTypeNotation = "application/vnd.cncf.notary.signature" ) +var cosignTagRule = regexp.MustCompile(`sha256\-.+\.sig`) + +func IsCosignTag(tag string) bool { + return cosignTagRule.MatchString(tag) +} + func Contains[T comparable](elems []T, v T) bool { for _, s := range elems { if v == s { diff --git a/pkg/extensions/extension_image_trust_test.go b/pkg/extensions/extension_image_trust_test.go index af3b24fc..a0dc6d30 100644 --- a/pkg/extensions/extension_image_trust_test.go +++ b/pkg/extensions/extension_image_trust_test.go @@ -131,23 +131,23 @@ func TestSignatureUploadAndVerificationAWS(t *testing.T) { cacheTablename := "BlobTable" + uuid.String() repoMetaTablename := "RepoMetadataTable" + uuid.String() - manifestDataTablename := "ManifestDataTable" + uuid.String() versionTablename := "Version" + uuid.String() - indexDataTablename := "IndexDataTable" + uuid.String() userDataTablename := "UserDataTable" + uuid.String() apiKeyTablename := "ApiKeyTable" + uuid.String() + imageMetaTablename := "imageMetaTable" + uuid.String() + repoBlobsInfoTablename := "repoBlobsInfoTable" + uuid.String() cacheDriverParams := map[string]interface{}{ - "name": "dynamoDB", - "endpoint": os.Getenv("DYNAMODBMOCK_ENDPOINT"), - "region": "us-east-2", - "cacheTablename": cacheTablename, - "repoMetaTablename": repoMetaTablename, - "manifestDataTablename": manifestDataTablename, - "indexDataTablename": indexDataTablename, - "userDataTablename": userDataTablename, - "apiKeyTablename": apiKeyTablename, - "versionTablename": versionTablename, + "name": "dynamoDB", + "endpoint": os.Getenv("DYNAMODBMOCK_ENDPOINT"), + "region": "us-east-2", + "cacheTablename": cacheTablename, + "repoMetaTablename": repoMetaTablename, + "imageMetaTablename": imageMetaTablename, + "repoBlobsInfoTablename": repoBlobsInfoTablename, + "userDataTablename": userDataTablename, + "apiKeyTablename": apiKeyTablename, + "versionTablename": versionTablename, } t.Logf("using dynamo driver options: %v", cacheDriverParams) diff --git a/pkg/extensions/imagetrust/image_trust.go b/pkg/extensions/imagetrust/image_trust.go index 273f2d90..3fd06fa1 100644 --- a/pkg/extensions/imagetrust/image_trust.go +++ b/pkg/extensions/imagetrust/image_trust.go @@ -5,7 +5,6 @@ package imagetrust import ( "context" - "encoding/json" "time" "github.com/aws/aws-sdk-go-v2/aws" @@ -158,18 +157,13 @@ func IsResourceExistsException(err error) bool { } func (imgTrustStore *ImageTrustStore) VerifySignature( - signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, manifestContent []byte, + signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageMeta mTypes.ImageMeta, repo string, ) (string, time.Time, bool, error) { - var manifest ispec.Manifest - if err := json.Unmarshal(manifestContent, &manifest); err != nil { - return "", time.Time{}, false, err - } - desc := ispec.Descriptor{ - MediaType: manifest.MediaType, - Digest: manifestDigest, - Size: int64(len(manifestContent)), + MediaType: imageMeta.MediaType, + Digest: imageMeta.Digest, + Size: imageMeta.Size, } if manifestDigest.String() == "" { @@ -190,7 +184,7 @@ func (imgTrustStore *ImageTrustStore) VerifySignature( func NewTaskGenerator(metaDB mTypes.MetaDB, log log.Logger) scheduler.TaskGenerator { return &sigValidityTaskGenerator{ - repos: []mTypes.RepoMetadata{}, + repos: []mTypes.RepoMeta{}, metaDB: metaDB, repoIndex: -1, log: log, @@ -198,7 +192,7 @@ func NewTaskGenerator(metaDB mTypes.MetaDB, log log.Logger) scheduler.TaskGenera } type sigValidityTaskGenerator struct { - repos []mTypes.RepoMetadata + repos []mTypes.RepoMeta metaDB mTypes.MetaDB repoIndex int done bool @@ -209,7 +203,7 @@ func (gen *sigValidityTaskGenerator) Next() (scheduler.Task, error) { if len(gen.repos) == 0 { ctx := context.Background() - repos, err := gen.metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { + repos, err := gen.metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMeta) bool { return true }) if err != nil { @@ -243,18 +237,18 @@ func (gen *sigValidityTaskGenerator) IsReady() bool { func (gen *sigValidityTaskGenerator) Reset() { gen.done = false gen.repoIndex = -1 - gen.repos = []mTypes.RepoMetadata{} + gen.repos = []mTypes.RepoMeta{} gen.log.Info().Msg("finished resetting task generator for updating signatures validity") } type validityTask struct { metaDB mTypes.MetaDB - repo mTypes.RepoMetadata + repo mTypes.RepoMeta log log.Logger } -func NewValidityTask(metaDB mTypes.MetaDB, repo mTypes.RepoMetadata, log log.Logger) *validityTask { +func NewValidityTask(metaDB mTypes.MetaDB, repo mTypes.RepoMeta, log log.Logger) *validityTask { return &validityTask{metaDB, repo, log} } diff --git a/pkg/extensions/imagetrust/image_trust_disabled.go b/pkg/extensions/imagetrust/image_trust_disabled.go index dda917a3..9a6cf24e 100644 --- a/pkg/extensions/imagetrust/image_trust_disabled.go +++ b/pkg/extensions/imagetrust/image_trust_disabled.go @@ -7,6 +7,8 @@ import ( "time" godigest "github.com/opencontainers/go-digest" + + mTypes "zotregistry.io/zot/pkg/meta/types" ) func NewLocalImageTrustStore(dir string) (*imageTrustDisabled, error) { @@ -20,7 +22,7 @@ func NewAWSImageTrustStore(region, endpoint string) (*imageTrustDisabled, error) type imageTrustDisabled struct{} func (imgTrustStore *imageTrustDisabled) VerifySignature( - signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, manifestContent []byte, + signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageMeta mTypes.ImageMeta, repo string, ) (string, time.Time, bool, error) { return "", time.Time{}, false, nil diff --git a/pkg/extensions/imagetrust/image_trust_disabled_test.go b/pkg/extensions/imagetrust/image_trust_disabled_test.go index 1140f7cc..c146df57 100644 --- a/pkg/extensions/imagetrust/image_trust_disabled_test.go +++ b/pkg/extensions/imagetrust/image_trust_disabled_test.go @@ -27,15 +27,13 @@ func TestImageTrust(t *testing.T) { repo := "repo" - image := CreateRandomImage() //nolint:staticcheck - manifestContent := image.ManifestDescriptor.Data - manifestDigest := image.ManifestDescriptor.Digest + image := CreateRandomImage() localImgTrustStore, err := imagetrust.NewLocalImageTrustStore(rootDir) So(err, ShouldBeNil) author, expTime, ok, err := localImgTrustStore.VerifySignature("cosign", - []byte(""), "", manifestDigest, manifestContent, repo, + []byte(""), "", image.Digest(), image.AsImageMeta(), repo, ) So(author, ShouldBeEmpty) So(expTime, ShouldBeZeroValue) @@ -54,7 +52,7 @@ func TestImageTrust(t *testing.T) { So(err, ShouldBeNil) author, expTime, ok, err = cloudImgTrustStore.VerifySignature("cosign", - []byte(""), "", manifestDigest, manifestContent, repo, + []byte(""), "", image.Digest(), image.AsImageMeta(), repo, ) So(author, ShouldBeEmpty) So(expTime, ShouldBeZeroValue) diff --git a/pkg/extensions/imagetrust/image_trust_test.go b/pkg/extensions/imagetrust/image_trust_test.go index 892b21e6..d436b055 100644 --- a/pkg/extensions/imagetrust/image_trust_test.go +++ b/pkg/extensions/imagetrust/image_trust_test.go @@ -150,31 +150,21 @@ func TestInitCosignAndNotationDirs(t *testing.T) { } func TestVerifySignatures(t *testing.T) { - Convey("wrong manifest content", t, func() { - manifestContent := []byte("wrong json") - - imgTrustStore := &imagetrust.ImageTrustStore{} - _, _, _, err := imgTrustStore.VerifySignature("", []byte(""), "", "", manifestContent, "repo") - So(err, ShouldNotBeNil) - }) - Convey("empty manifest digest", t, func() { image := CreateRandomImage() - manifestContent := image.ManifestDescriptor.Data imgTrustStore := &imagetrust.ImageTrustStore{} - _, _, _, err := imgTrustStore.VerifySignature("", []byte(""), "", "", manifestContent, "repo") + _, _, _, err := imgTrustStore.VerifySignature("", []byte(""), "", "", image.AsImageMeta(), "repo") So(err, ShouldNotBeNil) So(err, ShouldEqual, zerr.ErrBadManifestDigest) }) Convey("wrong signature type", t, func() { image := CreateRandomImage() - manifestContent := image.ManifestDescriptor.Data - manifestDigest := image.ManifestDescriptor.Digest imgTrustStore := &imagetrust.ImageTrustStore{} - _, _, _, err := imgTrustStore.VerifySignature("wrongType", []byte(""), "", manifestDigest, manifestContent, "repo") + _, _, _, err := imgTrustStore.VerifySignature("wrongType", []byte(""), "", image.Digest(), image.AsImageMeta(), + "repo") So(err, ShouldNotBeNil) So(err, ShouldEqual, zerr.ErrInvalidSignatureType) }) @@ -184,15 +174,13 @@ func TestVerifySignatures(t *testing.T) { tag := "test" //nolint:goconst image := CreateRandomImage() - manifestContent := image.ManifestDescriptor.Data - manifestDigest := image.ManifestDescriptor.Digest Convey("cosignDir is not set", func() { imgTrustStore := &imagetrust.ImageTrustStore{ CosignStorage: &imagetrust.PublicKeyLocalStorage{}, } - _, _, _, err := imgTrustStore.VerifySignature("cosign", []byte(""), "", manifestDigest, manifestContent, repo) + _, _, _, err := imgTrustStore.VerifySignature("cosign", []byte(""), "", image.Digest(), image.AsImageMeta(), repo) So(err, ShouldNotBeNil) So(err, ShouldEqual, zerr.ErrSignConfigDirNotSet) }) @@ -212,7 +200,7 @@ func TestVerifySignatures(t *testing.T) { CosignStorage: pubKeyStorage, } - _, _, _, err = imgTrustStore.VerifySignature("cosign", []byte(""), "", manifestDigest, manifestContent, repo) + _, _, _, err = imgTrustStore.VerifySignature("cosign", []byte(""), "", image.Digest(), image.AsImageMeta(), repo) So(err, ShouldNotBeNil) }) @@ -232,8 +220,8 @@ func TestVerifySignatures(t *testing.T) { CosignStorage: pubKeyStorage, } - _, _, isTrusted, err := imgTrustStore.VerifySignature("cosign", []byte(""), "", manifestDigest, - manifestContent, repo) + _, _, isTrusted, err := imgTrustStore.VerifySignature("cosign", []byte(""), "", image.Digest(), image.AsImageMeta(), + repo) So(err, ShouldBeNil) So(isTrusted, ShouldBeFalse) }) @@ -282,7 +270,7 @@ func TestVerifySignatures(t *testing.T) { AnnotationOptions: options.AnnotationOptions{Annotations: []string{fmt.Sprintf("tag=%s", tag)}}, Upload: true, }, - []string{fmt.Sprintf("localhost:%s/%s@%s", port, repo, manifestDigest.String())}) + []string{fmt.Sprintf("localhost:%s/%s@%s", port, repo, image.DigestStr())}) So(err, ShouldBeNil) err = os.Remove(path.Join(cosignDir, "cosign.key")) @@ -299,7 +287,7 @@ func TestVerifySignatures(t *testing.T) { var sigKey string for _, manifest := range index.Manifests { - if manifest.Digest != manifestDigest { + if manifest.Digest != image.Digest() { blobContent, err := ctlr.StoreController.DefaultStore.GetBlobContent(repo, manifest.Digest) So(err, ShouldBeNil) @@ -320,8 +308,8 @@ func TestVerifySignatures(t *testing.T) { } // signature is trusted - author, _, isTrusted, err := imgTrustStore.VerifySignature("cosign", rawSignature, sigKey, manifestDigest, - manifestContent, repo) + author, _, isTrusted, err := imgTrustStore.VerifySignature("cosign", rawSignature, sigKey, image.Digest(), + image.AsImageMeta(), repo) So(err, ShouldBeNil) So(isTrusted, ShouldBeTrue) So(author, ShouldNotBeEmpty) @@ -332,16 +320,14 @@ func TestVerifySignatures(t *testing.T) { repo := "repo" //nolint:goconst tag := "test" //nolint:goconst image := CreateRandomImage() - manifestContent := image.ManifestDescriptor.Data - manifestDigest := image.ManifestDescriptor.Digest Convey("notationDir is not set", func() { imgTrustStore := &imagetrust.ImageTrustStore{ NotationStorage: &imagetrust.CertificateLocalStorage{}, } - _, _, _, err := imgTrustStore.VerifySignature("notation", []byte("signature"), "", manifestDigest, - manifestContent, repo) + _, _, _, err := imgTrustStore.VerifySignature("notation", []byte("signature"), "", image.Digest(), + image.AsImageMeta(), repo) So(err, ShouldNotBeNil) So(err, ShouldEqual, zerr.ErrSignConfigDirNotSet) }) @@ -356,8 +342,8 @@ func TestVerifySignatures(t *testing.T) { NotationStorage: certStorage, } - _, _, isTrusted, err := imgTrustStore.VerifySignature("notation", []byte(""), "", manifestDigest, - manifestContent, repo) + _, _, isTrusted, err := imgTrustStore.VerifySignature("notation", []byte(""), "", image.Digest(), + image.AsImageMeta(), repo) So(err, ShouldNotBeNil) So(isTrusted, ShouldBeFalse) }) @@ -377,8 +363,8 @@ func TestVerifySignatures(t *testing.T) { NotationStorage: certStorage, } - _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", manifestDigest, - manifestContent, repo) + _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", image.Digest(), + image.AsImageMeta(), repo) So(err, ShouldNotBeNil) }) @@ -399,8 +385,8 @@ func TestVerifySignatures(t *testing.T) { NotationStorage: certStorage, } - _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", manifestDigest, manifestContent, - repo) + _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", image.Digest(), + image.AsImageMeta(), repo) So(err, ShouldNotBeNil) }) @@ -437,10 +423,10 @@ func TestVerifySignatures(t *testing.T) { err = signature.GenerateNotationCerts(notationDir, "notation-sign-test") So(err, ShouldBeNil) - // sign the image - image := fmt.Sprintf("localhost:%s/%s", port, fmt.Sprintf("%s:%s", repo, tag)) + // sign the imageURL + imageURL := fmt.Sprintf("localhost:%s/%s", port, fmt.Sprintf("%s:%s", repo, tag)) - err = signature.SignWithNotation("notation-sign-test", image, notationDir, true) + err = signature.SignWithNotation("notation-sign-test", imageURL, notationDir, true) So(err, ShouldBeNil) err = test.CopyFiles(path.Join(notationDir, "notation", "truststore"), path.Join(notationDir, "truststore")) @@ -481,7 +467,7 @@ func TestVerifySignatures(t *testing.T) { var sigKey string for _, manifest := range index.Manifests { - if manifest.Digest != manifestDigest { + if manifest.Digest != image.Digest() { blobContent, err := ctlr.StoreController.DefaultStore.GetBlobContent(repo, manifest.Digest) So(err, ShouldBeNil) @@ -502,8 +488,8 @@ func TestVerifySignatures(t *testing.T) { } // signature is trusted - author, _, isTrusted, err := imgTrustStore.VerifySignature("notation", rawSignature, sigKey, manifestDigest, - manifestContent, repo) + author, _, isTrusted, err := imgTrustStore.VerifySignature("notation", rawSignature, sigKey, image.Digest(), + image.AsImageMeta(), repo) So(err, ShouldBeNil) So(isTrusted, ShouldBeTrue) So(author, ShouldNotBeEmpty) @@ -512,8 +498,8 @@ func TestVerifySignatures(t *testing.T) { So(err, ShouldBeNil) // signature is not trusted - author, _, isTrusted, err = imgTrustStore.VerifySignature("notation", rawSignature, sigKey, manifestDigest, - manifestContent, repo) + author, _, isTrusted, err = imgTrustStore.VerifySignature("notation", rawSignature, sigKey, image.Digest(), + image.AsImageMeta(), repo) So(err, ShouldNotBeNil) So(isTrusted, ShouldBeFalse) So(author, ShouldNotBeEmpty) @@ -977,9 +963,6 @@ func TestAWSTrustStore(t *testing.T) { repo := "repo" image := CreateRandomImage() - manifestContent := image.ManifestDescriptor.Data - manifestDigest := image.ManifestDescriptor.Digest - secretsManagerMock := mocks.SecretsManagerMock{ CreateSecretFn: func(ctx context.Context, params *secretsmanager.CreateSecretInput, optFns ...func(*secretsmanager.Options), @@ -1001,8 +984,8 @@ func TestAWSTrustStore(t *testing.T) { NotationStorage: notationStorage, } - _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", manifestDigest, - manifestContent, repo) + _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", image.Digest(), + image.AsImageMeta(), repo) So(err, ShouldNotBeNil) }) @@ -1010,9 +993,6 @@ func TestAWSTrustStore(t *testing.T) { repo := "repo" image := CreateRandomImage() - manifestContent := image.ManifestDescriptor.Data - manifestDigest := image.ManifestDescriptor.Digest - secretsManagerMock := mocks.SecretsManagerMock{ CreateSecretFn: func(ctx context.Context, params *secretsmanager.CreateSecretInput, optFns ...func(*secretsmanager.Options), @@ -1034,8 +1014,8 @@ func TestAWSTrustStore(t *testing.T) { NotationStorage: notationStorage, } - _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", manifestDigest, - manifestContent, repo) + _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", image.Digest(), + image.AsImageMeta(), repo) So(err, ShouldNotBeNil) secretsManagerCacheMock = mocks.SecretsManagerCacheMock{ @@ -1051,8 +1031,8 @@ func TestAWSTrustStore(t *testing.T) { NotationStorage: notationStorage, } - _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", manifestDigest, - manifestContent, repo) + _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", image.Digest(), + image.AsImageMeta(), repo) So(err, ShouldNotBeNil) secretsManagerCacheMock = mocks.SecretsManagerCacheMock{ @@ -1068,8 +1048,8 @@ func TestAWSTrustStore(t *testing.T) { NotationStorage: notationStorage, } - _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", manifestDigest, - manifestContent, repo) + _, _, _, err = imgTrustStore.VerifySignature("notation", []byte("signature"), "", image.Digest(), + image.AsImageMeta(), repo) So(err, ShouldNotBeNil) }) @@ -1080,22 +1060,22 @@ func TestAWSTrustStore(t *testing.T) { } repoMetaTablename := "RepoMetadataTable" + uuid.String() - manifestDataTablename := "ManifestDataTable" + uuid.String() versionTablename := "Version" + uuid.String() - indexDataTablename := "IndexDataTable" + uuid.String() userDataTablename := "UserDataTable" + uuid.String() apiKeyTablename := "ApiKeyTable" + uuid.String() + imageMetaTablename := "imageMetaTable" + uuid.String() + repoBlobsInfoTablename := "repoBlobsInfoTable" + uuid.String() dynamoDBDriverParams := map[string]interface{}{ - "name": "dynamoDB", - "endpoint": os.Getenv("DYNAMODBMOCK_ENDPOINT"), - "region": "us-east-2", - "repometatablename": repoMetaTablename, - "manifestdatatablename": manifestDataTablename, - "indexdatatablename": indexDataTablename, - "userdatatablename": userDataTablename, - "apikeytablename": apiKeyTablename, - "versiontablename": versionTablename, + "name": "dynamoDB", + "endpoint": os.Getenv("DYNAMODBMOCK_ENDPOINT"), + "region": "us-east-2", + "repometatablename": repoMetaTablename, + "imagemetatablename": imageMetaTablename, + "repoblobsinfotablename": repoBlobsInfoTablename, + "userdatatablename": userDataTablename, + "apikeytablename": apiKeyTablename, + "versiontablename": versionTablename, } t.Logf("using dynamo driver options: %v", dynamoDBDriverParams) @@ -1237,8 +1217,6 @@ func RunVerificationTests(t *testing.T, dbDriverParams map[string]interface{}) { Convey("verify cosign signature is trusted", func() { image := CreateRandomImage() - manifestContent := image.ManifestDescriptor.Data - manifestDigest := image.ManifestDescriptor.Digest err = UploadImage(image, baseURL, repo, tag) So(err, ShouldBeNil) @@ -1264,7 +1242,7 @@ func RunVerificationTests(t *testing.T, dbDriverParams map[string]interface{}) { AnnotationOptions: options.AnnotationOptions{Annotations: []string{fmt.Sprintf("tag=%s", tag)}}, Upload: true, }, - []string{fmt.Sprintf("localhost:%s/%s@%s", port, repo, manifestDigest.String())}) + []string{fmt.Sprintf("localhost:%s/%s@%s", port, repo, image.DigestStr())}) So(err, ShouldBeNil) indexContent, err := ctlr.StoreController.DefaultStore.GetIndexContent(repo) @@ -1278,7 +1256,7 @@ func RunVerificationTests(t *testing.T, dbDriverParams map[string]interface{}) { var sigKey string for _, manifest := range index.Manifests { - if manifest.Digest != manifestDigest { + if manifest.Digest != image.Digest() { blobContent, err := ctlr.StoreController.DefaultStore.GetBlobContent(repo, manifest.Digest) So(err, ShouldBeNil) @@ -1308,8 +1286,8 @@ func RunVerificationTests(t *testing.T, dbDriverParams map[string]interface{}) { imageTrustStore := ctlr.MetaDB.ImageTrustStore() // signature is trusted - author, _, isTrusted, err := imageTrustStore.VerifySignature("cosign", rawSignature, sigKey, manifestDigest, - manifestContent, repo) + author, _, isTrusted, err := imageTrustStore.VerifySignature("cosign", rawSignature, sigKey, image.Digest(), + image.AsImageMeta(), repo) So(err, ShouldBeNil) So(isTrusted, ShouldBeTrue) So(author, ShouldNotBeEmpty) @@ -1317,8 +1295,6 @@ func RunVerificationTests(t *testing.T, dbDriverParams map[string]interface{}) { Convey("verify notation signature is trusted", func() { image := CreateRandomImage() - manifestContent := image.ManifestDescriptor.Data - manifestDigest := image.ManifestDescriptor.Digest err = UploadImage(image, baseURL, repo, tag) So(err, ShouldBeNil) @@ -1366,7 +1342,7 @@ func RunVerificationTests(t *testing.T, dbDriverParams map[string]interface{}) { t.Logf("Processing manifest %v", notationSig) if notationSig.Config.MediaType != notreg.ArtifactTypeNotation || - notationSig.Subject.Digest != manifestDigest { + notationSig.Subject.Digest != image.Digest() { continue } @@ -1400,8 +1376,8 @@ func RunVerificationTests(t *testing.T, dbDriverParams map[string]interface{}) { imageTrustStore := ctlr.MetaDB.ImageTrustStore() // signature is trusted - author, _, isTrusted, err := imageTrustStore.VerifySignature("notation", rawSignature, sigKey, manifestDigest, - manifestContent, repo) + author, _, isTrusted, err := imageTrustStore.VerifySignature("notation", rawSignature, sigKey, image.Digest(), + image.AsImageMeta(), repo) So(err, ShouldBeNil) So(isTrusted, ShouldBeTrue) So(author, ShouldEqual, "CN=cert,O=Notary,L=Seattle,ST=WA,C=US") diff --git a/pkg/extensions/search/convert/convert_internal_test.go b/pkg/extensions/search/convert/convert_internal_test.go index fe99ae68..17521db3 100644 --- a/pkg/extensions/search/convert/convert_internal_test.go +++ b/pkg/extensions/search/convert/convert_internal_test.go @@ -4,12 +4,10 @@ package convert import ( "context" - "encoding/json" "errors" "testing" "github.com/99designs/gqlgen/graphql" - godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" @@ -17,7 +15,7 @@ import ( "zotregistry.io/zot/pkg/extensions/search/gql_generated" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/boltdb" - mTypes "zotregistry.io/zot/pkg/meta/types" + . "zotregistry.io/zot/pkg/test/image-utils" "zotregistry.io/zot/pkg/test/mocks" ) @@ -34,33 +32,20 @@ func TestCVEConvert(t *testing.T) { metaDB, err := boltdb.New(boltDB, log.NewLogger("debug", "")) So(err, ShouldBeNil) - configBlob, err := json.Marshal(ispec.Image{}) + image := CreateImageWith(). + Layers([]Layer{{ + MediaType: ispec.MediaTypeImageLayerGzip, + Digest: ispec.MediaTypeEmptyJSON, + Blob: ispec.DescriptorEmptyJSON.Data, + }}).DefaultConfig().Build() + + err = metaDB.SetRepoReference("repo1", "0.1.0", image.AsImageMeta()) So(err, ShouldBeNil) - manifestBlob, err := json.Marshal(ispec.Manifest{ - Layers: []ispec.Descriptor{ - { - MediaType: ispec.MediaTypeImageLayerGzip, - Size: 0, - Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"), - }, - }, - }) + repoMetaList, err := metaDB.SearchRepos(context.Background(), "") So(err, ShouldBeNil) - repoMeta11 := mTypes.ManifestMetadata{ - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - } - - digest11 := godigest.FromString("abc1") - err = metaDB.SetManifestMeta("repo1", digest11, repoMeta11) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - reposMeta, manifestMetaMap, _, err := metaDB.SearchRepos(context.Background(), "") - So(err, ShouldBeNil) + imageMeta, err := metaDB.FilterImageMeta(context.Background(), []string{image.DigestStr()}) ctx := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) @@ -86,8 +71,8 @@ func TestCVEConvert(t *testing.T) { So(imageSummary, ShouldBeNil) So(graphql.GetErrors(ctx), ShouldBeNil) - imageSummary, _, err = ImageManifest2ImageSummary(ctx, "repo1", "0.1.0", digest11, reposMeta[0], - manifestMetaMap[digest11.String()]) + imageSummary, _, err = ImageManifest2ImageSummary(ctx, GetFullImageMeta("0.1.0", repoMetaList[0], + imageMeta[image.DigestStr()])) So(err, ShouldBeNil) So(imageSummary, ShouldNotBeNil) @@ -177,8 +162,8 @@ func TestCVEConvert(t *testing.T) { So(repoSummary, ShouldBeNil) So(graphql.GetErrors(ctx), ShouldBeNil) - imageSummary, _, err := ImageManifest2ImageSummary(ctx, "repo1", "0.1.0", digest11, reposMeta[0], - manifestMetaMap[digest11.String()]) + imageSummary, _, err := ImageManifest2ImageSummary(ctx, GetFullImageMeta("0.1.0", repoMetaList[0], + imageMeta[image.DigestStr()])) So(err, ShouldBeNil) So(imageSummary, ShouldNotBeNil) @@ -235,8 +220,8 @@ func TestCVEConvert(t *testing.T) { So(manifestSummary, ShouldBeNil) So(graphql.GetErrors(ctx), ShouldBeNil) - imageSummary, _, err := ImageManifest2ImageSummary(ctx, "repo1", "0.1.0", digest11, reposMeta[0], - manifestMetaMap[digest11.String()]) + imageSummary, _, err := ImageManifest2ImageSummary(ctx, GetFullImageMeta("0.1.0", repoMetaList[0], + imageMeta[image.DigestStr()])) So(err, ShouldBeNil) manifestSummary = imageSummary.Manifests[0] diff --git a/pkg/extensions/search/convert/convert_test.go b/pkg/extensions/search/convert/convert_test.go index cc52fcfc..befc74a8 100644 --- a/pkg/extensions/search/convert/convert_test.go +++ b/pkg/extensions/search/convert/convert_test.go @@ -4,21 +4,19 @@ package convert_test import ( "context" - "encoding/json" "errors" "testing" "time" - "github.com/99designs/gqlgen/graphql" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" "zotregistry.io/zot/pkg/extensions/search/convert" - cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/extensions/search/gql_generated" "zotregistry.io/zot/pkg/extensions/search/pagination" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/boltdb" mTypes "zotregistry.io/zot/pkg/meta/types" . "zotregistry.io/zot/pkg/test/image-utils" "zotregistry.io/zot/pkg/test/mocks" @@ -27,187 +25,6 @@ import ( var ErrTestError = errors.New("TestError") -func TestConvertErrors(t *testing.T) { - Convey("ImageIndex2ImageSummary errors", t, func() { - ctx := graphql.WithResponseContext(context.Background(), - graphql.DefaultErrorPresenter, graphql.DefaultRecover) - - _, _, err := convert.ImageIndex2ImageSummary( - ctx, - "repo", - "tag", - godigest.FromString("indexDigest"), - mTypes.RepoMetadata{}, - mTypes.IndexData{ - IndexBlob: []byte("bad json"), - }, - map[string]mTypes.ManifestMetadata{}, - ) - So(err, ShouldNotBeNil) - }) - - Convey("ImageIndex2ImageSummary cve scanning", t, func() { - ctx := graphql.WithResponseContext(context.Background(), - graphql.DefaultErrorPresenter, graphql.DefaultRecover) - - _, _, err := convert.ImageIndex2ImageSummary( - ctx, - "repo", - "tag", - godigest.FromString("indexDigest"), - mTypes.RepoMetadata{}, - mTypes.IndexData{ - IndexBlob: []byte("{}"), - }, - map[string]mTypes.ManifestMetadata{}, - ) - So(err, ShouldBeNil) - }) - - Convey("ImageManifest2ImageSummary", t, func() { - ctx := graphql.WithResponseContext(context.Background(), - graphql.DefaultErrorPresenter, graphql.DefaultRecover) - configBlob, err := json.Marshal(ispec.Image{ - Platform: ispec.Platform{ - OS: "os", - Architecture: "arch", - Variant: "var", - }, - }) - So(err, ShouldBeNil) - - _, _, err = convert.ImageManifest2ImageSummary( - ctx, - "repo", - "tag", - godigest.FromString("manifestDigest"), - mTypes.RepoMetadata{}, - mTypes.ManifestMetadata{ - ManifestBlob: []byte("{}"), - ConfigBlob: configBlob, - }, - ) - So(err, ShouldBeNil) - }) - - Convey("ImageManifest2ManifestSummary", t, func() { - ctx := graphql.WithResponseContext(context.Background(), - graphql.DefaultErrorPresenter, graphql.DefaultRecover) - - // with bad config json, shouldn't error when unmarshaling - _, _, _, err := convert.ImageManifest2ManifestSummary( - ctx, - "repo", - "tag", - ispec.Descriptor{ - Digest: "dig", - MediaType: ispec.MediaTypeImageManifest, - }, - mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{}, - Statistics: map[string]mTypes.DescriptorStatistics{}, - Signatures: map[string]mTypes.ManifestSignatures{}, - Referrers: map[string][]mTypes.ReferrerInfo{}, - }, - mTypes.ManifestMetadata{ - ManifestBlob: []byte(`{}`), - ConfigBlob: []byte("bad json"), - }, - nil, - ) - So(err, ShouldBeNil) - - // CVE scan using platform - configBlob, err := json.Marshal(ispec.Image{ - Platform: ispec.Platform{ - OS: "os", - Architecture: "arch", - Variant: "var", - }, - }) - So(err, ShouldBeNil) - - _, _, _, err = convert.ImageManifest2ManifestSummary( - ctx, - "repo", - "tag", - ispec.Descriptor{ - Digest: "dig", - MediaType: ispec.MediaTypeImageManifest, - }, - mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{}, - Statistics: map[string]mTypes.DescriptorStatistics{}, - Signatures: map[string]mTypes.ManifestSignatures{"dig": {"cosine": []mTypes.SignatureInfo{{}}}}, - Referrers: map[string][]mTypes.ReferrerInfo{}, - }, - mTypes.ManifestMetadata{ - ManifestBlob: []byte("{}"), - ConfigBlob: configBlob, - }, - nil, - ) - So(err, ShouldBeNil) - }) - - Convey("RepoMeta2ExpandedRepoInfo", t, func() { - ctx := graphql.WithResponseContext(context.Background(), - graphql.DefaultErrorPresenter, graphql.DefaultRecover) - - // with bad config json, error while unmarshaling - _, imageSummaries := convert.RepoMeta2ExpandedRepoInfo( - ctx, - mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tag1": {Digest: "dig", MediaType: ispec.MediaTypeImageManifest}, - }, - }, - map[string]mTypes.ManifestMetadata{ - "dig": { - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("bad json"), - }, - }, - map[string]mTypes.IndexData{}, - convert.SkipQGLField{ - Vulnerabilities: false, - }, - mocks.CveInfoMock{ - GetCVESummaryForImageMediaFn: func(repo, digest, mediaType string) (cvemodel.ImageCVESummary, error) { - return cvemodel.ImageCVESummary{}, ErrTestError - }, - }, log.NewLogger("debug", ""), - ) - So(len(imageSummaries), ShouldEqual, 1) - - // cveInfo present no error - _, imageSummaries = convert.RepoMeta2ExpandedRepoInfo( - ctx, - mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tag1": {Digest: "dig", MediaType: ispec.MediaTypeImageManifest}, - }, - }, - map[string]mTypes.ManifestMetadata{ - "dig": { - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("{}"), - }, - }, - map[string]mTypes.IndexData{}, - convert.SkipQGLField{ - Vulnerabilities: false, - }, - mocks.CveInfoMock{ - GetCVESummaryForImageMediaFn: func(repo, digest, mediaType string) (cvemodel.ImageCVESummary, error) { - return cvemodel.ImageCVESummary{}, ErrTestError - }, - }, log.NewLogger("debug", ""), - ) - So(len(imageSummaries), ShouldEqual, 1) - }) -} - func TestUpdateLastUpdatedTimestamp(t *testing.T) { Convey("Image summary is the first image checked for the repo", t, func() { before := time.Time{} @@ -300,14 +117,25 @@ func TestLabels(t *testing.T) { func TestGetSignaturesInfo(t *testing.T) { Convey("Test get signatures info - cosign", t, func() { - indexDigest := godigest.FromString("123") - repoMeta := mTypes.RepoMetadata{ - Signatures: map[string]mTypes.ManifestSignatures{string(indexDigest): {"cosign": []mTypes.SignatureInfo{{ - LayersInfo: []mTypes.LayerInfo{{LayerContent: []byte{}, LayerDigest: "", SignatureKey: "", Signer: "author"}}, - }}}}, + digest := godigest.FromString("dig") + signatures := map[string]mTypes.ManifestSignatures{ + digest.String(): { + "cosign": []mTypes.SignatureInfo{ + { + LayersInfo: []mTypes.LayerInfo{ + { + LayerContent: []byte{}, + LayerDigest: "", + SignatureKey: "", + Signer: "author", + }, + }, + }, + }, + }, } - signaturesSummary := convert.GetSignaturesInfo(true, repoMeta, indexDigest) + signaturesSummary := convert.GetSignaturesInfo(true, signatures[digest.String()]) So(signaturesSummary, ShouldNotBeEmpty) So(*signaturesSummary[0].Author, ShouldEqual, "author") So(*signaturesSummary[0].IsTrusted, ShouldEqual, true) @@ -315,22 +143,26 @@ func TestGetSignaturesInfo(t *testing.T) { }) Convey("Test get signatures info - notation", t, func() { - indexDigest := godigest.FromString("123") - repoMeta := mTypes.RepoMetadata{ - Signatures: map[string]mTypes.ManifestSignatures{string(indexDigest): {"notation": []mTypes.SignatureInfo{{ - LayersInfo: []mTypes.LayerInfo{ + digest := godigest.FromString("dig") + signatures := map[string]mTypes.ManifestSignatures{ + digest.String(): { + "notation": []mTypes.SignatureInfo{ { - LayerContent: []byte{}, - LayerDigest: "", - SignatureKey: "", - Signer: "author", - Date: time.Now().AddDate(0, 0, -1), + LayersInfo: []mTypes.LayerInfo{ + { + LayerContent: []byte{}, + LayerDigest: "", + SignatureKey: "", + Signer: "author", + Date: time.Now().AddDate(0, 0, -1), + }, + }, }, }, - }}}}, + }, } - signaturesSummary := convert.GetSignaturesInfo(true, repoMeta, indexDigest) + signaturesSummary := convert.GetSignaturesInfo(true, signatures[digest.String()]) So(signaturesSummary, ShouldNotBeEmpty) So(*signaturesSummary[0].Author, ShouldEqual, "author") So(*signaturesSummary[0].IsTrusted, ShouldEqual, false) @@ -422,6 +254,18 @@ func ref[T any](val T) *T { func TestPaginatedConvert(t *testing.T) { ctx := context.Background() + tempDir := t.TempDir() + + driver, err := boltdb.GetBoltDriver(boltdb.DBParameters{RootDir: tempDir}) + if err != nil { + t.FailNow() + } + + metaDB, err := boltdb.New(driver, log.NewLogger("debug", "")) + if err != nil { + t.FailNow() + } + var ( badBothImage = CreateImageWith().DefaultLayers().ImageConfig( ispec.Image{Platform: ispec.Platform{OS: "bad-os", Architecture: "bad-arch"}}).Build() @@ -432,8 +276,9 @@ func TestPaginatedConvert(t *testing.T) { goodImage = CreateImageWith().DefaultLayers().ImageConfig( ispec.Image{Platform: ispec.Platform{OS: "good-os", Architecture: "good-arch"}}).Build() - randomImage1 = CreateRandomImage() - randomImage2 = CreateRandomImage() + randomImage1 = CreateRandomImage() + randomImage2 = CreateRandomImage() + signatureDigest = godigest.FromString("signature") badMultiArch = CreateMultiarchWith().Images( []Image{badBothImage, badOsImage, badArchImage, randomImage1}).Build() @@ -441,14 +286,14 @@ func TestPaginatedConvert(t *testing.T) { []Image{badOsImage, badArchImage, randomImage2, goodImage}).Build() ) - reposMeta, manifestMetaMap, indexDataMap := ociutils.GetMetadataForRepos( + ctx, err = ociutils.InitializeTestMetaDB(ctx, metaDB, ociutils.Repo{ Name: "repo1-only-images", Images: []ociutils.RepoImage{ - {Image: goodImage, Tag: "goodImage"}, - {Image: badOsImage, Tag: "badOsImage"}, - {Image: badArchImage, Tag: "badArchImage"}, - {Image: badBothImage, Tag: "badBothImage"}, + {Image: goodImage, Reference: "goodImage"}, + {Image: badOsImage, Reference: "badOsImage"}, + {Image: badArchImage, Reference: "badArchImage"}, + {Image: badBothImage, Reference: "badBothImage"}, }, IsBookmarked: true, IsStarred: true, @@ -456,9 +301,9 @@ func TestPaginatedConvert(t *testing.T) { ociutils.Repo{ Name: "repo2-only-bad-images", Images: []ociutils.RepoImage{ - {Image: randomImage1, Tag: "randomImage1"}, - {Image: randomImage2, Tag: "randomImage2"}, - {Image: badBothImage, Tag: "badBothImage"}, + {Image: randomImage1, Reference: "randomImage1"}, + {Image: randomImage2, Reference: "randomImage2"}, + {Image: badBothImage, Reference: "badBothImage"}, }, IsBookmarked: true, IsStarred: true, @@ -466,8 +311,8 @@ func TestPaginatedConvert(t *testing.T) { ociutils.Repo{ Name: "repo3-only-multiarch", MultiArchImages: []ociutils.RepoMultiArchImage{ - {MultiarchImage: badMultiArch, Tag: "badMultiArch"}, - {MultiarchImage: goodMultiArch, Tag: "goodMultiArch"}, + {MultiarchImage: badMultiArch, Reference: "badMultiArch"}, + {MultiarchImage: goodMultiArch, Reference: "goodMultiArch"}, }, IsBookmarked: true, IsStarred: true, @@ -475,44 +320,56 @@ func TestPaginatedConvert(t *testing.T) { ociutils.Repo{ Name: "repo4-not-bookmarked-or-starred", Images: []ociutils.RepoImage{ - {Image: goodImage, Tag: "goodImage"}, + {Image: goodImage, Reference: "goodImage"}, }, MultiArchImages: []ociutils.RepoMultiArchImage{ - {MultiarchImage: goodMultiArch, Tag: "goodMultiArch"}, + {MultiarchImage: goodMultiArch, Reference: "goodMultiArch"}, }, }, ociutils.Repo{ Name: "repo5-signed", Images: []ociutils.RepoImage{ - {Image: goodImage, Tag: "goodImage"}, // is fake signed by the image below - {Image: CreateFakeTestSignature(goodImage.DescriptorRef())}, + {Image: goodImage, Reference: "goodImage"}, // is fake signed by the image below + }, + Signatures: map[string]mTypes.ManifestSignatures{ + goodImage.DigestStr(): ociutils.GetFakeSignatureInfo(signatureDigest.String()), }, }, ) + if err != nil { + t.FailNow() + } skipCVE := convert.SkipQGLField{Vulnerabilities: true} Convey("PaginatedRepoMeta2RepoSummaries filtering and sorting", t, func() { // Test different combinations of the filter + repoMetaList, err := metaDB.FilterRepos(ctx, mTypes.AcceptAllRepoNames, mTypes.AcceptAllRepoMeta) + So(err, ShouldBeNil) + imageMeta, err := metaDB.FilterImageMeta(ctx, mTypes.GetLatestImageDigests(repoMetaList)) + So(err, ShouldBeNil) reposSum, pageInfo, err := convert.PaginatedRepoMeta2RepoSummaries( - ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + ctx, repoMetaList, imageMeta, mTypes.Filter{ Os: []*string{ref("good-os")}, Arch: []*string{ref("good-arch")}, IsBookmarked: ref(true), IsStarred: ref(true), }, - pagination.PageInput{SortBy: pagination.AlphabeticAsc}, + pagination.PageInput{SortBy: pagination.AlphabeticAsc}, mocks.CveInfoMock{}, skipCVE, ) So(err, ShouldBeNil) So(len(reposSum), ShouldEqual, 2) So(*reposSum[0].Name, ShouldResemble, "repo1-only-images") So(*reposSum[1].Name, ShouldResemble, "repo3-only-multiarch") So(pageInfo.ItemCount, ShouldEqual, 2) + So(pageInfo.ItemCount, ShouldEqual, 2) + So(pageInfo.ItemCount, ShouldEqual, 2) + So(pageInfo.ItemCount, ShouldEqual, 2) reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries( - ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + ctx, repoMetaList, imageMeta, mTypes.Filter{ Os: []*string{ref("good-os")}, Arch: []*string{ref("good-arch")}, @@ -520,18 +377,18 @@ func TestPaginatedConvert(t *testing.T) { IsStarred: ref(true), HasToBeSigned: ref(true), }, - pagination.PageInput{SortBy: pagination.AlphabeticAsc}, + pagination.PageInput{SortBy: pagination.AlphabeticAsc}, mocks.CveInfoMock{}, skipCVE, ) So(err, ShouldBeNil) So(len(reposSum), ShouldEqual, 0) So(pageInfo.ItemCount, ShouldEqual, 0) reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries( - ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + ctx, repoMetaList, imageMeta, mTypes.Filter{ HasToBeSigned: ref(true), }, - pagination.PageInput{SortBy: pagination.AlphabeticAsc}, + pagination.PageInput{SortBy: pagination.AlphabeticAsc}, mocks.CveInfoMock{}, skipCVE, ) So(err, ShouldBeNil) So(len(reposSum), ShouldEqual, 1) @@ -540,8 +397,8 @@ func TestPaginatedConvert(t *testing.T) { // no filter reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries( - ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, - mTypes.Filter{}, pagination.PageInput{SortBy: pagination.AlphabeticAsc}, + ctx, repoMetaList, imageMeta, + mTypes.Filter{}, pagination.PageInput{SortBy: pagination.AlphabeticAsc}, mocks.CveInfoMock{}, skipCVE, ) So(err, ShouldBeNil) So(len(reposSum), ShouldEqual, 5) @@ -554,8 +411,8 @@ func TestPaginatedConvert(t *testing.T) { // no filter opposite sorting reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries( - ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, - mTypes.Filter{}, pagination.PageInput{SortBy: pagination.AlphabeticDsc}, + ctx, repoMetaList, imageMeta, + mTypes.Filter{}, pagination.PageInput{SortBy: pagination.AlphabeticDsc}, mocks.CveInfoMock{}, skipCVE, ) So(err, ShouldBeNil) So(len(reposSum), ShouldEqual, 5) @@ -568,14 +425,14 @@ func TestPaginatedConvert(t *testing.T) { // add pagination reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries( - ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + ctx, repoMetaList, imageMeta, mTypes.Filter{ Os: []*string{ref("good-os")}, Arch: []*string{ref("good-arch")}, IsBookmarked: ref(true), IsStarred: ref(true), }, - pagination.PageInput{Limit: 1, Offset: 0, SortBy: pagination.AlphabeticAsc}, + pagination.PageInput{Limit: 1, Offset: 0, SortBy: pagination.AlphabeticAsc}, mocks.CveInfoMock{}, skipCVE, ) So(err, ShouldBeNil) So(len(reposSum), ShouldEqual, 1) @@ -584,14 +441,14 @@ func TestPaginatedConvert(t *testing.T) { So(pageInfo.TotalCount, ShouldEqual, 2) reposSum, pageInfo, err = convert.PaginatedRepoMeta2RepoSummaries( - ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + ctx, repoMetaList, imageMeta, mTypes.Filter{ Os: []*string{ref("good-os")}, Arch: []*string{ref("good-arch")}, IsBookmarked: ref(true), IsStarred: ref(true), }, - pagination.PageInput{Limit: 1, Offset: 1, SortBy: pagination.AlphabeticAsc}, + pagination.PageInput{Limit: 1, Offset: 1, SortBy: pagination.AlphabeticAsc}, mocks.CveInfoMock{}, skipCVE, ) So(err, ShouldBeNil) So(len(reposSum), ShouldEqual, 1) @@ -601,8 +458,11 @@ func TestPaginatedConvert(t *testing.T) { }) Convey("PaginatedRepoMeta2ImageSummaries filtering and sorting", t, func() { - imgSum, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries( - ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + fullImageMetaList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta) + So(err, ShouldBeNil) + + imgSum, pageInfo, err := convert.PaginatedFullImageMeta2ImageSummaries( + ctx, fullImageMetaList, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{ Os: []*string{ref("good-os")}, Arch: []*string{ref("good-arch")}, @@ -624,8 +484,8 @@ func TestPaginatedConvert(t *testing.T) { So(pageInfo.ItemCount, ShouldEqual, 5) // add page of size 2 - imgSum, pageInfo, err = convert.PaginatedRepoMeta2ImageSummaries( - ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + imgSum, pageInfo, err = convert.PaginatedFullImageMeta2ImageSummaries( + ctx, fullImageMetaList, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{ Os: []*string{ref("good-os")}, Arch: []*string{ref("good-arch")}, @@ -642,8 +502,8 @@ func TestPaginatedConvert(t *testing.T) { So(pageInfo.TotalCount, ShouldEqual, 5) // next page - imgSum, pageInfo, err = convert.PaginatedRepoMeta2ImageSummaries( - ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + imgSum, pageInfo, err = convert.PaginatedFullImageMeta2ImageSummaries( + ctx, fullImageMetaList, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{ Os: []*string{ref("good-os")}, Arch: []*string{ref("good-arch")}, @@ -660,8 +520,8 @@ func TestPaginatedConvert(t *testing.T) { So(pageInfo.TotalCount, ShouldEqual, 5) // last page - imgSum, pageInfo, err = convert.PaginatedRepoMeta2ImageSummaries( - ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + imgSum, pageInfo, err = convert.PaginatedFullImageMeta2ImageSummaries( + ctx, fullImageMetaList, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{ Os: []*string{ref("good-os")}, Arch: []*string{ref("good-arch")}, @@ -676,8 +536,8 @@ func TestPaginatedConvert(t *testing.T) { So(pageInfo.TotalCount, ShouldEqual, 5) // has to be signed - imgSum, pageInfo, err = convert.PaginatedRepoMeta2ImageSummaries( - ctx, reposMeta, manifestMetaMap, indexDataMap, skipCVE, mocks.CveInfoMock{}, + imgSum, pageInfo, err = convert.PaginatedFullImageMeta2ImageSummaries( + ctx, fullImageMetaList, skipCVE, mocks.CveInfoMock{}, mTypes.Filter{ Os: []*string{ref("good-os")}, Arch: []*string{ref("good-arch")}, @@ -697,6 +557,16 @@ func TestIndexAnnotations(t *testing.T) { Convey("Test ImageIndex2ImageSummary annotations logic", t, func() { ctx := context.Background() + tempDir := t.TempDir() + + driver, err := boltdb.GetBoltDriver(boltdb.DBParameters{RootDir: tempDir}) + if err != nil { + t.FailNow() + } + + metaDB, err := boltdb.New(driver, log.NewLogger("debug", "")) + So(err, ShouldBeNil) + configLabels := map[string]string{ ispec.AnnotationDescription: "ConfigDescription", ispec.AnnotationLicenses: "ConfigLicenses", @@ -746,17 +616,22 @@ func TestIndexAnnotations(t *testing.T) { []Image{imageWithManifestAndConfigAnnotations}, ).Annotations(indexAnnotations).Build() - repoMeta, manifestMetadata, indexData := ociutils.GetMetadataForRepos(ociutils.Repo{ - Name: "repo", - MultiArchImages: []ociutils.RepoMultiArchImage{ - {MultiarchImage: indexWithAnnotations, Tag: "tag"}, - }, - }) + ctx, err = ociutils.InitializeTestMetaDB(ctx, metaDB, + ociutils.Repo{ + Name: "repo", + MultiArchImages: []ociutils.RepoMultiArchImage{ + {MultiarchImage: indexWithAnnotations, Reference: "tag"}, + }, + }) + So(err, ShouldBeNil) - digest := indexWithAnnotations.Digest() + repoMeta, err := metaDB.GetRepoMeta(ctx, "repo") + So(err, ShouldBeNil) + imageMeta, err := metaDB.FilterImageMeta(ctx, []string{indexWithAnnotations.DigestStr()}) + So(err, ShouldBeNil) - imageSummary, _, err := convert.ImageIndex2ImageSummary(ctx, "repo", "tag", digest, repoMeta[0], - indexData[digest.String()], manifestMetadata) + imageSummary, _, err := convert.ImageIndex2ImageSummary(ctx, convert.GetFullImageMeta("tag", repoMeta, + imageMeta[indexWithAnnotations.DigestStr()])) So(err, ShouldBeNil) So(*imageSummary.Description, ShouldResemble, "IndexDescription") So(*imageSummary.Licenses, ShouldResemble, "IndexLicenses") @@ -766,19 +641,30 @@ func TestIndexAnnotations(t *testing.T) { So(*imageSummary.Vendor, ShouldResemble, "IndexVendor") So(*imageSummary.Authors, ShouldResemble, "IndexAuthors") + err = metaDB.ResetDB() + So(err, ShouldBeNil) // -------------------------------------------------------- indexWithManifestAndConfigAnnotations := CreateMultiarchWith().Images( []Image{imageWithManifestAndConfigAnnotations, CreateRandomImage(), CreateRandomImage()}, ).Build() - repoMeta, manifestMetadata, indexData = ociutils.GetMetadataForRepos(ociutils.Repo{ - Name: "repo", - MultiArchImages: []ociutils.RepoMultiArchImage{{MultiarchImage: indexWithManifestAndConfigAnnotations}}, + ctx, err = ociutils.InitializeTestMetaDB(ctx, metaDB, ociutils.Repo{ + Name: "repo", + MultiArchImages: []ociutils.RepoMultiArchImage{ + {MultiarchImage: indexWithManifestAndConfigAnnotations, Reference: "tag"}, + }, }) - digest = indexWithManifestAndConfigAnnotations.Digest() + So(err, ShouldBeNil) - imageSummary, _, err = convert.ImageIndex2ImageSummary(ctx, "repo", "tag", digest, - repoMeta[0], indexData[digest.String()], manifestMetadata) + digest := indexWithManifestAndConfigAnnotations.DigestStr() + + repoMeta, err = metaDB.GetRepoMeta(ctx, "repo") + So(err, ShouldBeNil) + imageMeta, err = metaDB.FilterImageMeta(ctx, []string{digest}) + So(err, ShouldBeNil) + + imageSummary, _, err = convert.ImageIndex2ImageSummary(ctx, convert.GetFullImageMeta("tag", repoMeta, + imageMeta[digest])) So(err, ShouldBeNil) So(*imageSummary.Description, ShouldResemble, "ManifestDescription") So(*imageSummary.Licenses, ShouldResemble, "ManifestLicenses") @@ -787,19 +673,31 @@ func TestIndexAnnotations(t *testing.T) { So(*imageSummary.Documentation, ShouldResemble, "ManifestDocumentation") So(*imageSummary.Vendor, ShouldResemble, "ManifestVendor") So(*imageSummary.Authors, ShouldResemble, "ManifestAuthors") + + err = metaDB.ResetDB() + So(err, ShouldBeNil) // -------------------------------------------------------- indexWithConfigAnnotations := CreateMultiarchWith().Images( []Image{imageWithConfigAnnotations, CreateRandomImage(), CreateRandomImage()}, ).Build() - repoMeta, manifestMetadata, indexData = ociutils.GetMetadataForRepos(ociutils.Repo{ - Name: "repo", - MultiArchImages: []ociutils.RepoMultiArchImage{{MultiarchImage: indexWithConfigAnnotations, Tag: "tag"}}, + ctx, err = ociutils.InitializeTestMetaDB(ctx, metaDB, ociutils.Repo{ + Name: "repo", + MultiArchImages: []ociutils.RepoMultiArchImage{ + {MultiarchImage: indexWithConfigAnnotations, Reference: "tag"}, + }, }) - digest = indexWithConfigAnnotations.Digest() + So(err, ShouldBeNil) - imageSummary, _, err = convert.ImageIndex2ImageSummary(ctx, "repo", "tag", digest, - repoMeta[0], indexData[digest.String()], manifestMetadata) + digest = indexWithConfigAnnotations.DigestStr() + + repoMeta, err = metaDB.GetRepoMeta(ctx, "repo") + So(err, ShouldBeNil) + imageMeta, err = metaDB.FilterImageMeta(ctx, []string{digest}) + So(err, ShouldBeNil) + + imageSummary, _, err = convert.ImageIndex2ImageSummary(ctx, convert.GetFullImageMeta("tag", repoMeta, + imageMeta[digest])) So(err, ShouldBeNil) So(*imageSummary.Description, ShouldResemble, "ConfigDescription") So(*imageSummary.Licenses, ShouldResemble, "ConfigLicenses") @@ -808,6 +706,9 @@ func TestIndexAnnotations(t *testing.T) { So(*imageSummary.Documentation, ShouldResemble, "ConfigDocumentation") So(*imageSummary.Vendor, ShouldResemble, "ConfigVendor") So(*imageSummary.Authors, ShouldResemble, "ConfigAuthors") + + err = metaDB.ResetDB() + So(err, ShouldBeNil) //-------------------------------------------------------- indexWithMixAnnotations := CreateMultiarchWith().Images( @@ -834,14 +735,23 @@ func TestIndexAnnotations(t *testing.T) { }, ).Build() - repoMeta, manifestMetadata, indexData = ociutils.GetMetadataForRepos(ociutils.Repo{ - Name: "repo", - MultiArchImages: []ociutils.RepoMultiArchImage{{MultiarchImage: indexWithMixAnnotations, Tag: "tag"}}, + ctx, err = ociutils.InitializeTestMetaDB(ctx, metaDB, ociutils.Repo{ + Name: "repo", + MultiArchImages: []ociutils.RepoMultiArchImage{ + {MultiarchImage: indexWithMixAnnotations, Reference: "tag"}, + }, }) - digest = indexWithMixAnnotations.Digest() + So(err, ShouldBeNil) - imageSummary, _, err = convert.ImageIndex2ImageSummary(ctx, "repo", "tag", digest, - repoMeta[0], indexData[digest.String()], manifestMetadata) + digest = indexWithMixAnnotations.DigestStr() + + repoMeta, err = metaDB.GetRepoMeta(ctx, "repo") + So(err, ShouldBeNil) + imageMeta, err = metaDB.FilterImageMeta(ctx, []string{digest}) + So(err, ShouldBeNil) + + imageSummary, _, err = convert.ImageIndex2ImageSummary(ctx, convert.GetFullImageMeta("tag", repoMeta, + imageMeta[digest])) So(err, ShouldBeNil) So(*imageSummary.Description, ShouldResemble, "ConfigDescription") So(*imageSummary.Licenses, ShouldResemble, "ConfigLicenses") @@ -851,17 +761,28 @@ func TestIndexAnnotations(t *testing.T) { So(*imageSummary.Documentation, ShouldResemble, "IndexDocumentation") So(*imageSummary.Source, ShouldResemble, "IndexSource") + err = metaDB.ResetDB() + So(err, ShouldBeNil) //-------------------------------------------------------- indexWithNoAnnotations := CreateRandomMultiarch() - repoMeta, manifestMetadata, indexData = ociutils.GetMetadataForRepos(ociutils.Repo{ - Name: "repo", - MultiArchImages: []ociutils.RepoMultiArchImage{{MultiarchImage: indexWithNoAnnotations, Tag: "tag"}}, + ctx, err = ociutils.InitializeTestMetaDB(ctx, metaDB, ociutils.Repo{ + Name: "repo", + MultiArchImages: []ociutils.RepoMultiArchImage{ + {MultiarchImage: indexWithNoAnnotations, Reference: "tag"}, + }, }) - digest = indexWithNoAnnotations.Digest() + So(err, ShouldBeNil) - imageSummary, _, err = convert.ImageIndex2ImageSummary(ctx, "repo", "tag", digest, - repoMeta[0], indexData[digest.String()], manifestMetadata) + digest = indexWithNoAnnotations.DigestStr() + + repoMeta, err = metaDB.GetRepoMeta(ctx, "repo") + So(err, ShouldBeNil) + imageMeta, err = metaDB.FilterImageMeta(ctx, []string{digest}) + So(err, ShouldBeNil) + + imageSummary, _, err = convert.ImageIndex2ImageSummary(ctx, convert.GetFullImageMeta("tag", repoMeta, + imageMeta[digest])) So(err, ShouldBeNil) So(*imageSummary.Description, ShouldBeBlank) So(*imageSummary.Licenses, ShouldBeBlank) @@ -870,94 +791,8 @@ func TestIndexAnnotations(t *testing.T) { So(*imageSummary.Title, ShouldBeBlank) So(*imageSummary.Documentation, ShouldBeBlank) So(*imageSummary.Source, ShouldBeBlank) - }) -} - -func TestDownloadCount(t *testing.T) { - Convey("manifest", t, func() { - repoMeta, manifestMetaMap, indexDataMap := ociutils.GetMetadataForRepos( - ociutils.Repo{ - Name: "repo", - Images: []ociutils.RepoImage{ - { - Image: CreateRandomImage(), - Tag: "10-downloads", - Statistics: mTypes.DescriptorStatistics{ - DownloadCount: 10, - }, - }, - }, - }, - ) - - repoSummary := convert.RepoMeta2RepoSummary(context.Background(), repoMeta[0], manifestMetaMap, indexDataMap) - So(*repoSummary.DownloadCount, ShouldEqual, 10) - So(*repoSummary.NewestImage.DownloadCount, ShouldEqual, 10) - }) - - Convey("index", t, func() { - img1, img2, img3 := CreateRandomImage(), CreateRandomImage(), CreateRandomImage() - multiArch := CreateMultiarchWith().Images([]Image{img1, img2, img3}).Build() - - repoMeta, manifestMetaMap, indexDataMap := ociutils.GetMetadataForRepos( - ociutils.Repo{ - Name: "repo", - MultiArchImages: []ociutils.RepoMultiArchImage{ - { - MultiarchImage: multiArch, - Tag: "160-multiarch", - ImageStatistics: map[string]mTypes.DescriptorStatistics{ - img1.DigestStr(): {DownloadCount: 10}, - img2.DigestStr(): {DownloadCount: 20}, - img3.DigestStr(): {DownloadCount: 30}, - multiArch.DigestStr(): {DownloadCount: 100}, - }, - }, - }, - }, - ) - - repoSummary := convert.RepoMeta2RepoSummary(context.Background(), repoMeta[0], manifestMetaMap, indexDataMap) - So(*repoSummary.DownloadCount, ShouldEqual, 100) - So(*repoSummary.NewestImage.DownloadCount, ShouldEqual, 100) - }) - - Convey("index + manifest mixed", t, func() { - img1 := CreateRandomImage() - img2 := CreateRandomImage() - img3 := CreateImageWith().DefaultLayers().ImageConfig( - ispec.Image{Created: DateRef(2020, 1, 1, 1, 1, 1, 0, time.UTC)}, - ).Build() - - multiArch := CreateMultiarchWith().Images([]Image{img1, img2, img3}).Build() - - repoMeta, manifestMetaMap, indexDataMap := ociutils.GetMetadataForRepos( - ociutils.Repo{ - Name: "repo", - Images: []ociutils.RepoImage{ - { - Image: CreateRandomImage(), - Tag: "5-downloads", - Statistics: mTypes.DescriptorStatistics{DownloadCount: 5}, - }, - }, - MultiArchImages: []ociutils.RepoMultiArchImage{ - { - MultiarchImage: multiArch, - Tag: "160-multiarch", - ImageStatistics: map[string]mTypes.DescriptorStatistics{ - img1.DigestStr(): {DownloadCount: 10}, - img2.DigestStr(): {DownloadCount: 20}, - img3.DigestStr(): {DownloadCount: 30}, - multiArch.DigestStr(): {DownloadCount: 100}, - }, - }, - }, - }, - ) - - repoSummary := convert.RepoMeta2RepoSummary(context.Background(), repoMeta[0], manifestMetaMap, indexDataMap) - So(*repoSummary.DownloadCount, ShouldEqual, 105) - So(*repoSummary.NewestImage.DownloadCount, ShouldEqual, 100) + + err = metaDB.ResetDB() + So(err, ShouldBeNil) }) } diff --git a/pkg/extensions/search/convert/metadb.go b/pkg/extensions/search/convert/metadb.go index 5ff8f032..b3e9e38e 100644 --- a/pkg/extensions/search/convert/metadb.go +++ b/pkg/extensions/search/convert/metadb.go @@ -2,15 +2,11 @@ package convert import ( "context" - "encoding/json" - "fmt" "sort" "strconv" - "strings" "time" "github.com/99designs/gqlgen/graphql" - godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/vektah/gqlparser/v2/gqlerror" @@ -20,7 +16,6 @@ import ( "zotregistry.io/zot/pkg/extensions/search/gql_generated" "zotregistry.io/zot/pkg/extensions/search/pagination" "zotregistry.io/zot/pkg/log" - mcommon "zotregistry.io/zot/pkg/meta/common" mTypes "zotregistry.io/zot/pkg/meta/types" ) @@ -28,120 +23,6 @@ type SkipQGLField struct { Vulnerabilities bool } -func RepoMeta2RepoSummary(ctx context.Context, repoMeta mTypes.RepoMetadata, - manifestMetaMap map[string]mTypes.ManifestMetadata, indexDataMap map[string]mTypes.IndexData, -) *gql_generated.RepoSummary { - var ( - repoName = repoMeta.Name - repoLastUpdatedTimestamp = time.Time{} - repoPlatformsSet = map[string]*gql_generated.Platform{} - repoVendorsSet = map[string]bool{} - lastUpdatedImageSummary *gql_generated.ImageSummary - repoDownloadCount = 0 - repoStarCount = repoMeta.Stars // total number of stars - repoIsUserStarred = repoMeta.IsStarred // value specific to the current user - repoIsUserBookMarked = repoMeta.IsBookmarked // value specific to the current user - - // map used to keep track of all blobs of a repo without duplicates as - // some images may have the same layers - repoBlob2Size = make(map[string]int64, 10) - - // made up of all manifests, configs and image layers - size = int64(0) - ) - - for tag, descriptor := range repoMeta.Tags { - imageSummary, imageBlobsMap, err := Descriptor2ImageSummary(ctx, descriptor, repoMeta.Name, - tag, repoMeta, manifestMetaMap, indexDataMap, - ) - if err != nil { - continue - } - - for blobDigest, blobSize := range imageBlobsMap { - repoBlob2Size[blobDigest] = blobSize - } - - for _, manifestSummary := range imageSummary.Manifests { - if *manifestSummary.Platform.Os != "" || *manifestSummary.Platform.Arch != "" { - opSys, arch := *manifestSummary.Platform.Os, *manifestSummary.Platform.Arch - - platformString := strings.TrimSpace(fmt.Sprintf("%s %s", opSys, arch)) - repoPlatformsSet[platformString] = &gql_generated.Platform{Os: &opSys, Arch: &arch} - } - } - - repoDownloadCount += *imageSummary.DownloadCount - - if *imageSummary.Vendor != "" { - repoVendorsSet[*imageSummary.Vendor] = true - } - - lastUpdatedImageSummary = UpdateLastUpdatedTimestamp(&repoLastUpdatedTimestamp, lastUpdatedImageSummary, imageSummary) - } - - // calculate repo size = sum all manifest, config and layer blobs sizes - for _, blobSize := range repoBlob2Size { - size += blobSize - } - - repoSize := strconv.FormatInt(size, 10) - - repoPlatforms := make([]*gql_generated.Platform, 0, len(repoPlatformsSet)) - - for _, platform := range repoPlatformsSet { - repoPlatforms = append(repoPlatforms, platform) - } - - repoVendors := make([]*string, 0, len(repoVendorsSet)) - - for vendor := range repoVendorsSet { - vendor := vendor - repoVendors = append(repoVendors, &vendor) - } - - return &gql_generated.RepoSummary{ - Name: &repoName, - LastUpdated: &repoLastUpdatedTimestamp, - Size: &repoSize, - Platforms: repoPlatforms, - Vendors: repoVendors, - NewestImage: lastUpdatedImageSummary, - DownloadCount: &repoDownloadCount, - StarCount: &repoStarCount, - IsBookmarked: &repoIsUserBookMarked, - IsStarred: &repoIsUserStarred, - Rank: &repoMeta.Rank, - } -} - -func PaginatedRepoMeta2RepoSummaries(ctx context.Context, reposMeta []mTypes.RepoMetadata, - manifestMetaMap map[string]mTypes.ManifestMetadata, indexDataMap map[string]mTypes.IndexData, - skip SkipQGLField, cveInfo cveinfo.CveInfo, filter mTypes.Filter, pageInput pagination.PageInput, -) ([]*gql_generated.RepoSummary, zcommon.PageInfo, error) { - reposPageFinder, err := pagination.NewRepoSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy) - if err != nil { - return []*gql_generated.RepoSummary{}, zcommon.PageInfo{}, err - } - - for _, repoMeta := range reposMeta { - repoSummary := RepoMeta2RepoSummary(ctx, repoMeta, manifestMetaMap, indexDataMap) - - if RepoSumAcceptedByFilter(repoSummary, filter) { - reposPageFinder.Add(repoSummary) - } - } - - page, pageInfo := reposPageFinder.Page() - - // CVE scanning is expensive, only scan for the current page - for _, repoSummary := range page { - updateRepoSummaryVulnerabilities(ctx, repoSummary, skip, cveInfo) - } - - return page, pageInfo, nil -} - func UpdateLastUpdatedTimestamp(repoLastUpdatedTimestamp *time.Time, lastUpdatedImageSummary *gql_generated.ImageSummary, imageSummary *gql_generated.ImageSummary, ) *gql_generated.ImageSummary { @@ -159,223 +40,6 @@ func UpdateLastUpdatedTimestamp(repoLastUpdatedTimestamp *time.Time, return newLastUpdatedImageSummary } -func Descriptor2ImageSummary(ctx context.Context, descriptor mTypes.Descriptor, repo, tag string, - repoMeta mTypes.RepoMetadata, manifestMetaMap map[string]mTypes.ManifestMetadata, - indexDataMap map[string]mTypes.IndexData, -) (*gql_generated.ImageSummary, map[string]int64, error) { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - return ImageManifest2ImageSummary(ctx, repo, tag, godigest.Digest(descriptor.Digest), - repoMeta, manifestMetaMap[descriptor.Digest]) - case ispec.MediaTypeImageIndex: - return ImageIndex2ImageSummary(ctx, repo, tag, godigest.Digest(descriptor.Digest), - repoMeta, indexDataMap[descriptor.Digest], manifestMetaMap) - default: - return &gql_generated.ImageSummary{}, map[string]int64{}, zerr.ErrMediaTypeNotSupported - } -} - -func ImageIndex2ImageSummary(ctx context.Context, repo, tag string, indexDigest godigest.Digest, - repoMeta mTypes.RepoMetadata, indexData mTypes.IndexData, manifestMetaMap map[string]mTypes.ManifestMetadata, -) (*gql_generated.ImageSummary, map[string]int64, error) { - var indexContent ispec.Index - - err := json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return &gql_generated.ImageSummary{}, map[string]int64{}, err - } - - var ( - indexLastUpdated time.Time - isSigned bool - totalIndexSize int64 - indexSize string - totalDownloadCount int - manifestAnnotations *ImageAnnotations - manifestSummaries = make([]*gql_generated.ManifestSummary, 0, len(indexContent.Manifests)) - indexBlobs = make(map[string]int64, 0) - - indexDigestStr = indexDigest.String() - indexMediaType = ispec.MediaTypeImageIndex - ) - - for _, descriptor := range indexContent.Manifests { - manifestSummary, manifestBlobs, annotations, err := ImageManifest2ManifestSummary(ctx, repo, tag, descriptor, - repoMeta, manifestMetaMap[descriptor.Digest.String()], repoMeta.Referrers[descriptor.Digest.String()]) - if err != nil { - return &gql_generated.ImageSummary{}, map[string]int64{}, err - } - - manifestSize := int64(0) - - for digest, size := range manifestBlobs { - indexBlobs[digest] = size - manifestSize += size - } - - if indexLastUpdated.Before(*manifestSummary.LastUpdated) { - indexLastUpdated = *manifestSummary.LastUpdated - } - - if manifestAnnotations == nil { - manifestAnnotations = annotations - } - - totalIndexSize += manifestSize - - manifestSummaries = append(manifestSummaries, manifestSummary) - } - - totalDownloadCount += repoMeta.Statistics[indexDigestStr].DownloadCount - - for _, signatures := range repoMeta.Signatures[indexDigest.String()] { - if len(signatures) > 0 { - isSigned = true - } - } - - indexSize = strconv.FormatInt(totalIndexSize, 10) - - signaturesInfo := GetSignaturesInfo(isSigned, repoMeta, indexDigest) - - if manifestAnnotations == nil { - // The index doesn't have manifests - manifestAnnotations = &ImageAnnotations{} - } - - annotations := GetIndexAnnotations(indexContent.Annotations, manifestAnnotations) - - indexSummary := gql_generated.ImageSummary{ - RepoName: &repo, - Tag: &tag, - Digest: &indexDigestStr, - MediaType: &indexMediaType, - Manifests: manifestSummaries, - LastUpdated: &indexLastUpdated, - IsSigned: &isSigned, - SignatureInfo: signaturesInfo, - Size: &indexSize, - DownloadCount: &totalDownloadCount, - Description: &annotations.Description, - Title: &annotations.Title, - Documentation: &annotations.Documentation, - Licenses: &annotations.Licenses, - Labels: &annotations.Labels, - Source: &annotations.Source, - Vendor: &annotations.Vendor, - Authors: &annotations.Authors, - Referrers: getReferrers(repoMeta.Referrers[indexDigest.String()]), - } - - return &indexSummary, indexBlobs, nil -} - -func ImageManifest2ImageSummary(ctx context.Context, repo, tag string, digest godigest.Digest, - repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata, -) (*gql_generated.ImageSummary, map[string]int64, error) { - var ( - manifestContent ispec.Manifest - manifestDigest = digest.String() - mediaType = ispec.MediaTypeImageManifest - ) - - err := json.Unmarshal(manifestMeta.ManifestBlob, &manifestContent) - if err != nil { - graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal manifest blob for image: %s:%s, manifest digest: %s, "+ - "error: %s", repo, tag, manifestDigest, err.Error())) - - return &gql_generated.ImageSummary{}, map[string]int64{}, err - } - - configContent := mcommon.InitializeImageConfig(manifestMeta.ConfigBlob) - - var ( - repoName = repo - configDigest = manifestContent.Config.Digest.String() - configSize = manifestContent.Config.Size - artifactType = zcommon.GetManifestArtifactType(manifestContent) - imageLastUpdated = zcommon.GetImageLastUpdated(configContent) - downloadCount = repoMeta.Statistics[digest.String()].DownloadCount - isSigned = false - ) - - opSys := configContent.OS - arch := configContent.Architecture - variant := configContent.Variant - - if variant != "" { - arch = arch + "/" + variant - } - - platform := gql_generated.Platform{Os: &opSys, Arch: &arch} - - for _, signatures := range repoMeta.Signatures[digest.String()] { - if len(signatures) > 0 { - isSigned = true - } - } - - size, imageBlobsMap := getImageBlobsInfo( - manifestDigest, int64(len(manifestMeta.ManifestBlob)), - configDigest, configSize, - manifestContent.Layers) - imageSize := strconv.FormatInt(size, 10) - - annotations := GetAnnotations(manifestContent.Annotations, configContent.Config.Labels) - - authors := annotations.Authors - if authors == "" { - authors = configContent.Author - } - - historyEntries, err := getAllHistory(manifestContent, configContent) - if err != nil { - graphql.AddError(ctx, gqlerror.Errorf("error generating history on tag %s in repo %s: "+ - "manifest digest: %s, error: %s", tag, repo, manifestDigest, err.Error())) - } - - signaturesInfo := GetSignaturesInfo(isSigned, repoMeta, digest) - - manifestSummary := gql_generated.ManifestSummary{ - Digest: &manifestDigest, - ConfigDigest: &configDigest, - LastUpdated: &imageLastUpdated, - Size: &imageSize, - IsSigned: &isSigned, - SignatureInfo: signaturesInfo, - Platform: &platform, - DownloadCount: &downloadCount, - Layers: getLayersSummaries(manifestContent), - History: historyEntries, - Referrers: getReferrers(repoMeta.Referrers[manifestDigest]), - ArtifactType: &artifactType, - } - - imageSummary := gql_generated.ImageSummary{ - RepoName: &repoName, - Tag: &tag, - Digest: &manifestDigest, - MediaType: &mediaType, - Manifests: []*gql_generated.ManifestSummary{&manifestSummary}, - LastUpdated: &imageLastUpdated, - IsSigned: &isSigned, - SignatureInfo: signaturesInfo, - Size: &imageSize, - DownloadCount: &downloadCount, - Description: &annotations.Description, - Title: &annotations.Title, - Documentation: &annotations.Documentation, - Licenses: &annotations.Licenses, - Labels: &annotations.Labels, - Source: &annotations.Source, - Vendor: &annotations.Vendor, - Authors: &authors, - Referrers: getReferrers(repoMeta.Referrers[manifestDigest]), - } - - return &imageSummary, imageBlobsMap, nil -} - func getReferrers(referrersInfo []mTypes.ReferrerInfo) []*gql_generated.Referrer { referrers := make([]*gql_generated.Referrer, 0, len(referrersInfo)) @@ -410,84 +74,6 @@ func getAnnotationsFromMap(annotationsMap map[string]string) []*gql_generated.An return annotations } -func ImageManifest2ManifestSummary(ctx context.Context, repo, tag string, descriptor ispec.Descriptor, - repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata, - referrersInfo []mTypes.ReferrerInfo, -) (*gql_generated.ManifestSummary, map[string]int64, *ImageAnnotations, error) { - var ( - manifestContent ispec.Manifest - digest = descriptor.Digest - ) - - err := json.Unmarshal(manifestMeta.ManifestBlob, &manifestContent) - if err != nil { - graphql.AddError(ctx, gqlerror.Errorf("can't unmarshal manifest blob for image: %s:%s, manifest digest: %s, "+ - "error: %s", repo, tag, digest, err.Error())) - - return &gql_generated.ManifestSummary{}, map[string]int64{}, &ImageAnnotations{}, err - } - - configContent := mcommon.InitializeImageConfig(manifestMeta.ConfigBlob) - annotations := GetAnnotations(manifestContent.Annotations, configContent.Config.Labels) - - var ( - manifestDigestStr = digest.String() - configDigest = manifestContent.Config.Digest.String() - configSize = manifestContent.Config.Size - artifactType = zcommon.GetManifestArtifactType(manifestContent) - imageLastUpdated = zcommon.GetImageLastUpdated(configContent) - downloadCount = repoMeta.Statistics[digest.String()].DownloadCount - isSigned = false - ) - - opSys := configContent.OS - arch := configContent.Architecture - variant := configContent.Variant - - if variant != "" { - arch = arch + "/" + variant - } - - platform := gql_generated.Platform{Os: &opSys, Arch: &arch} - - size, imageBlobsMap := getImageBlobsInfo( - manifestDigestStr, int64(len(manifestMeta.ManifestBlob)), - configDigest, configSize, - manifestContent.Layers) - imageSize := strconv.FormatInt(size, 10) - - historyEntries, err := getAllHistory(manifestContent, configContent) - if err != nil { - graphql.AddError(ctx, gqlerror.Errorf("error generating history on tag %s in repo %s: "+ - "manifest digest: %s, error: %s", tag, repo, manifestDigestStr, err.Error())) - } - - for _, signatures := range repoMeta.Signatures[manifestDigestStr] { - if len(signatures) > 0 { - isSigned = true - } - } - - signaturesInfo := GetSignaturesInfo(isSigned, repoMeta, digest) - - manifestSummary := gql_generated.ManifestSummary{ - Digest: &manifestDigestStr, - ConfigDigest: &configDigest, - LastUpdated: &imageLastUpdated, - Size: &imageSize, - Platform: &platform, - DownloadCount: &downloadCount, - Layers: getLayersSummaries(manifestContent), - History: historyEntries, - IsSigned: &isSigned, - SignatureInfo: signaturesInfo, - Referrers: getReferrers(referrersInfo), - ArtifactType: &artifactType, - } - - return &manifestSummary, imageBlobsMap, &annotations, nil -} - func getImageBlobsInfo(manifestDigest string, manifestSize int64, configDigest string, configSize int64, layers []ispec.Descriptor, ) (int64, map[string]int64) { @@ -511,9 +97,8 @@ func getImageBlobsInfo(manifestDigest string, manifestSize int64, configDigest s return imageSize, imageBlobsMap } -func RepoMeta2ImageSummaries(ctx context.Context, repoMeta mTypes.RepoMetadata, - manifestMetaMap map[string]mTypes.ManifestMetadata, indexDataMap map[string]mTypes.IndexData, - skip SkipQGLField, cveInfo cveinfo.CveInfo, +func RepoMeta2ImageSummaries(ctx context.Context, repoMeta mTypes.RepoMeta, + imageMeta map[string]mTypes.ImageMeta, skip SkipQGLField, cveInfo cveinfo.CveInfo, ) []*gql_generated.ImageSummary { imageSummaries := make([]*gql_generated.ImageSummary, 0, len(repoMeta.Tags)) @@ -531,8 +116,7 @@ func RepoMeta2ImageSummaries(ctx context.Context, repoMeta mTypes.RepoMetadata, for _, tag := range tags { descriptor := repoMeta.Tags[tag] - imageSummary, _, err := Descriptor2ImageSummary(ctx, descriptor, repoMeta.Name, tag, - repoMeta, manifestMetaMap, indexDataMap) + imageSummary, _, err := FullImageMeta2ImageSummary(ctx, GetFullImageMeta(tag, repoMeta, imageMeta[descriptor.Digest])) if err != nil { continue } @@ -546,69 +130,16 @@ func RepoMeta2ImageSummaries(ctx context.Context, repoMeta mTypes.RepoMetadata, return imageSummaries } -func PaginatedRepoMeta2ImageSummaries(ctx context.Context, reposMeta []mTypes.RepoMetadata, - manifestMetaMap map[string]mTypes.ManifestMetadata, indexDataMap map[string]mTypes.IndexData, - skip SkipQGLField, cveInfo cveinfo.CveInfo, filter mTypes.Filter, pageInput pagination.PageInput, -) ([]*gql_generated.ImageSummary, zcommon.PageInfo, error) { - imagePageFinder, err := pagination.NewImgSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy) - if err != nil { - return []*gql_generated.ImageSummary{}, zcommon.PageInfo{}, err - } - - for _, repoMeta := range reposMeta { - for tag := range repoMeta.Tags { - descriptor := repoMeta.Tags[tag] - - imageSummary, _, err := Descriptor2ImageSummary(ctx, descriptor, repoMeta.Name, tag, - repoMeta, manifestMetaMap, indexDataMap) - if err != nil { - continue - } - - if ImgSumAcceptedByFilter(imageSummary, filter) { - imagePageFinder.Add(imageSummary) - } - } - } - - page, pageInfo := imagePageFinder.Page() - - for _, imageSummary := range page { - // CVE scanning is expensive, only scan for this page - updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo) - } - - return page, pageInfo, nil -} - -func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta mTypes.RepoMetadata, - manifestMetaMap map[string]mTypes.ManifestMetadata, indexDataMap map[string]mTypes.IndexData, - skip SkipQGLField, cveInfo cveinfo.CveInfo, log log.Logger, +func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta mTypes.RepoMeta, + imageMetaMap map[string]mTypes.ImageMeta, skip SkipQGLField, cveInfo cveinfo.CveInfo, log log.Logger, ) (*gql_generated.RepoSummary, []*gql_generated.ImageSummary) { - var ( - repoName = repoMeta.Name - repoLastUpdatedTimestamp = time.Time{} - repoPlatformsSet = map[string]*gql_generated.Platform{} - repoVendorsSet = map[string]bool{} - lastUpdatedImageSummary *gql_generated.ImageSummary - repoDownloadCount = 0 - repoStarCount = repoMeta.Stars // total number of stars - isStarred = repoMeta.IsStarred // value specific to the current user - isBookmarked = repoMeta.IsBookmarked // value specific to the current user - - // map used to keep track of all blobs of a repo without duplicates as - // some images may have the same layers - repoBlob2Size = make(map[string]int64, 10) - - // made up of all manifests, configs and image layers - size = int64(0) - - imageSummaries = make([]*gql_generated.ImageSummary, 0, len(repoMeta.Tags)) - ) + repoName := repoMeta.Name + imageSummaries := make([]*gql_generated.ImageSummary, 0, len(repoMeta.Tags)) for tag, descriptor := range repoMeta.Tags { - imageSummary, imageBlobs, err := Descriptor2ImageSummary(ctx, descriptor, repoName, tag, - repoMeta, manifestMetaMap, indexDataMap) + imageMeta := imageMetaMap[descriptor.Digest] + + imageSummary, _, err := FullImageMeta2ImageSummary(ctx, GetFullImageMeta(tag, repoMeta, imageMeta)) if err != nil { log.Error().Str("repository", repoName).Str("reference", tag). Msg("metadb: error while converting descriptor for image") @@ -616,65 +147,47 @@ func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta mTypes.RepoMetadata continue } - for _, manifestSummary := range imageSummary.Manifests { - opSys, arch := *manifestSummary.Platform.Os, *manifestSummary.Platform.Arch - if opSys != "" || arch != "" { - platformString := strings.TrimSpace(fmt.Sprintf("%s %s", opSys, arch)) - repoPlatformsSet[platformString] = &gql_generated.Platform{Os: &opSys, Arch: &arch} - } - - updateRepoBlobsMap(imageBlobs, repoBlob2Size) - } - - if *imageSummary.Vendor != "" { - repoVendorsSet[*imageSummary.Vendor] = true - } - updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo) - lastUpdatedImageSummary = UpdateLastUpdatedTimestamp(&repoLastUpdatedTimestamp, lastUpdatedImageSummary, imageSummary) - - repoDownloadCount += *imageSummary.DownloadCount - imageSummaries = append(imageSummaries, imageSummary) } - // calculate repo size = sum all manifest, config and layer blobs sizes - for _, blobSize := range repoBlob2Size { - size += blobSize + repoSummary := RepoMeta2RepoSummary(ctx, repoMeta, imageMetaMap) + + updateRepoSummaryVulnerabilities(ctx, repoSummary, skip, cveInfo) + + return repoSummary, imageSummaries +} + +func GetFullImageMeta(tag string, repoMeta mTypes.RepoMeta, imageMeta mTypes.ImageMeta, +) mTypes.FullImageMeta { + return mTypes.FullImageMeta{ + Repo: repoMeta.Name, + Tag: tag, + MediaType: imageMeta.MediaType, + Digest: imageMeta.Digest, + Size: imageMeta.Size, + Index: imageMeta.Index, + Manifests: GetFullManifestMeta(repoMeta, imageMeta.Manifests), + Referrers: repoMeta.Referrers[imageMeta.Digest.String()], + Statistics: repoMeta.Statistics[imageMeta.Digest.String()], + Signatures: repoMeta.Signatures[imageMeta.Digest.String()], + } +} + +func GetFullManifestMeta(repoMeta mTypes.RepoMeta, manifests []mTypes.ManifestData) []mTypes.FullManifestMeta { + results := make([]mTypes.FullManifestMeta, 0, len(manifests)) + + for i := range manifests { + results = append(results, mTypes.FullManifestMeta{ + ManifestData: manifests[i], + Referrers: repoMeta.Referrers[manifests[i].Digest.String()], + Statistics: repoMeta.Statistics[manifests[i].Digest.String()], + Signatures: repoMeta.Signatures[manifests[i].Digest.String()], + }) } - repoSize := strconv.FormatInt(size, 10) - - repoPlatforms := make([]*gql_generated.Platform, 0, len(repoPlatformsSet)) - - for _, platform := range repoPlatformsSet { - repoPlatforms = append(repoPlatforms, platform) - } - - repoVendors := make([]*string, 0, len(repoVendorsSet)) - - for vendor := range repoVendorsSet { - vendor := vendor - repoVendors = append(repoVendors, &vendor) - } - - summary := &gql_generated.RepoSummary{ - Name: &repoName, - LastUpdated: &repoLastUpdatedTimestamp, - Size: &repoSize, - Platforms: repoPlatforms, - Vendors: repoVendors, - NewestImage: lastUpdatedImageSummary, - DownloadCount: &repoDownloadCount, - StarCount: &repoStarCount, - IsBookmarked: &isBookmarked, - IsStarred: &isStarred, - } - - updateRepoSummaryVulnerabilities(ctx, summary, skip, cveInfo) - - return summary, imageSummaries + return results } func StringMap2Annotations(strMap map[string]string) []*gql_generated.Annotation { @@ -736,15 +249,14 @@ func GetPreloadString(prefix, name string) string { return name } -func GetSignaturesInfo(isSigned bool, repoMeta mTypes.RepoMetadata, indexDigest godigest.Digest, -) []*gql_generated.SignatureSummary { +func GetSignaturesInfo(isSigned bool, signatures mTypes.ManifestSignatures) []*gql_generated.SignatureSummary { signaturesInfo := []*gql_generated.SignatureSummary{} if !isSigned { return signaturesInfo } - for sigType, signatures := range repoMeta.Signatures[indexDigest.String()] { + for sigType, signatures := range signatures { for _, sig := range signatures { for _, layer := range sig.LayersInfo { var ( @@ -776,3 +288,341 @@ func GetSignaturesInfo(isSigned bool, repoMeta mTypes.RepoMetadata, indexDigest return signaturesInfo } + +func PaginatedRepoMeta2RepoSummaries(ctx context.Context, repoMetaList []mTypes.RepoMeta, + imageMetaMap map[string]mTypes.ImageMeta, filter mTypes.Filter, pageInput pagination.PageInput, + cveInfo cveinfo.CveInfo, skip SkipQGLField, +) ([]*gql_generated.RepoSummary, zcommon.PageInfo, error) { + reposPageFinder, err := pagination.NewRepoSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy) + if err != nil { + return []*gql_generated.RepoSummary{}, zcommon.PageInfo{}, err + } + + for _, repoMeta := range repoMetaList { + repoSummary := RepoMeta2RepoSummary(ctx, repoMeta, imageMetaMap) + + if RepoSumAcceptedByFilter(repoSummary, filter) { + reposPageFinder.Add(repoSummary) + } + } + + page, pageInfo := reposPageFinder.Page() + + // CVE scanning is expensive, only scan for the current page + for _, repoSummary := range page { + updateRepoSummaryVulnerabilities(ctx, repoSummary, skip, cveInfo) + } + + return page, pageInfo, nil +} + +func RepoMeta2RepoSummary(ctx context.Context, repoMeta mTypes.RepoMeta, + imageMetaMap map[string]mTypes.ImageMeta, +) *gql_generated.RepoSummary { + var ( + repoName = repoMeta.Name + repoLastUpdatedTimestamp = deref(repoMeta.LastUpdatedImage, mTypes.LastUpdatedImage{}).LastUpdated + repoPlatforms = repoMeta.Platforms + repoVendors = repoMeta.Vendors + repoDownloadCount = repoMeta.DownloadCount + repoStarCount = repoMeta.StarCount + repoIsUserStarred = repoMeta.IsStarred // value specific to the current user + repoIsUserBookMarked = repoMeta.IsBookmarked // value specific to the current user + repoSize = repoMeta.Size + lastUpdatedImageMeta = imageMetaMap[repoMeta.LastUpdatedImage.Digest] + lastUpdatedTag = repoMeta.LastUpdatedImage.Tag + ) + + if repoLastUpdatedTimestamp == nil { + repoLastUpdatedTimestamp = &time.Time{} + } + + imageSummary, _, err := FullImageMeta2ImageSummary(ctx, GetFullImageMeta(lastUpdatedTag, repoMeta, + lastUpdatedImageMeta)) + _ = err + + return &gql_generated.RepoSummary{ + Name: &repoName, + LastUpdated: repoLastUpdatedTimestamp, + Size: ref(strconv.FormatInt(repoSize, 10)), + Platforms: getGqlPlatforms(repoPlatforms), + Vendors: getGqlVendors(repoVendors), + NewestImage: imageSummary, + DownloadCount: &repoDownloadCount, + StarCount: &repoStarCount, + IsBookmarked: &repoIsUserBookMarked, + IsStarred: &repoIsUserStarred, + Rank: ref(repoMeta.Rank), + } +} + +func getGqlVendors(repoVendors []string) []*string { + result := make([]*string, 0, len(repoVendors)) + + for i := range repoVendors { + result = append(result, &repoVendors[i]) + } + + return result +} + +func getGqlPlatforms(repoPlatforms []ispec.Platform) []*gql_generated.Platform { + result := make([]*gql_generated.Platform, 0, len(repoPlatforms)) + + for i := range repoPlatforms { + result = append(result, &gql_generated.Platform{ + Os: ref(repoPlatforms[i].OS), + Arch: ref(getArch(repoPlatforms[i].Architecture, repoPlatforms[i].Variant)), + }) + } + + return result +} + +type ( + ManifestDigest = string + BlobDigest = string +) + +func FullImageMeta2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullImageMeta, +) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) { + switch fullImageMeta.MediaType { + case ispec.MediaTypeImageManifest: + return ImageManifest2ImageSummary(ctx, fullImageMeta) + case ispec.MediaTypeImageIndex: + return ImageIndex2ImageSummary(ctx, fullImageMeta) + default: + return nil, nil, zerr.ErrMediaTypeNotSupported + } +} + +func ImageIndex2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullImageMeta, +) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) { + var ( + repo = fullImageMeta.Repo + tag = fullImageMeta.Tag + indexLastUpdated time.Time + isSigned = isImageSigned(fullImageMeta.Signatures) + indexSize = int64(0) + manifestAnnotations *ImageAnnotations + manifestSummaries = make([]*gql_generated.ManifestSummary, 0, len(fullImageMeta.Manifests)) + indexBlobs = map[string]int64{} + + indexDigestStr = fullImageMeta.Digest.String() + indexMediaType = ispec.MediaTypeImageIndex + ) + + for _, imageManifest := range fullImageMeta.Manifests { + imageManifestSummary, manifestBlobs, err := ImageManifest2ImageSummary(ctx, mTypes.FullImageMeta{ + Repo: fullImageMeta.Repo, + Tag: fullImageMeta.Tag, + MediaType: ispec.MediaTypeImageManifest, + Digest: imageManifest.Digest, + Size: imageManifest.Size, + Manifests: []mTypes.FullManifestMeta{imageManifest}, + Referrers: imageManifest.Referrers, + Statistics: imageManifest.Statistics, + Signatures: imageManifest.Signatures, + }) + if err != nil { + return &gql_generated.ImageSummary{}, map[string]int64{}, err + } + + manifestSize := int64(0) + + for digest, size := range manifestBlobs { + indexBlobs[digest] = size + manifestSize += size + } + + if indexLastUpdated.Before(*imageManifestSummary.LastUpdated) { + indexLastUpdated = *imageManifestSummary.LastUpdated + } + + annotations := GetAnnotations(imageManifest.Manifest.Annotations, imageManifest.Config.Config.Labels) + if manifestAnnotations == nil { + manifestAnnotations = &annotations + } + + indexSize += manifestSize + + manifestSummaries = append(manifestSummaries, imageManifestSummary.Manifests[0]) + } + + signaturesInfo := GetSignaturesInfo(isSigned, fullImageMeta.Signatures) + + if manifestAnnotations == nil { + manifestAnnotations = &ImageAnnotations{} + } + + annotations := GetIndexAnnotations(fullImageMeta.Index.Annotations, manifestAnnotations) + + indexSummary := gql_generated.ImageSummary{ + RepoName: &repo, + Tag: &tag, + Digest: &indexDigestStr, + MediaType: &indexMediaType, + Manifests: manifestSummaries, + LastUpdated: &indexLastUpdated, + IsSigned: &isSigned, + SignatureInfo: signaturesInfo, + Size: ref(strconv.FormatInt(indexSize, 10)), + DownloadCount: ref(fullImageMeta.Statistics.DownloadCount), + Description: &annotations.Description, + Title: &annotations.Title, + Documentation: &annotations.Documentation, + Licenses: &annotations.Licenses, + Labels: &annotations.Labels, + Source: &annotations.Source, + Vendor: &annotations.Vendor, + Authors: &annotations.Authors, + Referrers: getReferrers(fullImageMeta.Referrers), + } + + return &indexSummary, indexBlobs, nil +} + +func ImageManifest2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullImageMeta, +) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) { + manifest := fullImageMeta.Manifests[0] + + var ( + repoName = fullImageMeta.Repo + tag = fullImageMeta.Tag + configDigest = manifest.Manifest.Config.Digest.String() + configSize = manifest.Manifest.Config.Size + manifestDigest = manifest.Digest.String() + manifestSize = manifest.Size + mediaType = manifest.Manifest.MediaType + artifactType = zcommon.GetManifestArtifactType(fullImageMeta.Manifests[0].Manifest) + platform = getPlatform(manifest.Config.Platform) + imageLastUpdated = zcommon.GetImageLastUpdated(manifest.Config) + downloadCount = fullImageMeta.Statistics.DownloadCount + isSigned = isImageSigned(fullImageMeta.Signatures) + ) + + imageSize, imageBlobsMap := getImageBlobsInfo(manifestDigest, manifestSize, configDigest, configSize, + manifest.Manifest.Layers) + imageSizeStr := strconv.FormatInt(imageSize, 10) + annotations := GetAnnotations(manifest.Manifest.Annotations, manifest.Config.Config.Labels) + + authors := annotations.Authors + if authors == "" { + authors = manifest.Config.Author + } + + historyEntries, err := getAllHistory(manifest.Manifest, manifest.Config) + if err != nil { + graphql.AddError(ctx, gqlerror.Errorf("error generating history on tag %s in repo %s: "+ + "manifest digest: %s, error: %s", tag, repoName, manifest.Digest, err.Error())) + } + + signaturesInfo := GetSignaturesInfo(isSigned, fullImageMeta.Signatures) + + manifestSummary := gql_generated.ManifestSummary{ + Digest: &manifestDigest, + ConfigDigest: &configDigest, + LastUpdated: &imageLastUpdated, + Size: &imageSizeStr, + IsSigned: &isSigned, + SignatureInfo: signaturesInfo, + Platform: &platform, + DownloadCount: &downloadCount, + Layers: getLayersSummaries(manifest.Manifest), + History: historyEntries, + Referrers: getReferrers(fullImageMeta.Referrers), + ArtifactType: &artifactType, + } + + imageSummary := gql_generated.ImageSummary{ + RepoName: &repoName, + Tag: &tag, + Digest: &manifestDigest, + MediaType: &mediaType, + Manifests: []*gql_generated.ManifestSummary{&manifestSummary}, + LastUpdated: &imageLastUpdated, + IsSigned: &isSigned, + SignatureInfo: signaturesInfo, + Size: &imageSizeStr, + DownloadCount: &downloadCount, + Description: &annotations.Description, + Title: &annotations.Title, + Documentation: &annotations.Documentation, + Licenses: &annotations.Licenses, + Labels: &annotations.Labels, + Source: &annotations.Source, + Vendor: &annotations.Vendor, + Authors: &authors, + Referrers: manifestSummary.Referrers, + } + + return &imageSummary, imageBlobsMap, nil +} + +func isImageSigned(manifestSignatures mTypes.ManifestSignatures) bool { + for _, signatures := range manifestSignatures { + if len(signatures) > 0 { + return true + } + } + + return false +} + +func getPlatform(platform ispec.Platform) gql_generated.Platform { + return gql_generated.Platform{ + Os: ref(platform.OS), + Arch: ref(getArch(platform.Architecture, platform.Variant)), + } +} + +func getArch(arch string, variant string) string { + if variant != "" { + arch = arch + "/" + variant + } + + return arch +} + +func ref[T any](val T) *T { + ref := val + + return &ref +} + +func deref[T any](pointer *T, defaultVal T) T { + if pointer != nil { + return *pointer + } + + return defaultVal +} + +func PaginatedFullImageMeta2ImageSummaries(ctx context.Context, imageMetaList []mTypes.FullImageMeta, skip SkipQGLField, + cveInfo cveinfo.CveInfo, filter mTypes.Filter, pageInput pagination.PageInput, +) ([]*gql_generated.ImageSummary, zcommon.PageInfo, error) { + imagePageFinder, err := pagination.NewImgSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy) + if err != nil { + return []*gql_generated.ImageSummary{}, zcommon.PageInfo{}, err + } + + for _, imageMeta := range imageMetaList { + imageSummary, _, err := FullImageMeta2ImageSummary(ctx, imageMeta) + if err != nil { + continue + } + + if ImgSumAcceptedByFilter(imageSummary, filter) { + imagePageFinder.Add(imageSummary) + } + } + + page, pageInfo := imagePageFinder.Page() + + for _, imageSummary := range page { + // CVE scanning is expensive, only scan for this page + updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo) + } + + return page, pageInfo, nil +} diff --git a/pkg/extensions/search/convert/oci.go b/pkg/extensions/search/convert/oci.go index a4f7a37b..1df370f4 100644 --- a/pkg/extensions/search/convert/oci.go +++ b/pkg/extensions/search/convert/oci.go @@ -9,19 +9,6 @@ import ( "zotregistry.io/zot/pkg/extensions/search/gql_generated" ) -// updateRepoBlobsMap adds all the image blobs and their respective size to the repo blobs map -// and returnes the total size of the image. -func updateRepoBlobsMap(imageBlobs map[string]int64, repoBlob2Size map[string]int64) int64 { - imgSize := int64(0) - - for digest, size := range imageBlobs { - repoBlob2Size[digest] = size - imgSize += size - } - - return imgSize -} - func getLayersSummaries(manifestContent ispec.Manifest) []*gql_generated.LayerSummary { layers := make([]*gql_generated.LayerSummary, 0, len(manifestContent.Layers)) diff --git a/pkg/extensions/search/cve/cve.go b/pkg/extensions/search/cve/cve.go index f4747cff..a4acb8b4 100644 --- a/pkg/extensions/search/cve/cve.go +++ b/pkg/extensions/search/cve/cve.go @@ -1,7 +1,7 @@ package cveinfo import ( - "encoding/json" + "context" "sort" "strings" "time" @@ -9,6 +9,7 @@ import ( godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" + zerr "zotregistry.io/zot/errors" zcommon "zotregistry.io/zot/pkg/common" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/extensions/search/cve/trivy" @@ -57,7 +58,7 @@ func NewCVEInfo(scanner Scanner, metaDB mTypes.MetaDB, log log.Logger) *BaseCveI func (cveinfo BaseCveInfo) GetImageListForCVE(repo, cveID string) ([]cvemodel.TagInfo, error) { imgList := make([]cvemodel.TagInfo, 0) - repoMeta, err := cveinfo.MetaDB.GetRepoMeta(repo) + repoMeta, err := cveinfo.MetaDB.GetRepoMeta(context.Background(), repo) if err != nil { cveinfo.Log.Error().Err(err).Str("repository", repo).Str("cve-id", cveID). Msg("unable to get list of tags from repo") @@ -105,7 +106,7 @@ func (cveinfo BaseCveInfo) GetImageListForCVE(repo, cveID string) ([]cvemodel.Ta } func (cveinfo BaseCveInfo) GetImageListWithCVEFixed(repo, cveID string) ([]cvemodel.TagInfo, error) { - repoMeta, err := cveinfo.MetaDB.GetRepoMeta(repo) + repoMeta, err := cveinfo.MetaDB.GetRepoMeta(context.Background(), repo) if err != nil { cveinfo.Log.Error().Err(err).Str("repository", repo).Str("cve-id", cveID). Msg("unable to get list of tags from repo") @@ -287,19 +288,16 @@ func getIndexContent(metaDB mTypes.MetaDB, indexDigestStr string) (ispec.Index, return ispec.Index{}, err } - indexData, err := metaDB.GetIndexData(indexDigest) + indexData, err := metaDB.GetImageMeta(indexDigest) if err != nil { return ispec.Index{}, err } - var indexContent ispec.Index - - err = json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return ispec.Index{}, err + if indexData.Index == nil { + return ispec.Index{}, zerr.ErrUnexpectedMediaType } - return indexContent, nil + return *indexData.Index, nil } func getConfigAndDigest(metaDB mTypes.MetaDB, manifestDigestStr string) (ispec.Image, godigest.Digest, error) { @@ -308,17 +306,17 @@ func getConfigAndDigest(metaDB mTypes.MetaDB, manifestDigestStr string) (ispec.I return ispec.Image{}, "", err } - manifestData, err := metaDB.GetManifestData(manifestDigest) + manifestData, err := metaDB.GetImageMeta(manifestDigest) if err != nil { return ispec.Image{}, "", err } - var configContent ispec.Image + // we'll fail the execution if the config is not compatible with ispec.Image because we can't scan this type of images. + if manifestData.Manifests[0].Manifest.Config.MediaType != ispec.MediaTypeImageConfig { + return ispec.Image{}, "", zerr.ErrUnexpectedMediaType + } - // we'll fail the execution if the config is not compatibe with ispec.Image because we can't scan this type of images. - err = json.Unmarshal(manifestData.ConfigBlob, &configContent) - - return configContent, manifestDigest, err + return manifestData.Manifests[0].Config, manifestDigest, err } func filterCVEList(cveMap map[string]cvemodel.CVE, searchedCVE string, pageFinder *CvePageFinder) { diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 662c969b..37c7aa1b 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -5,6 +5,7 @@ package cveinfo_test import ( + "context" "encoding/json" "fmt" "io" @@ -385,8 +386,8 @@ func TestImageFormat(t *testing.T) { log := log.NewLogger("debug", "") metaDB := &mocks.MetaDBMock{ - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{ Tags: map[string]mTypes.Descriptor{ "tag": { MediaType: ispec.MediaTypeImageIndex, @@ -395,8 +396,12 @@ func TestImageFormat(t *testing.T) { }, }, nil }, - GetIndexDataFn: func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{IndexBlob: []byte(`{}`)}, nil + GetImageMetaFn: func(digest godigest.Digest) (mTypes.ImageMeta, error) { + return mTypes.ImageMeta{ + MediaType: ispec.MediaTypeImageIndex, + Digest: godigest.FromString("digest"), + Index: &ispec.Index{}, + }, nil }, } storeController := storage.StoreController{ @@ -765,73 +770,32 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo image11 := CreateImageWith().DefaultLayers(). ImageConfig(ispec.Image{Created: DateRef(2008, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() - repoMeta11 := mTypes.ManifestMetadata{ - ManifestBlob: image11.ManifestDescriptor.Data, - ConfigBlob: image11.ConfigDescriptor.Data, - DownloadCount: 0, - Signatures: mTypes.ManifestSignatures{}, - } - - err = metaDB.SetManifestMeta(repo1, image11.ManifestDescriptor.Digest, repoMeta11) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.1.0", image11.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, "0.1.0", image11.AsImageMeta()) So(err, ShouldBeNil) image12 := CreateImageWith().DefaultLayers(). ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() - repoMeta12 := mTypes.ManifestMetadata{ - ManifestBlob: image12.ManifestDescriptor.Data, - ConfigBlob: image12.ConfigDescriptor.Data, - DownloadCount: 0, - Signatures: mTypes.ManifestSignatures{}, - } - - err = metaDB.SetManifestMeta(repo1, image12.ManifestDescriptor.Digest, repoMeta12) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.0", image12.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, "1.0.0", image12.AsImageMeta()) So(err, ShouldBeNil) image13 := CreateImageWith().DefaultLayers(). ImageConfig(ispec.Image{Created: DateRef(2010, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() - repoMeta13 := mTypes.ManifestMetadata{ - ManifestBlob: image13.ManifestDescriptor.Data, - ConfigBlob: image13.ConfigDescriptor.Data, - DownloadCount: 0, - Signatures: mTypes.ManifestSignatures{}, - } - - err = metaDB.SetManifestMeta(repo1, image13.ManifestDescriptor.Digest, repoMeta13) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.1.0", image13.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, "1.1.0", image13.AsImageMeta()) So(err, ShouldBeNil) image14 := CreateImageWith().DefaultLayers(). ImageConfig(ispec.Image{Created: DateRef(2011, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() - repoMeta14 := mTypes.ManifestMetadata{ - ManifestBlob: image14.ManifestDescriptor.Data, - ConfigBlob: image14.ConfigDescriptor.Data, - } - - err = metaDB.SetManifestMeta(repo1, image14.ManifestDescriptor.Digest, repoMeta14) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.1", image14.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, "1.0.1", image14.AsImageMeta()) So(err, ShouldBeNil) // Create metadb data for scannable image with no vulnerabilities image61 := CreateImageWith().DefaultLayers(). ImageConfig(ispec.Image{Created: DateRef(2016, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() - repoMeta61 := mTypes.ManifestMetadata{ - ManifestBlob: image61.ManifestDescriptor.Data, - ConfigBlob: image61.ConfigDescriptor.Data, - } - - err = metaDB.SetManifestMeta(repo6, image61.ManifestDescriptor.Digest, repoMeta61) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo6, "1.0.0", image61.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo6, "1.0.0", image61.AsImageMeta()) So(err, ShouldBeNil) // Create metadb data for image not supporting scanning @@ -841,106 +805,59 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo Digest: godigest.FromBytes([]byte{10, 10, 10}), }}).ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() - repoMeta21 := mTypes.ManifestMetadata{ - ManifestBlob: image21.ManifestDescriptor.Data, - ConfigBlob: image21.ConfigDescriptor.Data, - } - - err = metaDB.SetManifestMeta(repo2, image21.ManifestDescriptor.Digest, repoMeta21) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, "1.0.0", image21.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo2, "1.0.0", image21.AsImageMeta()) So(err, ShouldBeNil) // Create metadb data for invalid images/negative tests - manifestBlob31 := []byte("invalid manifest blob") - So(err, ShouldBeNil) - - repoMeta31 := mTypes.ManifestMetadata{ - ManifestBlob: manifestBlob31, - } - - digest31 := godigest.FromBytes(manifestBlob31) - err = metaDB.SetManifestMeta(repo3, digest31, repoMeta31) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, "invalid-manifest", digest31, ispec.MediaTypeImageManifest) + image := CreateRandomImage() + err = metaDB.SetRepoReference(repo3, "invalid-manifest", image.AsImageMeta()) So(err, ShouldBeNil) image41 := CreateImageWith().DefaultLayers(). CustomConfigBlob([]byte("invalid config blob"), ispec.MediaTypeImageConfig).Build() - repoMeta41 := mTypes.ManifestMetadata{ - ManifestBlob: image41.ManifestDescriptor.Data, - ConfigBlob: image41.ConfigDescriptor.Data, - } - - err = metaDB.SetManifestMeta(repo4, image41.ManifestDescriptor.Digest, repoMeta41) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo4, "invalid-config", image41.ManifestDescriptor.Digest, - ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo4, "invalid-config", image41.AsImageMeta()) So(err, ShouldBeNil) digest51 := godigest.FromString("abc8") - err = metaDB.SetRepoReference(repo5, "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest) + randomImgData := CreateRandomImage().AsImageMeta() + randomImgData.Digest = digest51 + randomImgData.Manifests[0].Digest = digest51 + err = metaDB.SetRepoReference(repo5, "nonexitent-manifest", randomImgData) So(err, ShouldBeNil) // Create metadb data for scannable image which errors during scan image71 := CreateImageWith().DefaultLayers(). ImageConfig(ispec.Image{Created: DateRef(2000, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() - repoMeta71 := mTypes.ManifestMetadata{ - ManifestBlob: image71.ManifestDescriptor.Data, - ConfigBlob: image71.ConfigDescriptor.Data, - } - - err = metaDB.SetManifestMeta(repo7, image71.ManifestDescriptor.Digest, repoMeta71) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo7, "1.0.0", image71.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo7, "1.0.0", image71.AsImageMeta()) So(err, ShouldBeNil) // create multiarch image with vulnerabilities multiarchImage := CreateRandomMultiarch() - err = metaDB.SetIndexData( - multiarchImage.IndexDescriptor.Digest, - mTypes.IndexData{IndexBlob: multiarchImage.IndexDescriptor.Data}, - ) + err = metaDB.SetRepoReference(repoMultiarch, multiarchImage.Images[0].DigestStr(), + multiarchImage.Images[0].AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetManifestData( - multiarchImage.Images[0].ManifestDescriptor.Digest, - mTypes.ManifestData{ - ManifestBlob: multiarchImage.Images[0].ManifestDescriptor.Data, - ConfigBlob: multiarchImage.Images[0].ConfigDescriptor.Data, + err = metaDB.SetRepoReference(repoMultiarch, multiarchImage.Images[1].DigestStr(), + multiarchImage.Images[1].AsImageMeta()) + So(err, ShouldBeNil) + + err = metaDB.SetRepoReference(repoMultiarch, multiarchImage.Images[2].DigestStr(), + multiarchImage.Images[2].AsImageMeta()) + So(err, ShouldBeNil) + + err = metaDB.SetRepoReference(repoMultiarch, "tagIndex", multiarchImage.AsImageMeta()) + So(err, ShouldBeNil) + + err = metaDB.SetRepoMeta("repo-with-bad-tag-digest", mTypes.RepoMeta{ + Name: "repo-with-bad-tag-digest", + Tags: map[string]mTypes.Descriptor{ + "tag": {MediaType: ispec.MediaTypeImageManifest, Digest: godigest.FromString("1").String()}, }, - ) + }) So(err, ShouldBeNil) - - err = metaDB.SetManifestData( - multiarchImage.Images[1].ManifestDescriptor.Digest, - mTypes.ManifestData{ - ManifestBlob: multiarchImage.Images[1].ManifestDescriptor.Data, - ConfigBlob: multiarchImage.Images[1].ConfigDescriptor.Data, - }, - ) - So(err, ShouldBeNil) - - err = metaDB.SetManifestData( - multiarchImage.Images[2].ManifestDescriptor.Digest, - mTypes.ManifestData{ - ManifestBlob: multiarchImage.Images[2].ManifestDescriptor.Data, - ConfigBlob: multiarchImage.Images[2].ConfigDescriptor.Data, - }, - ) - So(err, ShouldBeNil) - - err = metaDB.SetRepoReference( - repoMultiarch, - "tagIndex", - multiarchImage.IndexDescriptor.Digest, - ispec.MediaTypeImageIndex, - ) - So(err, ShouldBeNil) - // Keep a record of all the image references / digest pairings // This is normally done in MetaDB, but we want to verify // the whole flow, including MetaDB @@ -966,18 +883,6 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo image21Media := image21.ManifestDescriptor.MediaType image21Name := repo2 + ":1.0.0" imageMap[image21Name] = image21Digest - image31Digest := digest31.String() - image31Media := ispec.MediaTypeImageManifest - image31Name := repo3 + ":invalid-manifest" - imageMap[image31Name] = image31Digest - image41Digest := image41.ManifestDescriptor.Digest.String() - image41Media := image41.ManifestDescriptor.MediaType - image41Name := repo4 + ":invalid-config" - imageMap[image41Name] = image41Digest - image51Digest := digest51.String() - image51Media := ispec.MediaTypeImageManifest - image51Name := repo5 + ":nonexitent-manifest" - imageMap[image51Name] = digest51.String() image61Digest := image61.ManifestDescriptor.Digest.String() image61Media := image61.ManifestDescriptor.MediaType image61Name := repo6 + ":1.0.0" @@ -1151,7 +1056,7 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo // Almost same logic compared to actual Trivy specific implementation imageDir, inputTag := repo, reference - repoMeta, err := metaDB.GetRepoMeta(imageDir) + repoMeta, err := metaDB.GetRepoMeta(context.Background(), imageDir) if err != nil { return false, err } @@ -1174,19 +1079,12 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo return false, err } - manifestData, err := metaDB.GetManifestData(manifestDigest) + manifestData, err := metaDB.GetImageMeta(manifestDigest) if err != nil { return false, err } - var manifestContent ispec.Manifest - - err = json.Unmarshal(manifestData.ManifestBlob, &manifestContent) - if err != nil { - return false, zerr.ErrScanNotSupported - } - - for _, imageLayer := range manifestContent.Layers { + for _, imageLayer := range manifestData.Manifests[0].Manifest.Layers { switch imageLayer.MediaType { case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer): @@ -1203,12 +1101,6 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo if repo == repo2 && digest == image21Digest { return false, zerr.ErrScanNotSupported } - if repo == repo3 && digest == image31Digest { - return false, zerr.ErrTagMetaNotFound - } - if repo == repo5 && digest == image51Digest { - return false, zerr.ErrManifestDataNotFound - } if repo == repo100 { return false, zerr.ErrRepoMetaNotFound } @@ -1295,20 +1187,6 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo So(pageInfo.ItemCount, ShouldEqual, 0) So(pageInfo.TotalCount, ShouldEqual, 0) - // Config not valid - cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo4, "invalid-config", "", pageInput) - So(err, ShouldBeNil) - So(len(cveList), ShouldEqual, 0) - So(pageInfo.ItemCount, ShouldEqual, 0) - So(pageInfo.TotalCount, ShouldEqual, 0) - - // Manifest is not found - cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo5, "nonexitent-manifest", "", pageInput) - So(err, ShouldEqual, zerr.ErrManifestDataNotFound) - So(len(cveList), ShouldEqual, 0) - So(pageInfo.ItemCount, ShouldEqual, 0) - So(pageInfo.TotalCount, ShouldEqual, 0) - // Scan failed cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo7, "1.0.0", "", pageInput) So(err, ShouldEqual, ErrFailedScan) @@ -1316,6 +1194,13 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo So(pageInfo.ItemCount, ShouldEqual, 0) So(pageInfo.TotalCount, ShouldEqual, 0) + // Tag is not found + cveList, pageInfo, err = cveInfo.GetCVEListForImage("repo-with-bad-tag-digest", "tag", "", pageInput) + So(err, ShouldEqual, zerr.ErrImageMetaNotFound) + So(len(cveList), ShouldEqual, 0) + So(pageInfo.ItemCount, ShouldEqual, 0) + So(pageInfo.TotalCount, ShouldEqual, 0) + // Repo is not found cveList, pageInfo, err = cveInfo.GetCVEListForImage(repo100, "1.0.0", "", pageInput) So(err, ShouldEqual, zerr.ErrRepoMetaNotFound) @@ -1364,24 +1249,6 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo So(cveSummary.Count, ShouldEqual, 0) So(cveSummary.MaxSeverity, ShouldEqual, "") - // Tag is not found - cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo3, image31Digest, image31Media) - So(err, ShouldEqual, zerr.ErrTagMetaNotFound) - So(cveSummary.Count, ShouldEqual, 0) - So(cveSummary.MaxSeverity, ShouldEqual, "") - - // Config not valid - cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo4, image41Digest, image41Media) - So(err, ShouldBeNil) - So(cveSummary.Count, ShouldEqual, 0) - So(cveSummary.MaxSeverity, ShouldEqual, "NONE") - - // Manifest is not found - cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo5, image51Digest, image51Media) - So(err, ShouldEqual, zerr.ErrManifestDataNotFound) - So(cveSummary.Count, ShouldEqual, 0) - So(cveSummary.MaxSeverity, ShouldEqual, "") - // Scan failed cveSummary, err = cveInfo.GetCVESummaryForImageMedia(repo5, image71Digest, image71Media) So(err, ShouldBeNil) @@ -1430,16 +1297,6 @@ func TestCVEStruct(t *testing.T) { //nolint:gocyclo So(err, ShouldBeNil) So(len(tagList), ShouldEqual, 0) - // Tag is not found, but we should not error - tagList, err = cveInfo.GetImageListWithCVEFixed(repo3, "CVE101") - So(err, ShouldBeNil) - So(len(tagList), ShouldEqual, 0) - - // Manifest is not found, we just consider exclude it from the fixed list - tagList, err = cveInfo.GetImageListWithCVEFixed(repo5, "CVE101") - So(err, ShouldBeNil) - So(len(tagList), ShouldEqual, 0) - // Repo is not found, there could potentially be unaffected tags in the repo // but we can't access their data tagList, err = cveInfo.GetImageListWithCVEFixed(repo100, "CVE100") @@ -1719,158 +1576,6 @@ func TestFixedTagsWithIndex(t *testing.T) { }) } -func TestImageListWithCVEFixedErrors(t *testing.T) { - indexDigest := godigest.FromString("index") - manifestDigest := "sha256:1111111111111111111111111111111111111111111111111111111111111111" - - Convey("Errors", t, func() { - storeController := storage.StoreController{} - storeController.DefaultStore = mocks.MockedImageStore{} - - metaDB := mocks.MetaDBMock{} - log := log.NewLogger("debug", "") - - Convey("getIndexContent errors", func() { - metaDB.GetRepoMetaFn = func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tag": { - Digest: indexDigest.String(), - MediaType: ispec.MediaTypeImageIndex, - }, - }, - }, nil - } - metaDB.GetIndexDataFn = func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{}, zerr.ErrIndexDataNotFount - } - - scanner := cveinfo.NewScanner(storeController, metaDB, "", "", log) - cveInfo := cveinfo.NewCVEInfo(scanner, metaDB, log) - - _, err := cveInfo.GetImageListWithCVEFixed("repo", Vulnerability1ID) - So(err, ShouldBeNil) - }) - - Convey("getIndexContent bad indexDigest", func() { - metaDB.GetRepoMetaFn = func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tag": { - Digest: "bad digest", - MediaType: ispec.MediaTypeImageIndex, - }, - }, - }, nil - } - metaDB.GetIndexDataFn = func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{}, zerr.ErrIndexDataNotFount - } - - scanner := cveinfo.NewScanner(storeController, metaDB, "", "", log) - cveInfo := cveinfo.NewCVEInfo(scanner, metaDB, log) - - _, err := cveInfo.GetImageListWithCVEFixed("repo", Vulnerability1ID) - So(err, ShouldBeNil) - }) - - Convey("getIndexContent bad index content", func() { - metaDB.GetRepoMetaFn = func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tag": { - Digest: indexDigest.String(), - MediaType: ispec.MediaTypeImageIndex, - }, - }, - }, nil - } - metaDB.GetIndexDataFn = func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{IndexBlob: []byte(`bad index`)}, nil - } - - scanner := cveinfo.NewScanner(storeController, metaDB, "", "", log) - cveInfo := cveinfo.NewCVEInfo(scanner, metaDB, log) - - _, err := cveInfo.GetImageListWithCVEFixed("repo", Vulnerability1ID) - So(err, ShouldBeNil) - }) - - Convey("getTagInfoForManifest bad manifest digest", func() { - metaDB.GetRepoMetaFn = func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tag": { - Digest: "bad digest", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - }, nil - } - - scanner := cveinfo.NewScanner(storeController, metaDB, "", "", log) - cveInfo := cveinfo.NewCVEInfo(scanner, metaDB, log) - - _, err := cveInfo.GetImageListWithCVEFixed("repo", Vulnerability1ID) - So(err, ShouldBeNil) - }) - - Convey("getTagInfoForManifest fails for index", func() { - metaDB.GetRepoMetaFn = func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tag": { - Digest: indexDigest.String(), - MediaType: ispec.MediaTypeImageIndex, - }, - }, - }, nil - } - metaDB.GetIndexDataFn = func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{ - IndexBlob: []byte(fmt.Sprintf(`{ - "manifests": [ - { - "digest": "%s", - "mediaType": "application/vnd.oci.image.manifest.v1+json" - } - ]}`, manifestDigest)), - }, nil - } - metaDB.GetManifestDataFn = func(manifestDigest godigest.Digest) (mTypes.ManifestData, error) { - return mTypes.ManifestData{}, zerr.ErrManifestDataNotFound - } - - scanner := cveinfo.NewScanner(storeController, metaDB, "", "", log) - cveInfo := cveinfo.NewCVEInfo(scanner, metaDB, log) - - tagsInfo, err := cveInfo.GetImageListWithCVEFixed("repo", Vulnerability1ID) - So(err, ShouldBeNil) - So(tagsInfo, ShouldBeEmpty) - }) - - Convey("media type not supported", func() { - metaDB.GetRepoMetaFn = func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tag": { - Digest: godigest.FromString("media type").String(), - MediaType: "bad media type", - }, - }, - }, nil - } - - scanner := cveinfo.NewScanner(storeController, metaDB, "", "", log) - cveInfo := cveinfo.NewCVEInfo(scanner, metaDB, log) - - tagsInfo, err := cveInfo.GetImageListWithCVEFixed("repo", Vulnerability1ID) - So(err, ShouldBeNil) - So(tagsInfo, ShouldBeEmpty) - }) - }) -} - func TestGetCVESummaryForImageMediaErrors(t *testing.T) { Convey("Errors", t, func() { storeController := storage.StoreController{} diff --git a/pkg/extensions/search/cve/pagination_test.go b/pkg/extensions/search/cve/pagination_test.go index ab518c2c..c4d87afc 100644 --- a/pkg/extensions/search/cve/pagination_test.go +++ b/pkg/extensions/search/cve/pagination_test.go @@ -4,13 +4,11 @@ package cveinfo_test import ( - "encoding/json" "fmt" "sort" "testing" "time" - godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" @@ -18,7 +16,7 @@ import ( cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/boltdb" - mTypes "zotregistry.io/zot/pkg/meta/types" + . "zotregistry.io/zot/pkg/test/image-utils" "zotregistry.io/zot/pkg/test/mocks" ) @@ -36,70 +34,26 @@ func TestCVEPagination(t *testing.T) { // Create metadb data for scannable image with vulnerabilities timeStamp11 := time.Date(2008, 1, 1, 12, 0, 0, 0, time.UTC) - configBlob11, err := json.Marshal(ispec.Image{ - Created: &timeStamp11, - }) - So(err, ShouldBeNil) + image := CreateImageWith(). + Layers([]Layer{{ + MediaType: ispec.MediaTypeImageLayerGzip, + Digest: ispec.DescriptorEmptyJSON.Digest, + Blob: ispec.DescriptorEmptyJSON.Data, + }}).ImageConfig(ispec.Image{Created: &timeStamp11}).Build() - manifestBlob11, err := json.Marshal(ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: ispec.MediaTypeImageConfig, - Size: 0, - Digest: godigest.FromBytes(configBlob11), - }, - Layers: []ispec.Descriptor{ - { - MediaType: ispec.MediaTypeImageLayerGzip, - Size: 0, - Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"), - }, - }, - }) - So(err, ShouldBeNil) - - repoMeta11 := mTypes.ManifestMetadata{ - ManifestBlob: manifestBlob11, - ConfigBlob: configBlob11, - } - - digest11 := godigest.FromBytes(manifestBlob11) - err = metaDB.SetManifestMeta("repo1", digest11, repoMeta11) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "0.1.0", digest11, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("repo1", "0.1.0", image.AsImageMeta()) So(err, ShouldBeNil) timeStamp12 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC) - configBlob12, err := json.Marshal(ispec.Image{ - Created: &timeStamp12, - }) - So(err, ShouldBeNil) + image2 := CreateImageWith(). + Layers([]Layer{{ + MediaType: ispec.MediaTypeImageLayerGzip, + Digest: ispec.DescriptorEmptyJSON.Digest, + Blob: ispec.DescriptorEmptyJSON.Data, + }}).ImageConfig(ispec.Image{Created: &timeStamp12}).Build() - manifestBlob12, err := json.Marshal(ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: ispec.MediaTypeImageConfig, - Size: 0, - Digest: godigest.FromBytes(configBlob12), - }, - Layers: []ispec.Descriptor{ - { - MediaType: ispec.MediaTypeImageLayerGzip, - Size: 0, - Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"), - }, - }, - }) - So(err, ShouldBeNil) - - repoMeta12 := mTypes.ManifestMetadata{ - ManifestBlob: manifestBlob12, - ConfigBlob: configBlob12, - } - - digest12 := godigest.FromBytes(manifestBlob12) - err = metaDB.SetManifestMeta("repo1", digest12, repoMeta12) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "1.0.0", digest12, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("repo1", "1.0.0", image2.AsImageMeta()) So(err, ShouldBeNil) // MetaDB loaded with initial data, mock the scanner diff --git a/pkg/extensions/search/cve/scan.go b/pkg/extensions/search/cve/scan.go index cacf084a..3f3e1980 100644 --- a/pkg/extensions/search/cve/scan.go +++ b/pkg/extensions/search/cve/scan.go @@ -4,8 +4,6 @@ import ( "context" "sync" - godigest "github.com/opencontainers/go-digest" - "zotregistry.io/zot/pkg/log" mTypes "zotregistry.io/zot/pkg/meta/types" reqCtx "zotregistry.io/zot/pkg/requestcontext" @@ -43,13 +41,13 @@ type scanTaskGenerator struct { } func (gen *scanTaskGenerator) getMatcherFunc() mTypes.FilterFunc { - return func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { + return func(repoMeta mTypes.RepoMeta, imageMeta mTypes.ImageMeta) bool { // Note this matcher will return information based on scan status of manifests // An index scan aggregates results of manifest scans // If at least one of its manifests can be scanned, // the index and its tag will be returned by the caller function too repoName := repoMeta.Name - manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String() + manifestDigest := imageMeta.Digest.String() if gen.isScheduled(manifestDigest) { // We skip this manifest as it has already scheduled @@ -121,18 +119,18 @@ func (gen *scanTaskGenerator) Next() (scheduler.Task, error) { userAc.SetIsAdmin(true) ctx := userAc.DeriveContext(context.Background()) - // Obtain a list of repos with unscanned scannable manifests + // Obtain a list of repos with un-scanned scannable manifests // We may implement a method to return just 1 match at some point - reposMeta, _, _, err := gen.metaDB.FilterTags(ctx, gen.getMatcherFunc()) + imageMeta, err := gen.metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, gen.getMatcherFunc()) if err != nil { - // Do not crash the generator for potential repodb inconistencies + // Do not crash the generator for potential metadb inconsistencies // as there may be scannable images not yet scanned gen.log.Warn().Err(err).Msg("Scheduled CVE scan: error while obtaining repo metadata") } - // no reposMeta are returned, all results are in already in cache + // no imageMeta are returned, all results are in already in cache // or manifests cannot be scanned - if len(reposMeta) == 0 { + if len(imageMeta) == 0 { gen.log.Info().Msg("Scheduled CVE scan: finished for available images") gen.done = true @@ -140,23 +138,14 @@ func (gen *scanTaskGenerator) Next() (scheduler.Task, error) { return nil, nil } - // Since reposMeta will always contain just unscanned images we can pick - // any repo and any tag out of the resulting matches - repoMeta := reposMeta[0] - - var digest string - - // Pick any tag - for _, descriptor := range repoMeta.Tags { - digest = descriptor.Digest - - break - } + // Since imageMeta will always contain just un-scanned images we can pick + // any image out of the resulting matches + digest := imageMeta[0].Digest.String() // Mark the digest as scheduled so it is skipped on next generator run gen.setScheduled(digest, true) - return newScanTask(gen, repoMeta.Name, digest), nil + return newScanTask(gen, imageMeta[0].Repo, digest), nil } func (gen *scanTaskGenerator) IsDone() bool { diff --git a/pkg/extensions/search/cve/scan_test.go b/pkg/extensions/search/cve/scan_test.go index ccb45dfc..bc50a994 100644 --- a/pkg/extensions/search/cve/scan_test.go +++ b/pkg/extensions/search/cve/scan_test.go @@ -5,7 +5,6 @@ package cveinfo_test import ( "context" - "encoding/json" "errors" "io" "os" @@ -69,77 +68,38 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo metaDB, err := boltdb.New(boltDriver, log.NewLogger("debug", "")) So(err, ShouldBeNil) + // Refactor Idea: We can use InitializeTestMetaDB + // Create metadb data for scannable image with vulnerabilities image11 := CreateImageWith().DefaultLayers(). ImageConfig(ispec.Image{Created: DateRef(2008, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() - repoMeta11 := mTypes.ManifestMetadata{ - ManifestBlob: image11.ManifestDescriptor.Data, - ConfigBlob: image11.ConfigDescriptor.Data, - DownloadCount: 0, - Signatures: mTypes.ManifestSignatures{}, - } - - err = metaDB.SetManifestMeta("repo1", image11.ManifestDescriptor.Digest, repoMeta11) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "0.1.0", image11.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("repo1", "0.1.0", image11.AsImageMeta()) So(err, ShouldBeNil) image12 := CreateImageWith().DefaultLayers(). ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() - repoMeta12 := mTypes.ManifestMetadata{ - ManifestBlob: image12.ManifestDescriptor.Data, - ConfigBlob: image12.ConfigDescriptor.Data, - DownloadCount: 0, - Signatures: mTypes.ManifestSignatures{}, - } - - err = metaDB.SetManifestMeta("repo1", image12.ManifestDescriptor.Digest, repoMeta12) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "1.0.0", image12.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("repo1", "1.0.0", image12.AsImageMeta()) So(err, ShouldBeNil) image13 := CreateImageWith().DefaultLayers(). ImageConfig(ispec.Image{Created: DateRef(2010, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() - repoMeta13 := mTypes.ManifestMetadata{ - ManifestBlob: image13.ManifestDescriptor.Data, - ConfigBlob: image13.ConfigDescriptor.Data, - DownloadCount: 0, - Signatures: mTypes.ManifestSignatures{}, - } - - err = metaDB.SetManifestMeta("repo1", image13.ManifestDescriptor.Digest, repoMeta13) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "1.1.0", image13.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("repo1", "1.1.0", image13.AsImageMeta()) So(err, ShouldBeNil) image14 := CreateImageWith().DefaultLayers(). ImageConfig(ispec.Image{Created: DateRef(2011, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() - repoMeta14 := mTypes.ManifestMetadata{ - ManifestBlob: image14.ManifestDescriptor.Data, - ConfigBlob: image14.ConfigDescriptor.Data, - } - - err = metaDB.SetManifestMeta("repo1", image14.ManifestDescriptor.Digest, repoMeta14) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo1", "1.0.1", image14.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("repo1", "1.0.1", image14.AsImageMeta()) So(err, ShouldBeNil) // Create metadb data for scannable image with no vulnerabilities image61 := CreateImageWith().DefaultLayers(). ImageConfig(ispec.Image{Created: DateRef(2016, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() - repoMeta61 := mTypes.ManifestMetadata{ - ManifestBlob: image61.ManifestDescriptor.Data, - ConfigBlob: image61.ConfigDescriptor.Data, - } - - err = metaDB.SetManifestMeta("repo6", image61.ManifestDescriptor.Digest, repoMeta61) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo6", "1.0.0", image61.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("repo6", "1.0.0", image61.AsImageMeta()) So(err, ShouldBeNil) // Create metadb data for image not supporting scanning @@ -149,104 +109,58 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo Digest: godigest.FromBytes([]byte{10, 10, 10}), }}).ImageConfig(ispec.Image{Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() - repoMeta21 := mTypes.ManifestMetadata{ - ManifestBlob: image21.ManifestDescriptor.Data, - ConfigBlob: image21.ConfigDescriptor.Data, - } - - err = metaDB.SetManifestMeta("repo2", image21.ManifestDescriptor.Digest, repoMeta21) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo2", "1.0.0", image21.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("repo2", "1.0.0", image21.AsImageMeta()) So(err, ShouldBeNil) // Create metadb data for invalid images/negative tests - manifestBlob31 := []byte("invalid manifest blob") - So(err, ShouldBeNil) + img := CreateRandomImage() + digest31 := img.Digest() - repoMeta31 := mTypes.ManifestMetadata{ - ManifestBlob: manifestBlob31, - } - - digest31 := godigest.FromBytes(manifestBlob31) - err = metaDB.SetManifestMeta("repo3", digest31, repoMeta31) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo3", "invalid-manifest", digest31, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("repo3", "invalid-manifest", img.AsImageMeta()) So(err, ShouldBeNil) image41 := CreateImageWith().DefaultLayers(). CustomConfigBlob([]byte("invalid config blob"), ispec.MediaTypeImageConfig).Build() - repoMeta41 := mTypes.ManifestMetadata{ - ManifestBlob: image41.ManifestDescriptor.Data, - ConfigBlob: image41.ConfigDescriptor.Data, - } - - err = metaDB.SetManifestMeta("repo4", image41.ManifestDescriptor.Digest, repoMeta41) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo4", "invalid-config", image41.ManifestDescriptor.Digest, - ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("repo4", "invalid-config", image41.AsImageMeta()) So(err, ShouldBeNil) - digest51 := godigest.FromString("abc8") - err = metaDB.SetRepoReference("repo5", "nonexitent-manifest", digest51, ispec.MediaTypeImageManifest) + image15 := CreateRandomMultiarch() + + digest51 := image15.Digest() + err = metaDB.SetRepoReference("repo5", "nonexitent-manifests-for-multiarch", image15.AsImageMeta()) So(err, ShouldBeNil) // Create metadb data for scannable image which errors during scan image71 := CreateImageWith().DefaultLayers(). ImageConfig(ispec.Image{Created: DateRef(2000, 1, 1, 12, 0, 0, 0, time.UTC)}).Build() - repoMeta71 := mTypes.ManifestMetadata{ - ManifestBlob: image71.ManifestDescriptor.Data, - ConfigBlob: image71.ConfigDescriptor.Data, - } - - err = metaDB.SetManifestMeta("repo7", image71.ManifestDescriptor.Digest, repoMeta71) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo7", "1.0.0", image71.ManifestDescriptor.Digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("repo7", "1.0.0", image71.AsImageMeta()) So(err, ShouldBeNil) // Create multiarch image with vulnerabilities multiarchImage := CreateRandomMultiarch() - err = metaDB.SetIndexData( - multiarchImage.IndexDescriptor.Digest, - mTypes.IndexData{IndexBlob: multiarchImage.IndexDescriptor.Data}, - ) + err = metaDB.SetRepoReference(repoIndex, multiarchImage.Images[0].DigestStr(), + multiarchImage.Images[0].AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(repoIndex, multiarchImage.Images[1].DigestStr(), + multiarchImage.Images[1].AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(repoIndex, multiarchImage.Images[2].DigestStr(), + multiarchImage.Images[2].AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetManifestData( - multiarchImage.Images[0].ManifestDescriptor.Digest, - mTypes.ManifestData{ - ManifestBlob: multiarchImage.Images[0].ManifestDescriptor.Data, - ConfigBlob: multiarchImage.Images[0].ConfigDescriptor.Data, + err = metaDB.SetRepoReference(repoIndex, "tagIndex", multiarchImage.AsImageMeta()) + So(err, ShouldBeNil) + + err = metaDB.SetRepoMeta("repo-with-bad-tag-digest", mTypes.RepoMeta{ + Name: "repo-with-bad-tag-digest", + Tags: map[string]mTypes.Descriptor{ + "tag": {MediaType: ispec.MediaTypeImageManifest, Digest: godigest.FromString("1").String()}, + "tag-multi-arch": {MediaType: ispec.MediaTypeImageIndex, Digest: godigest.FromString("2").String()}, }, - ) - So(err, ShouldBeNil) - - err = metaDB.SetManifestData( - multiarchImage.Images[1].ManifestDescriptor.Digest, - mTypes.ManifestData{ - ManifestBlob: multiarchImage.Images[1].ManifestDescriptor.Data, - ConfigBlob: multiarchImage.Images[1].ConfigDescriptor.Data, - }, - ) - So(err, ShouldBeNil) - - err = metaDB.SetManifestData( - multiarchImage.Images[2].ManifestDescriptor.Digest, - mTypes.ManifestData{ - ManifestBlob: multiarchImage.Images[2].ManifestDescriptor.Data, - ConfigBlob: multiarchImage.Images[2].ConfigDescriptor.Data, - }, - ) - So(err, ShouldBeNil) - - err = metaDB.SetRepoReference( - repoIndex, - "tagIndex", - multiarchImage.IndexDescriptor.Digest, - ispec.MediaTypeImageIndex, - ) + }) So(err, ShouldBeNil) // Keep a record of all the image references / digest pairings @@ -274,7 +188,7 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo image41Digest := image41.ManifestDescriptor.Digest.String() image41Name := "repo4:invalid-config" imageMap[image41Name] = image41Digest - image51Name := "repo5:nonexitent-manifest" + image51Name := "repo5:nonexitent-manifest-for-multiarch" imageMap[image51Name] = digest51.String() image61Digest := image61.ManifestDescriptor.Digest.String() image61Name := "repo6:1.0.0" @@ -296,7 +210,7 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo imageMap[indexM3Name] = indexM3Digest // Initialize a test CVE cache - cache := cvecache.NewCveCache(10, logger) + cache := cvecache.NewCveCache(20, logger) // MetaDB loaded with initial data, now mock the scanner // Setup test CVE data in mock scanner @@ -440,7 +354,7 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo // Almost same logic compared to actual Trivy specific implementation imageDir, inputTag := repo, reference - repoMeta, err := metaDB.GetRepoMeta(imageDir) + repoMeta, err := metaDB.GetRepoMeta(context.Background(), imageDir) if err != nil { return false, err } @@ -463,19 +377,12 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo return false, err } - manifestData, err := metaDB.GetManifestData(manifestDigest) + manifestData, err := metaDB.GetImageMeta(manifestDigest) if err != nil { return false, err } - var manifestContent ispec.Manifest - - err = json.Unmarshal(manifestData.ManifestBlob, &manifestContent) - if err != nil { - return false, zerr.ErrScanNotSupported - } - - for _, imageLayer := range manifestContent.Layers { + for _, imageLayer := range manifestData.Manifests[0].Manifest.Layers { switch imageLayer.MediaType { case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer): @@ -531,7 +438,7 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo // Make sure the scanner generator has completed despite errors found, err := test.ReadLogFileAndSearchString(logPath, - "Scheduled CVE scan: finished for available images", 20*time.Second) + "Scheduled CVE scan: finished for available images", 40*time.Second) So(err, ShouldBeNil) So(found, ShouldBeTrue) @@ -546,13 +453,12 @@ func TestScanGeneratorWithMockedData(t *testing.T) { //nolint: gocyclo t.Log("expecting " + image + " " + digestStr + " to be present in cache") So(scanner.IsResultCached(digestStr), ShouldBeTrue) } else { - // We don't cache results for unscannable manifests + // We don't cache results for un-scannable manifests t.Log("expecting " + image + " " + digestStr + " to be absent from cache") So(scanner.IsResultCached(digestStr), ShouldBeFalse) } } - // Make sure the scanner generator is catching the metadb error for repo5:nonexitent-manifest found, err = test.ReadLogFileAndSearchString(logPath, "Scheduled CVE scan: error while obtaining repo metadata", 20*time.Second) So(err, ShouldBeNil) diff --git a/pkg/extensions/search/cve/trivy/scanner.go b/pkg/extensions/search/cve/trivy/scanner.go index ca70c742..6ae1a743 100644 --- a/pkg/extensions/search/cve/trivy/scanner.go +++ b/pkg/extensions/search/cve/trivy/scanner.go @@ -2,7 +2,6 @@ package trivy import ( "context" - "encoding/json" "fmt" "os" "path" @@ -26,7 +25,6 @@ import ( cvecache "zotregistry.io/zot/pkg/extensions/search/cve/cache" cvemodel "zotregistry.io/zot/pkg/extensions/search/cve/model" "zotregistry.io/zot/pkg/log" - mcommon "zotregistry.io/zot/pkg/meta/common" mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/storage" ) @@ -193,7 +191,7 @@ func (scanner Scanner) IsImageFormatScannable(repo, ref string) (bool, error) { ) if zcommon.IsTag(ref) { - imgDescriptor, err := mcommon.GetImageDescriptor(scanner.metaDB, repo, ref) + imgDescriptor, err := getImageDescriptor(scanner.metaDB, repo, ref) if err != nil { return false, err } @@ -203,7 +201,7 @@ func (scanner Scanner) IsImageFormatScannable(repo, ref string) (bool, error) { } else { var found bool - found, mediaType = mcommon.FindMediaTypeForDigest(scanner.metaDB, godigest.Digest(ref)) + found, mediaType = findMediaTypeForDigest(scanner.metaDB, godigest.Digest(ref)) if !found { return false, zerr.ErrManifestNotFound } @@ -224,7 +222,7 @@ func (scanner Scanner) IsImageMediaScannable(repo, digestStr, mediaType string) return ok, nil case ispec.MediaTypeImageIndex: - ok, err := scanner.isIndexScanable(digestStr) + ok, err := scanner.isIndexScannable(digestStr) if err != nil { return ok, fmt.Errorf("image '%s' %w", image, err) } @@ -240,21 +238,16 @@ func (scanner Scanner) isManifestScanable(digestStr string) (bool, error) { return true, nil } - manifestData, err := scanner.metaDB.GetManifestData(godigest.Digest(digestStr)) + manifestData, err := scanner.metaDB.GetImageMeta(godigest.Digest(digestStr)) if err != nil { return false, err } - var manifestContent ispec.Manifest - - err = json.Unmarshal(manifestData.ManifestBlob, &manifestContent) - if err != nil { - scanner.log.Error().Err(err).Msg("unable to unmashal manifest blob") - - return false, zerr.ErrScanNotSupported + if manifestData.MediaType != ispec.MediaTypeImageManifest { + return false, zerr.ErrUnexpectedMediaType } - for _, imageLayer := range manifestContent.Layers { + for _, imageLayer := range manifestData.Manifests[0].Manifest.Layers { switch imageLayer.MediaType { case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer): continue @@ -266,34 +259,54 @@ func (scanner Scanner) isManifestScanable(digestStr string) (bool, error) { return true, nil } -func (scanner Scanner) isIndexScanable(digestStr string) (bool, error) { +func (scanner Scanner) isManifestDataScannable(manifestData mTypes.ManifestData) (bool, error) { + if scanner.cache.Get(manifestData.Digest.String()) != nil { + return true, nil + } + + if manifestData.Manifest.MediaType != ispec.MediaTypeImageManifest { + return false, zerr.ErrScanNotSupported + } + + for _, imageLayer := range manifestData.Manifest.Layers { + switch imageLayer.MediaType { + case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer): + continue + default: + return false, zerr.ErrScanNotSupported + } + } + + return true, nil +} + +func (scanner Scanner) isIndexScannable(digestStr string) (bool, error) { if scanner.cache.Get(digestStr) != nil { return true, nil } - indexData, err := scanner.metaDB.GetIndexData(godigest.Digest(digestStr)) + indexData, err := scanner.metaDB.GetImageMeta(godigest.Digest(digestStr)) if err != nil { return false, err } - var indexContent ispec.Index - - err = json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return false, err + if indexData.MediaType != ispec.MediaTypeImageIndex || indexData.Index == nil { + return false, zerr.ErrUnexpectedMediaType } + indexContent := *indexData.Index + if len(indexContent.Manifests) == 0 { return true, nil } - for _, manifest := range indexContent.Manifests { - isScannable, err := scanner.isManifestScanable(manifest.Digest.String()) + for _, manifest := range indexData.Manifests { + isScannable, err := scanner.isManifestDataScannable(manifest) if err != nil { continue } - // if at least 1 manifest is scanable, the whole index is scanable + // if at least 1 manifest is scannable, the whole index is scannable if isScannable { return true, nil } @@ -323,7 +336,7 @@ func (scanner Scanner) ScanImage(image string) (map[string]cvemodel.CVE, error) digest = ref if isTag { - imgDescriptor, err := mcommon.GetImageDescriptor(scanner.metaDB, repo, ref) + imgDescriptor, err := getImageDescriptor(scanner.metaDB, repo, ref) if err != nil { return map[string]cvemodel.CVE{}, err } @@ -333,7 +346,7 @@ func (scanner Scanner) ScanImage(image string) (map[string]cvemodel.CVE, error) } else { var found bool - found, mediaType = mcommon.FindMediaTypeForDigest(scanner.metaDB, godigest.Digest(ref)) + found, mediaType = findMediaTypeForDigest(scanner.metaDB, godigest.Digest(ref)) if !found { return map[string]cvemodel.CVE{}, zerr.ErrManifestNotFound } @@ -441,21 +454,18 @@ func (scanner Scanner) scanIndex(repo, digest string) (map[string]cvemodel.CVE, return cachedMap, nil } - indexData, err := scanner.metaDB.GetIndexData(godigest.Digest(digest)) + indexData, err := scanner.metaDB.GetImageMeta(godigest.Digest(digest)) if err != nil { return map[string]cvemodel.CVE{}, err } - var indexContent ispec.Index - - err = json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return map[string]cvemodel.CVE{}, err + if indexData.Index == nil { + return map[string]cvemodel.CVE{}, zerr.ErrUnexpectedMediaType } indexCveIDMap := map[string]cvemodel.CVE{} - for _, manifest := range indexContent.Manifests { + for _, manifest := range indexData.Index.Manifests { if isScannable, err := scanner.isManifestScanable(manifest.Digest.String()); isScannable && err == nil { manifestCveIDMap, err := scanner.scanManifest(repo, manifest.Digest.String()) if err != nil { @@ -567,6 +577,31 @@ func (scanner Scanner) checkDBPresence() error { return nil } +func getImageDescriptor(metaDB mTypes.MetaDB, repo, tag string) (mTypes.Descriptor, error) { + repoMeta, err := metaDB.GetRepoMeta(context.Background(), repo) + if err != nil { + return mTypes.Descriptor{}, err + } + + imageDescriptor, ok := repoMeta.Tags[tag] + if !ok { + return mTypes.Descriptor{}, zerr.ErrTagMetaNotFound + } + + return imageDescriptor, nil +} + +// findMediaTypeForDigest will look into the buckets for a certain digest. Depending on which bucket that +// digest is found the corresponding mediatype is returned. +func findMediaTypeForDigest(metaDB mTypes.MetaDB, digest godigest.Digest) (bool, string) { + imageMeta, err := metaDB.GetImageMeta(digest) + if err == nil { + return true, imageMeta.MediaType + } + + return false, "" +} + func convertSeverity(detectedSeverity string) string { trivySeverity, _ := dbTypes.NewSeverity(detectedSeverity) diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index 4d81ef97..5801c5bb 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -22,7 +22,7 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta" "zotregistry.io/zot/pkg/meta/boltdb" - mTypes "zotregistry.io/zot/pkg/meta/types" + "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/imagestore" "zotregistry.io/zot/pkg/storage/local" @@ -288,114 +288,31 @@ func TestImageScannable(t *testing.T) { // Create metadb data for scannable image timeStamp := time.Date(2008, 1, 1, 12, 0, 0, 0, time.UTC) - validConfigBlob, err := json.Marshal(ispec.Image{ + validConfig := ispec.Image{ Created: &timeStamp, - }) - if err != nil { - panic(err) } - validManifestBlob, err := json.Marshal(ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: ispec.MediaTypeImageConfig, - Size: 0, - Digest: godigest.FromBytes(validConfigBlob), - }, - Layers: []ispec.Descriptor{ - { - MediaType: ispec.MediaTypeImageLayerGzip, - Size: 0, - Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"), - }, - }, - }) - if err != nil { - panic(err) - } + validImage := CreateImageWith(). + Layers([]Layer{{ + MediaType: ispec.MediaTypeImageLayerGzip, + Digest: ispec.DescriptorEmptyJSON.Digest, + Blob: ispec.DescriptorEmptyJSON.Data, + }}).ImageConfig(validConfig).Build() - validRepoMeta := mTypes.ManifestData{ - ManifestBlob: validManifestBlob, - ConfigBlob: validConfigBlob, - } - - digestValidManifest := godigest.FromBytes(validManifestBlob) - - err = metaDB.SetManifestData(digestValidManifest, validRepoMeta) - if err != nil { - panic(err) - } - - err = metaDB.SetRepoReference("repo1", "valid", digestValidManifest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("repo1", "valid", validImage.AsImageMeta()) if err != nil { panic(err) } // Create MetaDB data for manifest with unscannable layers - manifestBlobUnscannableLayer, err := json.Marshal(ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: ispec.MediaTypeImageConfig, - Size: 0, - Digest: godigest.FromBytes(validConfigBlob), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "unscannable_media_type", - Size: 0, - Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"), - }, - }, - }) - if err != nil { - panic(err) - } + imageWithUnscannableLayer := CreateImageWith(). + Layers([]Layer{{ + MediaType: "unscannable_media_type", + Digest: ispec.DescriptorEmptyJSON.Digest, + Blob: ispec.DescriptorEmptyJSON.Data, + }}).ImageConfig(validConfig).Build() - repoMetaUnscannableLayer := mTypes.ManifestData{ - ManifestBlob: manifestBlobUnscannableLayer, - ConfigBlob: validConfigBlob, - } - - digestManifestUnscannableLayer := godigest.FromBytes(manifestBlobUnscannableLayer) - - err = metaDB.SetManifestData(digestManifestUnscannableLayer, repoMetaUnscannableLayer) - if err != nil { - panic(err) - } - - err = metaDB.SetRepoReference("repo1", "unscannable-layer", digestManifestUnscannableLayer, - ispec.MediaTypeImageManifest) - if err != nil { - panic(err) - } - - // Create MetaDB data for unmarshable manifest - unmarshableManifestBlob := []byte("Some string") - repoMetaUnmarshable := mTypes.ManifestData{ - ManifestBlob: unmarshableManifestBlob, - ConfigBlob: validConfigBlob, - } - - digestUnmarshableManifest := godigest.FromBytes(unmarshableManifestBlob) - - err = metaDB.SetManifestData(digestUnmarshableManifest, repoMetaUnmarshable) - if err != nil { - panic(err) - } - - err = metaDB.SetRepoReference("repo1", "unmarshable", digestUnmarshableManifest, ispec.MediaTypeImageManifest) - if err != nil { - panic(err) - } - - // Manifest meta cannot be found - digestMissingManifest := godigest.FromBytes([]byte("Some other string")) - - err = metaDB.SetRepoReference("repo1", "missing", digestMissingManifest, ispec.MediaTypeImageManifest) - if err != nil { - panic(err) - } - - // RepoMeta contains invalid digest - err = metaDB.SetRepoReference("repo1", "invalid-digest", "invalid", ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("repo1", "unscannable-layer", imageWithUnscannableLayer.AsImageMeta()) if err != nil { panic(err) } @@ -423,18 +340,6 @@ func TestImageScannable(t *testing.T) { So(result, ShouldBeFalse) }) - Convey("Image with unmarshable manifests should be unscannable", t, func() { - result, err := scanner.IsImageFormatScannable("repo1", "unmarshable") - So(err, ShouldNotBeNil) - So(result, ShouldBeFalse) - }) - - Convey("Image with missing manifest meta should be unscannable", t, func() { - result, err := scanner.IsImageFormatScannable("repo1", "missing") - So(err, ShouldNotBeNil) - So(result, ShouldBeFalse) - }) - Convey("Image with invalid manifest digest should be unscannable", t, func() { result, err := scanner.IsImageFormatScannable("repo1", "invalid-digest") So(err, ShouldNotBeNil) @@ -523,14 +428,14 @@ func TestIsIndexScanable(t *testing.T) { scanner.cache.Add("digest", make(map[string]model.CVE)) - found, err := scanner.isIndexScanable("digest") + found, err := scanner.isIndexScannable("digest") So(err, ShouldBeNil) So(found, ShouldBeTrue) }) }) } -func TestScanIndexErrors(t *testing.T) { +func TestIsIndexScannableErrors(t *testing.T) { Convey("Errors", t, func() { storeController := storage.StoreController{} storeController.DefaultStore = mocks.MockedImageStore{} @@ -538,107 +443,22 @@ func TestScanIndexErrors(t *testing.T) { metaDB := mocks.MetaDBMock{} log := log.NewLogger("debug", "") - Convey("GetIndexData fails", func() { - metaDB.GetIndexDataFn = func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{}, godigest.ErrDigestUnsupported + Convey("all manifests of a index are not scannable", func() { + unscannableLayer := []Layer{{MediaType: "unscannable-layer-type", Digest: godigest.FromString("123")}} + img1 := CreateImageWith().Layers(unscannableLayer).RandomConfig().Build() + img2 := CreateImageWith().Layers(unscannableLayer).RandomConfig().Build() + multiarch := CreateMultiarchWith().Images([]Image{img1, img2}).Build() + + metaDB.GetImageMetaFn = func(digest godigest.Digest) (types.ImageMeta, error) { + return map[string]types.ImageMeta{ + img1.DigestStr(): img1.AsImageMeta(), + img2.DigestStr(): img2.AsImageMeta(), + multiarch.DigestStr(): multiarch.AsImageMeta(), + }[digest.String()], nil } scanner := NewScanner(storeController, metaDB, "", "", log) - - _, err := scanner.scanIndex("repo", "digest") - So(err, ShouldNotBeNil) - }) - - Convey("Bad Index Blob, Unamrshal fails", func() { - metaDB.GetIndexDataFn = func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{ - IndexBlob: []byte(`bad-blob`), - }, nil - } - - scanner := NewScanner(storeController, metaDB, "", "", log) - - _, err := scanner.scanIndex("repo", "digest") - So(err, ShouldNotBeNil) - }) - }) -} - -func TestIsIndexScanableErrors(t *testing.T) { - Convey("Errors", t, func() { - storeController := storage.StoreController{} - storeController.DefaultStore = mocks.MockedImageStore{} - - metaDB := mocks.MetaDBMock{} - log := log.NewLogger("debug", "") - - Convey("GetIndexData errors", func() { - metaDB.GetIndexDataFn = func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{}, zerr.ErrManifestDataNotFound - } - scanner := NewScanner(storeController, metaDB, "", "", log) - - _, err := scanner.isIndexScanable("digest") - So(err, ShouldNotBeNil) - }) - - Convey("bad index data, can't unmarshal", func() { - metaDB.GetIndexDataFn = func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{IndexBlob: []byte(`bad`)}, nil - } - scanner := NewScanner(storeController, metaDB, "", "", log) - - ok, err := scanner.isIndexScanable("digest") - So(err, ShouldNotBeNil) - So(ok, ShouldBeFalse) - }) - - Convey("is Manifest Scanable errors", func() { - metaDB.GetIndexDataFn = func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{IndexBlob: []byte(`{ - "manifests": [{ - "digest": "digest2" - }, - { - "digest": "digest1" - } - ] - }`)}, nil - } - metaDB.GetManifestDataFn = func(manifestDigest godigest.Digest) (mTypes.ManifestData, error) { - switch manifestDigest { - case "digest1": - return mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - }, nil - case "digest2": - return mTypes.ManifestData{}, zerr.ErrBadBlob - } - - return mTypes.ManifestData{}, nil - } - scanner := NewScanner(storeController, metaDB, "", "", log) - - ok, err := scanner.isIndexScanable("digest") - So(err, ShouldBeNil) - So(ok, ShouldBeTrue) - }) - - Convey("is Manifest Scanable returns false because no manifest is scanable", func() { - metaDB.GetIndexDataFn = func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{IndexBlob: []byte(`{ - "manifests": [{ - "digest": "digest2" - } - ] - }`)}, nil - } - metaDB.GetManifestDataFn = func(manifestDigest godigest.Digest) (mTypes.ManifestData, error) { - return mTypes.ManifestData{}, zerr.ErrBadBlob - } - scanner := NewScanner(storeController, metaDB, "", "", log) - - ok, err := scanner.isIndexScanable("digest") + ok, err := scanner.isIndexScannable(multiarch.DigestStr()) So(err, ShouldBeNil) So(ok, ShouldBeFalse) }) diff --git a/pkg/extensions/search/cve/trivy/scanner_test.go b/pkg/extensions/search/cve/trivy/scanner_test.go index c03b8218..028549c4 100644 --- a/pkg/extensions/search/cve/trivy/scanner_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_test.go @@ -11,7 +11,6 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" - zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/api" "zotregistry.io/zot/pkg/api/config" extconf "zotregistry.io/zot/pkg/extensions/config" @@ -20,13 +19,11 @@ import ( "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta" "zotregistry.io/zot/pkg/meta/boltdb" - mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/local" . "zotregistry.io/zot/pkg/test/common" "zotregistry.io/zot/pkg/test/deprecated" . "zotregistry.io/zot/pkg/test/image-utils" - "zotregistry.io/zot/pkg/test/mocks" ) func TestScanBigTestFile(t *testing.T) { @@ -130,31 +127,6 @@ func TestScanningByDigest(t *testing.T) { }) } -func TestScannerErrors(t *testing.T) { - digest := godigest.FromString("dig") - - Convey("Errors", t, func() { - storeController := storage.StoreController{} - storeController.DefaultStore = mocks.MockedImageStore{} - - metaDB := mocks.MetaDBMock{} - log := log.NewLogger("debug", "") - - Convey("IsImageFormatSanable", func() { - metaDB.GetManifestDataFn = func(manifestDigest godigest.Digest) (mTypes.ManifestData, error) { - return mTypes.ManifestData{}, zerr.ErrManifestDataNotFound - } - metaDB.GetIndexDataFn = func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{}, zerr.ErrManifestDataNotFound - } - scanner := trivy.NewScanner(storeController, metaDB, "", "", log) - - _, err := scanner.ScanImage("repo@" + digest.String()) - So(err, ShouldNotBeNil) - }) - }) -} - func TestVulnerableLayer(t *testing.T) { Convey("Vulnerable layer", t, func() { vulnerableLayer, err := GetLayerWithVulnerability() diff --git a/pkg/extensions/search/cve/update_test.go b/pkg/extensions/search/cve/update_test.go index e9c0849e..8cdf9554 100644 --- a/pkg/extensions/search/cve/update_test.go +++ b/pkg/extensions/search/cve/update_test.go @@ -40,8 +40,8 @@ func TestCVEDBGenerator(t *testing.T) { sch := scheduler.NewScheduler(cfg, logger) metaDB := &mocks.MetaDBMock{ - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{ Tags: map[string]mTypes.Descriptor{ "tag": {MediaType: ispec.MediaTypeImageIndex}, }, diff --git a/pkg/extensions/search/resolver.go b/pkg/extensions/search/resolver.go index 5bbd0daa..a6949ac8 100644 --- a/pkg/extensions/search/resolver.go +++ b/pkg/extensions/search/resolver.go @@ -6,13 +6,11 @@ package search import ( "context" - "encoding/json" "errors" "fmt" "sort" "strings" - "github.com/99designs/gqlgen/graphql" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/vektah/gqlparser/v2/gqlerror" @@ -75,18 +73,14 @@ func NewResolver(log log.Logger, storeController storage.StoreController, } func FilterByDigest(digest string) mTypes.FilterFunc { - return func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { + // imageMeta will always contain 1 manifest + return func(repoMeta mTypes.RepoMeta, imageMeta mTypes.ImageMeta) bool { lookupDigest := digest contains := false - var manifest ispec.Manifest + manifest := imageMeta.Manifests[0] - err := json.Unmarshal(manifestMeta.ManifestBlob, &manifest) - if err != nil { - return false - } - - manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String() + manifestDigest := manifest.Digest.String() // Check the image manifest in index.json matches the search digest // This is a blob with mediaType application/vnd.oci.image.manifest.v1+json @@ -96,13 +90,13 @@ func FilterByDigest(digest string) mTypes.FilterFunc { // Check the image config matches the search digest // This is a blob with mediaType application/vnd.oci.image.config.v1+json - if strings.Contains(manifest.Config.Digest.String(), lookupDigest) { + if strings.Contains(manifest.Manifest.Config.Digest.String(), lookupDigest) { contains = true } // Check to see if the individual layers in the oci image manifest match the digest // These are blobs with mediaType application/vnd.oci.image.layer.v1.tar+gzip - for _, layer := range manifest.Layers { + for _, layer := range manifest.Manifest.Layers { if strings.Contains(layer.Digest.String(), lookupDigest) { contains = true } @@ -124,21 +118,20 @@ func getImageListForDigest(ctx context.Context, digest string, metaDB mTypes.Met } pageInput := pagination.PageInput{ - Limit: safeDereferencing(requestedPage.Limit, 0), - Offset: safeDereferencing(requestedPage.Offset, 0), + Limit: deref(requestedPage.Limit, 0), + Offset: deref(requestedPage.Offset, 0), SortBy: pagination.SortCriteria( - safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance), + deref(requestedPage.SortBy, gql_generated.SortCriteriaRelevance), ), } - // get all repos - reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, FilterByDigest(digest)) + fullImageMetaList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, FilterByDigest(digest)) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } - imageSummaries, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, - indexDataMap, skip, cveInfo, mTypes.Filter{}, pageInput) + imageSummaries, pageInfo, err := convert.PaginatedFullImageMeta2ImageSummaries(ctx, fullImageMetaList, skip, + cveInfo, mTypes.Filter{}, pageInput) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } @@ -157,7 +150,7 @@ func getImageSummary(ctx context.Context, repo, tag string, digest *string, skip ) ( *gql_generated.ImageSummary, error, ) { - repoMeta, err := metaDB.GetRepoMeta(repo) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo) if err != nil { return nil, err } @@ -167,108 +160,23 @@ func getImageSummary(ctx context.Context, repo, tag string, digest *string, skip return nil, gqlerror.Errorf("can't find image: %s:%s", repo, tag) } - for t := range repoMeta.Tags { - if t != tag { - delete(repoMeta.Tags, t) + repoMeta.Tags = map[string]mTypes.Descriptor{tag: manifestDescriptor} + + imageDigest := manifestDescriptor.Digest + if digest != nil { + imageDigest = *digest + repoMeta.Tags[tag] = mTypes.Descriptor{ + Digest: imageDigest, + MediaType: ispec.MediaTypeImageManifest, } } - var ( - manifestMetaMap = map[string]mTypes.ManifestMetadata{} - indexDataMap = map[string]mTypes.IndexData{} - ) - - switch manifestDescriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDigest := manifestDescriptor.Digest - - if digest != nil && *digest != manifestDigest { - return nil, fmt.Errorf("resolver: can't get ManifestData for digest %s for image '%s:%s' %w", - manifestDigest, repo, tag, zerr.ErrManifestDataNotFound) - } - - manifestData, err := metaDB.GetManifestData(godigest.Digest(manifestDigest)) - if err != nil { - return nil, err - } - - manifestMetaMap[manifestDigest] = mTypes.ManifestMetadata{ - ManifestBlob: manifestData.ManifestBlob, - ConfigBlob: manifestData.ConfigBlob, - } - case ispec.MediaTypeImageIndex: - indexDigest := manifestDescriptor.Digest - - indexData, err := metaDB.GetIndexData(godigest.Digest(indexDigest)) - if err != nil { - return nil, err - } - - var indexContent ispec.Index - - err = json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return nil, err - } - - if digest != nil { - manifestDigest := *digest - - digestFound := false - - for _, manifest := range indexContent.Manifests { - if manifest.Digest.String() == manifestDigest { - digestFound = true - - break - } - } - - if !digestFound { - return nil, fmt.Errorf("resolver: can't get ManifestData for digest %s for image '%s:%s' %w", - manifestDigest, repo, tag, zerr.ErrManifestDataNotFound) - } - - manifestData, err := metaDB.GetManifestData(godigest.Digest(manifestDigest)) - if err != nil { - return nil, fmt.Errorf("resolver: can't get ManifestData for digest %s for image '%s:%s' %w", - manifestDigest, repo, tag, err) - } - - manifestMetaMap[manifestDigest] = mTypes.ManifestMetadata{ - ManifestBlob: manifestData.ManifestBlob, - ConfigBlob: manifestData.ConfigBlob, - } - - // We update the tag descriptor to be the manifest descriptor with digest specified in the - // 'digest' parameter. We treat it as a standalone image. - repoMeta.Tags[tag] = mTypes.Descriptor{ - Digest: manifestDigest, - MediaType: ispec.MediaTypeImageManifest, - } - - break - } - - for _, manifest := range indexContent.Manifests { - manifestData, err := metaDB.GetManifestData(manifest.Digest) - if err != nil { - return nil, fmt.Errorf("resolver: can't get ManifestData for digest %s for image '%s:%s' %w", - manifest.Digest, repo, tag, err) - } - - manifestMetaMap[manifest.Digest.String()] = mTypes.ManifestMetadata{ - ManifestBlob: manifestData.ManifestBlob, - ConfigBlob: manifestData.ConfigBlob, - } - } - - indexDataMap[indexDigest] = indexData - default: - log.Error().Str("mediaType", manifestDescriptor.MediaType).Msg("resolver: media type not supported") + imageMetaMap, err := metaDB.FilterImageMeta(ctx, []string{imageDigest}) + if err != nil { + return &gql_generated.ImageSummary{}, err } - imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, manifestMetaMap, indexDataMap, skipCVE, cveInfo) + imageSummaries := convert.RepoMeta2ImageSummaries(ctx, repoMeta, imageMetaMap, skipCVE, cveInfo) if len(imageSummaries) == 0 { return &gql_generated.ImageSummary{}, nil @@ -290,10 +198,10 @@ func getCVEListForImage( } pageInput := cvemodel.PageInput{ - Limit: safeDereferencing(requestedPage.Limit, 0), - Offset: safeDereferencing(requestedPage.Offset, 0), + Limit: deref(requestedPage.Limit, 0), + Offset: deref(requestedPage.Offset, 0), SortBy: cvemodel.SortCriteria( - safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaSeverity), + deref(requestedPage.SortBy, gql_generated.SortCriteriaSeverity), ), } @@ -352,8 +260,8 @@ func getCVEListForImage( } func FilterByTagInfo(tagsInfo []cvemodel.TagInfo) mTypes.FilterFunc { - return func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String() + return func(repoMeta mTypes.RepoMeta, imageMeta mTypes.ImageMeta) bool { + manifestDigest := imageMeta.Manifests[0].Digest.String() for _, tagInfo := range tagsInfo { switch tagInfo.Descriptor.MediaType { @@ -375,12 +283,12 @@ func FilterByTagInfo(tagsInfo []cvemodel.TagInfo) mTypes.FilterFunc { } func FilterByRepoAndTagInfo(repo string, tagsInfo []cvemodel.TagInfo) mTypes.FilterFunc { - return func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { + return func(repoMeta mTypes.RepoMeta, imageMeta mTypes.ImageMeta) bool { if repoMeta.Name != repo { return false } - manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String() + manifestDigest := imageMeta.Manifests[0].Digest.String() for _, tagInfo := range tagsInfo { switch tagInfo.Descriptor.MediaType { @@ -414,7 +322,7 @@ func getImageListForCVE( // Infinite page to make sure we scan all repos in advance, before filtering results // The CVE scan logic is called from here, not in the actual filter, // this is because we shouldn't keep the DB locked while we wait on scan results - reposMeta, err := metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true }) + reposMeta, err := metaDB.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMeta) bool { return true }) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } @@ -457,21 +365,21 @@ func getImageListForCVE( // Actual page requested by user pageInput := pagination.PageInput{ - Limit: safeDereferencing(requestedPage.Limit, 0), - Offset: safeDereferencing(requestedPage.Offset, 0), + Limit: deref(requestedPage.Limit, 0), + Offset: deref(requestedPage.Offset, 0), SortBy: pagination.SortCriteria( - safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), + deref(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), ), } // get all repos - reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, FilterByTagInfo(affectedImages)) + fullImageMetaList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, FilterByTagInfo(affectedImages)) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } - imageSummaries, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, - indexDataMap, skip, cveInfo, localFilter, pageInput) + imageSummaries, pageInfo, err := convert.PaginatedFullImageMeta2ImageSummaries(ctx, fullImageMetaList, + skip, cveInfo, localFilter, pageInput) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } @@ -530,21 +438,21 @@ func getImageListWithCVEFixed( // Actual page requested by user pageInput := pagination.PageInput{ - Limit: safeDereferencing(requestedPage.Limit, 0), - Offset: safeDereferencing(requestedPage.Offset, 0), + Limit: deref(requestedPage.Limit, 0), + Offset: deref(requestedPage.Offset, 0), SortBy: pagination.SortCriteria( - safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), + deref(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), ), } // get all repos - reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, FilterByRepoAndTagInfo(repo, tagsInfo)) + fullImageMetaList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, FilterByRepoAndTagInfo(repo, tagsInfo)) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } - imageSummaries, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, - indexDataMap, skip, cveInfo, localFilter, pageInput) + imageSummaries, pageInfo, err := convert.PaginatedFullImageMeta2ImageSummaries(ctx, fullImageMetaList, + skip, cveInfo, localFilter, pageInput) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } @@ -576,20 +484,25 @@ func repoListWithNewestImage( } pageInput := pagination.PageInput{ - Limit: safeDereferencing(requestedPage.Limit, 0), - Offset: safeDereferencing(requestedPage.Offset, 0), + Limit: deref(requestedPage.Limit, 0), + Offset: deref(requestedPage.Offset, 0), SortBy: pagination.SortCriteria( - safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), + deref(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), ), } - reposMeta, manifestMetaMap, indexDataMap, err := metaDB.SearchRepos(ctx, "") + repoMetaList, err := metaDB.SearchRepos(ctx, "") if err != nil { return &gql_generated.PaginatedReposResult{}, err } - repos, pageInfo, err := convert.PaginatedRepoMeta2RepoSummaries(ctx, reposMeta, manifestMetaMap, indexDataMap, - skip, cveInfo, mTypes.Filter{}, pageInput) + imageMetaMap, err := metaDB.FilterImageMeta(ctx, mTypes.GetLatestImageDigests(repoMetaList)) + if err != nil { + return &gql_generated.PaginatedReposResult{}, err + } + + repos, pageInfo, err := convert.PaginatedRepoMeta2RepoSummaries(ctx, repoMetaList, imageMetaMap, + mTypes.Filter{}, pageInput, cveInfo, skip) if err != nil { return &gql_generated.PaginatedReposResult{}, err } @@ -611,16 +524,16 @@ func getBookmarkedRepos( requestedPage *gql_generated.PageInput, metaDB mTypes.MetaDB, ) (*gql_generated.PaginatedReposResult, error) { - repoNames, err := metaDB.GetBookmarkedRepos(ctx) + bookmarkedRepos, err := metaDB.GetBookmarkedRepos(ctx) if err != nil { return &gql_generated.PaginatedReposResult{}, err } - filterFn := func(repoMeta mTypes.RepoMetadata) bool { - return zcommon.Contains(repoNames, repoMeta.Name) + filterByName := func(repo string) bool { + return zcommon.Contains(bookmarkedRepos, repo) } - return getFilteredPaginatedRepos(ctx, cveInfo, filterFn, log, requestedPage, metaDB) + return getFilteredPaginatedRepos(ctx, cveInfo, filterByName, log, requestedPage, metaDB) } func getStarredRepos( @@ -630,13 +543,13 @@ func getStarredRepos( requestedPage *gql_generated.PageInput, metaDB mTypes.MetaDB, ) (*gql_generated.PaginatedReposResult, error) { - repoNames, err := metaDB.GetStarredRepos(ctx) + starredRepos, err := metaDB.GetStarredRepos(ctx) if err != nil { return &gql_generated.PaginatedReposResult{}, err } - filterFn := func(repoMeta mTypes.RepoMetadata) bool { - return zcommon.Contains(repoNames, repoMeta.Name) + filterFn := func(repo string) bool { + return zcommon.Contains(starredRepos, repo) } return getFilteredPaginatedRepos(ctx, cveInfo, filterFn, log, requestedPage, metaDB) @@ -645,7 +558,7 @@ func getStarredRepos( func getFilteredPaginatedRepos( ctx context.Context, cveInfo cveinfo.CveInfo, - filterFn mTypes.FilterRepoFunc, + filterFn mTypes.FilterRepoNameFunc, log log.Logger, //nolint:unparam // may be used by devs for debugging requestedPage *gql_generated.PageInput, metaDB mTypes.MetaDB, @@ -659,20 +572,25 @@ func getFilteredPaginatedRepos( } pageInput := pagination.PageInput{ - Limit: safeDereferencing(requestedPage.Limit, 0), - Offset: safeDereferencing(requestedPage.Offset, 0), + Limit: deref(requestedPage.Limit, 0), + Offset: deref(requestedPage.Offset, 0), SortBy: pagination.SortCriteria( - safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), + deref(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), ), } - reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterRepos(ctx, filterFn) + repoMetaList, err := metaDB.FilterRepos(ctx, filterFn, mTypes.AcceptAllRepoMeta) if err != nil { return &gql_generated.PaginatedReposResult{}, err } - repos, pageInfo, err := convert.PaginatedRepoMeta2RepoSummaries(ctx, reposMeta, manifestMetaMap, indexDataMap, - skip, cveInfo, mTypes.Filter{}, pageInput) + latestImageMeta, err := metaDB.FilterImageMeta(ctx, mTypes.GetLatestImageDigests(repoMetaList)) + if err != nil { + return &gql_generated.PaginatedReposResult{}, err + } + + repos, pageInfo, err := convert.PaginatedRepoMeta2RepoSummaries(ctx, repoMetaList, latestImageMeta, + mTypes.Filter{}, pageInput, cveInfo, skip) if err != nil { return &gql_generated.PaginatedReposResult{}, err } @@ -716,22 +634,31 @@ func globalSearch(ctx context.Context, query string, metaDB mTypes.MetaDB, filte } pageInput := pagination.PageInput{ - Limit: safeDereferencing(requestedPage.Limit, 0), - Offset: safeDereferencing(requestedPage.Offset, 0), + Limit: deref(requestedPage.Limit, 0), + Offset: deref(requestedPage.Offset, 0), SortBy: pagination.SortCriteria( - safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance), + deref(requestedPage.SortBy, gql_generated.SortCriteriaRelevance), ), } - reposMeta, manifestMetaMap, indexDataMap, err := metaDB.SearchRepos(ctx, query) + repoMetaList, err := metaDB.SearchRepos(ctx, query) if err != nil { - return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err + return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, + []*gql_generated.LayerSummary{}, err } - repos, pageInfo, err := convert.PaginatedRepoMeta2RepoSummaries(ctx, reposMeta, manifestMetaMap, indexDataMap, - skip, cveInfo, localFilter, pageInput) + imageMetaMap, err := metaDB.FilterImageMeta(ctx, mTypes.GetLatestImageDigests(repoMetaList)) if err != nil { - return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err + return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, + []*gql_generated.LayerSummary{}, err + } + + repos, pageInfo, err := convert.PaginatedRepoMeta2RepoSummaries(ctx, repoMetaList, imageMetaMap, localFilter, + pageInput, cveInfo, + skip) + if err != nil { + return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, + []*gql_generated.LayerSummary{}, err } paginatedRepos.Page = &gql_generated.PageInfo{ @@ -746,20 +673,20 @@ func globalSearch(ctx context.Context, query string, metaDB mTypes.MetaDB, filte } pageInput := pagination.PageInput{ - Limit: safeDereferencing(requestedPage.Limit, 0), - Offset: safeDereferencing(requestedPage.Offset, 0), + Limit: deref(requestedPage.Limit, 0), + Offset: deref(requestedPage.Offset, 0), SortBy: pagination.SortCriteria( - safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance), + deref(requestedPage.SortBy, gql_generated.SortCriteriaRelevance), ), } - reposMeta, manifestMetaMap, indexDataMap, err := metaDB.SearchTags(ctx, query) + fullImageMetaList, err := metaDB.SearchTags(ctx, query) if err != nil { return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err } - imageSummaries, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, - indexDataMap, skip, cveInfo, localFilter, pageInput) + imageSummaries, pageInfo, err := convert.PaginatedFullImageMeta2ImageSummaries(ctx, fullImageMetaList, skip, cveInfo, + localFilter, pageInput) if err != nil { return &gql_generated.PaginatedReposResult{}, []*gql_generated.ImageSummary{}, []*gql_generated.LayerSummary{}, err } @@ -790,10 +717,10 @@ func derivedImageList(ctx context.Context, image string, digest *string, metaDB } pageInput := pagination.PageInput{ - Limit: safeDereferencing(requestedPage.Limit, 0), - Offset: safeDereferencing(requestedPage.Offset, 0), + Limit: deref(requestedPage.Limit, 0), + Offset: deref(requestedPage.Offset, 0), SortBy: pagination.SortCriteria( - safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), + deref(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), ), } @@ -820,13 +747,13 @@ func derivedImageList(ctx context.Context, image string, digest *string, metaDB } // we need all available tags - reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, filterDerivedImages(searchedImage)) + fullImageMetaList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, filterDerivedImages(searchedImage)) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } - derivedList, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, indexDataMap, - skip, cveInfo, mTypes.Filter{}, pageInput) + derivedList, pageInfo, err := convert.PaginatedFullImageMeta2ImageSummaries(ctx, fullImageMetaList, skip, cveInfo, + mTypes.Filter{}, pageInput) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } @@ -841,25 +768,20 @@ func derivedImageList(ctx context.Context, image string, digest *string, metaDB } func filterDerivedImages(image *gql_generated.ImageSummary) mTypes.FilterFunc { - return func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { + return func(repoMeta mTypes.RepoMeta, imageMeta mTypes.ImageMeta) bool { var addImageToList bool - var imageManifest ispec.Manifest - - err := json.Unmarshal(manifestMeta.ManifestBlob, &imageManifest) - if err != nil { - return false - } + imageManifest := imageMeta.Manifests[0] for i := range image.Manifests { - manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String() + manifestDigest := imageManifest.Digest.String() if manifestDigest == *image.Manifests[i].Digest { return false } imageLayers := image.Manifests[i].Layers addImageToList = false - layers := imageManifest.Layers + layers := imageManifest.Manifest.Layers sameLayer := 0 @@ -895,10 +817,10 @@ func baseImageList(ctx context.Context, image string, digest *string, metaDB mTy } pageInput := pagination.PageInput{ - Limit: safeDereferencing(requestedPage.Limit, 0), - Offset: safeDereferencing(requestedPage.Offset, 0), + Limit: deref(requestedPage.Limit, 0), + Offset: deref(requestedPage.Offset, 0), SortBy: pagination.SortCriteria( - safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), + deref(requestedPage.SortBy, gql_generated.SortCriteriaUpdateTime), ), } @@ -926,12 +848,12 @@ func baseImageList(ctx context.Context, image string, digest *string, metaDB mTy } // we need all available tags - reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, filterBaseImages(searchedImage)) + fullImageMetaList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, filterBaseImages(searchedImage)) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } - baseList, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, indexDataMap, + baseList, pageInfo, err := convert.PaginatedFullImageMeta2ImageSummaries(ctx, fullImageMetaList, skip, cveInfo, mTypes.Filter{}, pageInput) if err != nil { return &gql_generated.PaginatedImagesResult{}, err @@ -947,25 +869,20 @@ func baseImageList(ctx context.Context, image string, digest *string, metaDB mTy } func filterBaseImages(image *gql_generated.ImageSummary) mTypes.FilterFunc { - return func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { + return func(repoMeta mTypes.RepoMeta, imageMeta mTypes.ImageMeta) bool { var addImageToList bool - var manifestContent ispec.Manifest - - err := json.Unmarshal(manifestMeta.ManifestBlob, &manifestContent) - if err != nil { - return false - } + manifest := imageMeta.Manifests[0] for i := range image.Manifests { - manifestDigest := godigest.FromBytes(manifestMeta.ManifestBlob).String() + manifestDigest := manifest.Digest.String() if manifestDigest == *image.Manifests[i].Digest { return false } addImageToList = true - for _, l := range manifestContent.Layers { + for _, l := range manifest.Manifest.Layers { foundLayer := false for _, k := range image.Manifests[i].Layers { @@ -996,7 +913,7 @@ func validateGlobalSearchInput(query string, filter *gql_generated.Filter, requestedPage *gql_generated.PageInput, ) error { if len(query) > querySizeLimit { - return fmt.Errorf("global-search: max string size limit exeeded for query parameter. max=%d current=%d %w", + return fmt.Errorf("global-search: max string size limit exceeded for query parameter. max=%d current=%d %w", querySizeLimit, len(query), zerr.ErrInvalidRequestParams) } @@ -1020,14 +937,14 @@ func checkFilter(filter *gql_generated.Filter) error { for _, arch := range filter.Arch { if len(*arch) > querySizeLimit { - return fmt.Errorf("global-search: max string size limit exeeded for arch parameter. max=%d current=%d %w", + return fmt.Errorf("global-search: max string size limit exceeded for arch parameter. max=%d current=%d %w", querySizeLimit, len(*arch), zerr.ErrInvalidRequestParams) } } for _, osSys := range filter.Os { if len(*osSys) > querySizeLimit { - return fmt.Errorf("global-search: max string size limit exeeded for os parameter. max=%d current=%d %w", + return fmt.Errorf("global-search: max string size limit exceeded for os parameter. max=%d current=%d %w", querySizeLimit, len(*osSys), zerr.ErrInvalidRequestParams) } } @@ -1119,92 +1036,28 @@ func expandedRepoInfo(ctx context.Context, repo string, metaDB mTypes.MetaDB, cv return &gql_generated.RepoInfo{}, nil //nolint:nilerr // don't give details to a potential attacker } - repoMeta, err := metaDB.GetUserRepoMeta(ctx, repo) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo) if err != nil { log.Error().Err(err).Str("repository", repo).Msg("resolver: can't retrieve repoMeta for repo") return &gql_generated.RepoInfo{}, err } - var ( - manifestMetaMap = map[string]mTypes.ManifestMetadata{} - indexDataMap = map[string]mTypes.IndexData{} - ) + tagsDigests := []string{} - for tag, descriptor := range repoMeta.Tags { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - digest := descriptor.Digest - - if _, alreadyDownloaded := manifestMetaMap[digest]; alreadyDownloaded { - continue - } - - manifestData, err := metaDB.GetManifestData(godigest.Digest(digest)) - if err != nil { - graphql.AddError(ctx, fmt.Errorf("resolver: failed to get manifest meta for image %s:%s with manifest digest %s %w", - repo, tag, digest, err)) - - continue - } - - manifestMetaMap[digest] = mTypes.ManifestMetadata{ - ManifestBlob: manifestData.ManifestBlob, - ConfigBlob: manifestData.ConfigBlob, - } - case ispec.MediaTypeImageIndex: - digest := descriptor.Digest - - if _, alreadyDownloaded := indexDataMap[digest]; alreadyDownloaded { - continue - } - - indexData, err := metaDB.GetIndexData(godigest.Digest(digest)) - if err != nil { - graphql.AddError(ctx, fmt.Errorf("resolver: failed to get manifest meta for image %s:%s with manifest digest %s %w", - repo, tag, digest, err)) - - continue - } - - var indexContent ispec.Index - - err = json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - graphql.AddError(ctx, fmt.Errorf("resolver: failed to unmarshal index content for image %s:%s with digest %s %w", - repo, tag, digest, err)) - - continue - } - - var errorOccured bool - - for _, descriptor := range indexContent.Manifests { - manifestData, err := metaDB.GetManifestData(descriptor.Digest) - if err != nil { - graphql.AddError(ctx, - fmt.Errorf("resolver: failed to get manifest meta with digest '%s' for multiarch image %s:%s %w", - digest, repo, tag, err), - ) - - errorOccured = true - - break - } - - manifestMetaMap[descriptor.Digest.String()] = mTypes.ManifestMetadata{ - ManifestBlob: manifestData.ManifestBlob, - ConfigBlob: manifestData.ConfigBlob, - } - } - - if errorOccured { - continue - } - - indexDataMap[digest] = indexData - default: + for i := range repoMeta.Tags { + if i == "" { + continue } + + tagsDigests = append(tagsDigests, repoMeta.Tags[i].Digest) + } + + imageMetaMap, err := metaDB.FilterImageMeta(ctx, tagsDigests) + if err != nil { + log.Error().Err(err).Str("repository", repo).Msg("resolver: can't retrieve imageMeta for repo") + + return &gql_generated.RepoInfo{}, err } skip := convert.SkipQGLField{ @@ -1212,7 +1065,7 @@ func expandedRepoInfo(ctx context.Context, repo string, metaDB mTypes.MetaDB, cv canSkipField(convert.GetPreloads(ctx), "Images.Vulnerabilities"), } - repoSummary, imageSummaries := convert.RepoMeta2ExpandedRepoInfo(ctx, repoMeta, manifestMetaMap, indexDataMap, + repoSummary, imageSummaries := convert.RepoMeta2ExpandedRepoInfo(ctx, repoMeta, imageMetaMap, skip, cveInfo, log) dateSortedImages := make(timeSlice, 0, len(imageSummaries)) @@ -1239,7 +1092,7 @@ func (p timeSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func safeDereferencing[T any](pointer *T, defaultVal T) T { +func deref[T any](pointer *T, defaultVal T) T { if pointer != nil { return *pointer } @@ -1263,23 +1116,28 @@ func getImageList(ctx context.Context, repo string, metaDB mTypes.MetaDB, cveInf } pageInput := pagination.PageInput{ - Limit: safeDereferencing(requestedPage.Limit, 0), - Offset: safeDereferencing(requestedPage.Offset, 0), + Limit: deref(requestedPage.Limit, 0), + Offset: deref(requestedPage.Offset, 0), SortBy: pagination.SortCriteria( - safeDereferencing(requestedPage.SortBy, gql_generated.SortCriteriaRelevance), + deref(requestedPage.SortBy, gql_generated.SortCriteriaRelevance), ), } - reposMeta, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return repoMeta.Name == repo || repo == "" - }) + var matchRepoName mTypes.FilterRepoTagFunc + + if repo == "" { + matchRepoName = mTypes.AcceptAllRepoTag + } else { + matchRepoName = func(repoName, tag string) bool { return repoName == repo } + } + + imageMeta, err := metaDB.FilterTags(ctx, matchRepoName, mTypes.AcceptAllImageMeta) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } - imageList, pageInfo, err := convert.PaginatedRepoMeta2ImageSummaries(ctx, reposMeta, manifestMetaMap, - indexDataMap, skip, cveInfo, mTypes.Filter{}, pageInput) + imageList, pageInfo, err := convert.PaginatedFullImageMeta2ImageSummaries(ctx, imageMeta, skip, + cveInfo, mTypes.Filter{}, pageInput) if err != nil { return &gql_generated.PaginatedImagesResult{}, err } diff --git a/pkg/extensions/search/resolver_test.go b/pkg/extensions/search/resolver_test.go index eedcce38..02d8bd5a 100644 --- a/pkg/extensions/search/resolver_test.go +++ b/pkg/extensions/search/resolver_test.go @@ -4,16 +4,13 @@ package search //nolint import ( "context" - "encoding/json" "errors" "fmt" - "strings" "testing" "time" "github.com/99designs/gqlgen/graphql" godigest "github.com/opencontainers/go-digest" - "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" @@ -24,25 +21,25 @@ import ( "zotregistry.io/zot/pkg/extensions/search/gql_generated" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/boltdb" + mConvert "zotregistry.io/zot/pkg/meta/convert" mTypes "zotregistry.io/zot/pkg/meta/types" reqCtx "zotregistry.io/zot/pkg/requestcontext" "zotregistry.io/zot/pkg/storage" + . "zotregistry.io/zot/pkg/test/image-utils" "zotregistry.io/zot/pkg/test/mocks" + ociutils "zotregistry.io/zot/pkg/test/oci-utils" ) var ErrTestError = errors.New("TestError") -func TestGlobalSearch(t *testing.T) { +func TestResolverGlobalSearch(t *testing.T) { Convey("globalSearch", t, func() { const query = "repo1" Convey("MetaDB SearchRepos error", func() { mockMetaDB := mocks.MetaDBMock{ SearchReposFn: func(ctx context.Context, searchText string, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - return make([]mTypes.RepoMetadata, 0), make(map[string]mTypes.ManifestMetadata), - map[string]mTypes.IndexData{}, ErrTestError + ) ([]mTypes.RepoMeta, error) { + return []mTypes.RepoMeta{}, ErrTestError }, } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, @@ -73,249 +70,10 @@ func TestGlobalSearch(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("MetaDB SearchRepo is successful", func() { - mockMetaDB := mocks.MetaDBMock{ - SearchReposFn: func(ctx context.Context, searchText string, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - repos := []mTypes.RepoMetadata{ - { - Name: "repo1", - Tags: map[string]mTypes.Descriptor{ - "1.0.1": { - Digest: "digestTag1.0.1", - MediaType: ispec.MediaTypeImageManifest, - }, - "1.0.2": { - Digest: "digestTag1.0.2", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - Signatures: map[string]mTypes.ManifestSignatures{ - "digestTag1.0.1": { - "cosign": []mTypes.SignatureInfo{ - {SignatureManifestDigest: "testSignature", LayersInfo: []mTypes.LayerInfo{}}, - }, - }, - }, - Stars: 100, - }, - } - - createTime := time.Now() - configBlob1, err := json.Marshal(ispec.Image{ - Config: ispec.ImageConfig{ - Labels: map[string]string{ - ispec.AnnotationVendor: "TestVendor1", - }, - }, - Created: &createTime, - }) - So(err, ShouldBeNil) - - configBlob2, err := json.Marshal(ispec.Image{ - Config: ispec.ImageConfig{ - Labels: map[string]string{ - ispec.AnnotationVendor: "TestVendor2", - }, - }, - }) - So(err, ShouldBeNil) - - manifestBlob, err := json.Marshal(ispec.Manifest{}) - So(err, ShouldBeNil) - - manifestsMeta := map[string]mTypes.ManifestMetadata{ - "digestTag1.0.1": { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob1, - }, - "digestTag1.0.2": { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob2, - }, - } - - return repos, manifestsMeta, map[string]mTypes.IndexData{}, nil - }, - } - - const query = "repo1" - limit := 1 - offset := 0 - sortCriteria := gql_generated.SortCriteriaAlphabeticAsc - pageInput := gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, - } - - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, - graphql.DefaultRecover) - mockCve := mocks.CveInfoMock{} - repos, images, layers, err := globalSearch(responseContext, query, mockMetaDB, - &gql_generated.Filter{}, &pageInput, mockCve, log.NewLogger("debug", "")) - So(err, ShouldBeNil) - So(images, ShouldBeEmpty) - So(layers, ShouldBeEmpty) - So(repos.Results, ShouldNotBeEmpty) - So(len(repos.Results[0].Vendors), ShouldEqual, 2) - }) - - Convey("MetaDB SearchRepo Bad manifest referenced", func() { - mockMetaDB := mocks.MetaDBMock{ - SearchReposFn: func(ctx context.Context, searchText string, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - repos := []mTypes.RepoMetadata{ - { - Name: "repo1", - Tags: map[string]mTypes.Descriptor{ - "1.0.1": { - Digest: "digestTag1.0.1", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - Signatures: map[string]mTypes.ManifestSignatures{ - "digestTag1.0.1": { - "cosign": []mTypes.SignatureInfo{ - {SignatureManifestDigest: "testSignature", LayersInfo: []mTypes.LayerInfo{}}, - }, - }, - }, - Stars: 100, - }, - } - - configBlob, err := json.Marshal(ispec.Image{}) - So(err, ShouldBeNil) - - manifestsMeta := map[string]mTypes.ManifestMetadata{ - "digestTag1.0.1": { - ManifestBlob: []byte("bad manifest blob"), - ConfigBlob: configBlob, - }, - } - - return repos, manifestsMeta, map[string]mTypes.IndexData{}, nil - }, - } - - query := "repo1" - limit := 1 - offset := 0 - sortCriteria := gql_generated.SortCriteriaAlphabeticAsc - pageInput := gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, - } - - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, - graphql.DefaultRecover) - mockCve := mocks.CveInfoMock{} - - repos, images, layers, err := globalSearch(responseContext, query, mockMetaDB, - &gql_generated.Filter{}, &pageInput, mockCve, log.NewLogger("debug", "")) - So(err, ShouldBeNil) - So(images, ShouldBeEmpty) - So(layers, ShouldBeEmpty) - So(repos, ShouldNotBeEmpty) - - query = "repo1:1.0.1" - - responseContext = graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, - graphql.DefaultRecover) - repos, images, layers, err = globalSearch(responseContext, query, mockMetaDB, - &gql_generated.Filter{}, &pageInput, mockCve, log.NewLogger("debug", "")) - So(err, ShouldBeNil) - So(images, ShouldBeEmpty) - So(layers, ShouldBeEmpty) - So(repos.Results, ShouldBeEmpty) - }) - - Convey("MetaDB SearchRepo good manifest referenced and bad config blob", func() { - mockMetaDB := mocks.MetaDBMock{ - SearchReposFn: func(ctx context.Context, searchText string, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - repos := []mTypes.RepoMetadata{ - { - Name: "repo1", - Tags: map[string]mTypes.Descriptor{ - "1.0.1": { - Digest: "digestTag1.0.1", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - Signatures: map[string]mTypes.ManifestSignatures{ - "digestTag1.0.1": { - "cosign": []mTypes.SignatureInfo{ - {SignatureManifestDigest: "testSignature", LayersInfo: []mTypes.LayerInfo{}}, - }, - }, - }, - Stars: 100, - }, - } - - manifestBlob, err := json.Marshal(ispec.Manifest{}) - So(err, ShouldBeNil) - - manifestsMeta := map[string]mTypes.ManifestMetadata{ - "digestTag1.0.1": { - ManifestBlob: manifestBlob, - ConfigBlob: []byte("bad config blob"), - }, - } - - return repos, manifestsMeta, map[string]mTypes.IndexData{}, nil - }, - } - - query := "repo1" - limit := 1 - offset := 0 - sortCriteria := gql_generated.SortCriteriaAlphabeticAsc - pageInput := gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, - } - - mockCve := mocks.CveInfoMock{} - - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, - graphql.DefaultRecover) - repos, images, layers, err := globalSearch(responseContext, query, mockMetaDB, - &gql_generated.Filter{}, &pageInput, mockCve, log.NewLogger("debug", "")) - So(err, ShouldBeNil) - So(images, ShouldBeEmpty) - So(layers, ShouldBeEmpty) - So(repos.Results, ShouldNotBeEmpty) - - query = "repo1:1.0.1" - responseContext = graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, - graphql.DefaultRecover) - repos, images, layers, err = globalSearch(responseContext, query, mockMetaDB, - &gql_generated.Filter{}, &pageInput, mockCve, log.NewLogger("debug", "")) - So(err, ShouldBeNil) - So(images, ShouldBeEmpty) - So(layers, ShouldBeEmpty) - So(repos.Results, ShouldBeEmpty) - }) - Convey("MetaDB SearchTags gives error", func() { mockMetaDB := mocks.MetaDBMock{ - SearchTagsFn: func(ctx context.Context, searchText string, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - return make([]mTypes.RepoMetadata, 0), make(map[string]mTypes.ManifestMetadata), - map[string]mTypes.IndexData{}, ErrTestError + SearchTagsFn: func(ctx context.Context, searchText string) ([]mTypes.FullImageMeta, error) { + return []mTypes.FullImageMeta{}, ErrTestError }, } const query = "repo1:1.0.1" @@ -330,90 +88,6 @@ func TestGlobalSearch(t *testing.T) { So(layers, ShouldBeEmpty) So(repos.Results, ShouldBeEmpty) }) - - Convey("MetaDB SearchTags is successful", func() { - mockMetaDB := mocks.MetaDBMock{ - SearchTagsFn: func(ctx context.Context, searchText string, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - repos := []mTypes.RepoMetadata{ - { - Name: "repo1", - Tags: map[string]mTypes.Descriptor{ - "1.0.1": { - Digest: "digestTag1.0.1", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - Signatures: map[string]mTypes.ManifestSignatures{ - "digestTag1.0.1": { - "cosign": []mTypes.SignatureInfo{ - {SignatureManifestDigest: "testSignature", LayersInfo: []mTypes.LayerInfo{}}, - }, - }, - }, - Stars: 100, - }, - } - - configBlob1, err := json.Marshal(ispec.Image{ - Config: ispec.ImageConfig{ - Labels: map[string]string{ - ispec.AnnotationVendor: "TestVendor1", - }, - }, - }) - So(err, ShouldBeNil) - - configBlob2, err := json.Marshal(ispec.Image{ - Config: ispec.ImageConfig{ - Labels: map[string]string{ - ispec.AnnotationVendor: "TestVendor2", - }, - }, - }) - So(err, ShouldBeNil) - - manifestBlob, err := json.Marshal(ispec.Manifest{}) - So(err, ShouldBeNil) - - manifestsMeta := map[string]mTypes.ManifestMetadata{ - "digestTag1.0.1": { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob1, - }, - "digestTag1.0.2": { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob2, - }, - } - - return repos, manifestsMeta, map[string]mTypes.IndexData{}, nil - }, - } - - const query = "repo1:1.0.1" - limit := 1 - offset := 0 - sortCriteria := gql_generated.SortCriteriaAlphabeticAsc - pageInput := gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, - } - - mockCve := mocks.CveInfoMock{} - - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, - graphql.DefaultRecover) - repos, images, layers, err := globalSearch(responseContext, query, mockMetaDB, - &gql_generated.Filter{}, &pageInput, mockCve, log.NewLogger("debug", "")) - So(err, ShouldBeNil) - So(images, ShouldNotBeEmpty) - So(layers, ShouldBeEmpty) - So(repos.Results, ShouldBeEmpty) - }) }) } @@ -421,24 +95,18 @@ func TestRepoListWithNewestImage(t *testing.T) { Convey("RepoListWithNewestImage", t, func() { Convey("MetaDB SearchRepos error", func() { mockMetaDB := mocks.MetaDBMock{ - SearchReposFn: func(ctx context.Context, searchText string, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error, - ) { - return make([]mTypes.RepoMetadata, 0), make(map[string]mTypes.ManifestMetadata), - map[string]mTypes.IndexData{}, ErrTestError + SearchReposFn: func(ctx context.Context, searchText string) ([]mTypes.RepoMeta, error) { + return []mTypes.RepoMeta{}, ErrTestError }, } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) mockCve := mocks.CveInfoMock{} - limit := 1 - offset := 0 - sortCriteria := gql_generated.SortCriteriaUpdateTime pageInput := gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, + Limit: ref(1), + Offset: ref(0), + SortBy: ref(gql_generated.SortCriteriaUpdateTime), } repos, err := repoListWithNewestImage(responseContext, mockCve, log.NewLogger("debug", ""), &pageInput, mockMetaDB) So(err, ShouldNotBeNil) @@ -458,164 +126,90 @@ func TestRepoListWithNewestImage(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("MetaDB SearchRepo bad manifest referenced", func() { - mockMetaDB := mocks.MetaDBMock{ - SearchReposFn: func(ctx context.Context, searchText string, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - repos := []mTypes.RepoMetadata{ - { - Name: "repo1", - Tags: map[string]mTypes.Descriptor{ - "1.0.1": { - Digest: "digestTag1.0.1", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - Signatures: map[string]mTypes.ManifestSignatures{ - "digestTag1.0.1": { - "cosign": []mTypes.SignatureInfo{ - {SignatureManifestDigest: "testSignature", LayersInfo: []mTypes.LayerInfo{}}, - }, - }, - }, - Stars: 100, - }, - { - Name: "repo2", - Tags: map[string]mTypes.Descriptor{ - "1.0.2": { - Digest: "digestTag1.0.2", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - Signatures: map[string]mTypes.ManifestSignatures{ - "digestTag1.0.1": { - "cosign": []mTypes.SignatureInfo{ - {SignatureManifestDigest: "testSignature", LayersInfo: []mTypes.LayerInfo{}}, - }, - }, - }, - Stars: 100, - }, - } - - configBlob1, err := json.Marshal(ispec.Image{ - Config: ispec.ImageConfig{ - Labels: map[string]string{}, - }, - }) - So(err, ShouldBeNil) - - manifestsMeta := map[string]mTypes.ManifestMetadata{ - "digestTag1.0.1": { - ManifestBlob: []byte("bad manifest blob"), - ConfigBlob: configBlob1, - }, - "digestTag1.0.2": { - ManifestBlob: []byte("bad manifest blob"), - ConfigBlob: configBlob1, - }, - } - - return repos, manifestsMeta, map[string]mTypes.IndexData{}, nil - }, - } - - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, - graphql.DefaultRecover) - mockCve := mocks.CveInfoMock{} - - limit := 1 - offset := 0 - sortCriteria := gql_generated.SortCriteriaUpdateTime - pageInput := gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, - } - repos, err := repoListWithNewestImage(responseContext, mockCve, log.NewLogger("debug", ""), &pageInput, mockMetaDB) - So(err, ShouldBeNil) - So(repos.Results, ShouldNotBeEmpty) - }) - Convey("Working SearchRepo function", func() { createTime := time.Now() createTime2 := createTime.Add(time.Second) + img1 := CreateImageWith().DefaultLayers(). + ImageConfig(ispec.Image{ + Config: ispec.ImageConfig{ + Labels: map[string]string{}, + }, + Created: &createTime, + }).Build() + + img2 := CreateImageWith().DefaultLayers(). + ImageConfig(ispec.Image{ + Config: ispec.ImageConfig{ + Labels: map[string]string{}, + }, + Created: &createTime2, + }).Build() + mockMetaDB := mocks.MetaDBMock{ - SearchReposFn: func(ctx context.Context, searchText string, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - repos := []mTypes.RepoMetadata{ + SearchReposFn: func(ctx context.Context, searchText string) ([]mTypes.RepoMeta, error) { + repos := []mTypes.RepoMeta{ { Name: "repo1", Tags: map[string]mTypes.Descriptor{ "1.0.1": { - Digest: "digestTag1.0.1", + Digest: img1.DigestStr(), MediaType: ispec.MediaTypeImageManifest, }, }, Signatures: map[string]mTypes.ManifestSignatures{ - "digestTag1.0.1": { + img1.DigestStr(): { "cosign": []mTypes.SignatureInfo{ {SignatureManifestDigest: "testSignature", LayersInfo: []mTypes.LayerInfo{}}, }, }, }, - Stars: 100, + StarCount: 100, + LastUpdatedImage: &mTypes.LastUpdatedImage{ + Descriptor: mTypes.Descriptor{ + Digest: img1.DigestStr(), + MediaType: ispec.MediaTypeImageManifest, + }, + Tag: "1.0.1", + LastUpdated: &createTime, + }, }, { Name: "repo2", Tags: map[string]mTypes.Descriptor{ "1.0.2": { - Digest: "digestTag1.0.2", + Digest: img2.DigestStr(), MediaType: ispec.MediaTypeImageManifest, }, }, Signatures: map[string]mTypes.ManifestSignatures{ - "digestTag1.0.1": { + img1.DigestStr(): { "cosign": []mTypes.SignatureInfo{ {SignatureManifestDigest: "testSignature", LayersInfo: []mTypes.LayerInfo{}}, }, }, }, - Stars: 100, + StarCount: 100, + LastUpdatedImage: &mTypes.LastUpdatedImage{ + Descriptor: mTypes.Descriptor{ + Digest: img2.DigestStr(), + MediaType: ispec.MediaTypeImageManifest, + }, + Tag: "1.0.2", + LastUpdated: &createTime2, + }, }, } - configBlob1, err := json.Marshal(ispec.Image{ - Config: ispec.ImageConfig{ - Labels: map[string]string{}, - }, - Created: &createTime, - }) - So(err, ShouldBeNil) - - configBlob2, err := json.Marshal(ispec.Image{ - Config: ispec.ImageConfig{ - Labels: map[string]string{}, - }, - Created: &createTime2, - }) - So(err, ShouldBeNil) - - manifestBlob, err := json.Marshal(ispec.Manifest{}) - So(err, ShouldBeNil) - - manifestsMeta := map[string]mTypes.ManifestMetadata{ - "digestTag1.0.1": { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob1, - }, - "digestTag1.0.2": { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob2, - }, - } - - return repos, manifestsMeta, map[string]mTypes.IndexData{}, nil + return repos, nil + }, + FilterImageMetaFn: func(ctx context.Context, digests []string, + ) (map[string]mTypes.ImageMeta, error) { + return map[string]mTypes.ImageMeta{ + img1.DigestStr(): mConvert.GetImageManifestMeta(img1.Manifest, img1.Config, + img1.ManifestDescriptor.Size, img1.ManifestDescriptor.Digest), + img2.DigestStr(): mConvert.GetImageManifestMeta(img2.Manifest, img2.Config, + img2.ManifestDescriptor.Size, img2.ManifestDescriptor.Digest), + }, nil }, } Convey("MetaDB missing requestedPage", func() { @@ -628,13 +222,10 @@ func TestRepoListWithNewestImage(t *testing.T) { }) Convey("MetaDB SearchRepo is successful", func() { - limit := 2 - offset := 0 - sortCriteria := gql_generated.SortCriteriaUpdateTime pageInput := gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, + Limit: ref(2), + Offset: ref(0), + SortBy: ref(gql_generated.SortCriteriaUpdateTime), } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, @@ -691,59 +282,42 @@ func TestGetStarredRepos(t *testing.T) { }) } -func TestGetFilteredPaginatedRepos(t *testing.T) { - Convey("getFilteredPaginatedRepos FilterRepos fails", t, func() { - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, - graphql.DefaultRecover) - _, err := getFilteredPaginatedRepos( - responseContext, - mocks.CveInfoMock{}, - func(repoMeta mTypes.RepoMetadata) bool { return true }, - log.NewLogger("debug", ""), - nil, - mocks.MetaDBMock{ - FilterReposFn: func(ctx context.Context, filter mTypes.FilterRepoFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - ErrTestError - }, - }, - ) - So(err, ShouldNotBeNil) - }) +func getTestRepoMetaWithImages(repo string, images []Image) mTypes.RepoMeta { + tags := map[string]mTypes.Descriptor{"": {}} + statistics := map[string]mTypes.DescriptorStatistics{"": {}} + signatures := map[string]mTypes.ManifestSignatures{"": {}} + referrers := map[string][]mTypes.ReferrerInfo{"": {}} - Convey("Paginated convert fails", t, func() { - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, - graphql.DefaultRecover) - _, err := getFilteredPaginatedRepos(responseContext, - mocks.CveInfoMock{}, - func(repoMeta mTypes.RepoMetadata) bool { return true }, - log.NewLogger("debug", ""), - &gql_generated.PageInput{Limit: ref(-1)}, - mocks.MetaDBMock{}, - ) - So(err, ShouldNotBeNil) - }) + for i := range images { + tags[images[i].DigestStr()] = mTypes.Descriptor{} + statistics[images[i].DigestStr()] = mTypes.DescriptorStatistics{} + signatures[images[i].DigestStr()] = mTypes.ManifestSignatures{} + referrers[images[i].DigestStr()] = []mTypes.ReferrerInfo{} + } + + return mTypes.RepoMeta{ + Name: repo, + Tags: tags, + Statistics: statistics, + Signatures: signatures, + Referrers: referrers, + } } func TestImageListForDigest(t *testing.T) { Convey("getImageList", t, func() { Convey("no page requested, FilterTagsFn returns error", func() { - mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - ErrTestError + mockMetaDB := mocks.MetaDBMock{ + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, + filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + return []mTypes.FullImageMeta{}, ErrTestError }, } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) - _, err := getImageListForDigest(responseContext, "invalid", mockSearchDB, mocks.CveInfoMock{}, nil) + _, err := getImageListForDigest(responseContext, "invalid", mockMetaDB, mocks.CveInfoMock{}, nil) So(err, ShouldNotBeNil) }) @@ -755,667 +329,118 @@ func TestImageListForDigest(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("invalid manifest blob", func() { - mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - repos := []mTypes.RepoMetadata{ - { - Name: "test", - Tags: map[string]mTypes.Descriptor{ - "1.0.1": {Digest: "digestTag1.0.1", MediaType: ispec.MediaTypeImageManifest}, - }, - Stars: 100, - }, - } - - configBlob, err := json.Marshal(ispec.Image{ - Config: ispec.ImageConfig{ - Labels: map[string]string{}, - }, - }) - So(err, ShouldBeNil) - manifestBlob := []byte("invalid") - - manifestsMetaData := map[string]mTypes.ManifestMetadata{ - "digestTag1.0.1": { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - DownloadCount: 0, - }, - } - - return repos, manifestsMetaData, map[string]mTypes.IndexData{}, nil - }, - } - - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, - graphql.DefaultRecover) - - imageList, err := getImageListForDigest(responseContext, "test", mockSearchDB, mocks.CveInfoMock{}, nil) - So(err, ShouldBeNil) - So(imageList.Results, ShouldBeEmpty) - }) - Convey("valid imageListForDigest returned for matching manifest digest", func() { - manifestBlob, err := json.Marshal(ispec.Manifest{}) - So(err, ShouldBeNil) + img1, img2 := CreateRandomImage(), CreateRandomImage() + mockMetaDB := mocks.MetaDBMock{ + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, + filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + fullImageMetaList := []mTypes.ImageMeta{img1.AsImageMeta(), img2.AsImageMeta()} + repoMeta := getTestRepoMetaWithImages("repo", []Image{img1, img2}) + tags := []string{"tag1", "tag2"} - manifestDigest := godigest.FromBytes(manifestBlob).String() + acceptedImages := []mTypes.FullImageMeta{} - mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - repos := []mTypes.RepoMetadata{ - { - Name: "test", - Tags: map[string]mTypes.Descriptor{ - "1.0.1": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest}, - }, - Stars: 100, - }, - } - - configBlob, err := json.Marshal(ispec.ImageConfig{}) - So(err, ShouldBeNil) - - manifestsMetaData := map[string]mTypes.ManifestMetadata{ - manifestDigest: { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - DownloadCount: 0, - }, - } - matchedTags := repos[0].Tags - for tag, manifestDescriptor := range repos[0].Tags { - if !filterFunc(repos[0], manifestsMetaData[manifestDescriptor.Digest]) { - delete(matchedTags, tag) - delete(manifestsMetaData, manifestDescriptor.Digest) + for i := range fullImageMetaList { + if filterFunc(repoMeta, fullImageMetaList[i]) { + acceptedImages = append(acceptedImages, + convert.GetFullImageMeta(tags[i], repoMeta, fullImageMetaList[i])) continue } } - repos[0].Tags = matchedTags - - return repos, manifestsMetaData, map[string]mTypes.IndexData{}, nil + return acceptedImages, nil }, } - limit := 1 - offset := 0 - sortCriteria := gql_generated.SortCriteriaAlphabeticAsc pageInput := gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, + Limit: ref(1), + Offset: ref(0), + SortBy: ref(gql_generated.SortCriteriaAlphabeticAsc), } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) - imageSummaries, err := getImageListForDigest(responseContext, manifestDigest, - mockSearchDB, mocks.CveInfoMock{}, &pageInput) + imageSummaries, err := getImageListForDigest(responseContext, img1.DigestStr(), + mockMetaDB, mocks.CveInfoMock{}, &pageInput) So(err, ShouldBeNil) So(len(imageSummaries.Results), ShouldEqual, 1) imageSummaries, err = getImageListForDigest(responseContext, "invalid", - mockSearchDB, mocks.CveInfoMock{}, &pageInput) + mockMetaDB, mocks.CveInfoMock{}, &pageInput) So(err, ShouldBeNil) So(len(imageSummaries.Results), ShouldEqual, 0) - }) - Convey("valid imageListForDigest returned for matching config digest", func() { - manifestBlob, err := json.Marshal(ispec.Manifest{}) - So(err, ShouldBeNil) - - manifestDigest := godigest.FromBytes(manifestBlob).String() - - configBlob, err := json.Marshal(ispec.Image{}) - So(err, ShouldBeNil) - - configDigest := godigest.FromBytes(configBlob) - - mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - repos := []mTypes.RepoMetadata{ - { - Name: "test", - Tags: map[string]mTypes.Descriptor{ - "1.0.1": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest}, - }, - Stars: 100, - }, - } - - manifestBlob, err := json.Marshal(ispec.Manifest{ - Config: ispec.Descriptor{ - Digest: configDigest, - }, - }) - So(err, ShouldBeNil) - - manifestsMetaData := map[string]mTypes.ManifestMetadata{ - manifestDigest: { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - DownloadCount: 0, - }, - } - - matchedTags := repos[0].Tags - for tag, manifestDescriptor := range repos[0].Tags { - if !filterFunc(repos[0], manifestsMetaData[manifestDescriptor.Digest]) { - delete(matchedTags, tag) - delete(manifestsMetaData, manifestDescriptor.Digest) - - continue - } - } - - repos[0].Tags = matchedTags - - return repos, manifestsMetaData, map[string]mTypes.IndexData{}, nil - }, - } - - limit := 1 - offset := 0 - sortCriteria := gql_generated.SortCriteriaAlphabeticAsc - pageInput := gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, - } - - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, - graphql.DefaultRecover) - - imageSummaries, err := getImageListForDigest(responseContext, configDigest.String(), - mockSearchDB, mocks.CveInfoMock{}, &pageInput) + imageSummaries, err = getImageListForDigest(responseContext, img1.Manifest.Config.Digest.String(), + mockMetaDB, mocks.CveInfoMock{}, &pageInput) So(err, ShouldBeNil) So(len(imageSummaries.Results), ShouldEqual, 1) - }) - Convey("valid imageListForDigest returned for matching layer digest", func() { - manifestBlob, err := json.Marshal(ispec.Manifest{}) - So(err, ShouldBeNil) - - manifestDigest := godigest.FromBytes(manifestBlob).String() - - configBlob, err := json.Marshal(ispec.Image{}) - So(err, ShouldBeNil) - - layerDigest := godigest.Digest("validDigest") - - mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - repos := []mTypes.RepoMetadata{ - { - Name: "test", - Tags: map[string]mTypes.Descriptor{ - "1.0.1": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest}, - }, - Stars: 100, - }, - } - - manifestBlob, err := json.Marshal(ispec.Manifest{ - Layers: []ispec.Descriptor{ - { - Digest: layerDigest, - }, - }, - }) - So(err, ShouldBeNil) - - manifestsMetaData := map[string]mTypes.ManifestMetadata{ - manifestDigest: { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - DownloadCount: 0, - }, - } - - matchedTags := repos[0].Tags - for tag, manifestDescriptor := range repos[0].Tags { - if !filterFunc(repos[0], manifestsMetaData[manifestDescriptor.Digest]) { - delete(matchedTags, tag) - delete(manifestsMetaData, manifestDescriptor.Digest) - - continue - } - } - - repos[0].Tags = matchedTags - - return repos, manifestsMetaData, map[string]mTypes.IndexData{}, nil - }, - } - - limit := 1 - offset := 0 - sortCriteria := gql_generated.SortCriteriaAlphabeticAsc - pageInput := gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, - } - - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, - graphql.DefaultRecover) - - imageSummaries, err := getImageListForDigest(responseContext, layerDigest.String(), - mockSearchDB, mocks.CveInfoMock{}, &pageInput) + imageSummaries, err = getImageListForDigest(responseContext, img1.Manifest.Layers[0].Digest.String(), + mockMetaDB, mocks.CveInfoMock{}, &pageInput) So(err, ShouldBeNil) So(len(imageSummaries.Results), ShouldEqual, 1) }) Convey("valid imageListForDigest, multiple matching tags", func() { - manifestBlob, err := json.Marshal(ispec.Manifest{}) - So(err, ShouldBeNil) + img1 := CreateRandomImage() - manifestDigest := godigest.FromBytes(manifestBlob).String() + mockMetaDB := mocks.MetaDBMock{ + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, + filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + fullImageMetaList := []mTypes.ImageMeta{img1.AsImageMeta()} + repoMeta := getTestRepoMetaWithImages("repo", []Image{img1, img1}) + tags := []string{"tag1", "tag2"} - configBlob, err := json.Marshal(ispec.Image{}) - So(err, ShouldBeNil) + acceptedImages := []mTypes.FullImageMeta{} - mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - repos := []mTypes.RepoMetadata{ - { - Name: "test", - Tags: map[string]mTypes.Descriptor{ - "1.0.1": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest}, - "1.0.2": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest}, - }, - Stars: 100, - }, - } + for i := range fullImageMetaList { + if filterFunc(repoMeta, fullImageMetaList[i]) { + acceptedImages = append(acceptedImages, + convert.GetFullImageMeta(tags[i], repoMeta, fullImageMetaList[i])) - manifestsMetaData := map[string]mTypes.ManifestMetadata{ - manifestDigest: { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - DownloadCount: 0, - }, - } - - for i, repo := range repos { - matchedTags := repo.Tags - - for tag, manifestDescriptor := range repo.Tags { - if !filterFunc(repo, manifestsMetaData[manifestDescriptor.Digest]) { - delete(matchedTags, tag) - delete(manifestsMetaData, manifestDescriptor.Digest) - - continue - } + continue } - - repos[i].Tags = matchedTags } - return repos, manifestsMetaData, map[string]mTypes.IndexData{}, nil + return acceptedImages, nil }, } - limit := 1 - offset := 0 - sortCriteria := gql_generated.SortCriteriaAlphabeticAsc pageInput := gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, + Limit: ref(1), + Offset: ref(0), + SortBy: ref(gql_generated.SortCriteriaAlphabeticAsc), } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) - imageSummaries, err := getImageListForDigest(responseContext, manifestDigest, - mockSearchDB, mocks.CveInfoMock{}, &pageInput) - So(err, ShouldBeNil) - So(len(imageSummaries.Results), ShouldEqual, 1) - }) - - Convey("valid imageListForDigest, multiple matching tags limited by pageInput", func() { - manifestBlob, err := json.Marshal(ispec.Manifest{}) - So(err, ShouldBeNil) - - manifestDigest := godigest.FromBytes(manifestBlob).String() - - configBlob, err := json.Marshal(ispec.Image{}) - So(err, ShouldBeNil) - - mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error, - ) { - repos := []mTypes.RepoMetadata{ - { - Name: "test", - Tags: map[string]mTypes.Descriptor{ - "1.0.1": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest}, - "1.0.2": {Digest: manifestDigest, MediaType: ispec.MediaTypeImageManifest}, - }, - Stars: 100, - }, - } - - manifestsMetaData := map[string]mTypes.ManifestMetadata{ - manifestDigest: { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - DownloadCount: 0, - }, - } - - for i, repo := range repos { - matchedTags := repo.Tags - - for tag, manifestDescriptor := range repo.Tags { - if !filterFunc(repo, manifestsMetaData[manifestDescriptor.Digest]) { - delete(matchedTags, tag) - delete(manifestsMetaData, manifestDescriptor.Digest) - - continue - } - } - - repos[i].Tags = matchedTags - - repos = append(repos, repo) - } - - return repos, manifestsMetaData, map[string]mTypes.IndexData{}, nil - }, - } - - limit := 1 - offset := 0 - sortCriteria := gql_generated.SortCriteriaAlphabeticAsc - pageInput := gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, - } - - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, - graphql.DefaultRecover) - - imageSummaries, err := getImageListForDigest(responseContext, manifestDigest, - mockSearchDB, mocks.CveInfoMock{}, &pageInput) + imageSummaries, err := getImageListForDigest(responseContext, img1.DigestStr(), + mockMetaDB, mocks.CveInfoMock{}, &pageInput) So(err, ShouldBeNil) So(len(imageSummaries.Results), ShouldEqual, 1) }) }) } -func TestGetImageSummary(t *testing.T) { - Convey("GetImageSummary", t, func() { - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, - graphql.DefaultRecover) - - Convey("Media Type: ImageManifest", func() { - Convey("metaDB.GetManifestMeta fails", func() { - var ( - metaDB = mocks.MetaDBMock{ - GetManifestDataFn: func(manifestDigest godigest.Digest) (mTypes.ManifestData, error) { - return mTypes.ManifestData{}, ErrTestError - }, - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tag": {MediaType: ispec.MediaTypeImageManifest, Digest: "digest"}, - }, - }, nil - }, - } - - log = log.NewLogger("debug", "") - - skip = convert.SkipQGLField{ - Vulnerabilities: true, - } - ) - - _, err := getImageSummary(responseContext, "repo", "tag", nil, skip, metaDB, mocks.CveInfoMock{}, log) - So(err, ShouldNotBeNil) - }) - - Convey("0 len return", func() { - var ( - metaDB = mocks.MetaDBMock{ - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tag": {MediaType: ispec.MediaTypeImageManifest, Digest: "digest"}, - }, - }, nil - }, - } - - log = log.NewLogger("debug", "") - - skip = convert.SkipQGLField{ - Vulnerabilities: true, - } - ) - - _, err := getImageSummary(responseContext, "repo", "tag", nil, skip, metaDB, mocks.CveInfoMock{}, log) - So(err, ShouldBeNil) - }) - - Convey("digest != nil && *digest != actual image digest", func() { - var ( - metaDB = mocks.MetaDBMock{ - GetManifestMetaFn: func(repo string, manifestDigest godigest.Digest) (mTypes.ManifestMetadata, error) { - return mTypes.ManifestMetadata{}, ErrTestError - }, - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tag": {MediaType: ispec.MediaTypeImageManifest, Digest: "digest"}, - }, - }, nil - }, - } - - log = log.NewLogger("debug", "") - - digest = "wrongDigest" - - skip = convert.SkipQGLField{ - Vulnerabilities: true, - } - ) - - _, err := getImageSummary(responseContext, "repo", "tag", &digest, skip, metaDB, mocks.CveInfoMock{}, log) - So(err, ShouldNotBeNil) - }) - }) - - Convey("Media Type: ImageIndex", func() { - Convey("metaDB.GetIndexData fails", func() { - var ( - metaDB = mocks.MetaDBMock{ - GetIndexDataFn: func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{}, ErrTestError - }, - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tag": {MediaType: ispec.MediaTypeImageIndex, Digest: "digest"}, - }, - }, nil - }, - } - - log = log.NewLogger("debug", "") - - skip = convert.SkipQGLField{ - Vulnerabilities: true, - } - ) - - _, err := getImageSummary(responseContext, "repo", "tag", nil, skip, metaDB, mocks.CveInfoMock{}, log) - So(err, ShouldNotBeNil) - }) - - Convey("json.Unmarshal(indexData.IndexBlob, &indexContent) fails", func() { - var ( - metaDB = mocks.MetaDBMock{ - GetIndexDataFn: func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{ - IndexBlob: []byte("bad json"), - }, nil - }, - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tag": {MediaType: ispec.MediaTypeImageIndex, Digest: "digest"}, - }, - }, nil - }, - } - - log = log.NewLogger("debug", "") - - skip = convert.SkipQGLField{ - Vulnerabilities: true, - } - ) - - _, err := getImageSummary(responseContext, "repo", "tag", nil, skip, metaDB, mocks.CveInfoMock{}, log) - So(err, ShouldNotBeNil) - }) - - Convey("digest != nil", func() { - index := ispec.Index{ - Manifests: []ispec.Descriptor{ - { - Digest: "digest", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - } - - indexBlob, err := json.Marshal(index) - So(err, ShouldBeNil) - - metaDB := mocks.MetaDBMock{ - GetIndexDataFn: func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{ - IndexBlob: indexBlob, - }, nil - }, - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tag": {MediaType: ispec.MediaTypeImageIndex, Digest: "digest"}, - }, - }, nil - }, - } - - log := log.NewLogger("debug", "") - - goodDigest := "goodDigest" - - Convey("digest not found", func() { - wrongDigest := "wrongDigest" - - skip := convert.SkipQGLField{ - Vulnerabilities: true, - } - - _, err = getImageSummary(responseContext, "repo", "tag", &wrongDigest, skip, metaDB, mocks.CveInfoMock{}, log) - So(err, ShouldNotBeNil) - }) - - Convey("GetManifestData error", func() { - metaDB.GetManifestDataFn = func(manifestDigest godigest.Digest) (mTypes.ManifestData, error) { - return mTypes.ManifestData{}, ErrTestError - } - - skip := convert.SkipQGLField{ - Vulnerabilities: true, - } - - _, err = getImageSummary(responseContext, "repo", "tag", &goodDigest, skip, metaDB, mocks.CveInfoMock{}, log) - So(err, ShouldNotBeNil) - }) - }) - }) - - Convey("Media Type: not supported", func() { - var ( - metaDB = mocks.MetaDBMock{ - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tag": {MediaType: "unknown", Digest: "digest"}, - }, - }, nil - }, - } - - log = log.NewLogger("debug", "") - - skip = convert.SkipQGLField{ - Vulnerabilities: true, - } - ) - - _, err := getImageSummary(responseContext, "repo", "tag", nil, skip, metaDB, mocks.CveInfoMock{}, log) - So(err, ShouldBeNil) - }) - }) -} - -func TestFilterBaseImagesFn(t *testing.T) { - Convey("FilterBaseImages", t, func() { - filterFunc := filterBaseImages(&gql_generated.ImageSummary{}) - ok := filterFunc( - mTypes.RepoMetadata{}, - mTypes.ManifestMetadata{ - ManifestBlob: []byte("bad json"), - }, - ) - So(ok, ShouldBeFalse) - }) -} - -func TestImageList(t *testing.T) { +func TestImageListError(t *testing.T) { Convey("getImageList", t, func() { - testLogger := log.NewLogger("debug", "") + testLogger := log.NewLogger("debug", "/dev/null") Convey("no page requested, SearchRepoFn returns error", func() { - mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, - map[string]mTypes.IndexData{}, ErrTestError + mockMetaDB := mocks.MetaDBMock{ + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + return []mTypes.FullImageMeta{}, ErrTestError }, } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) - _, err := getImageList(responseContext, "test", mockSearchDB, mocks.CveInfoMock{}, nil, testLogger) + _, err := getImageList(responseContext, "test", mockMetaDB, mocks.CveInfoMock{}, nil, testLogger) So(err, ShouldNotBeNil) }) @@ -1430,81 +455,43 @@ func TestImageList(t *testing.T) { }) Convey("valid repoList returned", func() { - mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - repos := []mTypes.RepoMetadata{ - { - Name: "test", - Tags: map[string]mTypes.Descriptor{ - "1.0.1": { - Digest: "digestTag1.0.1", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - Signatures: map[string]mTypes.ManifestSignatures{ - "digestTag1.0.1": { - "cosign": []mTypes.SignatureInfo{ - {SignatureManifestDigest: "testSignature", LayersInfo: []mTypes.LayerInfo{}}, - }, - }, - }, - Stars: 100, - }, + mockMetaDB := mocks.MetaDBMock{ + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + repoName := "correct-repo" + + if !filterRepoTag(repoName, "tag") { + return []mTypes.FullImageMeta{}, nil } - configBlob, err := json.Marshal(ispec.Image{ - Config: ispec.ImageConfig{ - Labels: map[string]string{}, - }, - }) - So(err, ShouldBeNil) - - manifestBlob, err := json.Marshal(ispec.Manifest{}) - So(err, ShouldBeNil) - - manifestsMetaData := map[string]mTypes.ManifestMetadata{ - "digestTag1.0.1": { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - DownloadCount: 0, - Signatures: mTypes.ManifestSignatures{ - "cosign": []mTypes.SignatureInfo{ - {SignatureManifestDigest: "digestSignature1"}, - }, - }, - }, + image := CreateDefaultImage() + repoMeta := mTypes.RepoMeta{ + Name: "repo", + Tags: map[string]mTypes.Descriptor{image.DigestStr(): { + Digest: image.DigestStr(), + MediaType: ispec.MediaTypeImageManifest, + }}, } - if !filterFunc(repos[0], manifestsMetaData["digestTag1.0.1"]) { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, - map[string]mTypes.IndexData{}, nil - } - - return repos, manifestsMetaData, map[string]mTypes.IndexData{}, nil + return []mTypes.FullImageMeta{convert.GetFullImageMeta("tag", repoMeta, image.AsImageMeta())}, nil }, } - limit := 1 - offset := 0 - sortCriteria := gql_generated.SortCriteriaAlphabeticAsc pageInput := gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, + Limit: ref(1), + Offset: ref(0), + SortBy: ref(gql_generated.SortCriteriaAlphabeticAsc), } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) - imageSummaries, err := getImageList(responseContext, "test", mockSearchDB, + imageSummaries, err := getImageList(responseContext, "correct-repo", mockMetaDB, mocks.CveInfoMock{}, &pageInput, testLogger) So(err, ShouldBeNil) So(len(imageSummaries.Results), ShouldEqual, 1) - imageSummaries, err = getImageList(responseContext, "invalid", mockSearchDB, + imageSummaries, err = getImageList(responseContext, "invalid", mockMetaDB, mocks.CveInfoMock{}, &pageInput, testLogger) So(err, ShouldBeNil) So(len(imageSummaries.Results), ShouldEqual, 0) @@ -1580,35 +567,21 @@ func TestQueryResolverErrors(t *testing.T) { graphql.DefaultRecover) Convey("GlobalSearch error bad requested page", func() { - resolverConfig := NewResolver( - log, - storage.StoreController{}, - mocks.MetaDBMock{}, - mocks.CveInfoMock{}, - ) - - resolver := queryResolver{ - resolverConfig, - } - - limit := -1 - offset := 0 - sortCriteria := gql_generated.SortCriteriaAlphabeticAsc + resolverConfig := NewResolver(log, storage.StoreController{}, mocks.MetaDBMock{}, mocks.CveInfoMock{}) + resolver := queryResolver{resolverConfig} pageInput := gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, + Limit: ref(-1), + Offset: ref(0), + SortBy: ref(gql_generated.SortCriteriaAlphabeticAsc), } _, err := resolver.GlobalSearch(ctx, "some_string", &gql_generated.Filter{}, &pageInput) So(err, ShouldNotBeNil) - limit = 0 - offset = -1 pageInput = gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, + Limit: ref(0), + Offset: ref(-1), + SortBy: ref(gql_generated.SortCriteriaAlphabeticAsc), } _, err = resolver.GlobalSearch(ctx, "some_string", &gql_generated.Filter{}, &pageInput) @@ -1622,17 +595,15 @@ func TestQueryResolverErrors(t *testing.T) { DefaultStore: mocks.MockedImageStore{}, }, mocks.MetaDBMock{ - GetMultipleRepoMetaFn: func(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool, - ) ([]mTypes.RepoMetadata, error) { - return []mTypes.RepoMetadata{}, ErrTestError + GetMultipleRepoMetaFn: func(ctx context.Context, filter func(repoMeta mTypes.RepoMeta) bool, + ) ([]mTypes.RepoMeta, error) { + return []mTypes.RepoMeta{}, ErrTestError }, }, mocks.CveInfoMock{}, ) - qr := queryResolver{ - resolverConfig, - } + qr := queryResolver{resolverConfig} _, err := qr.ImageListForCve(ctx, "cve1", &gql_generated.Filter{}, &gql_generated.PageInput{}) So(err, ShouldNotBeNil) @@ -1645,21 +616,15 @@ func TestQueryResolverErrors(t *testing.T) { DefaultStore: mocks.MockedImageStore{}, }, mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - ErrTestError + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + return []mTypes.FullImageMeta{}, ErrTestError }, }, mocks.CveInfoMock{}, ) - qr := queryResolver{ - resolverConfig, - } + qr := queryResolver{resolverConfig} _, err := qr.ImageListForCve(ctx, "cve1", &gql_generated.Filter{}, &gql_generated.PageInput{}) So(err, ShouldNotBeNil) @@ -1672,21 +637,15 @@ func TestQueryResolverErrors(t *testing.T) { DefaultStore: mocks.MockedImageStore{}, }, mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - ErrTestError + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + return []mTypes.FullImageMeta{}, ErrTestError }, }, mocks.CveInfoMock{}, ) - qr := queryResolver{ - resolverConfig, - } + qr := queryResolver{resolverConfig} _, err := qr.ImageListWithCVEFixed(ctx, "cve1", "image", &gql_generated.Filter{}, &gql_generated.PageInput{}) So(err, ShouldNotBeNil) @@ -1700,18 +659,14 @@ func TestQueryResolverErrors(t *testing.T) { }, mocks.MetaDBMock{ SearchReposFn: func(ctx context.Context, searchText string, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - return nil, nil, nil, ErrTestError + ) ([]mTypes.RepoMeta, error) { + return nil, ErrTestError }, }, mocks.CveInfoMock{}, ) - qr := queryResolver{ - resolverConfig, - } + qr := queryResolver{resolverConfig} _, err := qr.RepoListWithNewestImage(ctx, &gql_generated.PageInput{}) So(err, ShouldNotBeNil) @@ -1723,18 +678,14 @@ func TestQueryResolverErrors(t *testing.T) { storage.StoreController{}, mocks.MetaDBMock{ SearchReposFn: func(ctx context.Context, searchText string, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - return nil, nil, nil, ErrTestError + ) ([]mTypes.RepoMeta, error) { + return nil, ErrTestError }, }, mocks.CveInfoMock{}, ) - qr := queryResolver{ - resolverConfig, - } + qr := queryResolver{resolverConfig} _, err := qr.RepoListWithNewestImage(ctx, &gql_generated.PageInput{}) So(err, ShouldNotBeNil) @@ -1745,21 +696,15 @@ func TestQueryResolverErrors(t *testing.T) { log, storage.StoreController{}, mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - ErrTestError + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + return []mTypes.FullImageMeta{}, ErrTestError }, }, mocks.CveInfoMock{}, ) - qr := queryResolver{ - resolverConfig, - } + qr := queryResolver{resolverConfig} _, err := qr.ImageList(ctx, "repo", &gql_generated.PageInput{}) So(err, ShouldNotBeNil) @@ -1779,16 +724,14 @@ func TestQueryResolverErrors(t *testing.T) { }, }, mocks.MetaDBMock{ - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{}, ErrTestError + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{}, ErrTestError }, }, mocks.CveInfoMock{}, ) - qr := queryResolver{ - resolverConfig, - } + qr := queryResolver{resolverConfig} _, err := qr.DerivedImageList(ctx, "repo:tag", nil, &gql_generated.PageInput{}) So(err, ShouldNotBeNil) @@ -1808,71 +751,48 @@ func TestQueryResolverErrors(t *testing.T) { }, }, mocks.MetaDBMock{ - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{}, ErrTestError + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{}, ErrTestError }, }, mocks.CveInfoMock{}, ) - qr := queryResolver{ - resolverConfig, - } + qr := queryResolver{resolverConfig} _, err := qr.BaseImageList(ctx, "repo:tag", nil, &gql_generated.PageInput{}) So(err, ShouldNotBeNil) }) Convey("DerivedImageList and BaseImage List FilterTags() errors", func() { - configBlob, err := json.Marshal(ispec.Image{ - Config: ispec.ImageConfig{ - Labels: map[string]string{}, - }, - }) - So(err, ShouldBeNil) - - manifest := ispec.Manifest{} - - manifestBlob, err := json.Marshal(manifest) - So(err, ShouldBeNil) - - manifestDigest := godigest.FromBytes(manifestBlob) + image := CreateDefaultImage() resolverConfig := NewResolver( log, storage.StoreController{}, mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - ErrTestError + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + return []mTypes.FullImageMeta{}, ErrTestError }, - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{ Name: "repo", Tags: map[string]mTypes.Descriptor{ - "tag": {Digest: manifestDigest.String(), MediaType: ispec.MediaTypeImageManifest}, + "tag": {Digest: image.DigestStr(), MediaType: ispec.MediaTypeImageManifest}, }, }, nil }, - GetManifestMetaFn: func(repo string, manifestDigest godigest.Digest) (mTypes.ManifestMetadata, error) { - return mTypes.ManifestMetadata{ - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - }, nil + GetImageMetaFn: func(digest godigest.Digest) (mTypes.ImageMeta, error) { + return image.AsImageMeta(), nil }, }, mocks.CveInfoMock{}, ) - resolver := queryResolver{ - resolverConfig, - } + resolver := queryResolver{resolverConfig} - _, err = resolver.DerivedImageList(ctx, "repo:tag", nil, &gql_generated.PageInput{}) + _, err := resolver.DerivedImageList(ctx, "repo:tag", nil, &gql_generated.PageInput{}) So(err, ShouldNotBeNil) _, err = resolver.BaseImageList(ctx, "repo:tag", nil, &gql_generated.PageInput{}) @@ -1893,9 +813,7 @@ func TestQueryResolverErrors(t *testing.T) { mocks.CveInfoMock{}, ) - qr := queryResolver{ - resolverConfig, - } + qr := queryResolver{resolverConfig} _, err := qr.Referrers(ctx, "repo", "", nil) So(err, ShouldNotBeNil) @@ -1904,180 +822,78 @@ func TestQueryResolverErrors(t *testing.T) { } func TestCVEResolvers(t *testing.T) { //nolint:gocyclo - params := boltdb.DBParameters{ - RootDir: t.TempDir(), - } - + ctx := context.Background() + log := log.NewLogger("debug", "") LINUX := "linux" AMD := "amd" ARM := "arm64" - boltDriver, err := boltdb.GetBoltDriver(params) + boltDriver, err := boltdb.GetBoltDriver(boltdb.DBParameters{RootDir: t.TempDir()}) if err != nil { panic(err) } - log := log.NewLogger("debug", "") - metaDB, err := boltdb.New(boltDriver, log) if err != nil { panic(err) } - // Create metadb data for scannable image with vulnerabilities - // Create manifest metadata first - timeStamp1 := time.Date(2008, 1, 1, 12, 0, 0, 0, time.UTC) - - configBlob1, err := json.Marshal(ispec.Image{ - Created: &timeStamp1, + image1 := CreateImageWith().RandomLayers(5, 2).ImageConfig(ispec.Image{ + Created: DateRef(2008, 1, 1, 12, 0, 0, 0, time.UTC), Platform: ispec.Platform{ Architecture: AMD, OS: LINUX, }, - }) - if err != nil { - panic(err) - } + }).Build() + digest1 := image1.Digest() - manifestBlob1, err := json.Marshal(ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: ispec.MediaTypeImageConfig, - Size: 0, - Digest: godigest.FromBytes(configBlob1), - }, - Layers: []ispec.Descriptor{ - { - MediaType: ispec.MediaTypeImageLayerGzip, - Size: 0, - Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"), - }, - }, - }) - if err != nil { - panic(err) - } - - repoMeta1 := mTypes.ManifestData{ - ManifestBlob: manifestBlob1, - ConfigBlob: configBlob1, - } - - digest1 := godigest.FromBytes(manifestBlob1) - - err = metaDB.SetManifestData(digest1, repoMeta1) - if err != nil { - panic(err) - } - - timeStamp2 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC) - - configBlob2, err := json.Marshal(ispec.Image{ - Created: &timeStamp2, + image2 := CreateImageWith().RandomLayers(5, 2).ImageConfig(ispec.Image{ + Created: DateRef(2009, 1, 1, 12, 0, 0, 0, time.UTC), Platform: ispec.Platform{ Architecture: AMD, OS: LINUX, }, - }) - if err != nil { - panic(err) - } + }).Build() + digest2 := image2.Digest() - manifestBlob2, err := json.Marshal(ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: ispec.MediaTypeImageConfig, - Size: 0, - Digest: godigest.FromBytes(configBlob2), - }, - Layers: []ispec.Descriptor{ - { - MediaType: ispec.MediaTypeImageLayerGzip, - Size: 0, - Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"), - }, - }, - }) - if err != nil { - panic(err) - } - - repoMeta2 := mTypes.ManifestData{ - ManifestBlob: manifestBlob2, - ConfigBlob: configBlob2, - } - - digest2 := godigest.FromBytes(manifestBlob2) - - err = metaDB.SetManifestData(digest2, repoMeta2) - if err != nil { - panic(err) - } - - timeStamp3 := time.Date(2010, 1, 1, 12, 0, 0, 0, time.UTC) - - configBlob3, err := json.Marshal(ispec.Image{ - Created: &timeStamp3, + image3 := CreateImageWith().RandomLayers(5, 2).ImageConfig(ispec.Image{ + Created: DateRef(2010, 1, 1, 12, 0, 0, 0, time.UTC), Platform: ispec.Platform{ Architecture: ARM, OS: LINUX, }, - }) - if err != nil { - panic(err) - } + }).Build() + digest3 := image3.Digest() - manifestBlob3, err := json.Marshal(ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: ispec.MediaTypeImageConfig, - Size: 0, - Digest: godigest.FromBytes(configBlob3), - }, - Layers: []ispec.Descriptor{ - { - MediaType: ispec.MediaTypeImageLayerGzip, - Size: 0, - Digest: godigest.NewDigestFromEncoded(godigest.SHA256, "digest"), + ctx, err = ociutils.InitializeTestMetaDB(ctx, metaDB, + ociutils.Repo{ + Name: "repo1", Images: []ociutils.RepoImage{ + {Image: image1, Reference: "1.0.0"}, + {Image: image2, Reference: "1.0.1"}, + {Image: image3, Reference: "1.1.0"}, + {Image: image3, Reference: "latest"}, }, }, - }) + ociutils.Repo{ + Name: "repo2", Images: []ociutils.RepoImage{ + {Image: image1, Reference: "2.0.0"}, + {Image: image2, Reference: "2.0.1"}, + {Image: image3, Reference: "2.1.0"}, + {Image: image3, Reference: "latest"}, + }, + }, + ociutils.Repo{ + Name: "repo3", Images: []ociutils.RepoImage{ + {Image: image2, Reference: "3.0.1"}, + {Image: image3, Reference: "3.1.0"}, + {Image: image3, Reference: "latest"}, + }, + }, + ) if err != nil { panic(err) } - repoMeta3 := mTypes.ManifestData{ - ManifestBlob: manifestBlob3, - ConfigBlob: configBlob3, - } - - digest3 := godigest.FromBytes(manifestBlob3) - - err = metaDB.SetManifestData(digest3, repoMeta3) - if err != nil { - panic(err) - } - - // Create the repo metadata using previously defined manifests - tagsMap := map[string]godigest.Digest{} - tagsMap["repo1:1.0.0"] = digest1 - tagsMap["repo1:1.0.1"] = digest2 - tagsMap["repo1:1.1.0"] = digest3 - tagsMap["repo1:latest"] = digest3 - tagsMap["repo2:2.0.0"] = digest1 - tagsMap["repo2:2.0.1"] = digest2 - tagsMap["repo2:2.1.0"] = digest3 - tagsMap["repo2:latest"] = digest3 - tagsMap["repo3:3.0.1"] = digest2 - tagsMap["repo3:3.1.0"] = digest3 - tagsMap["repo3:latest"] = digest3 - - for image, digest := range tagsMap { - repo, tag := common.GetImageDirAndTag(image) - - err := metaDB.SetRepoReference(repo, tag, digest, ispec.MediaTypeImageManifest) - if err != nil { - panic(err) - } - } - getCveResults := func(digestStr string) map[string]cvemodel.CVE { if digestStr == digest1.String() { return map[string]cvemodel.CVE{ @@ -2144,17 +960,19 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo // Setup test CVE data in mock scanner scanner := mocks.CveScannerMock{ ScanImageFn: func(image string) (map[string]cvemodel.CVE, error) { - digest, ok := tagsMap[image] - if !ok { - if !strings.Contains(image, "@") { - return map[string]cvemodel.CVE{}, nil - } + repo, ref, _, _ := common.GetRepoReference(image) - _, digestStr := common.GetImageDirAndDigest(image) - digest = godigest.Digest(digestStr) + if common.IsDigest(ref) { + return getCveResults(ref), nil } - return getCveResults(digest.String()), nil + repoMeta, _ := metaDB.GetRepoMeta(context.Background(), repo) + + if _, ok := repoMeta.Tags[ref]; !ok { + panic("unexpected tag '" + ref + "', test might be wrong") + } + + return getCveResults(repoMeta.Tags[ref].Digest), nil }, GetCachedResultFn: func(digestStr string) map[string]cvemodel.CVE { return getCveResults(digestStr) @@ -2172,12 +990,11 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo Convey("Get CVE list for image ", t, func() { Convey("Unpaginated request to get all CVEs in an image", func() { - sortCriteria := gql_generated.SortCriteriaAlphabeticAsc pageInput := &gql_generated.PageInput{ - SortBy: &sortCriteria, + SortBy: ref(gql_generated.SortCriteriaAlphabeticAsc), } - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + responseContext := graphql.WithResponseContext(ctx, graphql.DefaultErrorPresenter, graphql.DefaultRecover) dig := godigest.FromString("dig") @@ -2267,7 +1084,7 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo Limit: ref(-1), } - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + responseContext := graphql.WithResponseContext(ctx, graphql.DefaultErrorPresenter, graphql.DefaultRecover) _, err = getCVEListForImage(responseContext, "repo1:1.1.0", cveInfo, pageInput, "", log) @@ -2277,7 +1094,7 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo Convey("Get a list of images affected by a particular CVE ", t, func() { Convey("Unpaginated request", func() { - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + responseContext := graphql.WithResponseContext(ctx, graphql.DefaultErrorPresenter, graphql.DefaultRecover) images, err := getImageListForCVE(responseContext, "CVE1", cveInfo, nil, nil, metaDB, log) @@ -2327,7 +1144,7 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo Limit: ref(-1), } - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + responseContext := graphql.WithResponseContext(ctx, graphql.DefaultErrorPresenter, graphql.DefaultRecover) _, err = getImageListForCVE(responseContext, "repo1:1.1.0", cveInfo, &gql_generated.Filter{}, @@ -2336,7 +1153,7 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo }) Convey("Paginated requests", func() { - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + responseContext := graphql.WithResponseContext(ctx, graphql.DefaultErrorPresenter, graphql.DefaultRecover, ) @@ -2546,7 +1363,7 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo Convey("Get a list of images where a particular CVE is fixed", t, func() { Convey("Unpaginated request", func() { - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + responseContext := graphql.WithResponseContext(ctx, graphql.DefaultErrorPresenter, graphql.DefaultRecover) images, err := getImageListWithCVEFixed(responseContext, "CVE1", "repo1", cveInfo, nil, nil, metaDB, log) @@ -2583,13 +1400,13 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo Limit: ref(-1), } - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + responseContext := graphql.WithResponseContext(ctx, graphql.DefaultErrorPresenter, graphql.DefaultRecover) _, err = getImageListWithCVEFixed(responseContext, "cve", "repo1:1.1.0", cveInfo, &gql_generated.Filter{}, pageInput, mocks.MetaDBMock{ - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{ Tags: map[string]mTypes.Descriptor{ "1.1.0": { Digest: godigest.FromString("str").String(), @@ -2603,7 +1420,7 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo }) Convey("Paginated requests", func() { - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, + responseContext := graphql.WithResponseContext(ctx, graphql.DefaultErrorPresenter, graphql.DefaultRecover, ) @@ -2778,7 +1595,7 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo Convey("Errors for cve resolvers", t, func() { _, err := getImageListForCVE( - context.Background(), + ctx, "id", mocks.CveInfoMock{ GetImageListForCVEFn: func(repo, cveID string) ([]cvemodel.TagInfo, error) { @@ -2788,9 +1605,9 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo nil, nil, mocks.MetaDBMock{ - GetMultipleRepoMetaFn: func(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool, - ) ([]mTypes.RepoMetadata, error) { - return []mTypes.RepoMetadata{{}}, nil + GetMultipleRepoMetaFn: func(ctx context.Context, filter func(repoMeta mTypes.RepoMeta) bool, + ) ([]mTypes.RepoMeta, error) { + return []mTypes.RepoMeta{{}}, nil }, }, log, @@ -2799,45 +1616,36 @@ func TestCVEResolvers(t *testing.T) { //nolint:gocyclo }) } -func getPageInput(limit int, offset int) *gql_generated.PageInput { - sortCriteria := gql_generated.SortCriteriaAlphabeticAsc - - return &gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, - } -} - -func TestDerivedImageList(t *testing.T) { +func TestMockedDerivedImageList(t *testing.T) { Convey("MetaDB FilterTags error", t, func() { - mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - return make([]mTypes.RepoMetadata, 0), make(map[string]mTypes.ManifestMetadata), - make(map[string]mTypes.IndexData), ErrTestError + log := log.NewLogger("debug", "/dev/null") + + image := CreateRandomImage() + mockMetaDB := mocks.MetaDBMock{ + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + return []mTypes.FullImageMeta{}, ErrTestError }, - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{}, ErrTestError + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{}, ErrTestError }, - GetManifestMetaFn: func(repo string, manifestDigest godigest.Digest) (mTypes.ManifestMetadata, error) { - return mTypes.ManifestMetadata{}, ErrTestError + FilterImageMetaFn: func(ctx context.Context, digests []string) (map[string]mTypes.ImageMeta, error) { + return map[string]mTypes.ImageMeta{image.DigestStr(): image.AsImageMeta()}, nil }, } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) mockCve := mocks.CveInfoMock{} - images, err := derivedImageList(responseContext, "repo1:1.0.1", nil, mockSearchDB, &gql_generated.PageInput{}, - mockCve, log.NewLogger("debug", "")) + images, err := derivedImageList(responseContext, "repo1:1.0.1", nil, mockMetaDB, &gql_generated.PageInput{}, + mockCve, log) So(err, ShouldNotBeNil) So(images.Results, ShouldBeEmpty) }) Convey("paginated fail", t, func() { + log := log.NewLogger("debug", "/dev/null") + image := CreateRandomImage() pageInput := &gql_generated.PageInput{ Limit: ref(-1), } @@ -2847,223 +1655,133 @@ func TestDerivedImageList(t *testing.T) { _, err := derivedImageList(responseContext, "repo1:1.0.1", nil, mocks.MetaDBMock{ - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{ Tags: map[string]mTypes.Descriptor{ "1.0.1": { - Digest: godigest.FromString("str").String(), + Digest: image.DigestStr(), MediaType: ispec.MediaTypeImageManifest, }, }, }, nil }, + FilterImageMetaFn: func(ctx context.Context, digests []string) (map[string]mTypes.ImageMeta, error) { + return map[string]mTypes.ImageMeta{image.DigestStr(): image.AsImageMeta()}, nil + }, }, pageInput, - mocks.CveInfoMock{}, log.NewLogger("debug", "")) + mocks.CveInfoMock{}, log) So(err, ShouldNotBeNil) }) //nolint: dupl Convey("MetaDB FilterTags no repo available", t, func() { - configBlob, err := json.Marshal(ispec.Image{ - Config: ispec.ImageConfig{ - Labels: map[string]string{}, + log := log.NewLogger("debug", "/dev/null") + image := CreateDefaultImage() + + mockMetaDB := mocks.MetaDBMock{ + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + return []mTypes.FullImageMeta{}, nil }, - }) - So(err, ShouldBeNil) - - manifest := ispec.Manifest{} - - manifestBlob, err := json.Marshal(manifest) - So(err, ShouldBeNil) - - manifestDigest := godigest.FromBytes(manifestBlob) - - mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - nil - }, - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{ Name: "repo1", Tags: map[string]mTypes.Descriptor{ - "1.0.1": {Digest: manifestDigest.String(), MediaType: ispec.MediaTypeImageManifest}, + "1.0.1": {Digest: image.DigestStr(), MediaType: ispec.MediaTypeImageManifest}, }, }, nil }, - GetManifestMetaFn: func(repo string, manifestDigest godigest.Digest) (mTypes.ManifestMetadata, error) { - return mTypes.ManifestMetadata{ - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, + FilterImageMetaFn: func(ctx context.Context, digests []string) (map[string]mTypes.ImageMeta, error) { + return map[string]mTypes.ImageMeta{ + digests[0]: image.AsImageMeta(), }, nil }, + GetImageMetaFn: func(digest godigest.Digest) (mTypes.ImageMeta, error) { + return image.AsImageMeta(), nil + }, } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) mockCve := mocks.CveInfoMock{} - images, err := derivedImageList(responseContext, "repo1:1.0.1", nil, mockSearchDB, &gql_generated.PageInput{}, - mockCve, log.NewLogger("debug", "")) + images, err := derivedImageList(responseContext, "repo1:1.0.1", nil, mockMetaDB, &gql_generated.PageInput{}, + mockCve, log) So(err, ShouldBeNil) So(images.Results, ShouldBeEmpty) }) //nolint: dupl Convey("derived image list working", t, func() { - configBlob, err := json.Marshal(ispec.Image{ - Config: ispec.ImageConfig{ - Labels: map[string]string{}, - }, - }) - So(err, ShouldBeNil) + log := log.NewLogger("debug", "/dev/null") + layer1 := []byte{10, 11, 10, 11} + layer2 := []byte{11, 11, 11, 11} + layer3 := []byte{10, 10, 10, 11} + layer4 := []byte{13, 14, 15, 11} - configDigest := godigest.FromBytes(configBlob) + image := CreateImageWith(). + LayerBlobs([][]byte{ + layer1, + layer2, + layer3, + }).DefaultConfig().Build() - layers := [][]byte{ - {10, 11, 10, 11}, - {11, 11, 11, 11}, - {10, 10, 10, 11}, - {13, 14, 15, 11}, + derivedImage := CreateImageWith(). + LayerBlobs([][]byte{ + layer1, + layer2, + layer3, + layer4, + }).DefaultConfig().Build() + + imageMetaMap := map[string]mTypes.ImageMeta{ + image.DigestStr(): image.AsImageMeta(), + derivedImage.DigestStr(): derivedImage.AsImageMeta(), } - manifestBlob, err := json.Marshal(ispec.Manifest{ - Versioned: specs.Versioned{ - SchemaVersion: 2, - }, - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: configDigest, - Size: int64(len(configBlob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[0]), - Size: int64(len(layers[0])), - }, - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[1]), - Size: int64(len(layers[1])), - }, - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[2]), - Size: int64(len(layers[2])), - }, - }, - }) - So(err, ShouldBeNil) - - derivedManifestBlob, err := json.Marshal(ispec.Manifest{ - Versioned: specs.Versioned{ - SchemaVersion: 2, - }, - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: configDigest, - Size: int64(len(configBlob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[0]), - Size: int64(len(layers[0])), - }, - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[1]), - Size: int64(len(layers[1])), - }, - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[2]), - Size: int64(len(layers[2])), - }, - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[3]), - Size: int64(len(layers[3])), - }, - }, - }) - So(err, ShouldBeNil) - - manifestsMeta := map[string]mTypes.ManifestMetadata{ - "digestTag1.0.1": { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - DownloadCount: 100, - Signatures: make(mTypes.ManifestSignatures), - }, - "digestTag1.0.2": { - ManifestBlob: derivedManifestBlob, - ConfigBlob: configBlob, - DownloadCount: 100, - Signatures: make(mTypes.ManifestSignatures), - }, - "digestTag1.0.3": { - ManifestBlob: derivedManifestBlob, - ConfigBlob: configBlob, - DownloadCount: 100, - Signatures: make(mTypes.ManifestSignatures), - }, - } - manifestDigest := godigest.FromBytes(manifestBlob) - - mockSearchDB := mocks.MetaDBMock{ - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ + mockMetaDB := mocks.MetaDBMock{ + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{ Name: "repo1", Tags: map[string]mTypes.Descriptor{ - "1.0.1": {Digest: manifestDigest.String(), MediaType: ispec.MediaTypeImageManifest}, + "1.0.1": {Digest: image.DigestStr(), MediaType: ispec.MediaTypeImageManifest}, }, }, nil }, - GetManifestDataFn: func(manifestDigest godigest.Digest) (mTypes.ManifestData, error) { - return mTypes.ManifestData{ - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - }, nil + GetImageMetaFn: func(digest godigest.Digest) (mTypes.ImageMeta, error) { + return imageMetaMap[digest.String()], nil }, - FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - repos := []mTypes.RepoMetadata{ - { - Name: "repo1", - Tags: map[string]mTypes.Descriptor{ - "1.0.1": {Digest: "digestTag1.0.1", MediaType: ispec.MediaTypeImageManifest}, - "1.0.2": {Digest: "digestTag1.0.2", MediaType: ispec.MediaTypeImageManifest}, - "1.0.3": {Digest: "digestTag1.0.3", MediaType: ispec.MediaTypeImageManifest}, - }, - Stars: 100, - }, + FilterImageMetaFn: func(ctx context.Context, digests []string) (map[string]mTypes.ImageMeta, error) { + result := map[string]mTypes.ImageMeta{} + + for _, digest := range digests { + result[digest] = imageMetaMap[digest] } - for i, repo := range repos { - matchedTags := repo.Tags + return result, nil + }, + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + fullImageMetaList := []mTypes.FullImageMeta{} + repos := []mTypes.RepoMeta{{ + Name: "repo1", + Tags: map[string]mTypes.Descriptor{ + "1.0.1": {Digest: image.DigestStr(), MediaType: ispec.MediaTypeImageManifest}, + "1.0.2": {Digest: derivedImage.DigestStr(), MediaType: ispec.MediaTypeImageManifest}, + "1.0.3": {Digest: derivedImage.DigestStr(), MediaType: ispec.MediaTypeImageManifest}, + }, + }} + for _, repo := range repos { for tag, descriptor := range repo.Tags { - if !filterFunc(repo, manifestsMeta[descriptor.Digest]) { - delete(matchedTags, tag) - delete(manifestsMeta, descriptor.Digest) - - continue + if filterFunc(repo, imageMetaMap[descriptor.Digest]) { + fullImageMetaList = append(fullImageMetaList, + convert.GetFullImageMeta(tag, repo, imageMetaMap[descriptor.Digest])) } } - - repos[i].Tags = matchedTags } - return repos, manifestsMeta, map[string]mTypes.IndexData{}, nil + return fullImageMetaList, nil }, } @@ -3073,79 +1791,74 @@ func TestDerivedImageList(t *testing.T) { mockCve := mocks.CveInfoMock{} Convey("valid derivedImageList, results not affected by pageInput", func() { - images, err := derivedImageList(responseContext, "repo1:1.0.1", nil, mockSearchDB, &gql_generated.PageInput{}, - mockCve, log.NewLogger("debug", "")) + images, err := derivedImageList(responseContext, "repo1:1.0.1", nil, mockMetaDB, &gql_generated.PageInput{}, + mockCve, log) So(err, ShouldBeNil) So(images.Results, ShouldNotBeEmpty) So(len(images.Results), ShouldEqual, 2) }) Convey("valid derivedImageList, results affected by pageInput", func() { - limit := 1 - offset := 0 - sortCriteria := gql_generated.SortCriteriaAlphabeticAsc pageInput := gql_generated.PageInput{ - Limit: &limit, - Offset: &offset, - SortBy: &sortCriteria, + Limit: ref(1), + Offset: ref(0), + SortBy: ref(gql_generated.SortCriteriaAlphabeticAsc), } - images, err := derivedImageList(responseContext, "repo1:1.0.1", nil, mockSearchDB, &pageInput, - mockCve, log.NewLogger("debug", "")) + images, err := derivedImageList(responseContext, "repo1:1.0.1", nil, mockMetaDB, &pageInput, + mockCve, log) So(err, ShouldBeNil) So(images.Results, ShouldNotBeEmpty) - So(len(images.Results), ShouldEqual, limit) + So(len(images.Results), ShouldEqual, 1) }) }) } -func TestBaseImageList(t *testing.T) { +func TestMockedBaseImageList(t *testing.T) { Convey("MetaDB FilterTags error", t, func() { - mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - ErrTestError + mockMetaDB := mocks.MetaDBMock{ + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + return []mTypes.FullImageMeta{}, ErrTestError }, - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{}, ErrTestError + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{}, ErrTestError }, - GetManifestDataFn: func(manifestDigest godigest.Digest) (mTypes.ManifestData, error) { - return mTypes.ManifestData{}, ErrTestError + FilterImageMetaFn: func(ctx context.Context, digests []string) (map[string]mTypes.ImageMeta, error) { + return map[string]mTypes.ImageMeta{}, ErrTestError }, } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) mockCve := mocks.CveInfoMock{} - images, err := baseImageList(responseContext, "repo1:1.0.2", nil, mockSearchDB, &gql_generated.PageInput{}, + images, err := baseImageList(responseContext, "repo1:1.0.2", nil, mockMetaDB, &gql_generated.PageInput{}, mockCve, log.NewLogger("debug", "")) So(err, ShouldNotBeNil) So(images.Results, ShouldBeEmpty) }) Convey("paginated fail", t, func() { - pageInput := &gql_generated.PageInput{ - Limit: ref(-1), - } + image := CreateDefaultImage() + pageInput := &gql_generated.PageInput{Limit: ref(-1)} responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) _, err := baseImageList(responseContext, "repo1:1.0.2", nil, mocks.MetaDBMock{ - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{ Tags: map[string]mTypes.Descriptor{ "1.0.2": { - Digest: godigest.FromString("str").String(), + Digest: image.DigestStr(), MediaType: ispec.MediaTypeImageManifest, }, }, }, nil }, + FilterImageMetaFn: func(ctx context.Context, digests []string) (map[string]mTypes.ImageMeta, error) { + return map[string]mTypes.ImageMeta{image.DigestStr(): image.AsImageMeta()}, nil + }, }, pageInput, mocks.CveInfoMock{}, log.NewLogger("debug", "")) So(err, ShouldNotBeNil) @@ -3153,49 +1866,30 @@ func TestBaseImageList(t *testing.T) { //nolint: dupl Convey("MetaDB FilterTags no repo available", t, func() { - configBlob, err := json.Marshal(ispec.Image{ - Config: ispec.ImageConfig{ - Labels: map[string]string{}, + image := CreateDefaultImage() + + mockMetaDB := mocks.MetaDBMock{ + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + return []mTypes.FullImageMeta{}, nil }, - }) - So(err, ShouldBeNil) - - manifest := ispec.Manifest{} - - manifestBlob, err := json.Marshal(manifest) - So(err, ShouldBeNil) - - manifestDigest := godigest.FromBytes(manifestBlob) - - mockSearchDB := mocks.MetaDBMock{ - FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - nil - }, - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{ Name: "repo1", Tags: map[string]mTypes.Descriptor{ - "1.0.2": {Digest: manifestDigest.String(), MediaType: ispec.MediaTypeImageManifest}, + "1.0.2": {Digest: image.DigestStr(), MediaType: ispec.MediaTypeImageManifest}, }, }, nil }, - GetManifestDataFn: func(manifestDigest godigest.Digest) (mTypes.ManifestData, error) { - return mTypes.ManifestData{ - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - }, nil + FilterImageMetaFn: func(ctx context.Context, digests []string) (map[string]mTypes.ImageMeta, error) { + return map[string]mTypes.ImageMeta{image.DigestStr(): image.AsImageMeta()}, nil }, } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) mockCve := mocks.CveInfoMock{} - images, err := baseImageList(responseContext, "repo1:1.0.2", nil, mockSearchDB, &gql_generated.PageInput{}, + images, err := baseImageList(responseContext, "repo1:1.0.2", nil, mockMetaDB, &gql_generated.PageInput{}, mockCve, log.NewLogger("debug", "")) So(err, ShouldBeNil) So(images.Results, ShouldBeEmpty) @@ -3203,149 +1897,65 @@ func TestBaseImageList(t *testing.T) { //nolint: dupl Convey("base image list working", t, func() { - configBlob, err := json.Marshal(ispec.Image{ - Config: ispec.ImageConfig{ - Labels: map[string]string{}, - }, - }) - So(err, ShouldBeNil) + layer1 := []byte{10, 11, 10, 11} + layer2 := []byte{11, 11, 11, 11} + layer3 := []byte{10, 10, 10, 11} + layer4 := []byte{13, 14, 15, 11} - configDigest := godigest.FromBytes(configBlob) + image := CreateImageWith(). + LayerBlobs([][]byte{ + layer1, + layer2, + layer3, + }).DefaultConfig().Build() - layers := [][]byte{ - {10, 11, 10, 11}, - {11, 11, 11, 11}, - {10, 10, 10, 11}, - {13, 14, 15, 11}, + derivedImage := CreateImageWith(). + LayerBlobs([][]byte{ + layer1, + layer2, + layer3, + layer4, + }).DefaultConfig().Build() + + imageMetaMap := map[string]mTypes.ImageMeta{ + image.DigestStr(): image.AsImageMeta(), + derivedImage.DigestStr(): derivedImage.AsImageMeta(), } - manifestBlob, err := json.Marshal(ispec.Manifest{ - Versioned: specs.Versioned{ - SchemaVersion: 2, - }, - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: configDigest, - Size: int64(len(configBlob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[0]), - Size: int64(len(layers[0])), - }, - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[1]), - Size: int64(len(layers[1])), - }, - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[2]), - Size: int64(len(layers[2])), - }, - }, - }) - So(err, ShouldBeNil) - - derivedManifestBlob, err := json.Marshal(ispec.Manifest{ - Versioned: specs.Versioned{ - SchemaVersion: 2, - }, - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: configDigest, - Size: int64(len(configBlob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[0]), - Size: int64(len(layers[0])), - }, - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[1]), - Size: int64(len(layers[1])), - }, - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[2]), - Size: int64(len(layers[2])), - }, - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[3]), - Size: int64(len(layers[3])), - }, - }, - }) - So(err, ShouldBeNil) - - manifestsMeta := map[string]mTypes.ManifestMetadata{ - "digestTag1.0.1": { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - DownloadCount: 100, - Signatures: make(mTypes.ManifestSignatures), - }, - "digestTag1.0.2": { - ManifestBlob: derivedManifestBlob, - ConfigBlob: configBlob, - DownloadCount: 100, - Signatures: make(mTypes.ManifestSignatures), - }, - } - derivedManifestDigest := godigest.FromBytes(derivedManifestBlob) - - mockSearchDB := mocks.MetaDBMock{ - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ + mockMetaDB := mocks.MetaDBMock{ + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{ Name: "repo1", Tags: map[string]mTypes.Descriptor{ - "1.0.2": {Digest: derivedManifestDigest.String(), MediaType: ispec.MediaTypeImageManifest}, + "1.0.2": {Digest: derivedImage.DigestStr(), MediaType: ispec.MediaTypeImageManifest}, }, }, nil }, - GetManifestDataFn: func(manifestDigest godigest.Digest) (mTypes.ManifestData, error) { - return mTypes.ManifestData{ - ManifestBlob: derivedManifestBlob, - ConfigBlob: configBlob, - }, nil + FilterImageMetaFn: func(ctx context.Context, digests []string) (map[string]mTypes.ImageMeta, error) { + return imageMetaMap, nil }, - FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - repos := []mTypes.RepoMetadata{ - { - Name: "repo1", - Tags: map[string]mTypes.Descriptor{ - "1.0.1": {Digest: "digestTag1.0.1", MediaType: ispec.MediaTypeImageManifest}, - "1.0.3": {Digest: "digestTag1.0.1", MediaType: ispec.MediaTypeImageManifest}, - "1.0.2": {Digest: "digestTag1.0.2", MediaType: ispec.MediaTypeImageManifest}, - }, - Stars: 100, + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + fullImageMetaList := []mTypes.FullImageMeta{} + repos := []mTypes.RepoMeta{{ + Name: "repo1", + Tags: map[string]mTypes.Descriptor{ + "1.0.1": {Digest: image.DigestStr(), MediaType: ispec.MediaTypeImageManifest}, + "1.0.3": {Digest: image.DigestStr(), MediaType: ispec.MediaTypeImageManifest}, + "1.0.2": {Digest: derivedImage.DigestStr(), MediaType: ispec.MediaTypeImageManifest}, }, - } - - for i, repo := range repos { - matchedTags := repo.Tags + }} + for _, repo := range repos { for tag, descriptor := range repo.Tags { - if !filterFunc(repo, manifestsMeta[descriptor.Digest]) { - delete(matchedTags, tag) - delete(manifestsMeta, descriptor.Digest) - - continue + if filterFunc(repo, imageMetaMap[descriptor.Digest]) { + fullImageMetaList = append(fullImageMetaList, + convert.GetFullImageMeta(tag, repo, imageMetaMap[descriptor.Digest])) } } - - repos[i].Tags = matchedTags } - return repos, manifestsMeta, map[string]mTypes.IndexData{}, nil + return fullImageMetaList, nil }, } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, @@ -3354,7 +1964,7 @@ func TestBaseImageList(t *testing.T) { mockCve := mocks.CveInfoMock{} Convey("valid baseImageList, results not affected by pageInput", func() { - images, err := baseImageList(responseContext, "repo1:1.0.2", nil, mockSearchDB, + images, err := baseImageList(responseContext, "repo1:1.0.2", nil, mockMetaDB, &gql_generated.PageInput{}, mockCve, log.NewLogger("debug", "")) So(err, ShouldBeNil) So(images.Results, ShouldNotBeEmpty) @@ -3374,7 +1984,7 @@ func TestBaseImageList(t *testing.T) { SortBy: &sortCriteria, } - images, err := baseImageList(responseContext, "repo1:1.0.2", nil, mockSearchDB, + images, err := baseImageList(responseContext, "repo1:1.0.2", nil, mockMetaDB, &pageInput, mockCve, log.NewLogger("debug", "")) So(err, ShouldBeNil) So(images.Results, ShouldNotBeEmpty) @@ -3385,237 +1995,77 @@ func TestBaseImageList(t *testing.T) { //nolint: dupl Convey("filterTags working, no base image list found", t, func() { - configBlob, err := json.Marshal(ispec.Image{ - Config: ispec.ImageConfig{ - Labels: map[string]string{}, - }, - }) - So(err, ShouldBeNil) + layer1 := []byte{10, 11, 10, 11} + layer2 := []byte{11, 11, 11, 11} + layer3 := []byte{10, 10, 10, 11} + layer4 := []byte{13, 14, 15, 11} - configDigest := godigest.FromBytes(configBlob) + image := CreateImageWith(). + LayerBlobs([][]byte{ + layer1, + layer2, + layer3, + }).DefaultConfig().Build() - layers := [][]byte{ - {10, 11, 10, 11}, - {11, 11, 11, 11}, - {10, 10, 10, 11}, - {13, 14, 15, 11}, + derivedImage := CreateImageWith(). + LayerBlobs([][]byte{ + layer4, + }).DefaultConfig().Build() + + imageMetaMap := map[string]mTypes.ImageMeta{ + image.DigestStr(): image.AsImageMeta(), + derivedImage.DigestStr(): derivedImage.AsImageMeta(), } - manifestBlob, err := json.Marshal(ispec.Manifest{ - Versioned: specs.Versioned{ - SchemaVersion: 2, - }, - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: configDigest, - Size: int64(len(configBlob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[0]), - Size: int64(len(layers[0])), - }, - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[1]), - Size: int64(len(layers[1])), - }, - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[2]), - Size: int64(len(layers[2])), - }, - }, - }) - So(err, ShouldBeNil) - - derivedManifestBlob, err := json.Marshal(ispec.Manifest{ - Versioned: specs.Versioned{ - SchemaVersion: 2, - }, - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: configDigest, - Size: int64(len(configBlob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[3]), - Size: int64(len(layers[3])), - }, - }, - }) - So(err, ShouldBeNil) - - manifestsMeta := map[string]mTypes.ManifestMetadata{ - "digestTag1.0.1": { - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - DownloadCount: 100, - Signatures: make(mTypes.ManifestSignatures), - }, - "digestTag1.0.2": { - ManifestBlob: derivedManifestBlob, - ConfigBlob: configBlob, - DownloadCount: 100, - Signatures: make(mTypes.ManifestSignatures), - }, - } - derivedManifestDigest := godigest.FromBytes(derivedManifestBlob) - - mockSearchDB := mocks.MetaDBMock{ - GetRepoMetaFn: func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ + mockMetaDB := mocks.MetaDBMock{ + GetRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + return mTypes.RepoMeta{ Name: "repo1", Tags: map[string]mTypes.Descriptor{ - "1.0.2": {Digest: derivedManifestDigest.String(), MediaType: ispec.MediaTypeImageManifest}, + "1.0.2": {Digest: derivedImage.DigestStr(), MediaType: ispec.MediaTypeImageManifest}, }, }, nil }, - GetManifestDataFn: func(manifestDigest godigest.Digest) (mTypes.ManifestData, error) { - return mTypes.ManifestData{ - ManifestBlob: derivedManifestBlob, - ConfigBlob: configBlob, - }, nil + FilterImageMetaFn: func(ctx context.Context, digests []string) (map[string]mTypes.ImageMeta, error) { + return imageMetaMap, nil }, - FilterTagsFn: func(ctx context.Context, - filterFunc mTypes.FilterFunc, - ) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, - ) { - repos := []mTypes.RepoMetadata{ - { - Name: "repo1", - Tags: map[string]mTypes.Descriptor{ - "1.0.1": {Digest: "digestTag1.0.1", MediaType: ispec.MediaTypeImageManifest}, - "1.0.2": {Digest: "digestTag1.0.2", MediaType: ispec.MediaTypeImageManifest}, - }, - Stars: 100, + FilterTagsFn: func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, filterFunc mTypes.FilterFunc, + ) ([]mTypes.FullImageMeta, error) { + fullImageMetaList := []mTypes.FullImageMeta{} + repos := []mTypes.RepoMeta{{ + Name: "repo1", + Tags: map[string]mTypes.Descriptor{ + "1.0.1": {Digest: image.DigestStr(), MediaType: ispec.MediaTypeImageManifest}, + "1.0.2": {Digest: derivedImage.DigestStr(), MediaType: ispec.MediaTypeImageManifest}, }, - } - - for i, repo := range repos { - matchedTags := repo.Tags + }} + for _, repo := range repos { for tag, descriptor := range repo.Tags { - if !filterFunc(repo, manifestsMeta[descriptor.Digest]) { - delete(matchedTags, tag) - delete(manifestsMeta, descriptor.Digest) - - continue + if filterFunc(repo, imageMetaMap[descriptor.Digest]) { + fullImageMetaList = append(fullImageMetaList, + convert.GetFullImageMeta(tag, repo, imageMetaMap[descriptor.Digest])) } } - - repos[i].Tags = matchedTags } - return repos, manifestsMeta, map[string]mTypes.IndexData{}, nil + return fullImageMetaList, nil }, } responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, graphql.DefaultRecover) mockCve := mocks.CveInfoMock{} - images, err := baseImageList(responseContext, "repo1:1.0.2", nil, mockSearchDB, &gql_generated.PageInput{}, + images, err := baseImageList(responseContext, "repo1:1.0.2", nil, mockMetaDB, &gql_generated.PageInput{}, mockCve, log.NewLogger("debug", "")) So(err, ShouldBeNil) So(images.Results, ShouldBeEmpty) }) } -func TestExpandedRepoInfo(t *testing.T) { +func TestExpandedRepoInfoErrors(t *testing.T) { log := log.NewLogger("debug", "") - Convey("ExpandedRepoInfo Errors", t, func() { - responseContext := graphql.WithResponseContext(context.Background(), graphql.DefaultErrorPresenter, - graphql.DefaultRecover) - - metaDB := mocks.MetaDBMock{ - GetUserRepoMetaFn: func(ctx context.Context, repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{ - "tagManifest": { - Digest: "errorDigest", - MediaType: ispec.MediaTypeImageManifest, - }, - "tagIndex": { - Digest: "digestIndex", - MediaType: ispec.MediaTypeImageIndex, - }, - "tagGetIndexError": { - Digest: "errorIndexDigest", - MediaType: ispec.MediaTypeImageIndex, - }, - "tagGoodIndexBadManifests": { - Digest: "goodIndexBadManifests", - MediaType: ispec.MediaTypeImageIndex, - }, - "tagGoodIndex1GoodManifest": { - Digest: "goodIndexGoodManifest", - MediaType: ispec.MediaTypeImageIndex, - }, - "tagGoodIndex2GoodManifest": { - Digest: "goodIndexGoodManifest", - MediaType: ispec.MediaTypeImageIndex, - }, - }, - }, nil - }, - GetManifestDataFn: func(manifestDigest godigest.Digest) (mTypes.ManifestData, error) { - switch manifestDigest { - case "errorDigest": - return mTypes.ManifestData{}, ErrTestError - default: - return mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("{}"), - }, nil - } - }, - GetIndexDataFn: func(indexDigest godigest.Digest) (mTypes.IndexData, error) { - goodIndexBadManifestsBlob, err := json.Marshal(ispec.Index{ - Manifests: []ispec.Descriptor{ - { - Digest: "errorDigest", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - }) - So(err, ShouldBeNil) - - goodIndexGoodManifestBlob, err := json.Marshal(ispec.Index{ - Manifests: []ispec.Descriptor{ - { - Digest: "goodManifest", - MediaType: ispec.MediaTypeImageManifest, - }, - }, - }) - So(err, ShouldBeNil) - - switch indexDigest { - case "errorIndexDigest": - return mTypes.IndexData{}, ErrTestError - case "goodIndexBadManifests": - return mTypes.IndexData{ - IndexBlob: goodIndexBadManifestsBlob, - }, nil - case "goodIndexGoodManifest": - return mTypes.IndexData{ - IndexBlob: goodIndexGoodManifestBlob, - }, nil - default: - return mTypes.IndexData{}, nil - } - }, - } - - _, err := expandedRepoInfo(responseContext, "repo", metaDB, mocks.CveInfoMock{}, log) - So(err, ShouldBeNil) - }) - Convey("Access error", t, func() { userAc := reqCtx.NewUserAccessControl() userAc.SetUsername("user") @@ -3633,54 +2083,18 @@ func TestExpandedRepoInfo(t *testing.T) { }) } -func TestFilterFunctions(t *testing.T) { - Convey("Filter Functions", t, func() { - Convey("FilterByDigest bad manifest blob", func() { - filterFunc := FilterByDigest("digest") - ok := filterFunc( - mTypes.RepoMetadata{}, - mTypes.ManifestMetadata{ - ManifestBlob: []byte("bad blob"), - }, - ) - So(ok, ShouldBeFalse) - }) - - Convey("filterDerivedImages bad manifest blob", func() { - filterFunc := filterDerivedImages(&gql_generated.ImageSummary{}) - ok := filterFunc( - mTypes.RepoMetadata{}, - mTypes.ManifestMetadata{ - ManifestBlob: []byte("bad blob"), - }, - ) - So(ok, ShouldBeFalse) - }) - - Convey("FilterByTagInfo", func() { - fFunc := FilterByTagInfo([]cvemodel.TagInfo{ - { - Descriptor: cvemodel.Descriptor{ - MediaType: ispec.MediaTypeImageIndex, - }, - Manifests: []cvemodel.DescriptorInfo{ - { - Descriptor: cvemodel.Descriptor{ - Digest: godigest.FromString("{}"), - }, - }, - }, - }, - }) - - ok := fFunc(mTypes.RepoMetadata{}, mTypes.ManifestMetadata{ManifestBlob: []byte("{}")}) - So(ok, ShouldBeTrue) - }) - }) -} - -func ref[T any](val T) *T { - ref := val +func ref[T any](input T) *T { + ref := input return &ref } + +func getPageInput(limit int, offset int) *gql_generated.PageInput { + sortCriteria := gql_generated.SortCriteriaAlphabeticAsc + + return &gql_generated.PageInput{ + Limit: &limit, + Offset: &offset, + SortBy: &sortCriteria, + } +} diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index f704e48c..56b9b2b5 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -13,7 +13,6 @@ import ( "net/url" "os" "path" - "regexp" "strconv" "strings" "testing" @@ -305,7 +304,7 @@ func getMockCveScanner(metaDB mTypes.MetaDB) cveinfo.Scanner { imageDir := repo inputTag := reference - repoMeta, err := metaDB.GetRepoMeta(imageDir) + repoMeta, err := metaDB.GetRepoMeta(context.Background(), imageDir) if err != nil { return false, err } @@ -328,19 +327,12 @@ func getMockCveScanner(metaDB mTypes.MetaDB) cveinfo.Scanner { return false, err } - manifestData, err := metaDB.GetManifestData(manifestDigest) + manifestData, err := metaDB.GetImageMeta(manifestDigest) if err != nil { return false, err } - var manifestContent ispec.Manifest - - err = json.Unmarshal(manifestData.ManifestBlob, &manifestContent) - if err != nil { - return false, zerr.ErrScanNotSupported - } - - for _, imageLayer := range manifestContent.Layers { + for _, imageLayer := range manifestData.Manifests[0].Manifest.Layers { switch imageLayer.MediaType { case ispec.MediaTypeImageLayerGzip, ispec.MediaTypeImageLayer, string(regTypes.DockerLayer): @@ -1070,24 +1062,17 @@ func TestGetReferrersGQL(t *testing.T) { defer ctlrManager.StopServer() // Upload the index referrer - - targetImg, err := deprecated.GetRandomImage() //nolint:staticcheck - So(err, ShouldBeNil) + targetImg := CreateRandomImage() targetDigest := targetImg.Digest() - err = UploadImage(targetImg, baseURL, "repo", targetDigest.String()) - So(err, ShouldBeNil) - - indexReferrer, err := deprecated.GetRandomMultiarchImage("ref") //nolint:staticcheck + err := UploadImage(targetImg, baseURL, "repo", targetImg.DigestStr()) So(err, ShouldBeNil) artifactType := "com.artifact.art/type" - indexReferrer.Index.ArtifactType = artifactType - indexReferrer.Index.Subject = &ispec.Descriptor{ - MediaType: ispec.MediaTypeImageManifest, - Digest: targetDigest, - } - + indexReferrer := CreateMultiarchWith().RandomImages(2). + ArtifactType(artifactType). + Subject(targetImg.DescriptorRef()). + Build() indexReferrerDigest := indexReferrer.Digest() err = UploadMultiarchImage(indexReferrer, baseURL, "repo", "ref") @@ -3203,7 +3188,9 @@ func TestGlobalSearch(t *testing.T) { Vendors NewestImage { RepoName Tag LastUpdated Size + Digest Manifests{ + Digest ConfigDigest LastUpdated Size Platform { Os Arch } History { @@ -4495,67 +4482,24 @@ func TestMetaDBWhenPushingImages(t *testing.T) { ctlrManager.StartAndWait(port) defer ctlrManager.StopServer() - Convey("SetManifestMeta fails", func() { - ctlr.MetaDB = mocks.MetaDBMock{ - SetManifestDataFn: func(manifestDigest godigest.Digest, mm mTypes.ManifestData) error { - return ErrTestError - }, - } - config1, layers1, manifest1, err := deprecated.GetImageComponents(100) //nolint:staticcheck - So(err, ShouldBeNil) - - configBlob, err := json.Marshal(config1) - So(err, ShouldBeNil) - - ctlr.StoreController.DefaultStore = mocks.MockedImageStore{ - NewBlobUploadFn: ctlr.StoreController.DefaultStore.NewBlobUpload, - PutBlobChunkFn: ctlr.StoreController.DefaultStore.PutBlobChunk, - GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) { - return configBlob, nil - }, - DeleteImageManifestFn: func(repo, reference string, dc bool) error { - return ErrTestError - }, - } - - err = UploadImage( - Image{ - Manifest: manifest1, - Config: config1, - Layers: layers1, - }, baseURL, "repo1", "1.0.1", - ) - So(err, ShouldNotBeNil) - }) - Convey("SetManifestMeta succeeds but SetRepoReference fails", func() { ctlr.MetaDB = mocks.MetaDBMock{ - SetRepoReferenceFn: func(repo, reference string, manifestDigest godigest.Digest, mediaType string) error { + SetRepoReferenceFn: func(repo, reference string, imageMeta mTypes.ImageMeta) error { return ErrTestError }, } - config1, layers1, manifest1, err := deprecated.GetImageComponents(100) //nolint:staticcheck - So(err, ShouldBeNil) - - configBlob, err := json.Marshal(config1) - So(err, ShouldBeNil) + image := CreateRandomImage() ctlr.StoreController.DefaultStore = mocks.MockedImageStore{ NewBlobUploadFn: ctlr.StoreController.DefaultStore.NewBlobUpload, PutBlobChunkFn: ctlr.StoreController.DefaultStore.PutBlobChunk, GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) { - return configBlob, nil + return image.ConfigDescriptor.Data, nil }, } - err = UploadImage( - Image{ - Manifest: manifest1, - Config: config1, - Layers: layers1, - }, baseURL, "repo1", "1.0.1", - ) + err := UploadImage(image, baseURL, "repo1", "1.0.1") So(err, ShouldNotBeNil) }) }) @@ -4590,15 +4534,9 @@ func RunMetaDBIndexTests(baseURL, port string) { Convey("Push test index", func() { const repo = "repo" - multiarchImage, err := deprecated.GetRandomMultiarchImage("tag1") //nolint:staticcheck - So(err, ShouldBeNil) + multiarchImage := CreateRandomMultiarch() - indexBlob, err := json.Marshal(multiarchImage.Index) - So(err, ShouldBeNil) - - indexDigest := godigest.FromBytes(indexBlob) - - err = UploadMultiarchImage(multiarchImage, baseURL, repo, "tag1") + err := UploadMultiarchImage(multiarchImage, baseURL, repo, "tag1") So(err, ShouldBeNil) query := ` @@ -4634,7 +4572,7 @@ func RunMetaDBIndexTests(baseURL, port string) { responseImage := responseImages[0] So(len(responseImage.Manifests), ShouldEqual, 3) - err = signature.SignImageUsingCosign(fmt.Sprintf("repo@%s", indexDigest), port) + err = signature.SignImageUsingCosign(fmt.Sprintf("repo@%s", multiarchImage.DigestStr()), port) So(err, ShouldBeNil) resp, err = resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query)) @@ -4654,7 +4592,7 @@ func RunMetaDBIndexTests(baseURL, port string) { So(responseImage.IsSigned, ShouldBeTrue) // remove signature - cosignTag := "sha256-" + indexDigest.Encoded() + ".sig" + cosignTag := "sha256-" + multiarchImage.Digest().Encoded() + ".sig" _, err = resty.R().Delete(baseURL + "/v2/" + "repo" + "/manifests/" + cosignTag) So(err, ShouldBeNil) @@ -5409,9 +5347,7 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { for _, manifest := range indexContent.Manifests { tag := manifest.Annotations[ispec.AnnotationRefName] - cosignTagRule := regexp.MustCompile(`sha256\-.+\.sig`) - - if cosignTagRule.MatchString(tag) { + if zcommon.IsCosignTag(tag) { signatureTag = tag } } @@ -5685,7 +5621,10 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { } ctlr.MetaDB = mocks.MetaDBMock{ - RemoveRepoReferenceFn: func(repo, reference string, manifestDigest godigest.Digest) error { return ErrTestError }, + RemoveRepoReferenceFn: func(repo, reference string, manifestDigest godigest.Digest, + ) error { + return ErrTestError + }, } resp, err = resty.R().Delete(baseURL + "/v2/" + "repo1" + "/manifests/" + @@ -6438,16 +6377,6 @@ func TestUploadingArtifactsWithDifferentMediaType(t *testing.T) { err = UploadImage(defaultImage, baseURL, "repo", "default-image") So(err, ShouldBeNil) - manifestData, err := ctlr.MetaDB.GetManifestData(imageWithIncompatibleConfig.ManifestDescriptor.Digest) - So(err, ShouldBeNil) - So(manifestData.ConfigBlob, ShouldEqual, imageWithIncompatibleConfig.ConfigDescriptor.Data) - So(manifestData.ManifestBlob, ShouldEqual, imageWithIncompatibleConfig.ManifestDescriptor.Data) - - manifestData, err = ctlr.MetaDB.GetManifestData(defaultImage.ManifestDescriptor.Digest) - So(err, ShouldBeNil) - So(manifestData.ConfigBlob, ShouldEqual, defaultImage.ConfigDescriptor.Data) - So(manifestData.ManifestBlob, ShouldEqual, defaultImage.ManifestDescriptor.Data) - query := ` { GlobalSearch(query:"repo:incompatible-image"){ diff --git a/pkg/extensions/search/userprefs_test.go b/pkg/extensions/search/userprefs_test.go index 3844ec54..7d09a8e7 100644 --- a/pkg/extensions/search/userprefs_test.go +++ b/pkg/extensions/search/userprefs_test.go @@ -18,13 +18,11 @@ import ( "zotregistry.io/zot/pkg/api/constants" "zotregistry.io/zot/pkg/common" extconf "zotregistry.io/zot/pkg/extensions/config" - "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/log" - "zotregistry.io/zot/pkg/storage" - "zotregistry.io/zot/pkg/storage/local" test "zotregistry.io/zot/pkg/test/common" "zotregistry.io/zot/pkg/test/deprecated" . "zotregistry.io/zot/pkg/test/image-utils" + . "zotregistry.io/zot/pkg/test/oci-utils" ) //nolint:dupl @@ -535,25 +533,15 @@ func TestChangingRepoState(t *testing.T) { ctlr := api.NewController(conf) - img, err := deprecated.GetRandomImage() //nolint:staticcheck + img := CreateRandomImage() + storeCtlr := GetDefaultStoreController(conf.Storage.RootDirectory, log.NewLogger("debug", "")) + + err := WriteImageToFileSystem(img, accesibleRepo, "tag", storeCtlr) if err != nil { t.FailNow() } - // ------ Create the test repos - defaultStore := local.NewImageStore(conf.Storage.RootDirectory, false, false, - log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) - - err = WriteImageToFileSystem(img, accesibleRepo, "tag", storage.StoreController{ - DefaultStore: defaultStore, - }) - if err != nil { - t.FailNow() - } - - err = WriteImageToFileSystem(img, forbiddenRepo, "tag", storage.StoreController{ - DefaultStore: defaultStore, - }) + err = WriteImageToFileSystem(img, forbiddenRepo, "tag", storeCtlr) if err != nil { t.FailNow() } diff --git a/pkg/extensions/sync/references/cosign.go b/pkg/extensions/sync/references/cosign.go index 1b842b6e..1903e85a 100644 --- a/pkg/extensions/sync/references/cosign.go +++ b/pkg/extensions/sync/references/cosign.go @@ -153,21 +153,9 @@ func (ref CosignReference) SyncReferences(ctx context.Context, localRepo, remote ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr). Msg("metaDB: trying to sync cosign reference for image") - isSig, sigType, signedManifestDig, err := storage.CheckIsImageSignature(localRepo, manifestBuf, - cosignTag) - if err != nil { - return refsDigests, fmt.Errorf("failed to check if cosign reference '%s@%s' is a signature: %w", localRepo, - cosignTag, err) - } - - if isSig { - err = addSigToMeta(ref.metaDB, localRepo, sigType, cosignTag, signedManifestDig, referenceDigest, - manifestBuf, imageStore, ref.log) - } else { - err = meta.SetImageMetaFromInput(localRepo, cosignTag, ispec.MediaTypeImageManifest, - referenceDigest, manifestBuf, ref.storeController.GetImageStore(localRepo), - ref.metaDB, ref.log) - } + err = meta.SetImageMetaFromInput(localRepo, cosignTag, ispec.MediaTypeImageManifest, + referenceDigest, manifestBuf, ref.storeController.GetImageStore(localRepo), + ref.metaDB, ref.log) if err != nil { return refsDigests, fmt.Errorf("failed to set metadata for cosign reference in '%s@%s': %w", diff --git a/pkg/extensions/sync/references/oci.go b/pkg/extensions/sync/references/oci.go index beac3031..6519b05e 100644 --- a/pkg/extensions/sync/references/oci.go +++ b/pkg/extensions/sync/references/oci.go @@ -137,22 +137,9 @@ func (ref OciReferences) SyncReferences(ctx context.Context, localRepo, remoteRe ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr). Msg("metaDB: trying to add oci references for image") - isSig, sigType, signedManifestDig, err := storage.CheckIsImageSignature(localRepo, referenceBuf, - referrer.Digest.String()) - if err != nil { - return refsDigests, fmt.Errorf("failed to check if oci reference '%s@%s' is a signature: %w", localRepo, - referrer.Digest.String(), err) - } - - if isSig { - err = addSigToMeta(ref.metaDB, localRepo, sigType, referrer.Digest.String(), signedManifestDig, referenceDigest, - referenceBuf, imageStore, ref.log) - } else { - err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType, - referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo), - ref.metaDB, ref.log) - } - + err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType, + referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo), + ref.metaDB, ref.log) if err != nil { return refsDigests, fmt.Errorf("failed to set metadata for oci reference in '%s@%s': %w", localRepo, subjectDigestStr, err) diff --git a/pkg/extensions/sync/references/referrers_tag.go b/pkg/extensions/sync/references/referrers_tag.go index 4628add3..2633e15b 100644 --- a/pkg/extensions/sync/references/referrers_tag.go +++ b/pkg/extensions/sync/references/referrers_tag.go @@ -113,22 +113,9 @@ func (ref TagReferences) SyncReferences(ctx context.Context, localRepo, remoteRe ref.log.Debug().Str("repository", localRepo).Str("subject", subjectDigestStr). Msg("metaDB: trying to add oci references for image") - isSig, sigType, signedManifestDig, err := storage.CheckIsImageSignature(localRepo, referenceBuf, - referrer.Digest.String()) - if err != nil { - return refsDigests, fmt.Errorf("failed to check if oci reference '%s@%s' is a signature: %w", localRepo, - referrer.Digest.String(), err) - } - - if isSig { - err = addSigToMeta(ref.metaDB, localRepo, sigType, referrer.Digest.String(), signedManifestDig, referenceDigest, - referenceBuf, imageStore, ref.log) - } else { - err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType, - referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo), - ref.metaDB, ref.log) - } - + err = meta.SetImageMetaFromInput(localRepo, referenceDigest.String(), referrer.MediaType, + referenceDigest, referenceBuf, ref.storeController.GetImageStore(localRepo), + ref.metaDB, ref.log) if err != nil { return refsDigests, fmt.Errorf("failed to set metadata for oci reference in '%s@%s': %w", localRepo, subjectDigestStr, err) diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index 9b6c2968..f28d3c3e 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -27,6 +27,7 @@ import ( "zotregistry.io/zot/pkg/extensions/monitoring" client "zotregistry.io/zot/pkg/extensions/sync/httpclient" "zotregistry.io/zot/pkg/log" + mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/cache" storageConstants "zotregistry.io/zot/pkg/storage/constants" @@ -336,8 +337,8 @@ func TestLocalRegistry(t *testing.T) { Convey("trigger metaDB error on index manifest in CommitImage()", func() { registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, mocks.MetaDBMock{ - SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error { - if Reference == "1.0" { + SetRepoReferenceFn: func(repo string, reference string, imageMeta mTypes.ImageMeta) error { + if reference == "1.0" { return zerr.ErrRepoMetaNotFound } @@ -351,7 +352,7 @@ func TestLocalRegistry(t *testing.T) { Convey("trigger metaDB error on image manifest in CommitImage()", func() { registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, mocks.MetaDBMock{ - SetRepoReferenceFn: func(repo, Reference string, manifestDigest godigest.Digest, mediaType string) error { + SetRepoReferenceFn: func(repo, reference string, imageMeta mTypes.ImageMeta) error { return zerr.ErrRepoMetaNotFound }, }, log) diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 2236ca82..aed6c076 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -877,9 +877,7 @@ func TestOnDemand(t *testing.T) { return nil }, - SetRepoReferenceFn: func(repo, reference string, manifestDigest godigest.Digest, - mediaType string, - ) error { + SetRepoReferenceFn: func(repo, reference string, imageMeta mTypes.ImageMeta) error { if strings.HasPrefix(reference, "sha256-") && (strings.HasSuffix(reference, remote.SignatureTagSuffix) || strings.HasSuffix(reference, remote.SBOMTagSuffix)) || @@ -1019,8 +1017,8 @@ func TestOnDemand(t *testing.T) { // metadb fails for syncReferrersTag" dctlr.MetaDB = mocks.MetaDBMock{ - SetManifestDataFn: func(manifestDigest godigest.Digest, mm mTypes.ManifestData) error { - if manifestDigest.String() == ociRefImage.ManifestDescriptor.Digest.String() { + SetRepoReferenceFn: func(repo, reference string, imageMeta mTypes.ImageMeta) error { + if imageMeta.Digest.String() == ociRefImage.ManifestDescriptor.Digest.String() { return sync.ErrTestError } @@ -4670,7 +4668,7 @@ func TestSyncedSignaturesMetaDB(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusOK) - repoMeta, err := dctlr.MetaDB.GetRepoMeta(repoName) + repoMeta, err := dctlr.MetaDB.GetRepoMeta(context.Background(), repoName) So(err, ShouldBeNil) So(repoMeta.Tags, ShouldContainKey, tag) So(len(repoMeta.Tags), ShouldEqual, 1) diff --git a/pkg/meta/boltdb/boltdb.go b/pkg/meta/boltdb/boltdb.go index ca14e456..b3a223a2 100644 --- a/pkg/meta/boltdb/boltdb.go +++ b/pkg/meta/boltdb/boltdb.go @@ -11,12 +11,16 @@ import ( godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" "go.etcd.io/bbolt" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/api/constants" zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/common" + mConvert "zotregistry.io/zot/pkg/meta/convert" + proto_go "zotregistry.io/zot/pkg/meta/proto/gen" mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/meta/version" reqCtx "zotregistry.io/zot/pkg/requestcontext" @@ -41,21 +45,6 @@ func New(boltDB *bbolt.DB, log log.Logger) (*BoltDB, error) { return err } - _, err = transaction.CreateBucketIfNotExists([]byte(ManifestDataBucket)) - if err != nil { - return err - } - - _, err = transaction.CreateBucketIfNotExists([]byte(IndexDataBucket)) - if err != nil { - return err - } - - _, err = transaction.CreateBucketIfNotExists([]byte(RepoMetadataBucket)) - if err != nil { - return err - } - _, err = transaction.CreateBucketIfNotExists([]byte(UserDataBucket)) if err != nil { return err @@ -66,6 +55,21 @@ func New(boltDB *bbolt.DB, log log.Logger) (*BoltDB, error) { return err } + _, err = transaction.CreateBucketIfNotExists([]byte(ImageMetaBuck)) + if err != nil { + return err + } + + _, err = transaction.CreateBucketIfNotExists([]byte(RepoMetaBuck)) + if err != nil { + return err + } + + _, err = transaction.CreateBucketIfNotExists([]byte(RepoBlobsBuck)) + if err != nil { + return err + } + return nil }) if err != nil { @@ -80,26 +84,30 @@ func New(boltDB *bbolt.DB, log log.Logger) (*BoltDB, error) { }, nil } -func (bdw *BoltDB) ImageTrustStore() mTypes.ImageTrustStore { - return bdw.imgTrustStore -} - -func (bdw *BoltDB) SetImageTrustStore(imgTrustStore mTypes.ImageTrustStore) { - bdw.imgTrustStore = imgTrustStore -} - -func (bdw *BoltDB) SetManifestData(manifestDigest godigest.Digest, manifestData mTypes.ManifestData) error { +func (bdw *BoltDB) SetImageMeta(digest godigest.Digest, imageMeta mTypes.ImageMeta) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(ManifestDataBucket)) + buck := tx.Bucket([]byte(ImageMetaBuck)) - mdBlob, err := json.Marshal(manifestData) - if err != nil { - return fmt.Errorf("metadb: error while calculating blob for manifest with digest %s %w", manifestDigest, err) + protoImageMeta := &proto_go.ImageMeta{} + + switch imageMeta.MediaType { + case ispec.MediaTypeImageManifest: + manifest := imageMeta.Manifests[0] + + protoImageMeta = mConvert.GetProtoImageManifestData(manifest.Manifest, manifest.Config, + manifest.Size, manifest.Digest.String()) + case ispec.MediaTypeImageIndex: + protoImageMeta = mConvert.GetProtoImageIndexMeta(*imageMeta.Index, imageMeta.Size, imageMeta.Digest.String()) } - err = buck.Put([]byte(manifestDigest), mdBlob) + pImageMetaBlob, err := proto.Marshal(protoImageMeta) if err != nil { - return fmt.Errorf("metadb: error while setting manifest data with for digest %s %w", manifestDigest, err) + return fmt.Errorf("metadb: error while calculating blob for manifest with digest %s %w", digest, err) + } + + err = buck.Put([]byte(digest), pImageMetaBlob) + if err != nil { + return fmt.Errorf("metadb: error while setting manifest data with for digest %s %w", digest, err) } return nil @@ -108,217 +116,731 @@ func (bdw *BoltDB) SetManifestData(manifestDigest godigest.Digest, manifestData return err } -func (bdw *BoltDB) GetManifestData(manifestDigest godigest.Digest) (mTypes.ManifestData, error) { - var manifestData mTypes.ManifestData - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(ManifestDataBucket)) - - mdBlob := buck.Get([]byte(manifestDigest)) - - if len(mdBlob) == 0 { - return zerr.ErrManifestDataNotFound - } - - err := json.Unmarshal(mdBlob, &manifestData) - if err != nil { - return fmt.Errorf("metadb: error while unmashaling manifest meta for digest %s %w", manifestDigest, err) - } - - return nil - }) - - return manifestData, err -} - -func (bdw *BoltDB) SetManifestMeta(repo string, manifestDigest godigest.Digest, manifestMeta mTypes.ManifestMetadata, +func (bdw *BoltDB) SetRepoReference(repo string, reference string, imageMeta mTypes.ImageMeta, ) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - dataBuck := tx.Bucket([]byte(ManifestDataBucket)) - repoBuck := tx.Bucket([]byte(RepoMetadataBucket)) + if err := common.ValidateRepoReferenceInput(repo, reference, imageMeta.Digest); err != nil { + return err + } - repoMeta := mTypes.RepoMetadata{ - Name: repo, - Tags: map[string]mTypes.Descriptor{}, - Statistics: map[string]mTypes.DescriptorStatistics{}, - Signatures: map[string]mTypes.ManifestSignatures{}, - Referrers: map[string][]mTypes.ReferrerInfo{}, + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + repoBuck := tx.Bucket([]byte(RepoMetaBuck)) + repoBlobsBuck := tx.Bucket([]byte(RepoBlobsBuck)) + imageBuck := tx.Bucket([]byte(ImageMetaBuck)) + + // 1. Add image data to db if needed + + protoImageMeta := mConvert.GetProtoImageMeta(imageMeta) + + imageMetaBlob, err := proto.Marshal(protoImageMeta) + if err != nil { + return err + } + + err = imageBuck.Put([]byte(imageMeta.Digest), imageMetaBlob) + if err != nil { + return err } repoMetaBlob := repoBuck.Get([]byte(repo)) + + protoRepoMeta := &proto_go.RepoMeta{ + Name: repo, + Tags: map[string]*proto_go.TagDescriptor{"": {}}, // This is done so Protobuf can initialize a non-nil map + Statistics: map[string]*proto_go.DescriptorStatistics{"": {}}, + Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}}, + Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, + } + if len(repoMetaBlob) > 0 { - err := json.Unmarshal(repoMetaBlob, &repoMeta) + err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) if err != nil { return err } } - mdBlob, err := json.Marshal(mTypes.ManifestData{ - ManifestBlob: manifestMeta.ManifestBlob, - ConfigBlob: manifestMeta.ConfigBlob, - }) - if err != nil { - return fmt.Errorf("metadb: error while calculating blob for manifest with digest %s %w", manifestDigest, err) + // 2. Referrers + if subject := mConvert.GetImageSubject(protoImageMeta); subject != nil { + refInfo := &proto_go.ReferrersInfo{} + if protoRepoMeta.Referrers[subject.Digest.String()] != nil { + refInfo = protoRepoMeta.Referrers[subject.Digest.String()] + } + + foundReferrer := false + + for i := range refInfo.List { + if refInfo.List[i].Digest == mConvert.GetImageDigestStr(protoImageMeta) { + foundReferrer = true + refInfo.List[i].Count += 1 + + break + } + } + + if !foundReferrer { + refInfo.List = append(refInfo.List, &proto_go.ReferrerInfo{ + Count: 1, + MediaType: protoImageMeta.MediaType, + Digest: mConvert.GetImageDigestStr(protoImageMeta), + ArtifactType: mConvert.GetImageArtifactType(protoImageMeta), + Size: mConvert.GetImageManifestSize(protoImageMeta), + Annotations: mConvert.GetImageAnnotations(protoImageMeta), + }) + } + + protoRepoMeta.Referrers[subject.Digest.String()] = refInfo } - err = dataBuck.Put([]byte(manifestDigest), mdBlob) - if err != nil { - return fmt.Errorf("metadb: error while setting manifest meta with for digest %s %w", manifestDigest, err) - } - - updatedRepoMeta := common.UpdateManifestMeta(repoMeta, manifestDigest, manifestMeta) - - updatedRepoMetaBlob, err := json.Marshal(updatedRepoMeta) - if err != nil { - return fmt.Errorf("metadb: error while calculating blob for updated repo meta '%s' %w", repo, err) - } - - return repoBuck.Put([]byte(repo), updatedRepoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) GetManifestMeta(repo string, manifestDigest godigest.Digest) (mTypes.ManifestMetadata, error) { - var manifestMetadata mTypes.ManifestMetadata - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - dataBuck := tx.Bucket([]byte(ManifestDataBucket)) - repoBuck := tx.Bucket([]byte(RepoMetadataBucket)) - - mdBlob := dataBuck.Get([]byte(manifestDigest)) - - if len(mdBlob) == 0 { - return zerr.ErrManifestMetaNotFound - } - - var manifestData mTypes.ManifestData - - err := json.Unmarshal(mdBlob, &manifestData) - if err != nil { - return fmt.Errorf("metadb: error while unmashaling manifest meta for digest %s %w", manifestDigest, err) - } - - var repoMeta mTypes.RepoMetadata - - repoMetaBlob := repoBuck.Get([]byte(repo)) - if len(repoMetaBlob) > 0 { - err = json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return fmt.Errorf("metadb: error while unmashaling manifest meta for digest %s %w", manifestDigest, err) + // 3. Update tag + if !common.ReferenceIsDigest(reference) { + protoRepoMeta.Tags[reference] = &proto_go.TagDescriptor{ + Digest: imageMeta.Digest.String(), + MediaType: imageMeta.MediaType, } } - manifestMetadata.ManifestBlob = manifestData.ManifestBlob - manifestMetadata.ConfigBlob = manifestData.ConfigBlob - manifestMetadata.DownloadCount = repoMeta.Statistics[manifestDigest.String()].DownloadCount - - manifestMetadata.Signatures = mTypes.ManifestSignatures{} - if repoMeta.Signatures[manifestDigest.String()] != nil { - manifestMetadata.Signatures = repoMeta.Signatures[manifestDigest.String()] + if _, ok := protoRepoMeta.Statistics[imageMeta.Digest.String()]; !ok { + protoRepoMeta.Statistics[imageMeta.Digest.String()] = &proto_go.DescriptorStatistics{DownloadCount: 0} } - return nil - }) + if _, ok := protoRepoMeta.Signatures[imageMeta.Digest.String()]; !ok { + protoRepoMeta.Signatures[imageMeta.Digest.String()] = &proto_go.ManifestSignatures{ + Map: map[string]*proto_go.SignaturesInfo{"": {}}, + } + } - return manifestMetadata, err -} + if _, ok := protoRepoMeta.Referrers[imageMeta.Digest.String()]; !ok { + protoRepoMeta.Referrers[imageMeta.Digest.String()] = &proto_go.ReferrersInfo{ + List: []*proto_go.ReferrerInfo{}, + } + } -func (bdw *BoltDB) SetIndexData(indexDigest godigest.Digest, indexMetadata mTypes.IndexData) error { - // we make the assumption that the oci layout is consistent and all manifests refferenced inside the - // index are present - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(IndexDataBucket)) + // 4. Blobs + repoBlobsBytes := repoBlobsBuck.Get([]byte(protoRepoMeta.Name)) - imBlob, err := json.Marshal(indexMetadata) + repoBlobs := &proto_go.RepoBlobs{} + + if repoBlobsBytes == nil { + repoBlobs.Blobs = map[string]*proto_go.BlobInfo{} + } else { + err := proto.Unmarshal(repoBlobsBytes, repoBlobs) + if err != nil { + return err + } + } + + protoRepoMeta, repoBlobs, err = common.AddImageMetaToRepoMeta(protoRepoMeta, repoBlobs, reference, imageMeta) if err != nil { - return fmt.Errorf("metadb: error while calculating blob for manifest with digest %s %w", indexDigest, err) + return err } - err = buck.Put([]byte(indexDigest), imBlob) + repoBlobsBytes, err = proto.Marshal(repoBlobs) if err != nil { - return fmt.Errorf("metadb: error while setting manifest meta with for digest %s %w", indexDigest, err) + return err } - return nil + err = repoBlobsBuck.Put([]byte(protoRepoMeta.Name), repoBlobsBytes) + if err != nil { + return err + } + + repoMetaBlob, err = proto.Marshal(protoRepoMeta) + if err != nil { + return err + } + + return repoBuck.Put([]byte(repo), repoMetaBlob) }) return err } -func (bdw *BoltDB) GetIndexData(indexDigest godigest.Digest) (mTypes.IndexData, error) { - var indexMetadata mTypes.IndexData +func (bdw *BoltDB) FilterImageMeta(ctx context.Context, digests []string, +) (map[string]mTypes.ImageMeta, error) { + imageMetaMap := map[string]mTypes.ImageMeta{} - err := bdw.DB.View(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(IndexDataBucket)) + err := bdw.DB.View(func(transaction *bbolt.Tx) error { + imageBuck := transaction.Bucket([]byte(ImageMetaBuck)) - mmBlob := buck.Get([]byte(indexDigest)) + for _, digest := range digests { + protoImageMeta, err := fetchProtoImageMeta(imageBuck, digest) + if err != nil { + return err + } - if len(mmBlob) == 0 { - return zerr.ErrManifestMetaNotFound - } + if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { + manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) - err := json.Unmarshal(mmBlob, &indexMetadata) - if err != nil { - return fmt.Errorf("metadb: error while unmashaling manifest meta for digest %s %w", indexDigest, err) + for _, manifest := range protoImageMeta.Index.Index.Manifests { + imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest) + if err != nil { + return err + } + + manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + } + + protoImageMeta.Manifests = manifestDataList + } + + imageMetaMap[digest] = mConvert.GetImageMeta(protoImageMeta) } return nil }) - return indexMetadata, err + return imageMetaMap, err } -func (bdw BoltDB) SetReferrer(repo string, referredDigest godigest.Digest, referrer mTypes.ReferrerInfo) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) +func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, +) ([]mTypes.RepoMeta, error) { + repos := []mTypes.RepoMeta{} + + err := bdw.DB.View(func(transaction *bbolt.Tx) error { + var ( + repoBuck = transaction.Bucket([]byte(RepoMetaBuck)) + userBookmarks = getUserBookmarks(ctx, transaction) + userStars = getUserStars(ctx, transaction) + ) + + cursor := repoBuck.Cursor() + + for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { + if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { + continue + } + + rank := common.RankRepoName(searchText, string(repoName)) + if rank == -1 { + continue + } + + var protoRepoMeta proto_go.RepoMeta + + err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) + if err != nil { + return err + } + + delete(protoRepoMeta.Tags, "") + + protoRepoMeta.Rank = int32(rank) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) + + repos = append(repos, mConvert.GetRepoMeta(&protoRepoMeta)) + } + + return nil + }) + + return repos, err +} + +func fetchProtoImageMeta(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageMeta, error) { + imageMetaBlob := imageBuck.Get([]byte(digest)) + + if len(imageMetaBlob) == 0 { + return nil, zerr.ErrImageMetaNotFound + } + + imageMeta := proto_go.ImageMeta{} + + err := proto.Unmarshal(imageMetaBlob, &imageMeta) + if err != nil { + return nil, err + } + + return &imageMeta, nil +} + +func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, +) ([]mTypes.FullImageMeta, error) { + images := []mTypes.FullImageMeta{} + + searchedRepo, searchedTag, err := common.GetRepoTag(searchText) + if err != nil { + return []mTypes.FullImageMeta{}, + fmt.Errorf("metadb: error while parsing search text, invalid format %w", err) + } + + err = bdw.DB.View(func(transaction *bbolt.Tx) error { + var ( + repoBuck = transaction.Bucket([]byte(RepoMetaBuck)) + imageBuck = transaction.Bucket([]byte(ImageMetaBuck)) + userBookmarks = getUserBookmarks(ctx, transaction) + userStars = getUserStars(ctx, transaction) + ) + + repoName, repoMetaBlob := repoBuck.Cursor().Seek([]byte(searchedRepo)) + + if string(repoName) != searchedRepo { + return nil + } + + if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { + return err + } + + protoRepoMeta := &proto_go.RepoMeta{} + + err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + if err != nil { + return err + } + + delete(protoRepoMeta.Tags, "") + + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) + + for tag, descriptor := range protoRepoMeta.Tags { + if !strings.HasPrefix(tag, searchedTag) || tag == "" { + continue + } + + var protoImageMeta *proto_go.ImageMeta + + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestDigest := descriptor.Digest + + imageManifestData, err := fetchProtoImageMeta(imageBuck, manifestDigest) + if err != nil { + return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", + manifestDigest, err) + } + + protoImageMeta = imageManifestData + case ispec.MediaTypeImageIndex: + indexDigest := descriptor.Digest + + imageIndexData, err := fetchProtoImageMeta(imageBuck, indexDigest) + if err != nil { + return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", + indexDigest, err) + } + + manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests)) + + for _, manifest := range imageIndexData.Index.Index.Manifests { + imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest) + if err != nil { + return err + } + + manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + } + + imageIndexData.Manifests = manifestDataList + + protoImageMeta = imageIndexData + default: + bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") + + continue + } + + images = append(images, mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, protoImageMeta)) + } + + return nil + }) + + return images, err +} + +func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, + filterFunc mTypes.FilterFunc, +) ([]mTypes.FullImageMeta, error) { + images := []mTypes.FullImageMeta{} + + err := bdw.DB.View(func(transaction *bbolt.Tx) error { + var ( + repoBuck = transaction.Bucket([]byte(RepoMetaBuck)) + imageMetaBuck = transaction.Bucket([]byte(ImageMetaBuck)) + userBookmarks = getUserBookmarks(ctx, transaction) + userStars = getUserStars(ctx, transaction) + viewError error + ) + + cursor := repoBuck.Cursor() + repoName, repoMetaBlob := cursor.First() + + for ; repoName != nil; repoName, repoMetaBlob = cursor.Next() { + if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { + continue + } + + protoRepoMeta := &proto_go.RepoMeta{} + + err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + if err != nil { + viewError = errors.Join(viewError, err) + + continue + } + + delete(protoRepoMeta.Tags, "") + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) + repoMeta := mConvert.GetRepoMeta(protoRepoMeta) + + for tag, descriptor := range protoRepoMeta.Tags { + if !filterRepoTag(string(repoName), tag) { + continue + } + + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestDigest := descriptor.Digest + + imageManifestData, err := fetchProtoImageMeta(imageMetaBuck, manifestDigest) + if err != nil { + viewError = errors.Join(viewError, err) + + continue + } + + imageMeta := mConvert.GetImageMeta(imageManifestData) + + if filterFunc(repoMeta, imageMeta) { + images = append(images, mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, imageManifestData)) + } + case ispec.MediaTypeImageIndex: + indexDigest := descriptor.Digest + + imageIndexData, err := fetchProtoImageMeta(imageMetaBuck, indexDigest) + if err != nil { + viewError = errors.Join(viewError, err) + + continue + } + + matchedManifests := []*proto_go.ManifestMeta{} + + for _, manifest := range imageIndexData.Index.Index.Manifests { + manifestDigest := manifest.Digest + + imageManifestData, err := fetchProtoImageMeta(imageMetaBuck, manifestDigest) + if err != nil { + viewError = errors.Join(viewError, err) + + continue + } + + imageMeta := mConvert.GetImageMeta(imageManifestData) + + if filterFunc(repoMeta, imageMeta) { + matchedManifests = append(matchedManifests, imageManifestData.Manifests[0]) + } + } + + if len(matchedManifests) > 0 { + imageIndexData.Manifests = matchedManifests + + images = append(images, mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, imageIndexData)) + } + default: + bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") + + continue + } + } + } + + return viewError + }) + + return images, err +} + +func (bdw *BoltDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRepoNameFunc, + filter mTypes.FilterFullRepoFunc, +) ([]mTypes.RepoMeta, error) { + repos := []mTypes.RepoMeta{} + + err := bdw.DB.View(func(tx *bbolt.Tx) error { + var ( + buck = tx.Bucket([]byte(RepoMetaBuck)) + cursor = buck.Cursor() + userBookmarks = getUserBookmarks(ctx, tx) + userStars = getUserStars(ctx, tx) + ) + + for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { + if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { + continue + } + + if !acceptName(string(repoName)) { + continue + } + + repoMeta := proto_go.RepoMeta{} + + err := proto.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) + repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) + + fullRepoMeta := mConvert.GetRepoMeta(&repoMeta) + + if filter(fullRepoMeta) { + repos = append(repos, fullRepoMeta) + } + } + + return nil + }) + if err != nil { + return []mTypes.RepoMeta{}, err + } + + return repos, err +} + +func (bdw *BoltDB) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + var protoRepoMeta proto_go.RepoMeta + + err := bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + userBookmarks := getUserBookmarks(ctx, tx) + userStars := getUserStars(ctx, tx) repoMetaBlob := buck.Get([]byte(repo)) // object not found - if len(repoMetaBlob) == 0 { - var err error + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound + } - // create a new object - repoMeta := mTypes.RepoMetadata{ - Name: repo, - Tags: map[string]mTypes.Descriptor{}, - Statistics: map[string]mTypes.DescriptorStatistics{}, - Signatures: map[string]mTypes.ManifestSignatures{}, - Referrers: map[string][]mTypes.ReferrerInfo{ - referredDigest.String(): { - referrer, - }, - }, + // object found + err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) + if err != nil { + return err + } + + delete(protoRepoMeta.Tags, "") + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repo) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, repo) + + return nil + }) + + return mConvert.GetRepoMeta(&protoRepoMeta), err +} + +func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string) (mTypes.FullImageMeta, error) { + protoRepoMeta := &proto_go.RepoMeta{} + protoImageMeta := &proto_go.ImageMeta{} + + err := bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + imageBuck := tx.Bucket([]byte(ImageMetaBuck)) + userBookmarks := getUserBookmarks(ctx, tx) + userStars := getUserStars(ctx, tx) + + repoMetaBlob := buck.Get([]byte(repo)) + + // object not found + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound + } + + // object found + err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + if err != nil { + return err + } + + delete(protoRepoMeta.Tags, "") + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repo) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, repo) + + descriptor, ok := protoRepoMeta.Tags[tag] + if !ok { + return zerr.ErrImageMetaNotFound + } + + protoImageMeta, err = fetchProtoImageMeta(imageBuck, descriptor.Digest) + if err != nil { + return err + } + + if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { + manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) + + for _, manifest := range protoImageMeta.Index.Index.Manifests { + imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest) + if err != nil { + return err + } + + manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) } - repoMetaBlob, err = json.Marshal(repoMeta) + protoImageMeta.Manifests = manifestDataList + } + + return nil + }) + + return mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, protoImageMeta), err +} + +func (bdw *BoltDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, error) { + imageMeta := mTypes.ImageMeta{} + + err := bdw.DB.View(func(tx *bbolt.Tx) error { + imageBuck := tx.Bucket([]byte(ImageMetaBuck)) + + protoImageMeta, err := fetchProtoImageMeta(imageBuck, digest.String()) + if err != nil { + return err + } + + if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { + manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) + + for _, manifest := range protoImageMeta.Index.Index.Manifests { + imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest) + if err != nil { + return err + } + + manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + } + + protoImageMeta.Manifests = manifestDataList + } + + imageMeta = mConvert.GetImageMeta(protoImageMeta) + + return nil + }) + + return imageMeta, err +} + +func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta mTypes.RepoMeta) bool, +) ([]mTypes.RepoMeta, error) { + foundRepos := []mTypes.RepoMeta{} + + err := bdw.DB.View(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + cursor := buck.Cursor() + + for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { + if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { + continue + } + + protoRepoMeta := proto_go.RepoMeta{} + + err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) + if err != nil { + return err + } + + delete(protoRepoMeta.Tags, "") + + repoMeta := mConvert.GetRepoMeta(&protoRepoMeta) + + if filter(repoMeta) { + foundRepos = append(foundRepos, repoMeta) + } + } + + return nil + }) + + return foundRepos, err +} + +func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, + sygMeta mTypes.SignatureMetadata, +) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + repoMetaBlob := buck.Get([]byte(repo)) + + if len(repoMetaBlob) == 0 { + var err error + // create a new object + repoMeta := proto_go.RepoMeta{ + Name: repo, + Tags: map[string]*proto_go.TagDescriptor{"": {}}, + Signatures: map[string]*proto_go.ManifestSignatures{ + signedManifestDigest.String(): { + Map: map[string]*proto_go.SignaturesInfo{ + sygMeta.SignatureType: { + List: []*proto_go.SignatureInfo{ + { + SignatureManifestDigest: sygMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), + }, + }, + }, + }, + }, + }, + Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, + Statistics: map[string]*proto_go.DescriptorStatistics{"": {}}, + } + + repoMetaBlob, err = proto.Marshal(&repoMeta) if err != nil { return err } return buck.Put([]byte(repo), repoMetaBlob) } - var repoMeta mTypes.RepoMetadata - err := json.Unmarshal(repoMetaBlob, &repoMeta) + protoRepoMeta := &proto_go.RepoMeta{} + + err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) if err != nil { return err } - referrers := repoMeta.Referrers[referredDigest.String()] + var ( + manifestSignatures *proto_go.ManifestSignatures + found bool + ) - for i := range referrers { - if referrers[i].Digest == referrer.Digest { - return nil + if manifestSignatures, found = protoRepoMeta.Signatures[signedManifestDigest.String()]; !found { + manifestSignatures = &proto_go.ManifestSignatures{Map: map[string]*proto_go.SignaturesInfo{"": {}}} + } + + signatureSlice := &proto_go.SignaturesInfo{List: []*proto_go.SignatureInfo{}} + if sigSlice, found := manifestSignatures.Map[sygMeta.SignatureType]; found { + signatureSlice = sigSlice + } + + if !common.ProtoSignatureAlreadyExists(signatureSlice.List, sygMeta) { + switch sygMeta.SignatureType { + case zcommon.NotationSignature: + signatureSlice.List = append(signatureSlice.List, &proto_go.SignatureInfo{ + SignatureManifestDigest: sygMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), + }) + case zcommon.CosignSignature: + signatureSlice.List = []*proto_go.SignatureInfo{{ + SignatureManifestDigest: sygMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), + }} } } - referrers = append(referrers, referrer) + manifestSignatures.Map[sygMeta.SignatureType] = signatureSlice + protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures - repoMeta.Referrers[referredDigest.String()] = referrers - - repoMetaBlob, err = json.Marshal(repoMeta) + repoMetaBlob, err = proto.Marshal(protoRepoMeta) if err != nil { return err } @@ -329,73 +851,201 @@ func (bdw BoltDB) SetReferrer(repo string, referredDigest godigest.Digest, refer return err } -func (bdw BoltDB) DeleteReferrer(repo string, referredDigest godigest.Digest, - referrerDigest godigest.Digest, +func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Digest, + sigMeta mTypes.SignatureMetadata, ) error { - return bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBlob := buck.Get([]byte(repo)) - - if len(repoMetaBlob) == 0 { - return zerr.ErrRepoMetaNotFound + if repoMetaBlob == nil { + return zerr.ErrManifestMetaNotFound } - var repoMeta mTypes.RepoMetadata + protoRepoMeta := proto_go.RepoMeta{} - err := json.Unmarshal(repoMetaBlob, &repoMeta) + err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) if err != nil { return err } - referrers := repoMeta.Referrers[referredDigest.String()] + manifestSignatures, found := protoRepoMeta.Signatures[signedManifestDigest.String()] + if !found { + return zerr.ErrManifestMetaNotFound + } - for i := range referrers { - if referrers[i].Digest == referrerDigest.String() { - referrers = append(referrers[:i], referrers[i+1:]...) + signatureSlice := manifestSignatures.Map[sigMeta.SignatureType] - break + newSignatureSlice := make([]*proto_go.SignatureInfo, 0, len(signatureSlice.List)) + + for _, sigInfo := range signatureSlice.List { + if sigInfo.SignatureManifestDigest != sigMeta.SignatureDigest { + newSignatureSlice = append(newSignatureSlice, sigInfo) } } - repoMeta.Referrers[referredDigest.String()] = referrers + manifestSignatures.Map[sigMeta.SignatureType] = &proto_go.SignaturesInfo{List: newSignatureSlice} - repoMetaBlob, err = json.Marshal(repoMeta) + protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures + + repoMetaBlob, err = proto.Marshal(&protoRepoMeta) if err != nil { return err } return buck.Put([]byte(repo), repoMetaBlob) }) + + return err } -func (bdw BoltDB) GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string, +func (bdw *BoltDB) IncrementRepoStars(repo string) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + repoMetaBlob := buck.Get([]byte(repo)) + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound + } + + var repoMeta proto_go.RepoMeta + + err := proto.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + repoMeta.Stars++ + + repoMetaBlob, err = proto.Marshal(&repoMeta) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) DecrementRepoStars(repo string) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + repoMetaBlob := buck.Get([]byte(repo)) + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound + } + + var repoMeta proto_go.RepoMeta + + err := proto.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + if repoMeta.Stars == 0 { + return nil + } + + repoMeta.Stars-- + + repoMetaBlob, err = proto.Marshal(&repoMeta) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) SetRepoMeta(repo string, repoMeta mTypes.RepoMeta) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + repoMeta.Name = repo + + repoMetaBlob, err := proto.Marshal(mConvert.GetProtoRepoMeta(repoMeta)) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) ResetRepoReferences(repo string) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + + repoMetaBlob := buck.Get([]byte(repo)) + + if repoMetaBlob == nil { + return nil + } + + protoRepoMeta := &proto_go.RepoMeta{} + + err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + if err != nil { + return err + } + + repoMetaBlob, err = proto.Marshal(&proto_go.RepoMeta{ + Name: repo, + Statistics: protoRepoMeta.Statistics, + Stars: protoRepoMeta.Stars, + Tags: map[string]*proto_go.TagDescriptor{"": {}}, + Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}}, + Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, + }) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string, ) ([]mTypes.ReferrerInfo, error) { referrersInfoResult := []mTypes.ReferrerInfo{} err := bdw.DB.View(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) + buck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBlob := buck.Get([]byte(repo)) if len(repoMetaBlob) == 0 { return zerr.ErrRepoMetaNotFound } - var repoMeta mTypes.RepoMetadata + var repoMeta proto_go.RepoMeta - err := json.Unmarshal(repoMetaBlob, &repoMeta) + err := proto.Unmarshal(repoMetaBlob, &repoMeta) if err != nil { return err } - referrersInfo := repoMeta.Referrers[referredDigest.String()] + referrersInfo := repoMeta.Referrers[referredDigest.String()].List for i := range referrersInfo { if !common.MatchesArtifactTypes(referrersInfo[i].ArtifactType, artifactTypes) { continue } - referrersInfoResult = append(referrersInfoResult, referrersInfo[i]) + referrersInfoResult = append(referrersInfoResult, mTypes.ReferrerInfo{ + Digest: referrersInfo[i].Digest, + MediaType: referrersInfo[i].MediaType, + ArtifactType: referrersInfo[i].ArtifactType, + Size: int(referrersInfo[i].Size), + Annotations: referrersInfo[i].Annotations, + }) } return nil @@ -404,34 +1054,191 @@ func (bdw BoltDB) GetReferrersInfo(repo string, referredDigest godigest.Digest, return referrersInfoResult, err } -/* - RemoveRepoReference removes the tag from RepoMetadata if the reference is a tag, - -it also removes its corresponding digest from Statistics, Signatures and Referrers if there are no tags -pointing to it. -If the reference is a digest then it will remove the digest from Statistics, Signatures and Referrers only -if there are no tags pointing to the digest, otherwise it's noop. -*/ -func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error { +func (bdw *BoltDB) IncrementImageDownloads(repo string, reference string) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) + buck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBlob := buck.Get([]byte(repo)) - - repoMeta := mTypes.RepoMetadata{ - Name: repo, - Tags: map[string]mTypes.Descriptor{}, - Statistics: map[string]mTypes.DescriptorStatistics{}, - Signatures: map[string]mTypes.ManifestSignatures{}, - Referrers: map[string][]mTypes.ReferrerInfo{}, + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound } - // object not found - if len(repoMetaBlob) > 0 { - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err + var repoMeta proto_go.RepoMeta + + err := proto.Unmarshal(repoMetaBlob, &repoMeta) + if err != nil { + return err + } + + manifestDigest := reference + + if !common.ReferenceIsDigest(reference) { + // search digest for tag + descriptor, found := repoMeta.Tags[reference] + + if !found { + return zerr.ErrManifestMetaNotFound } + + manifestDigest = descriptor.Digest + } + + manifestStatistics, ok := repoMeta.Statistics[manifestDigest] + if !ok { + return zerr.ErrManifestMetaNotFound + } + + manifestStatistics.DownloadCount++ + repoMeta.Statistics[manifestDigest] = manifestStatistics + + repoMetaBlob, err = proto.Marshal(&repoMeta) + if err != nil { + return err + } + + return buck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) UpdateSignaturesValidity(repo string, manifestDigest godigest.Digest) error { + err := bdw.DB.Update(func(transaction *bbolt.Tx) error { + imgTrustStore := bdw.ImageTrustStore() + + if imgTrustStore == nil { + return nil + } + + // get ManifestData of signed manifest + imageMetaBuck := transaction.Bucket([]byte(ImageMetaBuck)) + idBlob := imageMetaBuck.Get([]byte(manifestDigest)) + + if len(idBlob) == 0 { + // manifest meta not found, updating signatures with details about validity and author will not be performed + return nil + } + + protoImageMeta := proto_go.ImageMeta{} + + err := proto.Unmarshal(idBlob, &protoImageMeta) + if err != nil { + return err + } + + // update signatures with details about validity and author + repoBuck := transaction.Bucket([]byte(RepoMetaBuck)) + + repoMetaBlob := repoBuck.Get([]byte(repo)) + if repoMetaBlob == nil { + return zerr.ErrRepoMetaNotFound + } + + protoRepoMeta := proto_go.RepoMeta{} + + err = proto.Unmarshal(repoMetaBlob, &protoRepoMeta) + if err != nil { + return err + } + + manifestSignatures := proto_go.ManifestSignatures{Map: map[string]*proto_go.SignaturesInfo{"": {}}} + for sigType, sigs := range protoRepoMeta.Signatures[manifestDigest.String()].Map { + signaturesInfo := []*proto_go.SignatureInfo{} + + for _, sigInfo := range sigs.List { + layersInfo := []*proto_go.LayersInfo{} + + for _, layerInfo := range sigInfo.LayersInfo { + author, date, isTrusted, _ := imgTrustStore.VerifySignature(sigType, layerInfo.LayerContent, + layerInfo.SignatureKey, manifestDigest, mConvert.GetImageMeta(&protoImageMeta), repo) + + if isTrusted { + layerInfo.Signer = author + } + + if !date.IsZero() { + layerInfo.Signer = author + layerInfo.Date = timestamppb.New(date) + } + + layersInfo = append(layersInfo, layerInfo) + } + + signaturesInfo = append(signaturesInfo, &proto_go.SignatureInfo{ + SignatureManifestDigest: sigInfo.SignatureManifestDigest, + LayersInfo: layersInfo, + }) + } + + manifestSignatures.Map[sigType] = &proto_go.SignaturesInfo{List: signaturesInfo} + } + + protoRepoMeta.Signatures[manifestDigest.String()] = &manifestSignatures + + repoMetaBlob, err = proto.Marshal(&protoRepoMeta) + if err != nil { + return err + } + + return repoBuck.Put([]byte(repo), repoMetaBlob) + }) + + return err +} + +func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error { + err := bdw.DB.Update(func(tx *bbolt.Tx) error { + buck := tx.Bucket([]byte(RepoMetaBuck)) + imageMetaBuck := tx.Bucket([]byte(ImageMetaBuck)) + repoBlobsBuck := tx.Bucket([]byte(RepoBlobsBuck)) + + repoMetaBlob := buck.Get([]byte(repo)) + if repoMetaBlob == nil { + return nil + } + + repoMeta := &proto_go.RepoMeta{} + + err := proto.Unmarshal(repoMetaBlob, repoMeta) + if err != nil { + return err + } + + imageMeta, err := fetchProtoImageMeta(imageMetaBuck, manifestDigest.String()) + if err != nil { + if errors.Is(err, zerr.ErrImageMetaNotFound) { + return nil + } + + return err + } + + // Remove Referrers + if subject := mConvert.GetImageSubject(imageMeta); subject != nil { + referredDigest := subject.Digest.String() + refInfo := &proto_go.ReferrersInfo{} + + if repoMeta.Referrers[referredDigest] != nil { + refInfo = repoMeta.Referrers[referredDigest] + } + + referrers := refInfo.List + + for i := range referrers { + if referrers[i].Digest == manifestDigest.String() { + referrers[i].Count -= 1 + + if referrers[i].Count == 0 || common.ReferenceIsDigest(reference) { + referrers = append(referrers[:i], referrers[i+1:]...) + } + + break + } + } + + refInfo.List = referrers + + repoMeta.Referrers[referredDigest] = refInfo } if !common.ReferenceIsDigest(reference) { @@ -460,516 +1267,35 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go delete(repoMeta.Referrers, manifestDigest.String()) } - repoMetaBlob, err := json.Marshal(repoMeta) - if err != nil { - return err - } + repoBlobsBytes := repoBlobsBuck.Get([]byte(repoMeta.Name)) - return buck.Put([]byte(repo), repoMetaBlob) - }) + repoBlobs := &proto_go.RepoBlobs{} - return err -} - -func (bdw *BoltDB) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, - mediaType string, -) error { - if err := common.ValidateRepoReferenceInput(repo, reference, manifestDigest); err != nil { - return err - } - - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) - - repoMetaBlob := buck.Get([]byte(repo)) - - repoMeta := mTypes.RepoMetadata{ - Name: repo, - Tags: map[string]mTypes.Descriptor{}, - Statistics: map[string]mTypes.DescriptorStatistics{}, - Signatures: map[string]mTypes.ManifestSignatures{}, - Referrers: map[string][]mTypes.ReferrerInfo{}, - } - - // object not found - if len(repoMetaBlob) > 0 { - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - } - - if !common.ReferenceIsDigest(reference) { - repoMeta.Tags[reference] = mTypes.Descriptor{ - Digest: manifestDigest.String(), - MediaType: mediaType, - } - } - - if _, ok := repoMeta.Statistics[manifestDigest.String()]; !ok { - repoMeta.Statistics[manifestDigest.String()] = mTypes.DescriptorStatistics{DownloadCount: 0} - } - - if _, ok := repoMeta.Signatures[manifestDigest.String()]; !ok { - repoMeta.Signatures[manifestDigest.String()] = mTypes.ManifestSignatures{} - } - - if _, ok := repoMeta.Referrers[manifestDigest.String()]; !ok { - repoMeta.Referrers[manifestDigest.String()] = []mTypes.ReferrerInfo{} - } - - repoMetaBlob, err := json.Marshal(repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) GetRepoMeta(repo string) (mTypes.RepoMetadata, error) { - var repoMeta mTypes.RepoMetadata - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) - - repoMetaBlob := buck.Get([]byte(repo)) - - // object not found - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } - - // object found - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - return nil - }) - - return repoMeta, err -} - -func (bdw *BoltDB) GetUserRepoMeta(ctx context.Context, repo string) (mTypes.RepoMetadata, error) { - var repoMeta mTypes.RepoMetadata - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) - userBookmarks := getUserBookmarks(ctx, tx) - userStars := getUserStars(ctx, tx) - - repoMetaBlob := buck.Get([]byte(repo)) - - // object not found - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } - - // object found - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repo) - repoMeta.IsStarred = zcommon.Contains(userStars, repo) - - return nil - }) - - return repoMeta, err -} - -func (bdw *BoltDB) SetRepoMeta(repo string, repoMeta mTypes.RepoMetadata) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) - - repoMeta.Name = repo - - repoMetaBlob, err := json.Marshal(repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) DeleteRepoTag(repo string, tag string) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) - - repoMetaBlob := buck.Get([]byte(repo)) - - // object not found - if repoMetaBlob == nil { - return nil - } - - // object found - var repoMeta mTypes.RepoMetadata - - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - delete(repoMeta.Tags, tag) - - repoMetaBlob, err = json.Marshal(repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) IncrementRepoStars(repo string) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) - - repoMetaBlob := buck.Get([]byte(repo)) - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } - - var repoMeta mTypes.RepoMetadata - - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - repoMeta.Stars++ - - repoMetaBlob, err = json.Marshal(repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) DecrementRepoStars(repo string) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) - - repoMetaBlob := buck.Get([]byte(repo)) - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } - - var repoMeta mTypes.RepoMetadata - - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - if repoMeta.Stars > 0 { - repoMeta.Stars-- - } - - repoMetaBlob, err = json.Marshal(repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) GetRepoStars(repo string) (int, error) { - stars := 0 - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) - - buck.Get([]byte(repo)) - repoMetaBlob := buck.Get([]byte(repo)) - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } - - var repoMeta mTypes.RepoMetadata - - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - stars = repoMeta.Stars - - return nil - }) - - return stars, err -} - -func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool, -) ([]mTypes.RepoMetadata, error) { - foundRepos := []mTypes.RepoMetadata{} - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) - - cursor := buck.Cursor() - - for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { - if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { - continue - } - - repoMeta := mTypes.RepoMetadata{} - - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - if filter(repoMeta) { - foundRepos = append(foundRepos, repoMeta) - } - } - - return nil - }) - - return foundRepos, err -} - -func (bdw *BoltDB) IncrementImageDownloads(repo string, reference string) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) - - repoMetaBlob := buck.Get([]byte(repo)) - if repoMetaBlob == nil { - return zerr.ErrManifestMetaNotFound - } - - var repoMeta mTypes.RepoMetadata - - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - manifestDigest := reference - - if !common.ReferenceIsDigest(reference) { - // search digest for tag - descriptor, found := repoMeta.Tags[reference] - - if !found { - return zerr.ErrManifestMetaNotFound - } - - manifestDigest = descriptor.Digest - } - - manifestStatistics := repoMeta.Statistics[manifestDigest] - manifestStatistics.DownloadCount++ - repoMeta.Statistics[manifestDigest] = manifestStatistics - - repoMetaBlob, err = json.Marshal(repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) UpdateSignaturesValidity(repo string, manifestDigest godigest.Digest) error { - err := bdw.DB.Update(func(transaction *bbolt.Tx) error { - imgTrustStore := bdw.ImageTrustStore() - - if imgTrustStore == nil { - return nil - } - - // get ManifestData of signed manifest - manifestBuck := transaction.Bucket([]byte(ManifestDataBucket)) - mdBlob := manifestBuck.Get([]byte(manifestDigest)) - - var blob []byte - - if len(mdBlob) != 0 { - var manifestData mTypes.ManifestData - - err := json.Unmarshal(mdBlob, &manifestData) - if err != nil { - return fmt.Errorf("metadb: %w error while unmashaling manifest meta for digest %s", err, manifestDigest) - } - - blob = manifestData.ManifestBlob + if repoBlobsBytes == nil { + repoBlobs.Blobs = map[string]*proto_go.BlobInfo{} } else { - var indexData mTypes.IndexData - - indexBuck := transaction.Bucket([]byte(IndexDataBucket)) - idBlob := indexBuck.Get([]byte(manifestDigest)) - - if len(idBlob) == 0 { - // manifest meta not found, updating signatures with details about validity and author will not be performed - return nil - } - - err := json.Unmarshal(idBlob, &indexData) - if err != nil { - return fmt.Errorf("metadb: %w error while unmashaling index meta for digest %s", err, manifestDigest) - } - - blob = indexData.IndexBlob - } - - // update signatures with details about validity and author - repoBuck := transaction.Bucket([]byte(RepoMetadataBucket)) - - repoMetaBlob := repoBuck.Get([]byte(repo)) - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } - - var repoMeta mTypes.RepoMetadata - - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - manifestSignatures := mTypes.ManifestSignatures{} - for sigType, sigs := range repoMeta.Signatures[manifestDigest.String()] { - signaturesInfo := []mTypes.SignatureInfo{} - - for _, sigInfo := range sigs { - layersInfo := []mTypes.LayerInfo{} - - for _, layerInfo := range sigInfo.LayersInfo { - author, date, isTrusted, _ := imgTrustStore.VerifySignature(sigType, layerInfo.LayerContent, - layerInfo.SignatureKey, manifestDigest, blob, repo) - - if isTrusted { - layerInfo.Signer = author - } - - if !date.IsZero() { - layerInfo.Signer = author - layerInfo.Date = date - } - - layersInfo = append(layersInfo, layerInfo) - } - - signaturesInfo = append(signaturesInfo, mTypes.SignatureInfo{ - SignatureManifestDigest: sigInfo.SignatureManifestDigest, - LayersInfo: layersInfo, - }) - } - - manifestSignatures[sigType] = signaturesInfo - } - - repoMeta.Signatures[manifestDigest.String()] = manifestSignatures - - repoMetaBlob, err = json.Marshal(repoMeta) - if err != nil { - return err - } - - return repoBuck.Put([]byte(repo), repoMetaBlob) - }) - - return err -} - -func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, - sygMeta mTypes.SignatureMetadata, -) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) - - repoMetaBlob := buck.Get([]byte(repo)) - - if len(repoMetaBlob) == 0 { - var err error - // create a new object - repoMeta := mTypes.RepoMetadata{ - Name: repo, - Tags: map[string]mTypes.Descriptor{}, - Signatures: map[string]mTypes.ManifestSignatures{ - signedManifestDigest.String(): { - sygMeta.SignatureType: []mTypes.SignatureInfo{ - { - SignatureManifestDigest: sygMeta.SignatureDigest, - LayersInfo: sygMeta.LayersInfo, - }, - }, - }, - }, - Statistics: map[string]mTypes.DescriptorStatistics{}, - Referrers: map[string][]mTypes.ReferrerInfo{}, - } - - repoMetaBlob, err = json.Marshal(repoMeta) + err := proto.Unmarshal(repoBlobsBytes, repoBlobs) if err != nil { return err } - - return buck.Put([]byte(repo), repoMetaBlob) } - var repoMeta mTypes.RepoMetadata - - err := json.Unmarshal(repoMetaBlob, &repoMeta) + repoMeta, repoBlobs, err = common.RemoveImageFromRepoMeta(repoMeta, repoBlobs, reference) if err != nil { return err } - var ( - manifestSignatures mTypes.ManifestSignatures - found bool - ) - - if manifestSignatures, found = repoMeta.Signatures[signedManifestDigest.String()]; !found { - manifestSignatures = mTypes.ManifestSignatures{} + repoBlobsBytes, err = proto.Marshal(repoBlobs) + if err != nil { + return err } - signatureSlice := manifestSignatures[sygMeta.SignatureType] - if !common.SignatureAlreadyExists(signatureSlice, sygMeta) { - if sygMeta.SignatureType == zcommon.NotationSignature { - signatureSlice = append(signatureSlice, mTypes.SignatureInfo{ - SignatureManifestDigest: sygMeta.SignatureDigest, - LayersInfo: sygMeta.LayersInfo, - }) - } else if sygMeta.SignatureType == zcommon.CosignSignature { - signatureSlice = []mTypes.SignatureInfo{{ - SignatureManifestDigest: sygMeta.SignatureDigest, - LayersInfo: sygMeta.LayersInfo, - }} - } + err = repoBlobsBuck.Put([]byte(repoMeta.Name), repoBlobsBytes) + if err != nil { + return err } - manifestSignatures[sygMeta.SignatureType] = signatureSlice - - repoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures - - repoMetaBlob, err = json.Marshal(repoMeta) + repoMetaBlob, err = proto.Marshal(repoMeta) if err != nil { return err } @@ -980,518 +1306,12 @@ func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godige return err } -func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Digest, - sigMeta mTypes.SignatureMetadata, -) error { - err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetadataBucket)) - - repoMetaBlob := buck.Get([]byte(repo)) - if repoMetaBlob == nil { - return zerr.ErrManifestMetaNotFound - } - - var repoMeta mTypes.RepoMetadata - - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - sigType := sigMeta.SignatureType - - var ( - manifestSignatures mTypes.ManifestSignatures - found bool - ) - - if manifestSignatures, found = repoMeta.Signatures[signedManifestDigest.String()]; !found { - return zerr.ErrManifestMetaNotFound - } - - signatureSlice := manifestSignatures[sigType] - - newSignatureSlice := make([]mTypes.SignatureInfo, 0, len(signatureSlice)-1) - - for _, sigDigest := range signatureSlice { - if sigDigest.SignatureManifestDigest != sigMeta.SignatureDigest { - newSignatureSlice = append(newSignatureSlice, sigDigest) - } - } - - manifestSignatures[sigType] = newSignatureSlice - - repoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures - - repoMetaBlob, err = json.Marshal(repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) - }) - - return err +func (bdw *BoltDB) ImageTrustStore() mTypes.ImageTrustStore { + return bdw.imgTrustStore } -func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { - var ( - foundRepos = make([]mTypes.RepoMetadata, 0) - manifestMetadataMap = make(map[string]mTypes.ManifestMetadata) - indexDataMap = make(map[string]mTypes.IndexData) - ) - - err := bdw.DB.View(func(transaction *bbolt.Tx) error { - var ( - repoBuck = transaction.Bucket([]byte(RepoMetadataBucket)) - indexBuck = transaction.Bucket([]byte(IndexDataBucket)) - manifestBuck = transaction.Bucket([]byte(ManifestDataBucket)) - userBookmarks = getUserBookmarks(ctx, transaction) - userStars = getUserStars(ctx, transaction) - ) - - cursor := repoBuck.Cursor() - - for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { - if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { - continue - } - - var repoMeta mTypes.RepoMetadata - - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - rank := common.RankRepoName(searchText, repoMeta.Name) - if rank == -1 { - continue - } - - repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) - repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) - repoMeta.Rank = rank - - for tag, descriptor := range repoMeta.Tags { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDigest := descriptor.Digest - - manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, - manifestMetadataMap, manifestBuck) - if err != nil { - return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", - manifestDigest, err) - } - - manifestMetadataMap[descriptor.Digest] = manifestMeta - case ispec.MediaTypeImageIndex: - indexDigest := descriptor.Digest - - indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck) - if err != nil { - return fmt.Errorf("metadb: error fetching index data for index with digest %s %w", - indexDigest, err) - } - - var indexContent ispec.Index - - err = json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return fmt.Errorf("metadb: error while unmashaling index content for %s:%s %w", - repoName, tag, err) - } - - for _, manifest := range indexContent.Manifests { - manifestDigest := manifest.Digest - - manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest.String(), - manifestMetadataMap, manifestBuck) - if err != nil { - return err - } - - manifestMetadataMap[manifest.Digest.String()] = manifestMeta - } - - indexDataMap[indexDigest] = indexData - default: - bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") - - continue - } - } - - foundRepos = append(foundRepos, repoMeta) - } - - return nil - }) - - return foundRepos, manifestMetadataMap, indexDataMap, err -} - -func fetchManifestMetaWithCheck(repoMeta mTypes.RepoMetadata, manifestDigest string, - manifestMetadataMap map[string]mTypes.ManifestMetadata, manifestBuck *bbolt.Bucket, -) (mTypes.ManifestMetadata, error) { - manifestMeta, manifestDownloaded := manifestMetadataMap[manifestDigest] - - if !manifestDownloaded { - var manifestData mTypes.ManifestData - - manifestDataBlob := manifestBuck.Get([]byte(manifestDigest)) - if manifestDataBlob == nil { - return mTypes.ManifestMetadata{}, zerr.ErrManifestMetaNotFound - } - - err := json.Unmarshal(manifestDataBlob, &manifestData) - if err != nil { - return mTypes.ManifestMetadata{}, fmt.Errorf("metadb: error while unmarshaling manifest metadata for digest %s %w", - manifestDigest, err) - } - - manifestMeta = NewManifestMetadata(manifestDigest, repoMeta, manifestData) - } - - return manifestMeta, nil -} - -func fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]mTypes.IndexData, - indexBuck *bbolt.Bucket, -) (mTypes.IndexData, error) { - var ( - indexData mTypes.IndexData - err error - ) - - indexData, indexExists := indexDataMap[indexDigest] - - if !indexExists { - indexDataBlob := indexBuck.Get([]byte(indexDigest)) - if indexDataBlob == nil { - return mTypes.IndexData{}, zerr.ErrIndexDataNotFount - } - - err := json.Unmarshal(indexDataBlob, &indexData) - if err != nil { - return mTypes.IndexData{}, - fmt.Errorf("metadb: error while unmashaling index data for digest %s %w", indexDigest, err) - } - } - - return indexData, err -} - -func NewManifestMetadata(manifestDigest string, repoMeta mTypes.RepoMetadata, - manifestData mTypes.ManifestData, -) mTypes.ManifestMetadata { - manifestMeta := mTypes.ManifestMetadata{ - ManifestBlob: manifestData.ManifestBlob, - ConfigBlob: manifestData.ConfigBlob, - } - - manifestMeta.DownloadCount = repoMeta.Statistics[manifestDigest].DownloadCount - - manifestMeta.Signatures = mTypes.ManifestSignatures{} - if repoMeta.Signatures[manifestDigest] != nil { - manifestMeta.Signatures = repoMeta.Signatures[manifestDigest] - } - - return manifestMeta -} - -func (bdw *BoltDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error, -) { - var ( - foundRepos = make([]mTypes.RepoMetadata, 0) - manifestMetadataMap = make(map[string]mTypes.ManifestMetadata) - indexDataMap = make(map[string]mTypes.IndexData) - ) - - err := bdw.DB.View(func(transaction *bbolt.Tx) error { - var ( - repoBuck = transaction.Bucket([]byte(RepoMetadataBucket)) - indexBuck = transaction.Bucket([]byte(IndexDataBucket)) - manifestBuck = transaction.Bucket([]byte(ManifestDataBucket)) - cursor = repoBuck.Cursor() - userBookmarks = getUserBookmarks(ctx, transaction) - userStars = getUserStars(ctx, transaction) - viewError error - ) - - repoName, repoMetaBlob := cursor.First() - - for ; repoName != nil; repoName, repoMetaBlob = cursor.Next() { - if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { - continue - } - - repoMeta := mTypes.RepoMetadata{} - - if err := json.Unmarshal(repoMetaBlob, &repoMeta); err != nil { - viewError = errors.Join(viewError, err) - - continue - } - - repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) - repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) - - matchedTags := make(map[string]mTypes.Descriptor) - // take all manifestsMeta - for tag, descriptor := range repoMeta.Tags { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDigest := descriptor.Digest - - manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck) - if err != nil { - err = fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", manifestDigest, err) - viewError = errors.Join(viewError, err) - - continue - } - - if filterFunc(repoMeta, manifestMeta) { - matchedTags[tag] = descriptor - manifestMetadataMap[manifestDigest] = manifestMeta - } - case ispec.MediaTypeImageIndex: - indexDigest := descriptor.Digest - - indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck) - if err != nil { - err = fmt.Errorf("metadb: error while getting index data for digest %s %w", indexDigest, err) - viewError = errors.Join(viewError, err) - - continue - } - - var indexContent ispec.Index - - err = json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - err = fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err) - viewError = errors.Join(viewError, err) - - continue - } - - matchedManifests := []ispec.Descriptor{} - - for _, manifest := range indexContent.Manifests { - manifestDigest := manifest.Digest.String() - - manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck) - if err != nil { - err = fmt.Errorf("metadb: error while getting manifest data for digest %s %w", manifestDigest, err) - viewError = errors.Join(viewError, err) - - continue - } - - if filterFunc(repoMeta, manifestMeta) { - matchedManifests = append(matchedManifests, manifest) - manifestMetadataMap[manifestDigest] = manifestMeta - } - } - - if len(matchedManifests) > 0 { - indexContent.Manifests = matchedManifests - - indexBlob, err := json.Marshal(indexContent) - if err != nil { - viewError = errors.Join(viewError, err) - - continue - } - - indexData.IndexBlob = indexBlob - - indexDataMap[indexDigest] = indexData - matchedTags[tag] = descriptor - } - default: - bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") - - continue - } - } - - if len(matchedTags) == 0 { - continue - } - - repoMeta.Tags = matchedTags - - foundRepos = append(foundRepos, repoMeta) - } - - return viewError - }) - - return foundRepos, manifestMetadataMap, indexDataMap, err -} - -func (bdw *BoltDB) FilterRepos(ctx context.Context, filter mTypes.FilterRepoFunc) ( - []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error, -) { - foundRepos := make([]mTypes.RepoMetadata, 0) - - err := bdw.DB.View(func(tx *bbolt.Tx) error { - var ( - buck = tx.Bucket([]byte(RepoMetadataBucket)) - cursor = buck.Cursor() - userBookmarks = getUserBookmarks(ctx, tx) - userStars = getUserStars(ctx, tx) - ) - - for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { - if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { - continue - } - - repoMeta := mTypes.RepoMetadata{} - - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) - repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) - - if filter(repoMeta) { - foundRepos = append(foundRepos, repoMeta) - } - } - - return nil - }) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, err - } - - foundManifestMetadataMap, foundIndexDataMap, err := common.FetchDataForRepos(bdw, foundRepos) - - return foundRepos, foundManifestMetadataMap, foundIndexDataMap, err -} - -func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { - var ( - foundRepos = make([]mTypes.RepoMetadata, 0) - manifestMetadataMap = make(map[string]mTypes.ManifestMetadata) - indexDataMap = make(map[string]mTypes.IndexData) - ) - - searchedRepo, searchedTag, err := common.GetRepoTag(searchText) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - fmt.Errorf("metadb: error while parsing search text, invalid format %w", err) - } - - err = bdw.DB.View(func(transaction *bbolt.Tx) error { - var ( - repoBuck = transaction.Bucket([]byte(RepoMetadataBucket)) - indexBuck = transaction.Bucket([]byte(IndexDataBucket)) - manifestBuck = transaction.Bucket([]byte(ManifestDataBucket)) - userBookmarks = getUserBookmarks(ctx, transaction) - userStars = getUserStars(ctx, transaction) - ) - - repoName, repoMetaBlob := repoBuck.Cursor().Seek([]byte(searchedRepo)) - - if string(repoName) != searchedRepo { - return nil - } - - if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { - return err - } - - repoMeta := mTypes.RepoMetadata{} - - err := json.Unmarshal(repoMetaBlob, &repoMeta) - if err != nil { - return err - } - - repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) - repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) - - matchedTags := make(map[string]mTypes.Descriptor) - - for tag, descriptor := range repoMeta.Tags { - if !strings.HasPrefix(tag, searchedTag) { - continue - } - - matchedTags[tag] = descriptor - - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDigest := descriptor.Digest - - manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck) - if err != nil { - return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", - manifestDigest, err) - } - - manifestMetadataMap[descriptor.Digest] = manifestMeta - case ispec.MediaTypeImageIndex: - indexDigest := descriptor.Digest - - indexData, err := fetchIndexDataWithCheck(indexDigest, indexDataMap, indexBuck) - if err != nil { - return fmt.Errorf("metadb: error fetching index data for index with digest %s %w", - indexDigest, err) - } - - var indexContent ispec.Index - - err = json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return fmt.Errorf("metadb: error collecting filter data for index with digest %s %w", - indexDigest, err) - } - - for _, manifest := range indexContent.Manifests { - manifestDigest := manifest.Digest.String() - - manifestMeta, err := fetchManifestMetaWithCheck(repoMeta, manifestDigest, manifestMetadataMap, manifestBuck) - if err != nil { - return fmt.Errorf("metadb: error fetching from db manifest meta for manifest with digest %s %w", - manifestDigest, err) - } - - manifestMetadataMap[manifestDigest] = manifestMeta - } - - indexDataMap[indexDigest] = indexData - default: - bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") - - continue - } - } - - if len(matchedTags) == 0 { - return nil - } - - repoMeta.Tags = matchedTags - - foundRepos = append(foundRepos, repoMeta) - - return nil - }) - - return foundRepos, manifestMetadataMap, indexDataMap, err +func (bdw *BoltDB) SetImageTrustStore(imgTrustStore mTypes.ImageTrustStore) { + bdw.imgTrustStore = imgTrustStore } func (bdw *BoltDB) ToggleStarRepo(ctx context.Context, repo string) (mTypes.ToggleState, error) { @@ -1531,16 +1351,16 @@ func (bdw *BoltDB) ToggleStarRepo(ctx context.Context, repo string) (mTypes.Togg return err } - repoBuck := tx.Bucket([]byte(RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBlob := repoBuck.Get([]byte(repo)) if repoMetaBlob == nil { return zerr.ErrRepoMetaNotFound } - var repoMeta mTypes.RepoMetadata + repoMeta := &proto_go.RepoMeta{} - err = json.Unmarshal(repoMetaBlob, &repoMeta) + err = proto.Unmarshal(repoMetaBlob, repoMeta) if err != nil { return err } @@ -1552,7 +1372,7 @@ func (bdw *BoltDB) ToggleStarRepo(ctx context.Context, repo string) (mTypes.Togg repoMeta.Stars-- } - repoMetaBlob, err = json.Marshal(repoMeta) + repoMetaBlob, err = proto.Marshal(repoMeta) if err != nil { return err } @@ -2087,3 +1907,51 @@ func (bdw *BoltDB) DeleteUserData(ctx context.Context) error { return err } + +func (bdw *BoltDB) ResetDB() error { + err := bdw.DB.Update(func(transaction *bbolt.Tx) error { + err := resetBucket(transaction, RepoMetaBuck) + if err != nil { + return err + } + + err = resetBucket(transaction, ImageMetaBuck) + if err != nil { + return err + } + + err = resetBucket(transaction, RepoBlobsBuck) + if err != nil { + return err + } + + err = resetBucket(transaction, UserAPIKeysBucket) + if err != nil { + return err + } + + err = resetBucket(transaction, UserDataBucket) + if err != nil { + return err + } + + return nil + }) + + return err +} + +func resetBucket(transaction *bbolt.Tx, bucketName string) error { + bucket := transaction.Bucket([]byte(bucketName)) + + if bucket != nil { + err := transaction.DeleteBucket([]byte(bucketName)) + if err != nil { + return err + } + } + + _, err := transaction.CreateBucketIfNotExists([]byte(bucketName)) + + return err +} diff --git a/pkg/meta/boltdb/boltdb_test.go b/pkg/meta/boltdb/boltdb_test.go index 4729e5a8..f606df1d 100644 --- a/pkg/meta/boltdb/boltdb_test.go +++ b/pkg/meta/boltdb/boltdb_test.go @@ -4,29 +4,25 @@ import ( "context" "crypto/rand" "encoding/base64" - "encoding/json" "math" "testing" "time" "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" "go.etcd.io/bbolt" zerr "zotregistry.io/zot/errors" - zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/boltdb" mTypes "zotregistry.io/zot/pkg/meta/types" reqCtx "zotregistry.io/zot/pkg/requestcontext" - . "zotregistry.io/zot/pkg/test/image-utils" ) type imgTrustStore struct{} func (its imgTrustStore) VerifySignature( - signatureType string, rawSignature []byte, sigKey string, manifestDigest digest.Digest, manifestContent []byte, + signatureType string, rawSignature []byte, sigKey string, manifestDigest digest.Digest, imageMeta mTypes.ImageMeta, repo string, ) (string, time.Time, bool, error) { return "", time.Time{}, false, nil @@ -47,14 +43,6 @@ func TestWrapperErrors(t *testing.T) { boltdbWrapper.SetImageTrustStore(imgTrustStore{}) - repoMeta := mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{}, - Signatures: map[string]mTypes.ManifestSignatures{}, - } - - repoMetaBlob, err := json.Marshal(repoMeta) - So(err, ShouldBeNil) - userAc := reqCtx.NewUserAccessControl() userAc.SetUsername("test") @@ -245,604 +233,6 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("GetManifestData", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - dataBuck := tx.Bucket([]byte(boltdb.ManifestDataBucket)) - - return dataBuck.Put([]byte("digest1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - _, err = boltdbWrapper.GetManifestData("digest1") - So(err, ShouldNotBeNil) - - _, err = boltdbWrapper.GetManifestMeta("repo1", "digest1") - So(err, ShouldNotBeNil) - }) - - Convey("SetManifestMeta", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - dataBuck := tx.Bucket([]byte(boltdb.ManifestDataBucket)) - - err := dataBuck.Put([]byte("digest1"), repoMetaBlob) - if err != nil { - return err - } - - return repoBuck.Put([]byte("repo1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetManifestMeta("repo1", "digest1", mTypes.ManifestMetadata{}) - So(err, ShouldNotBeNil) - - _, err = boltdbWrapper.GetManifestMeta("repo1", "digest1") - So(err, ShouldNotBeNil) - }) - - Convey("FilterRepos", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - err := buck.Put([]byte("badRepo"), []byte("bad repo")) - So(err, ShouldBeNil) - - return nil - }) - So(err, ShouldBeNil) - - _, _, _, err = boltdbWrapper.FilterRepos(context.Background(), - func(repoMeta mTypes.RepoMetadata) bool { return true }) - So(err, ShouldNotBeNil) - }) - - Convey("SetReferrer", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetReferrer("repo", "ref", mTypes.ReferrerInfo{}) - So(err, ShouldNotBeNil) - }) - - Convey("DeleteReferrer", func() { - Convey("RepoMeta not found", func() { - err := boltdbWrapper.DeleteReferrer("r", "dig", "dig") - So(err, ShouldNotBeNil) - }) - - Convey("bad repo meta blob", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.DeleteReferrer("repo", "dig", "dig") - So(err, ShouldNotBeNil) - }) - }) - - Convey("SetRepoReference", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetRepoReference("repo1", "tag", "digest", ispec.MediaTypeImageManifest) - So(err, ShouldNotBeNil) - }) - - Convey("GetRepoMeta", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - _, err = boltdbWrapper.GetRepoMeta("repo1") - So(err, ShouldNotBeNil) - }) - - Convey("DeleteRepoTag", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.DeleteRepoTag("repo1", "tag") - So(err, ShouldNotBeNil) - }) - - Convey("GetReferrersInfo", func() { - _, err = boltdbWrapper.GetReferrersInfo("repo1", "tag", nil) - So(err, ShouldNotBeNil) - - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - _, err = boltdbWrapper.GetReferrersInfo("repo1", "tag", nil) - So(err, ShouldNotBeNil) - }) - - Convey("IncrementRepoStars", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.IncrementRepoStars("repo2") - So(err, ShouldNotBeNil) - - err = boltdbWrapper.IncrementRepoStars("repo1") - So(err, ShouldNotBeNil) - }) - - Convey("DecrementRepoStars", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.DecrementRepoStars("repo2") - So(err, ShouldNotBeNil) - - err = boltdbWrapper.DecrementRepoStars("repo1") - So(err, ShouldNotBeNil) - }) - - Convey("GetRepoStars", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - _, err = boltdbWrapper.GetRepoStars("repo1") - So(err, ShouldNotBeNil) - }) - - Convey("GetMultipleRepoMeta", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - _, err = boltdbWrapper.GetMultipleRepoMeta(context.TODO(), func(repoMeta mTypes.RepoMetadata) bool { - return true - }) - So(err, ShouldNotBeNil) - }) - - Convey("IncrementImageDownloads", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.IncrementImageDownloads("repo2", "tag") - So(err, ShouldNotBeNil) - - err = boltdbWrapper.IncrementImageDownloads("repo1", "tag") - So(err, ShouldNotBeNil) - - err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), repoMetaBlob) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.IncrementImageDownloads("repo1", "tag") - So(err, ShouldNotBeNil) - }) - - Convey("AddManifestSignature", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.AddManifestSignature("repo1", digest.FromString("dig"), - mTypes.SignatureMetadata{}) - So(err, ShouldNotBeNil) - - err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), repoMetaBlob) - }) - So(err, ShouldBeNil) - - // signatures not found - err = boltdbWrapper.AddManifestSignature("repo1", digest.FromString("dig"), - mTypes.SignatureMetadata{}) - So(err, ShouldBeNil) - - // - err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - repoMeta := mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{}, - Signatures: map[string]mTypes.ManifestSignatures{ - "digest1": { - "cosgin": {{}}, - }, - "digest2": { - "notation": {{}}, - }, - }, - } - - repoMetaBlob, err := json.Marshal(repoMeta) - So(err, ShouldBeNil) - - return repoBuck.Put([]byte("repo1"), repoMetaBlob) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.AddManifestSignature("repo1", digest.FromString("dig"), - mTypes.SignatureMetadata{ - SignatureType: "cosign", - SignatureDigest: "digest1", - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.AddManifestSignature("repo1", digest.FromString("dig"), - mTypes.SignatureMetadata{ - SignatureType: "cosign", - SignatureDigest: "digest2", - }) - So(err, ShouldBeNil) - - repoData, err := boltdbWrapper.GetRepoMeta("repo1") - So(err, ShouldBeNil) - So(len(repoData.Signatures[string(digest.FromString("dig"))][zcommon.CosignSignature]), - ShouldEqual, 1) - So(repoData.Signatures[string(digest.FromString("dig"))][zcommon.CosignSignature][0].SignatureManifestDigest, - ShouldEqual, "digest2") - - err = boltdbWrapper.AddManifestSignature("repo1", digest.FromString("dig"), - mTypes.SignatureMetadata{ - SignatureType: "notation", - SignatureDigest: "digest2", - }) - So(err, ShouldBeNil) - }) - - Convey("DeleteSignature", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.DeleteSignature("repo2", digest.FromString("dig"), - mTypes.SignatureMetadata{}) - So(err, ShouldNotBeNil) - - err = boltdbWrapper.DeleteSignature("repo1", digest.FromString("dig"), - mTypes.SignatureMetadata{}) - So(err, ShouldNotBeNil) - - err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - repoMeta := mTypes.RepoMetadata{ - Tags: map[string]mTypes.Descriptor{}, - Signatures: map[string]mTypes.ManifestSignatures{ - "digest1": { - "cosgin": []mTypes.SignatureInfo{ - { - SignatureManifestDigest: "sigDigest1", - }, - { - SignatureManifestDigest: "sigDigest2", - }, - }, - }, - "digest2": { - "notation": {{}}, - }, - }, - } - - repoMetaBlob, err := json.Marshal(repoMeta) - So(err, ShouldBeNil) - - return repoBuck.Put([]byte("repo1"), repoMetaBlob) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.DeleteSignature("repo1", "digest1", - mTypes.SignatureMetadata{ - SignatureType: "cosgin", - SignatureDigest: "sigDigest2", - }) - So(err, ShouldBeNil) - }) - - Convey("SearchRepos", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "") - So(err, ShouldNotBeNil) - - err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - dataBuck := tx.Bucket([]byte(boltdb.ManifestDataBucket)) - - err := dataBuck.Put([]byte("dig1"), []byte("wrong json")) - if err != nil { - return err - } - - repoMeta := mTypes.RepoMetadata{ - Name: "repo1", - Tags: map[string]mTypes.Descriptor{ - "tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest}, - }, - Signatures: map[string]mTypes.ManifestSignatures{}, - } - repoMetaBlob, err := json.Marshal(repoMeta) - So(err, ShouldBeNil) - - err = repoBuck.Put([]byte("repo1"), repoMetaBlob) - if err != nil { - return err - } - - repoMeta = mTypes.RepoMetadata{ - Name: "repo2", - Tags: map[string]mTypes.Descriptor{ - "tag2": {Digest: "dig2", MediaType: ispec.MediaTypeImageManifest}, - }, - Signatures: map[string]mTypes.ManifestSignatures{}, - } - repoMetaBlob, err = json.Marshal(repoMeta) - So(err, ShouldBeNil) - - return repoBuck.Put([]byte("repo2"), repoMetaBlob) - }) - So(err, ShouldBeNil) - - _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo1") - So(err, ShouldNotBeNil) - - _, _, _, err = boltdbWrapper.SearchRepos(context.Background(), "repo2") - So(err, ShouldNotBeNil) - }) - - Convey("Index Errors", func() { - Convey("Bad index data", func() { - indexDigest := digest.FromString("indexDigest") - - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck - So(err, ShouldBeNil) - - err = setBadIndexData(boltdbWrapper.DB, indexDigest.String()) - So(err, ShouldBeNil) - - _, _, _, err = boltdbWrapper.SearchRepos(ctx, "") - So(err, ShouldNotBeNil) - - _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:") - So(err, ShouldNotBeNil) - }) - - Convey("Bad indexBlob in IndexData", func() { - indexDigest := digest.FromString("indexDigest") - - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck - So(err, ShouldBeNil) - - err = boltdbWrapper.SetIndexData(indexDigest, mTypes.IndexData{ - IndexBlob: []byte("bad json"), - }) - So(err, ShouldBeNil) - - _, _, _, err = boltdbWrapper.SearchRepos(ctx, "") - So(err, ShouldNotBeNil) - - _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:") - So(err, ShouldNotBeNil) - }) - }) - - Convey("SearchTags", func() { - ctx := context.Background() - - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - _, _, _, err = boltdbWrapper.SearchTags(ctx, "") - So(err, ShouldNotBeNil) - - _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:") - So(err, ShouldNotBeNil) - - err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - dataBuck := tx.Bucket([]byte(boltdb.ManifestDataBucket)) - - manifestMeta := mTypes.ManifestMetadata{ - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("wrong json"), - Signatures: mTypes.ManifestSignatures{}, - } - - manifestMetaBlob, err := json.Marshal(manifestMeta) - if err != nil { - return err - } - - err = dataBuck.Put([]byte("dig1"), manifestMetaBlob) - if err != nil { - return err - } - - err = dataBuck.Put([]byte("wrongManifestData"), []byte("wrong json")) - if err != nil { - return err - } - - // manifest data doesn't exist - repoMeta = mTypes.RepoMetadata{ - Name: "repo1", - Tags: map[string]mTypes.Descriptor{ - "tag2": {Digest: "dig2", MediaType: ispec.MediaTypeImageManifest}, - }, - Signatures: map[string]mTypes.ManifestSignatures{}, - } - repoMetaBlob, err = json.Marshal(repoMeta) - So(err, ShouldBeNil) - - err = repoBuck.Put([]byte("repo1"), repoMetaBlob) - if err != nil { - return err - } - - // manifest data is wrong - repoMeta = mTypes.RepoMetadata{ - Name: "repo2", - Tags: map[string]mTypes.Descriptor{ - "tag2": {Digest: "wrongManifestData", MediaType: ispec.MediaTypeImageManifest}, - }, - Signatures: map[string]mTypes.ManifestSignatures{}, - } - repoMetaBlob, err = json.Marshal(repoMeta) - So(err, ShouldBeNil) - - err = repoBuck.Put([]byte("repo2"), repoMetaBlob) - if err != nil { - return err - } - - repoMeta = mTypes.RepoMetadata{ - Name: "repo3", - Tags: map[string]mTypes.Descriptor{ - "tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest}, - }, - Signatures: map[string]mTypes.ManifestSignatures{}, - } - repoMetaBlob, err = json.Marshal(repoMeta) - So(err, ShouldBeNil) - - return repoBuck.Put([]byte("repo3"), repoMetaBlob) - }) - So(err, ShouldBeNil) - - _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo1:") - So(err, ShouldNotBeNil) - - _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo2:") - So(err, ShouldNotBeNil) - }) - - Convey("FilterTags Index errors", func() { - Convey("FilterTags bad IndexData", func() { - indexDigest := digest.FromString("indexDigest") - - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck - So(err, ShouldBeNil) - - err = setBadIndexData(boltdbWrapper.DB, indexDigest.String()) - So(err, ShouldBeNil) - - _, _, _, err = boltdbWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) - So(err, ShouldNotBeNil) - }) - - Convey("FilterTags bad indexBlob in IndexData", func() { - indexDigest := digest.FromString("indexDigest") - - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck - So(err, ShouldBeNil) - - err = boltdbWrapper.SetIndexData(indexDigest, mTypes.IndexData{ - IndexBlob: []byte("bad json"), - }) - So(err, ShouldBeNil) - - _, _, _, err = boltdbWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) - So(err, ShouldNotBeNil) - }) - - Convey("FilterTags didn't match any index manifest", func() { - var ( - indexDigest = digest.FromString("indexDigest") - manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1") - manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2") - ) - - err := boltdbWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck - So(err, ShouldBeNil) - - indexBlob, err := GetIndexBlobWithManifests([]digest.Digest{ - manifestDigestFromIndex1, manifestDigestFromIndex2, - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetIndexData(indexDigest, mTypes.IndexData{ - IndexBlob: indexBlob, - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("{}"), - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("{}"), - }) - So(err, ShouldBeNil) - - _, _, _, err = boltdbWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return false }) - So(err, ShouldBeNil) - }) - }) - Convey("ToggleStarRepo bad context errors", func() { uacKey := reqCtx.GetContextKey() ctx := context.WithValue(context.Background(), uacKey, "bad context") @@ -860,7 +250,7 @@ func TestWrapperErrors(t *testing.T) { ctx := userAc.DeriveContext(context.Background()) err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(boltdb.RepoMetaBuck)) err := repoBuck.Put([]byte("repo"), []byte("bad repo")) So(err, ShouldBeNil) @@ -976,23 +366,6 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("Unsuported type", func() { - digest := digest.FromString("digest") - - err := boltdbWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck - So(err, ShouldBeNil) - - _, _, _, err = boltdbWrapper.SearchRepos(ctx, "") - So(err, ShouldBeNil) - - _, _, _, err = boltdbWrapper.SearchTags(ctx, "repo:") - So(err, ShouldBeNil) - - _, _, _, err = boltdbWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) - So(err, ShouldBeNil) - }) - Convey("GetUserRepoMeta unmarshal error", func() { userAc := reqCtx.NewUserAccessControl() userAc.SetUsername("username") @@ -1002,7 +375,7 @@ func TestWrapperErrors(t *testing.T) { ctx := userAc.DeriveContext(context.Background()) err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) + repoBuck := tx.Bucket([]byte(boltdb.RepoMetaBuck)) err := repoBuck.Put([]byte("repo"), []byte("bad repo")) So(err, ShouldBeNil) @@ -1011,78 +384,8 @@ func TestWrapperErrors(t *testing.T) { }) So(err, ShouldBeNil) - _, err := boltdbWrapper.GetUserRepoMeta(ctx, "repo") + _, err := boltdbWrapper.GetRepoMeta(ctx, "repo") So(err, ShouldNotBeNil) }) - - Convey("UpdateSignaturesValidity", func() { - Convey("manifestMeta of signed manifest not found", func() { - err := boltdbWrapper.UpdateSignaturesValidity("repo", digest.FromString("dig")) - So(err, ShouldBeNil) - }) - - Convey("repoMeta of signed manifest not found", func() { - // repo Meta not found - err := boltdbWrapper.SetManifestData(digest.FromString("dig"), mTypes.ManifestData{ - ManifestBlob: []byte("Bad Manifest"), - ConfigBlob: []byte("Bad Manifest"), - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.UpdateSignaturesValidity("repo", digest.FromString("dig")) - So(err, ShouldNotBeNil) - }) - - Convey("manifest - bad content", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - dataBuck := tx.Bucket([]byte(boltdb.ManifestDataBucket)) - - return dataBuck.Put([]byte("digest1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.UpdateSignaturesValidity("repo1", "digest1") - So(err, ShouldNotBeNil) - }) - - Convey("index - bad content", func() { - err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - dataBuck := tx.Bucket([]byte(boltdb.IndexDataBucket)) - - return dataBuck.Put([]byte("digest1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.UpdateSignaturesValidity("repo1", "digest1") - So(err, ShouldNotBeNil) - }) - - Convey("repo - bad content", func() { - // repo Meta not found - err := boltdbWrapper.SetManifestData(digest.FromString("dig"), mTypes.ManifestData{ - ManifestBlob: []byte("Bad Manifest"), - ConfigBlob: []byte("Bad Manifest"), - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error { - repoBuck := tx.Bucket([]byte(boltdb.RepoMetadataBucket)) - - return repoBuck.Put([]byte("repo1"), []byte("wrong json")) - }) - So(err, ShouldBeNil) - - err = boltdbWrapper.UpdateSignaturesValidity("repo1", digest.FromString("dig")) - So(err, ShouldNotBeNil) - }) - }) - }) -} - -func setBadIndexData(dB *bbolt.DB, digest string) error { - return dB.Update(func(tx *bbolt.Tx) error { - indexDataBuck := tx.Bucket([]byte(boltdb.IndexDataBucket)) - - return indexDataBuck.Put([]byte(digest), []byte("bad json")) }) } diff --git a/pkg/meta/boltdb/buckets.go b/pkg/meta/boltdb/buckets.go index 84649902..90188974 100644 --- a/pkg/meta/boltdb/buckets.go +++ b/pkg/meta/boltdb/buckets.go @@ -2,10 +2,10 @@ package boltdb // MetadataDB. const ( - ManifestDataBucket = "ManifestData" - IndexDataBucket = "IndexData" - RepoMetadataBucket = "RepoMetadata" - UserDataBucket = "UserData" - VersionBucket = "Version" - UserAPIKeysBucket = "UserAPIKeys" + ImageMetaBuck = "ImageMeta" + RepoMetaBuck = "RepoMeta" + RepoBlobsBuck = "RepoBlobsMeta" + UserDataBucket = "UserData" + VersionBucket = "Version" + UserAPIKeysBucket = "UserAPIKeys" ) diff --git a/pkg/meta/common/common.go b/pkg/meta/common/common.go index f97dfa01..a1c09334 100644 --- a/pkg/meta/common/common.go +++ b/pkg/meta/common/common.go @@ -1,8 +1,6 @@ package common import ( - "encoding/json" - "fmt" "strings" "time" @@ -10,28 +8,23 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" zerr "zotregistry.io/zot/errors" + zcommon "zotregistry.io/zot/pkg/common" + mConvert "zotregistry.io/zot/pkg/meta/convert" + proto_go "zotregistry.io/zot/pkg/meta/proto/gen" mTypes "zotregistry.io/zot/pkg/meta/types" ) -func UpdateManifestMeta(repoMeta mTypes.RepoMetadata, manifestDigest godigest.Digest, - manifestMeta mTypes.ManifestMetadata, -) mTypes.RepoMetadata { - updatedRepoMeta := repoMeta - - updatedStatistics := repoMeta.Statistics[manifestDigest.String()] - updatedStatistics.DownloadCount = manifestMeta.DownloadCount - updatedRepoMeta.Statistics[manifestDigest.String()] = updatedStatistics - - if manifestMeta.Signatures == nil { - manifestMeta.Signatures = mTypes.ManifestSignatures{} +func SignatureAlreadyExists(signatureSlice []mTypes.SignatureInfo, sm mTypes.SignatureMetadata) bool { + for _, sigInfo := range signatureSlice { + if sm.SignatureDigest == sigInfo.SignatureManifestDigest { + return true + } } - updatedRepoMeta.Signatures[manifestDigest.String()] = manifestMeta.Signatures - - return updatedRepoMeta + return false } -func SignatureAlreadyExists(signatureSlice []mTypes.SignatureInfo, sm mTypes.SignatureMetadata) bool { +func ProtoSignatureAlreadyExists(signatureSlice []*proto_go.SignatureInfo, sm mTypes.SignatureMetadata) bool { for _, sigInfo := range signatureSlice { if sm.SignatureDigest == sigInfo.SignatureManifestDigest { return true @@ -74,7 +67,7 @@ const ( ) // RankRepoName associates a rank to a given repoName given a searchText. -// The imporance of the value grows inversly proportional to the int value it has. +// The importance of the value grows inversely proportional to the int value it has. // For example: rank(1) > rank(10) > rank(100)... func RankRepoName(searchText string, repoName string) int { searchText = strings.Trim(searchText, "/") @@ -89,7 +82,7 @@ func RankRepoName(searchText string, repoName string) int { return perfectMatchPriority } - // searchText containst just 1 diretory name + // searchText contains just 1 directory name if len(searchTextSlice) == 1 { lastNameInRepoPath := repoNameSlice[len(repoNameSlice)-1] @@ -157,21 +150,6 @@ func GetRepoTag(searchText string) (string, string, error) { return repo, tag, nil } -func GetReferredSubject(descriptorBlob []byte) (godigest.Digest, bool) { - var manifest ispec.Manifest - - err := json.Unmarshal(descriptorBlob, &manifest) - if err != nil { - return "", false - } - - if manifest.Subject == nil || manifest.Subject.Digest.String() == "" { - return "", false - } - - return manifest.Subject.Digest, true -} - func MatchesArtifactTypes(descriptorMediaType string, artifactTypes []string) bool { if len(artifactTypes) == 0 { return true @@ -208,139 +186,199 @@ func CheckImageLastUpdated(repoLastUpdated time.Time, isSigned bool, noImageChec return repoLastUpdated, noImageChecked, isSigned } -func FilterDataByRepo(foundRepos []mTypes.RepoMetadata, manifestMetadataMap map[string]mTypes.ManifestMetadata, - indexDataMap map[string]mTypes.IndexData, -) (map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { - var ( - foundManifestMetadataMap = make(map[string]mTypes.ManifestMetadata) - foundindexDataMap = make(map[string]mTypes.IndexData) - ) +func AddImageMetaToRepoMeta(repoMeta *proto_go.RepoMeta, repoBlobs *proto_go.RepoBlobs, reference string, + imageMeta mTypes.ImageMeta, +) (*proto_go.RepoMeta, *proto_go.RepoBlobs, error) { + switch imageMeta.MediaType { + case ispec.MediaTypeImageManifest: + manifestData := imageMeta.Manifests[0] - // keep just the manifestMeta we need - for _, repoMeta := range foundRepos { - for _, descriptor := range repoMeta.Tags { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - foundManifestMetadataMap[descriptor.Digest] = manifestMetadataMap[descriptor.Digest] - case ispec.MediaTypeImageIndex: - indexData := indexDataMap[descriptor.Digest] + vendor := GetVendor(manifestData.Manifest.Annotations) + if vendor == "" { + vendor = GetVendor(manifestData.Manifest.Annotations) + } - var indexContent ispec.Index + vendors := []string{} + if vendor != "" { + vendors = append(vendors, vendor) + } - err := json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - fmt.Errorf("metadb: error while getting manifest data for digest %s %w", descriptor.Digest, err) - } + platforms := []*proto_go.Platform{getProtoPlatform(&manifestData.Config.Platform)} + if platforms[0].OS == "" && platforms[0].Architecture == "" { + platforms = []*proto_go.Platform{} + } - for _, manifestDescriptor := range indexContent.Manifests { - manifestDigest := manifestDescriptor.Digest.String() + subBlobs := []string{manifestData.Manifest.Config.Digest.String()} + repoBlobs.Blobs[manifestData.Manifest.Config.Digest.String()] = &proto_go.BlobInfo{ + Size: manifestData.Manifest.Config.Size, + } - foundManifestMetadataMap[manifestDigest] = manifestMetadataMap[manifestDigest] - } + for _, layer := range manifestData.Manifest.Layers { + subBlobs = append(subBlobs, layer.Digest.String()) + repoBlobs.Blobs[layer.Digest.String()] = &proto_go.BlobInfo{Size: layer.Size} + } - foundindexDataMap[descriptor.Digest] = indexData - default: - continue + lastUpdated := zcommon.GetImageLastUpdated(manifestData.Config) + + repoBlobs.Blobs[imageMeta.Digest.String()] = &proto_go.BlobInfo{ + Size: imageMeta.Size, + Vendors: vendors, + Platforms: platforms, + SubBlobs: subBlobs, + LastUpdated: mConvert.GetProtoTime(&lastUpdated), + } + case ispec.MediaTypeImageIndex: + subBlobs := []string{} + for _, manifest := range imageMeta.Index.Manifests { + subBlobs = append(subBlobs, manifest.Digest.String()) + } + + repoBlobs.Blobs[imageMeta.Digest.String()] = &proto_go.BlobInfo{ + Size: imageMeta.Size, + SubBlobs: subBlobs, + } + } + + // update info only when a tag is added + if zcommon.IsDigest(reference) { + return repoMeta, repoBlobs, nil + } + + size, platforms, vendors := recalculateAggregateFields(repoMeta, repoBlobs) + repoMeta.Vendors = vendors + repoMeta.Platforms = platforms + repoMeta.Size = int32(size) + + imageBlobInfo := repoBlobs.Blobs[imageMeta.Digest.String()] + repoMeta.LastUpdatedImage = mConvert.GetProtoEarlierUpdatedImage(repoMeta.LastUpdatedImage, + &proto_go.RepoLastUpdatedImage{ + LastUpdated: imageBlobInfo.LastUpdated, + MediaType: imageMeta.MediaType, + Digest: imageMeta.Digest.String(), + Tag: reference, + }) + + return repoMeta, repoBlobs, nil +} + +func RemoveImageFromRepoMeta(repoMeta *proto_go.RepoMeta, repoBlobs *proto_go.RepoBlobs, ref string, +) (*proto_go.RepoMeta, *proto_go.RepoBlobs, error) { + var updatedLastImage *proto_go.RepoLastUpdatedImage + + updatedBlobs := map[string]*proto_go.BlobInfo{} + updatedSize := int64(0) + updatedVendors := []string{} + updatedPlatforms := []*proto_go.Platform{} + + for tag, descriptor := range repoMeta.Tags { + if descriptor.Digest == "" { + continue + } + + queue := []string{descriptor.Digest} + + mConvert.GetProtoEarlierUpdatedImage(updatedLastImage, &proto_go.RepoLastUpdatedImage{ + LastUpdated: repoBlobs.Blobs[descriptor.Digest].LastUpdated, + MediaType: descriptor.MediaType, + Digest: descriptor.Digest, + Tag: tag, + }) + + for len(queue) > 0 { + currentBlob := queue[0] + queue = queue[1:] + + if _, found := updatedBlobs[currentBlob]; !found { + blobInfo := repoBlobs.Blobs[currentBlob] + + updatedBlobs[currentBlob] = blobInfo + updatedSize += blobInfo.Size + updatedVendors = mConvert.AddVendors(updatedVendors, blobInfo.Vendors) + updatedPlatforms = mConvert.AddProtoPlatforms(updatedPlatforms, blobInfo.Platforms) + + queue = append(queue, blobInfo.SubBlobs...) } } } - return foundManifestMetadataMap, foundindexDataMap, nil + repoMeta.Size = int32(updatedSize) + repoMeta.Vendors = updatedVendors + repoMeta.Platforms = updatedPlatforms + repoMeta.LastUpdatedImage = updatedLastImage + + repoBlobs.Blobs = updatedBlobs + + return repoMeta, repoBlobs, nil } -func FetchDataForRepos(metaDB mTypes.MetaDB, foundRepos []mTypes.RepoMetadata, -) (map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { - foundManifestMetadataMap := map[string]mTypes.ManifestMetadata{} - foundIndexDataMap := map[string]mTypes.IndexData{} +func recalculateAggregateFields(repoMeta *proto_go.RepoMeta, repoBlobs *proto_go.RepoBlobs, +) (int64, []*proto_go.Platform, []string) { + size := int64(0) + platforms := []*proto_go.Platform{} + vendors := []string{} + blobsMap := map[string]struct{}{} - for idx := range foundRepos { - for _, descriptor := range foundRepos[idx].Tags { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestData, err := metaDB.GetManifestData(godigest.Digest(descriptor.Digest)) - if err != nil { - return map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, err + for _, descriptor := range repoMeta.Tags { + if descriptor.Digest == "" { + continue + } + + queue := []string{descriptor.Digest} + + for len(queue) > 0 { + currentBlob := queue[0] + queue = queue[1:] + + if _, found := blobsMap[currentBlob]; !found { + blobInfo := repoBlobs.Blobs[currentBlob] + if blobInfo == nil { + continue } - foundManifestMetadataMap[descriptor.Digest] = mTypes.ManifestMetadata{ - ManifestBlob: manifestData.ManifestBlob, - ConfigBlob: manifestData.ConfigBlob, - } - case ispec.MediaTypeImageIndex: - indexData, err := metaDB.GetIndexData(godigest.Digest(descriptor.Digest)) - if err != nil { - return map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, err - } + blobsMap[currentBlob] = struct{}{} + size += blobInfo.Size + vendors = mConvert.AddVendors(vendors, blobInfo.Vendors) + platforms = mConvert.AddProtoPlatforms(platforms, blobInfo.Platforms) - var indexContent ispec.Index - - err = json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return map[string]mTypes.ManifestMetadata{}, - map[string]mTypes.IndexData{}, - fmt.Errorf("metadb: error while getting index data for digest %s %w", descriptor.Digest, err) - } - - for _, manifestDescriptor := range indexContent.Manifests { - manifestDigest := manifestDescriptor.Digest.String() - - manifestData, err := metaDB.GetManifestData(manifestDescriptor.Digest) - if err != nil { - return map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, err - } - - foundManifestMetadataMap[manifestDigest] = mTypes.ManifestMetadata{ - ManifestBlob: manifestData.ManifestBlob, - ConfigBlob: manifestData.ConfigBlob, - } - } - - foundIndexDataMap[descriptor.Digest] = indexData + queue = append(queue, blobInfo.SubBlobs...) } } } - return foundManifestMetadataMap, foundIndexDataMap, nil + return size, platforms, vendors } -// FindMediaTypeForDigest will look into the buckets for a certain digest. Depending on which bucket that -// digest is found the corresponding mediatype is returned. -func FindMediaTypeForDigest(metaDB mTypes.MetaDB, digest godigest.Digest) (bool, string) { - _, err := metaDB.GetManifestData(digest) - if err == nil { - return true, ispec.MediaTypeImageManifest +func getProtoPlatform(platform *ispec.Platform) *proto_go.Platform { + if platform == nil { + return nil } - _, err = metaDB.GetIndexData(digest) - if err == nil { - return true, ispec.MediaTypeImageIndex + return &proto_go.Platform{ + Architecture: getArch(platform.Architecture, platform.Variant), + OS: platform.OS, } - - return false, "" } -func GetImageDescriptor(metaDB mTypes.MetaDB, repo, tag string) (mTypes.Descriptor, error) { - repoMeta, err := metaDB.GetRepoMeta(repo) - if err != nil { - return mTypes.Descriptor{}, err +func getArch(arch string, variant string) string { + if variant != "" { + arch = arch + "/" + variant } - imageDescriptor, ok := repoMeta.Tags[tag] - if !ok { - return mTypes.Descriptor{}, zerr.ErrTagMetaNotFound - } - - return imageDescriptor, nil + return arch } -func InitializeImageConfig(blob []byte) ispec.Image { - var configContent ispec.Image +func GetVendor(annotations map[string]string) string { + return GetAnnotationValue(annotations, ispec.AnnotationVendor, "org.label-schema.vendor") +} - err := json.Unmarshal(blob, &configContent) - if err != nil { - return ispec.Image{} +func GetAnnotationValue(annotations map[string]string, annotationKey, labelKey string) string { + value, ok := annotations[annotationKey] + if !ok || value == "" { + value, ok = annotations[labelKey] + if !ok { + value = "" + } } - return configContent + return value } diff --git a/pkg/meta/common/common_test.go b/pkg/meta/common/common_test.go index 6c7c9efe..01ffbf39 100644 --- a/pkg/meta/common/common_test.go +++ b/pkg/meta/common/common_test.go @@ -5,23 +5,15 @@ import ( "testing" "time" - "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" "zotregistry.io/zot/pkg/meta/common" mTypes "zotregistry.io/zot/pkg/meta/types" - "zotregistry.io/zot/pkg/test/mocks" ) var ErrTestError = errors.New("test error") func TestUtils(t *testing.T) { - Convey("GetReferredSubject", t, func() { - _, err := common.GetReferredSubject([]byte("bad json")) - So(err, ShouldNotBeNil) - }) - Convey("MatchesArtifactTypes", t, func() { res := common.MatchesArtifactTypes("", nil) So(res, ShouldBeTrue) @@ -116,145 +108,4 @@ func TestUtils(t *testing.T) { So(res, ShouldEqual, false) }) - - Convey("FilterDataByRepo", t, func() { - Convey("Functionality", func() { - _, _, err := common.FilterDataByRepo( - []mTypes.RepoMetadata{{ - Tags: map[string]mTypes.Descriptor{ - "manifest": { - Digest: "manifestDigest", - MediaType: ispec.MediaTypeImageManifest, - }, - "index": { - Digest: "indexDigest", - MediaType: ispec.MediaTypeImageIndex, - }, - "rand": { - Digest: "randDigest", - MediaType: "rand", - }, - }, - }}, - map[string]mTypes.ManifestMetadata{}, - map[string]mTypes.IndexData{ - "indexDigest": { - IndexBlob: []byte(`{ - "manifests": [ - { - "digest": "manifestDigest" - } - ] - }`), - }, - }, - ) - So(err, ShouldBeNil) - }) - Convey("Errors", func() { - // Unmarshal index data error - _, _, err := common.FilterDataByRepo( - []mTypes.RepoMetadata{{ - Tags: map[string]mTypes.Descriptor{ - "tag": { - Digest: "indexDigest", - MediaType: ispec.MediaTypeImageIndex, - }, - }, - }}, - map[string]mTypes.ManifestMetadata{}, - map[string]mTypes.IndexData{ - "indexDigest": { - IndexBlob: []byte("bad blob"), - }, - }, - ) - - So(err, ShouldNotBeNil) - }) - }) - - Convey("FetchDataForRepos", t, func() { - Convey("Errors", func() { - // Unmarshal index data error - _, _, err := common.FetchDataForRepos( - mocks.MetaDBMock{ - GetIndexDataFn: func(indexDigest digest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{ - IndexBlob: []byte("bad blob"), - }, nil - }, - }, - []mTypes.RepoMetadata{{ - Tags: map[string]mTypes.Descriptor{ - "tag": { - Digest: "indexDigest", - MediaType: ispec.MediaTypeImageIndex, - }, - }, - }}, - ) - So(err, ShouldNotBeNil) - }) - }) -} - -func TestFetchDataForRepos(t *testing.T) { - Convey("GetReferredSubject", t, func() { - mockMetaDB := mocks.MetaDBMock{} - - Convey("GetManifestData errors", func() { - mockMetaDB.GetManifestDataFn = func(manifestDigest digest.Digest) (mTypes.ManifestData, error) { - return mTypes.ManifestData{}, ErrTestError - } - - _, _, err := common.FetchDataForRepos(mockMetaDB, []mTypes.RepoMetadata{ - { - Tags: map[string]mTypes.Descriptor{ - "tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageManifest}, - }, - }, - }) - So(err, ShouldNotBeNil) - }) - - Convey("GetIndexData errors", func() { - mockMetaDB.GetIndexDataFn = func(indexDigest digest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{}, ErrTestError - } - - _, _, err := common.FetchDataForRepos(mockMetaDB, []mTypes.RepoMetadata{ - { - Tags: map[string]mTypes.Descriptor{ - "tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageIndex}, - }, - }, - }) - So(err, ShouldNotBeNil) - }) - - Convey("GetIndexData ok, GetManifestData errors", func() { - mockMetaDB.GetIndexDataFn = func(indexDigest digest.Digest) (mTypes.IndexData, error) { - return mTypes.IndexData{ - IndexBlob: []byte(`{ - "manifests": [ - {"digest": "dig1"} - ] - }`), - }, nil - } - mockMetaDB.GetManifestDataFn = func(manifestDigest digest.Digest) (mTypes.ManifestData, error) { - return mTypes.ManifestData{}, ErrTestError - } - - _, _, err := common.FetchDataForRepos(mockMetaDB, []mTypes.RepoMetadata{ - { - Tags: map[string]mTypes.Descriptor{ - "tag1": {Digest: "dig1", MediaType: ispec.MediaTypeImageIndex}, - }, - }, - }) - So(err, ShouldNotBeNil) - }) - }) } diff --git a/pkg/meta/convert/convert.go b/pkg/meta/convert/convert.go new file mode 100644 index 00000000..9bc76052 --- /dev/null +++ b/pkg/meta/convert/convert.go @@ -0,0 +1,611 @@ +package convert + +import ( + "time" + + godigest "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/specs-go" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + "google.golang.org/protobuf/types/known/timestamppb" + + "zotregistry.io/zot/pkg/common" + proto_go "zotregistry.io/zot/pkg/meta/proto/gen" + mTypes "zotregistry.io/zot/pkg/meta/types" +) + +func GetHistory(history []*proto_go.History) []ispec.History { + if history == nil { + return nil + } + + results := make([]ispec.History, 0, len(history)) + + for _, his := range history { + results = append(results, ispec.History{ + Created: ref(his.Created.AsTime()), + CreatedBy: deref(his.CreatedBy, ""), + Author: deref(his.Author, ""), + Comment: deref(his.Comment, ""), + EmptyLayer: deref(his.EmptyLayer, false), + }) + } + + return results +} + +func GetImageArtifactType(imageMeta *proto_go.ImageMeta) string { + switch imageMeta.MediaType { + case ispec.MediaTypeImageManifest: + manifestArtifactType := deref(imageMeta.Manifests[0].Manifest.ArtifactType, "") + if manifestArtifactType != "" { + return manifestArtifactType + } + + return imageMeta.Manifests[0].Manifest.Config.MediaType + case ispec.MediaTypeImageIndex: + return deref(imageMeta.Index.Index.ArtifactType, "") + default: + return "" + } +} + +func GetImageManifestSize(imageMeta *proto_go.ImageMeta) int64 { + switch imageMeta.MediaType { + case ispec.MediaTypeImageManifest: + return imageMeta.Manifests[0].Size + case ispec.MediaTypeImageIndex: + return imageMeta.Index.Size + default: + return 0 + } +} + +func GetImageDigest(imageMeta *proto_go.ImageMeta) godigest.Digest { + switch imageMeta.MediaType { + case ispec.MediaTypeImageManifest: + return godigest.Digest(imageMeta.Manifests[0].Digest) + case ispec.MediaTypeImageIndex: + return godigest.Digest(imageMeta.Index.Digest) + default: + return "" + } +} + +func GetImageDigestStr(imageMeta *proto_go.ImageMeta) string { + switch imageMeta.MediaType { + case ispec.MediaTypeImageManifest: + return imageMeta.Manifests[0].Digest + case ispec.MediaTypeImageIndex: + return imageMeta.Index.Digest + default: + return "" + } +} + +func GetImageAnnotations(imageMeta *proto_go.ImageMeta) map[string]string { + switch imageMeta.MediaType { + case ispec.MediaTypeImageManifest: + return imageMeta.Manifests[0].Manifest.Annotations + case ispec.MediaTypeImageIndex: + return imageMeta.Index.Index.Annotations + default: + return map[string]string{} + } +} + +func GetImageSubject(imageMeta *proto_go.ImageMeta) *ispec.Descriptor { + switch imageMeta.MediaType { + case ispec.MediaTypeImageManifest: + if imageMeta.Manifests[0].Manifest.Subject == nil { + return nil + } + + return GetDescriptorRef(imageMeta.Manifests[0].Manifest.Subject) + case ispec.MediaTypeImageIndex: + if imageMeta.Index.Index.Subject == nil { + return nil + } + + return GetDescriptorRef(imageMeta.Index.Index.Subject) + default: + return nil + } +} + +func GetDescriptorRef(descriptor *proto_go.Descriptor) *ispec.Descriptor { + if descriptor == nil { + return nil + } + + platform := GetPlatformRef(descriptor.Platform) + + return &ispec.Descriptor{ + MediaType: descriptor.MediaType, + Digest: godigest.Digest(descriptor.Digest), + Size: descriptor.Size, + URLs: descriptor.URLs, + Data: descriptor.Data, + Platform: platform, + ArtifactType: deref(descriptor.ArtifactType, ""), + Annotations: descriptor.Annotations, + } +} + +func GetPlatform(platform *proto_go.Platform) ispec.Platform { + if platform == nil { + return ispec.Platform{} + } + + return ispec.Platform{ + Architecture: platform.Architecture, + OS: platform.OS, + OSVersion: deref(platform.OSVersion, ""), + OSFeatures: platform.OSFeatures, + Variant: deref(platform.Variant, ""), + } +} + +func GetPlatformRef(platform *proto_go.Platform) *ispec.Platform { + if platform == nil { + return nil + } + + return &ispec.Platform{ + Architecture: platform.Architecture, + OS: platform.OS, + OSVersion: deref(platform.OSVersion, ""), + OSFeatures: platform.OSFeatures, + Variant: deref(platform.Variant, ""), + } +} + +func GetLayers(descriptors []*proto_go.Descriptor) []ispec.Descriptor { + results := make([]ispec.Descriptor, 0, len(descriptors)) + + for _, desc := range descriptors { + results = append(results, ispec.Descriptor{ + MediaType: desc.MediaType, + Digest: godigest.Digest(desc.Digest), + Size: desc.Size, + }) + } + + return results +} + +func GetSubject(subj *proto_go.Descriptor) *ispec.Descriptor { + if subj == nil { + return nil + } + + return &ispec.Descriptor{ + MediaType: subj.MediaType, + Digest: godigest.Digest(subj.Digest), + Size: subj.Size, + } +} + +func GetReferrers(refs map[string]*proto_go.ReferrersInfo) map[string][]mTypes.ReferrerInfo { + results := map[string][]mTypes.ReferrerInfo{} + + for digest, ref := range refs { + referrers := []mTypes.ReferrerInfo{} + + for _, dbRef := range ref.List { + referrers = append(referrers, mTypes.ReferrerInfo{ + Digest: dbRef.Digest, + MediaType: dbRef.MediaType, + ArtifactType: dbRef.ArtifactType, + Size: int(dbRef.Size), + Annotations: dbRef.Annotations, + }) + } + + results[digest] = referrers + } + + return results +} + +func GetImageReferrers(refs *proto_go.ReferrersInfo) []mTypes.ReferrerInfo { + if refs == nil { + return []mTypes.ReferrerInfo{} + } + + results := []mTypes.ReferrerInfo{} + + for _, dbRef := range refs.List { + results = append(results, mTypes.ReferrerInfo{ + Digest: dbRef.Digest, + MediaType: dbRef.MediaType, + ArtifactType: dbRef.ArtifactType, + Size: int(dbRef.Size), + Annotations: dbRef.Annotations, + }) + } + + return results +} + +func GetSignatures(sigs map[string]*proto_go.ManifestSignatures) map[string]mTypes.ManifestSignatures { + results := map[string]mTypes.ManifestSignatures{} + + for digest, dbSignatures := range sigs { + imageSignatures := mTypes.ManifestSignatures{} + + for signatureName, signatureInfo := range dbSignatures.Map { + imageSignatures[signatureName] = GetSignaturesInfo(signatureInfo.List) + } + + results[digest] = imageSignatures + } + + return results +} + +func GetImageSignatures(sigs *proto_go.ManifestSignatures) mTypes.ManifestSignatures { + if sigs == nil { + return mTypes.ManifestSignatures{} + } + + results := mTypes.ManifestSignatures{} + + for signatureName, signatureInfo := range sigs.Map { + results[signatureName] = GetSignaturesInfo(signatureInfo.List) + } + + return results +} + +func GetSignaturesInfo(sigsInfo []*proto_go.SignatureInfo) []mTypes.SignatureInfo { + results := []mTypes.SignatureInfo{} + + for _, siginfo := range sigsInfo { + results = append(results, mTypes.SignatureInfo{ + SignatureManifestDigest: siginfo.SignatureManifestDigest, + LayersInfo: GetLayersInfo(siginfo.LayersInfo), + }) + } + + return results +} + +func GetLayersInfo(layersInfo []*proto_go.LayersInfo) []mTypes.LayerInfo { + results := []mTypes.LayerInfo{} + + for _, layerInfo := range layersInfo { + date := time.Time{} + + if layerInfo.Date != nil { + date = layerInfo.Date.AsTime() + } + + results = append(results, mTypes.LayerInfo{ + LayerDigest: layerInfo.LayerDigest, + LayerContent: layerInfo.LayerContent, + SignatureKey: layerInfo.SignatureKey, + Signer: layerInfo.Signer, + Date: date, + }) + } + + return results +} + +func GetStatisticsMap(stats map[string]*proto_go.DescriptorStatistics) map[string]mTypes.DescriptorStatistics { + results := map[string]mTypes.DescriptorStatistics{} + + for digest, stat := range stats { + results[digest] = mTypes.DescriptorStatistics{ + DownloadCount: int(stat.DownloadCount), + } + } + + return results +} + +func GetImageStatistics(stats *proto_go.DescriptorStatistics) mTypes.DescriptorStatistics { + if stats == nil { + return mTypes.DescriptorStatistics{} + } + + return mTypes.DescriptorStatistics{ + DownloadCount: int(stats.DownloadCount), + } +} + +func GetImageManifestMeta(manifestContent ispec.Manifest, configContent ispec.Image, size int64, + digest godigest.Digest, +) mTypes.ImageMeta { + return mTypes.ImageMeta{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: size, + Manifests: []mTypes.ManifestData{ + { + Digest: digest, + Size: size, + Config: configContent, + Manifest: manifestContent, + }, + }, + } +} + +func GetImageIndexMeta(indexContent ispec.Index, size int64, digest godigest.Digest) mTypes.ImageMeta { + return mTypes.ImageMeta{ + MediaType: ispec.MediaTypeImageIndex, + Index: &indexContent, + Manifests: GetManifests(indexContent.Manifests), + Size: size, + Digest: digest, + } +} + +func GetTags(tags map[string]*proto_go.TagDescriptor) map[string]mTypes.Descriptor { + resultMap := map[string]mTypes.Descriptor{} + + for tag, tagDescriptor := range tags { + resultMap[tag] = mTypes.Descriptor{ + Digest: tagDescriptor.Digest, + MediaType: tagDescriptor.MediaType, + } + } + + return resultMap +} + +func GetManifests(descriptors []ispec.Descriptor) []mTypes.ManifestData { + manifestList := []mTypes.ManifestData{} + + for _, manifest := range descriptors { + manifestList = append(manifestList, mTypes.ManifestData{ + Digest: manifest.Digest, + Size: manifest.Size, + }) + } + + return manifestList +} + +func GetTime(time *timestamppb.Timestamp) *time.Time { + if time == nil { + return nil + } + + return ref(time.AsTime()) +} + +func GetFullImageMetaFromProto(tag string, protoRepoMeta *proto_go.RepoMeta, protoImageMeta *proto_go.ImageMeta, +) mTypes.FullImageMeta { + imageMeta := GetImageMeta(protoImageMeta) + imageDigest := imageMeta.Digest.String() + + return mTypes.FullImageMeta{ + Repo: protoRepoMeta.Name, + Tag: tag, + MediaType: imageMeta.MediaType, + Digest: imageMeta.Digest, + Size: imageMeta.Size, + Index: imageMeta.Index, + Manifests: GetFullManifestData(protoRepoMeta, imageMeta.Manifests), + IsStarred: protoRepoMeta.IsStarred, + IsBookmarked: protoRepoMeta.IsBookmarked, + + Referrers: GetImageReferrers(protoRepoMeta.Referrers[imageDigest]), + Statistics: GetImageStatistics(protoRepoMeta.Statistics[imageDigest]), + Signatures: GetImageSignatures(protoRepoMeta.Signatures[imageDigest]), + } +} + +func GetFullManifestData(protoRepoMeta *proto_go.RepoMeta, manifestData []mTypes.ManifestData, +) []mTypes.FullManifestMeta { + results := []mTypes.FullManifestMeta{} + + for i := range manifestData { + results = append(results, mTypes.FullManifestMeta{ + ManifestData: manifestData[i], + Referrers: GetImageReferrers(protoRepoMeta.Referrers[manifestData[i].Digest.String()]), + Statistics: GetImageStatistics(protoRepoMeta.Statistics[manifestData[i].Digest.String()]), + Signatures: GetImageSignatures(protoRepoMeta.Signatures[manifestData[i].Digest.String()]), + }) + } + + return results +} + +func GetRepoMeta(protoRepoMeta *proto_go.RepoMeta) mTypes.RepoMeta { + repoDownloads := int32(0) + + for _, descriptor := range protoRepoMeta.Tags { + if statistic := protoRepoMeta.Statistics[descriptor.Digest]; statistic != nil { + repoDownloads += statistic.DownloadCount + } + } + + return mTypes.RepoMeta{ + Name: protoRepoMeta.Name, + Tags: GetTags(protoRepoMeta.Tags), + Rank: int(protoRepoMeta.Rank), + Size: int64(protoRepoMeta.Size), + Platforms: GetPlatforms(protoRepoMeta.Platforms), + Vendors: protoRepoMeta.Vendors, + IsStarred: protoRepoMeta.IsStarred, + IsBookmarked: protoRepoMeta.IsBookmarked, + StarCount: int(protoRepoMeta.Stars), + DownloadCount: int(repoDownloads), + LastUpdatedImage: GetLastUpdatedImage(protoRepoMeta.LastUpdatedImage), + Statistics: GetStatisticsMap(protoRepoMeta.Statistics), + Signatures: GetSignatures(protoRepoMeta.Signatures), + Referrers: GetReferrers(protoRepoMeta.Referrers), + } +} + +func GetPlatforms(platforms []*proto_go.Platform) []ispec.Platform { + result := []ispec.Platform{} + + for i := range platforms { + result = append(result, GetPlatform(platforms[i])) + } + + return result +} + +func AddProtoPlatforms(platforms []*proto_go.Platform, newPlatforms []*proto_go.Platform) []*proto_go.Platform { + for _, newPlatform := range newPlatforms { + if !ContainsProtoPlatform(platforms, newPlatform) { + platforms = append(platforms, newPlatform) + } + } + + return platforms +} + +func ContainsProtoPlatform(platforms []*proto_go.Platform, platform *proto_go.Platform) bool { + for i := range platforms { + if platforms[i].OS == platform.OS && platforms[i].Architecture == platform.Architecture { + return true + } + } + + return false +} + +func AddVendors(vendors []string, newVendors []string) []string { + for _, newVendor := range newVendors { + if !common.Contains(vendors, newVendor) { + vendors = append(vendors, newVendor) + } + } + + return vendors +} + +func GetLastUpdatedImage(protoLastUpdated *proto_go.RepoLastUpdatedImage) *mTypes.LastUpdatedImage { + if protoLastUpdated == nil { + return nil + } + + return &mTypes.LastUpdatedImage{ + Descriptor: mTypes.Descriptor{ + Digest: protoLastUpdated.Digest, + MediaType: protoLastUpdated.MediaType, + }, + Tag: protoLastUpdated.Tag, + LastUpdated: GetTime(protoLastUpdated.LastUpdated), + } +} + +func GetImageMeta(dbImageMeta *proto_go.ImageMeta) mTypes.ImageMeta { + imageMeta := mTypes.ImageMeta{ + MediaType: dbImageMeta.MediaType, + Size: GetImageManifestSize(dbImageMeta), + Digest: GetImageDigest(dbImageMeta), + } + + if dbImageMeta.MediaType == ispec.MediaTypeImageIndex { + manifests := make([]ispec.Descriptor, 0, len(dbImageMeta.Manifests)) + + for _, manifest := range dbImageMeta.Manifests { + manifests = append(manifests, ispec.Descriptor{ + MediaType: deref(manifest.Manifest.MediaType, ""), + Digest: godigest.Digest(manifest.Digest), + Size: manifest.Size, + }) + } + + imageMeta.Index = &ispec.Index{ + Versioned: specs.Versioned{SchemaVersion: int(dbImageMeta.Index.Index.Versioned.GetSchemaVersion())}, + MediaType: ispec.MediaTypeImageIndex, + Manifests: manifests, + Subject: GetImageSubject(dbImageMeta), + ArtifactType: GetImageArtifactType(dbImageMeta), + Annotations: GetImageAnnotations(dbImageMeta), + } + } + + manifestDataList := make([]mTypes.ManifestData, 0, len(dbImageMeta.Manifests)) + + for _, manifest := range dbImageMeta.Manifests { + manifestDataList = append(manifestDataList, mTypes.ManifestData{ + Size: manifest.Size, + Digest: godigest.Digest(manifest.Digest), + Manifest: ispec.Manifest{ + Versioned: specs.Versioned{SchemaVersion: int(manifest.Manifest.Versioned.GetSchemaVersion())}, + MediaType: deref(manifest.Manifest.MediaType, ""), + ArtifactType: deref(manifest.Manifest.ArtifactType, ""), + Config: ispec.Descriptor{ + MediaType: manifest.Manifest.Config.MediaType, + Size: manifest.Manifest.Config.Size, + Digest: godigest.Digest(manifest.Manifest.Config.Digest), + }, + Layers: GetLayers(manifest.Manifest.Layers), + Subject: GetSubject(manifest.Manifest.Subject), + Annotations: manifest.Manifest.Annotations, + }, + Config: ispec.Image{ + Created: GetTime(manifest.Config.Created), + Author: deref(manifest.Config.Author, ""), + Platform: GetPlatform(manifest.Config.Platform), + Config: ispec.ImageConfig{ + User: manifest.Config.Config.User, + ExposedPorts: GetExposedPorts(manifest.Config.Config.ExposedPorts), + Env: manifest.Config.Config.Env, + Entrypoint: manifest.Config.Config.Entrypoint, + Cmd: manifest.Config.Config.Cmd, + Volumes: GetConfigVolumes(manifest.Config.Config.Volumes), + WorkingDir: deref(manifest.Config.Config.WorkingDir, ""), + Labels: manifest.Config.Config.Labels, + StopSignal: deref(manifest.Config.Config.StopSignal, ""), + }, + RootFS: ispec.RootFS{ + Type: manifest.Config.RootFS.Type, + DiffIDs: GetDiffIDs(manifest.Config.RootFS.DiffIDs), + }, + History: GetHistory(manifest.Config.History), + }, + }) + } + + imageMeta.Manifests = manifestDataList + + return imageMeta +} + +func GetExposedPorts(exposedPorts map[string]*proto_go.EmptyMessage) map[string]struct{} { + if exposedPorts == nil { + return nil + } + + result := map[string]struct{}{} + + for key := range exposedPorts { + result[key] = struct{}{} + } + + return result +} + +func GetConfigVolumes(configVolumes map[string]*proto_go.EmptyMessage) map[string]struct{} { + if configVolumes == nil { + return nil + } + + result := map[string]struct{}{} + + for key := range configVolumes { + result[key] = struct{}{} + } + + return result +} + +func GetDiffIDs(diffIDs []string) []godigest.Digest { + result := make([]godigest.Digest, 0, len(diffIDs)) + + for i := range diffIDs { + result = append(result, godigest.Digest(diffIDs[i])) + } + + return result +} diff --git a/pkg/meta/convert/convert_proto.go b/pkg/meta/convert/convert_proto.go new file mode 100644 index 00000000..5450ec13 --- /dev/null +++ b/pkg/meta/convert/convert_proto.go @@ -0,0 +1,392 @@ +package convert + +import ( + "time" + + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + "google.golang.org/protobuf/types/known/timestamppb" + + "zotregistry.io/zot/pkg/common" + proto_go "zotregistry.io/zot/pkg/meta/proto/gen" + mTypes "zotregistry.io/zot/pkg/meta/types" +) + +func GetProtoRepoMeta(repo mTypes.RepoMeta) *proto_go.RepoMeta { + return &proto_go.RepoMeta{ + Name: repo.Name, + Tags: GetProtoTags(repo.Tags), + Statistics: GetProtoStatistics(repo.Statistics), + Signatures: GetProtoSignatures(repo.Signatures), + Referrers: GetProtoReferrers(repo.Referrers), + Size: int32(repo.Size), + Vendors: repo.Vendors, + Platforms: GetProtoPlatforms(repo.Platforms), + LastUpdatedImage: GetProtoLastUpdatedImage(repo.LastUpdatedImage), + } +} + +func GetProtoImageMeta(imageMeta mTypes.ImageMeta) *proto_go.ImageMeta { + switch imageMeta.MediaType { + case ispec.MediaTypeImageManifest: + if len(imageMeta.Manifests) == 0 { + return nil + } + manifestData := imageMeta.Manifests[0] + + return GetProtoImageManifestData(manifestData.Manifest, manifestData.Config, manifestData.Size, + manifestData.Digest.String()) + case ispec.MediaTypeImageIndex: + if imageMeta.Index == nil { + return nil + } + + return GetProtoImageIndexMeta(*imageMeta.Index, imageMeta.Size, imageMeta.Digest.String()) + default: + return nil + } +} + +func GetProtoImageManifestData(manifestContent ispec.Manifest, configContent ispec.Image, size int64, digest string, +) *proto_go.ImageMeta { + return &proto_go.ImageMeta{ + MediaType: ispec.MediaTypeImageManifest, + Manifests: []*proto_go.ManifestMeta{GetProtoManifestMeta(manifestContent, configContent, size, digest)}, + Index: nil, + } +} + +func GetProtoManifestMeta(manifestContent ispec.Manifest, configContent ispec.Image, size int64, digest string, +) *proto_go.ManifestMeta { + return &proto_go.ManifestMeta{ + Digest: digest, + Size: size, + Manifest: &proto_go.Manifest{ + Versioned: &proto_go.Versioned{SchemaVersion: int32(manifestContent.SchemaVersion)}, + Config: &proto_go.Descriptor{ + Digest: manifestContent.Config.Digest.String(), + Size: manifestContent.Config.Size, + MediaType: manifestContent.Config.MediaType, + }, + MediaType: ref(ispec.MediaTypeImageManifest), + ArtifactType: &manifestContent.ArtifactType, + Layers: getProtoManifestLayers(manifestContent.Layers), + Subject: getProtoDesc(manifestContent.Subject), + Annotations: manifestContent.Annotations, + }, + Config: &proto_go.Image{ + Created: GetProtoTime(configContent.Created), + Author: &configContent.Author, + Platform: GetProtoPlatform(&configContent.Platform), + Config: &proto_go.ImageConfig{ + User: configContent.Config.User, + ExposedPorts: getProtoExposedPorts(configContent.Config.ExposedPorts), + Env: configContent.Config.Env, + Entrypoint: configContent.Config.Entrypoint, + Cmd: configContent.Config.Cmd, + Volumes: getProtoConfigVolumes(configContent.Config.Volumes), + WorkingDir: &configContent.Config.WorkingDir, + Labels: configContent.Config.Labels, + StopSignal: &configContent.Config.StopSignal, + }, + RootFS: &proto_go.RootFS{ + Type: configContent.RootFS.Type, + DiffIDs: getProtoDiffIDs(configContent.RootFS.DiffIDs), + }, + History: getProtoHistory(configContent.History), + }, + } +} + +func GetProtoImageIndexMeta(indexContent ispec.Index, size int64, digest string) *proto_go.ImageMeta { + return &proto_go.ImageMeta{ + MediaType: ispec.MediaTypeImageIndex, + Index: &proto_go.IndexMeta{ + Size: size, + Digest: digest, + Index: &proto_go.Index{ + Versioned: &proto_go.Versioned{SchemaVersion: int32(indexContent.Versioned.SchemaVersion)}, + MediaType: ref(ispec.MediaTypeImageIndex), + ArtifactType: ref(common.GetIndexArtifactType(indexContent)), + Manifests: getProtoManifestList(indexContent.Manifests), + Subject: getProtoDesc(indexContent.Subject), + Annotations: indexContent.Annotations, + }, + }, + } +} + +func GetProtoStatistics(stats map[string]mTypes.DescriptorStatistics) map[string]*proto_go.DescriptorStatistics { + results := map[string]*proto_go.DescriptorStatistics{} + + for digest, stat := range stats { + results[digest] = &proto_go.DescriptorStatistics{ + DownloadCount: int32(stat.DownloadCount), + } + } + + return results +} + +func GetProtoPlatforms(platforms []ispec.Platform) []*proto_go.Platform { + result := []*proto_go.Platform{} + + for i := range platforms { + result = append(result, &proto_go.Platform{ + OS: platforms[i].OS, + Architecture: platforms[i].Architecture, + }) + } + + return result +} + +func GetProtoReferrers(refs map[string][]mTypes.ReferrerInfo) map[string]*proto_go.ReferrersInfo { + results := map[string]*proto_go.ReferrersInfo{} + + for digest, ref := range refs { + referrersInfoList := []*proto_go.ReferrerInfo{} + + for _, dbRef := range ref { + referrersInfoList = append(referrersInfoList, GetProtoReferrerInfo(dbRef)) + } + + results[digest] = &proto_go.ReferrersInfo{List: referrersInfoList} + } + + return results +} + +func GetProtoSignatures(sigs map[string]mTypes.ManifestSignatures) map[string]*proto_go.ManifestSignatures { + results := map[string]*proto_go.ManifestSignatures{} + + for digest, dbSignatures := range sigs { + imageSignatures := &proto_go.ManifestSignatures{Map: map[string]*proto_go.SignaturesInfo{}} + + for signatureName, signatureInfo := range dbSignatures { + imageSignatures.Map[signatureName] = &proto_go.SignaturesInfo{List: GetProtoSignaturesInfo(signatureInfo)} + } + + results[digest] = imageSignatures + } + + return results +} + +func GetProtoSignaturesInfo(sigsInfo []mTypes.SignatureInfo) []*proto_go.SignatureInfo { + results := []*proto_go.SignatureInfo{} + + for _, sigInfo := range sigsInfo { + results = append(results, &proto_go.SignatureInfo{ + SignatureManifestDigest: sigInfo.SignatureManifestDigest, + LayersInfo: GetProtoLayersInfo(sigInfo.LayersInfo), + }) + } + + return results +} + +func GetProtoLayersInfo(layersInfo []mTypes.LayerInfo) []*proto_go.LayersInfo { + result := make([]*proto_go.LayersInfo, 0, len(layersInfo)) + + for _, layerInfo := range layersInfo { + result = append(result, &proto_go.LayersInfo{ + LayerDigest: layerInfo.LayerDigest, + LayerContent: layerInfo.LayerContent, + SignatureKey: layerInfo.SignatureKey, + Signer: layerInfo.Signer, + Date: timestamppb.New(layerInfo.Date), + }) + } + + return result +} + +func getProtoManifestLayers(layers []ispec.Descriptor) []*proto_go.Descriptor { + protoLayers := []*proto_go.Descriptor{} + + for _, layer := range layers { + layer := layer + + protoLayers = append(protoLayers, getProtoDesc(&layer)) + } + + return protoLayers +} + +func getProtoDesc(descriptor *ispec.Descriptor) *proto_go.Descriptor { + if descriptor == nil { + return nil + } + + return &proto_go.Descriptor{ + MediaType: descriptor.MediaType, + Digest: descriptor.Digest.String(), + Size: descriptor.Size, + URLs: descriptor.URLs, + Annotations: descriptor.Annotations, + Data: descriptor.Data, + Platform: GetProtoPlatform(descriptor.Platform), + ArtifactType: &descriptor.ArtifactType, + } +} + +func getProtoManifestList(manifests []ispec.Descriptor) []*proto_go.Descriptor { + result := make([]*proto_go.Descriptor, 0, len(manifests)) + + for _, manifest := range manifests { + result = append(result, &proto_go.Descriptor{ + MediaType: manifest.MediaType, + Digest: manifest.Digest.String(), + Size: manifest.Size, + URLs: manifest.URLs, + Annotations: manifest.Annotations, + Data: manifest.Data, + Platform: GetProtoPlatform(manifest.Platform), + ArtifactType: ref(manifest.ArtifactType), + }) + } + + return result +} + +func GetProtoPlatform(platform *ispec.Platform) *proto_go.Platform { + if platform == nil { + return nil + } + + return &proto_go.Platform{ + Architecture: platform.Architecture, + OS: platform.OS, + OSVersion: ref(platform.OSVersion), + OSFeatures: platform.OSFeatures, + Variant: ref(platform.Variant), + } +} + +func getProtoHistory(historySlice []ispec.History) []*proto_go.History { + protoHistory := []*proto_go.History{} + + for _, history := range historySlice { + history := history + + protoHistory = append(protoHistory, &proto_go.History{ + Created: GetProtoTime(history.Created), + CreatedBy: &history.CreatedBy, + Author: &history.Author, + Comment: &history.Comment, + EmptyLayer: &history.EmptyLayer, + }) + } + + return protoHistory +} + +func getProtoDiffIDs(digests []godigest.Digest) []string { + digestsStr := []string{} + + for _, digest := range digests { + digestsStr = append(digestsStr, digest.String()) + } + + return digestsStr +} + +func getProtoExposedPorts(exposedPorts map[string]struct{}) map[string]*proto_go.EmptyMessage { + protoPorts := map[string]*proto_go.EmptyMessage{} + + for i := range exposedPorts { + protoPorts[i] = &proto_go.EmptyMessage{} + } + + return protoPorts +} + +func getProtoConfigVolumes(volumes map[string]struct{}) map[string]*proto_go.EmptyMessage { + protoVolumes := map[string]*proto_go.EmptyMessage{} + + for i := range volumes { + protoVolumes[i] = &proto_go.EmptyMessage{} + } + + return protoVolumes +} + +func GetProtoReferrerInfo(referrer mTypes.ReferrerInfo) *proto_go.ReferrerInfo { + return &proto_go.ReferrerInfo{ + Digest: referrer.Digest, + MediaType: referrer.MediaType, + ArtifactType: referrer.ArtifactType, + Size: int64(referrer.Size), + Annotations: referrer.Annotations, + } +} + +func GetProtoTime(time *time.Time) *timestamppb.Timestamp { + if time == nil { + return nil + } + + return timestamppb.New(*time) +} + +func GetProtoTags(tags map[string]mTypes.Descriptor) map[string]*proto_go.TagDescriptor { + resultMap := map[string]*proto_go.TagDescriptor{} + + for tag, tagDescriptor := range tags { + resultMap[tag] = &proto_go.TagDescriptor{ + Digest: tagDescriptor.Digest, + MediaType: tagDescriptor.MediaType, + } + } + + return resultMap +} + +func GetProtoLastUpdatedImage(lastUpdatedImage *mTypes.LastUpdatedImage) *proto_go.RepoLastUpdatedImage { + if lastUpdatedImage == nil { + return nil + } + + return &proto_go.RepoLastUpdatedImage{ + LastUpdated: GetProtoTime(lastUpdatedImage.LastUpdated), + MediaType: lastUpdatedImage.MediaType, + Digest: lastUpdatedImage.Digest, + Tag: lastUpdatedImage.Tag, + } +} + +func GetProtoEarlierUpdatedImage(repoLastImage *proto_go.RepoLastUpdatedImage, lastImage *proto_go.RepoLastUpdatedImage, +) *proto_go.RepoLastUpdatedImage { + if repoLastImage == nil { + return lastImage + } + + if lastImage == nil || lastImage.LastUpdated == nil { + return repoLastImage + } + + if repoLastImage.LastUpdated == nil { + return lastImage + } + + if repoLastImage.LastUpdated.AsTime().Before(lastImage.LastUpdated.AsTime()) { + return lastImage + } + + return repoLastImage +} + +func ref[T any](input T) *T { + ref := input + + return &ref +} + +func deref[T any](pointer *T, defaultVal T) T { + if pointer != nil { + return *pointer + } + + return defaultVal +} diff --git a/pkg/meta/dynamodb/dynamodb.go b/pkg/meta/dynamodb/dynamodb.go index 84c20e2d..99cb7815 100644 --- a/pkg/meta/dynamodb/dynamodb.go +++ b/pkg/meta/dynamodb/dynamodb.go @@ -2,7 +2,6 @@ package dynamodb import ( "context" - "encoding/json" "errors" "fmt" "strings" @@ -14,46 +13,48 @@ import ( "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/api/constants" zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta/common" + mConvert "zotregistry.io/zot/pkg/meta/convert" + proto_go "zotregistry.io/zot/pkg/meta/proto/gen" mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/meta/version" reqCtx "zotregistry.io/zot/pkg/requestcontext" ) -var errMetaDB = errors.New("metadb: error while constructing manifest meta") - type DynamoDB struct { - Client *dynamodb.Client - APIKeyTablename string - RepoMetaTablename string - IndexDataTablename string - ManifestDataTablename string - UserDataTablename string - VersionTablename string - Patches []func(client *dynamodb.Client, tableNames map[string]string) error - imgTrustStore mTypes.ImageTrustStore - Log log.Logger + Client *dynamodb.Client + APIKeyTablename string + RepoMetaTablename string + RepoBlobsTablename string + ImageMetaTablename string + UserDataTablename string + VersionTablename string + Patches []func(client *dynamodb.Client, tableNames map[string]string) error + imgTrustStore mTypes.ImageTrustStore + Log log.Logger } func New( client *dynamodb.Client, params DBDriverParameters, log log.Logger, ) (*DynamoDB, error) { dynamoWrapper := DynamoDB{ - Client: client, - RepoMetaTablename: params.RepoMetaTablename, - ManifestDataTablename: params.ManifestDataTablename, - IndexDataTablename: params.IndexDataTablename, - VersionTablename: params.VersionTablename, - UserDataTablename: params.UserDataTablename, - APIKeyTablename: params.APIKeyTablename, - Patches: version.GetDynamoDBPatches(), - imgTrustStore: nil, - Log: log, + Client: client, + VersionTablename: params.VersionTablename, + UserDataTablename: params.UserDataTablename, + APIKeyTablename: params.APIKeyTablename, + RepoMetaTablename: params.RepoMetaTablename, + ImageMetaTablename: params.ImageMetaTablename, + RepoBlobsTablename: params.RepoBlobsInfoTablename, + Patches: version.GetDynamoDBPatches(), + imgTrustStore: nil, + Log: log, } err := dynamoWrapper.createVersionTable() @@ -61,27 +62,27 @@ func New( return nil, err } - err = dynamoWrapper.createRepoMetaTable() + err = dynamoWrapper.createTable(dynamoWrapper.RepoMetaTablename) if err != nil { return nil, err } - err = dynamoWrapper.createManifestDataTable() + err = dynamoWrapper.createTable(dynamoWrapper.RepoBlobsTablename) if err != nil { return nil, err } - err = dynamoWrapper.createIndexDataTable() + err = dynamoWrapper.createTable(dynamoWrapper.ImageMetaTablename) if err != nil { return nil, err } - err = dynamoWrapper.createUserDataTable() + err = dynamoWrapper.createTable(dynamoWrapper.UserDataTablename) if err != nil { return nil, err } - err = dynamoWrapper.createAPIKeyTable() + err = dynamoWrapper.createTable(dynamoWrapper.APIKeyTablename) if err != nil { return nil, err } @@ -98,147 +99,794 @@ func (dwr *DynamoDB) SetImageTrustStore(imgTrustStore mTypes.ImageTrustStore) { dwr.imgTrustStore = imgTrustStore } -func (dwr *DynamoDB) SetManifestData(manifestDigest godigest.Digest, manifestData mTypes.ManifestData) error { - mdAttributeValue, err := attributevalue.Marshal(manifestData) +func (dwr *DynamoDB) SetProtoImageMeta(digest godigest.Digest, protoImageMeta *proto_go.ImageMeta) error { + bytes, err := proto.Marshal(protoImageMeta) + if err != nil { + return err + } + + mdAttributeValue, err := attributevalue.Marshal(bytes) if err != nil { return err } _, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ ExpressionAttributeNames: map[string]string{ - "#MD": "ManifestData", + "#ID": "ImageMeta", }, ExpressionAttributeValues: map[string]types.AttributeValue{ - ":ManifestData": mdAttributeValue, + ":ImageMeta": mdAttributeValue, }, Key: map[string]types.AttributeValue{ - "Digest": &types.AttributeValueMemberS{ - Value: manifestDigest.String(), + "Key": &types.AttributeValueMemberS{ + Value: mConvert.GetImageDigestStr(protoImageMeta), }, }, - TableName: aws.String(dwr.ManifestDataTablename), - UpdateExpression: aws.String("SET #MD = :ManifestData"), + TableName: aws.String(dwr.ImageMetaTablename), + UpdateExpression: aws.String("SET #ID = :ImageMeta"), }) return err } -func (dwr *DynamoDB) GetManifestData(manifestDigest godigest.Digest) (mTypes.ManifestData, error) { - resp, err := dwr.Client.GetItem(context.Background(), &dynamodb.GetItemInput{ - TableName: aws.String(dwr.ManifestDataTablename), +func (dwr *DynamoDB) SetImageMeta(digest godigest.Digest, imageMeta mTypes.ImageMeta) error { + return dwr.SetProtoImageMeta(imageMeta.Digest, mConvert.GetProtoImageMeta(imageMeta)) +} + +func (dwr *DynamoDB) GetProtoImageMeta(ctx context.Context, digest godigest.Digest) (*proto_go.ImageMeta, error) { + resp, err := dwr.Client.GetItem(ctx, &dynamodb.GetItemInput{ + TableName: aws.String(dwr.ImageMetaTablename), Key: map[string]types.AttributeValue{ - "Digest": &types.AttributeValueMemberS{Value: manifestDigest.String()}, + "Key": &types.AttributeValueMemberS{Value: digest.String()}, }, }) if err != nil { - return mTypes.ManifestData{}, err + return nil, err + } + + blob := []byte{} + + if resp.Item == nil { + return nil, zerr.ErrImageMetaNotFound + } + + err = attributevalue.Unmarshal(resp.Item["ImageMeta"], &blob) + if err != nil { + return nil, err + } + + imageMeta := &proto_go.ImageMeta{} + + err = proto.Unmarshal(blob, imageMeta) + if err != nil { + return nil, err + } + + return imageMeta, nil +} + +func (dwr *DynamoDB) setProtoRepoMeta(repo string, repoMeta *proto_go.RepoMeta) error { + repoMeta.Name = repo + + repoMetaBlob, err := proto.Marshal(repoMeta) + if err != nil { + return err + } + + repoAttributeValue, err := attributevalue.Marshal(repoMetaBlob) + if err != nil { + return err + } + + _, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ + ExpressionAttributeNames: map[string]string{ + "#RM": "RepoMetadata", + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":RepoMetadata": repoAttributeValue, + }, + Key: map[string]types.AttributeValue{ + "Key": &types.AttributeValueMemberS{ + Value: repo, + }, + }, + TableName: aws.String(dwr.RepoMetaTablename), + UpdateExpression: aws.String("SET #RM = :RepoMetadata"), + }) + + return err +} + +func (dwr *DynamoDB) getProtoRepoMeta(ctx context.Context, repo string) (*proto_go.RepoMeta, error) { + resp, err := dwr.Client.GetItem(ctx, &dynamodb.GetItemInput{ + TableName: aws.String(dwr.RepoMetaTablename), + Key: map[string]types.AttributeValue{ + "Key": &types.AttributeValueMemberS{Value: repo}, + }, + }) + if err != nil { + return nil, err } if resp.Item == nil { - return mTypes.ManifestData{}, zerr.ErrManifestDataNotFound + return nil, zerr.ErrRepoMetaNotFound } - var manifestData mTypes.ManifestData + blob := []byte{} - err = attributevalue.Unmarshal(resp.Item["ManifestData"], &manifestData) + err = attributevalue.Unmarshal(resp.Item["RepoMetadata"], &blob) if err != nil { - return mTypes.ManifestData{}, err + return nil, err } - return manifestData, nil + repoMeta := &proto_go.RepoMeta{} + + err = proto.Unmarshal(blob, repoMeta) + if err != nil { + return nil, err + } + + return repoMeta, nil } -func (dwr *DynamoDB) SetManifestMeta(repo string, manifestDigest godigest.Digest, manifestMeta mTypes.ManifestMetadata, -) error { - if manifestMeta.Signatures == nil { - manifestMeta.Signatures = mTypes.ManifestSignatures{} +func (dwr *DynamoDB) SetRepoReference(repo string, reference string, imageMeta mTypes.ImageMeta) error { + if err := common.ValidateRepoReferenceInput(repo, reference, imageMeta.Digest); err != nil { + return err } - repoMeta, err := dwr.GetRepoMeta(repo) + // 1. Add image data to db if needed + protoImageMeta := mConvert.GetProtoImageMeta(imageMeta) + + err := dwr.SetProtoImageMeta(imageMeta.Digest, protoImageMeta) + if err != nil { + return err + } + + repoMeta, err := dwr.getProtoRepoMeta(context.Background(), repo) if err != nil { if !errors.Is(err, zerr.ErrRepoMetaNotFound) { return err } - repoMeta = mTypes.RepoMetadata{ + repoMeta = &proto_go.RepoMeta{ Name: repo, - Tags: map[string]mTypes.Descriptor{}, - Statistics: map[string]mTypes.DescriptorStatistics{}, - Signatures: map[string]mTypes.ManifestSignatures{}, - Referrers: map[string][]mTypes.ReferrerInfo{}, + Tags: map[string]*proto_go.TagDescriptor{"": {}}, + Statistics: map[string]*proto_go.DescriptorStatistics{"": {}}, + Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}}, + Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, } } - err = dwr.SetManifestData(manifestDigest, mTypes.ManifestData{ - ManifestBlob: manifestMeta.ManifestBlob, - ConfigBlob: manifestMeta.ConfigBlob, + // 2. Referrers + if subject := mConvert.GetImageSubject(protoImageMeta); subject != nil { + refInfo := &proto_go.ReferrersInfo{} + if repoMeta.Referrers[subject.Digest.String()] != nil { + refInfo = repoMeta.Referrers[subject.Digest.String()] + } + + foundReferrer := false + + for i := range refInfo.List { + if refInfo.List[i].Digest == mConvert.GetImageDigestStr(protoImageMeta) { + foundReferrer = true + refInfo.List[i].Count += 1 + + break + } + } + + if !foundReferrer { + refInfo.List = append(refInfo.List, &proto_go.ReferrerInfo{ + Count: 1, + MediaType: protoImageMeta.MediaType, + Digest: mConvert.GetImageDigestStr(protoImageMeta), + ArtifactType: mConvert.GetImageArtifactType(protoImageMeta), + Size: mConvert.GetImageManifestSize(protoImageMeta), + Annotations: mConvert.GetImageAnnotations(protoImageMeta), + }) + } + + repoMeta.Referrers[subject.Digest.String()] = refInfo + } + + // 3. Update tag + if !common.ReferenceIsDigest(reference) { + repoMeta.Tags[reference] = &proto_go.TagDescriptor{ + Digest: imageMeta.Digest.String(), + MediaType: imageMeta.MediaType, + } + } + + if _, ok := repoMeta.Statistics[imageMeta.Digest.String()]; !ok { + repoMeta.Statistics[imageMeta.Digest.String()] = &proto_go.DescriptorStatistics{DownloadCount: 0} + } + + if _, ok := repoMeta.Signatures[imageMeta.Digest.String()]; !ok { + repoMeta.Signatures[imageMeta.Digest.String()] = &proto_go.ManifestSignatures{ + Map: map[string]*proto_go.SignaturesInfo{"": {}}, + } + } + + if _, ok := repoMeta.Referrers[imageMeta.Digest.String()]; !ok { + repoMeta.Referrers[imageMeta.Digest.String()] = &proto_go.ReferrersInfo{ + List: []*proto_go.ReferrerInfo{}, + } + } + + // 4. Blobs + repoBlobs, err := dwr.getRepoBlobsInfo(repo) + if err != nil { + return err + } + + repoMeta, repoBlobs, err = common.AddImageMetaToRepoMeta(repoMeta, repoBlobs, reference, imageMeta) + if err != nil { + return err + } + + err = dwr.setRepoBlobsInfo(repo, repoBlobs) + if err != nil { + return err + } + + return dwr.setProtoRepoMeta(repo, repoMeta) +} + +func (dwr *DynamoDB) getRepoBlobsInfo(repo string) (*proto_go.RepoBlobs, error) { + resp, err := dwr.Client.GetItem(context.Background(), &dynamodb.GetItemInput{ + TableName: aws.String(dwr.RepoBlobsTablename), + Key: map[string]types.AttributeValue{ + "Key": &types.AttributeValueMemberS{Value: repo}, + }, }) if err != nil { - return err + return nil, err } - updatedRepoMeta := common.UpdateManifestMeta(repoMeta, manifestDigest, manifestMeta) + if resp.Item == nil { + return &proto_go.RepoBlobs{Name: repo, Blobs: map[string]*proto_go.BlobInfo{"": {}}}, nil + } - err = dwr.SetRepoMeta(repo, updatedRepoMeta) + repoBlobsBytes := []byte{} + + err = attributevalue.Unmarshal(resp.Item["RepoBlobsInfo"], &repoBlobsBytes) + if err != nil { + return nil, err + } + + repoBlobs := &proto_go.RepoBlobs{} + if repoBlobsBytes == nil { + repoBlobs.Blobs = map[string]*proto_go.BlobInfo{} + } else { + err := proto.Unmarshal(repoBlobsBytes, repoBlobs) + if err != nil { + return nil, err + } + } + + return repoBlobs, nil +} + +func (dwr *DynamoDB) setRepoBlobsInfo(repo string, repoBlobs *proto_go.RepoBlobs) error { + bytes, err := proto.Marshal(repoBlobs) if err != nil { return err } + mdAttributeValue, err := attributevalue.Marshal(bytes) + if err != nil { + return err + } + + _, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ + ExpressionAttributeNames: map[string]string{ + "#RBI": "RepoBlobsInfo", + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":RepoBlobsInfo": mdAttributeValue, + }, + Key: map[string]types.AttributeValue{ + "Key": &types.AttributeValueMemberS{ + Value: repo, + }, + }, + TableName: aws.String(dwr.RepoBlobsTablename), + UpdateExpression: aws.String("SET #RBI = :RepoBlobsInfo"), + }) + return err } -func (dwr *DynamoDB) GetManifestMeta(repo string, manifestDigest godigest.Digest, -) (mTypes.ManifestMetadata, error) { //nolint:contextcheck - manifestData, err := dwr.GetManifestData(manifestDigest) - if err != nil { - if errors.Is(err, zerr.ErrManifestDataNotFound) { - return mTypes.ManifestMetadata{}, zerr.ErrManifestMetaNotFound +func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string) ([]mTypes.RepoMeta, error) { + repos := []mTypes.RepoMeta{} + + userBookmarks := getUserBookmarks(ctx, dwr) + userStars := getUserStars(ctx, dwr) + + repoMetaAttributeIterator := NewBaseDynamoAttributesIterator( + dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, + ) + + repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) + + for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { + if err != nil { + return []mTypes.RepoMeta{}, err } - return mTypes.ManifestMetadata{}, - fmt.Errorf("%w for manifest '%s' from repo '%s'", errMetaDB, manifestDigest, repo) + repoMetaBlob := []byte{} + + err := attributevalue.Unmarshal(repoMetaAttribute, &repoMetaBlob) + if err != nil { + return []mTypes.RepoMeta{}, err + } + + protoRepoMeta := &proto_go.RepoMeta{} + + err = proto.Unmarshal(repoMetaBlob, protoRepoMeta) + if err != nil { + return []mTypes.RepoMeta{}, err + } + + if ok, err := reqCtx.RepoIsUserAvailable(ctx, protoRepoMeta.Name); !ok || err != nil { + continue + } + + rank := common.RankRepoName(searchText, protoRepoMeta.Name) + if rank == -1 { + continue + } + + protoRepoMeta.Rank = int32(rank) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) + + repos = append(repos, mConvert.GetRepoMeta(protoRepoMeta)) } - repoMeta, err := dwr.GetRepoMeta(repo) + return repos, nil +} + +func (dwr *DynamoDB) SearchTags(ctx context.Context, searchText string) ([]mTypes.FullImageMeta, error) { + images := []mTypes.FullImageMeta{} + userBookmarks := getUserBookmarks(ctx, dwr) + userStars := getUserStars(ctx, dwr) + + searchedRepo, searchedTag, err := common.GetRepoTag(searchText) + if err != nil { + return []mTypes.FullImageMeta{}, + fmt.Errorf("metadb: error while parsing search text, invalid format %w", err) + } + + if ok, err := reqCtx.RepoIsUserAvailable(ctx, searchedRepo); !ok || err != nil { + return []mTypes.FullImageMeta{}, err + } + + protoRepoMeta, err := dwr.getProtoRepoMeta(ctx, searchedRepo) if err != nil { if errors.Is(err, zerr.ErrRepoMetaNotFound) { - return mTypes.ManifestMetadata{}, zerr.ErrManifestMetaNotFound + return []mTypes.FullImageMeta{}, nil } - return mTypes.ManifestMetadata{}, - fmt.Errorf("%w for manifest '%s' from repo '%s'", errMetaDB, manifestDigest, repo) + return nil, err } - manifestMetadata := mTypes.ManifestMetadata{} + delete(protoRepoMeta.Tags, "") - manifestMetadata.ManifestBlob = manifestData.ManifestBlob - manifestMetadata.ConfigBlob = manifestData.ConfigBlob - manifestMetadata.DownloadCount = repoMeta.Statistics[manifestDigest.String()].DownloadCount + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, searchedRepo) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, searchedRepo) - manifestMetadata.Signatures = mTypes.ManifestSignatures{} + for tag, descriptor := range protoRepoMeta.Tags { + if !strings.HasPrefix(tag, searchedTag) { + continue + } - if repoMeta.Signatures[manifestDigest.String()] != nil { - manifestMetadata.Signatures = repoMeta.Signatures[manifestDigest.String()] + var protoImageMeta *proto_go.ImageMeta + + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestDigest := descriptor.Digest + + imageManifestData, err := dwr.GetProtoImageMeta(ctx, godigest.Digest(manifestDigest)) + if err != nil { + return []mTypes.FullImageMeta{}, + fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", manifestDigest, err) + } + + protoImageMeta = imageManifestData + case ispec.MediaTypeImageIndex: + indexDigest := godigest.Digest(descriptor.Digest) + + imageIndexData, err := dwr.GetProtoImageMeta(ctx, indexDigest) + if err != nil { + return []mTypes.FullImageMeta{}, + fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", indexDigest, err) + } + + manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests)) + + for _, manifest := range imageIndexData.Index.Index.Manifests { + manifestDigest := godigest.Digest(manifest.Digest) + + imageManifestData, err := dwr.GetProtoImageMeta(ctx, manifestDigest) + if err != nil { + return []mTypes.FullImageMeta{}, err + } + + manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + } + + imageIndexData.Manifests = manifestDataList + + protoImageMeta = imageIndexData + default: + dwr.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") + + continue + } + + images = append(images, mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, protoImageMeta)) } - return manifestMetadata, nil + return images, err +} + +func (dwr *DynamoDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, + filterFunc mTypes.FilterFunc, +) ([]mTypes.FullImageMeta, error) { + images := []mTypes.FullImageMeta{} + userBookmarks := getUserBookmarks(ctx, dwr) + userStars := getUserStars(ctx, dwr) + + var viewError error + + repoMetaAttributeIterator := NewBaseDynamoAttributesIterator( + dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, + ) + + repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) + + for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { + if err != nil { + viewError = errors.Join(viewError, err) + + continue + } + + protoRepoMeta, err := getProtoRepoMetaFromAttribute(repoMetaAttribute) + if err != nil { + viewError = errors.Join(viewError, err) + + continue + } + + if ok, err := reqCtx.RepoIsUserAvailable(ctx, protoRepoMeta.Name); !ok || err != nil { + continue + } + + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) + repoMeta := mConvert.GetRepoMeta(protoRepoMeta) + + for tag, descriptor := range repoMeta.Tags { + if !filterRepoTag(repoMeta.Name, tag) { + continue + } + + switch descriptor.MediaType { + case ispec.MediaTypeImageManifest: + manifestDigest := descriptor.Digest + + imageManifestData, err := dwr.GetProtoImageMeta(ctx, godigest.Digest(manifestDigest)) + if err != nil { + viewError = errors.Join(viewError, err) + + continue + } + + imageMeta := mConvert.GetImageMeta(imageManifestData) + + if filterFunc(repoMeta, imageMeta) { + images = append(images, mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, imageManifestData)) + } + case ispec.MediaTypeImageIndex: + indexDigest := descriptor.Digest + + imageIndexData, err := dwr.GetProtoImageMeta(ctx, godigest.Digest(indexDigest)) + if err != nil { + viewError = errors.Join(viewError, err) + + continue + } + + matchedManifests := []*proto_go.ManifestMeta{} + + for _, manifest := range imageIndexData.Index.Index.Manifests { + manifestDigest := manifest.Digest + + imageManifestData, err := dwr.GetProtoImageMeta(ctx, godigest.Digest(manifestDigest)) + if err != nil { + viewError = errors.Join(viewError, err) + + continue + } + + imageMeta := mConvert.GetImageMeta(imageManifestData) + + if filterFunc(repoMeta, imageMeta) { + matchedManifests = append(matchedManifests, imageManifestData.Manifests[0]) + } + } + + if len(matchedManifests) > 0 { + imageIndexData.Manifests = matchedManifests + + images = append(images, mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, imageIndexData)) + } + default: + dwr.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") + + continue + } + } + } + + viewError = errors.Join(viewError, err) + + return images, viewError +} + +func getProtoRepoMetaFromAttribute(repoMetaAttribute types.AttributeValue) (*proto_go.RepoMeta, error) { + blob := []byte{} + + err := attributevalue.Unmarshal(repoMetaAttribute, &blob) + if err != nil { + return nil, err + } + + protoRepoMeta := &proto_go.RepoMeta{} + + err = proto.Unmarshal(blob, protoRepoMeta) + if err != nil { + return nil, err + } + + return protoRepoMeta, nil +} + +func getProtoImageMetaFromAttribute(imageMetaAttribute types.AttributeValue) (*proto_go.ImageMeta, error) { + blob := []byte{} + + err := attributevalue.Unmarshal(imageMetaAttribute, &blob) + if err != nil { + return nil, err + } + + protoImageMeta := &proto_go.ImageMeta{} + + err = proto.Unmarshal(blob, protoImageMeta) + if err != nil { + return nil, err + } + + return protoImageMeta, nil +} + +func (dwr *DynamoDB) ResetRepoReferences(repo string) error { + protoRepoMeta, err := dwr.getProtoRepoMeta(context.Background(), repo) + if err != nil { + return err + } + + return dwr.setProtoRepoMeta(repo, &proto_go.RepoMeta{ + Name: repo, + Statistics: protoRepoMeta.Statistics, + Stars: protoRepoMeta.Stars, + Tags: map[string]*proto_go.TagDescriptor{"": {}}, + Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, + Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}}, + }) +} + +func (dwr *DynamoDB) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + protoRepoMeta, err := dwr.getProtoRepoMeta(ctx, repo) + if err != nil { + return mTypes.RepoMeta{}, err + } + + delete(protoRepoMeta.Tags, "") + + userBookmarks := getUserBookmarks(ctx, dwr) + userStars := getUserStars(ctx, dwr) + + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repo) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, repo) + + return mConvert.GetRepoMeta(protoRepoMeta), nil +} + +func (dwr *DynamoDB) GetFullImageMeta(ctx context.Context, repo string, tag string) (mTypes.FullImageMeta, error) { + protoRepoMeta, err := dwr.getProtoRepoMeta(ctx, repo) + if err != nil { + return mTypes.FullImageMeta{}, err + } + + delete(protoRepoMeta.Tags, "") + + bookmarks, stars := dwr.getUserBookmarksAndStars(ctx) + + protoRepoMeta.IsBookmarked = zcommon.Contains(bookmarks, repo) + protoRepoMeta.IsStarred = zcommon.Contains(stars, repo) + + descriptor, ok := protoRepoMeta.Tags[tag] + if !ok { + return mTypes.FullImageMeta{}, zerr.ErrImageMetaNotFound + } + + protoImageMeta, err := dwr.GetProtoImageMeta(ctx, godigest.Digest(descriptor.Digest)) + if err != nil { + return mTypes.FullImageMeta{}, err + } + + if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { + manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) + + for _, manifest := range protoImageMeta.Index.Index.Manifests { + imageManifestData, err := dwr.GetProtoImageMeta(ctx, godigest.Digest(manifest.Digest)) + if err != nil { + return mTypes.FullImageMeta{}, err + } + + manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + } + + protoImageMeta.Manifests = manifestDataList + } + + return mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, protoImageMeta), nil +} + +func (dwr *DynamoDB) getUserBookmarksAndStars(ctx context.Context) ([]string, []string) { + userData, err := dwr.GetUserData(ctx) + if err != nil { + return []string{}, []string{} + } + + return userData.BookmarkedRepos, userData.StarredRepos +} + +func (dwr *DynamoDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, error) { + protoImageMeta, err := dwr.GetProtoImageMeta(context.Background(), digest) + if err != nil { + return mTypes.ImageMeta{}, err + } + + if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { + manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) + + for _, manifest := range protoImageMeta.Index.Index.Manifests { + manifestDigest := godigest.Digest(manifest.Digest) + + imageManifestData, err := dwr.GetProtoImageMeta(context.Background(), manifestDigest) + if err != nil { + return mTypes.ImageMeta{}, err + } + + manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + } + + protoImageMeta.Manifests = manifestDataList + } + + return mConvert.GetImageMeta(protoImageMeta), nil +} + +func (dwr *DynamoDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta mTypes.RepoMeta) bool, +) ([]mTypes.RepoMeta, error) { + var ( + foundRepos = []mTypes.RepoMeta{} + repoMetaAttributeIterator AttributesIterator + ) + + repoMetaAttributeIterator = NewBaseDynamoAttributesIterator( + dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, + ) + + repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) + + for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { + if err != nil { + return []mTypes.RepoMeta{}, err + } + + repoMetaBlob := []byte{} + + err := attributevalue.Unmarshal(repoMetaAttribute, &repoMetaBlob) + if err != nil { + return []mTypes.RepoMeta{}, err + } + + protoRepoMeta := &proto_go.RepoMeta{} + + err = proto.Unmarshal(repoMetaBlob, protoRepoMeta) + if err != nil { + return []mTypes.RepoMeta{}, err + } + + delete(protoRepoMeta.Tags, "") + + if ok, err := reqCtx.RepoIsUserAvailable(ctx, protoRepoMeta.Name); !ok || err != nil { + continue + } + + repoMeta := mConvert.GetRepoMeta(protoRepoMeta) + + if filter(repoMeta) { + foundRepos = append(foundRepos, repoMeta) + } + } + + return foundRepos, err +} + +func (dwr *DynamoDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRepoNameFunc, + filterFunc mTypes.FilterFullRepoFunc, +) ([]mTypes.RepoMeta, error) { + repos := []mTypes.RepoMeta{} + userBookmarks := getUserBookmarks(ctx, dwr) + userStars := getUserStars(ctx, dwr) + + repoMetaAttributeIterator := NewBaseDynamoAttributesIterator( + dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, + ) + + repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) + + for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { + if err != nil { + return []mTypes.RepoMeta{}, + err + } + + protoRepoMeta, err := getProtoRepoMetaFromAttribute(repoMetaAttribute) + if err != nil { + return nil, err + } + + if ok, err := reqCtx.RepoIsUserAvailable(ctx, protoRepoMeta.Name); !ok || err != nil { + continue + } + + if !acceptName(protoRepoMeta.Name) { + continue + } + + protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) + protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) + + fullRepoMeta := mConvert.GetRepoMeta(protoRepoMeta) + + if filterFunc(fullRepoMeta) { + repos = append(repos, fullRepoMeta) + } + } + + return repos, err } func (dwr *DynamoDB) IncrementRepoStars(repo string) error { - repoMeta, err := dwr.GetRepoMeta(repo) + repoMeta, err := dwr.getProtoRepoMeta(context.Background(), repo) if err != nil { return err } repoMeta.Stars++ - err = dwr.SetRepoMeta(repo, repoMeta) - - return err + return dwr.setProtoRepoMeta(repo, repoMeta) } func (dwr *DynamoDB) DecrementRepoStars(repo string) error { - repoMeta, err := dwr.GetRepoMeta(repo) + repoMeta, err := dwr.getProtoRepoMeta(context.Background(), repo) if err != nil { return err } @@ -247,192 +895,23 @@ func (dwr *DynamoDB) DecrementRepoStars(repo string) error { repoMeta.Stars-- } - err = dwr.SetRepoMeta(repo, repoMeta) - - return err + return dwr.setProtoRepoMeta(repo, repoMeta) } -func (dwr *DynamoDB) GetRepoStars(repo string) (int, error) { - repoMeta, err := dwr.GetRepoMeta(repo) - if err != nil { - return 0, err - } +func (dwr *DynamoDB) SetRepoMeta(repo string, repoMeta mTypes.RepoMeta) error { + protoRepoMeta := mConvert.GetProtoRepoMeta(repoMeta) - return repoMeta.Stars, nil + return dwr.setProtoRepoMeta(repo, protoRepoMeta) } -func (dwr *DynamoDB) SetIndexData(indexDigest godigest.Digest, indexData mTypes.IndexData) error { - indexAttributeValue, err := attributevalue.Marshal(indexData) - if err != nil { - return err - } - - _, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ - ExpressionAttributeNames: map[string]string{ - "#ID": "IndexData", - }, - ExpressionAttributeValues: map[string]types.AttributeValue{ - ":IndexData": indexAttributeValue, - }, - Key: map[string]types.AttributeValue{ - "IndexDigest": &types.AttributeValueMemberS{ - Value: indexDigest.String(), - }, - }, - TableName: aws.String(dwr.IndexDataTablename), - UpdateExpression: aws.String("SET #ID = :IndexData"), - }) - - return err -} - -func (dwr *DynamoDB) GetIndexData(indexDigest godigest.Digest) (mTypes.IndexData, error) { - resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ - TableName: aws.String(dwr.IndexDataTablename), - Key: map[string]types.AttributeValue{ - "IndexDigest": &types.AttributeValueMemberS{ - Value: indexDigest.String(), - }, - }, - }) - if err != nil { - return mTypes.IndexData{}, err - } - - if resp.Item == nil { - return mTypes.IndexData{}, zerr.ErrRepoMetaNotFound - } - - var indexData mTypes.IndexData - - err = attributevalue.Unmarshal(resp.Item["IndexData"], &indexData) - if err != nil { - return mTypes.IndexData{}, err - } - - return indexData, nil -} - -func (dwr DynamoDB) SetReferrer(repo string, referredDigest godigest.Digest, referrer mTypes.ReferrerInfo) error { - resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ - TableName: aws.String(dwr.RepoMetaTablename), - Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{Value: repo}, - }, - }) - if err != nil { - return err - } - - repoMeta := mTypes.RepoMetadata{ - Name: repo, - Tags: map[string]mTypes.Descriptor{}, - Statistics: map[string]mTypes.DescriptorStatistics{}, - Signatures: map[string]mTypes.ManifestSignatures{}, - Referrers: map[string][]mTypes.ReferrerInfo{}, - } - - if resp.Item != nil { - err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta) - if err != nil { - return err - } - } - - referrers := repoMeta.Referrers[referredDigest.String()] - - for i := range referrers { - if referrers[i].Digest == referrer.Digest { - return nil - } - } - - referrers = append(referrers, referrer) - - repoMeta.Referrers[referredDigest.String()] = referrers - - return dwr.SetRepoMeta(repo, repoMeta) -} - -func (dwr DynamoDB) GetReferrers(repo string, referredDigest godigest.Digest) ([]mTypes.ReferrerInfo, error) { - resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ - TableName: aws.String(dwr.RepoMetaTablename), - Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{Value: repo}, - }, - }) +func (dwr *DynamoDB) GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string, +) ([]mTypes.ReferrerInfo, error) { + repoMeta, err := dwr.GetRepoMeta(context.Background(), repo) if err != nil { return []mTypes.ReferrerInfo{}, err } - repoMeta := mTypes.RepoMetadata{ - Name: repo, - Tags: map[string]mTypes.Descriptor{}, - Statistics: map[string]mTypes.DescriptorStatistics{}, - Signatures: map[string]mTypes.ManifestSignatures{}, - Referrers: map[string][]mTypes.ReferrerInfo{}, - } - - if resp.Item != nil { - err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta) - if err != nil { - return []mTypes.ReferrerInfo{}, err - } - } - - return repoMeta.Referrers[referredDigest.String()], nil -} - -func (dwr DynamoDB) DeleteReferrer(repo string, referredDigest godigest.Digest, - referrerDigest godigest.Digest, -) error { - resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ - TableName: aws.String(dwr.RepoMetaTablename), - Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{Value: repo}, - }, - }) - if err != nil { - return err - } - - repoMeta := mTypes.RepoMetadata{ - Name: repo, - Tags: map[string]mTypes.Descriptor{}, - Statistics: map[string]mTypes.DescriptorStatistics{}, - Signatures: map[string]mTypes.ManifestSignatures{}, - Referrers: map[string][]mTypes.ReferrerInfo{}, - } - - if resp.Item != nil { - err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta) - if err != nil { - return err - } - } - - referrers := repoMeta.Referrers[referredDigest.String()] - - for i := range referrers { - if referrers[i].Digest == referrerDigest.String() { - referrers = append(referrers[:i], referrers[i+1:]...) - - break - } - } - - repoMeta.Referrers[referredDigest.String()] = referrers - - return dwr.SetRepoMeta(repo, repoMeta) -} - -func (dwr DynamoDB) GetReferrersInfo(repo string, referredDigest godigest.Digest, - artifactTypes []string, -) ([]mTypes.ReferrerInfo, error) { - referrersInfo, err := dwr.GetReferrers(repo, referredDigest) - if err != nil { - return nil, err - } + referrersInfo := repoMeta.Referrers[referredDigest.String()] filteredResults := make([]mTypes.ReferrerInfo, 0, len(referrersInfo)) @@ -447,243 +926,8 @@ func (dwr DynamoDB) GetReferrersInfo(repo string, referredDigest godigest.Digest return filteredResults, nil } -/* - RemoveRepoReference removes the tag from RepoMetadata if the reference is a tag, - -it also removes its corresponding digest from Statistics, Signatures and Referrers if there are no tags -pointing to it. -If the reference is a digest then it will remove the digest from Statistics, Signatures and Referrers only -if there are no tags pointing to the digest, otherwise it's noop. -*/ -func (dwr *DynamoDB) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest, -) error { - resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ - TableName: aws.String(dwr.RepoMetaTablename), - Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{Value: repo}, - }, - }) - if err != nil { - return err - } - - repoMeta := mTypes.RepoMetadata{ - Name: repo, - Tags: map[string]mTypes.Descriptor{}, - Statistics: map[string]mTypes.DescriptorStatistics{}, - Signatures: map[string]mTypes.ManifestSignatures{}, - Referrers: map[string][]mTypes.ReferrerInfo{}, - } - - if resp.Item != nil { - err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta) - if err != nil { - return err - } - } - - if !common.ReferenceIsDigest(reference) { - delete(repoMeta.Tags, reference) - } else { - // find all tags pointing to this digest - tags := []string{} - for tag, desc := range repoMeta.Tags { - if desc.Digest == reference { - tags = append(tags, tag) - } - } - - // remove all tags - for _, tag := range tags { - delete(repoMeta.Tags, tag) - } - } - - /* try to find at least one tag pointing to manifestDigest - if not found then we can also remove everything related to this digest */ - var foundTag bool - - for _, desc := range repoMeta.Tags { - if desc.Digest == manifestDigest.String() { - foundTag = true - } - } - - if !foundTag { - delete(repoMeta.Statistics, manifestDigest.String()) - delete(repoMeta.Signatures, manifestDigest.String()) - delete(repoMeta.Referrers, manifestDigest.String()) - } - - err = dwr.SetRepoMeta(repo, repoMeta) - - return err -} - -func (dwr *DynamoDB) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, - mediaType string, -) error { - if err := common.ValidateRepoReferenceInput(repo, reference, manifestDigest); err != nil { - return err - } - - resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ - TableName: aws.String(dwr.RepoMetaTablename), - Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{Value: repo}, - }, - }) - if err != nil { - return err - } - - repoMeta := mTypes.RepoMetadata{ - Name: repo, - Tags: map[string]mTypes.Descriptor{}, - Statistics: map[string]mTypes.DescriptorStatistics{}, - Signatures: map[string]mTypes.ManifestSignatures{}, - Referrers: map[string][]mTypes.ReferrerInfo{}, - } - - if resp.Item != nil { - err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta) - if err != nil { - return err - } - } - - if !common.ReferenceIsDigest(reference) { - repoMeta.Tags[reference] = mTypes.Descriptor{ - Digest: manifestDigest.String(), - MediaType: mediaType, - } - } - - if _, ok := repoMeta.Statistics[manifestDigest.String()]; !ok { - repoMeta.Statistics[manifestDigest.String()] = mTypes.DescriptorStatistics{DownloadCount: 0} - } - - if _, ok := repoMeta.Signatures[manifestDigest.String()]; !ok { - repoMeta.Signatures[manifestDigest.String()] = mTypes.ManifestSignatures{} - } - - if _, ok := repoMeta.Referrers[manifestDigest.String()]; !ok { - repoMeta.Referrers[manifestDigest.String()] = []mTypes.ReferrerInfo{} - } - - err = dwr.SetRepoMeta(repo, repoMeta) - - return err -} - -func (dwr *DynamoDB) DeleteRepoTag(repo string, tag string) error { - resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ - TableName: aws.String(dwr.RepoMetaTablename), - Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{Value: repo}, - }, - }) - if err != nil { - return err - } - - if resp.Item == nil { - return nil - } - - var repoMeta mTypes.RepoMetadata - - err = attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta) - if err != nil { - return err - } - - delete(repoMeta.Tags, tag) - - repoAttributeValue, err := attributevalue.Marshal(repoMeta) - if err != nil { - return err - } - - _, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ - ExpressionAttributeNames: map[string]string{ - "#RM": "RepoMetadata", - }, - ExpressionAttributeValues: map[string]types.AttributeValue{ - ":RepoMetadata": repoAttributeValue, - }, - Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{ - Value: repo, - }, - }, - TableName: aws.String(dwr.RepoMetaTablename), - UpdateExpression: aws.String("SET #RM = :RepoMetadata"), - }) - - return err -} - -func (dwr *DynamoDB) GetRepoMeta(repo string) (mTypes.RepoMetadata, error) { - resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ - TableName: aws.String(dwr.RepoMetaTablename), - Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{Value: repo}, - }, - }) - if err != nil { - return mTypes.RepoMetadata{}, err - } - - if resp.Item == nil { - return mTypes.RepoMetadata{}, zerr.ErrRepoMetaNotFound - } - - var repoMeta mTypes.RepoMetadata - - err = attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta) - if err != nil { - return mTypes.RepoMetadata{}, err - } - - return repoMeta, nil -} - -func (dwr *DynamoDB) GetUserRepoMeta(ctx context.Context, repo string) (mTypes.RepoMetadata, error) { - resp, err := dwr.Client.GetItem(ctx, &dynamodb.GetItemInput{ - TableName: aws.String(dwr.RepoMetaTablename), - Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{Value: repo}, - }, - }) - if err != nil { - return mTypes.RepoMetadata{}, err - } - - if resp.Item == nil { - return mTypes.RepoMetadata{}, zerr.ErrRepoMetaNotFound - } - - var repoMeta mTypes.RepoMetadata - - err = attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta) - if err != nil { - return mTypes.RepoMetadata{}, err - } - - userData, err := dwr.GetUserData(ctx) - if err != nil { - return mTypes.RepoMetadata{}, err - } - - repoMeta.IsBookmarked = zcommon.Contains(userData.BookmarkedRepos, repo) - repoMeta.IsStarred = zcommon.Contains(userData.StarredRepos, repo) - - return repoMeta, nil -} - func (dwr *DynamoDB) IncrementImageDownloads(repo string, reference string) error { - repoMeta, err := dwr.GetRepoMeta(repo) + repoMeta, err := dwr.getProtoRepoMeta(context.Background(), repo) if err != nil { return err } @@ -701,11 +945,15 @@ func (dwr *DynamoDB) IncrementImageDownloads(repo string, reference string) erro descriptorDigest = descriptor.Digest } - manifestStatistics := repoMeta.Statistics[descriptorDigest] + manifestStatistics, ok := repoMeta.Statistics[descriptorDigest] + if !ok { + return zerr.ErrManifestMetaNotFound + } + manifestStatistics.DownloadCount++ repoMeta.Statistics[descriptorDigest] = manifestStatistics - return dwr.SetRepoMeta(repo, repoMeta) + return dwr.setProtoRepoMeta(repo, repoMeta) } func (dwr *DynamoDB) UpdateSignaturesValidity(repo string, manifestDigest godigest.Digest) error { @@ -715,42 +963,28 @@ func (dwr *DynamoDB) UpdateSignaturesValidity(repo string, manifestDigest godige return nil } - // get ManifestData of signed manifest - var blob []byte - - manifestData, err := dwr.GetManifestData(manifestDigest) - if err != nil { - if errors.Is(err, zerr.ErrManifestDataNotFound) { - indexData, err := dwr.GetIndexData(manifestDigest) - if err != nil { - return nil //nolint: nilerr - } - - blob = indexData.IndexBlob - } else { - return fmt.Errorf("%w for manifest '%s' from repo '%s'", errMetaDB, manifestDigest, repo) - } - } else { - blob = manifestData.ManifestBlob - } - - // update signatures with details about validity and author - repoMeta, err := dwr.GetRepoMeta(repo) + protoImageMeta, err := dwr.GetProtoImageMeta(context.Background(), manifestDigest) if err != nil { return err } - manifestSignatures := mTypes.ManifestSignatures{} + // update signatures with details about validity and author + protoRepoMeta, err := dwr.getProtoRepoMeta(context.Background(), repo) + if err != nil { + return err + } - for sigType, sigs := range repoMeta.Signatures[manifestDigest.String()] { - signaturesInfo := []mTypes.SignatureInfo{} + manifestSignatures := proto_go.ManifestSignatures{Map: map[string]*proto_go.SignaturesInfo{"": {}}} - for _, sigInfo := range sigs { - layersInfo := []mTypes.LayerInfo{} + for sigType, sigs := range protoRepoMeta.Signatures[manifestDigest.String()].Map { + signaturesInfo := []*proto_go.SignatureInfo{} + + for _, sigInfo := range sigs.List { + layersInfo := []*proto_go.LayersInfo{} for _, layerInfo := range sigInfo.LayersInfo { - author, date, isTrusted, _ := imgTrustStore.VerifySignature(sigType, layerInfo.LayerContent, layerInfo.SignatureKey, - manifestDigest, blob, repo) + author, date, isTrusted, _ := imgTrustStore.VerifySignature(sigType, layerInfo.LayerContent, + layerInfo.SignatureKey, manifestDigest, mConvert.GetImageMeta(protoImageMeta), repo) if isTrusted { layerInfo.Signer = author @@ -758,90 +992,98 @@ func (dwr *DynamoDB) UpdateSignaturesValidity(repo string, manifestDigest godige if !date.IsZero() { layerInfo.Signer = author - layerInfo.Date = date + layerInfo.Date = timestamppb.New(date) } layersInfo = append(layersInfo, layerInfo) } - signaturesInfo = append(signaturesInfo, mTypes.SignatureInfo{ + signaturesInfo = append(signaturesInfo, &proto_go.SignatureInfo{ SignatureManifestDigest: sigInfo.SignatureManifestDigest, LayersInfo: layersInfo, }) } - manifestSignatures[sigType] = signaturesInfo + manifestSignatures.Map[sigType] = &proto_go.SignaturesInfo{List: signaturesInfo} } - repoMeta.Signatures[manifestDigest.String()] = manifestSignatures + protoRepoMeta.Signatures[manifestDigest.String()] = &manifestSignatures - return dwr.SetRepoMeta(repoMeta.Name, repoMeta) + return dwr.setProtoRepoMeta(protoRepoMeta.Name, protoRepoMeta) } func (dwr *DynamoDB) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, sygMeta mTypes.SignatureMetadata, ) error { - repoMeta, err := dwr.GetRepoMeta(repo) + protoRepoMeta, err := dwr.getProtoRepoMeta(context.Background(), repo) if err != nil { if errors.Is(err, zerr.ErrRepoMetaNotFound) { - repoMeta = mTypes.RepoMetadata{ + protoRepoMeta = &proto_go.RepoMeta{ Name: repo, - Tags: map[string]mTypes.Descriptor{}, - Statistics: map[string]mTypes.DescriptorStatistics{}, - Signatures: map[string]mTypes.ManifestSignatures{ + Tags: map[string]*proto_go.TagDescriptor{"": {}}, + Statistics: map[string]*proto_go.DescriptorStatistics{"": {}}, + Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, + Signatures: map[string]*proto_go.ManifestSignatures{ signedManifestDigest.String(): { - sygMeta.SignatureType: []mTypes.SignatureInfo{ - { - SignatureManifestDigest: sygMeta.SignatureDigest, - LayersInfo: sygMeta.LayersInfo, + Map: map[string]*proto_go.SignaturesInfo{ + sygMeta.SignatureType: { + List: []*proto_go.SignatureInfo{ + { + SignatureManifestDigest: sygMeta.SignatureDigest, + LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), + }, + }, }, }, }, }, - Referrers: map[string][]mTypes.ReferrerInfo{}, } - return dwr.SetRepoMeta(repo, repoMeta) + return dwr.setProtoRepoMeta(repo, protoRepoMeta) } return err } var ( - manifestSignatures mTypes.ManifestSignatures + manifestSignatures *proto_go.ManifestSignatures found bool ) - if manifestSignatures, found = repoMeta.Signatures[signedManifestDigest.String()]; !found { - manifestSignatures = mTypes.ManifestSignatures{} + if manifestSignatures, found = protoRepoMeta.Signatures[signedManifestDigest.String()]; !found { + manifestSignatures = &proto_go.ManifestSignatures{Map: map[string]*proto_go.SignaturesInfo{"": {}}} } - signatureSlice := manifestSignatures[sygMeta.SignatureType] - if !common.SignatureAlreadyExists(signatureSlice, sygMeta) { - if sygMeta.SignatureType == zcommon.NotationSignature { - signatureSlice = append(signatureSlice, mTypes.SignatureInfo{ + signatureSlice := &proto_go.SignaturesInfo{List: []*proto_go.SignatureInfo{}} + if sigSlice, found := manifestSignatures.Map[sygMeta.SignatureType]; found { + signatureSlice = sigSlice + } + + if !common.ProtoSignatureAlreadyExists(signatureSlice.List, sygMeta) { + switch sygMeta.SignatureType { + case zcommon.NotationSignature: + signatureSlice.List = append(signatureSlice.List, &proto_go.SignatureInfo{ SignatureManifestDigest: sygMeta.SignatureDigest, - LayersInfo: sygMeta.LayersInfo, + LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), }) - } else if sygMeta.SignatureType == zcommon.CosignSignature { - signatureSlice = []mTypes.SignatureInfo{{ + case zcommon.CosignSignature: + signatureSlice.List = []*proto_go.SignatureInfo{{ SignatureManifestDigest: sygMeta.SignatureDigest, - LayersInfo: sygMeta.LayersInfo, + LayersInfo: mConvert.GetProtoLayersInfo(sygMeta.LayersInfo), }} } } - manifestSignatures[sygMeta.SignatureType] = signatureSlice + manifestSignatures.Map[sygMeta.SignatureType] = signatureSlice + protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures - repoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures - - return dwr.SetRepoMeta(repoMeta.Name, repoMeta) + return dwr.setProtoRepoMeta(protoRepoMeta.Name, protoRepoMeta) } func (dwr *DynamoDB) DeleteSignature(repo string, signedManifestDigest godigest.Digest, sigMeta mTypes.SignatureMetadata, ) error { - repoMeta, err := dwr.GetRepoMeta(repo) + protoRepoMeta, err := dwr.getProtoRepoMeta(context.Background(), repo) if err != nil { return err } @@ -849,167 +1091,175 @@ func (dwr *DynamoDB) DeleteSignature(repo string, signedManifestDigest godigest. sigType := sigMeta.SignatureType var ( - manifestSignatures mTypes.ManifestSignatures + manifestSignatures *proto_go.ManifestSignatures found bool ) - if manifestSignatures, found = repoMeta.Signatures[signedManifestDigest.String()]; !found { + if manifestSignatures, found = protoRepoMeta.Signatures[signedManifestDigest.String()]; !found { return zerr.ErrManifestMetaNotFound } - signatureSlice := manifestSignatures[sigType] + signatureSlice := manifestSignatures.Map[sigType] - newSignatureSlice := make([]mTypes.SignatureInfo, 0, len(signatureSlice)-1) + newSignatureSlice := make([]*proto_go.SignatureInfo, 0, len(signatureSlice.List)-1) - for _, sigDigest := range signatureSlice { + for _, sigDigest := range signatureSlice.List { if sigDigest.SignatureManifestDigest != sigMeta.SignatureDigest { newSignatureSlice = append(newSignatureSlice, sigDigest) } } - manifestSignatures[sigType] = newSignatureSlice + manifestSignatures.Map[sigMeta.SignatureType] = &proto_go.SignaturesInfo{List: newSignatureSlice} - repoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures + protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures - err = dwr.SetRepoMeta(repoMeta.Name, repoMeta) - - return err + return dwr.setProtoRepoMeta(protoRepoMeta.Name, protoRepoMeta) } -func (dwr *DynamoDB) GetMultipleRepoMeta(ctx context.Context, - filter func(repoMeta mTypes.RepoMetadata) bool, -) ([]mTypes.RepoMetadata, error) { - var ( - foundRepos = []mTypes.RepoMetadata{} - repoMetaAttributeIterator AttributesIterator - ) - - repoMetaAttributeIterator = NewBaseDynamoAttributesIterator( - dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, - ) - - repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) - - for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { - if err != nil { - return []mTypes.RepoMetadata{}, err - } - - var repoMeta mTypes.RepoMetadata - - err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta) - if err != nil { - return []mTypes.RepoMetadata{}, err - } - - if ok, err := reqCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil { - continue - } - - if filter(repoMeta) { - foundRepos = append(foundRepos, repoMeta) - } +func (dwr *DynamoDB) FilterImageMeta(ctx context.Context, digests []string, +) (map[string]mTypes.ImageMeta, error) { + imageMetaAttributes, err := dwr.fetchImageMetaAttributesByDigest(ctx, digests) + if err != nil { + return nil, err } - return foundRepos, err + results := map[string]mTypes.ImageMeta{} + + for _, attributes := range imageMetaAttributes { + protoImageMeta, err := getProtoImageMetaFromAttribute(attributes["ImageMeta"]) + if err != nil { + return nil, err + } + + if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { + manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) + + indexDigests := make([]string, 0, len(protoImageMeta.Index.Index.Manifests)) + for i := range protoImageMeta.Index.Index.Manifests { + indexDigests = append(indexDigests, protoImageMeta.Index.Index.Manifests[i].Digest) + } + + manifestsAttributes, err := dwr.fetchImageMetaAttributesByDigest(ctx, indexDigests) + if err != nil { + return nil, err + } + + for _, manifestAttribute := range manifestsAttributes { + imageManifestData, err := getProtoImageMetaFromAttribute(manifestAttribute["ImageMeta"]) + if err != nil { + return nil, err + } + + manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) + } + + protoImageMeta.Manifests = manifestDataList + } + + results[mConvert.GetImageDigestStr(protoImageMeta)] = mConvert.GetImageMeta(protoImageMeta) + } + + return results, nil } -func (dwr *DynamoDB) SearchRepos(ctx context.Context, searchText string, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { - var ( - repos = []mTypes.RepoMetadata{} - manifestMetadataMap = make(map[string]mTypes.ManifestMetadata) - indexDataMap = make(map[string]mTypes.IndexData) - repoMetaAttributeIterator AttributesIterator - - userBookmarks = getUserBookmarks(ctx, dwr) - userStars = getUserStars(ctx, dwr) - ) - - repoMetaAttributeIterator = NewBaseDynamoAttributesIterator( - dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, - ) - - repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) - - for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - err +func (dwr *DynamoDB) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest, +) error { + protoRepoMeta, err := dwr.getProtoRepoMeta(context.Background(), repo) + if err != nil { + if errors.Is(err, zerr.ErrRepoMetaNotFound) { + return nil } - var repoMeta mTypes.RepoMetadata + return err + } - err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - err + protoImageMeta, err := dwr.GetProtoImageMeta(context.TODO(), manifestDigest) + if err != nil { + if errors.Is(err, zerr.ErrImageMetaNotFound) { + return nil } - if ok, err := reqCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil { - continue + return err + } + + // Remove Referrers + if subject := mConvert.GetImageSubject(protoImageMeta); subject != nil { + referredDigest := subject.Digest.String() + refInfo := &proto_go.ReferrersInfo{} + + if protoRepoMeta.Referrers[referredDigest] != nil { + refInfo = protoRepoMeta.Referrers[referredDigest] } - rank := common.RankRepoName(searchText, repoMeta.Name) - if rank == -1 { - continue - } + referrers := refInfo.List - repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) - repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) - repoMeta.Rank = rank + for i := range referrers { + if referrers[i].Digest == manifestDigest.String() { + referrers[i].Count -= 1 - for _, descriptor := range repoMeta.Tags { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDigest := descriptor.Digest - - manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck - manifestMetadataMap) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - err + if referrers[i].Count == 0 || common.ReferenceIsDigest(reference) { + referrers = append(referrers[:i], referrers[i+1:]...) } - manifestMetadataMap[descriptor.Digest] = manifestMeta - case ispec.MediaTypeImageIndex: - indexData, err := dwr.fetchIndexDataWithCheck(descriptor.Digest, indexDataMap) //nolint:contextcheck - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - err - } - - var indexContent ispec.Index - - err = json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - fmt.Errorf("metadb: error while unmarshaling index content for digest %s %w", descriptor.Digest, err) - } - - for _, manifest := range indexContent.Manifests { - manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifest.Digest.String(), //nolint: contextcheck - manifestMetadataMap) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - err - } - - manifestMetadataMap[manifest.Digest.String()] = manifestMeta - } - - indexDataMap[descriptor.Digest] = indexData - default: - dwr.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") - - continue + break } } - repos = append(repos, repoMeta) + refInfo.List = referrers + + protoRepoMeta.Referrers[referredDigest] = refInfo } - return repos, manifestMetadataMap, indexDataMap, nil + if !common.ReferenceIsDigest(reference) { + delete(protoRepoMeta.Tags, reference) + } else { + // find all tags pointing to this digest + tags := []string{} + for tag, desc := range protoRepoMeta.Tags { + if desc.Digest == reference { + tags = append(tags, tag) + } + } + + // remove all tags + for _, tag := range tags { + delete(protoRepoMeta.Tags, tag) + } + } + + /* try to find at least one tag pointing to manifestDigest + if not found then we can also remove everything related to this digest */ + var foundTag bool + + for _, desc := range protoRepoMeta.Tags { + if desc.Digest == manifestDigest.String() { + foundTag = true + } + } + + if !foundTag { + delete(protoRepoMeta.Statistics, manifestDigest.String()) + delete(protoRepoMeta.Signatures, manifestDigest.String()) + delete(protoRepoMeta.Referrers, manifestDigest.String()) + } + + repoBlobsInfo, err := dwr.getRepoBlobsInfo(repo) + if err != nil { + return err + } + + protoRepoMeta, repoBlobsInfo, err = common.RemoveImageFromRepoMeta(protoRepoMeta, repoBlobsInfo, reference) + if err != nil { + return err + } + + err = dwr.setRepoBlobsInfo(repo, repoBlobsInfo) + if err != nil { + return err + } + err = dwr.setProtoRepoMeta(repo, protoRepoMeta) + + return err } func getUserStars(ctx context.Context, dwr *DynamoDB) []string { @@ -1030,641 +1280,6 @@ func getUserBookmarks(ctx context.Context, dwr *DynamoDB) []string { return bookmarkedRepos } -func (dwr *DynamoDB) fetchManifestMetaWithCheck(repoName string, manifestDigest string, - manifestMetadataMap map[string]mTypes.ManifestMetadata, -) (mTypes.ManifestMetadata, error) { - var ( - manifestMeta mTypes.ManifestMetadata - err error - ) - - manifestMeta, manifestDownloaded := manifestMetadataMap[manifestDigest] - - if !manifestDownloaded { - manifestMeta, err = dwr.GetManifestMeta(repoName, godigest.Digest(manifestDigest)) //nolint:contextcheck - if err != nil { - return mTypes.ManifestMetadata{}, err - } - } - - return manifestMeta, nil -} - -func (dwr *DynamoDB) fetchIndexDataWithCheck(indexDigest string, indexDataMap map[string]mTypes.IndexData, -) (mTypes.IndexData, error) { - var ( - indexData mTypes.IndexData - err error - ) - - indexData, indexExists := indexDataMap[indexDigest] - - if !indexExists { - indexData, err = dwr.GetIndexData(godigest.Digest(indexDigest)) //nolint:contextcheck - if err != nil { - return mTypes.IndexData{}, - fmt.Errorf("metadb: error while unmarshaling index data for digest %s \n%w", indexDigest, err) - } - } - - return indexData, err -} - -func (dwr *DynamoDB) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error, -) { - var ( - foundRepos = make([]mTypes.RepoMetadata, 0) - manifestMetadataMap = make(map[string]mTypes.ManifestMetadata) - indexDataMap = make(map[string]mTypes.IndexData) - repoMetaAttributeIterator AttributesIterator - userBookmarks = getUserBookmarks(ctx, dwr) - userStars = getUserStars(ctx, dwr) - aggregateError error - ) - - repoMetaAttributeIterator = NewBaseDynamoAttributesIterator( - dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, - ) - - repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) - if err != nil { - return foundRepos, manifestMetadataMap, indexDataMap, err - } - - for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { - if err != nil { - aggregateError = errors.Join(aggregateError, err) - - continue - } - - var repoMeta mTypes.RepoMetadata - - err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta) - if err != nil { - aggregateError = errors.Join(aggregateError, err) - - continue - } - - if ok, err := reqCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil { - continue - } - - repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) - repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) - - matchedTags := make(map[string]mTypes.Descriptor) - - for tag, descriptor := range repoMeta.Tags { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDigest := descriptor.Digest - - manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck - manifestMetadataMap) - if err != nil { - err = fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s \n%w", manifestDigest, err) - aggregateError = errors.Join(aggregateError, err) - - continue - } - - if filterFunc(repoMeta, manifestMeta) { - matchedTags[tag] = descriptor - manifestMetadataMap[manifestDigest] = manifestMeta - } - case ispec.MediaTypeImageIndex: - indexDigest := descriptor.Digest - - indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck - if err != nil { - err = fmt.Errorf("metadb: error while getting index data for digest %s %w", indexDigest, err) - aggregateError = errors.Join(aggregateError, err) - - continue - } - - var indexContent ispec.Index - - err = json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - err = fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err) - aggregateError = errors.Join(aggregateError, err) - - continue - } - - matchedManifests := []ispec.Descriptor{} - - for _, manifest := range indexContent.Manifests { - manifestDigest := manifest.Digest.String() - - manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck - manifestMetadataMap) - if err != nil { - err = fmt.Errorf("%w metadb: error while getting manifest data for digest %s", err, manifestDigest) - aggregateError = errors.Join(aggregateError, err) - - continue - } - - if filterFunc(repoMeta, manifestMeta) { - matchedManifests = append(matchedManifests, manifest) - manifestMetadataMap[manifestDigest] = manifestMeta - } - } - - if len(matchedManifests) > 0 { - indexContent.Manifests = matchedManifests - - indexBlob, err := json.Marshal(indexContent) - if err != nil { - aggregateError = errors.Join(aggregateError, err) - - continue - } - - indexData.IndexBlob = indexBlob - - indexDataMap[indexDigest] = indexData - matchedTags[tag] = descriptor - } - default: - dwr.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") - - continue - } - } - - if len(matchedTags) == 0 { - continue - } - - repoMeta.Tags = matchedTags - - foundRepos = append(foundRepos, repoMeta) - } - - return foundRepos, manifestMetadataMap, indexDataMap, aggregateError -} - -func (dwr *DynamoDB) FilterRepos(ctx context.Context, filter mTypes.FilterRepoFunc, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { - var ( - foundRepos = []mTypes.RepoMetadata{} - repoMetaAttributeIterator AttributesIterator - userBookmarks = getUserBookmarks(ctx, dwr) - userStars = getUserStars(ctx, dwr) - ) - - repoMetaAttributeIterator = NewBaseDynamoAttributesIterator( - dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, - ) - - repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) - - for ; repoMetaAttribute != nil; repoMetaAttribute, err = repoMetaAttributeIterator.Next(ctx) { - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - err - } - - var repoMeta mTypes.RepoMetadata - - err := attributevalue.Unmarshal(repoMetaAttribute, &repoMeta) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - err - } - - if ok, err := reqCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil { - continue - } - - repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) - repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) - - if filter(repoMeta) { - foundRepos = append(foundRepos, repoMeta) - } - } - - foundManifestMetadataMap, foundIndexDataMap, err := common.FetchDataForRepos(dwr, foundRepos) - - return foundRepos, foundManifestMetadataMap, foundIndexDataMap, err -} - -func (dwr *DynamoDB) SearchTags(ctx context.Context, searchText string, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error, -) { - var ( - foundRepos = make([]mTypes.RepoMetadata, 0, 1) - manifestMetadataMap = make(map[string]mTypes.ManifestMetadata) - indexDataMap = make(map[string]mTypes.IndexData) - repoMetaAttributeIterator AttributesIterator - userBookmarks = getUserBookmarks(ctx, dwr) - userStars = getUserStars(ctx, dwr) - ) - - repoMetaAttributeIterator = NewBaseDynamoAttributesIterator( - dwr.Client, dwr.RepoMetaTablename, "RepoMetadata", 0, dwr.Log, - ) - - searchedRepo, searchedTag, err := common.GetRepoTag(searchText) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - fmt.Errorf("metadb: error while parsing search text, invalid format %w", err) - } - - repoMetaAttribute, err := repoMetaAttributeIterator.First(ctx) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - err - } - - var repoMeta mTypes.RepoMetadata - - err = attributevalue.Unmarshal(repoMetaAttribute, &repoMeta) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - err - } - - if ok, err := reqCtx.RepoIsUserAvailable(ctx, repoMeta.Name); !ok || err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - err - } - - if repoMeta.Name != searchedRepo { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - err - } - - repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) - repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) - - matchedTags := make(map[string]mTypes.Descriptor) - - for tag, descriptor := range repoMeta.Tags { - if !strings.HasPrefix(tag, searchedTag) { - continue - } - - matchedTags[tag] = descriptor - - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestDigest := descriptor.Digest - - manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck - manifestMetadataMap) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - fmt.Errorf("metadb: error while unmashaling manifest metadata for digest %s %w", descriptor.Digest, err) - } - - manifestMetadataMap[descriptor.Digest] = manifestMeta - case ispec.MediaTypeImageIndex: - indexDigest := descriptor.Digest - - indexData, err := dwr.fetchIndexDataWithCheck(indexDigest, indexDataMap) //nolint:contextcheck - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - fmt.Errorf("%w", err) - } - - var indexContent ispec.Index - - err = json.Unmarshal(indexData.IndexBlob, &indexContent) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - fmt.Errorf("metadb: error while unmashaling index content for digest %s %w", indexDigest, err) - } - - for _, manifest := range indexContent.Manifests { - manifestDigest := manifest.Digest.String() - - manifestMeta, err := dwr.fetchManifestMetaWithCheck(repoMeta.Name, manifestDigest, //nolint:contextcheck - manifestMetadataMap) - if err != nil { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - fmt.Errorf("%w", err) - } - - manifestMetadataMap[manifestDigest] = manifestMeta - } - - indexDataMap[indexDigest] = indexData - default: - dwr.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") - - continue - } - } - - if len(matchedTags) == 0 { - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, map[string]mTypes.IndexData{}, - err - } - - repoMeta.Tags = matchedTags - - foundRepos = append(foundRepos, repoMeta) - - return foundRepos, manifestMetadataMap, indexDataMap, err -} - -func (dwr *DynamoDB) PatchDB() error { - DBVersion, err := dwr.getDBVersion() - if err != nil { - return fmt.Errorf("patching dynamo failed, error retrieving database version %w", err) - } - - if version.GetVersionIndex(DBVersion) == -1 { - return fmt.Errorf("DB has broken format, no version found %w", err) - } - - for patchIndex, patch := range dwr.Patches { - if patchIndex < version.GetVersionIndex(DBVersion) { - continue - } - - tableNames := map[string]string{ - "RepoMetaTablename": dwr.RepoMetaTablename, - "ManifestDataTablename": dwr.ManifestDataTablename, - "VersionTablename": dwr.VersionTablename, - } - - err := patch(dwr.Client, tableNames) - if err != nil { - return err - } - } - - return nil -} - -func (dwr *DynamoDB) SetRepoMeta(repo string, repoMeta mTypes.RepoMetadata) error { - repoMeta.Name = repo - - repoAttributeValue, err := attributevalue.Marshal(repoMeta) - if err != nil { - return err - } - - _, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ - ExpressionAttributeNames: map[string]string{ - "#RM": "RepoMetadata", - }, - ExpressionAttributeValues: map[string]types.AttributeValue{ - ":RepoMetadata": repoAttributeValue, - }, - Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{ - Value: repo, - }, - }, - TableName: aws.String(dwr.RepoMetaTablename), - UpdateExpression: aws.String("SET #RM = :RepoMetadata"), - }) - - return err -} - -func (dwr *DynamoDB) createRepoMetaTable() error { - _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ - TableName: aws.String(dwr.RepoMetaTablename), - AttributeDefinitions: []types.AttributeDefinition{ - { - AttributeName: aws.String("RepoName"), - AttributeType: types.ScalarAttributeTypeS, - }, - }, - KeySchema: []types.KeySchemaElement{ - { - AttributeName: aws.String("RepoName"), - KeyType: types.KeyTypeHash, - }, - }, - BillingMode: types.BillingModePayPerRequest, - }) - if err != nil { - if strings.Contains(err.Error(), "Table already exists") { - return nil - } - - return err - } - - return dwr.waitTableToBeCreated(dwr.RepoMetaTablename) -} - -func (dwr *DynamoDB) deleteRepoMetaTable() error { - _, err := dwr.Client.DeleteTable(context.Background(), &dynamodb.DeleteTableInput{ - TableName: aws.String(dwr.RepoMetaTablename), - }) - - if temp := new(types.ResourceNotFoundException); errors.As(err, &temp) { - return nil - } - - return dwr.waitTableToBeDeleted(dwr.RepoMetaTablename) -} - -func (dwr *DynamoDB) ResetRepoMetaTable() error { - err := dwr.deleteRepoMetaTable() - if err != nil { - return err - } - - return dwr.createRepoMetaTable() -} - -func (dwr *DynamoDB) waitTableToBeCreated(tableName string) error { - const maxWaitTime = 20 * time.Second - - waiter := dynamodb.NewTableExistsWaiter(dwr.Client) - - return waiter.Wait(context.Background(), &dynamodb.DescribeTableInput{ - TableName: &tableName, - }, maxWaitTime) -} - -func (dwr *DynamoDB) waitTableToBeDeleted(tableName string) error { - const maxWaitTime = 20 * time.Second - - waiter := dynamodb.NewTableNotExistsWaiter(dwr.Client) - - return waiter.Wait(context.Background(), &dynamodb.DescribeTableInput{ - TableName: &tableName, - }, maxWaitTime) -} - -func (dwr *DynamoDB) createManifestDataTable() error { - _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ - TableName: aws.String(dwr.ManifestDataTablename), - AttributeDefinitions: []types.AttributeDefinition{ - { - AttributeName: aws.String("Digest"), - AttributeType: types.ScalarAttributeTypeS, - }, - }, - KeySchema: []types.KeySchemaElement{ - { - AttributeName: aws.String("Digest"), - KeyType: types.KeyTypeHash, - }, - }, - BillingMode: types.BillingModePayPerRequest, - }) - if err != nil { - if strings.Contains(err.Error(), "Table already exists") { - return nil - } - - return err - } - - return dwr.waitTableToBeCreated(dwr.ManifestDataTablename) -} - -func (dwr *DynamoDB) createIndexDataTable() error { - _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ - TableName: aws.String(dwr.IndexDataTablename), - AttributeDefinitions: []types.AttributeDefinition{ - { - AttributeName: aws.String("IndexDigest"), - AttributeType: types.ScalarAttributeTypeS, - }, - }, - KeySchema: []types.KeySchemaElement{ - { - AttributeName: aws.String("IndexDigest"), - KeyType: types.KeyTypeHash, - }, - }, - BillingMode: types.BillingModePayPerRequest, - }) - if err != nil { - if strings.Contains(err.Error(), "Table already exists") { - return nil - } - - return err - } - - return dwr.waitTableToBeCreated(dwr.IndexDataTablename) -} - -func (dwr *DynamoDB) createVersionTable() error { - _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ - TableName: aws.String(dwr.VersionTablename), - AttributeDefinitions: []types.AttributeDefinition{ - { - AttributeName: aws.String("VersionKey"), - AttributeType: types.ScalarAttributeTypeS, - }, - }, - KeySchema: []types.KeySchemaElement{ - { - AttributeName: aws.String("VersionKey"), - KeyType: types.KeyTypeHash, - }, - }, - BillingMode: types.BillingModePayPerRequest, - }) - if err != nil { - if strings.Contains(err.Error(), "Table already exists") { - return nil - } - - return err - } - - err = dwr.waitTableToBeCreated(dwr.VersionTablename) - if err != nil { - return err - } - - if err == nil { - mdAttributeValue, err := attributevalue.Marshal(version.CurrentVersion) - if err != nil { - return err - } - - _, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ - ExpressionAttributeNames: map[string]string{ - "#V": "Version", - }, - ExpressionAttributeValues: map[string]types.AttributeValue{ - ":Version": mdAttributeValue, - }, - Key: map[string]types.AttributeValue{ - "VersionKey": &types.AttributeValueMemberS{ - Value: version.DBVersionKey, - }, - }, - TableName: aws.String(dwr.VersionTablename), - UpdateExpression: aws.String("SET #V = :Version"), - }) - - if err != nil { - return err - } - } - - return nil -} - -func (dwr *DynamoDB) getDBVersion() (string, error) { - resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ - TableName: aws.String(dwr.VersionTablename), - Key: map[string]types.AttributeValue{ - "VersionKey": &types.AttributeValueMemberS{Value: version.DBVersionKey}, - }, - }) - if err != nil { - return "", err - } - - if resp.Item == nil { - return "", nil - } - - var version string - - err = attributevalue.Unmarshal(resp.Item["Version"], &version) - if err != nil { - return "", err - } - - return version, nil -} - -func (dwr *DynamoDB) deleteManifestDataTable() error { - _, err := dwr.Client.DeleteTable(context.Background(), &dynamodb.DeleteTableInput{ - TableName: aws.String(dwr.ManifestDataTablename), - }) - - if temp := new(types.ResourceNotFoundException); errors.As(err, &temp) { - return nil - } - - return dwr.waitTableToBeDeleted(dwr.ManifestDataTablename) -} - -func (dwr *DynamoDB) ResetManifestDataTable() error { - err := dwr.deleteManifestDataTable() - if err != nil { - return err - } - - return dwr.createManifestDataTable() -} - func (dwr *DynamoDB) ToggleBookmarkRepo(ctx context.Context, repo string) ( mTypes.ToggleState, error, ) { @@ -1680,11 +1295,7 @@ func (dwr *DynamoDB) ToggleBookmarkRepo(ctx context.Context, repo string) ( } userData, err := dwr.GetUserData(ctx) - if err != nil { - if errors.Is(err, zerr.ErrUserDataNotFound) { - return mTypes.NotChanged, nil - } - + if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { return res, err } @@ -1749,7 +1360,7 @@ func (dwr *DynamoDB) ToggleStarRepo(ctx context.Context, repo string) ( } if res != mTypes.NotChanged { - repoMeta, err := dwr.GetRepoMeta(repo) //nolint:contextcheck + repoMeta, err := dwr.getProtoRepoMeta(ctx, repo) //nolint:contextcheck if err != nil { return mTypes.NotChanged, err } @@ -1761,7 +1372,12 @@ func (dwr *DynamoDB) ToggleStarRepo(ctx context.Context, repo string) ( repoMeta.Stars-- } - repoAttributeValue, err := attributevalue.Marshal(repoMeta) + repoMetaBlob, err := proto.Marshal(repoMeta) + if err != nil { + return mTypes.NotChanged, err + } + + repoAttributeValue, err := attributevalue.Marshal(repoMetaBlob) if err != nil { return mTypes.NotChanged, err } @@ -1783,7 +1399,7 @@ func (dwr *DynamoDB) ToggleStarRepo(ctx context.Context, repo string) ( ":UserData": userAttributeValue, }, Key: map[string]types.AttributeValue{ - "Identity": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: userid, }, }, @@ -1801,7 +1417,7 @@ func (dwr *DynamoDB) ToggleStarRepo(ctx context.Context, repo string) ( ":RepoMetadata": repoAttributeValue, }, Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: repo, }, }, @@ -1829,62 +1445,6 @@ func (dwr *DynamoDB) GetStarredRepos(ctx context.Context) ([]string, error) { return userMeta.StarredRepos, err } -func (dwr *DynamoDB) createUserDataTable() error { - _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ - TableName: aws.String(dwr.UserDataTablename), - AttributeDefinitions: []types.AttributeDefinition{ - { - AttributeName: aws.String("Identity"), - AttributeType: types.ScalarAttributeTypeS, - }, - }, - KeySchema: []types.KeySchemaElement{ - { - AttributeName: aws.String("Identity"), - KeyType: types.KeyTypeHash, - }, - }, - BillingMode: types.BillingModePayPerRequest, - }) - if err != nil { - if strings.Contains(err.Error(), "Table already exists") { - return nil - } - - return err - } - - return dwr.waitTableToBeCreated(dwr.UserDataTablename) -} - -func (dwr DynamoDB) createAPIKeyTable() error { - _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ - TableName: aws.String(dwr.APIKeyTablename), - AttributeDefinitions: []types.AttributeDefinition{ - { - AttributeName: aws.String("HashedKey"), - AttributeType: types.ScalarAttributeTypeS, - }, - }, - KeySchema: []types.KeySchemaElement{ - { - AttributeName: aws.String("HashedKey"), - KeyType: types.KeyTypeHash, - }, - }, - BillingMode: types.BillingModePayPerRequest, - }) - if err != nil { - if strings.Contains(err.Error(), "Table already exists") { - return nil - } - - return err - } - - return dwr.waitTableToBeCreated(dwr.APIKeyTablename) -} - func (dwr DynamoDB) SetUserGroups(ctx context.Context, groups []string) error { userData, err := dwr.GetUserData(ctx) if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { @@ -2033,7 +1593,7 @@ func (dwr DynamoDB) AddUserAPIKey(ctx context.Context, hashedKey string, apiKeyD ":UserData": userAttributeValue, }, Key: map[string]types.AttributeValue{ - "Identity": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: userid, }, }, @@ -2051,7 +1611,7 @@ func (dwr DynamoDB) AddUserAPIKey(ctx context.Context, hashedKey string, apiKeyD ":Identity": &types.AttributeValueMemberS{Value: userid}, }, Key: map[string]types.AttributeValue{ - "HashedKey": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: hashedKey, }, }, @@ -2078,7 +1638,7 @@ func (dwr DynamoDB) DeleteUserAPIKey(ctx context.Context, keyID string) error { _, err = dwr.Client.DeleteItem(ctx, &dynamodb.DeleteItemInput{ TableName: aws.String(dwr.APIKeyTablename), Key: map[string]types.AttributeValue{ - "HashedKey": &types.AttributeValueMemberS{Value: hash}, + "Key": &types.AttributeValueMemberS{Value: hash}, }, }) if err != nil { @@ -2100,7 +1660,7 @@ func (dwr DynamoDB) GetUserAPIKeyInfo(hashedKey string) (string, error) { resp, err := dwr.Client.GetItem(context.Background(), &dynamodb.GetItemInput{ TableName: aws.String(dwr.APIKeyTablename), Key: map[string]types.AttributeValue{ - "HashedKey": &types.AttributeValueMemberS{Value: hashedKey}, + "Key": &types.AttributeValueMemberS{Value: hashedKey}, }, }) if err != nil { @@ -2136,7 +1696,7 @@ func (dwr DynamoDB) GetUserData(ctx context.Context) (mTypes.UserData, error) { resp, err := dwr.Client.GetItem(ctx, &dynamodb.GetItemInput{ TableName: aws.String(dwr.UserDataTablename), Key: map[string]types.AttributeValue{ - "Identity": &types.AttributeValueMemberS{Value: userid}, + "Key": &types.AttributeValueMemberS{Value: userid}, }, }) if err != nil { @@ -2180,7 +1740,7 @@ func (dwr DynamoDB) SetUserData(ctx context.Context, userData mTypes.UserData) e ":UserData": userAttributeValue, }, Key: map[string]types.AttributeValue{ - "Identity": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: userid, }, }, @@ -2206,9 +1766,252 @@ func (dwr DynamoDB) DeleteUserData(ctx context.Context) error { _, err = dwr.Client.DeleteItem(ctx, &dynamodb.DeleteItemInput{ TableName: aws.String(dwr.UserDataTablename), Key: map[string]types.AttributeValue{ - "Identity": &types.AttributeValueMemberS{Value: userid}, + "Key": &types.AttributeValueMemberS{Value: userid}, }, }) return err } + +func (dwr *DynamoDB) fetchImageMetaAttributesByDigest(ctx context.Context, digests []string, +) ([]map[string]types.AttributeValue, error) { + resp, err := dwr.Client.BatchGetItem(ctx, &dynamodb.BatchGetItemInput{ + RequestItems: map[string]types.KeysAndAttributes{ + dwr.ImageMetaTablename: { + Keys: getBatchImageKeys(digests), + }, + }, + }) + if err != nil { + return nil, err + } + + if len(resp.Responses[dwr.ImageMetaTablename]) != len(digests) { + return nil, zerr.ErrImageMetaNotFound + } + + return resp.Responses[dwr.ImageMetaTablename], nil +} + +func getBatchImageKeys(digests []string) []map[string]types.AttributeValue { + result := []map[string]types.AttributeValue{} + + for _, digest := range digests { + result = append(result, map[string]types.AttributeValue{ + "Key": &types.AttributeValueMemberS{ + Value: digest, + }, + }) + } + + return result +} + +func (dwr *DynamoDB) PatchDB() error { + DBVersion, err := dwr.getDBVersion() + if err != nil { + return fmt.Errorf("patching dynamo failed, error retrieving database version %w", err) + } + + if version.GetVersionIndex(DBVersion) == -1 { + return fmt.Errorf("DB has broken format, no version found %w", err) + } + + for patchIndex, patch := range dwr.Patches { + if patchIndex < version.GetVersionIndex(DBVersion) { + continue + } + + tableNames := map[string]string{ + "RepoMetaTablename": dwr.RepoMetaTablename, + "VersionTablename": dwr.VersionTablename, + } + + err := patch(dwr.Client, tableNames) + if err != nil { + return err + } + } + + return nil +} + +func (dwr *DynamoDB) ResetDB() error { + err := dwr.ResetTable(dwr.APIKeyTablename) + if err != nil { + return err + } + + err = dwr.ResetTable(dwr.ImageMetaTablename) + if err != nil { + return err + } + + err = dwr.ResetTable(dwr.RepoBlobsTablename) + if err != nil { + return err + } + + err = dwr.ResetTable(dwr.RepoMetaTablename) + if err != nil { + return err + } + + err = dwr.ResetTable(dwr.UserDataTablename) + if err != nil { + return err + } + + return nil +} + +func (dwr *DynamoDB) ResetTable(tableName string) error { + err := dwr.deleteTable(tableName) + if err != nil { + return err + } + + return dwr.createTable(tableName) +} + +func (dwr *DynamoDB) createTable(tableName string) error { + _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ + TableName: aws.String(tableName), + AttributeDefinitions: []types.AttributeDefinition{ + { + AttributeName: aws.String("Key"), + AttributeType: types.ScalarAttributeTypeS, + }, + }, + KeySchema: []types.KeySchemaElement{ + { + AttributeName: aws.String("Key"), + KeyType: types.KeyTypeHash, + }, + }, + BillingMode: types.BillingModePayPerRequest, + }) + + if err != nil && !strings.Contains(err.Error(), "Table already exists") { + return err + } + + return dwr.waitTableToBeCreated(tableName) +} + +func (dwr *DynamoDB) deleteTable(tableName string) error { + _, err := dwr.Client.DeleteTable(context.Background(), &dynamodb.DeleteTableInput{ + TableName: aws.String(tableName), + }) + + if temp := new(types.ResourceNotFoundException); errors.As(err, &temp) { + return nil + } + + return dwr.waitTableToBeDeleted(tableName) +} + +func (dwr *DynamoDB) waitTableToBeCreated(tableName string) error { + const maxWaitTime = 20 * time.Second + + waiter := dynamodb.NewTableExistsWaiter(dwr.Client) + + return waiter.Wait(context.Background(), &dynamodb.DescribeTableInput{ + TableName: &tableName, + }, maxWaitTime) +} + +func (dwr *DynamoDB) waitTableToBeDeleted(tableName string) error { + const maxWaitTime = 20 * time.Second + + waiter := dynamodb.NewTableNotExistsWaiter(dwr.Client) + + return waiter.Wait(context.Background(), &dynamodb.DescribeTableInput{ + TableName: &tableName, + }, maxWaitTime) +} + +func (dwr *DynamoDB) createVersionTable() error { + _, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{ + TableName: aws.String(dwr.VersionTablename), + AttributeDefinitions: []types.AttributeDefinition{ + { + AttributeName: aws.String("Key"), + AttributeType: types.ScalarAttributeTypeS, + }, + }, + KeySchema: []types.KeySchemaElement{ + { + AttributeName: aws.String("Key"), + KeyType: types.KeyTypeHash, + }, + }, + BillingMode: types.BillingModePayPerRequest, + }) + if err != nil { + if strings.Contains(err.Error(), "Table already exists") { + return nil + } + + return err + } + + err = dwr.waitTableToBeCreated(dwr.VersionTablename) + if err != nil { + return err + } + + if err == nil { + mdAttributeValue, err := attributevalue.Marshal(version.CurrentVersion) + if err != nil { + return err + } + + _, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ + ExpressionAttributeNames: map[string]string{ + "#V": "Version", + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":Version": mdAttributeValue, + }, + Key: map[string]types.AttributeValue{ + "Key": &types.AttributeValueMemberS{ + Value: version.DBVersionKey, + }, + }, + TableName: aws.String(dwr.VersionTablename), + UpdateExpression: aws.String("SET #V = :Version"), + }) + + if err != nil { + return err + } + } + + return nil +} + +func (dwr *DynamoDB) getDBVersion() (string, error) { + resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{ + TableName: aws.String(dwr.VersionTablename), + Key: map[string]types.AttributeValue{ + "Key": &types.AttributeValueMemberS{Value: version.DBVersionKey}, + }, + }) + if err != nil { + return "", err + } + + if resp.Item == nil { + return "", nil + } + + var version string + + err = attributevalue.Unmarshal(resp.Item["Version"], &version) + if err != nil { + return "", err + } + + return version, nil +} diff --git a/pkg/meta/dynamodb/dynamodb_internal_test.go b/pkg/meta/dynamodb/dynamodb_internal_test.go index cad85a33..42157aab 100644 --- a/pkg/meta/dynamodb/dynamodb_internal_test.go +++ b/pkg/meta/dynamodb/dynamodb_internal_test.go @@ -29,8 +29,6 @@ func TestWrapperErrors(t *testing.T) { } repoMetaTablename := "RepoMetadataTable" + uuid.String() - manifestDataTablename := "ManifestDataTable" + uuid.String() - indexDataTablename := "IndexDataTable" + uuid.String() userDataTablename := "UserDataTable" + uuid.String() apiKeyTablename := "ApiKeyTable" + uuid.String() @@ -54,31 +52,23 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldBeNil) dynamoWrapper := DynamoDB{ - Client: dynamodb.NewFromConfig(cfg), - RepoMetaTablename: repoMetaTablename, - ManifestDataTablename: manifestDataTablename, - IndexDataTablename: indexDataTablename, - VersionTablename: versionTablename, - UserDataTablename: userDataTablename, - APIKeyTablename: apiKeyTablename, - Patches: version.GetDynamoDBPatches(), - Log: log.Logger{Logger: zerolog.New(os.Stdout)}, + Client: dynamodb.NewFromConfig(cfg), + RepoMetaTablename: repoMetaTablename, + VersionTablename: versionTablename, + UserDataTablename: userDataTablename, + APIKeyTablename: apiKeyTablename, + Patches: version.GetDynamoDBPatches(), + Log: log.Logger{Logger: zerolog.New(os.Stdout)}, } // The table creation should fail as the endpoint is not configured correctly - err = dynamoWrapper.createRepoMetaTable() - So(err, ShouldNotBeNil) - - err = dynamoWrapper.createManifestDataTable() - So(err, ShouldNotBeNil) - - err = dynamoWrapper.createIndexDataTable() + err = dynamoWrapper.createTable(dynamoWrapper.RepoMetaTablename) So(err, ShouldNotBeNil) err = dynamoWrapper.createVersionTable() So(err, ShouldNotBeNil) - err = dynamoWrapper.createAPIKeyTable() + err = dynamoWrapper.createTable(dynamoWrapper.APIKeyTablename) So(err, ShouldNotBeNil) }) @@ -98,21 +88,16 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldBeNil) dynamoWrapper := DynamoDB{ - Client: dynamodb.NewFromConfig(cfg), - RepoMetaTablename: repoMetaTablename, - ManifestDataTablename: manifestDataTablename, - VersionTablename: versionTablename, - IndexDataTablename: indexDataTablename, - UserDataTablename: userDataTablename, - Patches: version.GetDynamoDBPatches(), - Log: log.Logger{Logger: zerolog.New(os.Stdout)}, + Client: dynamodb.NewFromConfig(cfg), + RepoMetaTablename: repoMetaTablename, + VersionTablename: versionTablename, + UserDataTablename: userDataTablename, + Patches: version.GetDynamoDBPatches(), + Log: log.Logger{Logger: zerolog.New(os.Stdout)}, } // The tables were not created so delete calls fail, but dynamoWrapper should not error - err = dynamoWrapper.deleteRepoMetaTable() - So(err, ShouldBeNil) - - err = dynamoWrapper.deleteManifestDataTable() + err = dynamoWrapper.deleteTable(dynamoWrapper.RepoMetaTablename) So(err, ShouldBeNil) }) } diff --git a/pkg/meta/dynamodb/dynamodb_test.go b/pkg/meta/dynamodb/dynamodb_test.go index 4901d8ec..4d3b4c6e 100644 --- a/pkg/meta/dynamodb/dynamodb_test.go +++ b/pkg/meta/dynamodb/dynamodb_test.go @@ -3,7 +3,6 @@ package dynamodb_test import ( "context" "os" - "strings" "testing" "github.com/aws/aws-sdk-go-v2/aws" @@ -12,8 +11,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/dynamodb" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" guuid "github.com/gofrs/uuid" - "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/rs/zerolog" . "github.com/smartystreets/goconvey/convey" @@ -40,9 +37,9 @@ func TestIterator(t *testing.T) { } repoMetaTablename := "RepoMetadataTable" + uuid.String() - manifestDataTablename := "ManifestDataTable" + uuid.String() versionTablename := "Version" + uuid.String() - indexDataTablename := "IndexDataTable" + uuid.String() + imageMetaTablename := "ImageMeta" + uuid.String() + repoBlobsTablename := "RepoBlobs" + uuid.String() userDataTablename := "UserDataTable" + uuid.String() apiKeyTablename := "ApiKeyTable" + uuid.String() @@ -50,14 +47,14 @@ func TestIterator(t *testing.T) { Convey("TestIterator", t, func() { params := mdynamodb.DBDriverParameters{ - Endpoint: endpoint, - Region: region, - RepoMetaTablename: repoMetaTablename, - ManifestDataTablename: manifestDataTablename, - IndexDataTablename: indexDataTablename, - VersionTablename: versionTablename, - APIKeyTablename: apiKeyTablename, - UserDataTablename: userDataTablename, + Endpoint: endpoint, + Region: region, + RepoMetaTablename: repoMetaTablename, + ImageMetaTablename: imageMetaTablename, + RepoBlobsInfoTablename: repoBlobsTablename, + VersionTablename: versionTablename, + APIKeyTablename: apiKeyTablename, + UserDataTablename: userDataTablename, } client, err := mdynamodb.GetDynamoClient(params) So(err, ShouldBeNil) @@ -65,21 +62,21 @@ func TestIterator(t *testing.T) { dynamoWrapper, err := mdynamodb.New(client, params, log) So(err, ShouldBeNil) - So(dynamoWrapper.ResetManifestDataTable(), ShouldBeNil) - So(dynamoWrapper.ResetRepoMetaTable(), ShouldBeNil) + So(dynamoWrapper.ResetTable(dynamoWrapper.ImageMetaTablename), ShouldBeNil) + So(dynamoWrapper.ResetTable(dynamoWrapper.RepoMetaTablename), ShouldBeNil) - err = dynamoWrapper.SetRepoReference("repo1", "tag1", "manifestType", "manifestDigest1") + err = dynamoWrapper.SetRepoReference("repo1", "tag1", CreateRandomImage().AsImageMeta()) So(err, ShouldBeNil) - err = dynamoWrapper.SetRepoReference("repo2", "tag2", "manifestType", "manifestDigest2") + err = dynamoWrapper.SetRepoReference("repo2", "tag2", CreateRandomImage().AsImageMeta()) So(err, ShouldBeNil) - err = dynamoWrapper.SetRepoReference("repo3", "tag3", "manifestType", "manifestDigest3") + err = dynamoWrapper.SetRepoReference("repo3", "tag3", CreateRandomImage().AsImageMeta()) So(err, ShouldBeNil) repoMetaAttributeIterator := mdynamodb.NewBaseDynamoAttributesIterator( dynamoWrapper.Client, - repoMetaTablename, + dynamoWrapper.RepoMetaTablename, "RepoMetadata", 1, log, @@ -143,25 +140,25 @@ func TestWrapperErrors(t *testing.T) { } repoMetaTablename := "RepoMetadataTable" + uuid.String() - manifestDataTablename := "ManifestDataTable" + uuid.String() versionTablename := "Version" + uuid.String() - indexDataTablename := "IndexDataTable" + uuid.String() userDataTablename := "UserDataTable" + uuid.String() apiKeyTablename := "ApiKeyTable" + uuid.String() wrongTableName := "WRONG Tables" + imageMetaTablename := "ImageMeta" + uuid.String() + repoBlobsTablename := "RepoBlobs" + uuid.String() log := log.NewLogger("debug", "") Convey("Errors", t, func() { params := mdynamodb.DBDriverParameters{ //nolint:contextcheck - Endpoint: endpoint, - Region: region, - RepoMetaTablename: repoMetaTablename, - ManifestDataTablename: manifestDataTablename, - IndexDataTablename: indexDataTablename, - UserDataTablename: userDataTablename, - APIKeyTablename: apiKeyTablename, - VersionTablename: versionTablename, + Endpoint: endpoint, + Region: region, + RepoMetaTablename: repoMetaTablename, + ImageMetaTablename: imageMetaTablename, + RepoBlobsInfoTablename: repoBlobsTablename, + UserDataTablename: userDataTablename, + APIKeyTablename: apiKeyTablename, + VersionTablename: versionTablename, } client, err := mdynamodb.GetDynamoClient(params) //nolint:contextcheck So(err, ShouldBeNil) @@ -174,8 +171,9 @@ func TestWrapperErrors(t *testing.T) { dynamoWrapper.SetImageTrustStore(imgTrustStore) - So(dynamoWrapper.ResetManifestDataTable(), ShouldBeNil) //nolint:contextcheck - So(dynamoWrapper.ResetRepoMetaTable(), ShouldBeNil) //nolint:contextcheck + So(dynamoWrapper.ResetTable(dynamoWrapper.RepoMetaTablename), ShouldBeNil) //nolint:contextcheck + So(dynamoWrapper.ResetTable(dynamoWrapper.ImageMetaTablename), ShouldBeNil) //nolint:contextcheck + So(dynamoWrapper.ResetTable(dynamoWrapper.UserDataTablename), ShouldBeNil) //nolint:contextcheck userAc := reqCtx.NewUserAccessControl() userAc.SetUsername("test") @@ -240,7 +238,7 @@ func TestWrapperErrors(t *testing.T) { status, err := dynamoWrapper.ToggleBookmarkRepo(ctx, "repo") So(err, ShouldBeNil) - So(status, ShouldEqual, mTypes.NotChanged) + So(status, ShouldEqual, mTypes.Added) }) Convey("ToggleBookmarkRepo GetUserMeta client error", func() { @@ -476,608 +474,6 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("SetManifestData", func() { - dynamoWrapper.ManifestDataTablename = wrongTableName - - err := dynamoWrapper.SetManifestData("dig", mTypes.ManifestData{}) - So(err, ShouldNotBeNil) - }) - - Convey("GetManifestData", func() { - dynamoWrapper.ManifestDataTablename = wrongTableName - - _, err := dynamoWrapper.GetManifestData("dig") - So(err, ShouldNotBeNil) - }) - - Convey("GetManifestData unmarshal error", func() { - err := setBadManifestData(dynamoWrapper.Client, manifestDataTablename, "dig") - So(err, ShouldBeNil) - - _, err = dynamoWrapper.GetManifestData("dig") - So(err, ShouldNotBeNil) - }) - - Convey("GetIndexData", func() { - dynamoWrapper.IndexDataTablename = wrongTableName - - _, err := dynamoWrapper.GetIndexData("dig") - So(err, ShouldNotBeNil) - }) - - Convey("GetIndexData unmarshal error", func() { - err := setBadIndexData(dynamoWrapper.Client, indexDataTablename, "dig") - So(err, ShouldBeNil) - - _, err = dynamoWrapper.GetManifestData("dig") - So(err, ShouldNotBeNil) - }) - - Convey("SetManifestMeta GetRepoMeta error", func() { - err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo1") - So(err, ShouldBeNil) - - err = dynamoWrapper.SetManifestMeta("repo1", "dig", mTypes.ManifestMetadata{}) - So(err, ShouldNotBeNil) - }) - - Convey("GetManifestMeta GetManifestData not found error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") - So(err, ShouldBeNil) - - _, err = dynamoWrapper.GetManifestMeta("repo", "dig") - So(err, ShouldNotBeNil) - }) - - Convey("GetManifestMeta GetRepoMeta Not Found error", func() { - err := dynamoWrapper.SetManifestData("dig", mTypes.ManifestData{}) - So(err, ShouldBeNil) - - _, err = dynamoWrapper.GetManifestMeta("repoNotFound", "dig") - So(err, ShouldNotBeNil) - }) - - Convey("GetManifestMeta GetRepoMeta error", func() { - err := dynamoWrapper.SetManifestData("dig", mTypes.ManifestData{}) - So(err, ShouldBeNil) - - err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") - So(err, ShouldBeNil) - - _, err = dynamoWrapper.GetManifestMeta("repo", "dig") - So(err, ShouldNotBeNil) - }) - - Convey("SetRepoReference client error", func() { - dynamoWrapper.RepoMetaTablename = badTablename - digest := digest.FromString("str") - err := dynamoWrapper.SetRepoReference("repo", digest.String(), digest, ispec.MediaTypeImageManifest) - So(err, ShouldNotBeNil) - }) - - Convey("SetReferrer client error", func() { - dynamoWrapper.RepoMetaTablename = badTablename - err := dynamoWrapper.SetReferrer("repo", "", mTypes.ReferrerInfo{}) - So(err, ShouldNotBeNil) - }) - - Convey("SetReferrer bad repoMeta", func() { - err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") - So(err, ShouldBeNil) - - err = dynamoWrapper.SetReferrer("repo", "", mTypes.ReferrerInfo{}) - So(err, ShouldNotBeNil) - }) - - Convey("GetReferrers client error", func() { - dynamoWrapper.RepoMetaTablename = badTablename - _, err := dynamoWrapper.GetReferrers("repo", "") - So(err, ShouldNotBeNil) - }) - - Convey("GetReferrers bad repoMeta", func() { - err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") - So(err, ShouldBeNil) - - _, err = dynamoWrapper.GetReferrers("repo", "") - So(err, ShouldNotBeNil) - }) - - Convey("DeleteReferrer client error", func() { - dynamoWrapper.RepoMetaTablename = badTablename - err := dynamoWrapper.DeleteReferrer("repo", "", "") - So(err, ShouldNotBeNil) - }) - - Convey("DeleteReferrer bad repoMeta", func() { - err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") - So(err, ShouldBeNil) - - err = dynamoWrapper.DeleteReferrer("repo", "", "") - So(err, ShouldNotBeNil) - }) - - Convey("GetReferrersInfo GetReferrers errors", func() { - dynamoWrapper.RepoMetaTablename = badTablename - _, err := dynamoWrapper.GetReferrersInfo("repo", "", nil) - So(err, ShouldNotBeNil) - }) - - Convey("GetReferrersInfo getData fails", func() { - dynamoWrapper.ManifestDataTablename = badTablename - err = dynamoWrapper.SetReferrer("repo", "rf", mTypes.ReferrerInfo{ - Digest: "dig1", - MediaType: ispec.MediaTypeImageManifest, - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetReferrer("repo", "rf", mTypes.ReferrerInfo{ - Digest: "dig2", - MediaType: ispec.MediaTypeImageManifest, - }) - So(err, ShouldBeNil) - - _, err := dynamoWrapper.GetReferrersInfo("repo", "rf", nil) - So(err, ShouldBeNil) - }) - - Convey("GetReferrersInfo bad descriptor blob", func() { - err = dynamoWrapper.SetManifestData("dig3", mTypes.ManifestData{ - ManifestBlob: []byte("bad json"), - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetReferrer("repo", "rf", mTypes.ReferrerInfo{ - Digest: "dig2", - MediaType: ispec.MediaTypeImageManifest, - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetReferrer("repo", "rf", mTypes.ReferrerInfo{ - Digest: "dig3", - MediaType: ispec.MediaTypeImageManifest, - }) - So(err, ShouldBeNil) - - _, err := dynamoWrapper.GetReferrersInfo("repo", "rf", nil) - So(err, ShouldBeNil) - }) - - Convey("IncrementRepoStars GetRepoMeta error", func() { - err = dynamoWrapper.IncrementRepoStars("repo") - So(err, ShouldNotBeNil) - }) - - Convey("DecrementRepoStars GetRepoMeta error", func() { - err = dynamoWrapper.DecrementRepoStars("repo") - So(err, ShouldNotBeNil) - }) - - Convey("DeleteRepoTag Client.GetItem error", func() { - strSlice := make([]string, 10000) - repoName := strings.Join(strSlice, ".") - - err = dynamoWrapper.DeleteRepoTag(repoName, "tag") - So(err, ShouldNotBeNil) - }) - - Convey("DeleteRepoTag unmarshal error", func() { - err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") - So(err, ShouldBeNil) - - err = dynamoWrapper.DeleteRepoTag("repo", "tag") - So(err, ShouldNotBeNil) - }) - - Convey("GetRepoMeta Client.GetItem error", func() { - strSlice := make([]string, 10000) - repoName := strings.Join(strSlice, ".") - - _, err = dynamoWrapper.GetRepoMeta(repoName) - So(err, ShouldNotBeNil) - }) - - Convey("GetRepoMeta unmarshal error", func() { - err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") - So(err, ShouldBeNil) - - _, err = dynamoWrapper.GetRepoMeta("repo") - So(err, ShouldNotBeNil) - }) - - Convey("IncrementImageDownloads GetRepoMeta error", func() { - err = dynamoWrapper.IncrementImageDownloads("repoNotFound", "") - So(err, ShouldNotBeNil) - }) - - Convey("IncrementImageDownloads tag not found error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") - So(err, ShouldBeNil) - - err = dynamoWrapper.IncrementImageDownloads("repo", "notFoundTag") - So(err, ShouldNotBeNil) - }) - - Convey("UpdateSignaturesValidity GetManifestData error", func() { - err := setBadManifestData(dynamoWrapper.Client, manifestDataTablename, "dig") - So(err, ShouldBeNil) - - err = dynamoWrapper.UpdateSignaturesValidity("repo", "dig") - So(err, ShouldNotBeNil) - - err = dynamoWrapper.UpdateSignaturesValidity("repo", digest.FromString("dig")) - So(err, ShouldBeNil) - }) - - Convey("UpdateSignaturesValidity GetRepoMeta error", func() { - err := dynamoWrapper.SetManifestData("dig", mTypes.ManifestData{}) - So(err, ShouldBeNil) - - err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") - So(err, ShouldBeNil) - - err = dynamoWrapper.UpdateSignaturesValidity("repo", "dig") - So(err, ShouldNotBeNil) - }) - - Convey("AddManifestSignature GetRepoMeta error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") - So(err, ShouldBeNil) - - err = dynamoWrapper.AddManifestSignature("repoNotFound", "tag", mTypes.SignatureMetadata{}) - So(err, ShouldNotBeNil) - }) - - Convey("AddManifestSignature ManifestSignatures signedManifestDigest not found error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") - So(err, ShouldBeNil) - - err = dynamoWrapper.AddManifestSignature("repo", "tagNotFound", mTypes.SignatureMetadata{}) - So(err, ShouldNotBeNil) - }) - - Convey("AddManifestSignature SignatureType metadb.NotationType", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", "dig", "") - So(err, ShouldBeNil) - - err = dynamoWrapper.AddManifestSignature("repo", "tagNotFound", mTypes.SignatureMetadata{ - SignatureType: "notation", - }) - So(err, ShouldBeNil) - }) - - Convey("DeleteSignature GetRepoMeta error", func() { - err = dynamoWrapper.DeleteSignature("repoNotFound", "tagNotFound", mTypes.SignatureMetadata{}) - So(err, ShouldNotBeNil) - }) - - Convey("DeleteSignature sigDigest.SignatureManifestDigest != sigMeta.SignatureDigest true", func() { - err := setRepoMeta(dynamoWrapper.Client, repoMetaTablename, mTypes.RepoMetadata{ - Name: "repo", - Signatures: map[string]mTypes.ManifestSignatures{ - "tag1": { - "cosign": []mTypes.SignatureInfo{ - {SignatureManifestDigest: "dig1"}, - {SignatureManifestDigest: "dig2"}, - }, - }, - }, - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.DeleteSignature("repo", "tag1", mTypes.SignatureMetadata{ - SignatureDigest: "dig2", - SignatureType: "cosign", - }) - So(err, ShouldBeNil) - }) - - Convey("GetMultipleRepoMeta unmarshal error", func() { - err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck - So(err, ShouldBeNil) - - _, err = dynamoWrapper.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true }) - - So(err, ShouldNotBeNil) - }) - - Convey("SearchRepos repoMeta unmarshal error", func() { - err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.SearchRepos(ctx, "") - - So(err, ShouldNotBeNil) - }) - - Convey("SearchRepos bad tablename", func() { - dynamoWrapper.RepoMetaTablename = badTablename - - _, _, _, err = dynamoWrapper.SearchRepos(ctx, "") - - So(err, ShouldNotBeNil) - }) - - Convey("GetMultipleRepoMeta bad tablename", func() { - dynamoWrapper.RepoMetaTablename = badTablename - - _, err = dynamoWrapper.GetMultipleRepoMeta(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true }) - - So(err, ShouldNotBeNil) - }) - - Convey("FilterTags bad tablename", func() { - dynamoWrapper.RepoMetaTablename = badTablename - - _, _, _, err = dynamoWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return true - }) - - So(err, ShouldNotBeNil) - }) - - Convey("FilterRepos bad tablename", func() { - dynamoWrapper.RepoMetaTablename = badTablename - - _, _, _, err = dynamoWrapper.FilterRepos(ctx, func(repoMeta mTypes.RepoMetadata) bool { return true }) - - So(err, ShouldNotBeNil) - }) - - Convey("SearchTags bad tablename", func() { - dynamoWrapper.RepoMetaTablename = badTablename - - _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:tag") - - So(err, ShouldNotBeNil) - }) - - Convey("SearchRepos GetManifestMeta error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag1", "notFoundDigest", //nolint:contextcheck - ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.SearchRepos(ctx, "") - - So(err, ShouldNotBeNil) - }) - - Convey("Unsuported type", func() { - digest := digest.FromString("digest") - - err := dynamoWrapper.SetRepoReference("repo", "tag1", digest, "invalid type") //nolint:contextcheck - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.SearchRepos(ctx, "") - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:") - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) - So(err, ShouldBeNil) - }) - - Convey("SearchRepos bad index data", func() { - indexDigest := digest.FromString("indexDigest") - - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck - So(err, ShouldBeNil) - - err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.SearchRepos(ctx, "") - So(err, ShouldNotBeNil) - }) - - Convey("SearchRepos bad indexBlob in IndexData", func() { - indexDigest := digest.FromString("indexDigest") - - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck - So(err, ShouldBeNil) - - err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck - IndexBlob: []byte("bad json"), - }) - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.SearchRepos(ctx, "") - So(err, ShouldNotBeNil) - }) - - Convey("SearchTags repoMeta unmarshal error", func() { - err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:") - - So(err, ShouldNotBeNil) - }) - - Convey("SearchTags GetManifestMeta error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag1", "manifestNotFound", //nolint:contextcheck - ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:") - - So(err, ShouldNotBeNil) - }) - - Convey("SearchTags bad index data", func() { - indexDigest := digest.FromString("indexDigest") - - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck - So(err, ShouldBeNil) - - err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:") - So(err, ShouldNotBeNil) - }) - - Convey("SearchTags bad indexBlob in IndexData", func() { - indexDigest := digest.FromString("indexDigest") - - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck - So(err, ShouldBeNil) - - err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck - IndexBlob: []byte("bad json"), - }) - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.SearchTags(ctx, "repo:") - So(err, ShouldNotBeNil) - }) - - Convey("SearchRepos attr", func() { - err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck - So(err, ShouldBeNil) - - _, _, _, err := dynamoWrapper.SearchRepos(ctx, "repo") - So(err, ShouldNotBeNil) - }) - - Convey("FilterRepos attributevalue.Unmarshal(repoMetaAttribute) errors", func() { - dynamoWrapper.RepoMetaTablename = "bad-table-FilterRepos" - - _, _, _, err := dynamoWrapper.FilterRepos(ctx, func(repoMeta mTypes.RepoMetadata) bool { - return true - }) - So(err, ShouldNotBeNil) - }) - - Convey("SearchRepos bad RepoMeta table name", func() { - dynamoWrapper.RepoMetaTablename = "SearchRepos-bad-table" - - _, _, _, err := dynamoWrapper.SearchRepos(ctx, "repo") - So(err, ShouldNotBeNil) - }) - - Convey("FilterTags repoMeta unmarshal error", func() { - err = setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") //nolint:contextcheck - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return true - }) - - So(err, ShouldNotBeNil) - }) - - Convey("FilterTags bad RepoMeta table name", func() { - dynamoWrapper.RepoMetaTablename = "bad-table" - - _, _, _, err := dynamoWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return true - }) - - So(err, ShouldNotBeNil) - }) - - Convey("FilterTags manifestMeta not found", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag1", "manifestNotFound", //nolint:contextcheck - ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return true - }) - - So(err, ShouldNotBeNil) - }) - - Convey("FilterTags manifestMeta unmarshal error", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag1", "dig", ispec.MediaTypeImageManifest) //nolint:contextcheck - So(err, ShouldBeNil) - - err = setBadManifestData(dynamoWrapper.Client, manifestDataTablename, "dig") //nolint:contextcheck - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.FilterTags( - ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return true - }) - - So(err, ShouldNotBeNil) - }) - - Convey("FilterTags bad IndexData", func() { - indexDigest := digest.FromString("indexDigest") - - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck - So(err, ShouldBeNil) - - err = setBadIndexData(dynamoWrapper.Client, indexDataTablename, indexDigest.String()) //nolint:contextcheck - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) - So(err, ShouldNotBeNil) - }) - - Convey("FilterTags bad indexBlob in IndexData", func() { - indexDigest := digest.FromString("indexDigest") - - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck - So(err, ShouldBeNil) - - err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck - IndexBlob: []byte("bad json"), - }) - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) - So(err, ShouldNotBeNil) - }) - - Convey("FilterTags didn't match any index manifest", func() { - var ( - indexDigest = digest.FromString("indexDigest") - manifestDigestFromIndex1 = digest.FromString("manifestDigestFromIndex1") - manifestDigestFromIndex2 = digest.FromString("manifestDigestFromIndex2") - ) - - err := dynamoWrapper.SetRepoReference("repo", "tag1", indexDigest, ispec.MediaTypeImageIndex) //nolint:contextcheck - So(err, ShouldBeNil) - - indexBlob, err := GetIndexBlobWithManifests([]digest.Digest{ - manifestDigestFromIndex1, manifestDigestFromIndex2, - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetIndexData(indexDigest, mTypes.IndexData{ //nolint:contextcheck - IndexBlob: indexBlob, - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetManifestData(manifestDigestFromIndex1, mTypes.ManifestData{ //nolint:contextcheck - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("{}"), - }) - So(err, ShouldBeNil) - - err = dynamoWrapper.SetManifestData(manifestDigestFromIndex2, mTypes.ManifestData{ //nolint:contextcheck - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("{}"), - }) - So(err, ShouldBeNil) - - _, _, _, err = dynamoWrapper.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return false }) - So(err, ShouldBeNil) - }) - Convey("PatchDB dwr.getDBVersion errors", func() { dynamoWrapper.VersionTablename = badTablename @@ -1102,7 +498,7 @@ func TestWrapperErrors(t *testing.T) { Convey("ResetRepoMetaTable client errors", func() { dynamoWrapper.RepoMetaTablename = badTablename - err := dynamoWrapper.ResetRepoMetaTable() + err := dynamoWrapper.ResetTable(dynamoWrapper.RepoMetaTablename) So(err, ShouldNotBeNil) }) @@ -1112,61 +508,18 @@ func TestWrapperErrors(t *testing.T) { err := dynamoWrapper.PatchDB() So(err, ShouldNotBeNil) }) - - Convey("GetUserRepoMeta client.GetItem error", func() { - dynamoWrapper.RepoMetaTablename = badTablename - - _, err = dynamoWrapper.GetUserRepoMeta(ctx, "repo") - So(err, ShouldNotBeNil) - }) - - Convey("GetUserRepoMeta repoMeta not found", func() { - _, err = dynamoWrapper.GetUserRepoMeta(ctx, "unknown-repo-meta") - So(err, ShouldNotBeNil) - }) - - Convey("GetUserRepoMeta userMeta not found", func() { - err := dynamoWrapper.SetRepoReference("repo", "tag", digest.FromString("1"), ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - dynamoWrapper.UserDataTablename = badTablename - - userAc := reqCtx.NewUserAccessControl() - userAc.SetUsername("username") - userAc.SetGlobPatterns("read", map[string]bool{ - "repo": true, - }) - ctx := userAc.DeriveContext(context.Background()) - - _, err = dynamoWrapper.GetUserRepoMeta(ctx, "repo") - So(err, ShouldNotBeNil) - }) - - Convey("GetUserRepoMeta unmarshal error", func() { - err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo") - So(err, ShouldBeNil) - - userAc := reqCtx.NewUserAccessControl() - userAc.SetUsername("username") - userAc.SetGlobPatterns("read", map[string]bool{ - "repo": true, - }) - ctx := userAc.DeriveContext(context.Background()) - - _, err = dynamoWrapper.GetUserRepoMeta(ctx, "repo") - So(err, ShouldNotBeNil) - }) }) Convey("NewDynamoDBWrapper errors", t, func() { params := mdynamodb.DBDriverParameters{ //nolint:contextcheck - Endpoint: endpoint, - Region: region, - RepoMetaTablename: "", - ManifestDataTablename: manifestDataTablename, - IndexDataTablename: indexDataTablename, - UserDataTablename: userDataTablename, - APIKeyTablename: apiKeyTablename, - VersionTablename: versionTablename, + Endpoint: endpoint, + Region: region, + RepoMetaTablename: "", + ImageMetaTablename: imageMetaTablename, + RepoBlobsInfoTablename: repoBlobsTablename, + UserDataTablename: userDataTablename, + APIKeyTablename: apiKeyTablename, + VersionTablename: versionTablename, } client, err := mdynamodb.GetDynamoClient(params) So(err, ShouldBeNil) @@ -1175,14 +528,14 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) params = mdynamodb.DBDriverParameters{ //nolint:contextcheck - Endpoint: endpoint, - Region: region, - RepoMetaTablename: repoMetaTablename, - ManifestDataTablename: "", - IndexDataTablename: indexDataTablename, - UserDataTablename: userDataTablename, - APIKeyTablename: apiKeyTablename, - VersionTablename: versionTablename, + Endpoint: endpoint, + Region: region, + RepoMetaTablename: repoMetaTablename, + ImageMetaTablename: imageMetaTablename, + RepoBlobsInfoTablename: repoBlobsTablename, + UserDataTablename: userDataTablename, + APIKeyTablename: apiKeyTablename, + VersionTablename: "", } client, err = mdynamodb.GetDynamoClient(params) So(err, ShouldBeNil) @@ -1191,14 +544,14 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) params = mdynamodb.DBDriverParameters{ //nolint:contextcheck - Endpoint: endpoint, - Region: region, - RepoMetaTablename: repoMetaTablename, - ManifestDataTablename: manifestDataTablename, - IndexDataTablename: "", - UserDataTablename: userDataTablename, - APIKeyTablename: apiKeyTablename, - VersionTablename: versionTablename, + Endpoint: endpoint, + Region: region, + RepoMetaTablename: repoMetaTablename, + ImageMetaTablename: imageMetaTablename, + RepoBlobsInfoTablename: repoBlobsTablename, + VersionTablename: versionTablename, + UserDataTablename: "", + APIKeyTablename: apiKeyTablename, } client, err = mdynamodb.GetDynamoClient(params) So(err, ShouldBeNil) @@ -1207,62 +560,14 @@ func TestWrapperErrors(t *testing.T) { So(err, ShouldNotBeNil) params = mdynamodb.DBDriverParameters{ //nolint:contextcheck - Endpoint: endpoint, - Region: region, - RepoMetaTablename: repoMetaTablename, - ManifestDataTablename: manifestDataTablename, - IndexDataTablename: indexDataTablename, - UserDataTablename: userDataTablename, - APIKeyTablename: apiKeyTablename, - VersionTablename: "", - } - client, err = mdynamodb.GetDynamoClient(params) - So(err, ShouldBeNil) - - _, err = mdynamodb.New(client, params, log) - So(err, ShouldNotBeNil) - - params = mdynamodb.DBDriverParameters{ //nolint:contextcheck - Endpoint: endpoint, - Region: region, - RepoMetaTablename: repoMetaTablename, - ManifestDataTablename: manifestDataTablename, - IndexDataTablename: indexDataTablename, - VersionTablename: versionTablename, - UserDataTablename: userDataTablename, - APIKeyTablename: apiKeyTablename, - } - client, err = mdynamodb.GetDynamoClient(params) - So(err, ShouldBeNil) - - _, err = mdynamodb.New(client, params, log) - So(err, ShouldBeNil) - - params = mdynamodb.DBDriverParameters{ //nolint:contextcheck - Endpoint: endpoint, - Region: region, - RepoMetaTablename: repoMetaTablename, - ManifestDataTablename: manifestDataTablename, - IndexDataTablename: indexDataTablename, - VersionTablename: versionTablename, - UserDataTablename: "", - APIKeyTablename: apiKeyTablename, - } - client, err = mdynamodb.GetDynamoClient(params) - So(err, ShouldBeNil) - - _, err = mdynamodb.New(client, params, log) - So(err, ShouldNotBeNil) - - params = mdynamodb.DBDriverParameters{ //nolint:contextcheck - Endpoint: endpoint, - Region: region, - RepoMetaTablename: repoMetaTablename, - ManifestDataTablename: manifestDataTablename, - IndexDataTablename: indexDataTablename, - VersionTablename: versionTablename, - UserDataTablename: userDataTablename, - APIKeyTablename: "", + Endpoint: endpoint, + Region: region, + RepoMetaTablename: repoMetaTablename, + ImageMetaTablename: imageMetaTablename, + RepoBlobsInfoTablename: repoBlobsTablename, + VersionTablename: versionTablename, + UserDataTablename: userDataTablename, + APIKeyTablename: "", } client, err = mdynamodb.GetDynamoClient(params) So(err, ShouldBeNil) @@ -1272,81 +577,6 @@ func TestWrapperErrors(t *testing.T) { }) } -func setBadManifestData(client *dynamodb.Client, manifestDataTableName, digest string) error { - mdAttributeValue, err := attributevalue.Marshal("string") - if err != nil { - return err - } - - _, err = client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ - ExpressionAttributeNames: map[string]string{ - "#MD": "ManifestData", - }, - ExpressionAttributeValues: map[string]types.AttributeValue{ - ":ManifestData": mdAttributeValue, - }, - Key: map[string]types.AttributeValue{ - "Digest": &types.AttributeValueMemberS{ - Value: digest, - }, - }, - TableName: aws.String(manifestDataTableName), - UpdateExpression: aws.String("SET #MD = :ManifestData"), - }) - - return err -} - -func setBadRepoMeta(client *dynamodb.Client, repoMetadataTableName, repoName string) error { - repoAttributeValue, err := attributevalue.Marshal("string") - if err != nil { - return err - } - - _, err = client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ - ExpressionAttributeNames: map[string]string{ - "#RM": "RepoMetadata", - }, - ExpressionAttributeValues: map[string]types.AttributeValue{ - ":RepoMetadata": repoAttributeValue, - }, - Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{ - Value: repoName, - }, - }, - TableName: aws.String(repoMetadataTableName), - UpdateExpression: aws.String("SET #RM = :RepoMetadata"), - }) - - return err -} - -func setBadIndexData(client *dynamodb.Client, indexDataTableName, digest string) error { - mdAttributeValue, err := attributevalue.Marshal("string") - if err != nil { - return err - } - - _, err = client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ - ExpressionAttributeNames: map[string]string{ - "#ID": "IndexData", - }, - ExpressionAttributeValues: map[string]types.AttributeValue{ - ":IndexData": mdAttributeValue, - }, - Key: map[string]types.AttributeValue{ - "IndexDigest": &types.AttributeValueMemberS{ - Value: digest, - }, - }, - TableName: aws.String(indexDataTableName), - UpdateExpression: aws.String("SET #ID = :IndexData"), - }) - - return err -} - func setBadUserData(client *dynamodb.Client, userDataTablename, userID string) error { userAttributeValue, err := attributevalue.Marshal("string") if err != nil { @@ -1361,7 +591,7 @@ func setBadUserData(client *dynamodb.Client, userDataTablename, userID string) e ":UserData": userAttributeValue, }, Key: map[string]types.AttributeValue{ - "Identity": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: userID, }, }, @@ -1386,7 +616,7 @@ func setVersion(client *dynamodb.Client, versionTablename string, version string ":Version": mdAttributeValue, }, Key: map[string]types.AttributeValue{ - "VersionKey": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: "DBVersion", }, }, @@ -1396,28 +626,3 @@ func setVersion(client *dynamodb.Client, versionTablename string, version string return err } - -func setRepoMeta(client *dynamodb.Client, repoMetadataTableName string, repoMeta mTypes.RepoMetadata) error { - repoAttributeValue, err := attributevalue.Marshal(repoMeta) - if err != nil { - return err - } - - _, err = client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ - ExpressionAttributeNames: map[string]string{ - "#RM": "RepoMetadata", - }, - ExpressionAttributeValues: map[string]types.AttributeValue{ - ":RepoMetadata": repoAttributeValue, - }, - Key: map[string]types.AttributeValue{ - "RepoName": &types.AttributeValueMemberS{ - Value: repoMeta.Name, - }, - }, - TableName: aws.String(repoMetadataTableName), - UpdateExpression: aws.String("SET #RM = :RepoMetadata"), - }) - - return err -} diff --git a/pkg/meta/dynamodb/parameters.go b/pkg/meta/dynamodb/parameters.go index cd73f2be..48ecacd1 100644 --- a/pkg/meta/dynamodb/parameters.go +++ b/pkg/meta/dynamodb/parameters.go @@ -9,7 +9,7 @@ import ( ) type DBDriverParameters struct { - Endpoint, Region, RepoMetaTablename, ManifestDataTablename, IndexDataTablename, + Endpoint, Region, RepoMetaTablename, RepoBlobsInfoTablename, ImageMetaTablename, UserDataTablename, APIKeyTablename, VersionTablename string } diff --git a/pkg/meta/hooks.go b/pkg/meta/hooks.go index 3817f2c9..74d6d938 100644 --- a/pkg/meta/hooks.go +++ b/pkg/meta/hooks.go @@ -2,10 +2,10 @@ package meta import ( godigest "github.com/opencontainers/go-digest" + v1 "github.com/opencontainers/image-spec/specs-go/v1" zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/log" - "zotregistry.io/zot/pkg/meta/common" mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/storage" ) @@ -22,55 +22,9 @@ func OnUpdateManifest(repo, reference, mediaType string, digest godigest.Digest, imgStore := storeController.GetImageStore(repo) - // check if image is a signature - isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(repo, body, reference) + err := SetImageMetaFromInput(repo, reference, mediaType, digest, body, + imgStore, metaDB, log) if err != nil { - log.Error().Err(err).Msg("can't check if image is a signature or not") - - if err := imgStore.DeleteImageManifest(repo, reference, false); err != nil { - log.Error().Err(err).Str("manifest", reference).Str("repository", repo).Msg("couldn't remove image manifest in repo") - - return err - } - - return err - } - - metadataSuccessfullySet := true - - if isSignature { - layersInfo, errGetLayers := GetSignatureLayersInfo(repo, reference, digest.String(), signatureType, body, - imgStore, log) - if errGetLayers != nil { - metadataSuccessfullySet = false - err = errGetLayers - } else { - err = metaDB.AddManifestSignature(repo, signedManifestDigest, mTypes.SignatureMetadata{ - SignatureType: signatureType, - SignatureDigest: digest.String(), - LayersInfo: layersInfo, - }) - if err != nil { - log.Error().Err(err).Msg("metadb: error while putting repo meta") - metadataSuccessfullySet = false - } else { - err = metaDB.UpdateSignaturesValidity(repo, signedManifestDigest) - if err != nil { - log.Error().Err(err).Str("repository", repo).Str("reference", reference).Str("digest", - signedManifestDigest.String()).Msg("metadb: failed verify signatures validity for signed image") - metadataSuccessfullySet = false - } - } - } - } else { - err = SetImageMetaFromInput(repo, reference, mediaType, digest, body, - imgStore, metaDB, log) - if err != nil { - metadataSuccessfullySet = false - } - } - - if !metadataSuccessfullySet { log.Info().Str("tag", reference).Str("repository", repo).Msg("uploading image meta was unsuccessful for tag in repo") if err := imgStore.DeleteImageManifest(repo, reference, false); err != nil { @@ -130,15 +84,6 @@ func OnDeleteManifest(repo, reference, mediaType string, digest godigest.Digest, manageRepoMetaSuccessfully = false } - - if referredDigest, hasSubject := common.GetReferredSubject(manifestBlob); hasSubject { - err := metaDB.DeleteReferrer(repo, referredDigest, digest) - if err != nil { - log.Error().Err(err).Msg("metadb: error while deleting referrer") - - return err - } - } } if !manageRepoMetaSuccessfully { @@ -152,7 +97,7 @@ func OnDeleteManifest(repo, reference, mediaType string, digest godigest.Digest, } // OnDeleteManifest is called when a manifest is downloaded. It increments the download couter on that manifest. -func OnGetManifest(name, reference string, body []byte, +func OnGetManifest(name, reference, mediaType string, body []byte, storeController storage.StoreController, metaDB mTypes.MetaDB, log log.Logger, ) error { // check if image is a signature @@ -163,14 +108,20 @@ func OnGetManifest(name, reference string, body []byte, return err } - if !isSignature && !zcommon.IsReferrersTag(reference) { - err := metaDB.IncrementImageDownloads(name, reference) - if err != nil { - log.Error().Err(err).Str("repository", name).Str("reference", reference). - Msg("unexpected error for image") + if isSignature || zcommon.IsReferrersTag(reference) { + return nil + } - return err - } + if !(mediaType == v1.MediaTypeImageManifest || mediaType == v1.MediaTypeImageIndex) { + return nil + } + + err = metaDB.IncrementImageDownloads(name, reference) + if err != nil { + log.Error().Err(err).Str("repository", name).Str("reference", reference). + Msg("unexpected error for image") + + return err } return nil diff --git a/pkg/meta/hooks_test.go b/pkg/meta/hooks_test.go index 5f697905..bfea45e4 100644 --- a/pkg/meta/hooks_test.go +++ b/pkg/meta/hooks_test.go @@ -1,24 +1,19 @@ package meta_test import ( - "encoding/json" + "context" "errors" "testing" - notreg "github.com/notaryproject/notation-go/registry" - godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" - zerr "zotregistry.io/zot/errors" "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta" "zotregistry.io/zot/pkg/meta/boltdb" - mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/storage" "zotregistry.io/zot/pkg/storage/local" - "zotregistry.io/zot/pkg/test/deprecated" . "zotregistry.io/zot/pkg/test/image-utils" "zotregistry.io/zot/pkg/test/mocks" ) @@ -42,259 +37,38 @@ func TestOnUpdateManifest(t *testing.T) { metaDB, err := boltdb.New(boltDriver, log) So(err, ShouldBeNil) - config, layers, manifest, err := deprecated.GetRandomImageComponents(100) //nolint:staticcheck + image := CreateDefaultImage() + + err = WriteImageToFileSystem(CreateDefaultImage(), "repo", "tag1", storeController) So(err, ShouldBeNil) - err = WriteImageToFileSystem( - Image{ - Config: config, Manifest: manifest, Layers: layers, - }, "repo", "tag1", storeController) + err = meta.OnUpdateManifest("repo", "tag1", ispec.MediaTypeImageManifest, image.Digest(), + image.ManifestDescriptor.Data, storeController, metaDB, log) So(err, ShouldBeNil) - manifestBlob, err := json.Marshal(manifest) - So(err, ShouldBeNil) - - digest := godigest.FromBytes(manifestBlob) - - err = meta.OnUpdateManifest("repo", "tag1", "", digest, manifestBlob, storeController, metaDB, log) - So(err, ShouldBeNil) - - repoMeta, err := metaDB.GetRepoMeta("repo") + repoMeta, err := metaDB.GetRepoMeta(context.Background(), "repo") So(err, ShouldBeNil) So(repoMeta.Tags, ShouldContainKey, "tag1") }) - - Convey("metadataSuccessfullySet is false", t, func() { - rootDir := t.TempDir() - storeController := storage.StoreController{} - log := log.NewLogger("debug", "") - metrics := monitoring.NewMetricsServer(false, log) - storeController.DefaultStore = local.NewImageStore(rootDir, true, true, log, metrics, nil, nil) - - metaDB := mocks.MetaDBMock{ - SetManifestDataFn: func(manifestDigest godigest.Digest, mm mTypes.ManifestData) error { - return ErrTestError - }, - } - - err := meta.OnUpdateManifest("repo", "tag1", ispec.MediaTypeImageManifest, "digest", - []byte("{}"), storeController, metaDB, log) - So(err, ShouldNotBeNil) - }) } func TestUpdateErrors(t *testing.T) { Convey("Update operations", t, func() { - Convey("On UpdateManifest", func() { - imageStore := mocks.MockedImageStore{} - storeController := storage.StoreController{DefaultStore: &imageStore} - metaDB := mocks.MetaDBMock{} - log := log.NewLogger("debug", "") + imageStore := mocks.MockedImageStore{} + storeController := storage.StoreController{DefaultStore: &imageStore} + metaDB := mocks.MetaDBMock{} + log := log.NewLogger("debug", "") - Convey("CheckIsImageSignature errors", func() { - badManifestBlob := []byte("bad") - - imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return []byte{}, "", "", zerr.ErrManifestNotFound - } - - imageStore.DeleteImageManifestFn = func(repo, reference string, detectCollision bool) error { - return nil - } - - err := meta.OnUpdateManifest("repo", "tag1", "digest", "media", badManifestBlob, - storeController, metaDB, log) - So(err, ShouldNotBeNil) - }) - - Convey("IsReferrersTag true", func() { - err := meta.OnUpdateManifest("repo", "sha256-123", "digest", "media", []byte("bad"), - storeController, metaDB, log) - So(err, ShouldBeNil) - }) - - Convey("GetSignatureLayersInfo errors", func() { - // get notation signature layers info - badNotationManifestContent := ispec.Manifest{ - Subject: &ispec.Descriptor{ - Digest: "123", - }, - Config: ispec.Descriptor{MediaType: notreg.ArtifactTypeNotation}, - } - - badNotationManifestBlob, err := json.Marshal(badNotationManifestContent) - So(err, ShouldBeNil) - - imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return badNotationManifestBlob, "", "", nil - } - - err = meta.OnUpdateManifest("repo", "tag1", "", "digest", badNotationManifestBlob, - storeController, metaDB, log) - So(err, ShouldNotBeNil) - }) - - Convey("UpdateSignaturesValidity", func() { - notationManifestContent := ispec.Manifest{ - Subject: &ispec.Descriptor{ - Digest: "123", - }, - Config: ispec.Descriptor{MediaType: notreg.ArtifactTypeNotation}, - Layers: []ispec.Descriptor{{ - MediaType: ispec.MediaTypeImageLayer, - Digest: godigest.FromString("blob digest"), - }}, - } - - notationManifestBlob, err := json.Marshal(notationManifestContent) - So(err, ShouldBeNil) - - imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return notationManifestBlob, "", "", nil - } - - imageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) { - return []byte{}, nil - } - - metaDB.UpdateSignaturesValidityFn = func(repo string, manifestDigest godigest.Digest) error { - return ErrTestError - } - - err = meta.OnUpdateManifest("repo", "tag1", "", "digest", notationManifestBlob, - storeController, metaDB, log) - So(err, ShouldNotBeNil) - }) - }) - - Convey("On DeleteManifest", func() { - imageStore := mocks.MockedImageStore{} - storeController := storage.StoreController{DefaultStore: &imageStore} - metaDB := mocks.MetaDBMock{} - log := log.NewLogger("debug", "") - - Convey("CheckIsImageSignature errors", func() { - badManifestBlob := []byte("bad") - - imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return []byte{}, "", "", zerr.ErrManifestNotFound - } - - err := meta.OnDeleteManifest("repo", "tag1", "digest", "media", badManifestBlob, - storeController, metaDB, log) - So(err, ShouldNotBeNil) - }) - - Convey("IsReferrersTag true", func() { - err := meta.OnDeleteManifest("repo", "sha256-123", "digest", "media", []byte("bad"), - storeController, metaDB, log) - So(err, ShouldBeNil) - }) - - Convey("DeleteReferrers errors", func() { - metaDB.DeleteReferrerFn = func(repo string, referredDigest, referrerDigest godigest.Digest) error { - return ErrTestError - } - - err := meta.OnDeleteManifest("repo", "tag1", "digest", "media", - []byte(`{"subject": {"digest": "dig"}}`), - storeController, metaDB, log) - So(err, ShouldNotBeNil) - }) - }) - - Convey("On GetManifest", func() { - imageStore := mocks.MockedImageStore{} - storeController := storage.StoreController{DefaultStore: &imageStore} - metaDB := mocks.MetaDBMock{} - log := log.NewLogger("debug", "") - - Convey("CheckIsImageSignature errors", func() { - badManifestBlob := []byte("bad") - - imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return []byte{}, "", "", zerr.ErrManifestNotFound - } - - err := meta.OnGetManifest("repo", "tag1", badManifestBlob, - storeController, metaDB, log) - So(err, ShouldNotBeNil) - }) - }) - - Convey("SetImageMetaFromInput", func() { - imageStore := mocks.MockedImageStore{} - metaDB := mocks.MetaDBMock{} - log := log.NewLogger("debug", "") - - err := meta.SetImageMetaFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest", - []byte("BadManifestBlob"), imageStore, metaDB, log) - So(err, ShouldNotBeNil) - - // reference is digest - - manifestContent := ispec.Manifest{} - manifestBlob, err := json.Marshal(manifestContent) - So(err, ShouldBeNil) - - imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return manifestBlob, "", "", nil - } - imageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) { - return []byte("{}"), nil - } - - err = meta.SetImageMetaFromInput("repo", string(godigest.FromString("reference")), "", "digest", - manifestBlob, imageStore, metaDB, log) + Convey("IsReferrersTag true update", func() { + err := meta.OnUpdateManifest("repo", "sha256-123", "digest", "media", []byte("bad"), + storeController, metaDB, log) So(err, ShouldBeNil) }) - - Convey("SetImageMetaFromInput SetData errors", func() { - imageStore := mocks.MockedImageStore{} - log := log.NewLogger("debug", "") - - metaDB := mocks.MetaDBMock{ - SetManifestDataFn: func(manifestDigest godigest.Digest, mm mTypes.ManifestData) error { - return ErrTestError - }, - } - err := meta.SetImageMetaFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest", - []byte("{}"), imageStore, metaDB, log) - So(err, ShouldNotBeNil) - }) - - Convey("SetImageMetaFromInput SetIndexData errors", func() { - imageStore := mocks.MockedImageStore{} - log := log.NewLogger("debug", "") - - metaDB := mocks.MetaDBMock{ - SetIndexDataFn: func(digest godigest.Digest, indexData mTypes.IndexData) error { - return ErrTestError - }, - } - err := meta.SetImageMetaFromInput("repo", "ref", ispec.MediaTypeImageIndex, "digest", - []byte("{}"), imageStore, metaDB, log) - So(err, ShouldNotBeNil) - }) - - Convey("SetImageMetaFromInput SetReferrer errors", func() { - imageStore := mocks.MockedImageStore{ - GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) { - return []byte("{}"), nil - }, - } - log := log.NewLogger("debug", "") - - metaDB := mocks.MetaDBMock{ - SetReferrerFn: func(repo string, referredDigest godigest.Digest, referrer mTypes.ReferrerInfo) error { - return ErrTestError - }, - } - - err := meta.SetImageMetaFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest", - []byte(`{"subject": {"digest": "subjDigest"}}`), imageStore, metaDB, log) - So(err, ShouldNotBeNil) + Convey("IsReferrersTag true delete", func() { + err := meta.OnDeleteManifest("repo", "sha256-123", "digest", "media", []byte("bad"), + storeController, metaDB, log) + So(err, ShouldBeNil) }) }) } diff --git a/pkg/meta/meta.go b/pkg/meta/meta.go index 7d47c78e..ef685676 100644 --- a/pkg/meta/meta.go +++ b/pkg/meta/meta.go @@ -81,10 +81,10 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) m repoMetaTablename, ok := toStringIfOk(cacheDriverConfig, "repometatablename", log) allParametersOk = allParametersOk && ok - manifestDataTablename, ok := toStringIfOk(cacheDriverConfig, "manifestdatatablename", log) + repoBlobsInfoTablename, ok := toStringIfOk(cacheDriverConfig, "repoblobsinfotablename", log) allParametersOk = allParametersOk && ok - indexDataTablename, ok := toStringIfOk(cacheDriverConfig, "indexdatatablename", log) + imageMetaTablename, ok := toStringIfOk(cacheDriverConfig, "imagemetatablename", log) allParametersOk = allParametersOk && ok apiKeyTablename, ok := toStringIfOk(cacheDriverConfig, "apikeytablename", log) @@ -101,14 +101,14 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) m } return mdynamodb.DBDriverParameters{ - Endpoint: endpoint, - Region: region, - RepoMetaTablename: repoMetaTablename, - ManifestDataTablename: manifestDataTablename, - IndexDataTablename: indexDataTablename, - UserDataTablename: userDataTablename, - APIKeyTablename: apiKeyTablename, - VersionTablename: versionTablename, + Endpoint: endpoint, + Region: region, + RepoMetaTablename: repoMetaTablename, + RepoBlobsInfoTablename: repoBlobsInfoTablename, + ImageMetaTablename: imageMetaTablename, + UserDataTablename: userDataTablename, + APIKeyTablename: apiKeyTablename, + VersionTablename: versionTablename, } } diff --git a/pkg/meta/meta_test.go b/pkg/meta/meta_test.go index 7e9996df..08997c9b 100644 --- a/pkg/meta/meta_test.go +++ b/pkg/meta/meta_test.go @@ -5,9 +5,7 @@ package meta_test import ( "context" - "encoding/json" "fmt" - "math/rand" "os" "path" "testing" @@ -19,11 +17,12 @@ import ( "github.com/notaryproject/notation-go" "github.com/notaryproject/notation-go/signer" godigest "github.com/opencontainers/go-digest" - "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" + zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/imagetrust" + "zotregistry.io/zot/pkg/extensions/search/convert" "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/meta" "zotregistry.io/zot/pkg/meta/boltdb" @@ -31,10 +30,9 @@ import ( mdynamodb "zotregistry.io/zot/pkg/meta/dynamodb" mTypes "zotregistry.io/zot/pkg/meta/types" reqCtx "zotregistry.io/zot/pkg/requestcontext" - test "zotregistry.io/zot/pkg/test/common" - "zotregistry.io/zot/pkg/test/deprecated" + tCommon "zotregistry.io/zot/pkg/test/common" . "zotregistry.io/zot/pkg/test/image-utils" - signature "zotregistry.io/zot/pkg/test/signature" + "zotregistry.io/zot/pkg/test/signature" tskip "zotregistry.io/zot/pkg/test/skip" ) @@ -45,6 +43,8 @@ const ( ARM = "arm64" ) +func getManifestDigest(md mTypes.ManifestData) string { return md.Digest.String() } + func TestBoltDB(t *testing.T) { Convey("BoltDB creation", t, func() { boltDBParams := boltdb.DBParameters{RootDir: t.TempDir()} @@ -106,26 +106,24 @@ func TestDynamoDBWrapper(t *testing.T) { } repoMetaTablename := "RepoMetadataTable" + uuid.String() - manifestDataTablename := "ManifestDataTable" + uuid.String() versionTablename := "Version" + uuid.String() - indexDataTablename := "IndexDataTable" + uuid.String() userDataTablename := "UserDataTable" + uuid.String() apiKeyTablename := "ApiKeyTable" + uuid.String() + imageMetaTablename := "ImageMeta" + uuid.String() + repoBlobsTablename := "RepoBlobs" + uuid.String() Convey("DynamoDB Wrapper", t, func() { dynamoDBDriverParams := mdynamodb.DBDriverParameters{ - Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"), - RepoMetaTablename: repoMetaTablename, - ManifestDataTablename: manifestDataTablename, - IndexDataTablename: indexDataTablename, - VersionTablename: versionTablename, - UserDataTablename: userDataTablename, - APIKeyTablename: apiKeyTablename, - Region: "us-east-2", + Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"), + RepoMetaTablename: repoMetaTablename, + RepoBlobsInfoTablename: repoBlobsTablename, + ImageMetaTablename: imageMetaTablename, + VersionTablename: versionTablename, + UserDataTablename: userDataTablename, + APIKeyTablename: apiKeyTablename, + Region: "us-east-2", } - t.Logf("using dynamo driver options: %v", dynamoDBDriverParams) - dynamoClient, err := mdynamodb.GetDynamoClient(dynamoDBDriverParams) So(err, ShouldBeNil) @@ -141,15 +139,23 @@ func TestDynamoDBWrapper(t *testing.T) { dynamoDriver.SetImageTrustStore(imgTrustStore) resetDynamoDBTables := func() error { - err := dynamoDriver.ResetRepoMetaTable() + err := dynamoDriver.ResetTable(dynamoDriver.RepoMetaTablename) + if err != nil { + return err + } + + err = dynamoDriver.ResetTable(dynamoDriver.ImageMetaTablename) + if err != nil { + return err + } + + err = dynamoDriver.ResetTable(dynamoDriver.RepoBlobsTablename) if err != nil { return err } // Note: Tests are very slow if we reset the UserData table every new convey. We'll reset it as needed - err = dynamoDriver.ResetManifestDataTable() - return err } @@ -158,6 +164,8 @@ func TestDynamoDBWrapper(t *testing.T) { } func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func() error) { //nolint: thelper + ctx := context.Background() + Convey("Test MetaDB Interface implementation", func() { for _, prepFunc := range preparationFuncs { err := prepFunc() @@ -453,242 +461,258 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) }) - Convey("Test SetManifestData and GetManifestData", func() { - configBlob, manifestBlob, err := generateTestImage() + Convey("Test Setting an image by tag and retrieving data", func() { + imgData := CreateImageWith(). + DefaultLayers(). + ImageConfig(ispec.Image{ + Created: DateRef(2000, 10, 10, 10, 10, 10, 10, time.UTC), + Author: "author", + Platform: ispec.Platform{ + Architecture: "arch", + OS: "os", + OSVersion: "os-vers", + OSFeatures: []string{"os-features"}, + Variant: "variant", + }, + Config: ispec.ImageConfig{ + Labels: map[string]string{"test": "test"}, + Env: []string{"test"}, + ExposedPorts: map[string]struct{}{"test": {}}, + Volumes: map[string]struct{}{"test": {}}, + }, + RootFS: ispec.RootFS{ + DiffIDs: []godigest.Digest{godigest.FromString("test")}, + }, + }).Build().AsImageMeta() + + err := metaDB.SetImageMeta(imgData.Digest, imgData) So(err, ShouldBeNil) - manifestDigest := godigest.FromBytes(manifestBlob) + retrievedImgData, err := metaDB.GetImageMeta(imgData.Digest) + So(err, ShouldBeNil) + So(imgData, ShouldResemble, retrievedImgData) - err = metaDB.SetManifestData(manifestDigest, mTypes.ManifestData{ - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - }) + imgMulti := CreateRandomMultiarch() + + for i := range imgMulti.Images { + err = metaDB.SetImageMeta(imgMulti.Images[i].Digest(), imgMulti.Images[i].AsImageMeta()) + So(err, ShouldBeNil) + } + + err = metaDB.SetImageMeta(imgMulti.Digest(), imgMulti.AsImageMeta()) So(err, ShouldBeNil) - mm, err := metaDB.GetManifestData(manifestDigest) + retrievedImgMultiData, err := metaDB.GetImageMeta(imgMulti.Digest()) So(err, ShouldBeNil) - So(mm.ManifestBlob, ShouldResemble, manifestBlob) - So(mm.ConfigBlob, ShouldResemble, configBlob) + So(imgMulti.AsImageMeta(), ShouldEqual, retrievedImgMultiData) + + // set subject on multiarch }) - Convey("Test GetManifestMeta fails", func() { - _, err := metaDB.GetManifestMeta("repo", "bad digest") - So(err, ShouldNotBeNil) - }) + Convey("Set/Get RepoMeta", func() { + err := metaDB.SetRepoMeta("repo", mTypes.RepoMeta{ + Name: "repo", + Tags: map[string]mTypes.Descriptor{"tag": {Digest: "dig"}}, - Convey("Test SetManifestMeta", func() { - Convey("RepoMeta not found", func() { - var ( - manifestDigest = godigest.FromString("dig") - manifestBlob = []byte("manifestBlob") - configBlob = []byte("configBlob") - - signatures = mTypes.ManifestSignatures{ - "digest1": []mTypes.SignatureInfo{ - { - SignatureManifestDigest: "signatureDigest", - LayersInfo: []mTypes.LayerInfo{ - { - LayerDigest: "layerDigest", - LayerContent: []byte("layerContent"), - }, - }, - }, - }, - } - ) - - err := metaDB.SetManifestMeta("repo", manifestDigest, mTypes.ManifestMetadata{ - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - DownloadCount: 10, - Signatures: signatures, - }) - So(err, ShouldBeNil) - - manifestMeta, err := metaDB.GetManifestMeta("repo", manifestDigest) - So(err, ShouldBeNil) - - So(manifestMeta.ManifestBlob, ShouldResemble, manifestBlob) - So(manifestMeta.ConfigBlob, ShouldResemble, configBlob) - So(manifestMeta.DownloadCount, ShouldEqual, 10) - So(manifestMeta.Signatures, ShouldResemble, signatures) + Statistics: map[string]mTypes.DescriptorStatistics{}, + Signatures: map[string]mTypes.ManifestSignatures{}, + Referrers: map[string][]mTypes.ReferrerInfo{"digest": {{Digest: "dig"}}}, }) + So(err, ShouldBeNil) + repoMeta, err := metaDB.GetRepoMeta(ctx, "repo") + So(err, ShouldBeNil) + So(repoMeta.Name, ShouldResemble, "repo") + So(repoMeta.Tags, ShouldContainKey, "tag") }) Convey("Test SetRepoReference", func() { - // test behaviours var ( - repo1 = "repo1" - repo2 = "repo2" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") - - tag2 = "0.0.2" - manifestDigest2 = godigest.FromString("fake-manifes2") + repo1 = "repo1" + repo2 = "repo2" + tag1 = "0.0.1" + tag2 = "0.0.2" ) + img1 := CreateImageWith().RandomLayers(2, 10).RandomConfig(). + Annotations(map[string]string{"test": "annotation"}).Build() + imgData1 := img1.AsImageMeta() + img1Size := img1.ConfigDescriptor.Size + img1.ManifestDescriptor.Size + 2*10 + + img2 := CreateImageWith().LayerBlobs(img1.Layers).RandomConfig(). + Annotations(map[string]string{"test": "annotation"}).Build() + imgData2 := img2.AsImageMeta() + img2Size := img2.ConfigDescriptor.Size + img2.ManifestDescriptor.Size + 2*10 + + multiImages := []Image{ + CreateImageWith().RandomLayers(2, 10). + ImageConfig(ispec.Image{Platform: ispec.Platform{OS: "multi-os1", Architecture: "multi-arch1"}}). + Annotations(map[string]string{ispec.AnnotationVendor: "vendor1"}). + Build(), + CreateImageWith().RandomLayers(2, 10). + ImageConfig(ispec.Image{Platform: ispec.Platform{OS: "multi-os2", Architecture: "multi-arch2"}}). + Annotations(map[string]string{ispec.AnnotationVendor: "vendor2"}). + Build(), + } + + imgMulti := CreateMultiarchWith(). + Images(multiImages). + Annotations(map[string]string{ispec.AnnotationVendor: "vendor1"}).Build() + Convey("Setting a good repo", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(repo1, tag1, imgData1) So(err, ShouldBeNil) - repoMeta, err := metaDB.GetRepoMeta(repo1) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) So(repoMeta.Name, ShouldResemble, repo1) - So(repoMeta.Tags[tag1].Digest, ShouldEqual, manifestDigest1.String()) - - err = metaDB.SetRepoMeta(repo2, mTypes.RepoMetadata{Tags: map[string]mTypes.Descriptor{ - tag2: { - Digest: manifestDigest2.String(), - }, - }}) - So(err, ShouldBeNil) - - repoMeta, err = metaDB.GetRepoMeta(repo2) - So(err, ShouldBeNil) - So(repoMeta.Name, ShouldResemble, repo2) - So(repoMeta.Tags[tag2].Digest, ShouldEqual, manifestDigest2.String()) + So(repoMeta.Tags[tag1].Digest, ShouldEqual, img1.DigestStr()) }) - Convey("Setting a good repo using a digest", func() { - _, err := metaDB.GetRepoMeta(repo1) + Convey("Setting an index with it's manifests", func() { + _, err := metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldNotBeNil) - digest := godigest.FromString("digest") - err = metaDB.SetRepoReference(repo1, digest.String(), digest, - ispec.MediaTypeImageManifest) + for i := range imgMulti.Images { + err := metaDB.SetRepoReference(repo1, imgMulti.Images[i].DigestStr(), + imgMulti.Images[i].AsImageMeta()) + So(err, ShouldBeNil) + } + + err = metaDB.SetRepoReference(repo1, tag1, imgMulti.AsImageMeta()) So(err, ShouldBeNil) - repoMeta, err := metaDB.GetRepoMeta(repo1) + image1TotalSize := multiImages[0].ManifestDescriptor.Size + multiImages[0].ConfigDescriptor.Size + 2*10 + image2TotalSize := multiImages[1].ManifestDescriptor.Size + multiImages[1].ConfigDescriptor.Size + 2*10 + indexTotalSize := image1TotalSize + image2TotalSize + imgMulti.IndexDescriptor.Size + + repoMeta, err := metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) So(repoMeta.Name, ShouldResemble, repo1) - }) - - Convey("Set multiple tags for repo", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - repoMeta, err := metaDB.GetRepoMeta(repo1) - So(err, ShouldBeNil) - So(repoMeta.Tags[tag1].Digest, ShouldEqual, manifestDigest1.String()) - So(repoMeta.Tags[tag2].Digest, ShouldEqual, manifestDigest2.String()) + So(repoMeta.Platforms, ShouldContain, ispec.Platform{OS: "multi-os1", Architecture: "multi-arch1"}) + So(repoMeta.Platforms, ShouldContain, ispec.Platform{OS: "multi-os2", Architecture: "multi-arch2"}) + So(repoMeta.Vendors, ShouldContain, "vendor1") + So(repoMeta.Vendors, ShouldContain, "vendor2") + So(repoMeta.Size, ShouldEqual, indexTotalSize) }) Convey("Set multiple repos", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(repo1, tag1, imgData1) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo2, tag1, imgData2) So(err, ShouldBeNil) - repoMeta1, err := metaDB.GetRepoMeta(repo1) + repoMeta1, err := metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - repoMeta2, err := metaDB.GetRepoMeta(repo2) + repoMeta2, err := metaDB.GetRepoMeta(ctx, repo2) So(err, ShouldBeNil) - So(repoMeta1.Tags[tag1].Digest, ShouldResemble, manifestDigest1.String()) - So(repoMeta2.Tags[tag2].Digest, ShouldResemble, manifestDigest2.String()) + So(repoMeta1.Tags[tag1].Digest, ShouldResemble, imgData1.Digest.String()) + So(repoMeta2.Tags[tag1].Digest, ShouldResemble, imgData2.Digest.String()) + So(repoMeta1.Size, ShouldEqual, img1Size) + So(repoMeta2.Size, ShouldEqual, img2Size) }) - Convey("Setting a repo with invalid fields", func() { - Convey("Repo name is not valid", func() { - err := metaDB.SetRepoReference("", tag1, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldNotBeNil) - }) + Convey("Check repo blobs info for manifest image", func() { + image1 := CreateImageWith().RandomLayers(2, 10). + ImageConfig(ispec.Image{Platform: ispec.Platform{OS: "os1", Architecture: "arch1"}}). + Annotations(map[string]string{ispec.AnnotationVendor: "vendor1"}). + Build() + imageMeta1 := image1.AsImageMeta() - Convey("Tag is not valid", func() { - err := metaDB.SetRepoReference(repo1, "", manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldNotBeNil) - }) + layersSize := int64(2 * 10) + image1Size := imageMeta1.Manifests[0].Size + imageMeta1.Manifests[0].Manifest.Config.Size + layersSize - Convey("Manifest Digest is not valid", func() { - err := metaDB.SetRepoReference(repo1, tag1, "", ispec.MediaTypeImageManifest) - So(err, ShouldNotBeNil) - }) + err := metaDB.SetRepoReference(repo1, tag1, imageMeta1) + So(err, ShouldBeNil) + + repoMeta, err := metaDB.GetRepoMeta(ctx, repo1) + So(err, ShouldBeNil) + So(repoMeta.Vendors, ShouldContain, "vendor1") + So(repoMeta.Platforms, ShouldContain, ispec.Platform{OS: "os1", Architecture: "arch1"}) + So(repoMeta.Size, ShouldEqual, image1Size) + + image2 := CreateImageWith(). + LayerBlobs(image1.Layers). + ImageConfig(ispec.Image{Platform: ispec.Platform{OS: "os2", Architecture: "arch2"}}). + Annotations(map[string]string{ispec.AnnotationVendor: "vendor2"}). + Build() + imageMeta2 := image2.AsImageMeta() + + // the layers are the same so we add them once + repoSize := image1Size + image2.ManifestDescriptor.Size + image2.ConfigDescriptor.Size + + err = metaDB.SetRepoReference(repo1, tag2, imageMeta2) + So(err, ShouldBeNil) + + repoMeta, err = metaDB.GetRepoMeta(ctx, repo1) + So(err, ShouldBeNil) + So(repoMeta.Vendors, ShouldContain, "vendor1") + So(repoMeta.Vendors, ShouldContain, "vendor2") + So(repoMeta.Platforms, ShouldContain, ispec.Platform{OS: "os1", Architecture: "arch1"}) + So(repoMeta.Platforms, ShouldContain, ispec.Platform{OS: "os2", Architecture: "arch2"}) + So(repoMeta.Size, ShouldEqual, repoSize) }) }) - Convey("Test GetRepoMeta", func() { + Convey("Test RemoveRepoReference", func() { var ( - repo1 = "repo1" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") - - repo2 = "repo2" - tag2 = "0.0.2" - manifestDigest2 = godigest.FromString("fake-manifest2") - - InexistentRepo = "InexistentRepo" + repo = "repo1" + tag1 = "0.0.1" + tag2 = "0.0.2" ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + layersSize := int64(2 * 10) + + image1 := CreateImageWith(). + RandomLayers(2, 10). + ImageConfig(ispec.Image{Platform: ispec.Platform{OS: "os1", Architecture: "arch1"}}). + Annotations(map[string]string{ispec.AnnotationVendor: "vendor1"}). + Build() + imageMeta1 := image1.AsImageMeta() + image1Size := imageMeta1.Manifests[0].Size + imageMeta1.Manifests[0].Manifest.Config.Size + layersSize + + image2 := CreateImageWith(). + LayerBlobs(image1.Layers). + ImageConfig(ispec.Image{Platform: ispec.Platform{OS: "os2", Architecture: "arch2"}}). + Annotations(map[string]string{ispec.AnnotationVendor: "vendor2", "annotation": "test"}). + Build() + imageMeta2 := image2.AsImageMeta() + image2Size := imageMeta2.Manifests[0].Size + imageMeta2.Manifests[0].Manifest.Config.Size + layersSize + + totalRepoSize := image1Size + image2Size - layersSize + + err := metaDB.SetRepoReference(repo, tag1, imageMeta1) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - Convey("Get a existent repo", func() { - repoMeta1, err := metaDB.GetRepoMeta(repo1) - So(err, ShouldBeNil) - So(repoMeta1.Tags[tag1].Digest, ShouldResemble, manifestDigest1.String()) - - repoMeta2, err := metaDB.GetRepoMeta(repo2) - So(err, ShouldBeNil) - So(repoMeta2.Tags[tag2].Digest, ShouldResemble, manifestDigest2.String()) - }) - - Convey("Get a repo that doesn't exist", func() { - repoMeta, err := metaDB.GetRepoMeta(InexistentRepo) - So(err, ShouldNotBeNil) - So(repoMeta, ShouldBeZeroValue) - }) - }) - - Convey("Test RemoveRepoReference and DeleteRepoTag", func() { - var ( - repo = "repo1" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") - tag2 = "0.0.2" - manifestDigest2 = godigest.FromString("fake-manifest2") - ) - - err := metaDB.SetRepoReference(repo, tag1, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - err = metaDB.SetRepoReference(repo, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo, tag2, imageMeta2) So(err, ShouldBeNil) Convey("Delete reference from repo", func() { - _, err := metaDB.GetRepoMeta(repo) + err = metaDB.RemoveRepoReference(repo, tag1, imageMeta1.Digest) So(err, ShouldBeNil) - err = metaDB.RemoveRepoReference(repo, tag1, manifestDigest1) - So(err, ShouldBeNil) - - repoMeta, err := metaDB.GetRepoMeta(repo) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo) So(err, ShouldBeNil) _, ok := repoMeta.Tags[tag1] So(ok, ShouldBeFalse) - So(repoMeta.Tags[tag2].Digest, ShouldResemble, manifestDigest2.String()) + So(repoMeta.Size, ShouldEqual, image2Size) + So(repoMeta.Platforms, ShouldNotContain, ispec.Platform{OS: "os1", Architecture: "arch1"}) + So(repoMeta.Vendors, ShouldNotContain, "vendor1") }) - Convey("Delete a reference from repo", func() { - _, err := metaDB.GetRepoMeta(repo) + Convey("Delete a digest from repo", func() { + err = metaDB.RemoveRepoReference(repo, tag2, imageMeta2.Digest) So(err, ShouldBeNil) - // shouldn't do anything because there is tag1 pointing to it - err = metaDB.RemoveRepoReference(repo, manifestDigest1.String(), manifestDigest1) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo) So(err, ShouldBeNil) - repoMeta, err := metaDB.GetRepoMeta(repo) - So(err, ShouldBeNil) - - _, ok := repoMeta.Tags[tag1] + _, ok := repoMeta.Tags[tag2] So(ok, ShouldBeFalse) - So(repoMeta.Tags[tag2].Digest, ShouldResemble, manifestDigest2.String()) + So(repoMeta.Size, ShouldEqual, image1Size) + So(repoMeta.Platforms, ShouldNotContain, ispec.Platform{OS: "os2", Architecture: "arch2"}) + So(repoMeta.Vendors, ShouldNotContain, "vendor2") }) Convey("Delete inexistent reference from repo", func() { @@ -696,11 +720,12 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err := metaDB.RemoveRepoReference(repo, inexistentDigest.String(), inexistentDigest) So(err, ShouldBeNil) - repoMeta, err := metaDB.GetRepoMeta(repo) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo) So(err, ShouldBeNil) - So(repoMeta.Tags[tag1].Digest, ShouldResemble, manifestDigest1.String()) - So(repoMeta.Tags[tag2].Digest, ShouldResemble, manifestDigest2.String()) + So(repoMeta.Tags[tag1].Digest, ShouldResemble, imageMeta1.Digest.String()) + So(repoMeta.Tags[tag2].Digest, ShouldResemble, imageMeta2.Digest.String()) + So(repoMeta.Size, ShouldEqual, totalRepoSize) }) Convey("Delete reference from inexistent repo", func() { @@ -709,72 +734,48 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err := metaDB.RemoveRepoReference("InexistentRepo", inexistentDigest.String(), inexistentDigest) So(err, ShouldBeNil) - repoMeta, err := metaDB.GetRepoMeta(repo) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo) So(err, ShouldBeNil) - So(repoMeta.Tags[tag1].Digest, ShouldResemble, manifestDigest1.String()) - So(repoMeta.Tags[tag2].Digest, ShouldResemble, manifestDigest2.String()) - }) - - Convey("Delete from repo a tag", func() { - _, err := metaDB.GetRepoMeta(repo) - So(err, ShouldBeNil) - - err = metaDB.DeleteRepoTag(repo, tag1) - So(err, ShouldBeNil) - - repoMeta, err := metaDB.GetRepoMeta(repo) - So(err, ShouldBeNil) - - _, ok := repoMeta.Tags[tag1] - So(ok, ShouldBeFalse) - So(repoMeta.Tags[tag2].Digest, ShouldResemble, manifestDigest2.String()) - }) - - Convey("Delete inexistent tag from repo", func() { - err := metaDB.DeleteRepoTag(repo, "InexistentTag") - So(err, ShouldBeNil) - - repoMeta, err := metaDB.GetRepoMeta(repo) - So(err, ShouldBeNil) - - So(repoMeta.Tags[tag1].Digest, ShouldResemble, manifestDigest1.String()) - So(repoMeta.Tags[tag2].Digest, ShouldResemble, manifestDigest2.String()) - }) - - Convey("Delete tag from inexistent repo", func() { - err := metaDB.DeleteRepoTag("InexistentRepo", "InexistentTag") - So(err, ShouldBeNil) - - repoMeta, err := metaDB.GetRepoMeta(repo) - So(err, ShouldBeNil) - - So(repoMeta.Tags[tag1].Digest, ShouldResemble, manifestDigest1.String()) - So(repoMeta.Tags[tag2].Digest, ShouldResemble, manifestDigest2.String()) + So(repoMeta.Tags[tag1].Digest, ShouldResemble, imageMeta1.Digest.String()) + So(repoMeta.Tags[tag2].Digest, ShouldResemble, imageMeta2.Digest.String()) + So(repoMeta.Size, ShouldEqual, totalRepoSize) }) }) Convey("Test GetMultipleRepoMeta", func() { var ( - repo1 = "repo1" - repo2 = "repo2" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") - tag2 = "0.0.2" - manifestDigest2 = godigest.FromString("fake-manifest2") + repo1 = "repo1" + repo2 = "repo2" + tag1 = "0.0.1" + tag2 = "0.0.2" ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + image1 := CreateImageWith(). + RandomLayers(2, 10). + ImageConfig(ispec.Image{Platform: ispec.Platform{OS: "os1", Architecture: "arch1"}}). + Annotations(map[string]string{ispec.AnnotationVendor: "vendor1"}). + Build() + imageMeta1 := image1.AsImageMeta() + + image2 := CreateImageWith(). + LayerBlobs(image1.Layers). + ImageConfig(ispec.Image{Platform: ispec.Platform{OS: "os2", Architecture: "arch2"}}). + Annotations(map[string]string{ispec.AnnotationVendor: "vendor2"}). + Build() + imageMeta2 := image2.AsImageMeta() + + err := metaDB.SetRepoReference(repo1, tag1, imageMeta1) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, tag2, imageMeta2) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo2, tag2, imageMeta2) So(err, ShouldBeNil) - Convey("Get all Repometa", func() { - repoMetaSlice, err := metaDB.GetMultipleRepoMeta(context.TODO(), func(repoMeta mTypes.RepoMetadata) bool { + Convey("Get all RepoMeta", func() { + repoMetaSlice, err := metaDB.GetMultipleRepoMeta(context.TODO(), func(repoMeta mTypes.RepoMeta) bool { return true }) So(err, ShouldBeNil) @@ -782,7 +783,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) Convey("Get repo with a tag", func() { - repoMetaSlice, err := metaDB.GetMultipleRepoMeta(context.TODO(), func(repoMeta mTypes.RepoMetadata) bool { + repoMetaSlice, err := metaDB.GetMultipleRepoMeta(context.TODO(), func(repoMeta mTypes.RepoMeta) bool { for tag := range repoMeta.Tags { if tag == tag1 { return true @@ -793,113 +794,110 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) So(err, ShouldBeNil) So(len(repoMetaSlice), ShouldEqual, 1) - So(repoMetaSlice[0].Tags[tag1].Digest == manifestDigest1.String(), ShouldBeTrue) + So(repoMetaSlice[0].Tags[tag1].Digest == imageMeta1.Digest.String(), ShouldBeTrue) }) }) Convey("Test IncrementRepoStars", func() { var ( - repo1 = "repo1" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") + repo1 = "repo1" + tag1 = "0.0.1" + imageMeta = CreateDefaultImage().AsImageMeta() ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(repo1, tag1, imageMeta) So(err, ShouldBeNil) err = metaDB.IncrementRepoStars(repo1) So(err, ShouldBeNil) - repoMeta, err := metaDB.GetRepoMeta(repo1) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoMeta.Stars, ShouldEqual, 1) + So(repoMeta.StarCount, ShouldEqual, 1) err = metaDB.IncrementRepoStars(repo1) So(err, ShouldBeNil) - repoMeta, err = metaDB.GetRepoMeta(repo1) + repoMeta, err = metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoMeta.Stars, ShouldEqual, 2) + So(repoMeta.StarCount, ShouldEqual, 2) err = metaDB.IncrementRepoStars(repo1) So(err, ShouldBeNil) - repoMeta, err = metaDB.GetRepoMeta(repo1) + repoMeta, err = metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoMeta.Stars, ShouldEqual, 3) + So(repoMeta.StarCount, ShouldEqual, 3) }) Convey("Test DecrementRepoStars", func() { var ( - repo1 = "repo1" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") + repo1 = "repo1" + tag1 = "0.0.1" + imageMeta = CreateDefaultImage().AsImageMeta() ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(repo1, tag1, imageMeta) So(err, ShouldBeNil) err = metaDB.IncrementRepoStars(repo1) So(err, ShouldBeNil) - repoMeta, err := metaDB.GetRepoMeta(repo1) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoMeta.Stars, ShouldEqual, 1) + So(repoMeta.StarCount, ShouldEqual, 1) err = metaDB.DecrementRepoStars(repo1) So(err, ShouldBeNil) - repoMeta, err = metaDB.GetRepoMeta(repo1) + repoMeta, err = metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoMeta.Stars, ShouldEqual, 0) + So(repoMeta.StarCount, ShouldEqual, 0) err = metaDB.DecrementRepoStars(repo1) So(err, ShouldBeNil) - repoMeta, err = metaDB.GetRepoMeta(repo1) + repoMeta, err = metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoMeta.Stars, ShouldEqual, 0) + So(repoMeta.StarCount, ShouldEqual, 0) - _, err = metaDB.GetRepoMeta("badRepo") + _, err = metaDB.GetRepoMeta(ctx, "badRepo") So(err, ShouldNotBeNil) }) - Convey("Test GetRepoStars", func() { + Convey("Test Repo Stars", func() { var ( - repo1 = "repo1" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") + repo1 = "repo1" + tag1 = "0.0.1" ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(repo1, tag1, CreateDefaultImage().AsImageMeta()) So(err, ShouldBeNil) err = metaDB.IncrementRepoStars(repo1) So(err, ShouldBeNil) - - stars, err := metaDB.GetRepoStars(repo1) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(stars, ShouldEqual, 1) + So(repoMeta.StarCount, ShouldEqual, 1) err = metaDB.IncrementRepoStars(repo1) So(err, ShouldBeNil) + repoMeta, err = metaDB.GetRepoMeta(ctx, repo1) + So(err, ShouldBeNil) + So(repoMeta.StarCount, ShouldEqual, 2) + err = metaDB.IncrementRepoStars(repo1) So(err, ShouldBeNil) - - stars, err = metaDB.GetRepoStars(repo1) + repoMeta, err = metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(stars, ShouldEqual, 3) - - _, err = metaDB.GetRepoStars("badRepo") - So(err, ShouldNotBeNil) + So(repoMeta.StarCount, ShouldEqual, 3) }) Convey("Test repo stars for user", func() { var ( - repo1 = "repo1" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") - repo2 = "repo2" + repo1 = "repo1" + tag1 = "0.0.1" + repo2 = "repo2" ) userAc := reqCtx.NewUserAccessControl() @@ -910,7 +908,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) // "user1" - ctx1 := userAc.DeriveContext(context.Background()) + ctx1 := userAc.DeriveContext(ctx) userAc = reqCtx.NewUserAccessControl() userAc.SetUsername("user2") @@ -920,7 +918,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) // "user2" - ctx2 := userAc.DeriveContext(context.Background()) + ctx2 := userAc.DeriveContext(ctx) userAc = reqCtx.NewUserAccessControl() userAc.SetGlobPatterns("read", map[string]bool{ @@ -929,22 +927,14 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) // anonymous user - ctx3 := userAc.DeriveContext(context.Background()) + ctx3 := userAc.DeriveContext(ctx) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(repo1, tag1, CreateDefaultImage().AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo2, tag1, CreateDefaultImage().AsImageMeta()) So(err, ShouldBeNil) - starCount, err := metaDB.GetRepoStars(repo1) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 0) - - starCount, err = metaDB.GetRepoStars(repo2) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 0) - repos, err := metaDB.GetStarredRepos(ctx1) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 0) @@ -962,13 +952,9 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) So(toggleState, ShouldEqual, mTypes.Added) - repoMeta, err := metaDB.GetRepoMeta(repo1) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoMeta.Stars, ShouldEqual, 1) - - starCount, err = metaDB.GetRepoStars(repo1) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 1) + So(repoMeta.StarCount, ShouldEqual, 1) repos, err = metaDB.GetStarredRepos(ctx1) So(err, ShouldBeNil) @@ -988,13 +974,9 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) So(toggleState, ShouldEqual, mTypes.Added) - repoMeta, err = metaDB.GetRepoMeta(repo1) + repoMeta, err = metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoMeta.Stars, ShouldEqual, 2) - - starCount, err = metaDB.GetRepoStars(repo1) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 2) + So(repoMeta.StarCount, ShouldEqual, 2) repos, err = metaDB.GetStarredRepos(ctx1) So(err, ShouldBeNil) @@ -1015,13 +997,9 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) So(toggleState, ShouldEqual, mTypes.Added) - repoMeta, err = metaDB.GetRepoMeta(repo2) + repoMeta, err = metaDB.GetRepoMeta(ctx, repo2) So(err, ShouldBeNil) - So(repoMeta.Stars, ShouldEqual, 1) - - starCount, err = metaDB.GetRepoStars(repo2) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 1) + So(repoMeta.StarCount, ShouldEqual, 1) repos, err = metaDB.GetStarredRepos(ctx1) So(err, ShouldBeNil) @@ -1043,13 +1021,9 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) So(toggleState, ShouldEqual, mTypes.Removed) - repoMeta, err = metaDB.GetRepoMeta(repo1) + repoMeta, err = metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoMeta.Stars, ShouldEqual, 1) - - starCount, err = metaDB.GetRepoStars(repo1) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 1) + So(repoMeta.StarCount, ShouldEqual, 1) repos, err = metaDB.GetStarredRepos(ctx1) So(err, ShouldBeNil) @@ -1074,21 +1048,13 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) So(toggleState, ShouldEqual, mTypes.Removed) - repoMeta, err = metaDB.GetRepoMeta(repo1) + repoMeta, err = metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoMeta.Stars, ShouldEqual, 1) + So(repoMeta.StarCount, ShouldEqual, 1) - repoMeta, err = metaDB.GetRepoMeta(repo2) + repoMeta, err = metaDB.GetRepoMeta(ctx, repo2) So(err, ShouldBeNil) - So(repoMeta.Stars, ShouldEqual, 1) - - starCount, err = metaDB.GetRepoStars(repo1) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 1) - - starCount, err = metaDB.GetRepoStars(repo2) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 1) + So(repoMeta.StarCount, ShouldEqual, 1) repos, err = metaDB.GetStarredRepos(ctx1) So(err, ShouldBeNil) @@ -1104,15 +1070,11 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) So(len(repos), ShouldEqual, 0) - // Anonyous user attempts to toggle a star + // Anonymous user attempts to toggle a star toggleState, err = metaDB.ToggleStarRepo(ctx3, repo1) So(err, ShouldNotBeNil) So(toggleState, ShouldEqual, mTypes.NotChanged) - starCount, err = metaDB.GetRepoStars(repo1) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 1) - repos, err = metaDB.GetStarredRepos(ctx3) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 0) @@ -1122,10 +1084,6 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(err, ShouldBeNil) So(toggleState, ShouldEqual, mTypes.Removed) - starCount, err = metaDB.GetRepoStars(repo2) - So(err, ShouldBeNil) - So(starCount, ShouldEqual, 0) - repos, err = metaDB.GetStarredRepos(ctx3) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 0) @@ -1133,10 +1091,10 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func Convey("Test repo bookmarks for user", func() { var ( - repo1 = "repo1" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") - repo2 = "repo2" + repo1 = "repo1" + tag1 = "0.0.1" + repo2 = "repo2" + image1 = CreateRandomImage().AsImageMeta() ) userAc := reqCtx.NewUserAccessControl() @@ -1168,10 +1126,10 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func // anonymous user ctx3 := userAc.DeriveContext(context.Background()) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(repo1, tag1, image1) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo2, tag1, image1) So(err, ShouldBeNil) repos, err := metaDB.GetBookmarkedRepos(ctx1) @@ -1277,122 +1235,98 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func Convey("Test IncrementImageDownloads", func() { var ( - repo1 = "repo1" - tag1 = "0.0.1" + repo1 = "repo1" + tag1 = "0.0.1" + image1 = CreateRandomImage().AsImageMeta() ) - configBlob, manifestBlob, err := generateTestImage() - So(err, ShouldBeNil) - - manifestDigest := godigest.FromBytes(manifestBlob) - - err = metaDB.SetRepoReference(repo1, tag1, manifestDigest, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestDigest, mTypes.ManifestMetadata{ - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - }) + err := metaDB.SetRepoReference(repo1, tag1, image1) So(err, ShouldBeNil) err = metaDB.IncrementImageDownloads(repo1, tag1) So(err, ShouldBeNil) - repoMeta, err := metaDB.GetRepoMeta(repo1) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 1) + So(repoMeta.Statistics[image1.Digest.String()].DownloadCount, ShouldEqual, 1) err = metaDB.IncrementImageDownloads(repo1, tag1) So(err, ShouldBeNil) - repoMeta, err = metaDB.GetRepoMeta(repo1) + repoMeta, err = metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 2) - - _, err = metaDB.GetManifestMeta(repo1, "badManiestDigest") - So(err, ShouldNotBeNil) + So(repoMeta.Statistics[image1.Digest.String()].DownloadCount, ShouldEqual, 2) }) Convey("Test AddImageSignature", func() { var ( - repo1 = "repo1" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") + repo1 = "repo1" + tag1 = "0.0.1" + image1 = CreateRandomImage().AsImageMeta() ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(repo1, tag1, image1) So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo1, manifestDigest1, mTypes.ManifestMetadata{}) - So(err, ShouldBeNil) - - err = metaDB.AddManifestSignature(repo1, manifestDigest1, mTypes.SignatureMetadata{ + err = metaDB.AddManifestSignature(repo1, image1.Digest, mTypes.SignatureMetadata{ SignatureType: "cosign", SignatureDigest: "digest", + LayersInfo: []mTypes.LayerInfo{{LayerDigest: "layer-digest", LayerContent: []byte{10}}}, }) So(err, ShouldBeNil) - repoMeta, err := metaDB.GetRepoMeta(repo1) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoMeta.Signatures[manifestDigest1.String()]["cosign"][0].SignatureManifestDigest, + So(repoMeta.Signatures[image1.Digest.String()]["cosign"][0].SignatureManifestDigest, ShouldResemble, "digest") - _, err = metaDB.GetManifestMeta(repo1, "badDigest") - So(err, ShouldNotBeNil) + imageMeta, err := metaDB.GetImageMeta(image1.Digest) + + fullImageMeta := convert.GetFullImageMeta(tag1, repoMeta, imageMeta) + So(err, ShouldBeNil) + So(fullImageMeta.Signatures["cosign"][0].SignatureManifestDigest, ShouldResemble, "digest") + So(fullImageMeta.Signatures["cosign"][0].LayersInfo[0].LayerDigest, ShouldResemble, "layer-digest") + So(fullImageMeta.Signatures["cosign"][0].LayersInfo[0].LayerContent[0], ShouldEqual, 10) }) Convey("Test UpdateSignaturesValidity", func() { Convey("untrusted signature", func() { var ( - repo1 = "repo1" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("dig") + repo1 = "repo1" + tag1 = "0.0.1" + image1 = CreateRandomImage() ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestDigest1, mTypes.ManifestMetadata{ - ManifestBlob: []byte("Bad Manifest"), - ConfigBlob: []byte("Bad Manifest"), - }) + err := metaDB.SetRepoReference(repo1, tag1, image1.AsImageMeta()) So(err, ShouldBeNil) layerInfo := mTypes.LayerInfo{LayerDigest: "", LayerContent: []byte{}, SignatureKey: ""} - err = metaDB.AddManifestSignature(repo1, manifestDigest1, mTypes.SignatureMetadata{ + err = metaDB.AddManifestSignature(repo1, image1.Digest(), mTypes.SignatureMetadata{ SignatureType: "cosign", - SignatureDigest: string(manifestDigest1), + SignatureDigest: image1.DigestStr(), LayersInfo: []mTypes.LayerInfo{layerInfo}, }) So(err, ShouldBeNil) - err = metaDB.UpdateSignaturesValidity(repo1, manifestDigest1) + err = metaDB.UpdateSignaturesValidity(repo1, image1.Digest()) So(err, ShouldBeNil) - repoData, err := metaDB.GetRepoMeta(repo1) + repoData, err := metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoData.Signatures[string(manifestDigest1)]["cosign"][0].LayersInfo[0].Signer, + So(repoData.Signatures[image1.DigestStr()]["cosign"][0].LayersInfo[0].Signer, ShouldBeEmpty) - So(repoData.Signatures[string(manifestDigest1)]["cosign"][0].LayersInfo[0].Date, + So(repoData.Signatures[image1.DigestStr()]["cosign"][0].LayersInfo[0].Date, ShouldBeZeroValue) }) Convey("trusted signature", func() { - _, _, manifest, _ := deprecated.GetRandomImageComponents(10) //nolint:staticcheck - manifestContent, _ := json.Marshal(manifest) - manifestDigest := godigest.FromBytes(manifestContent) - repo := "repo" + image1 := CreateRandomImage() + repo := "repo1" tag := "0.0.1" - err := metaDB.SetRepoReference(repo, tag, manifestDigest, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo, manifestDigest, mTypes.ManifestMetadata{ - ManifestBlob: manifestContent, - ConfigBlob: []byte("configContent"), - }) + err := metaDB.SetRepoReference(repo, tag, image1.AsImageMeta()) So(err, ShouldBeNil) mediaType := jws.MediaTypeEnvelope @@ -1424,7 +1358,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func signingKeys, err := signature.LoadNotationSigningkeys(tdir) So(err, ShouldBeNil) - idx := test.Index(signingKeys.Keys, keyName) + idx := tCommon.Index(signingKeys.Keys, keyName) So(idx, ShouldBeGreaterThanOrEqualTo, 0) key := signingKeys.Keys[idx] @@ -1435,9 +1369,9 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func } descToSign := ispec.Descriptor{ - MediaType: manifest.MediaType, - Digest: manifestDigest, - Size: int64(len(manifestContent)), + MediaType: image1.Manifest.MediaType, + Digest: image1.Digest(), + Size: image1.ManifestDescriptor.Size, } ctx := context.Background() @@ -1450,7 +1384,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func LayerContent: sig, SignatureKey: mediaType, } - err = metaDB.AddManifestSignature(repo, manifestDigest, mTypes.SignatureMetadata{ + err = metaDB.AddManifestSignature(repo, image1.Digest(), mTypes.SignatureMetadata{ SignatureType: "notation", SignatureDigest: string(godigest.FromString("signature digest")), LayersInfo: []mTypes.LayerInfo{layerInfo}, @@ -1471,80 +1405,71 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func err = imagetrust.UploadCertificate(imgTrustStore.NotationStorage, certificateContent, "ca") So(err, ShouldBeNil) - err = metaDB.UpdateSignaturesValidity(repo, manifestDigest) //nolint:contextcheck + err = metaDB.UpdateSignaturesValidity(repo, image1.Digest()) //nolint:contextcheck So(err, ShouldBeNil) - repoData, err := metaDB.GetRepoMeta(repo) + repoData, err := metaDB.GetRepoMeta(ctx, repo) So(err, ShouldBeNil) - So(repoData.Signatures[string(manifestDigest)]["notation"][0].LayersInfo[0].Signer, + So(repoData.Signatures[image1.DigestStr()]["notation"][0].LayersInfo[0].Signer, ShouldNotBeEmpty) - So(repoData.Signatures[string(manifestDigest)]["notation"][0].LayersInfo[0].Date, + So(repoData.Signatures[image1.DigestStr()]["notation"][0].LayersInfo[0].Date, ShouldNotBeZeroValue) }) }) Convey("Test AddImageSignature with inverted order", func() { var ( - repo1 = "repo1" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") + repo1 = "repo1" + tag1 = "0.0.1" + image1 = CreateRandomImage() ) - err := metaDB.AddManifestSignature(repo1, manifestDigest1, mTypes.SignatureMetadata{ + err := metaDB.AddManifestSignature(repo1, image1.Digest(), mTypes.SignatureMetadata{ SignatureType: "cosign", SignatureDigest: "digest", }) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, tag1, image1.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetManifestData(manifestDigest1, mTypes.ManifestData{}) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - - repoMeta, err := metaDB.GetRepoMeta(repo1) - So(err, ShouldBeNil) - So(repoMeta.Signatures[manifestDigest1.String()]["cosign"][0].SignatureManifestDigest, + So(repoMeta.Signatures[image1.DigestStr()]["cosign"][0].SignatureManifestDigest, ShouldResemble, "digest") - - _, err = metaDB.GetManifestMeta(repo1, "badDigest") - So(err, ShouldNotBeNil) }) Convey("Test DeleteSignature", func() { var ( - repo1 = "repo1" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") + repo1 = "repo1" + tag1 = "0.0.1" + image1 = CreateRandomImage() ) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(repo1, tag1, image1.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetManifestData(manifestDigest1, mTypes.ManifestData{}) - So(err, ShouldBeNil) - - err = metaDB.AddManifestSignature(repo1, manifestDigest1, mTypes.SignatureMetadata{ + err = metaDB.AddManifestSignature(repo1, image1.Digest(), mTypes.SignatureMetadata{ SignatureType: "cosign", SignatureDigest: "digest", }) So(err, ShouldBeNil) - repoMeta, err := metaDB.GetRepoMeta(repo1) + repoMeta, err := metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoMeta.Signatures[manifestDigest1.String()]["cosign"][0].SignatureManifestDigest, + So(repoMeta.Signatures[image1.DigestStr()]["cosign"][0].SignatureManifestDigest, ShouldResemble, "digest") - err = metaDB.DeleteSignature(repo1, manifestDigest1, mTypes.SignatureMetadata{ + err = metaDB.DeleteSignature(repo1, image1.Digest(), mTypes.SignatureMetadata{ SignatureType: "cosign", SignatureDigest: "digest", }) So(err, ShouldBeNil) - repoMeta, err = metaDB.GetRepoMeta(repo1) + repoMeta, err = metaDB.GetRepoMeta(ctx, repo1) So(err, ShouldBeNil) - So(repoMeta.Signatures[manifestDigest1.String()]["cosign"], ShouldBeEmpty) + So(repoMeta.Signatures[image1.DigestStr()]["cosign"], ShouldBeEmpty) err = metaDB.DeleteSignature(repo1, "badDigest", mTypes.SignatureMetadata{ SignatureType: "cosign", @@ -1555,138 +1480,89 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func Convey("Test SearchRepos", func() { var ( - repo1 = "repo1" - repo2 = "repo2" - repo3 = "repo3" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") - tag2 = "0.0.2" - manifestDigest2 = godigest.FromString("fake-manifest2") - tag3 = "0.0.3" - manifestDigest3 = godigest.FromString("fake-manifest3") - ctx = context.Background() - emptyManifest ispec.Manifest - emptyConfig ispec.Manifest + repo1 = "repo1" + repo2 = "repo2" + repo3 = "repo3" + tag1 = "0.0.1" + tag2 = "0.0.2" + tag3 = "0.0.3" + image1 = CreateRandomImage() + image2 = CreateRandomImage() + image3 = CreateRandomImage() + ctx = context.Background() ) - emptyManifestBlob, err := json.Marshal(emptyManifest) - So(err, ShouldBeNil) - - emptyConfigBlob, err := json.Marshal(emptyConfig) - So(err, ShouldBeNil) - - emptyRepoMeta := mTypes.ManifestMetadata{ - ManifestBlob: emptyManifestBlob, - ConfigBlob: emptyConfigBlob, - } - + _ = repo3 Convey("Search all repos", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(repo1, tag1, image1.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, tag2, image2.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag3, manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo2, tag3, image3.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) - So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo1, manifestDigest2, emptyRepoMeta) - So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo1, manifestDigest3, emptyRepoMeta) + repoMetaList, err := metaDB.SearchRepos(ctx, "") So(err, ShouldBeNil) + So(len(repoMetaList), ShouldEqual, 2) - repos, manifestMetaMap, _, err := metaDB.SearchRepos(ctx, "") - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 2) - So(len(manifestMetaMap), ShouldEqual, 3) - So(manifestMetaMap, ShouldContainKey, manifestDigest1.String()) - So(manifestMetaMap, ShouldContainKey, manifestDigest2.String()) - So(manifestMetaMap, ShouldContainKey, manifestDigest3.String()) + So(repoMetaList[0].Tags[tag1].Digest, ShouldResemble, image1.DigestStr()) + So(repoMetaList[0].Tags[tag2].Digest, ShouldResemble, image2.DigestStr()) + So(repoMetaList[1].Tags[tag3].Digest, ShouldResemble, image3.DigestStr()) }) Convey("Search a repo by name", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(repo1, tag1, image1.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) + repoMetaList, err := metaDB.SearchRepos(ctx, repo1) So(err, ShouldBeNil) - - repos, manifestMetaMap, _, err := metaDB.SearchRepos(ctx, repo1) - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(len(manifestMetaMap), ShouldEqual, 1) - So(manifestMetaMap, ShouldContainKey, manifestDigest1.String()) + So(len(repoMetaList), ShouldEqual, 1) + So(repoMetaList[0].Tags[tag1].Digest, ShouldResemble, image1.DigestStr()) }) Convey("Search non-existing repo by name", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(repo1, tag1, image1.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, tag2, image2.AsImageMeta()) So(err, ShouldBeNil) - repos, manifestMetaMap, _, err := metaDB.SearchRepos(ctx, "RepoThatDoesntExist") + repoMetaList, err := metaDB.SearchRepos(ctx, "RepoThatDoesntExist") So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 0) - So(len(manifestMetaMap), ShouldEqual, 0) + So(len(repoMetaList), ShouldEqual, 0) }) Convey("Search with partial match", func() { - err := metaDB.SetRepoReference("alpine", tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference("alpine", tag1, image1.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("pine", tag2, manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("pine", tag2, image2.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("golang", tag3, manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("golang", tag3, image3.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetManifestMeta("alpine", manifestDigest1, emptyRepoMeta) + repoMetaList, err := metaDB.SearchRepos(ctx, "pine") So(err, ShouldBeNil) - err = metaDB.SetManifestMeta("pine", manifestDigest2, emptyRepoMeta) - So(err, ShouldBeNil) - err = metaDB.SetManifestMeta("golang", manifestDigest3, emptyRepoMeta) - So(err, ShouldBeNil) - - repos, manifestMetaMap, _, err := metaDB.SearchRepos(ctx, "pine") - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 2) - So(manifestMetaMap, ShouldContainKey, manifestDigest1.String()) - So(manifestMetaMap, ShouldContainKey, manifestDigest2.String()) - So(manifestMetaMap, ShouldNotContainKey, manifestDigest3.String()) + So(len(repoMetaList), ShouldEqual, 2) }) Convey("Search multiple repos that share manifests", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference("alpine", tag1, image1.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("pine", tag2, image1.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("golang", tag3, image1.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) + repoMetaList, err := metaDB.SearchRepos(ctx, "") So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo2, manifestDigest1, emptyRepoMeta) - So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo3, manifestDigest1, emptyRepoMeta) - So(err, ShouldBeNil) - - repos, manifestMetaMap, _, err := metaDB.SearchRepos(ctx, "") - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 3) - So(len(manifestMetaMap), ShouldEqual, 1) + So(len(repoMetaList), ShouldEqual, 3) }) Convey("Search repos with access control", func() { - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) + err := metaDB.SetRepoReference(repo1, tag1, image1.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo2, tag2, image2.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) - So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo2, manifestDigest1, emptyRepoMeta) - So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo3, manifestDigest1, emptyRepoMeta) + err = metaDB.SetRepoReference(repo3, tag3, image3.AsImageMeta()) So(err, ShouldBeNil) userAc := reqCtx.NewUserAccessControl() @@ -1698,153 +1574,84 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func ctx := userAc.DeriveContext(context.Background()) - repos, _, _, err := metaDB.SearchRepos(ctx, "repo") + repoMetaList, err := metaDB.SearchRepos(ctx, "repo") So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 2) - for _, k := range repos { + So(len(repoMetaList), ShouldEqual, 2) + for _, k := range repoMetaList { So(k.Name, ShouldBeIn, []string{repo1, repo2}) } }) Convey("Search Repos with Indexes", func() { var ( - tag4 = "0.0.4" - indexDigest = godigest.FromString("Multiarch") - manifestDigest1 = godigest.FromString("manifestDigest1") - manifestDigest2 = godigest.FromString("manifestDigest2") + tag4 = "0.0.4" + subImage1 = CreateRandomImage() + subImage2 = CreateRandomImage() + multiarch = CreateMultiarchWith().Images([]Image{subImage1, subImage2}).Build() - tag5 = "0.0.5" - manifestDigest3 = godigest.FromString("manifestDigest3") + tag5 = "0.0.5" + image1 = CreateRandomImage() ) - err := metaDB.SetManifestData(manifestDigest1, mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("{}"), - }) + err := metaDB.SetRepoReference("repo", subImage1.DigestStr(), subImage1.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference("repo", subImage2.DigestStr(), subImage2.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference("repo", tag4, multiarch.AsImageMeta()) So(err, ShouldBeNil) - config := ispec.Image{ - Platform: ispec.Platform{ - Architecture: "arch", - OS: "os", - }, - } - - confBlob, err := json.Marshal(config) + err = metaDB.SetRepoReference("repo", tag5, image1.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetManifestData(manifestDigest2, mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - ConfigBlob: confBlob, - }) - So(err, ShouldBeNil) - err = metaDB.SetManifestData(manifestDigest3, mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("{}"), - }) + repoMetaList, err := metaDB.SearchRepos(ctx, "repo") So(err, ShouldBeNil) - indexContent := ispec.Index{ - MediaType: ispec.MediaTypeImageIndex, - Manifests: []ispec.Descriptor{ - { - Digest: manifestDigest1, - }, - { - Digest: manifestDigest2, - }, - }, - } - - indexBlob, err := json.Marshal(indexContent) - So(err, ShouldBeNil) - - err = metaDB.SetIndexData(indexDigest, mTypes.IndexData{ - IndexBlob: indexBlob, - }) - So(err, ShouldBeNil) - - err = metaDB.SetRepoReference("repo", tag4, indexDigest, ispec.MediaTypeImageIndex) - So(err, ShouldBeNil) - - err = metaDB.SetRepoReference("repo", tag5, manifestDigest3, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - repos, manifestMetaMap, indexDataMap, err := metaDB.SearchRepos(ctx, "repo") - So(err, ShouldBeNil) - - So(len(repos), ShouldEqual, 1) - So(repos[0].Name, ShouldResemble, "repo") - So(repos[0].Tags, ShouldContainKey, tag4) - So(repos[0].Tags, ShouldContainKey, tag5) - So(manifestMetaMap, ShouldContainKey, manifestDigest1.String()) - So(manifestMetaMap, ShouldContainKey, manifestDigest2.String()) - So(manifestMetaMap, ShouldContainKey, manifestDigest3.String()) - So(indexDataMap, ShouldContainKey, indexDigest.String()) + So(len(repoMetaList), ShouldEqual, 1) + So(repoMetaList[0].Name, ShouldResemble, "repo") + So(repoMetaList[0].Tags, ShouldContainKey, tag4) + So(repoMetaList[0].Tags[tag4].MediaType, ShouldResemble, ispec.MediaTypeImageIndex) + So(repoMetaList[0].Tags, ShouldContainKey, tag5) + So(repoMetaList[0].Tags[tag5].MediaType, ShouldResemble, ispec.MediaTypeImageManifest) }) }) Convey("Test SearchTags", func() { var ( - repo1 = "repo1" - repo2 = "repo2" - manifestDigest1 = godigest.FromString("fake-manifest1") - manifestDigest2 = godigest.FromString("fake-manifest2") - manifestDigest3 = godigest.FromString("fake-manifest3") - ctx = context.Background() - emptyManifest ispec.Manifest - emptyConfig ispec.Manifest + repo1 = "repo1" + repo2 = "repo2" + image1 = CreateRandomImage() + image2 = CreateRandomImage() + image3 = CreateRandomImage() + ctx = context.Background() ) - emptyManifestBlob, err := json.Marshal(emptyManifest) + err := metaDB.SetRepoReference(repo1, "0.0.1", image1.AsImageMeta()) So(err, ShouldBeNil) - - emptyConfigBlob, err := json.Marshal(emptyConfig) + err = metaDB.SetRepoReference(repo1, "0.0.2", image3.AsImageMeta()) So(err, ShouldBeNil) - - emptyRepoMeta := mTypes.ManifestMetadata{ - ManifestBlob: emptyManifestBlob, - ConfigBlob: emptyConfigBlob, - } - - err = metaDB.SetRepoReference(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, "0.1.0", image2.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, "1.0.0", image2.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, "1.0.1", image2.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyRepoMeta) - So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo1, manifestDigest2, emptyRepoMeta) - So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo1, manifestDigest3, emptyRepoMeta) - So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo2, manifestDigest3, emptyRepoMeta) + err = metaDB.SetRepoReference(repo2, "0.0.1", image3.AsImageMeta()) So(err, ShouldBeNil) Convey("With exact match", func() { - repos, manifestMetaMap, _, err := metaDB.SearchTags(ctx, "repo1:0.0.1") + fullImageMetaList, err := metaDB.SearchTags(ctx, "repo1:0.0.1") So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(len(repos[0].Tags), ShouldEqual, 1) - So(repos[0].Tags, ShouldContainKey, "0.0.1") - So(manifestMetaMap, ShouldContainKey, manifestDigest1.String()) + So(len(fullImageMetaList), ShouldEqual, 1) + So(fullImageMetaList[0].Digest.String(), ShouldResemble, image1.DigestStr()) }) Convey("With no match", func() { - repos, _, _, err := metaDB.SearchTags(ctx, "repo1:badtag") + fullImageMetaList, err := metaDB.SearchTags(ctx, "repo1:badtag") So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 0) + So(len(fullImageMetaList), ShouldEqual, 0) }) - Convey("With no permision", func() { + Convey("With no permission", func() { userAc := reqCtx.NewUserAccessControl() userAc.SetUsername("user1") userAc.SetGlobPatterns("read", @@ -1855,76 +1662,51 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func ) ctx1 := userAc.DeriveContext(context.Background()) - repos, _, _, err := metaDB.SearchTags(ctx1, "repo1:0.0.1") + fullImageMetaList, err := metaDB.SearchTags(ctx1, "repo1:0.0.1") So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 0) + So(len(fullImageMetaList), ShouldEqual, 0) }) Convey("With partial repo path", func() { - repos, manifestMetaMap, _, err := metaDB.SearchTags(ctx, "repo:0.0.1") + fullImageMetaList, err := metaDB.SearchTags(ctx, "repo:0.0.1") So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 0) - So(len(manifestMetaMap), ShouldEqual, 0) + So(len(fullImageMetaList), ShouldEqual, 0) }) Convey("With partial tag", func() { - repos, manifestMetaMap, _, err := metaDB.SearchTags(ctx, "repo1:0.0") + fullImageMetaList, err := metaDB.SearchTags(ctx, "repo1:0.0") So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(len(repos[0].Tags), ShouldEqual, 2) - So(repos[0].Tags, ShouldContainKey, "0.0.2") - So(repos[0].Tags, ShouldContainKey, "0.0.1") - So(manifestMetaMap, ShouldContainKey, manifestDigest1.String()) - So(manifestMetaMap, ShouldContainKey, manifestDigest3.String()) + So(len(fullImageMetaList), ShouldEqual, 2) - repos, manifestMetaMap, _, err = metaDB.SearchTags(ctx, "repo1:0.") + tags := map[string]struct{}{} + for _, imageMeta := range fullImageMetaList { + tags[imageMeta.Tag] = struct{}{} + } + + So(tags, ShouldContainKey, "0.0.2") + So(tags, ShouldContainKey, "0.0.1") + + fullImageMetaList, err = metaDB.SearchTags(ctx, "repo1:0.") So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(len(repos[0].Tags), ShouldEqual, 3) - So(repos[0].Tags, ShouldContainKey, "0.0.1") - So(repos[0].Tags, ShouldContainKey, "0.0.2") - So(repos[0].Tags, ShouldContainKey, "0.1.0") - So(manifestMetaMap, ShouldContainKey, manifestDigest1.String()) - So(manifestMetaMap, ShouldContainKey, manifestDigest2.String()) - So(manifestMetaMap, ShouldContainKey, manifestDigest3.String()) + So(len(fullImageMetaList), ShouldEqual, 3) + + tags = map[string]struct{}{} + for _, imageMeta := range fullImageMetaList { + tags[imageMeta.Tag] = struct{}{} + } + + So(tags, ShouldContainKey, "0.0.1") + So(tags, ShouldContainKey, "0.0.2") + So(tags, ShouldContainKey, "0.1.0") }) Convey("With bad query", func() { - repos, manifestMetaMap, _, err := metaDB.SearchTags(ctx, "repo:0.0.1:test") + fullImageMetaList, err := metaDB.SearchTags(ctx, "repo:0.0.1:test") So(err, ShouldNotBeNil) - So(len(repos), ShouldEqual, 0) - So(len(manifestMetaMap), ShouldEqual, 0) + So(len(fullImageMetaList), ShouldEqual, 0) }) Convey("Search with access control", func() { - var ( - repo1 = "repo1" - repo2 = "repo2" - repo3 = "repo3" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") - tag2 = "0.0.2" - tag3 = "0.0.3" - ) - - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, tag2, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo3, tag3, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - config := ispec.Image{} - configBlob, err := json.Marshal(config) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestDigest1, mTypes.ManifestMetadata{ConfigBlob: configBlob}) - So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo2, manifestDigest1, mTypes.ManifestMetadata{ConfigBlob: configBlob}) - So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo3, manifestDigest1, mTypes.ManifestMetadata{ConfigBlob: configBlob}) - So(err, ShouldBeNil) - userAc := reqCtx.NewUserAccessControl() userAc.SetUsername("username") userAc.SetGlobPatterns("read", map[string]bool{ @@ -1934,297 +1716,205 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func ctx := userAc.DeriveContext(context.Background()) - repos, _, _, err := metaDB.SearchTags(ctx, "repo1:") + fullImageMetaList, err := metaDB.SearchTags(ctx, "repo1:") So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(repos[0].Name, ShouldResemble, repo1) + So(len(fullImageMetaList), ShouldEqual, 5) + So(fullImageMetaList[0].Repo, ShouldResemble, repo1) - repos, _, _, err = metaDB.SearchTags(ctx, "repo2:") + fullImageMetaList, err = metaDB.SearchTags(ctx, "repo2:") So(err, ShouldBeNil) - So(repos, ShouldBeEmpty) + So(fullImageMetaList, ShouldBeEmpty) }) Convey("Search Tags with Indexes", func() { var ( - tag4 = "0.0.4" - indexDigest = godigest.FromString("Multiarch") - manifestDigest1 = godigest.FromString("manifestDigest1") - manifestDigest2 = godigest.FromString("manifestDigest2") + tag4 = "0.0.4" + subImage1 = CreateRandomImage() + subImage2 = CreateRandomImage() + multiarch = CreateMultiarchWith().Images([]Image{subImage1, subImage2}).Build() - tag5 = "0.0.5" - manifestDigest3 = godigest.FromString("manifestDigest3") + tag5 = "0.0.5" + image5 = CreateRandomImage() - tag6 = "6.0.0" - manifestDigest4 = godigest.FromString("manifestDigest4") + tag6 = "6.0.0" + image6 = CreateRandomImage() ) - err := metaDB.SetManifestData(manifestDigest1, mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("{}"), - }) + err = metaDB.SetRepoReference("repo", subImage1.DigestStr(), subImage1.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference("repo", subImage2.DigestStr(), subImage2.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference("repo", tag4, multiarch.AsImageMeta()) So(err, ShouldBeNil) - config := ispec.Image{ - Platform: ispec.Platform{ - Architecture: "arch", - OS: "os", - }, + err = metaDB.SetRepoReference("repo", tag5, image5.AsImageMeta()) + So(err, ShouldBeNil) + + err = metaDB.SetRepoReference("repo", tag6, image6.AsImageMeta()) + So(err, ShouldBeNil) + + fullImageMetaList, err := metaDB.SearchTags(ctx, "repo:0.0") + So(err, ShouldBeNil) + + tags := map[string]struct{}{} + for _, imageMeta := range fullImageMetaList { + tags[imageMeta.Tag] = struct{}{} } - confBlob, err := json.Marshal(config) - So(err, ShouldBeNil) + So(len(fullImageMetaList), ShouldEqual, 2) + So(tags, ShouldContainKey, tag4) + So(tags, ShouldContainKey, tag5) + So(tags, ShouldNotContainKey, tag6) - err = metaDB.SetManifestData(manifestDigest2, mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - ConfigBlob: confBlob, - }) - So(err, ShouldBeNil) - err = metaDB.SetManifestData(manifestDigest3, mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("{}"), - }) - So(err, ShouldBeNil) + multiarchImageMeta := mTypes.FullImageMeta{} + found := false - err = metaDB.SetManifestData(manifestDigest4, mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("{}"), - }) - So(err, ShouldBeNil) + for _, imageMeta := range fullImageMetaList { + if imageMeta.MediaType == ispec.MediaTypeImageIndex { + multiarchImageMeta = imageMeta + found = true + } + } - indexBlob, err := GetIndexBlobWithManifests( - []godigest.Digest{ - manifestDigest1, - manifestDigest2, - }, - ) - So(err, ShouldBeNil) + So(found, ShouldBeTrue) + So(len(multiarchImageMeta.Manifests), ShouldEqual, 2) - err = metaDB.SetIndexData(indexDigest, mTypes.IndexData{ - IndexBlob: indexBlob, - }) - So(err, ShouldBeNil) + digests := []string{} + for _, manifest := range multiarchImageMeta.Manifests { + digests = append(digests, manifest.Digest.String()) + } - err = metaDB.SetRepoReference("repo", tag4, indexDigest, ispec.MediaTypeImageIndex) - So(err, ShouldBeNil) - - err = metaDB.SetRepoReference("repo", tag5, manifestDigest3, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - err = metaDB.SetRepoReference("repo", tag6, manifestDigest4, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - repos, manifestMetaMap, indexDataMap, err := metaDB.SearchTags(ctx, "repo:0.0") - So(err, ShouldBeNil) - - So(len(repos), ShouldEqual, 1) - So(repos[0].Name, ShouldResemble, "repo") - So(repos[0].Tags, ShouldContainKey, tag4) - So(repos[0].Tags, ShouldContainKey, tag5) - So(repos[0].Tags, ShouldNotContainKey, tag6) - So(manifestMetaMap, ShouldContainKey, manifestDigest1.String()) - So(manifestMetaMap, ShouldContainKey, manifestDigest2.String()) - So(manifestMetaMap, ShouldContainKey, manifestDigest3.String()) - So(manifestMetaMap, ShouldNotContainKey, manifestDigest4.String()) - So(indexDataMap, ShouldContainKey, indexDigest.String()) + So(digests, ShouldContain, subImage1.DigestStr()) + So(digests, ShouldContain, subImage2.DigestStr()) }) - }) - Convey("Paginated tag search", func() { - var ( - repo1 = "repo1" - tag1 = "0.0.1" - manifestDigest1 = godigest.FromString("fake-manifest1") - tag2 = "0.0.2" - tag3 = "0.0.3" - tag4 = "0.0.4" - tag5 = "0.0.5" - ) + Convey("With referrer", func() { + refImage := CreateRandomImageWith().Subject(image1.DescriptorRef()).Build() + err := metaDB.SetRepoReference(repo1, "ref-tag", refImage.AsImageMeta()) + So(err, ShouldBeNil) - err := metaDB.SetRepoReference(repo1, tag1, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag2, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag3, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag4, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, tag5, manifestDigest1, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - config := ispec.Image{} - configBlob, err := json.Marshal(config) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestDigest1, mTypes.ManifestMetadata{ConfigBlob: configBlob}) - So(err, ShouldBeNil) - - repos, _, _, err := metaDB.SearchTags(context.TODO(), "repo1:") - - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - keys := make([]string, 0, len(repos[0].Tags)) - for k := range repos[0].Tags { - keys = append(keys, k) - } - - repos, _, _, err = metaDB.SearchTags(context.TODO(), "repo1:") - - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - for k := range repos[0].Tags { - keys = append(keys, k) - } - - repos, _, _, err = metaDB.SearchTags(context.TODO(), "repo1:") - - So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - for k := range repos[0].Tags { - keys = append(keys, k) - } - - So(keys, ShouldContain, tag1) - So(keys, ShouldContain, tag2) - So(keys, ShouldContain, tag3) + fullImageMetaList, err := metaDB.SearchTags(ctx, "repo1:0.0.1") + So(err, ShouldBeNil) + So(len(fullImageMetaList), ShouldEqual, 1) + So(len(fullImageMetaList[0].Referrers), ShouldEqual, 1) + So(fullImageMetaList[0].Referrers[0].Digest, ShouldResemble, refImage.DigestStr()) + }) }) Convey("Test FilterTags", func() { var ( - repo1 = "repo1" - repo2 = "repo2" - manifestDigest1 = godigest.FromString("fake-manifest1") - manifestDigest2 = godigest.FromString("fake-manifest2") - manifestDigest3 = godigest.FromString("fake-manifest3") - indexDigest = godigest.FromString("index-digest") - manifestFromIndexDigest1 = godigest.FromString("fake-manifestFromIndexDigest1") - manifestFromIndexDigest2 = godigest.FromString("fake-manifestFromIndexDigest2") + repo1 = "repo1" + repo2 = "repo2" + image1 = CreateRandomImage() + image2 = CreateRandomImage() + image3 = CreateRandomImage() - emptyManifest ispec.Manifest - emptyConfig ispec.Image - ctx = context.Background() + subImage1 = CreateRandomImage() + subImage2 = CreateRandomImage() + multiarch = CreateMultiarchWith().Images([]Image{subImage1, subImage2}).Build() + ctx = context.Background() ) - emptyManifestBlob, err := json.Marshal(emptyManifest) + err := metaDB.SetRepoReference(repo1, subImage1.DigestStr(), subImage1.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(repo1, subImage2.DigestStr(), subImage2.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(repo1, "2.0.0", multiarch.AsImageMeta()) So(err, ShouldBeNil) - emptyConfigBlob, err := json.Marshal(emptyConfig) + err = metaDB.SetRepoReference(repo1, "0.0.1", image1.AsImageMeta()) So(err, ShouldBeNil) - - emptyManifestMeta := mTypes.ManifestMetadata{ - ManifestBlob: emptyManifestBlob, - ConfigBlob: emptyConfigBlob, - } - - emptyManifestData := mTypes.ManifestData{ - ManifestBlob: emptyManifestBlob, - ConfigBlob: emptyConfigBlob, - } - - err = metaDB.SetRepoReference(repo1, "2.0.0", indexDigest, ispec.MediaTypeImageIndex) + err = metaDB.SetRepoReference(repo1, "0.0.2", image3.AsImageMeta()) So(err, ShouldBeNil) - - indexBlob, err := GetIndexBlobWithManifests([]godigest.Digest{ - manifestFromIndexDigest1, - manifestFromIndexDigest2, - }) + err = metaDB.SetRepoReference(repo1, "0.1.0", image2.AsImageMeta()) So(err, ShouldBeNil) - - err = metaDB.SetIndexData(indexDigest, mTypes.IndexData{ - IndexBlob: indexBlob, - }) + err = metaDB.SetRepoReference(repo1, "1.0.0", image2.AsImageMeta()) So(err, ShouldBeNil) - - err = metaDB.SetRepoReference(repo1, "0.0.1", manifestDigest1, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo1, "1.0.1", image2.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.0.2", manifestDigest3, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "0.1.0", manifestDigest2, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.0", manifestDigest2, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo1, "1.0.1", manifestDigest2, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo2, "0.0.1", manifestDigest3, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - err = metaDB.SetManifestMeta(repo1, manifestDigest1, emptyManifestMeta) - So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo1, manifestDigest2, emptyManifestMeta) - So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo1, manifestDigest3, emptyManifestMeta) - So(err, ShouldBeNil) - err = metaDB.SetManifestMeta(repo2, manifestDigest3, emptyManifestMeta) - So(err, ShouldBeNil) - - err = metaDB.SetManifestData(manifestFromIndexDigest1, emptyManifestData) - So(err, ShouldBeNil) - err = metaDB.SetManifestData(manifestFromIndexDigest2, emptyManifestData) + err = metaDB.SetRepoReference(repo2, "0.0.1", image3.AsImageMeta()) So(err, ShouldBeNil) Convey("Return all tags", func() { - repos, manifestMetaMap, indexDataMap, err := metaDB.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return true - }) + fullImageMetaList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta) So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 2) - So(repos[0].Name, ShouldEqual, "repo1") - So(repos[1].Name, ShouldEqual, "repo2") - So(len(repos[0].Tags), ShouldEqual, 6) - So(len(repos[1].Tags), ShouldEqual, 1) - So(repos[0].Tags, ShouldContainKey, "0.0.1") - So(repos[0].Tags, ShouldContainKey, "0.0.2") - So(repos[0].Tags, ShouldContainKey, "0.1.0") - So(repos[0].Tags, ShouldContainKey, "1.0.0") - So(repos[0].Tags, ShouldContainKey, "1.0.1") - So(repos[0].Tags, ShouldContainKey, "2.0.0") - So(repos[1].Tags, ShouldContainKey, "0.0.1") + So(len(fullImageMetaList), ShouldEqual, 7) - So(manifestMetaMap, ShouldContainKey, manifestDigest1.String()) - So(manifestMetaMap, ShouldContainKey, manifestDigest2.String()) - So(manifestMetaMap, ShouldContainKey, manifestDigest3.String()) + tags := []string{} + indexImage := mTypes.FullImageMeta{} - So(indexDataMap, ShouldContainKey, indexDigest.String()) - So(manifestMetaMap, ShouldContainKey, manifestFromIndexDigest1.String()) - So(manifestMetaMap, ShouldContainKey, manifestFromIndexDigest2.String()) + for _, imageMeta := range fullImageMetaList { + tags = append(tags, imageMeta.Tag) + + if imageMeta.MediaType == ispec.MediaTypeImageIndex { + indexImage = imageMeta + } + } + + So(zcommon.Contains(tags, "0.0.1"), ShouldBeTrue) + So(zcommon.Contains(tags, "0.0.2"), ShouldBeTrue) + So(zcommon.Contains(tags, "0.1.0"), ShouldBeTrue) + So(zcommon.Contains(tags, "1.0.0"), ShouldBeTrue) + So(zcommon.Contains(tags, "1.0.1"), ShouldBeTrue) + So(zcommon.Contains(tags, "2.0.0"), ShouldBeTrue) + So(zcommon.Contains(tags, "0.0.1"), ShouldBeTrue) + + So(indexImage.Digest.String(), ShouldResemble, multiarch.DigestStr()) + + digests := []string{} + for _, manifest := range indexImage.Manifests { + digests = append(digests, manifest.Digest.String()) + } + + So(digests, ShouldContain, subImage1.DigestStr()) + So(digests, ShouldContain, subImage2.DigestStr()) }) Convey("Return all tags in a specific repo", func() { - repos, manifestMetaMap, indexDataMap, err := metaDB.FilterTags( - ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return repoMeta.Name == repo1 - }) + fullImageMetaList, err := metaDB.FilterTags(ctx, func(repo, tag string) bool { return repo == repo1 }, + mTypes.AcceptAllImageMeta) So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(repos[0].Name, ShouldEqual, repo1) - So(len(repos[0].Tags), ShouldEqual, 6) - So(repos[0].Tags, ShouldContainKey, "0.0.1") - So(repos[0].Tags, ShouldContainKey, "0.0.2") - So(repos[0].Tags, ShouldContainKey, "0.1.0") - So(repos[0].Tags, ShouldContainKey, "1.0.0") - So(repos[0].Tags, ShouldContainKey, "1.0.1") - So(repos[0].Tags, ShouldContainKey, "2.0.0") - So(manifestMetaMap, ShouldContainKey, manifestDigest1.String()) - So(manifestMetaMap, ShouldContainKey, manifestDigest2.String()) - So(manifestMetaMap, ShouldContainKey, manifestDigest3.String()) - So(indexDataMap, ShouldContainKey, indexDigest.String()) - So(manifestMetaMap, ShouldContainKey, manifestFromIndexDigest1.String()) - So(manifestMetaMap, ShouldContainKey, manifestFromIndexDigest2.String()) + So(len(fullImageMetaList), ShouldEqual, 6) + + tags := map[string]struct{}{} + indexImage := mTypes.FullImageMeta{} + + for _, imageMeta := range fullImageMetaList { + tags[imageMeta.Tag] = struct{}{} + + if imageMeta.MediaType == ispec.MediaTypeImageIndex { + indexImage = imageMeta + } + } + + So(tags, ShouldContainKey, "0.0.1") + So(tags, ShouldContainKey, "0.0.2") + So(tags, ShouldContainKey, "0.1.0") + So(tags, ShouldContainKey, "1.0.0") + So(tags, ShouldContainKey, "1.0.1") + So(tags, ShouldContainKey, "2.0.0") + + So(indexImage.Digest.String(), ShouldResemble, multiarch.DigestStr()) + + digests := []string{} + for _, manifest := range indexImage.Manifests { + digests = append(digests, manifest.Digest.String()) + } + + So(digests, ShouldContain, subImage1.DigestStr()) + So(digests, ShouldContain, subImage2.DigestStr()) }) Convey("Filter everything out", func() { - repos, manifestMetaMap, _, err := metaDB.FilterTags( - ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return false - }) + fullImageMetaList, err := metaDB.FilterTags(ctx, + func(repo, tag string) bool { return false }, + func(repoMeta mTypes.RepoMeta, imageMeta mTypes.ImageMeta) bool { return false }, + ) So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 0) - So(len(manifestMetaMap), ShouldEqual, 0) + So(len(fullImageMetaList), ShouldEqual, 0) }) Convey("Search with access control", func() { @@ -2237,243 +1927,288 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func ctx := userAc.DeriveContext(context.Background()) - repos, manifestMetaMap, _, err := metaDB.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { - return true - }) + fullImageMetaList, err := metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta) So(err, ShouldBeNil) - So(len(repos), ShouldEqual, 1) - So(repos[0].Name, ShouldResemble, repo2) - So(len(repos[0].Tags), ShouldEqual, 1) - So(repos[0].Tags, ShouldContainKey, "0.0.1") - So(manifestMetaMap, ShouldContainKey, manifestDigest3.String()) + So(len(fullImageMetaList), ShouldEqual, 1) + So(fullImageMetaList[0].Repo, ShouldResemble, repo2) + So(fullImageMetaList[0].Tag, ShouldResemble, "0.0.1") }) }) - Convey("Test index logic", func() { - multiArch, err := deprecated.GetRandomMultiarchImage("tag1") //nolint:staticcheck - So(err, ShouldBeNil) - - indexDigest := multiArch.Digest() - - indexData := multiArch.IndexData() - - err = metaDB.SetIndexData(indexDigest, indexData) - So(err, ShouldBeNil) - - result, err := metaDB.GetIndexData(indexDigest) - So(err, ShouldBeNil) - So(result, ShouldResemble, indexData) - - _, err = metaDB.GetIndexData(godigest.FromString("inexistent")) - So(err, ShouldNotBeNil) - }) - Convey("Test Referrers", func() { - image, err := deprecated.GetRandomImage() //nolint:staticcheck + image1 := CreateRandomImage() + + err := metaDB.SetRepoReference("repo", "tag", image1.AsImageMeta()) So(err, ShouldBeNil) - referredDigest := image.Digest() + // Artifact 1 with artifact type in Manifest + artifact1 := CreateImageWith(). + RandomLayers(10, 2).DefaultConfig(). + ArtifactType("art-type1"). + Subject(image1.DescriptorRef()). + Build() - manifestBlob, err := json.Marshal(image.Manifest) + err = metaDB.SetRepoReference("repo", artifact1.DigestStr(), artifact1.AsImageMeta()) So(err, ShouldBeNil) - configBlob, err := json.Marshal(image.Config) + // Artifact 2 with artifact type in Config media type + artifact2 := CreateImageWith(). + RandomLayers(10, 2). + ArtifactConfig("art-type2"). + Subject(image1.DescriptorRef()). + Build() + + err = metaDB.SetRepoReference("repo", artifact2.DigestStr(), artifact2.AsImageMeta()) So(err, ShouldBeNil) - manifestData := mTypes.ManifestData{ - ManifestBlob: manifestBlob, - ConfigBlob: configBlob, - } - - err = metaDB.SetManifestData(referredDigest, manifestData) - So(err, ShouldBeNil) - - err = metaDB.SetRepoReference("repo", "tag", referredDigest, ispec.MediaTypeImageManifest) - So(err, ShouldBeNil) - - // ------- Add Artifact 1 - - artifact1, err := deprecated.GetImageWithSubject( //nolint:staticcheck - referredDigest, - ispec.MediaTypeImageManifest, - ) - So(err, ShouldBeNil) - - artifactDigest1 := artifact1.Digest() - - err = metaDB.SetReferrer("repo", referredDigest, mTypes.ReferrerInfo{ - Digest: artifactDigest1.String(), - MediaType: ispec.MediaTypeImageManifest, - }) - So(err, ShouldBeNil) - - // ------- Add Artifact 2 - - artifact2, err := deprecated.GetImageWithSubject( //nolint:staticcheck - referredDigest, - ispec.MediaTypeImageManifest, - ) - So(err, ShouldBeNil) - - artifactDigest2 := artifact2.Digest() - - err = metaDB.SetReferrer("repo", referredDigest, mTypes.ReferrerInfo{ - Digest: artifactDigest2.String(), - MediaType: ispec.MediaTypeImageManifest, - }) - So(err, ShouldBeNil) - - // ------ GetReferrers - - referrers, err := metaDB.GetReferrersInfo("repo", referredDigest, nil) + // GetReferrers + referrers, err := metaDB.GetReferrersInfo("repo", image1.Digest(), nil) So(len(referrers), ShouldEqual, 2) So(referrers, ShouldContain, mTypes.ReferrerInfo{ - Digest: artifactDigest1.String(), - MediaType: ispec.MediaTypeImageManifest, + Digest: artifact1.DigestStr(), + MediaType: ispec.MediaTypeImageManifest, + ArtifactType: "art-type1", + Size: int(artifact1.ManifestDescriptor.Size), }) So(referrers, ShouldContain, mTypes.ReferrerInfo{ - Digest: artifactDigest2.String(), - MediaType: ispec.MediaTypeImageManifest, + Digest: artifact2.DigestStr(), + MediaType: ispec.MediaTypeImageManifest, + ArtifactType: "art-type2", + Size: int(artifact2.ManifestDescriptor.Size), }) So(err, ShouldBeNil) - // ------ DeleteReferrers - - err = metaDB.DeleteReferrer("repo", referredDigest, artifactDigest1) + // Delete the Referrers + err = metaDB.RemoveRepoReference("repo", artifact1.DigestStr(), artifact1.Digest()) So(err, ShouldBeNil) - err = metaDB.DeleteReferrer("repo", referredDigest, artifactDigest2) + referrers, err = metaDB.GetReferrersInfo("repo", image1.Digest(), nil) + So(err, ShouldBeNil) + So(len(referrers), ShouldEqual, 1) + + err = metaDB.RemoveRepoReference("repo", artifact2.DigestStr(), artifact2.Digest()) So(err, ShouldBeNil) - referrers, err = metaDB.GetReferrersInfo("repo", referredDigest, nil) + referrers, err = metaDB.GetReferrersInfo("repo", image1.Digest(), nil) So(err, ShouldBeNil) So(len(referrers), ShouldEqual, 0) }) - Convey("Test Referrers on empty Repo", func() { - repoMeta, err := metaDB.GetRepoMeta("repo") + Convey("Test Referrers add same one twice but with different tags - delete by tag then digest", func() { + _, err := metaDB.GetRepoMeta(ctx, "repo") So(err, ShouldNotBeNil) - So(repoMeta, ShouldResemble, mTypes.RepoMetadata{}) - referredDigest := godigest.FromString("referredDigest") - referrerDigest := godigest.FromString("referrerDigest") + tag := "tag1" + refTag := "refTag" + image := CreateRandomImage() + referrer := CreateRandomImageWith().Subject(image.DescriptorRef()).Build() - err = metaDB.SetReferrer("repo", referredDigest, mTypes.ReferrerInfo{ - Digest: referrerDigest.String(), - MediaType: ispec.MediaTypeImageManifest, - }) + err = metaDB.SetRepoReference("repo", tag, image.AsImageMeta()) So(err, ShouldBeNil) - repoMeta, err = metaDB.GetRepoMeta("repo") + err = metaDB.SetRepoReference("repo", refTag, referrer.AsImageMeta()) So(err, ShouldBeNil) - So(repoMeta.Referrers[referredDigest.String()][0].Digest, ShouldResemble, referrerDigest.String()) + + err = metaDB.SetRepoReference("repo", referrer.DigestStr(), referrer.AsImageMeta()) + So(err, ShouldBeNil) + + repoMeta, err := metaDB.GetRepoMeta(ctx, "repo") + So(err, ShouldBeNil) + So(len(repoMeta.Referrers[image.DigestStr()]), ShouldEqual, 1) + + err = metaDB.RemoveRepoReference("repo", refTag, referrer.Digest()) + So(err, ShouldBeNil) + + // we still have the untagged manifest + repoMeta, err = metaDB.GetRepoMeta(ctx, "repo") + So(err, ShouldBeNil) + So(len(repoMeta.Referrers[image.DigestStr()]), ShouldEqual, 1) + + err = metaDB.RemoveRepoReference("repo", referrer.DigestStr(), referrer.Digest()) + So(err, ShouldBeNil) + + repoMeta, err = metaDB.GetRepoMeta(ctx, "repo") + So(err, ShouldBeNil) + So(len(repoMeta.Referrers[image.DigestStr()]), ShouldEqual, 0) + }) + + Convey("Test Referrers add same one twice but with different tags - delete by digest", func() { + _, err := metaDB.GetRepoMeta(ctx, "repo") + So(err, ShouldNotBeNil) + + tag := "tag-1" + refTag := "refTag" + image := CreateRandomImage() + referrer := CreateRandomImageWith().Subject(image.DescriptorRef()).Build() + + err = metaDB.SetRepoReference("repo", tag, image.AsImageMeta()) + So(err, ShouldBeNil) + + err = metaDB.SetRepoReference("repo", refTag, referrer.AsImageMeta()) + So(err, ShouldBeNil) + + err = metaDB.SetRepoReference("repo", referrer.DigestStr(), referrer.AsImageMeta()) + So(err, ShouldBeNil) + + repoMeta, err := metaDB.GetRepoMeta(ctx, "repo") + So(err, ShouldBeNil) + So(len(repoMeta.Referrers[image.DigestStr()]), ShouldEqual, 1) + + // this should delete all references + err = metaDB.RemoveRepoReference("repo", referrer.DigestStr(), referrer.Digest()) + So(err, ShouldBeNil) + + repoMeta, err = metaDB.GetRepoMeta(ctx, "repo") + So(err, ShouldBeNil) + So(len(repoMeta.Referrers[image.DigestStr()]), ShouldEqual, 0) }) Convey("Test Referrers add same one twice", func() { - repoMeta, err := metaDB.GetRepoMeta("repo") + _, err := metaDB.GetRepoMeta(ctx, "repo") So(err, ShouldNotBeNil) - So(repoMeta, ShouldResemble, mTypes.RepoMetadata{}) - referredDigest := godigest.FromString("referredDigest") - referrerDigest := godigest.FromString("referrerDigest") + tag := "tag-ref" + image := CreateRandomImage() + referrer := CreateRandomImageWith().Subject(image.DescriptorRef()).Build() - err = metaDB.SetReferrer("repo", referredDigest, mTypes.ReferrerInfo{ - Digest: referrerDigest.String(), - MediaType: ispec.MediaTypeImageManifest, - }) + err = metaDB.SetRepoReference("repo", tag, image.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetReferrer("repo", referredDigest, mTypes.ReferrerInfo{ - Digest: referrerDigest.String(), - MediaType: ispec.MediaTypeImageManifest, - }) + err = metaDB.SetRepoReference("repo", referrer.DigestStr(), referrer.AsImageMeta()) So(err, ShouldBeNil) - repoMeta, err = metaDB.GetRepoMeta("repo") + err = metaDB.SetRepoReference("repo", referrer.DigestStr(), referrer.AsImageMeta()) So(err, ShouldBeNil) - So(len(repoMeta.Referrers[referredDigest.String()]), ShouldEqual, 1) + + repoMeta, err := metaDB.GetRepoMeta(ctx, "repo") + So(err, ShouldBeNil) + So(len(repoMeta.Referrers[image.DigestStr()]), ShouldEqual, 1) }) Convey("GetReferrersInfo", func() { - referredDigest := godigest.FromString("referredDigest") + repo := "repo" + tag := "tag" - err := metaDB.SetReferrer("repo", referredDigest, mTypes.ReferrerInfo{ - Digest: "inexistendManifestDigest", - MediaType: ispec.MediaTypeImageManifest, - }) + image := CreateRandomImage() + err := metaDB.SetRepoReference(repo, tag, image.AsImageMeta()) So(err, ShouldBeNil) - // ------- Set existent manifest and artifact manifest - err = metaDB.SetManifestData("goodManifest", mTypes.ManifestData{ - ManifestBlob: []byte(`{"artifactType": "unwantedType"}`), - ConfigBlob: []byte("{}"), - }) + referrerWantedType := CreateRandomImageWith(). + ArtifactType("wanted-type"). + Subject(image.DescriptorRef()).Build() + + referrerNotWantedType := CreateRandomImageWith(). + ArtifactType("not-wanted-type"). + Subject(image.DescriptorRef()).Build() + + err = metaDB.SetRepoReference(repo, referrerWantedType.DigestStr(), referrerWantedType.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(repo, referrerNotWantedType.DigestStr(), referrerNotWantedType.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetReferrer("repo", referredDigest, mTypes.ReferrerInfo{ - Digest: "goodManifestUnwanted", - MediaType: ispec.MediaTypeImageManifest, - ArtifactType: "unwantedType", - }) - So(err, ShouldBeNil) - - err = metaDB.SetReferrer("repo", referredDigest, mTypes.ReferrerInfo{ - Digest: "goodManifest", - MediaType: ispec.MediaTypeImageManifest, - ArtifactType: "wantedType", - }) - So(err, ShouldBeNil) - - referrerInfo, err := metaDB.GetReferrersInfo("repo", referredDigest, []string{"wantedType"}) + referrerInfo, err := metaDB.GetReferrersInfo("repo", image.Digest(), []string{"wanted-type"}) So(err, ShouldBeNil) So(len(referrerInfo), ShouldEqual, 1) - So(referrerInfo[0].ArtifactType, ShouldResemble, "wantedType") - So(referrerInfo[0].Digest, ShouldResemble, "goodManifest") + So(referrerInfo[0].ArtifactType, ShouldResemble, "wanted-type") + So(referrerInfo[0].Digest, ShouldResemble, referrerWantedType.DigestStr()) + }) + + Convey("FilterImageMeta", func() { + repo := "repo" + tag := "tag" + + Convey("Just manifests", func() { + image := CreateRandomImage() + err := metaDB.SetRepoReference(repo, tag, image.AsImageMeta()) + So(err, ShouldBeNil) + + imageMeta, err := metaDB.FilterImageMeta(ctx, []string{image.DigestStr()}) + So(err, ShouldBeNil) + So(imageMeta, ShouldContainKey, image.DigestStr()) + + _, err = metaDB.FilterImageMeta(ctx, []string{image.DigestStr(), "bad-digest"}) + So(err, ShouldNotBeNil) + }) + + Convey("Index", func() { + multi := CreateRandomMultiarch() + digests := []string{} + + for i := range multi.Images { + err := metaDB.SetRepoReference(repo, multi.Images[i].DigestStr(), multi.Images[i].AsImageMeta()) + So(err, ShouldBeNil) + + digests = append(digests, multi.Images[i].DigestStr()) + } + + err := metaDB.SetRepoReference(repo, tag, multi.AsImageMeta()) + So(err, ShouldBeNil) + + imageMeta, err := metaDB.FilterImageMeta(ctx, []string{multi.DigestStr()}) + So(err, ShouldBeNil) + So(imageMeta, ShouldContainKey, multi.DigestStr()) + + actualDigests := tCommon.AccumulateField(imageMeta[multi.DigestStr()].Manifests, getManifestDigest) + So(tCommon.ContainSameElements(actualDigests, digests), ShouldBeTrue) + }) + }) + + Convey("ResetDB", func() { + repo := "repo-reset" + tag := "tag-reset" + + image := CreateRandomImage() + referrer := CreateRandomImageWith().Subject(image.DescriptorRef()).Build() + err := metaDB.SetRepoReference(repo, tag, image.AsImageMeta()) + So(err, ShouldBeNil) + err = metaDB.SetRepoReference(repo, tag, referrer.AsImageMeta()) + So(err, ShouldBeNil) + + repoMeta, err := metaDB.GetRepoMeta(ctx, repo) + So(err, ShouldBeNil) + So(repoMeta.Tags, ShouldNotBeEmpty) + So(repoMeta.Statistics, ShouldNotBeEmpty) + So(repoMeta.Referrers, ShouldNotBeEmpty) + + err = metaDB.ResetDB() + So(err, ShouldBeNil) + + _, err = metaDB.GetRepoMeta(ctx, repo) + So(err, ShouldNotBeNil) }) Convey("FilterRepos", func() { - img, err := deprecated.GetRandomImage() //nolint:staticcheck - So(err, ShouldBeNil) - imgDigest := img.Digest() + repo := "repoFilter" + tag1 := "tag1" + tag2 := "tag2" - manifestData, err := NewManifestData(img.Manifest, img.Config) + image := CreateImageWith().DefaultLayers().PlatformConfig("image-platform", "image-os").Build() + err := metaDB.SetRepoReference(repo, tag1, image.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetManifestData(imgDigest, manifestData) - So(err, ShouldBeNil) - - multiarch, err := deprecated.GetRandomMultiarchImage("multi") //nolint:staticcheck - So(err, ShouldBeNil) - multiarchDigest := multiarch.Digest() - - indexData, err := NewIndexData(multiarch.Index) - So(err, ShouldBeNil) - - err = metaDB.SetIndexData(multiarchDigest, indexData) - So(err, ShouldBeNil) + multiarch := CreateMultiarchWith(). + Images([]Image{ + CreateImageWith().DefaultLayers().PlatformConfig("sub-platform1", "sub-os1").Build(), + CreateImageWith().DefaultLayers().PlatformConfig("sub-platform2", "sub-os2").Build(), + }).Build() for _, img := range multiarch.Images { - digest := img.Digest() - - indManData1, err := NewManifestData(multiarch.Images[0].Manifest, multiarch.Images[0].Config) - So(err, ShouldBeNil) - - err = metaDB.SetManifestData(digest, indManData1) + err := metaDB.SetRepoReference(repo, img.DigestStr(), img.AsImageMeta()) So(err, ShouldBeNil) } - err = metaDB.SetRepoReference("repo", img.DigestStr(), imgDigest, img.Manifest.MediaType) + err = metaDB.SetRepoReference(repo, tag2, multiarch.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference("repo", multiarch.DigestStr(), multiarchDigest, ispec.MediaTypeImageIndex) + repoMetaList, err := metaDB.FilterRepos(context.Background(), mTypes.AcceptAllRepoNames, + mTypes.AcceptAllRepoMeta) So(err, ShouldBeNil) - - repoMetas, _, _, err := metaDB.FilterRepos(context.Background(), - func(repoMeta mTypes.RepoMetadata) bool { return true }) - So(err, ShouldBeNil) - So(len(repoMetas), ShouldEqual, 1) + So(len(repoMetaList), ShouldEqual, 1) + repoMeta := repoMetaList[0] + So(repoMeta.Platforms, ShouldContain, ispec.Platform{Architecture: "image-platform", OS: "image-os"}) + So(repoMeta.Platforms, ShouldContain, ispec.Platform{Architecture: "sub-platform1", OS: "sub-os1"}) + So(repoMeta.Platforms, ShouldContain, ispec.Platform{Architecture: "sub-platform2", OS: "sub-os2"}) }) Convey("Test bookmarked/starred field present in returned RepoMeta", func() { @@ -2486,42 +2221,34 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func ctx := userAc.DeriveContext(context.Background()) - manifestDigest := godigest.FromString("dig") - err := metaDB.SetManifestData(manifestDigest, mTypes.ManifestData{ - ManifestBlob: []byte("{}"), - ConfigBlob: []byte("{}"), - }) + image := CreateRandomImage() + + err := metaDB.SetRepoReference(repo99, "tag", image.AsImageMeta()) So(err, ShouldBeNil) - err = metaDB.SetRepoReference(repo99, "tag", manifestDigest, ispec.MediaTypeImageManifest) + repoMetaList, err := metaDB.SearchRepos(ctx, repo99) So(err, ShouldBeNil) + So(len(repoMetaList), ShouldEqual, 1) + So(repoMetaList[0].IsBookmarked, ShouldBeFalse) + So(repoMetaList[0].IsStarred, ShouldBeFalse) - repoMetas, _, _, err := metaDB.SearchRepos(ctx, repo99) + fullImageMetaList, err := metaDB.SearchTags(ctx, repo99+":") So(err, ShouldBeNil) - So(len(repoMetas), ShouldEqual, 1) - So(repoMetas[0].IsBookmarked, ShouldBeFalse) - So(repoMetas[0].IsStarred, ShouldBeFalse) + So(len(fullImageMetaList), ShouldEqual, 1) + So(fullImageMetaList[0].IsBookmarked, ShouldBeFalse) + So(fullImageMetaList[0].IsStarred, ShouldBeFalse) - repoMetas, _, _, err = metaDB.SearchTags(ctx, repo99+":") + repoMetaList, err = metaDB.FilterRepos(ctx, mTypes.AcceptAllRepoNames, mTypes.AcceptAllRepoMeta) So(err, ShouldBeNil) - So(len(repoMetas), ShouldEqual, 1) - So(repoMetas[0].IsBookmarked, ShouldBeFalse) - So(repoMetas[0].IsStarred, ShouldBeFalse) + So(len(repoMetaList), ShouldEqual, 1) + So(repoMetaList[0].IsBookmarked, ShouldBeFalse) + So(repoMetaList[0].IsStarred, ShouldBeFalse) - repoMetas, _, _, err = metaDB.FilterRepos(ctx, func(repoMeta mTypes.RepoMetadata) bool { - return true - }) + fullImageMetaList, err = metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta) So(err, ShouldBeNil) - So(len(repoMetas), ShouldEqual, 1) - So(repoMetas[0].IsBookmarked, ShouldBeFalse) - So(repoMetas[0].IsStarred, ShouldBeFalse) - - repoMetas, _, _, err = metaDB.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) - So(err, ShouldBeNil) - So(len(repoMetas), ShouldEqual, 1) - So(repoMetas[0].IsBookmarked, ShouldBeFalse) - So(repoMetas[0].IsStarred, ShouldBeFalse) + So(len(fullImageMetaList), ShouldEqual, 1) + So(fullImageMetaList[0].IsBookmarked, ShouldBeFalse) + So(fullImageMetaList[0].IsStarred, ShouldBeFalse) _, err = metaDB.ToggleBookmarkRepo(ctx, repo99) So(err, ShouldBeNil) @@ -2529,35 +2256,35 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func _, err = metaDB.ToggleStarRepo(ctx, repo99) So(err, ShouldBeNil) - repoMetas, _, _, err = metaDB.SearchRepos(ctx, repo99) + repoMetaList, err = metaDB.SearchRepos(ctx, repo99) So(err, ShouldBeNil) - So(len(repoMetas), ShouldEqual, 1) - So(repoMetas[0].IsBookmarked, ShouldBeTrue) - So(repoMetas[0].IsStarred, ShouldBeTrue) + So(len(repoMetaList), ShouldEqual, 1) + So(repoMetaList[0].IsBookmarked, ShouldBeTrue) + So(repoMetaList[0].IsStarred, ShouldBeTrue) - repoMetas, _, _, err = metaDB.SearchTags(ctx, repo99+":") + fullImageMetaList, err = metaDB.SearchTags(ctx, repo99+":") So(err, ShouldBeNil) - So(len(repoMetas), ShouldEqual, 1) - So(repoMetas[0].IsBookmarked, ShouldBeTrue) - So(repoMetas[0].IsStarred, ShouldBeTrue) + So(len(fullImageMetaList), ShouldEqual, 1) + So(fullImageMetaList[0].IsBookmarked, ShouldBeTrue) + So(fullImageMetaList[0].IsStarred, ShouldBeTrue) - repoMetas, _, _, err = metaDB.FilterRepos(ctx, func(repoMeta mTypes.RepoMetadata) bool { - return true - }) + repoMetaList, err = metaDB.FilterRepos(ctx, mTypes.AcceptAllRepoNames, mTypes.AcceptAllRepoMeta) So(err, ShouldBeNil) - So(len(repoMetas), ShouldEqual, 1) - So(repoMetas[0].IsBookmarked, ShouldBeTrue) - So(repoMetas[0].IsStarred, ShouldBeTrue) + So(len(repoMetaList), ShouldEqual, 1) + So(repoMetaList[0].IsBookmarked, ShouldBeTrue) + So(repoMetaList[0].IsStarred, ShouldBeTrue) - repoMetas, _, _, err = metaDB.FilterTags(ctx, - func(repoMeta mTypes.RepoMetadata, manifestMeta mTypes.ManifestMetadata) bool { return true }) + fullImageMetaList, err = metaDB.FilterTags(ctx, mTypes.AcceptAllRepoTag, mTypes.AcceptAllImageMeta) So(err, ShouldBeNil) - So(len(repoMetas), ShouldEqual, 1) - So(repoMetas[0].IsBookmarked, ShouldBeTrue) - So(repoMetas[0].IsStarred, ShouldBeTrue) + So(len(fullImageMetaList), ShouldEqual, 1) + So(fullImageMetaList[0].IsBookmarked, ShouldBeTrue) + So(fullImageMetaList[0].IsStarred, ShouldBeTrue) }) Convey("Test GetUserRepoMeta", func() { + err := metaDB.ResetDB() + So(err, ShouldBeNil) + userAc := reqCtx.NewUserAccessControl() userAc.SetUsername("user1") userAc.SetGlobPatterns("read", map[string]bool{ @@ -2566,9 +2293,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func ctx := userAc.DeriveContext(context.Background()) - digest := godigest.FromString("1") - - err := metaDB.SetRepoReference("repo", "tag", digest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference("repo", "tag", CreateDefaultImage().AsImageMeta()) So(err, ShouldBeNil) _, err = metaDB.ToggleBookmarkRepo(ctx, "repo") @@ -2577,7 +2302,7 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func _, err = metaDB.ToggleStarRepo(ctx, "repo") So(err, ShouldBeNil) - repoMeta, err := metaDB.GetUserRepoMeta(ctx, "repo") + repoMeta, err := metaDB.GetRepoMeta(ctx, "repo") So(err, ShouldBeNil) So(repoMeta.IsBookmarked, ShouldBeTrue) So(repoMeta.IsStarred, ShouldBeTrue) @@ -2586,28 +2311,6 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func }) } -func NewManifestData(manifest ispec.Manifest, config ispec.Image) (mTypes.ManifestData, error) { - configBlob, err := json.Marshal(config) - if err != nil { - return mTypes.ManifestData{}, err - } - - manifest.Config.Digest = godigest.FromBytes(configBlob) - - manifestBlob, err := json.Marshal(manifest) - if err != nil { - return mTypes.ManifestData{}, err - } - - return mTypes.ManifestData{ManifestBlob: manifestBlob, ConfigBlob: configBlob}, nil -} - -func NewIndexData(index ispec.Index) (mTypes.IndexData, error) { - indexBlob, err := json.Marshal(index) - - return mTypes.IndexData{IndexBlob: indexBlob}, err -} - func TestRelevanceSorting(t *testing.T) { Convey("Test Relevance Sorting", t, func() { So(common.RankRepoName("alpine", "alpine"), ShouldEqual, 0) @@ -2629,78 +2332,19 @@ func TestRelevanceSorting(t *testing.T) { }) } -func generateTestImage() ([]byte, []byte, error) { - config := ispec.Image{ - Platform: ispec.Platform{ - Architecture: "amd64", - OS: LINUX, - }, - RootFS: ispec.RootFS{ - Type: "layers", - DiffIDs: []godigest.Digest{}, - }, - Author: "ZotUser", - } - - configBlob, err := json.Marshal(config) - if err != nil { - return []byte{}, []byte{}, err - } - - configDigest := godigest.FromBytes(configBlob) - - layers := [][]byte{ - make([]byte, 100), - } - - // init layers with random values - for i := range layers { - //nolint:gosec - _, err := rand.Read(layers[i]) //nolint:staticcheck - if err != nil { - return []byte{}, []byte{}, err - } - } - - manifest := ispec.Manifest{ - Versioned: specs.Versioned{ - SchemaVersion: 2, - }, - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: configDigest, - Size: int64(len(configBlob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: godigest.FromBytes(layers[0]), - Size: int64(len(layers[0])), - }, - }, - } - - manifestBlob, err := json.Marshal(manifest) - if err != nil { - return []byte{}, []byte{}, err - } - - return configBlob, manifestBlob, nil -} - func TestCreateDynamo(t *testing.T) { tskip.SkipDynamo(t) Convey("Create", t, func() { dynamoDBDriverParams := mdynamodb.DBDriverParameters{ - Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"), - RepoMetaTablename: "RepoMetadataTable", - ManifestDataTablename: "ManifestDataTable", - IndexDataTablename: "IndexDataTable", - UserDataTablename: "UserDataTable", - APIKeyTablename: "ApiKeyTable", - VersionTablename: "Version", - Region: "us-east-2", + Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"), + RepoMetaTablename: "RepoMetadataTable", + RepoBlobsInfoTablename: "RepoBlobs", + ImageMetaTablename: "ImageMeta", + UserDataTablename: "UserDataTable", + APIKeyTablename: "ApiKeyTable", + VersionTablename: "Version", + Region: "us-east-2", } client, err := mdynamodb.GetDynamoClient(dynamoDBDriverParams) diff --git a/pkg/meta/parse.go b/pkg/meta/parse.go index 6613d29b..d3797a27 100644 --- a/pkg/meta/parse.go +++ b/pkg/meta/parse.go @@ -3,7 +3,6 @@ package meta import ( "encoding/json" "errors" - "fmt" "time" godigest "github.com/opencontainers/go-digest" @@ -12,11 +11,17 @@ import ( zerr "zotregistry.io/zot/errors" zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/meta/convert" mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/storage" storageTypes "zotregistry.io/zot/pkg/storage/types" ) +const ( + CosignType = "cosign" + NotationType = "notation" +) + // ParseStorage will sync all repos found in the rootdirectory of the oci layout that zot was deployed on with the // ParseStorage database. func ParseStorage(metaDB mTypes.MetaDB, storeController storage.StoreController, log log.Logger) error { @@ -31,7 +36,10 @@ func ParseStorage(metaDB mTypes.MetaDB, storeController storage.StoreController, return err } - for _, repo := range allRepos { + for i, repo := range allRepos { + log.Info().Int("total", len(allRepos)).Int("progress", i).Str("current-repo", repo). + Msgf("parsing next repo '%s'", repo) + err := ParseRepo(repo, metaDB, storeController, log) if err != nil { log.Error().Err(err).Str("repository", repo).Msg("load-local-layout: failed to sync repo") @@ -70,75 +78,35 @@ func ParseRepo(repo string, metaDB mTypes.MetaDB, storeController storage.StoreC return err } - err = resetRepoMeta(repo, metaDB, log) + err = metaDB.ResetRepoReferences(repo) if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { log.Error().Err(err).Str("repository", repo).Msg("load-repo: failed to reset tag field in RepoMetadata for repo") return err } - for _, descriptor := range indexContent.Manifests { - tag := descriptor.Annotations[ispec.AnnotationRefName] + for _, manifest := range indexContent.Manifests { + tag := manifest.Annotations[ispec.AnnotationRefName] if zcommon.IsReferrersTag(tag) { continue } - descriptorBlob, err := getCachedBlob(repo, descriptor, metaDB, imageStore, log) + manifestBlob, _, _, err := imageStore.GetImageManifest(repo, manifest.Digest.String()) if err != nil { - log.Error().Err(err).Msg("load-repo: error checking manifestMeta in MetaDB") + log.Error().Err(err).Str("repository", repo).Str("digest", manifest.Digest.String()). + Msg("load-repo: failed to get blob for image") return err } - isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(repo, - descriptorBlob, tag) - if err != nil { - log.Error().Err(err).Str("repository", repo).Str("tag", tag). - Msg("load-repo: failed checking if image is signature for specified image") - - return err - } - - if isSignature { - layers, err := GetSignatureLayersInfo(repo, tag, descriptor.Digest.String(), signatureType, - descriptorBlob, imageStore, log) - if err != nil { - return err - } - - err = metaDB.AddManifestSignature(repo, signedManifestDigest, - mTypes.SignatureMetadata{ - SignatureType: signatureType, - SignatureDigest: descriptor.Digest.String(), - LayersInfo: layers, - }) - if err != nil { - log.Error().Err(err).Str("repository", repo).Str("tag", tag). - Str("manifestDigest", signedManifestDigest.String()). - Msg("load-repo: failed set signature meta for signed image") - - return err - } - - err = metaDB.UpdateSignaturesValidity(repo, signedManifestDigest) - if err != nil { - log.Error().Err(err).Str("repository", repo).Str("reference", tag).Str("digest", signedManifestDigest.String()).Msg( - "load-repo: failed verify signatures validity for signed image") - - return err - } - - continue - } - reference := tag if tag == "" { - reference = descriptor.Digest.String() + reference = manifest.Digest.String() } - err = SetImageMetaFromInput(repo, reference, descriptor.MediaType, descriptor.Digest, descriptorBlob, + err = SetImageMetaFromInput(repo, reference, manifest.MediaType, manifest.Digest, manifestBlob, imageStore, metaDB, log) if err != nil { log.Error().Err(err).Str("repository", repo).Str("tag", tag). @@ -151,32 +119,6 @@ func ParseRepo(repo string, metaDB mTypes.MetaDB, storeController storage.StoreC return nil } -// resetRepoMeta will delete all tags and non-user related information from a RepoMetadata. -// It is used to recalculate and keep MetaDB consistent with the layout in case of unexpected changes. -func resetRepoMeta(repo string, metaDB mTypes.MetaDB, log log.Logger) error { - repoMeta, err := metaDB.GetRepoMeta(repo) - if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { - log.Error().Err(err).Str("repository", repo).Msg("load-repo: failed to get RepoMeta for repo") - - return err - } - - if errors.Is(err, zerr.ErrRepoMetaNotFound) { - log.Info().Str("repository", repo).Msg("load-repo: RepoMeta not found for repo, new RepoMeta will be created") - - return nil - } - - return metaDB.SetRepoMeta(repo, mTypes.RepoMetadata{ - Name: repoMeta.Name, - Tags: map[string]mTypes.Descriptor{}, - Statistics: repoMeta.Statistics, - Signatures: map[string]mTypes.ManifestSignatures{}, - Referrers: map[string][]mTypes.ReferrerInfo{}, - Stars: repoMeta.Stars, - }) -} - func getAllRepos(storeController storage.StoreController) ([]string, error) { allRepos, err := storeController.DefaultStore.GetRepositories() if err != nil { @@ -197,43 +139,6 @@ func getAllRepos(storeController storage.StoreController) ([]string, error) { return allRepos, nil } -func getCachedBlob(repo string, descriptor ispec.Descriptor, metaDB mTypes.MetaDB, - imageStore storageTypes.ImageStore, log log.Logger, -) ([]byte, error) { - digest := descriptor.Digest - - descriptorBlob, err := getCachedBlobFromMetaDB(descriptor, metaDB) - - if err != nil || len(descriptorBlob) == 0 { - descriptorBlob, _, _, err = imageStore.GetImageManifest(repo, digest.String()) - if err != nil { - log.Error().Err(err).Str("repository", repo).Str("digest", digest.String()). - Msg("load-repo: failed to get blob for image") - - return nil, err - } - - return descriptorBlob, nil - } - - return descriptorBlob, nil -} - -func getCachedBlobFromMetaDB(descriptor ispec.Descriptor, metaDB mTypes.MetaDB) ([]byte, error) { - switch descriptor.MediaType { - case ispec.MediaTypeImageManifest: - manifestData, err := metaDB.GetManifestData(descriptor.Digest) - - return manifestData.ManifestBlob, err - case ispec.MediaTypeImageIndex: - indexData, err := metaDB.GetIndexData(descriptor.Digest) - - return indexData.IndexBlob, err - } - - return nil, nil -} - func GetSignatureLayersInfo(repo, tag, manifestDigest, signatureType string, manifestBlob []byte, imageStore storageTypes.ImageStore, log log.Logger, ) ([]mTypes.LayerInfo, error) { @@ -341,92 +246,82 @@ func getNotationSignatureLayersInfo( return layers, nil } -// NewManifestMeta takes raw data about an image and createa a new ManifestMetadate object. -func NewManifestData(repoName string, manifestBlob []byte, imageStore storageTypes.ImageStore, -) (mTypes.ManifestData, error) { - var ( - manifestContent ispec.Manifest - configContent ispec.Image - manifestData mTypes.ManifestData - ) - - err := json.Unmarshal(manifestBlob, &manifestContent) - if err != nil { - return mTypes.ManifestData{}, err - } - - var lockLatency time.Time - - imageStore.RLock(&lockLatency) - defer imageStore.RUnlock(&lockLatency) - - configBlob, err := imageStore.GetBlobContent(repoName, manifestContent.Config.Digest) - if err != nil { - return mTypes.ManifestData{}, err - } - - if manifestContent.Config.MediaType == ispec.MediaTypeImageConfig { - err = json.Unmarshal(configBlob, &configContent) - if err != nil { - return mTypes.ManifestData{}, err - } - } - - manifestData.ManifestBlob = manifestBlob - manifestData.ConfigBlob = configBlob - - return manifestData, nil -} - -func NewIndexData(repoName string, indexBlob []byte, imageStore storageTypes.ImageStore, -) mTypes.IndexData { - indexData := mTypes.IndexData{} - - indexData.IndexBlob = indexBlob - - return indexData -} - // SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag // (in case the reference is a tag). The function expects image manifests and indexes (multi arch images). -func SetImageMetaFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte, +func SetImageMetaFromInput(repo, reference, mediaType string, digest godigest.Digest, blob []byte, imageStore storageTypes.ImageStore, metaDB mTypes.MetaDB, log log.Logger, ) error { + var imageMeta mTypes.ImageMeta + switch mediaType { case ispec.MediaTypeImageManifest: - imageData, err := NewManifestData(repo, descriptorBlob, imageStore) + manifestContent := ispec.Manifest{} + configContent := ispec.Image{} + + err := json.Unmarshal(blob, &manifestContent) if err != nil { return err } - err = metaDB.SetManifestData(digest, imageData) - if err != nil { - log.Error().Err(err).Msg("metadb: error while putting manifest meta") + if manifestContent.Config.MediaType == ispec.MediaTypeImageConfig { + configBlob, err := imageStore.GetBlobContent(repo, manifestContent.Config.Digest) + if err != nil { + return err + } - return err + err = json.Unmarshal(configBlob, &configContent) + if err != nil { + return err + } } + + if isSig, sigType, signedManifestDigest := isSignature(reference, manifestContent); isSig { + layers, err := GetSignatureLayersInfo(repo, reference, digest.String(), sigType, + blob, imageStore, log) + if err != nil { + return err + } + + err = metaDB.AddManifestSignature(repo, signedManifestDigest, + mTypes.SignatureMetadata{ + SignatureType: sigType, + SignatureDigest: digest.String(), + LayersInfo: layers, + }) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("tag", reference). + Str("manifestDigest", signedManifestDigest.String()). + Msg("load-repo: failed set signature meta for signed image") + + return err + } + + err = metaDB.UpdateSignaturesValidity(repo, signedManifestDigest) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("reference", reference).Str("digest", + signedManifestDigest.String()).Msg("load-repo: failed verify signatures validity for signed image") + + return err + } + + return nil + } + + imageMeta = convert.GetImageManifestMeta(manifestContent, configContent, int64(len(blob)), digest) case ispec.MediaTypeImageIndex: - indexData := NewIndexData(repo, descriptorBlob, imageStore) + indexContent := ispec.Index{} - err := metaDB.SetIndexData(digest, indexData) + err := json.Unmarshal(blob, &indexContent) if err != nil { - log.Error().Err(err).Msg("metadb: error while putting index data") - return err } + + imageMeta = convert.GetImageIndexMeta(indexContent, int64(len(blob)), digest) + default: + return nil } - referredDigest, referrerInfo, hasSubject, err := GetReferredInfo(descriptorBlob, digest.String(), mediaType) - if hasSubject && err == nil { - err := metaDB.SetReferrer(repo, referredDigest, referrerInfo) - if err != nil { - log.Error().Err(err).Msg("metadb: error while settingg referrer") - - return err - } - } - - err = metaDB.SetRepoReference(repo, reference, digest, mediaType) + err := metaDB.SetRepoReference(repo, reference, imageMeta) if err != nil { log.Error().Err(err).Msg("metadb: error while putting repo meta") @@ -436,55 +331,24 @@ func SetImageMetaFromInput(repo, reference, mediaType string, digest godigest.Di return nil } -func GetReferredInfo(descriptorBlob []byte, referrerDigest, mediaType string, -) (godigest.Digest, mTypes.ReferrerInfo, bool, error) { - var ( - referrerInfo mTypes.ReferrerInfo - referrerSubject *ispec.Descriptor - ) +func isSignature(reference string, manifestContent ispec.Manifest) (bool, string, godigest.Digest) { + manifestArtifactType := zcommon.GetManifestArtifactType(manifestContent) - switch mediaType { - case ispec.MediaTypeImageManifest: - var manifestContent ispec.Manifest - - err := json.Unmarshal(descriptorBlob, &manifestContent) - if err != nil { - return "", referrerInfo, false, - fmt.Errorf("metadb: can't unmarshal manifest for digest %s: %w", referrerDigest, err) - } - - referrerSubject = manifestContent.Subject - - referrerInfo = mTypes.ReferrerInfo{ - Digest: referrerDigest, - MediaType: mediaType, - ArtifactType: zcommon.GetManifestArtifactType(manifestContent), - Size: len(descriptorBlob), - Annotations: manifestContent.Annotations, - } - case ispec.MediaTypeImageIndex: - var indexContent ispec.Index - - err := json.Unmarshal(descriptorBlob, &indexContent) - if err != nil { - return "", referrerInfo, false, - fmt.Errorf("metadb: can't unmarshal manifest for digest %s: %w", referrerDigest, err) - } - - referrerSubject = indexContent.Subject - - referrerInfo = mTypes.ReferrerInfo{ - Digest: referrerDigest, - MediaType: mediaType, - ArtifactType: zcommon.GetIndexArtifactType(indexContent), - Size: len(descriptorBlob), - Annotations: indexContent.Annotations, - } + // check notation signature + if manifestArtifactType == zcommon.ArtifactTypeNotation && manifestContent.Subject != nil { + return true, NotationType, manifestContent.Subject.Digest } - if referrerSubject == nil || referrerSubject.Digest.String() == "" { - return "", mTypes.ReferrerInfo{}, false, nil + if tag := reference; zcommon.IsCosignTag(reference) { + prefixLen := len("sha256-") + digestLen := 64 + signedImageManifestDigestEncoded := tag[prefixLen : prefixLen+digestLen] + + signedImageManifestDigest := godigest.NewDigestFromEncoded(godigest.SHA256, + signedImageManifestDigestEncoded) + + return true, CosignType, signedImageManifestDigest } - return referrerSubject.Digest, referrerInfo, true, nil + return false, "", "" } diff --git a/pkg/meta/parse_test.go b/pkg/meta/parse_test.go index 103d847e..79de709a 100644 --- a/pkg/meta/parse_test.go +++ b/pkg/meta/parse_test.go @@ -13,7 +13,6 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" . "github.com/smartystreets/goconvey/convey" - zerr "zotregistry.io/zot/errors" zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/monitoring" "zotregistry.io/zot/pkg/log" @@ -97,50 +96,6 @@ func TestParseStorageErrors(t *testing.T) { So(err, ShouldNotBeNil) }) - Convey("resetRepoMetaTags errors", func() { - imageStore.GetIndexContentFn = func(repo string) ([]byte, error) { - return []byte("{}"), nil - } - - Convey("metaDB.GetRepoMeta errors", func() { - metaDB.GetRepoMetaFn = func(repo string) (mTypes.RepoMetadata, error) { - return mTypes.RepoMetadata{}, ErrTestError - } - - err := meta.ParseRepo("repo", metaDB, storeController, log) - So(err, ShouldNotBeNil) - }) - }) - - Convey("isManifestMetaPresent errors", func() { - indexContent := ispec.Index{ - Manifests: []ispec.Descriptor{ - { - Digest: godigest.FromString("manifest1"), - MediaType: ispec.MediaTypeImageManifest, - Annotations: map[string]string{ - ispec.AnnotationRefName: "tag1", - }, - }, - }, - } - indexBlob, err := json.Marshal(indexContent) - So(err, ShouldBeNil) - - imageStore.GetIndexContentFn = func(repo string) ([]byte, error) { - return indexBlob, nil - } - - Convey("metaDB.GetManifestMeta errors", func() { - metaDB.GetManifestMetaFn = func(repo string, manifestDigest godigest.Digest) (mTypes.ManifestMetadata, error) { - return mTypes.ManifestMetadata{}, ErrTestError - } - - err = meta.ParseRepo("repo", metaDB, storeController, log) - So(err, ShouldNotBeNil) - }) - }) - Convey("manifestMetaIsPresent true", func() { indexContent := ispec.Index{ Manifests: []ispec.Descriptor{ @@ -161,7 +116,7 @@ func TestParseStorageErrors(t *testing.T) { } Convey("metaDB.SetRepoReference", func() { - metaDB.SetRepoReferenceFn = func(repo, tag string, manifestDigest godigest.Digest, mediaType string) error { + metaDB.SetRepoReferenceFn = func(repo, reference string, imageMeta mTypes.ImageMeta) error { return ErrTestError } @@ -169,212 +124,6 @@ func TestParseStorageErrors(t *testing.T) { So(err, ShouldNotBeNil) }) }) - - Convey("manifestMetaIsPresent false", func() { - indexContent := ispec.Index{ - Manifests: []ispec.Descriptor{ - { - Digest: godigest.FromString("manifest1"), - MediaType: ispec.MediaTypeImageManifest, - Annotations: map[string]string{ - ispec.AnnotationRefName: "tag1", - }, - }, - }, - } - indexBlob, err := json.Marshal(indexContent) - So(err, ShouldBeNil) - - imageStore.GetIndexContentFn = func(repo string) ([]byte, error) { - return indexBlob, nil - } - - metaDB.GetManifestMetaFn = func(repo string, manifestDigest godigest.Digest) (mTypes.ManifestMetadata, error) { - return mTypes.ManifestMetadata{}, zerr.ErrManifestMetaNotFound - } - - Convey("GetImageManifest errors", func() { - imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return nil, "", "", ErrTestError - } - err = meta.ParseRepo("repo", metaDB, storeController, log) - So(err, ShouldNotBeNil) - }) - - Convey("CheckIsImageSignature errors", func() { - // CheckIsImageSignature will fail because of a invalid json - imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return []byte("Invalid JSON"), "", "", nil - } - err = meta.ParseRepo("repo", metaDB, storeController, log) - So(err, ShouldNotBeNil) - }) - Convey("CheckIsImageSignature -> not signature", func() { - manifestContent := ispec.Manifest{} - manifestBlob, err := json.Marshal(manifestContent) - So(err, ShouldBeNil) - - imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return manifestBlob, "", "", nil - } - - Convey("imgStore.GetBlobContent errors", func() { - imageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) { - return nil, ErrTestError - } - - err = meta.ParseRepo("repo", metaDB, storeController, log) - So(err, ShouldNotBeNil) - }) - }) - - Convey("CheckIsImageSignature -> is signature", func() { - manifestContent := ispec.Manifest{ - Subject: &ispec.Descriptor{ - Digest: "123", - }, - ArtifactType: "application/vnd.cncf.notary.signature", - Layers: []ispec.Descriptor{{MediaType: ispec.MediaTypeImageLayer}}, - } - - manifestBlob, err := json.Marshal(manifestContent) - So(err, ShouldBeNil) - - imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return manifestBlob, "", "", nil - } - - metaDB.AddManifestSignatureFn = func(repo string, signedManifestDigest godigest.Digest, - sm mTypes.SignatureMetadata, - ) error { - return ErrTestError - } - - err = meta.ParseRepo("repo", metaDB, storeController, log) - So(err, ShouldNotBeNil) - - metaDB.AddManifestSignatureFn = func(repo string, signedManifestDigest godigest.Digest, - sm mTypes.SignatureMetadata, - ) error { - return nil - } - - metaDB.UpdateSignaturesValidityFn = func(repo string, signedManifestDigest godigest.Digest, - ) error { - return ErrTestError - } - - err = meta.ParseRepo("repo", metaDB, storeController, log) - So(err, ShouldNotBeNil) - }) - - Convey("GetSignatureLayersInfo errors", func() { - // get notation signature layers info - badNotationManifestContent := ispec.Manifest{ - Subject: &ispec.Descriptor{ - Digest: "123", - }, - ArtifactType: "application/vnd.cncf.notary.signature", - } - - badNotationManifestBlob, err := json.Marshal(badNotationManifestContent) - So(err, ShouldBeNil) - - imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return badNotationManifestBlob, "", "", nil - } - - // wrong number of layers of notation manifest - err = meta.ParseRepo("repo", metaDB, storeController, log) - So(err, ShouldNotBeNil) - - notationManifestContent := ispec.Manifest{ - Subject: &ispec.Descriptor{ - Digest: "123", - }, - ArtifactType: "application/vnd.cncf.notary.signature", - Layers: []ispec.Descriptor{{MediaType: ispec.MediaTypeImageLayer}}, - } - - notationManifestBlob, err := json.Marshal(notationManifestContent) - So(err, ShouldBeNil) - - imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return notationManifestBlob, "", "", nil - } - - imageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) { - return []byte{}, ErrTestError - } - - // unable to get layer content - err = meta.ParseRepo("repo", metaDB, storeController, log) - So(err, ShouldNotBeNil) - - _, _, cosignManifestContent, _ := deprecated.GetRandomImageComponents(10) //nolint:staticcheck - _, _, signedManifest, _ := deprecated.GetRandomImageComponents(10) //nolint:staticcheck - signatureTag, err := signature.GetCosignSignatureTagForManifest(signedManifest) - So(err, ShouldBeNil) - - cosignManifestContent.Annotations = map[string]string{ispec.AnnotationRefName: signatureTag} - - cosignManifestBlob, err := json.Marshal(cosignManifestContent) - So(err, ShouldBeNil) - - imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) { - return cosignManifestBlob, "", "", nil - } - - indexContent := ispec.Index{ - Manifests: []ispec.Descriptor{ - { - Digest: godigest.FromString("cosignSig"), - MediaType: ispec.MediaTypeImageManifest, - Annotations: map[string]string{ - ispec.AnnotationRefName: signatureTag, - }, - }, - }, - } - indexBlob, err := json.Marshal(indexContent) - So(err, ShouldBeNil) - - imageStore.GetIndexContentFn = func(repo string) ([]byte, error) { - return indexBlob, nil - } - - // unable to get layer content - err = meta.ParseRepo("repo", metaDB, storeController, log) - So(err, ShouldNotBeNil) - }) - - Convey("IsReferrersTag -> true", func() { - indexContent := ispec.Index{ - Manifests: []ispec.Descriptor{ - { - Digest: godigest.FromString("indx1"), - MediaType: ispec.MediaTypeImageIndex, - Annotations: map[string]string{ - ispec.AnnotationRefName: "sha256-123", - }, - }, - }, - } - indexBlob, err := json.Marshal(indexContent) - So(err, ShouldBeNil) - - imageStore.GetIndexContentFn = func(repo string) ([]byte, error) { - return indexBlob, nil - } - - metaDB.SetIndexDataFn = func(digest godigest.Digest, indexData mTypes.IndexData) error { - return ErrTestError - } - - err = meta.ParseRepo("repo", metaDB, storeController, log) - So(err, ShouldBeNil) - }) - }) }) } @@ -401,14 +150,14 @@ func TestParseStorageDynamoWrapper(t *testing.T) { rootDir := t.TempDir() params := dynamodb.DBDriverParameters{ - Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"), - Region: "us-east-2", - RepoMetaTablename: "RepoMetadataTable", - ManifestDataTablename: "ManifestDataTable", - IndexDataTablename: "IndexDataTable", - UserDataTablename: "UserDataTable", - APIKeyTablename: "ApiKeyTable", - VersionTablename: "Version", + Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"), + Region: "us-east-2", + RepoMetaTablename: "RepoMetadataTable", + RepoBlobsInfoTablename: "RepoBlobsInfoTablename", + ImageMetaTablename: "ImageMetaTablename", + UserDataTablename: "UserDataTable", + APIKeyTablename: "ApiKeyTable", + VersionTablename: "Version", } dynamoClient, err := dynamodb.GetDynamoClient(params) @@ -417,10 +166,13 @@ func TestParseStorageDynamoWrapper(t *testing.T) { dynamoWrapper, err := dynamodb.New(dynamoClient, params, log.NewLogger("debug", "")) So(err, ShouldBeNil) - err = dynamoWrapper.ResetManifestDataTable() + err = dynamoWrapper.ResetTable(dynamoWrapper.RepoMetaTablename) So(err, ShouldBeNil) - err = dynamoWrapper.ResetRepoMetaTable() + err = dynamoWrapper.ResetTable(dynamoWrapper.RepoBlobsTablename) + So(err, ShouldBeNil) + + err = dynamoWrapper.ResetTable(dynamoWrapper.ImageMetaTablename) So(err, ShouldBeNil) RunParseStorageTests(rootDir, dynamoWrapper) @@ -496,20 +248,20 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { So(err, ShouldBeNil) repos, err := metaDB.GetMultipleRepoMeta(context.Background(), - func(repoMeta mTypes.RepoMetadata) bool { return true }) + func(repoMeta mTypes.RepoMeta) bool { return true }) So(err, ShouldBeNil) So(len(repos), ShouldEqual, 1) So(len(repos[0].Tags), ShouldEqual, 2) - for _, descriptor := range repos[0].Tags { - manifestMeta, err := metaDB.GetManifestMeta(repo, godigest.Digest(descriptor.Digest)) + ctx := context.Background() + + for tag, descriptor := range repos[0].Tags { + imageManifestData, err := metaDB.GetFullImageMeta(ctx, repo, tag) So(err, ShouldBeNil) - So(manifestMeta.ManifestBlob, ShouldNotBeNil) - So(manifestMeta.ConfigBlob, ShouldNotBeNil) if descriptor.Digest == signedManifestDigest.String() { - So(manifestMeta.Signatures, ShouldNotBeEmpty) + So(imageManifestData.Signatures, ShouldNotBeEmpty) } } }) @@ -555,10 +307,8 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", "")) So(err, ShouldBeNil) - repos, err := metaDB.GetMultipleRepoMeta( - context.Background(), - func(repoMeta mTypes.RepoMetadata) bool { return true }, - ) + repos, err := metaDB.GetMultipleRepoMeta(context.Background(), + func(repoMeta mTypes.RepoMeta) bool { return true }) So(err, ShouldBeNil) for _, desc := range repos[0].Tags { @@ -577,15 +327,12 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { storeController := storage.StoreController{DefaultStore: imageStore} // add an image - image, err := deprecated.GetRandomImage() //nolint:staticcheck + image := CreateRandomImage() //nolint:staticcheck + + err := WriteImageToFileSystem(image, repo, "tag", storeController) So(err, ShouldBeNil) - manifestDigest := image.Digest() - - err = WriteImageToFileSystem(image, repo, "tag", storeController) - So(err, ShouldBeNil) - - err = metaDB.SetRepoReference(repo, "tag", manifestDigest, ispec.MediaTypeImageManifest) + err = metaDB.SetRepoReference(repo, "tag", image.AsImageMeta()) So(err, ShouldBeNil) err = metaDB.IncrementRepoStars(repo) @@ -597,30 +344,20 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { err = metaDB.IncrementImageDownloads(repo, "tag") So(err, ShouldBeNil) - repoMeta, err := metaDB.GetRepoMeta(repo) + repoMeta, err := metaDB.GetRepoMeta(context.Background(), repo) So(err, ShouldBeNil) - So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 3) - So(repoMeta.Stars, ShouldEqual, 1) + So(repoMeta.Statistics[image.DigestStr()].DownloadCount, ShouldEqual, 3) + So(repoMeta.StarCount, ShouldEqual, 1) err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", "")) So(err, ShouldBeNil) - repoMeta, err = metaDB.GetRepoMeta(repo) + repoMeta, err = metaDB.GetRepoMeta(context.Background(), repo) So(err, ShouldBeNil) - So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 3) - So(repoMeta.Stars, ShouldEqual, 1) - }) -} - -func TestGetReferredInfo(t *testing.T) { - Convey("GetReferredInfo error", t, func() { - _, _, _, err := meta.GetReferredInfo([]byte("bad json"), "digest", ispec.MediaTypeImageManifest) - So(err, ShouldNotBeNil) - - _, _, _, err = meta.GetReferredInfo([]byte("bad json"), "digest", ispec.MediaTypeImageIndex) - So(err, ShouldNotBeNil) + So(repoMeta.Statistics[image.DigestStr()].DownloadCount, ShouldEqual, 3) + So(repoMeta.StarCount, ShouldEqual, 1) }) } diff --git a/pkg/meta/proto/gen/config.pb.go b/pkg/meta/proto/gen/config.pb.go new file mode 100644 index 00000000..60cafd21 --- /dev/null +++ b/pkg/meta/proto/gen/config.pb.go @@ -0,0 +1,628 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.24.4 +// source: oci/config.proto + +package gen + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Image struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Created *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=Created,proto3,oneof" json:"Created,omitempty"` + Author *string `protobuf:"bytes,2,opt,name=Author,proto3,oneof" json:"Author,omitempty"` + Platform *Platform `protobuf:"bytes,3,opt,name=Platform,proto3" json:"Platform,omitempty"` + Config *ImageConfig `protobuf:"bytes,4,opt,name=Config,proto3,oneof" json:"Config,omitempty"` + RootFS *RootFS `protobuf:"bytes,5,opt,name=RootFS,proto3,oneof" json:"RootFS,omitempty"` + History []*History `protobuf:"bytes,6,rep,name=History,proto3" json:"History,omitempty"` +} + +func (x *Image) Reset() { + *x = Image{} + if protoimpl.UnsafeEnabled { + mi := &file_oci_config_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Image) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Image) ProtoMessage() {} + +func (x *Image) ProtoReflect() protoreflect.Message { + mi := &file_oci_config_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Image.ProtoReflect.Descriptor instead. +func (*Image) Descriptor() ([]byte, []int) { + return file_oci_config_proto_rawDescGZIP(), []int{0} +} + +func (x *Image) GetCreated() *timestamppb.Timestamp { + if x != nil { + return x.Created + } + return nil +} + +func (x *Image) GetAuthor() string { + if x != nil && x.Author != nil { + return *x.Author + } + return "" +} + +func (x *Image) GetPlatform() *Platform { + if x != nil { + return x.Platform + } + return nil +} + +func (x *Image) GetConfig() *ImageConfig { + if x != nil { + return x.Config + } + return nil +} + +func (x *Image) GetRootFS() *RootFS { + if x != nil { + return x.RootFS + } + return nil +} + +func (x *Image) GetHistory() []*History { + if x != nil { + return x.History + } + return nil +} + +type ImageConfig struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + ExposedPorts map[string]*EmptyMessage `protobuf:"bytes,1,rep,name=ExposedPorts,proto3" json:"ExposedPorts,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Volumes map[string]*EmptyMessage `protobuf:"bytes,2,rep,name=Volumes,proto3" json:"Volumes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Labels map[string]string `protobuf:"bytes,3,rep,name=Labels,proto3" json:"Labels,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + User string `protobuf:"bytes,4,opt,name=User,proto3" json:"User,omitempty"` + Env []string `protobuf:"bytes,5,rep,name=Env,proto3" json:"Env,omitempty"` + Entrypoint []string `protobuf:"bytes,6,rep,name=Entrypoint,proto3" json:"Entrypoint,omitempty"` + Cmd []string `protobuf:"bytes,7,rep,name=Cmd,proto3" json:"Cmd,omitempty"` + WorkingDir *string `protobuf:"bytes,8,opt,name=WorkingDir,proto3,oneof" json:"WorkingDir,omitempty"` + StopSignal *string `protobuf:"bytes,9,opt,name=StopSignal,proto3,oneof" json:"StopSignal,omitempty"` + ArgsEscaped bool `protobuf:"varint,10,opt,name=ArgsEscaped,proto3" json:"ArgsEscaped,omitempty"` +} + +func (x *ImageConfig) Reset() { + *x = ImageConfig{} + if protoimpl.UnsafeEnabled { + mi := &file_oci_config_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ImageConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ImageConfig) ProtoMessage() {} + +func (x *ImageConfig) ProtoReflect() protoreflect.Message { + mi := &file_oci_config_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ImageConfig.ProtoReflect.Descriptor instead. +func (*ImageConfig) Descriptor() ([]byte, []int) { + return file_oci_config_proto_rawDescGZIP(), []int{1} +} + +func (x *ImageConfig) GetExposedPorts() map[string]*EmptyMessage { + if x != nil { + return x.ExposedPorts + } + return nil +} + +func (x *ImageConfig) GetVolumes() map[string]*EmptyMessage { + if x != nil { + return x.Volumes + } + return nil +} + +func (x *ImageConfig) GetLabels() map[string]string { + if x != nil { + return x.Labels + } + return nil +} + +func (x *ImageConfig) GetUser() string { + if x != nil { + return x.User + } + return "" +} + +func (x *ImageConfig) GetEnv() []string { + if x != nil { + return x.Env + } + return nil +} + +func (x *ImageConfig) GetEntrypoint() []string { + if x != nil { + return x.Entrypoint + } + return nil +} + +func (x *ImageConfig) GetCmd() []string { + if x != nil { + return x.Cmd + } + return nil +} + +func (x *ImageConfig) GetWorkingDir() string { + if x != nil && x.WorkingDir != nil { + return *x.WorkingDir + } + return "" +} + +func (x *ImageConfig) GetStopSignal() string { + if x != nil && x.StopSignal != nil { + return *x.StopSignal + } + return "" +} + +func (x *ImageConfig) GetArgsEscaped() bool { + if x != nil { + return x.ArgsEscaped + } + return false +} + +type RootFS struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Type string `protobuf:"bytes,1,opt,name=Type,proto3" json:"Type,omitempty"` + DiffIDs []string `protobuf:"bytes,2,rep,name=DiffIDs,proto3" json:"DiffIDs,omitempty"` +} + +func (x *RootFS) Reset() { + *x = RootFS{} + if protoimpl.UnsafeEnabled { + mi := &file_oci_config_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RootFS) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RootFS) ProtoMessage() {} + +func (x *RootFS) ProtoReflect() protoreflect.Message { + mi := &file_oci_config_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RootFS.ProtoReflect.Descriptor instead. +func (*RootFS) Descriptor() ([]byte, []int) { + return file_oci_config_proto_rawDescGZIP(), []int{2} +} + +func (x *RootFS) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +func (x *RootFS) GetDiffIDs() []string { + if x != nil { + return x.DiffIDs + } + return nil +} + +type History struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Created *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=Created,proto3,oneof" json:"Created,omitempty"` + CreatedBy *string `protobuf:"bytes,2,opt,name=CreatedBy,proto3,oneof" json:"CreatedBy,omitempty"` + Author *string `protobuf:"bytes,3,opt,name=Author,proto3,oneof" json:"Author,omitempty"` + Comment *string `protobuf:"bytes,4,opt,name=Comment,proto3,oneof" json:"Comment,omitempty"` + EmptyLayer *bool `protobuf:"varint,5,opt,name=EmptyLayer,proto3,oneof" json:"EmptyLayer,omitempty"` +} + +func (x *History) Reset() { + *x = History{} + if protoimpl.UnsafeEnabled { + mi := &file_oci_config_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *History) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*History) ProtoMessage() {} + +func (x *History) ProtoReflect() protoreflect.Message { + mi := &file_oci_config_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use History.ProtoReflect.Descriptor instead. +func (*History) Descriptor() ([]byte, []int) { + return file_oci_config_proto_rawDescGZIP(), []int{3} +} + +func (x *History) GetCreated() *timestamppb.Timestamp { + if x != nil { + return x.Created + } + return nil +} + +func (x *History) GetCreatedBy() string { + if x != nil && x.CreatedBy != nil { + return *x.CreatedBy + } + return "" +} + +func (x *History) GetAuthor() string { + if x != nil && x.Author != nil { + return *x.Author + } + return "" +} + +func (x *History) GetComment() string { + if x != nil && x.Comment != nil { + return *x.Comment + } + return "" +} + +func (x *History) GetEmptyLayer() bool { + if x != nil && x.EmptyLayer != nil { + return *x.EmptyLayer + } + return false +} + +type EmptyMessage struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields +} + +func (x *EmptyMessage) Reset() { + *x = EmptyMessage{} + if protoimpl.UnsafeEnabled { + mi := &file_oci_config_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *EmptyMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EmptyMessage) ProtoMessage() {} + +func (x *EmptyMessage) ProtoReflect() protoreflect.Message { + mi := &file_oci_config_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EmptyMessage.ProtoReflect.Descriptor instead. +func (*EmptyMessage) Descriptor() ([]byte, []int) { + return file_oci_config_proto_rawDescGZIP(), []int{4} +} + +var File_oci_config_proto protoreflect.FileDescriptor + +var file_oci_config_proto_rawDesc = []byte{ + 0x0a, 0x10, 0x6f, 0x63, 0x69, 0x2f, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x12, 0x06, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x1a, 0x13, 0x6f, 0x63, 0x69, 0x2f, + 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, + 0x14, 0x6f, 0x63, 0x69, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc4, 0x02, 0x0a, 0x05, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, + 0x39, 0x0a, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, 0x07, + 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x88, 0x01, 0x01, 0x12, 0x1b, 0x0a, 0x06, 0x41, 0x75, + 0x74, 0x68, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x06, 0x41, 0x75, + 0x74, 0x68, 0x6f, 0x72, 0x88, 0x01, 0x01, 0x12, 0x2c, 0x0a, 0x08, 0x50, 0x6c, 0x61, 0x74, 0x66, + 0x6f, 0x72, 0x6d, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x6f, 0x63, 0x69, 0x5f, + 0x76, 0x31, 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x08, 0x50, 0x6c, 0x61, + 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x30, 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, 0x49, + 0x6d, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x48, 0x02, 0x52, 0x06, 0x43, 0x6f, + 0x6e, 0x66, 0x69, 0x67, 0x88, 0x01, 0x01, 0x12, 0x2b, 0x0a, 0x06, 0x52, 0x6f, 0x6f, 0x74, 0x46, + 0x53, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, + 0x2e, 0x52, 0x6f, 0x6f, 0x74, 0x46, 0x53, 0x48, 0x03, 0x52, 0x06, 0x52, 0x6f, 0x6f, 0x74, 0x46, + 0x53, 0x88, 0x01, 0x01, 0x12, 0x29, 0x0a, 0x07, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x18, + 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, 0x48, + 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x52, 0x07, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x42, + 0x0a, 0x0a, 0x08, 0x5f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x09, 0x0a, 0x07, 0x5f, + 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x43, 0x6f, 0x6e, 0x66, 0x69, + 0x67, 0x42, 0x09, 0x0a, 0x07, 0x5f, 0x52, 0x6f, 0x6f, 0x74, 0x46, 0x53, 0x22, 0x93, 0x05, 0x0a, + 0x0b, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x49, 0x0a, 0x0c, + 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, 0x49, 0x6d, 0x61, 0x67, + 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, 0x64, 0x50, + 0x6f, 0x72, 0x74, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x45, 0x78, 0x70, 0x6f, 0x73, + 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x73, 0x12, 0x3a, 0x0a, 0x07, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, + 0x31, 0x2e, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x56, 0x6f, + 0x6c, 0x75, 0x6d, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x56, 0x6f, 0x6c, 0x75, + 0x6d, 0x65, 0x73, 0x12, 0x37, 0x0a, 0x06, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, 0x03, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, 0x49, 0x6d, 0x61, + 0x67, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x12, 0x12, 0x0a, 0x04, + 0x55, 0x73, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x55, 0x73, 0x65, 0x72, + 0x12, 0x10, 0x0a, 0x03, 0x45, 0x6e, 0x76, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x03, 0x45, + 0x6e, 0x76, 0x12, 0x1e, 0x0a, 0x0a, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x18, 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x43, 0x6d, 0x64, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x03, 0x43, 0x6d, 0x64, 0x12, 0x23, 0x0a, 0x0a, 0x57, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, 0x44, + 0x69, 0x72, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x0a, 0x57, 0x6f, 0x72, 0x6b, + 0x69, 0x6e, 0x67, 0x44, 0x69, 0x72, 0x88, 0x01, 0x01, 0x12, 0x23, 0x0a, 0x0a, 0x53, 0x74, 0x6f, + 0x70, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, + 0x0a, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x88, 0x01, 0x01, 0x12, 0x20, + 0x0a, 0x0b, 0x41, 0x72, 0x67, 0x73, 0x45, 0x73, 0x63, 0x61, 0x70, 0x65, 0x64, 0x18, 0x0a, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0b, 0x41, 0x72, 0x67, 0x73, 0x45, 0x73, 0x63, 0x61, 0x70, 0x65, 0x64, + 0x1a, 0x55, 0x0a, 0x11, 0x45, 0x78, 0x70, 0x6f, 0x73, 0x65, 0x64, 0x50, 0x6f, 0x72, 0x74, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, + 0x45, 0x6d, 0x70, 0x74, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x50, 0x0a, 0x0c, 0x56, 0x6f, 0x6c, 0x75, 0x6d, + 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, + 0x31, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x39, 0x0a, 0x0b, 0x4c, 0x61, 0x62, + 0x65, 0x6c, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x57, 0x6f, 0x72, 0x6b, 0x69, 0x6e, 0x67, + 0x44, 0x69, 0x72, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x53, 0x74, 0x6f, 0x70, 0x53, 0x69, 0x67, 0x6e, + 0x61, 0x6c, 0x22, 0x36, 0x0a, 0x06, 0x52, 0x6f, 0x6f, 0x74, 0x46, 0x53, 0x12, 0x12, 0x0a, 0x04, + 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x18, 0x0a, 0x07, 0x44, 0x69, 0x66, 0x66, 0x49, 0x44, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x07, 0x44, 0x69, 0x66, 0x66, 0x49, 0x44, 0x73, 0x22, 0x88, 0x02, 0x0a, 0x07, 0x48, + 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x12, 0x39, 0x0a, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, + 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, 0x07, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x88, 0x01, + 0x01, 0x12, 0x21, 0x0a, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x79, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x09, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, + 0x79, 0x88, 0x01, 0x01, 0x12, 0x1b, 0x0a, 0x06, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x48, 0x02, 0x52, 0x06, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x88, 0x01, + 0x01, 0x12, 0x1d, 0x0a, 0x07, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x03, 0x52, 0x07, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x88, 0x01, 0x01, + 0x12, 0x23, 0x0a, 0x0a, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x08, 0x48, 0x04, 0x52, 0x0a, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x4c, 0x61, 0x79, + 0x65, 0x72, 0x88, 0x01, 0x01, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x64, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x79, 0x42, + 0x09, 0x0a, 0x07, 0x5f, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x43, + 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x0d, 0x0a, 0x0b, 0x5f, 0x45, 0x6d, 0x70, 0x74, 0x79, + 0x4c, 0x61, 0x79, 0x65, 0x72, 0x22, 0x0e, 0x0a, 0x0c, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x4d, 0x65, + 0x73, 0x73, 0x61, 0x67, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_oci_config_proto_rawDescOnce sync.Once + file_oci_config_proto_rawDescData = file_oci_config_proto_rawDesc +) + +func file_oci_config_proto_rawDescGZIP() []byte { + file_oci_config_proto_rawDescOnce.Do(func() { + file_oci_config_proto_rawDescData = protoimpl.X.CompressGZIP(file_oci_config_proto_rawDescData) + }) + return file_oci_config_proto_rawDescData +} + +var file_oci_config_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_oci_config_proto_goTypes = []interface{}{ + (*Image)(nil), // 0: oci_v1.Image + (*ImageConfig)(nil), // 1: oci_v1.ImageConfig + (*RootFS)(nil), // 2: oci_v1.RootFS + (*History)(nil), // 3: oci_v1.History + (*EmptyMessage)(nil), // 4: oci_v1.EmptyMessage + nil, // 5: oci_v1.ImageConfig.ExposedPortsEntry + nil, // 6: oci_v1.ImageConfig.VolumesEntry + nil, // 7: oci_v1.ImageConfig.LabelsEntry + (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp + (*Platform)(nil), // 9: oci_v1.Platform +} +var file_oci_config_proto_depIdxs = []int32{ + 8, // 0: oci_v1.Image.Created:type_name -> google.protobuf.Timestamp + 9, // 1: oci_v1.Image.Platform:type_name -> oci_v1.Platform + 1, // 2: oci_v1.Image.Config:type_name -> oci_v1.ImageConfig + 2, // 3: oci_v1.Image.RootFS:type_name -> oci_v1.RootFS + 3, // 4: oci_v1.Image.History:type_name -> oci_v1.History + 5, // 5: oci_v1.ImageConfig.ExposedPorts:type_name -> oci_v1.ImageConfig.ExposedPortsEntry + 6, // 6: oci_v1.ImageConfig.Volumes:type_name -> oci_v1.ImageConfig.VolumesEntry + 7, // 7: oci_v1.ImageConfig.Labels:type_name -> oci_v1.ImageConfig.LabelsEntry + 8, // 8: oci_v1.History.Created:type_name -> google.protobuf.Timestamp + 4, // 9: oci_v1.ImageConfig.ExposedPortsEntry.value:type_name -> oci_v1.EmptyMessage + 4, // 10: oci_v1.ImageConfig.VolumesEntry.value:type_name -> oci_v1.EmptyMessage + 11, // [11:11] is the sub-list for method output_type + 11, // [11:11] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name +} + +func init() { file_oci_config_proto_init() } +func file_oci_config_proto_init() { + if File_oci_config_proto != nil { + return + } + file_oci_descriptor_proto_init() + if !protoimpl.UnsafeEnabled { + file_oci_config_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Image); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_oci_config_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ImageConfig); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_oci_config_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RootFS); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_oci_config_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*History); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_oci_config_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EmptyMessage); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_oci_config_proto_msgTypes[0].OneofWrappers = []interface{}{} + file_oci_config_proto_msgTypes[1].OneofWrappers = []interface{}{} + file_oci_config_proto_msgTypes[3].OneofWrappers = []interface{}{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_oci_config_proto_rawDesc, + NumEnums: 0, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_oci_config_proto_goTypes, + DependencyIndexes: file_oci_config_proto_depIdxs, + MessageInfos: file_oci_config_proto_msgTypes, + }.Build() + File_oci_config_proto = out.File + file_oci_config_proto_rawDesc = nil + file_oci_config_proto_goTypes = nil + file_oci_config_proto_depIdxs = nil +} diff --git a/pkg/meta/proto/gen/descriptor.pb.go b/pkg/meta/proto/gen/descriptor.pb.go new file mode 100644 index 00000000..e2c1a054 --- /dev/null +++ b/pkg/meta/proto/gen/descriptor.pb.go @@ -0,0 +1,328 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.24.4 +// source: oci/descriptor.proto + +package gen + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Descriptor struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + MediaType string `protobuf:"bytes,1,opt,name=MediaType,proto3" json:"MediaType,omitempty"` + Digest string `protobuf:"bytes,2,opt,name=Digest,proto3" json:"Digest,omitempty"` + Size int64 `protobuf:"varint,3,opt,name=Size,proto3" json:"Size,omitempty"` + URLs []string `protobuf:"bytes,4,rep,name=URLs,proto3" json:"URLs,omitempty"` + Data []byte `protobuf:"bytes,5,opt,name=Data,proto3" json:"Data,omitempty"` + Platform *Platform `protobuf:"bytes,6,opt,name=Platform,proto3,oneof" json:"Platform,omitempty"` + ArtifactType *string `protobuf:"bytes,7,opt,name=ArtifactType,proto3,oneof" json:"ArtifactType,omitempty"` + Annotations map[string]string `protobuf:"bytes,8,rep,name=Annotations,proto3" json:"Annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *Descriptor) Reset() { + *x = Descriptor{} + if protoimpl.UnsafeEnabled { + mi := &file_oci_descriptor_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Descriptor) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Descriptor) ProtoMessage() {} + +func (x *Descriptor) ProtoReflect() protoreflect.Message { + mi := &file_oci_descriptor_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Descriptor.ProtoReflect.Descriptor instead. +func (*Descriptor) Descriptor() ([]byte, []int) { + return file_oci_descriptor_proto_rawDescGZIP(), []int{0} +} + +func (x *Descriptor) GetMediaType() string { + if x != nil { + return x.MediaType + } + return "" +} + +func (x *Descriptor) GetDigest() string { + if x != nil { + return x.Digest + } + return "" +} + +func (x *Descriptor) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *Descriptor) GetURLs() []string { + if x != nil { + return x.URLs + } + return nil +} + +func (x *Descriptor) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *Descriptor) GetPlatform() *Platform { + if x != nil { + return x.Platform + } + return nil +} + +func (x *Descriptor) GetArtifactType() string { + if x != nil && x.ArtifactType != nil { + return *x.ArtifactType + } + return "" +} + +func (x *Descriptor) GetAnnotations() map[string]string { + if x != nil { + return x.Annotations + } + return nil +} + +type Platform struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Architecture string `protobuf:"bytes,1,opt,name=Architecture,proto3" json:"Architecture,omitempty"` + OS string `protobuf:"bytes,2,opt,name=OS,proto3" json:"OS,omitempty"` + OSVersion *string `protobuf:"bytes,3,opt,name=OSVersion,proto3,oneof" json:"OSVersion,omitempty"` + OSFeatures []string `protobuf:"bytes,4,rep,name=OSFeatures,proto3" json:"OSFeatures,omitempty"` + Variant *string `protobuf:"bytes,5,opt,name=Variant,proto3,oneof" json:"Variant,omitempty"` +} + +func (x *Platform) Reset() { + *x = Platform{} + if protoimpl.UnsafeEnabled { + mi := &file_oci_descriptor_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Platform) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Platform) ProtoMessage() {} + +func (x *Platform) ProtoReflect() protoreflect.Message { + mi := &file_oci_descriptor_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Platform.ProtoReflect.Descriptor instead. +func (*Platform) Descriptor() ([]byte, []int) { + return file_oci_descriptor_proto_rawDescGZIP(), []int{1} +} + +func (x *Platform) GetArchitecture() string { + if x != nil { + return x.Architecture + } + return "" +} + +func (x *Platform) GetOS() string { + if x != nil { + return x.OS + } + return "" +} + +func (x *Platform) GetOSVersion() string { + if x != nil && x.OSVersion != nil { + return *x.OSVersion + } + return "" +} + +func (x *Platform) GetOSFeatures() []string { + if x != nil { + return x.OSFeatures + } + return nil +} + +func (x *Platform) GetVariant() string { + if x != nil && x.Variant != nil { + return *x.Variant + } + return "" +} + +var File_oci_descriptor_proto protoreflect.FileDescriptor + +var file_oci_descriptor_proto_rawDesc = []byte{ + 0x0a, 0x14, 0x6f, 0x63, 0x69, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x22, 0xff, + 0x02, 0x0a, 0x0a, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x12, 0x1c, 0x0a, + 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, + 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, 0x67, + 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x03, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x55, 0x52, 0x4c, 0x73, 0x18, + 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x55, 0x52, 0x4c, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x44, + 0x61, 0x74, 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x44, 0x61, 0x74, 0x61, 0x12, + 0x31, 0x0a, 0x08, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x10, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, + 0x6f, 0x72, 0x6d, 0x48, 0x00, 0x52, 0x08, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x88, + 0x01, 0x01, 0x12, 0x27, 0x0a, 0x0c, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, + 0x70, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x0c, 0x41, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x45, 0x0a, 0x0b, 0x41, + 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x23, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x42, 0x0b, 0x0a, 0x09, 0x5f, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x42, + 0x0f, 0x0a, 0x0d, 0x5f, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x22, 0xba, 0x01, 0x0a, 0x08, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x12, 0x22, 0x0a, + 0x0c, 0x41, 0x72, 0x63, 0x68, 0x69, 0x74, 0x65, 0x63, 0x74, 0x75, 0x72, 0x65, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0c, 0x41, 0x72, 0x63, 0x68, 0x69, 0x74, 0x65, 0x63, 0x74, 0x75, 0x72, + 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x4f, 0x53, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x4f, + 0x53, 0x12, 0x21, 0x0a, 0x09, 0x4f, 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x4f, 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x88, 0x01, 0x01, 0x12, 0x1e, 0x0a, 0x0a, 0x4f, 0x53, 0x46, 0x65, 0x61, 0x74, 0x75, 0x72, + 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x4f, 0x53, 0x46, 0x65, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x73, 0x12, 0x1d, 0x0a, 0x07, 0x56, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x74, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x07, 0x56, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x74, + 0x88, 0x01, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x4f, 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x56, 0x61, 0x72, 0x69, 0x61, 0x6e, 0x74, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_oci_descriptor_proto_rawDescOnce sync.Once + file_oci_descriptor_proto_rawDescData = file_oci_descriptor_proto_rawDesc +) + +func file_oci_descriptor_proto_rawDescGZIP() []byte { + file_oci_descriptor_proto_rawDescOnce.Do(func() { + file_oci_descriptor_proto_rawDescData = protoimpl.X.CompressGZIP(file_oci_descriptor_proto_rawDescData) + }) + return file_oci_descriptor_proto_rawDescData +} + +var file_oci_descriptor_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_oci_descriptor_proto_goTypes = []interface{}{ + (*Descriptor)(nil), // 0: oci_v1.Descriptor + (*Platform)(nil), // 1: oci_v1.Platform + nil, // 2: oci_v1.Descriptor.AnnotationsEntry +} +var file_oci_descriptor_proto_depIdxs = []int32{ + 1, // 0: oci_v1.Descriptor.Platform:type_name -> oci_v1.Platform + 2, // 1: oci_v1.Descriptor.Annotations:type_name -> oci_v1.Descriptor.AnnotationsEntry + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_oci_descriptor_proto_init() } +func file_oci_descriptor_proto_init() { + if File_oci_descriptor_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_oci_descriptor_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Descriptor); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_oci_descriptor_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Platform); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_oci_descriptor_proto_msgTypes[0].OneofWrappers = []interface{}{} + file_oci_descriptor_proto_msgTypes[1].OneofWrappers = []interface{}{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_oci_descriptor_proto_rawDesc, + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_oci_descriptor_proto_goTypes, + DependencyIndexes: file_oci_descriptor_proto_depIdxs, + MessageInfos: file_oci_descriptor_proto_msgTypes, + }.Build() + File_oci_descriptor_proto = out.File + file_oci_descriptor_proto_rawDesc = nil + file_oci_descriptor_proto_goTypes = nil + file_oci_descriptor_proto_depIdxs = nil +} diff --git a/pkg/meta/proto/gen/index.pb.go b/pkg/meta/proto/gen/index.pb.go new file mode 100644 index 00000000..9e5d2a2b --- /dev/null +++ b/pkg/meta/proto/gen/index.pb.go @@ -0,0 +1,217 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.24.4 +// source: oci/index.proto + +package gen + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Index struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Versioned *Versioned `protobuf:"bytes,1,opt,name=Versioned,proto3" json:"Versioned,omitempty"` + MediaType *string `protobuf:"bytes,2,opt,name=MediaType,proto3,oneof" json:"MediaType,omitempty"` + ArtifactType *string `protobuf:"bytes,3,opt,name=ArtifactType,proto3,oneof" json:"ArtifactType,omitempty"` + Manifests []*Descriptor `protobuf:"bytes,4,rep,name=Manifests,proto3" json:"Manifests,omitempty"` + Subject *Descriptor `protobuf:"bytes,5,opt,name=Subject,proto3,oneof" json:"Subject,omitempty"` + Annotations map[string]string `protobuf:"bytes,6,rep,name=Annotations,proto3" json:"Annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *Index) Reset() { + *x = Index{} + if protoimpl.UnsafeEnabled { + mi := &file_oci_index_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Index) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Index) ProtoMessage() {} + +func (x *Index) ProtoReflect() protoreflect.Message { + mi := &file_oci_index_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Index.ProtoReflect.Descriptor instead. +func (*Index) Descriptor() ([]byte, []int) { + return file_oci_index_proto_rawDescGZIP(), []int{0} +} + +func (x *Index) GetVersioned() *Versioned { + if x != nil { + return x.Versioned + } + return nil +} + +func (x *Index) GetMediaType() string { + if x != nil && x.MediaType != nil { + return *x.MediaType + } + return "" +} + +func (x *Index) GetArtifactType() string { + if x != nil && x.ArtifactType != nil { + return *x.ArtifactType + } + return "" +} + +func (x *Index) GetManifests() []*Descriptor { + if x != nil { + return x.Manifests + } + return nil +} + +func (x *Index) GetSubject() *Descriptor { + if x != nil { + return x.Subject + } + return nil +} + +func (x *Index) GetAnnotations() map[string]string { + if x != nil { + return x.Annotations + } + return nil +} + +var File_oci_index_proto protoreflect.FileDescriptor + +var file_oci_index_proto_rawDesc = []byte{ + 0x0a, 0x0f, 0x6f, 0x63, 0x69, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x06, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x1a, 0x14, 0x6f, 0x63, 0x69, 0x2f, 0x64, + 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, + 0x13, 0x6f, 0x63, 0x69, 0x2f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x96, 0x03, 0x0a, 0x05, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x2f, + 0x0a, 0x09, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x11, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x09, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x12, + 0x21, 0x0a, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x88, + 0x01, 0x01, 0x12, 0x27, 0x0a, 0x0c, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, + 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, 0x52, 0x0c, 0x41, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x30, 0x0a, 0x09, 0x4d, + 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x6f, 0x72, 0x52, 0x09, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x73, 0x12, 0x31, 0x0a, + 0x07, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, + 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x6f, 0x72, 0x48, 0x02, 0x52, 0x07, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x88, 0x01, 0x01, + 0x12, 0x40, 0x0a, 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, 0x49, + 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, + 0x42, 0x0f, 0x0a, 0x0d, 0x5f, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, + 0x65, 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_oci_index_proto_rawDescOnce sync.Once + file_oci_index_proto_rawDescData = file_oci_index_proto_rawDesc +) + +func file_oci_index_proto_rawDescGZIP() []byte { + file_oci_index_proto_rawDescOnce.Do(func() { + file_oci_index_proto_rawDescData = protoimpl.X.CompressGZIP(file_oci_index_proto_rawDescData) + }) + return file_oci_index_proto_rawDescData +} + +var file_oci_index_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_oci_index_proto_goTypes = []interface{}{ + (*Index)(nil), // 0: oci_v1.Index + nil, // 1: oci_v1.Index.AnnotationsEntry + (*Versioned)(nil), // 2: oci_v1.Versioned + (*Descriptor)(nil), // 3: oci_v1.Descriptor +} +var file_oci_index_proto_depIdxs = []int32{ + 2, // 0: oci_v1.Index.Versioned:type_name -> oci_v1.Versioned + 3, // 1: oci_v1.Index.Manifests:type_name -> oci_v1.Descriptor + 3, // 2: oci_v1.Index.Subject:type_name -> oci_v1.Descriptor + 1, // 3: oci_v1.Index.Annotations:type_name -> oci_v1.Index.AnnotationsEntry + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_oci_index_proto_init() } +func file_oci_index_proto_init() { + if File_oci_index_proto != nil { + return + } + file_oci_descriptor_proto_init() + file_oci_versioned_proto_init() + if !protoimpl.UnsafeEnabled { + file_oci_index_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Index); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_oci_index_proto_msgTypes[0].OneofWrappers = []interface{}{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_oci_index_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_oci_index_proto_goTypes, + DependencyIndexes: file_oci_index_proto_depIdxs, + MessageInfos: file_oci_index_proto_msgTypes, + }.Build() + File_oci_index_proto = out.File + file_oci_index_proto_rawDesc = nil + file_oci_index_proto_goTypes = nil + file_oci_index_proto_depIdxs = nil +} diff --git a/pkg/meta/proto/gen/manifest.pb.go b/pkg/meta/proto/gen/manifest.pb.go new file mode 100644 index 00000000..f4640ced --- /dev/null +++ b/pkg/meta/proto/gen/manifest.pb.go @@ -0,0 +1,229 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.24.4 +// source: oci/manifest.proto + +package gen + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Manifest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Versioned *Versioned `protobuf:"bytes,1,opt,name=Versioned,proto3" json:"Versioned,omitempty"` + MediaType *string `protobuf:"bytes,2,opt,name=MediaType,proto3,oneof" json:"MediaType,omitempty"` + ArtifactType *string `protobuf:"bytes,3,opt,name=ArtifactType,proto3,oneof" json:"ArtifactType,omitempty"` + Config *Descriptor `protobuf:"bytes,4,opt,name=Config,proto3" json:"Config,omitempty"` + Layers []*Descriptor `protobuf:"bytes,5,rep,name=Layers,proto3" json:"Layers,omitempty"` + Subject *Descriptor `protobuf:"bytes,6,opt,name=Subject,proto3,oneof" json:"Subject,omitempty"` + Annotations map[string]string `protobuf:"bytes,7,rep,name=Annotations,proto3" json:"Annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *Manifest) Reset() { + *x = Manifest{} + if protoimpl.UnsafeEnabled { + mi := &file_oci_manifest_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Manifest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Manifest) ProtoMessage() {} + +func (x *Manifest) ProtoReflect() protoreflect.Message { + mi := &file_oci_manifest_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Manifest.ProtoReflect.Descriptor instead. +func (*Manifest) Descriptor() ([]byte, []int) { + return file_oci_manifest_proto_rawDescGZIP(), []int{0} +} + +func (x *Manifest) GetVersioned() *Versioned { + if x != nil { + return x.Versioned + } + return nil +} + +func (x *Manifest) GetMediaType() string { + if x != nil && x.MediaType != nil { + return *x.MediaType + } + return "" +} + +func (x *Manifest) GetArtifactType() string { + if x != nil && x.ArtifactType != nil { + return *x.ArtifactType + } + return "" +} + +func (x *Manifest) GetConfig() *Descriptor { + if x != nil { + return x.Config + } + return nil +} + +func (x *Manifest) GetLayers() []*Descriptor { + if x != nil { + return x.Layers + } + return nil +} + +func (x *Manifest) GetSubject() *Descriptor { + if x != nil { + return x.Subject + } + return nil +} + +func (x *Manifest) GetAnnotations() map[string]string { + if x != nil { + return x.Annotations + } + return nil +} + +var File_oci_manifest_proto protoreflect.FileDescriptor + +var file_oci_manifest_proto_rawDesc = []byte{ + 0x0a, 0x12, 0x6f, 0x63, 0x69, 0x2f, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x1a, 0x14, 0x6f, 0x63, + 0x69, 0x2f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x1a, 0x13, 0x6f, 0x63, 0x69, 0x2f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, + 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0xc2, 0x03, 0x0a, 0x08, 0x4d, 0x61, 0x6e, 0x69, + 0x66, 0x65, 0x73, 0x74, 0x12, 0x2f, 0x0a, 0x09, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, + 0x2e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x52, 0x09, 0x56, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x12, 0x21, 0x0a, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, + 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x09, 0x4d, 0x65, 0x64, 0x69, + 0x61, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, 0x01, 0x12, 0x27, 0x0a, 0x0c, 0x41, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x48, 0x01, + 0x52, 0x0c, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x88, 0x01, + 0x01, 0x12, 0x2a, 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x12, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x06, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x2a, 0x0a, + 0x06, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, + 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, + 0x72, 0x52, 0x06, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x12, 0x31, 0x0a, 0x07, 0x53, 0x75, 0x62, + 0x6a, 0x65, 0x63, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6f, 0x63, 0x69, + 0x5f, 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x48, 0x02, + 0x52, 0x07, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x88, 0x01, 0x01, 0x12, 0x43, 0x0a, 0x0b, + 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x21, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, 0x4d, 0x61, 0x6e, 0x69, 0x66, + 0x65, 0x73, 0x74, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x42, 0x0c, 0x0a, 0x0a, 0x5f, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x42, + 0x0f, 0x0a, 0x0d, 0x5f, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, + 0x42, 0x0a, 0x0a, 0x08, 0x5f, 0x53, 0x75, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x62, 0x06, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_oci_manifest_proto_rawDescOnce sync.Once + file_oci_manifest_proto_rawDescData = file_oci_manifest_proto_rawDesc +) + +func file_oci_manifest_proto_rawDescGZIP() []byte { + file_oci_manifest_proto_rawDescOnce.Do(func() { + file_oci_manifest_proto_rawDescData = protoimpl.X.CompressGZIP(file_oci_manifest_proto_rawDescData) + }) + return file_oci_manifest_proto_rawDescData +} + +var file_oci_manifest_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_oci_manifest_proto_goTypes = []interface{}{ + (*Manifest)(nil), // 0: oci_v1.Manifest + nil, // 1: oci_v1.Manifest.AnnotationsEntry + (*Versioned)(nil), // 2: oci_v1.Versioned + (*Descriptor)(nil), // 3: oci_v1.Descriptor +} +var file_oci_manifest_proto_depIdxs = []int32{ + 2, // 0: oci_v1.Manifest.Versioned:type_name -> oci_v1.Versioned + 3, // 1: oci_v1.Manifest.Config:type_name -> oci_v1.Descriptor + 3, // 2: oci_v1.Manifest.Layers:type_name -> oci_v1.Descriptor + 3, // 3: oci_v1.Manifest.Subject:type_name -> oci_v1.Descriptor + 1, // 4: oci_v1.Manifest.Annotations:type_name -> oci_v1.Manifest.AnnotationsEntry + 5, // [5:5] is the sub-list for method output_type + 5, // [5:5] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name +} + +func init() { file_oci_manifest_proto_init() } +func file_oci_manifest_proto_init() { + if File_oci_manifest_proto != nil { + return + } + file_oci_descriptor_proto_init() + file_oci_versioned_proto_init() + if !protoimpl.UnsafeEnabled { + file_oci_manifest_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Manifest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_oci_manifest_proto_msgTypes[0].OneofWrappers = []interface{}{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_oci_manifest_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_oci_manifest_proto_goTypes, + DependencyIndexes: file_oci_manifest_proto_depIdxs, + MessageInfos: file_oci_manifest_proto_msgTypes, + }.Build() + File_oci_manifest_proto = out.File + file_oci_manifest_proto_rawDesc = nil + file_oci_manifest_proto_goTypes = nil + file_oci_manifest_proto_depIdxs = nil +} diff --git a/pkg/meta/proto/gen/meta.pb.go b/pkg/meta/proto/gen/meta.pb.go new file mode 100644 index 00000000..46a324f2 --- /dev/null +++ b/pkg/meta/proto/gen/meta.pb.go @@ -0,0 +1,1521 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.24.4 +// source: meta/meta.proto + +package gen + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type TagDescriptor struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + MediaType string `protobuf:"bytes,1,opt,name=MediaType,proto3" json:"MediaType,omitempty"` + Digest string `protobuf:"bytes,2,opt,name=Digest,proto3" json:"Digest,omitempty"` +} + +func (x *TagDescriptor) Reset() { + *x = TagDescriptor{} + if protoimpl.UnsafeEnabled { + mi := &file_meta_meta_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *TagDescriptor) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TagDescriptor) ProtoMessage() {} + +func (x *TagDescriptor) ProtoReflect() protoreflect.Message { + mi := &file_meta_meta_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TagDescriptor.ProtoReflect.Descriptor instead. +func (*TagDescriptor) Descriptor() ([]byte, []int) { + return file_meta_meta_proto_rawDescGZIP(), []int{0} +} + +func (x *TagDescriptor) GetMediaType() string { + if x != nil { + return x.MediaType + } + return "" +} + +func (x *TagDescriptor) GetDigest() string { + if x != nil { + return x.Digest + } + return "" +} + +type ImageMeta struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + MediaType string `protobuf:"bytes,1,opt,name=MediaType,proto3" json:"MediaType,omitempty"` + Manifests []*ManifestMeta `protobuf:"bytes,2,rep,name=Manifests,proto3" json:"Manifests,omitempty"` + Index *IndexMeta `protobuf:"bytes,3,opt,name=Index,proto3,oneof" json:"Index,omitempty"` +} + +func (x *ImageMeta) Reset() { + *x = ImageMeta{} + if protoimpl.UnsafeEnabled { + mi := &file_meta_meta_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ImageMeta) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ImageMeta) ProtoMessage() {} + +func (x *ImageMeta) ProtoReflect() protoreflect.Message { + mi := &file_meta_meta_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ImageMeta.ProtoReflect.Descriptor instead. +func (*ImageMeta) Descriptor() ([]byte, []int) { + return file_meta_meta_proto_rawDescGZIP(), []int{1} +} + +func (x *ImageMeta) GetMediaType() string { + if x != nil { + return x.MediaType + } + return "" +} + +func (x *ImageMeta) GetManifests() []*ManifestMeta { + if x != nil { + return x.Manifests + } + return nil +} + +func (x *ImageMeta) GetIndex() *IndexMeta { + if x != nil { + return x.Index + } + return nil +} + +type ManifestMeta struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Digest string `protobuf:"bytes,1,opt,name=Digest,proto3" json:"Digest,omitempty"` + Size int64 `protobuf:"varint,2,opt,name=Size,proto3" json:"Size,omitempty"` + Manifest *Manifest `protobuf:"bytes,3,opt,name=Manifest,proto3" json:"Manifest,omitempty"` + Config *Image `protobuf:"bytes,4,opt,name=Config,proto3" json:"Config,omitempty"` +} + +func (x *ManifestMeta) Reset() { + *x = ManifestMeta{} + if protoimpl.UnsafeEnabled { + mi := &file_meta_meta_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ManifestMeta) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ManifestMeta) ProtoMessage() {} + +func (x *ManifestMeta) ProtoReflect() protoreflect.Message { + mi := &file_meta_meta_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ManifestMeta.ProtoReflect.Descriptor instead. +func (*ManifestMeta) Descriptor() ([]byte, []int) { + return file_meta_meta_proto_rawDescGZIP(), []int{2} +} + +func (x *ManifestMeta) GetDigest() string { + if x != nil { + return x.Digest + } + return "" +} + +func (x *ManifestMeta) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *ManifestMeta) GetManifest() *Manifest { + if x != nil { + return x.Manifest + } + return nil +} + +func (x *ManifestMeta) GetConfig() *Image { + if x != nil { + return x.Config + } + return nil +} + +type IndexMeta struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Digest string `protobuf:"bytes,1,opt,name=Digest,proto3" json:"Digest,omitempty"` + Size int64 `protobuf:"varint,2,opt,name=Size,proto3" json:"Size,omitempty"` + Index *Index `protobuf:"bytes,3,opt,name=Index,proto3" json:"Index,omitempty"` +} + +func (x *IndexMeta) Reset() { + *x = IndexMeta{} + if protoimpl.UnsafeEnabled { + mi := &file_meta_meta_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *IndexMeta) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IndexMeta) ProtoMessage() {} + +func (x *IndexMeta) ProtoReflect() protoreflect.Message { + mi := &file_meta_meta_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IndexMeta.ProtoReflect.Descriptor instead. +func (*IndexMeta) Descriptor() ([]byte, []int) { + return file_meta_meta_proto_rawDescGZIP(), []int{3} +} + +func (x *IndexMeta) GetDigest() string { + if x != nil { + return x.Digest + } + return "" +} + +func (x *IndexMeta) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *IndexMeta) GetIndex() *Index { + if x != nil { + return x.Index + } + return nil +} + +type RepoLastUpdatedImage struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + LastUpdated *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=LastUpdated,proto3,oneof" json:"LastUpdated,omitempty"` + MediaType string `protobuf:"bytes,2,opt,name=MediaType,proto3" json:"MediaType,omitempty"` + Digest string `protobuf:"bytes,3,opt,name=Digest,proto3" json:"Digest,omitempty"` + Tag string `protobuf:"bytes,4,opt,name=Tag,proto3" json:"Tag,omitempty"` +} + +func (x *RepoLastUpdatedImage) Reset() { + *x = RepoLastUpdatedImage{} + if protoimpl.UnsafeEnabled { + mi := &file_meta_meta_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RepoLastUpdatedImage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RepoLastUpdatedImage) ProtoMessage() {} + +func (x *RepoLastUpdatedImage) ProtoReflect() protoreflect.Message { + mi := &file_meta_meta_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RepoLastUpdatedImage.ProtoReflect.Descriptor instead. +func (*RepoLastUpdatedImage) Descriptor() ([]byte, []int) { + return file_meta_meta_proto_rawDescGZIP(), []int{4} +} + +func (x *RepoLastUpdatedImage) GetLastUpdated() *timestamppb.Timestamp { + if x != nil { + return x.LastUpdated + } + return nil +} + +func (x *RepoLastUpdatedImage) GetMediaType() string { + if x != nil { + return x.MediaType + } + return "" +} + +func (x *RepoLastUpdatedImage) GetDigest() string { + if x != nil { + return x.Digest + } + return "" +} + +func (x *RepoLastUpdatedImage) GetTag() string { + if x != nil { + return x.Tag + } + return "" +} + +type RepoMeta struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Tags map[string]*TagDescriptor `protobuf:"bytes,2,rep,name=Tags,proto3" json:"Tags,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Statistics map[string]*DescriptorStatistics `protobuf:"bytes,3,rep,name=Statistics,proto3" json:"Statistics,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Signatures map[string]*ManifestSignatures `protobuf:"bytes,4,rep,name=Signatures,proto3" json:"Signatures,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Referrers map[string]*ReferrersInfo `protobuf:"bytes,5,rep,name=Referrers,proto3" json:"Referrers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + IsStarred bool `protobuf:"varint,6,opt,name=IsStarred,proto3" json:"IsStarred,omitempty"` + IsBookmarked bool `protobuf:"varint,7,opt,name=IsBookmarked,proto3" json:"IsBookmarked,omitempty"` + Rank int32 `protobuf:"varint,8,opt,name=Rank,proto3" json:"Rank,omitempty"` + Stars int32 `protobuf:"varint,9,opt,name=Stars,proto3" json:"Stars,omitempty"` + Size int32 `protobuf:"varint,10,opt,name=Size,proto3" json:"Size,omitempty"` + Vendors []string `protobuf:"bytes,11,rep,name=Vendors,proto3" json:"Vendors,omitempty"` + Platforms []*Platform `protobuf:"bytes,12,rep,name=Platforms,proto3" json:"Platforms,omitempty"` + LastUpdatedImage *RepoLastUpdatedImage `protobuf:"bytes,13,opt,name=LastUpdatedImage,proto3,oneof" json:"LastUpdatedImage,omitempty"` +} + +func (x *RepoMeta) Reset() { + *x = RepoMeta{} + if protoimpl.UnsafeEnabled { + mi := &file_meta_meta_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RepoMeta) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RepoMeta) ProtoMessage() {} + +func (x *RepoMeta) ProtoReflect() protoreflect.Message { + mi := &file_meta_meta_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RepoMeta.ProtoReflect.Descriptor instead. +func (*RepoMeta) Descriptor() ([]byte, []int) { + return file_meta_meta_proto_rawDescGZIP(), []int{5} +} + +func (x *RepoMeta) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *RepoMeta) GetTags() map[string]*TagDescriptor { + if x != nil { + return x.Tags + } + return nil +} + +func (x *RepoMeta) GetStatistics() map[string]*DescriptorStatistics { + if x != nil { + return x.Statistics + } + return nil +} + +func (x *RepoMeta) GetSignatures() map[string]*ManifestSignatures { + if x != nil { + return x.Signatures + } + return nil +} + +func (x *RepoMeta) GetReferrers() map[string]*ReferrersInfo { + if x != nil { + return x.Referrers + } + return nil +} + +func (x *RepoMeta) GetIsStarred() bool { + if x != nil { + return x.IsStarred + } + return false +} + +func (x *RepoMeta) GetIsBookmarked() bool { + if x != nil { + return x.IsBookmarked + } + return false +} + +func (x *RepoMeta) GetRank() int32 { + if x != nil { + return x.Rank + } + return 0 +} + +func (x *RepoMeta) GetStars() int32 { + if x != nil { + return x.Stars + } + return 0 +} + +func (x *RepoMeta) GetSize() int32 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *RepoMeta) GetVendors() []string { + if x != nil { + return x.Vendors + } + return nil +} + +func (x *RepoMeta) GetPlatforms() []*Platform { + if x != nil { + return x.Platforms + } + return nil +} + +func (x *RepoMeta) GetLastUpdatedImage() *RepoLastUpdatedImage { + if x != nil { + return x.LastUpdatedImage + } + return nil +} + +type RepoBlobs struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + Blobs map[string]*BlobInfo `protobuf:"bytes,2,rep,name=Blobs,proto3" json:"Blobs,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *RepoBlobs) Reset() { + *x = RepoBlobs{} + if protoimpl.UnsafeEnabled { + mi := &file_meta_meta_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RepoBlobs) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RepoBlobs) ProtoMessage() {} + +func (x *RepoBlobs) ProtoReflect() protoreflect.Message { + mi := &file_meta_meta_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RepoBlobs.ProtoReflect.Descriptor instead. +func (*RepoBlobs) Descriptor() ([]byte, []int) { + return file_meta_meta_proto_rawDescGZIP(), []int{6} +} + +func (x *RepoBlobs) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *RepoBlobs) GetBlobs() map[string]*BlobInfo { + if x != nil { + return x.Blobs + } + return nil +} + +// for example this is a manifest and it has a config, and layers +// or index and has manifests +type BlobInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Size int64 `protobuf:"varint,1,opt,name=Size,proto3" json:"Size,omitempty"` + Vendors []string `protobuf:"bytes,2,rep,name=Vendors,proto3" json:"Vendors,omitempty"` + SubBlobs []string `protobuf:"bytes,3,rep,name=SubBlobs,proto3" json:"SubBlobs,omitempty"` + Platforms []*Platform `protobuf:"bytes,4,rep,name=Platforms,proto3" json:"Platforms,omitempty"` + LastUpdated *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=LastUpdated,proto3,oneof" json:"LastUpdated,omitempty"` +} + +func (x *BlobInfo) Reset() { + *x = BlobInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_meta_meta_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BlobInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BlobInfo) ProtoMessage() {} + +func (x *BlobInfo) ProtoReflect() protoreflect.Message { + mi := &file_meta_meta_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BlobInfo.ProtoReflect.Descriptor instead. +func (*BlobInfo) Descriptor() ([]byte, []int) { + return file_meta_meta_proto_rawDescGZIP(), []int{7} +} + +func (x *BlobInfo) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *BlobInfo) GetVendors() []string { + if x != nil { + return x.Vendors + } + return nil +} + +func (x *BlobInfo) GetSubBlobs() []string { + if x != nil { + return x.SubBlobs + } + return nil +} + +func (x *BlobInfo) GetPlatforms() []*Platform { + if x != nil { + return x.Platforms + } + return nil +} + +func (x *BlobInfo) GetLastUpdated() *timestamppb.Timestamp { + if x != nil { + return x.LastUpdated + } + return nil +} + +type DescriptorStatistics struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + DownloadCount int32 `protobuf:"varint,1,opt,name=DownloadCount,proto3" json:"DownloadCount,omitempty"` +} + +func (x *DescriptorStatistics) Reset() { + *x = DescriptorStatistics{} + if protoimpl.UnsafeEnabled { + mi := &file_meta_meta_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *DescriptorStatistics) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DescriptorStatistics) ProtoMessage() {} + +func (x *DescriptorStatistics) ProtoReflect() protoreflect.Message { + mi := &file_meta_meta_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DescriptorStatistics.ProtoReflect.Descriptor instead. +func (*DescriptorStatistics) Descriptor() ([]byte, []int) { + return file_meta_meta_proto_rawDescGZIP(), []int{8} +} + +func (x *DescriptorStatistics) GetDownloadCount() int32 { + if x != nil { + return x.DownloadCount + } + return 0 +} + +type ReferrersInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + List []*ReferrerInfo `protobuf:"bytes,1,rep,name=list,proto3" json:"list,omitempty"` +} + +func (x *ReferrersInfo) Reset() { + *x = ReferrersInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_meta_meta_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ReferrersInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReferrersInfo) ProtoMessage() {} + +func (x *ReferrersInfo) ProtoReflect() protoreflect.Message { + mi := &file_meta_meta_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReferrersInfo.ProtoReflect.Descriptor instead. +func (*ReferrersInfo) Descriptor() ([]byte, []int) { + return file_meta_meta_proto_rawDescGZIP(), []int{9} +} + +func (x *ReferrersInfo) GetList() []*ReferrerInfo { + if x != nil { + return x.List + } + return nil +} + +type ReferrerInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Digest string `protobuf:"bytes,1,opt,name=Digest,proto3" json:"Digest,omitempty"` + Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` + MediaType string `protobuf:"bytes,3,opt,name=MediaType,proto3" json:"MediaType,omitempty"` + ArtifactType string `protobuf:"bytes,4,opt,name=ArtifactType,proto3" json:"ArtifactType,omitempty"` + Size int64 `protobuf:"varint,5,opt,name=Size,proto3" json:"Size,omitempty"` + Annotations map[string]string `protobuf:"bytes,6,rep,name=Annotations,proto3" json:"Annotations,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *ReferrerInfo) Reset() { + *x = ReferrerInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_meta_meta_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ReferrerInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ReferrerInfo) ProtoMessage() {} + +func (x *ReferrerInfo) ProtoReflect() protoreflect.Message { + mi := &file_meta_meta_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ReferrerInfo.ProtoReflect.Descriptor instead. +func (*ReferrerInfo) Descriptor() ([]byte, []int) { + return file_meta_meta_proto_rawDescGZIP(), []int{10} +} + +func (x *ReferrerInfo) GetDigest() string { + if x != nil { + return x.Digest + } + return "" +} + +func (x *ReferrerInfo) GetCount() int64 { + if x != nil { + return x.Count + } + return 0 +} + +func (x *ReferrerInfo) GetMediaType() string { + if x != nil { + return x.MediaType + } + return "" +} + +func (x *ReferrerInfo) GetArtifactType() string { + if x != nil { + return x.ArtifactType + } + return "" +} + +func (x *ReferrerInfo) GetSize() int64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *ReferrerInfo) GetAnnotations() map[string]string { + if x != nil { + return x.Annotations + } + return nil +} + +type ManifestSignatures struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Map map[string]*SignaturesInfo `protobuf:"bytes,1,rep,name=map,proto3" json:"map,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *ManifestSignatures) Reset() { + *x = ManifestSignatures{} + if protoimpl.UnsafeEnabled { + mi := &file_meta_meta_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ManifestSignatures) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ManifestSignatures) ProtoMessage() {} + +func (x *ManifestSignatures) ProtoReflect() protoreflect.Message { + mi := &file_meta_meta_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ManifestSignatures.ProtoReflect.Descriptor instead. +func (*ManifestSignatures) Descriptor() ([]byte, []int) { + return file_meta_meta_proto_rawDescGZIP(), []int{11} +} + +func (x *ManifestSignatures) GetMap() map[string]*SignaturesInfo { + if x != nil { + return x.Map + } + return nil +} + +type SignaturesInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + List []*SignatureInfo `protobuf:"bytes,1,rep,name=list,proto3" json:"list,omitempty"` +} + +func (x *SignaturesInfo) Reset() { + *x = SignaturesInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_meta_meta_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SignaturesInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SignaturesInfo) ProtoMessage() {} + +func (x *SignaturesInfo) ProtoReflect() protoreflect.Message { + mi := &file_meta_meta_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SignaturesInfo.ProtoReflect.Descriptor instead. +func (*SignaturesInfo) Descriptor() ([]byte, []int) { + return file_meta_meta_proto_rawDescGZIP(), []int{12} +} + +func (x *SignaturesInfo) GetList() []*SignatureInfo { + if x != nil { + return x.List + } + return nil +} + +type SignatureInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + SignatureManifestDigest string `protobuf:"bytes,1,opt,name=SignatureManifestDigest,proto3" json:"SignatureManifestDigest,omitempty"` + LayersInfo []*LayersInfo `protobuf:"bytes,2,rep,name=LayersInfo,proto3" json:"LayersInfo,omitempty"` +} + +func (x *SignatureInfo) Reset() { + *x = SignatureInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_meta_meta_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *SignatureInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SignatureInfo) ProtoMessage() {} + +func (x *SignatureInfo) ProtoReflect() protoreflect.Message { + mi := &file_meta_meta_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SignatureInfo.ProtoReflect.Descriptor instead. +func (*SignatureInfo) Descriptor() ([]byte, []int) { + return file_meta_meta_proto_rawDescGZIP(), []int{13} +} + +func (x *SignatureInfo) GetSignatureManifestDigest() string { + if x != nil { + return x.SignatureManifestDigest + } + return "" +} + +func (x *SignatureInfo) GetLayersInfo() []*LayersInfo { + if x != nil { + return x.LayersInfo + } + return nil +} + +type LayersInfo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + LayerDigest string `protobuf:"bytes,1,opt,name=LayerDigest,proto3" json:"LayerDigest,omitempty"` + LayerContent []byte `protobuf:"bytes,2,opt,name=LayerContent,proto3" json:"LayerContent,omitempty"` + SignatureKey string `protobuf:"bytes,3,opt,name=SignatureKey,proto3" json:"SignatureKey,omitempty"` + Signer string `protobuf:"bytes,4,opt,name=Signer,proto3" json:"Signer,omitempty"` + Date *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=Date,proto3" json:"Date,omitempty"` +} + +func (x *LayersInfo) Reset() { + *x = LayersInfo{} + if protoimpl.UnsafeEnabled { + mi := &file_meta_meta_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LayersInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LayersInfo) ProtoMessage() {} + +func (x *LayersInfo) ProtoReflect() protoreflect.Message { + mi := &file_meta_meta_proto_msgTypes[14] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LayersInfo.ProtoReflect.Descriptor instead. +func (*LayersInfo) Descriptor() ([]byte, []int) { + return file_meta_meta_proto_rawDescGZIP(), []int{14} +} + +func (x *LayersInfo) GetLayerDigest() string { + if x != nil { + return x.LayerDigest + } + return "" +} + +func (x *LayersInfo) GetLayerContent() []byte { + if x != nil { + return x.LayerContent + } + return nil +} + +func (x *LayersInfo) GetSignatureKey() string { + if x != nil { + return x.SignatureKey + } + return "" +} + +func (x *LayersInfo) GetSigner() string { + if x != nil { + return x.Signer + } + return "" +} + +func (x *LayersInfo) GetDate() *timestamppb.Timestamp { + if x != nil { + return x.Date + } + return nil +} + +var File_meta_meta_proto protoreflect.FileDescriptor + +var file_meta_meta_proto_rawDesc = []byte{ + 0x0a, 0x0f, 0x6d, 0x65, 0x74, 0x61, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x12, 0x07, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x76, 0x31, 0x1a, 0x10, 0x6f, 0x63, 0x69, 0x2f, + 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x12, 0x6f, 0x63, + 0x69, 0x2f, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x1a, 0x0f, 0x6f, 0x63, 0x69, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x1a, 0x13, 0x6f, 0x63, 0x69, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x14, 0x6f, 0x63, 0x69, 0x2f, 0x64, 0x65, 0x73, 0x63, + 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x45, 0x0a, 0x0d, + 0x54, 0x61, 0x67, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x12, 0x1c, 0x0a, + 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, + 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, 0x67, + 0x65, 0x73, 0x74, 0x22, 0x97, 0x01, 0x0a, 0x09, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x4d, 0x65, 0x74, + 0x61, 0x12, 0x1c, 0x0a, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, + 0x33, 0x0a, 0x09, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x4d, 0x61, 0x6e, + 0x69, 0x66, 0x65, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x52, 0x09, 0x4d, 0x61, 0x6e, 0x69, 0x66, + 0x65, 0x73, 0x74, 0x73, 0x12, 0x2d, 0x0a, 0x05, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x49, 0x6e, + 0x64, 0x65, 0x78, 0x4d, 0x65, 0x74, 0x61, 0x48, 0x00, 0x52, 0x05, 0x49, 0x6e, 0x64, 0x65, 0x78, + 0x88, 0x01, 0x01, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x22, 0x8f, 0x01, + 0x0a, 0x0c, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x16, + 0x0a, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x2c, 0x0a, 0x08, 0x4d, 0x61, + 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x6f, + 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52, 0x08, + 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x25, 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, + 0x31, 0x2e, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x52, 0x06, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, + 0x5c, 0x0a, 0x09, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x16, 0x0a, 0x06, + 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, + 0x67, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x23, 0x0a, 0x05, 0x49, 0x6e, 0x64, 0x65, + 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, + 0x2e, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x05, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x22, 0xb1, 0x01, + 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6f, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x41, 0x0a, 0x0b, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, + 0x64, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, 0x0b, 0x4c, 0x61, 0x73, 0x74, 0x55, + 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x88, 0x01, 0x01, 0x12, 0x1c, 0x0a, 0x09, 0x4d, 0x65, 0x64, + 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x4d, 0x65, + 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, + 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, + 0x10, 0x0a, 0x03, 0x54, 0x61, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x54, 0x61, + 0x67, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x64, 0x22, 0xa5, 0x07, 0x0a, 0x08, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x12, 0x12, + 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, + 0x6d, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x54, 0x61, 0x67, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1b, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4d, + 0x65, 0x74, 0x61, 0x2e, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x54, + 0x61, 0x67, 0x73, 0x12, 0x41, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x76, + 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, + 0x73, 0x74, 0x69, 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x53, 0x74, 0x61, 0x74, + 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x41, 0x0a, 0x0a, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, + 0x75, 0x72, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x6d, 0x65, 0x74, + 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x2e, 0x53, 0x69, + 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, 0x53, + 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x3e, 0x0a, 0x09, 0x52, 0x65, 0x66, + 0x65, 0x72, 0x72, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x6d, + 0x65, 0x74, 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x2e, + 0x52, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x09, + 0x52, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x72, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x49, 0x73, 0x53, + 0x74, 0x61, 0x72, 0x72, 0x65, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x49, 0x73, + 0x53, 0x74, 0x61, 0x72, 0x72, 0x65, 0x64, 0x12, 0x22, 0x0a, 0x0c, 0x49, 0x73, 0x42, 0x6f, 0x6f, + 0x6b, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x49, + 0x73, 0x42, 0x6f, 0x6f, 0x6b, 0x6d, 0x61, 0x72, 0x6b, 0x65, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x52, + 0x61, 0x6e, 0x6b, 0x18, 0x08, 0x20, 0x01, 0x28, 0x05, 0x52, 0x04, 0x52, 0x61, 0x6e, 0x6b, 0x12, + 0x14, 0x0a, 0x05, 0x53, 0x74, 0x61, 0x72, 0x73, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, + 0x53, 0x74, 0x61, 0x72, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x0a, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x56, 0x65, 0x6e, + 0x64, 0x6f, 0x72, 0x73, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x56, 0x65, 0x6e, 0x64, + 0x6f, 0x72, 0x73, 0x12, 0x2e, 0x0a, 0x09, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x73, + 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x2e, + 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x09, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, + 0x72, 0x6d, 0x73, 0x12, 0x4e, 0x0a, 0x10, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, + 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x4c, 0x61, 0x73, 0x74, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x48, 0x00, 0x52, 0x10, + 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, + 0x88, 0x01, 0x01, 0x1a, 0x4f, 0x0a, 0x09, 0x54, 0x61, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x54, 0x61, 0x67, 0x44, + 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x5c, 0x0a, 0x0f, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, + 0x63, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x33, 0x0a, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x5f, + 0x76, 0x31, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, + 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x1a, 0x5a, 0x0a, 0x0f, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, + 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x31, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x76, 0x31, + 0x2e, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, + 0x72, 0x65, 0x73, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x1a, 0x54, + 0x0a, 0x0e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x16, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, + 0x72, 0x72, 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x3a, 0x02, 0x38, 0x01, 0x42, 0x13, 0x0a, 0x11, 0x5f, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x64, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x22, 0xa1, 0x01, 0x0a, 0x09, 0x52, 0x65, + 0x70, 0x6f, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x33, 0x0a, 0x05, 0x42, + 0x6c, 0x6f, 0x62, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x6d, 0x65, 0x74, + 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x70, 0x6f, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x2e, 0x42, + 0x6c, 0x6f, 0x62, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x05, 0x42, 0x6c, 0x6f, 0x62, 0x73, + 0x1a, 0x4b, 0x0a, 0x0a, 0x42, 0x6c, 0x6f, 0x62, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x27, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x11, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x42, 0x6c, 0x6f, 0x62, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xd7, 0x01, + 0x0a, 0x08, 0x42, 0x6c, 0x6f, 0x62, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, + 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x18, + 0x0a, 0x07, 0x56, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x07, 0x56, 0x65, 0x6e, 0x64, 0x6f, 0x72, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x53, 0x75, 0x62, 0x42, + 0x6c, 0x6f, 0x62, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x53, 0x75, 0x62, 0x42, + 0x6c, 0x6f, 0x62, 0x73, 0x12, 0x2e, 0x0a, 0x09, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, + 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, + 0x2e, 0x50, 0x6c, 0x61, 0x74, 0x66, 0x6f, 0x72, 0x6d, 0x52, 0x09, 0x50, 0x6c, 0x61, 0x74, 0x66, + 0x6f, 0x72, 0x6d, 0x73, 0x12, 0x41, 0x0a, 0x0b, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x48, 0x00, 0x52, 0x0b, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x64, 0x88, 0x01, 0x01, 0x42, 0x0e, 0x0a, 0x0c, 0x5f, 0x4c, 0x61, 0x73, 0x74, + 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x22, 0x3c, 0x0a, 0x14, 0x44, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, + 0x24, 0x0a, 0x0d, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, + 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x3a, 0x0a, 0x0d, 0x52, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, + 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x29, 0x0a, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x52, + 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x72, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x6c, 0x69, 0x73, + 0x74, 0x22, 0x9c, 0x02, 0x0a, 0x0c, 0x52, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x72, 0x49, 0x6e, + 0x66, 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x43, 0x6f, + 0x75, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x43, 0x6f, 0x75, 0x6e, 0x74, + 0x12, 0x1c, 0x0a, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x09, 0x4d, 0x65, 0x64, 0x69, 0x61, 0x54, 0x79, 0x70, 0x65, 0x12, 0x22, + 0x0a, 0x0c, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, + 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x04, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x6d, 0x65, + 0x74, 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x52, 0x65, 0x66, 0x65, 0x72, 0x72, 0x65, 0x72, 0x49, 0x6e, + 0x66, 0x6f, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, + 0x74, 0x72, 0x79, 0x52, 0x0b, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x1a, 0x3e, 0x0a, 0x10, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0x9d, 0x01, 0x0a, 0x12, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x53, 0x69, 0x67, + 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x12, 0x36, 0x0a, 0x03, 0x6d, 0x61, 0x70, 0x18, 0x01, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x4d, + 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x73, 0x2e, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x03, 0x6d, 0x61, 0x70, 0x1a, + 0x4f, 0x0a, 0x08, 0x4d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x2d, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6d, + 0x65, 0x74, 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, + 0x22, 0x3c, 0x0a, 0x0e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x73, 0x49, 0x6e, + 0x66, 0x6f, 0x12, 0x2a, 0x0a, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x16, 0x2e, 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x22, 0x7e, + 0x0a, 0x0d, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, + 0x38, 0x0a, 0x17, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x4d, 0x61, 0x6e, 0x69, + 0x66, 0x65, 0x73, 0x74, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x17, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, 0x4d, 0x61, 0x6e, 0x69, 0x66, + 0x65, 0x73, 0x74, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x33, 0x0a, 0x0a, 0x4c, 0x61, 0x79, + 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, + 0x6d, 0x65, 0x74, 0x61, 0x5f, 0x76, 0x31, 0x2e, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x0a, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0xbe, + 0x01, 0x0a, 0x0a, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x20, 0x0a, + 0x0b, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x44, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, + 0x22, 0x0a, 0x0c, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0c, 0x4c, 0x61, 0x79, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x12, 0x22, 0x0a, 0x0c, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x74, 0x75, 0x72, 0x65, + 0x4b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x53, 0x69, 0x67, 0x6e, 0x61, + 0x74, 0x75, 0x72, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x65, + 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x72, 0x12, + 0x2e, 0x0a, 0x04, 0x44, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x04, 0x44, 0x61, 0x74, 0x65, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_meta_meta_proto_rawDescOnce sync.Once + file_meta_meta_proto_rawDescData = file_meta_meta_proto_rawDesc +) + +func file_meta_meta_proto_rawDescGZIP() []byte { + file_meta_meta_proto_rawDescOnce.Do(func() { + file_meta_meta_proto_rawDescData = protoimpl.X.CompressGZIP(file_meta_meta_proto_rawDescData) + }) + return file_meta_meta_proto_rawDescData +} + +var file_meta_meta_proto_msgTypes = make([]protoimpl.MessageInfo, 22) +var file_meta_meta_proto_goTypes = []interface{}{ + (*TagDescriptor)(nil), // 0: meta_v1.TagDescriptor + (*ImageMeta)(nil), // 1: meta_v1.ImageMeta + (*ManifestMeta)(nil), // 2: meta_v1.ManifestMeta + (*IndexMeta)(nil), // 3: meta_v1.IndexMeta + (*RepoLastUpdatedImage)(nil), // 4: meta_v1.RepoLastUpdatedImage + (*RepoMeta)(nil), // 5: meta_v1.RepoMeta + (*RepoBlobs)(nil), // 6: meta_v1.RepoBlobs + (*BlobInfo)(nil), // 7: meta_v1.BlobInfo + (*DescriptorStatistics)(nil), // 8: meta_v1.DescriptorStatistics + (*ReferrersInfo)(nil), // 9: meta_v1.ReferrersInfo + (*ReferrerInfo)(nil), // 10: meta_v1.ReferrerInfo + (*ManifestSignatures)(nil), // 11: meta_v1.ManifestSignatures + (*SignaturesInfo)(nil), // 12: meta_v1.SignaturesInfo + (*SignatureInfo)(nil), // 13: meta_v1.SignatureInfo + (*LayersInfo)(nil), // 14: meta_v1.LayersInfo + nil, // 15: meta_v1.RepoMeta.TagsEntry + nil, // 16: meta_v1.RepoMeta.StatisticsEntry + nil, // 17: meta_v1.RepoMeta.SignaturesEntry + nil, // 18: meta_v1.RepoMeta.ReferrersEntry + nil, // 19: meta_v1.RepoBlobs.BlobsEntry + nil, // 20: meta_v1.ReferrerInfo.AnnotationsEntry + nil, // 21: meta_v1.ManifestSignatures.MapEntry + (*Manifest)(nil), // 22: oci_v1.Manifest + (*Image)(nil), // 23: oci_v1.Image + (*Index)(nil), // 24: oci_v1.Index + (*timestamppb.Timestamp)(nil), // 25: google.protobuf.Timestamp + (*Platform)(nil), // 26: oci_v1.Platform +} +var file_meta_meta_proto_depIdxs = []int32{ + 2, // 0: meta_v1.ImageMeta.Manifests:type_name -> meta_v1.ManifestMeta + 3, // 1: meta_v1.ImageMeta.Index:type_name -> meta_v1.IndexMeta + 22, // 2: meta_v1.ManifestMeta.Manifest:type_name -> oci_v1.Manifest + 23, // 3: meta_v1.ManifestMeta.Config:type_name -> oci_v1.Image + 24, // 4: meta_v1.IndexMeta.Index:type_name -> oci_v1.Index + 25, // 5: meta_v1.RepoLastUpdatedImage.LastUpdated:type_name -> google.protobuf.Timestamp + 15, // 6: meta_v1.RepoMeta.Tags:type_name -> meta_v1.RepoMeta.TagsEntry + 16, // 7: meta_v1.RepoMeta.Statistics:type_name -> meta_v1.RepoMeta.StatisticsEntry + 17, // 8: meta_v1.RepoMeta.Signatures:type_name -> meta_v1.RepoMeta.SignaturesEntry + 18, // 9: meta_v1.RepoMeta.Referrers:type_name -> meta_v1.RepoMeta.ReferrersEntry + 26, // 10: meta_v1.RepoMeta.Platforms:type_name -> oci_v1.Platform + 4, // 11: meta_v1.RepoMeta.LastUpdatedImage:type_name -> meta_v1.RepoLastUpdatedImage + 19, // 12: meta_v1.RepoBlobs.Blobs:type_name -> meta_v1.RepoBlobs.BlobsEntry + 26, // 13: meta_v1.BlobInfo.Platforms:type_name -> oci_v1.Platform + 25, // 14: meta_v1.BlobInfo.LastUpdated:type_name -> google.protobuf.Timestamp + 10, // 15: meta_v1.ReferrersInfo.list:type_name -> meta_v1.ReferrerInfo + 20, // 16: meta_v1.ReferrerInfo.Annotations:type_name -> meta_v1.ReferrerInfo.AnnotationsEntry + 21, // 17: meta_v1.ManifestSignatures.map:type_name -> meta_v1.ManifestSignatures.MapEntry + 13, // 18: meta_v1.SignaturesInfo.list:type_name -> meta_v1.SignatureInfo + 14, // 19: meta_v1.SignatureInfo.LayersInfo:type_name -> meta_v1.LayersInfo + 25, // 20: meta_v1.LayersInfo.Date:type_name -> google.protobuf.Timestamp + 0, // 21: meta_v1.RepoMeta.TagsEntry.value:type_name -> meta_v1.TagDescriptor + 8, // 22: meta_v1.RepoMeta.StatisticsEntry.value:type_name -> meta_v1.DescriptorStatistics + 11, // 23: meta_v1.RepoMeta.SignaturesEntry.value:type_name -> meta_v1.ManifestSignatures + 9, // 24: meta_v1.RepoMeta.ReferrersEntry.value:type_name -> meta_v1.ReferrersInfo + 7, // 25: meta_v1.RepoBlobs.BlobsEntry.value:type_name -> meta_v1.BlobInfo + 12, // 26: meta_v1.ManifestSignatures.MapEntry.value:type_name -> meta_v1.SignaturesInfo + 27, // [27:27] is the sub-list for method output_type + 27, // [27:27] is the sub-list for method input_type + 27, // [27:27] is the sub-list for extension type_name + 27, // [27:27] is the sub-list for extension extendee + 0, // [0:27] is the sub-list for field type_name +} + +func init() { file_meta_meta_proto_init() } +func file_meta_meta_proto_init() { + if File_meta_meta_proto != nil { + return + } + file_oci_config_proto_init() + file_oci_manifest_proto_init() + file_oci_index_proto_init() + file_oci_descriptor_proto_init() + if !protoimpl.UnsafeEnabled { + file_meta_meta_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*TagDescriptor); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_meta_meta_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ImageMeta); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_meta_meta_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ManifestMeta); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_meta_meta_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*IndexMeta); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_meta_meta_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RepoLastUpdatedImage); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_meta_meta_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RepoMeta); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_meta_meta_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RepoBlobs); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_meta_meta_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BlobInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_meta_meta_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DescriptorStatistics); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_meta_meta_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReferrersInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_meta_meta_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ReferrerInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_meta_meta_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ManifestSignatures); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_meta_meta_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SignaturesInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_meta_meta_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SignatureInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_meta_meta_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LayersInfo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_meta_meta_proto_msgTypes[1].OneofWrappers = []interface{}{} + file_meta_meta_proto_msgTypes[4].OneofWrappers = []interface{}{} + file_meta_meta_proto_msgTypes[5].OneofWrappers = []interface{}{} + file_meta_meta_proto_msgTypes[7].OneofWrappers = []interface{}{} + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_meta_meta_proto_rawDesc, + NumEnums: 0, + NumMessages: 22, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_meta_meta_proto_goTypes, + DependencyIndexes: file_meta_meta_proto_depIdxs, + MessageInfos: file_meta_meta_proto_msgTypes, + }.Build() + File_meta_meta_proto = out.File + file_meta_meta_proto_rawDesc = nil + file_meta_meta_proto_goTypes = nil + file_meta_meta_proto_depIdxs = nil +} diff --git a/pkg/meta/proto/gen/versioned.pb.go b/pkg/meta/proto/gen/versioned.pb.go new file mode 100644 index 00000000..df7d9272 --- /dev/null +++ b/pkg/meta/proto/gen/versioned.pb.go @@ -0,0 +1,142 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.31.0 +// protoc v4.24.4 +// source: oci/versioned.proto + +package gen + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Versioned struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + SchemaVersion int32 `protobuf:"varint,1,opt,name=SchemaVersion,proto3" json:"SchemaVersion,omitempty"` +} + +func (x *Versioned) Reset() { + *x = Versioned{} + if protoimpl.UnsafeEnabled { + mi := &file_oci_versioned_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Versioned) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Versioned) ProtoMessage() {} + +func (x *Versioned) ProtoReflect() protoreflect.Message { + mi := &file_oci_versioned_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Versioned.ProtoReflect.Descriptor instead. +func (*Versioned) Descriptor() ([]byte, []int) { + return file_oci_versioned_proto_rawDescGZIP(), []int{0} +} + +func (x *Versioned) GetSchemaVersion() int32 { + if x != nil { + return x.SchemaVersion + } + return 0 +} + +var File_oci_versioned_proto protoreflect.FileDescriptor + +var file_oci_versioned_proto_rawDesc = []byte{ + 0x0a, 0x13, 0x6f, 0x63, 0x69, 0x2f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x6f, 0x63, 0x69, 0x5f, 0x76, 0x31, 0x22, 0x31, 0x0a, + 0x09, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x12, 0x24, 0x0a, 0x0d, 0x53, 0x63, + 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x0d, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_oci_versioned_proto_rawDescOnce sync.Once + file_oci_versioned_proto_rawDescData = file_oci_versioned_proto_rawDesc +) + +func file_oci_versioned_proto_rawDescGZIP() []byte { + file_oci_versioned_proto_rawDescOnce.Do(func() { + file_oci_versioned_proto_rawDescData = protoimpl.X.CompressGZIP(file_oci_versioned_proto_rawDescData) + }) + return file_oci_versioned_proto_rawDescData +} + +var file_oci_versioned_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_oci_versioned_proto_goTypes = []interface{}{ + (*Versioned)(nil), // 0: oci_v1.Versioned +} +var file_oci_versioned_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_oci_versioned_proto_init() } +func file_oci_versioned_proto_init() { + if File_oci_versioned_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_oci_versioned_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Versioned); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_oci_versioned_proto_rawDesc, + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_oci_versioned_proto_goTypes, + DependencyIndexes: file_oci_versioned_proto_depIdxs, + MessageInfos: file_oci_versioned_proto_msgTypes, + }.Build() + File_oci_versioned_proto = out.File + file_oci_versioned_proto_rawDesc = nil + file_oci_versioned_proto_goTypes = nil + file_oci_versioned_proto_depIdxs = nil +} diff --git a/pkg/meta/proto/meta/meta.proto b/pkg/meta/proto/meta/meta.proto new file mode 100644 index 00000000..579e5cf5 --- /dev/null +++ b/pkg/meta/proto/meta/meta.proto @@ -0,0 +1,115 @@ +syntax = "proto3"; +package meta_v1; + +import "oci/config.proto"; +import "oci/manifest.proto"; +import "oci/index.proto"; +import "oci/timestamp.proto"; +import "oci/descriptor.proto"; + +message TagDescriptor { + string MediaType = 1; + string Digest = 2; +} + +message ImageMeta { + string MediaType = 1; + repeated ManifestMeta Manifests = 2; + optional IndexMeta Index = 3; +} + +message ManifestMeta { + string Digest = 1; + int64 Size = 2; + oci_v1.Manifest Manifest = 3; + oci_v1.Image Config = 4; +} + +message IndexMeta { + string Digest = 1; + int64 Size = 2; + oci_v1.Index Index = 3; +} + +message RepoLastUpdatedImage { + optional google.protobuf.Timestamp LastUpdated = 1; + string MediaType = 2; + string Digest = 3; + string Tag = 4; +} + +message RepoMeta { + string Name = 1; + map Tags = 2; + + map Statistics = 3; + map Signatures = 4; + map Referrers = 5; + + bool IsStarred = 6; + bool IsBookmarked = 7; + int32 Rank = 8; + + int32 Stars = 9; + + int32 Size = 10; + repeated string Vendors = 11; + repeated oci_v1.Platform Platforms = 12; + optional RepoLastUpdatedImage LastUpdatedImage = 13; +} + +message RepoBlobs { + string Name = 1; + map Blobs = 2; +} + +// for example this is a manifest and it has a config, and layers +// or index and has manifests +message BlobInfo { + int64 Size = 1; + repeated string Vendors = 2; + repeated string SubBlobs = 3; + repeated oci_v1.Platform Platforms = 4; + + optional google.protobuf.Timestamp LastUpdated = 5; +} + +message DescriptorStatistics { + int32 DownloadCount = 1; +} + +message ReferrersInfo { + repeated ReferrerInfo list = 1; +} + +message ReferrerInfo { + string Digest = 1; + int64 Count = 2; + string MediaType = 3; + string ArtifactType = 4; + int64 Size = 5; + + map Annotations = 6; +} + +message ManifestSignatures { + map map = 1; +} + +message SignaturesInfo { + repeated SignatureInfo list = 1; +} + +message SignatureInfo { + string SignatureManifestDigest = 1; + repeated LayersInfo LayersInfo = 2; +} + +message LayersInfo { + string LayerDigest = 1; + bytes LayerContent = 2; + string SignatureKey = 3; + string Signer = 4; + + google.protobuf.Timestamp Date = 5; +} \ No newline at end of file diff --git a/pkg/meta/proto/oci/config.proto b/pkg/meta/proto/oci/config.proto new file mode 100644 index 00000000..7bec2d9a --- /dev/null +++ b/pkg/meta/proto/oci/config.proto @@ -0,0 +1,45 @@ +syntax = "proto3"; +package oci_v1; + +import "oci/timestamp.proto"; +import "oci/descriptor.proto"; + +// https://github.com/opencontainers/image-spec/blob/main/specs-go/v1/config.go + +message Image { + optional google.protobuf.Timestamp Created = 1; + optional string Author = 2; + Platform Platform = 3; + optional ImageConfig Config = 4; + optional RootFS RootFS = 5; + repeated History History = 6; +} + +message ImageConfig { + map ExposedPorts = 1; + map Volumes = 2; + map Labels = 3; + string User = 4; + repeated string Env = 5; + repeated string Entrypoint = 6; + repeated string Cmd = 7; + optional string WorkingDir = 8; + optional string StopSignal = 9; + bool ArgsEscaped = 10; +} + +message RootFS { + string Type = 1; + repeated string DiffIDs = 2; +} + +message History { + optional google.protobuf.Timestamp Created = 1; + + optional string CreatedBy = 2; + optional string Author = 3; + optional string Comment = 4; + optional bool EmptyLayer = 5; +} + +message EmptyMessage{} \ No newline at end of file diff --git a/pkg/meta/proto/oci/descriptor.proto b/pkg/meta/proto/oci/descriptor.proto new file mode 100644 index 00000000..0e99d324 --- /dev/null +++ b/pkg/meta/proto/oci/descriptor.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; +package oci_v1; + +// https://github.com/opencontainers/image-spec/blob/main/specs-go/v1/descriptor.go + +message Descriptor { + string MediaType = 1; + string Digest = 2; + int64 Size = 3; + repeated string URLs = 4; + bytes Data = 5; + optional Platform Platform = 6; + optional string ArtifactType = 7; + map Annotations = 8; +} + +message Platform { + string Architecture = 1; + string OS = 2; + optional string OSVersion = 3; + repeated string OSFeatures = 4; + optional string Variant = 5; +} \ No newline at end of file diff --git a/pkg/meta/proto/oci/index.proto b/pkg/meta/proto/oci/index.proto new file mode 100644 index 00000000..1740ef33 --- /dev/null +++ b/pkg/meta/proto/oci/index.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; +package oci_v1; + +import "oci/descriptor.proto"; +import "oci/versioned.proto"; + +// https://github.com/opencontainers/image-spec/blob/main/specs-go/v1/index.go + +message Index { + Versioned Versioned = 1; + optional string MediaType = 2; + optional string ArtifactType = 3; + repeated Descriptor Manifests = 4; + optional Descriptor Subject = 5; + map Annotations = 6; +} \ No newline at end of file diff --git a/pkg/meta/proto/oci/manifest.proto b/pkg/meta/proto/oci/manifest.proto new file mode 100644 index 00000000..a8c79312 --- /dev/null +++ b/pkg/meta/proto/oci/manifest.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; +package oci_v1; + +import "oci/descriptor.proto"; +import "oci/versioned.proto"; + +// https://github.com/opencontainers/image-spec/blob/main/specs-go/v1/manifest.go + +message Manifest { + Versioned Versioned = 1; + optional string MediaType = 2; + optional string ArtifactType = 3; + Descriptor Config = 4; + repeated Descriptor Layers = 5; + optional Descriptor Subject = 6; + map Annotations = 7; +} \ No newline at end of file diff --git a/pkg/meta/proto/oci/timestamp.proto b/pkg/meta/proto/oci/timestamp.proto new file mode 100644 index 00000000..fd0bc07d --- /dev/null +++ b/pkg/meta/proto/oci/timestamp.proto @@ -0,0 +1,144 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/protobuf/types/known/timestamppb"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TimestampProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; + +// A Timestamp represents a point in time independent of any time zone or local +// calendar, encoded as a count of seconds and fractions of seconds at +// nanosecond resolution. The count is relative to an epoch at UTC midnight on +// January 1, 1970, in the proleptic Gregorian calendar which extends the +// Gregorian calendar backwards to year one. +// +// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap +// second table is needed for interpretation, using a [24-hour linear +// smear](https://developers.google.com/time/smear). +// +// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By +// restricting to that range, we ensure that we can convert to and from [RFC +// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. +// +// # Examples +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// Example 5: Compute Timestamp from Java `Instant.now()`. +// +// Instant now = Instant.now(); +// +// Timestamp timestamp = +// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) +// .setNanos(now.getNano()).build(); +// +// Example 6: Compute Timestamp from current time in Python. +// +// timestamp = Timestamp() +// timestamp.GetCurrentTime() +// +// # JSON Mapping +// +// In JSON format, the Timestamp type is encoded as a string in the +// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +// where {year} is always expressed using four digits while {month}, {day}, +// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +// is required. A proto3 JSON serializer should always use UTC (as indicated by +// "Z") when printing the Timestamp type and a proto3 JSON parser should be +// able to accept both UTC and other timezones (as indicated by an offset). +// +// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +// 01:30 UTC on January 15, 2017. +// +// In JavaScript, one can convert a Date object to this format using the +// standard +// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) +// method. In Python, a standard `datetime.datetime` object can be converted +// to this format using +// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with +// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use +// the Joda Time's [`ISODateTimeFormat.dateTime()`]( +// http://joda-time.sourceforge.net/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime() +// ) to obtain a formatter capable of generating timestamps in this format. +// +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} diff --git a/pkg/meta/proto/oci/versioned.proto b/pkg/meta/proto/oci/versioned.proto new file mode 100644 index 00000000..0cbb10f3 --- /dev/null +++ b/pkg/meta/proto/oci/versioned.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; +package oci_v1; + +// https://github.com/opencontainers/image-spec/blob/main/specs-go/versioned.go + +message Versioned { + int32 SchemaVersion = 1; +} diff --git a/pkg/meta/types/types.go b/pkg/meta/types/types.go index 6e64280b..3d3b2090 100644 --- a/pkg/meta/types/types.go +++ b/pkg/meta/types/types.go @@ -5,18 +5,9 @@ import ( "time" godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// DetailedRepoMeta is a auxiliary structure used for sorting RepoMeta arrays by information -// that's not directly available in the RepoMetadata structure (ex. that needs to be calculated -// by iterating the manifests, etc.) -type DetailedRepoMeta struct { - RepoMetadata - Rank int - Downloads int - UpdateTime time.Time -} - // Used to model changes to an object after a call to the DB. type ToggleState int @@ -27,23 +18,109 @@ const ( ) type ( - FilterFunc func(repoMeta RepoMetadata, manifestMeta ManifestMetadata) bool - FilterRepoFunc func(repoMeta RepoMetadata) bool + // Currently imageMeta applied for indexes is applied for each manifest individually so imageMeta.manifests + // contains just 1 manifest. + FilterFunc func(repoMeta RepoMeta, imageMeta ImageMeta) bool + FilterRepoNameFunc func(repo string) bool + FilterFullRepoFunc func(repoMeta RepoMeta) bool + FilterRepoTagFunc func(repo, tag string) bool +) + +func AcceptAllRepoNames(repo string) bool { + return true +} + +func AcceptAllRepoMeta(repoMeta RepoMeta) bool { + return true +} + +func AcceptAllRepoTag(repo, tag string) bool { + return true +} + +func AcceptAllImageMeta(repoMeta RepoMeta, imageMeta ImageMeta) bool { + return true +} + +func GetLatestImageDigests(repoMetaList []RepoMeta) []string { + digests := make([]string, 0, len(repoMetaList)) + + for i := range repoMetaList { + if repoMetaList[i].LastUpdatedImage != nil { + digests = append(digests, repoMetaList[i].LastUpdatedImage.Digest) + } + } + + return digests +} + +type ( + ImageDigest = string ) type MetaDB interface { //nolint:interfacebloat UserDB + + SetImageMeta(digest godigest.Digest, imageMeta ImageMeta) error + + // SetRepoReference sets the given image data to the repo metadata. + SetRepoReference(repo string, reference string, imageMeta ImageMeta) error + + // SearchRepos searches for repos given a search string + SearchRepos(ctx context.Context, searchText string) ([]RepoMeta, error) + + // SearchTags searches for images(repo:tag) given a search string + SearchTags(ctx context.Context, searchText string) ([]FullImageMeta, error) + + // FilterTags filters for images given a filter function + FilterTags(ctx context.Context, filterRepoTag FilterRepoTagFunc, filterFunc FilterFunc, + ) ([]FullImageMeta, error) + + // FilterRepos filters for repos given a filter function + FilterRepos(ctx context.Context, rankName FilterRepoNameFunc, filterFunc FilterFullRepoFunc, + ) ([]RepoMeta, error) + + // GetRepoMeta returns the full information about a repo + GetRepoMeta(ctx context.Context, repo string) (RepoMeta, error) + + // GetFullImageMeta returns the full information about an image + GetFullImageMeta(ctx context.Context, repo string, tag string) (FullImageMeta, error) + + // GetImageMeta returns the raw information about an image + GetImageMeta(digest godigest.Digest) (ImageMeta, error) + + // GetMultipleRepoMeta returns information about all repositories as map[string]RepoMetadata filtered by the filter + // function + GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta RepoMeta) bool) ( + []RepoMeta, error) + + // AddManifestSignature adds signature metadata to a given manifest in the database + AddManifestSignature(repo string, signedManifestDigest godigest.Digest, sm SignatureMetadata) error + + // DeleteSignature deletes signature metadata to a given manifest from the database + DeleteSignature(repo string, signedManifestDigest godigest.Digest, sigMeta SignatureMetadata) error + + // UpdateSignaturesValidity checks and updates signatures validity of a given manifest + UpdateSignaturesValidity(repo string, manifestDigest godigest.Digest) error + // IncrementRepoStars adds 1 to the star count of an image IncrementRepoStars(repo string) error - // IncrementRepoStars subtracts 1 from the star count of an image + // DecrementRepoStars subtracts 1 from the star count of an image DecrementRepoStars(repo string) error - // GetRepoStars returns the total number of stars a repo has - GetRepoStars(repo string) (int, error) + // SetRepoMeta returns RepoMetadata of a repo from the database + SetRepoMeta(repo string, repoMeta RepoMeta) error - // SetRepoReference sets the reference of a manifest in the tag list of a repo - SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, mediaType string) error + // GetReferrersInfo returns a list of for all referrers of the given digest that match one of the + // artifact types. + GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string) ([]ReferrerInfo, error) + + // IncrementImageDownloads adds 1 to the download count of an image + IncrementImageDownloads(repo string, reference string) error + + // FilterImageMeta returns the image data for the given digests + FilterImageMeta(ctx context.Context, digests []string) (map[string]ImageMeta, error) /* RemoveRepoReference removes the tag from RepoMetadata if the reference is a tag, @@ -55,80 +132,12 @@ type MetaDB interface { //nolint:interfacebloat */ RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error - // DeleteRepoTag delets the tag from the tag list of a repo - DeleteRepoTag(repo string, tag string) error + // ResetRepoReferences resets all layout specific data (tags, signatures, referrers, etc.) but keep user and image + // specific metadata such as star count, downloads other statistics + ResetRepoReferences(repo string) error - // GetRepoMeta returns RepoMetadata of a repo from the database - GetRepoMeta(repo string) (RepoMetadata, error) - - // GetUserRepometa return RepoMetadata of a repo from the database along side specific information about the - // user - GetUserRepoMeta(ctx context.Context, repo string) (RepoMetadata, error) - - // GetRepoMeta returns RepoMetadata of a repo from the database - SetRepoMeta(repo string, repoMeta RepoMetadata) error - - // GetMultipleRepoMeta returns information about all repositories as map[string]RepoMetadata filtered by the filter - // function - GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta RepoMetadata) bool) ( - []RepoMetadata, error) - - // SetManifestData sets ManifestData for a given manifest in the database - SetManifestData(manifestDigest godigest.Digest, md ManifestData) error - - // GetManifestData return the manifest and its related config - GetManifestData(manifestDigest godigest.Digest) (ManifestData, error) - - // GetManifestMeta returns ManifestMetadata for a given manifest from the database - GetManifestMeta(repo string, manifestDigest godigest.Digest) (ManifestMetadata, error) - - // GetManifestMeta sets ManifestMetadata for a given manifest in the database - SetManifestMeta(repo string, manifestDigest godigest.Digest, mm ManifestMetadata) error - - // SetIndexData sets indexData for a given index in the database - SetIndexData(digest godigest.Digest, indexData IndexData) error - - // GetIndexData returns indexData for a given Index from the database - GetIndexData(indexDigest godigest.Digest) (IndexData, error) - - // SetReferrer adds a referrer to the referrers list of a manifest inside a repo - SetReferrer(repo string, referredDigest godigest.Digest, referrer ReferrerInfo) error - - // SetReferrer delets a referrer to the referrers list of a manifest inside a repo - DeleteReferrer(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest) error - - // GetReferrersInfo returnes a list of for all referrers of the given digest that match one of the - // artifact types. - GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string) ( - []ReferrerInfo, error) - - // IncrementManifestDownloads adds 1 to the download count of a manifest - IncrementImageDownloads(repo string, reference string) error - - // AddManifestSignature adds signature metadata to a given manifest in the database - AddManifestSignature(repo string, signedManifestDigest godigest.Digest, sm SignatureMetadata) error - - // DeleteSignature delets signature metadata to a given manifest from the database - DeleteSignature(repo string, signedManifestDigest godigest.Digest, sm SignatureMetadata) error - - // UpdateSignaturesValidity checks and updates signatures validity of a given manifest - UpdateSignaturesValidity(repo string, manifestDigest godigest.Digest) error - - // SearchRepos searches for repos given a search string - SearchRepos(ctx context.Context, searchText string) ( - []RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error) - - // SearchTags searches for images(repo:tag) given a search string - SearchTags(ctx context.Context, searchText string) ( - []RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error) - - // FilterRepos filters for repos given a filter function - FilterRepos(ctx context.Context, filter FilterRepoFunc) ( - []RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error) - - // FilterTags filters for images given a filter function - FilterTags(ctx context.Context, filterFunc FilterFunc) ( - []RepoMetadata, map[string]ManifestMetadata, map[string]IndexData, error) + // ResetDB will delete all data in the DB + ResetDB() error PatchDB() error @@ -176,25 +185,78 @@ type UserDB interface { //nolint:interfacebloat type ImageTrustStore interface { VerifySignature( - signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, manifestContent []byte, + signatureType string, rawSignature []byte, sigKey string, manifestDigest godigest.Digest, imageMeta ImageMeta, repo string, ) (string, time.Time, bool, error) } -type ManifestMetadata struct { - ManifestBlob []byte - ConfigBlob []byte - DownloadCount int - Signatures ManifestSignatures -} - -type IndexData struct { - IndexBlob []byte +// ImageMeta can store all data related to a image, multiarch or simple. Used for writing imaged to MetaDB. +type ImageMeta struct { + MediaType string // MediaType refers to the image descriptor, a manifest or a index (if multiarch) + Digest godigest.Digest // Digest refers to the image descriptor, a manifest or a index (if multiarch) + Size int64 // Size refers to the image descriptor, a manifest or a index (if multiarch) + Index *ispec.Index // If the image is multiarch the Index will be non-nil + Manifests []ManifestData // All manifests under the image, 1 for simple images and many for multiarch } +// ManifestData represents all data related to an image manifests (found from the image contents itself). type ManifestData struct { - ManifestBlob []byte - ConfigBlob []byte + Size int64 + Digest godigest.Digest + Manifest ispec.Manifest + Config ispec.Image +} + +type RepoMeta struct { + Name string + Tags map[string]Descriptor + + Statistics map[string]DescriptorStatistics + Signatures map[string]ManifestSignatures + Referrers map[string][]ReferrerInfo + + LastUpdatedImage *LastUpdatedImage + Platforms []ispec.Platform + Vendors []string + Size int64 + + IsStarred bool + IsBookmarked bool + Rank int + + StarCount int + DownloadCount int +} + +// FullImageMeta is a condensed structure of all information needed about an image when searching MetaDB. +type FullImageMeta struct { + Repo string + Tag string + MediaType string + Digest godigest.Digest + Size int64 + Index *ispec.Index + Manifests []FullManifestMeta + IsStarred bool + IsBookmarked bool + + Referrers []ReferrerInfo + Statistics DescriptorStatistics + Signatures ManifestSignatures +} + +type FullManifestMeta struct { + ManifestData + + Referrers []ReferrerInfo + Statistics DescriptorStatistics + Signatures ManifestSignatures +} + +type LastUpdatedImage struct { + Descriptor + Tag string + LastUpdated *time.Time } type ReferrerInfo struct { @@ -217,21 +279,6 @@ type DescriptorStatistics struct { type ManifestSignatures map[string][]SignatureInfo -type RepoMetadata struct { - Name string - Tags map[string]Descriptor - - Statistics map[string]DescriptorStatistics - Signatures map[string]ManifestSignatures - Referrers map[string][]ReferrerInfo - - IsStarred bool - IsBookmarked bool - Rank int - - Stars int -} - type LayerInfo struct { LayerDigest string LayerContent []byte diff --git a/pkg/meta/version/version_test.go b/pkg/meta/version/version_test.go index ec919ea9..e56918d2 100644 --- a/pkg/meta/version/version_test.go +++ b/pkg/meta/version/version_test.go @@ -124,14 +124,14 @@ func TestVersioningDynamoDB(t *testing.T) { Convey("Tests", t, func() { params := mdynamodb.DBDriverParameters{ - Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"), - Region: "us-east-2", - RepoMetaTablename: "RepoMetadataTable" + uuid.String(), - ManifestDataTablename: "ManifestDataTable" + uuid.String(), - IndexDataTablename: "IndexDataTable" + uuid.String(), - UserDataTablename: "UserDataTable" + uuid.String(), - APIKeyTablename: "ApiKeyTable" + uuid.String(), - VersionTablename: "Version" + uuid.String(), + Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"), + Region: "us-east-2", + RepoMetaTablename: "RepoMetadataTable" + uuid.String(), + RepoBlobsInfoTablename: "RepoBlobsInfoTablename" + uuid.String(), + ImageMetaTablename: "ImageMetaTablename" + uuid.String(), + UserDataTablename: "UserDataTable" + uuid.String(), + APIKeyTablename: "ApiKeyTable" + uuid.String(), + VersionTablename: "Version" + uuid.String(), } dynamoClient, err := mdynamodb.GetDynamoClient(params) @@ -142,8 +142,7 @@ func TestVersioningDynamoDB(t *testing.T) { dynamoWrapper, err := mdynamodb.New(dynamoClient, params, log) So(err, ShouldBeNil) - So(dynamoWrapper.ResetManifestDataTable(), ShouldBeNil) - So(dynamoWrapper.ResetRepoMetaTable(), ShouldBeNil) + So(dynamoWrapper.ResetTable(dynamoWrapper.RepoMetaTablename), ShouldBeNil) Convey("dbVersion is empty", func() { err := setDynamoDBVersion(dynamoWrapper.Client, params.VersionTablename, "") @@ -201,7 +200,7 @@ func setDynamoDBVersion(client *dynamodb.Client, versTable, vers string) error { ":Version": mdAttributeValue, }, Key: map[string]types.AttributeValue{ - "VersionKey": &types.AttributeValueMemberS{ + "Key": &types.AttributeValueMemberS{ Value: version.DBVersionKey, }, }, diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 56da20b7..7d3cf4cd 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -3,7 +3,6 @@ package storage import ( "encoding/json" "fmt" - "regexp" "strings" "github.com/docker/distribution/registry/storage/driver/factory" @@ -228,10 +227,7 @@ func CheckIsImageSignature(repoName string, manifestBlob []byte, reference strin return true, NotationType, manifestContent.Subject.Digest, nil } - // check cosign - cosignTagRule := regexp.MustCompile(`sha256\-.+\.sig`) - - if tag := reference; cosignTagRule.MatchString(reference) { + if tag := reference; zcommon.IsCosignTag(reference) { prefixLen := len("sha256-") digestLen := 64 signedImageManifestDigestEncoded := tag[prefixLen : prefixLen+digestLen] diff --git a/pkg/test/common/utils.go b/pkg/test/common/utils.go index 65696eb1..b33a0746 100644 --- a/pkg/test/common/utils.go +++ b/pkg/test/common/utils.go @@ -211,3 +211,35 @@ func GenerateRandomName() (string, int64) { return string(randomBytes), seed } + +func AccumulateField[R any, T any](list []T, accFunc func(T) R) []R { + result := make([]R, 0, len(list)) + + for i := range list { + result = append(result, accFunc(list[i])) + } + + return result +} + +func ContainSameElements[T comparable](list1, list2 []T) bool { + if len(list1) != len(list2) { + return false + } + + count1 := map[T]int{} + count2 := map[T]int{} + + for i := range list1 { + count1[list1[i]]++ + count2[list2[i]]++ + } + + for key := range count1 { + if count1[key] != count2[key] { + return false + } + } + + return true +} diff --git a/pkg/test/deprecated/deprecated.go b/pkg/test/deprecated/deprecated.go index d6bca087..4c6ed0ad 100644 --- a/pkg/test/deprecated/deprecated.go +++ b/pkg/test/deprecated/deprecated.go @@ -387,7 +387,7 @@ func GetRandomMultiarchImage(reference string) (image.MultiarchImage, error) { index.SchemaVersion = 2 return image.MultiarchImage{ - Index: index, Images: images, Reference: reference, + Index: index, Images: images, }, err } diff --git a/pkg/test/image-utils/images.go b/pkg/test/image-utils/images.go index 38587f07..3e46ba2d 100644 --- a/pkg/test/image-utils/images.go +++ b/pkg/test/image-utils/images.go @@ -11,6 +11,7 @@ import ( "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" + mTypes "zotregistry.io/zot/pkg/meta/types" storageConstants "zotregistry.io/zot/pkg/storage/constants" ) @@ -48,6 +49,9 @@ type ConfigBuilder interface { // an OCI artifact. // (see: https://github.com/opencontainers/image-spec/blob/main/manifest.md#guidelines-for-artifact-usage) ArtifactConfig(artifactType string) ManifestBuilder + // PlatformConfig is used when we're interesting in specifying only the platform of a manifest. + // Other fields of the config are random. + PlatformConfig(architecture, os string) ManifestBuilder // DefaultConfig sets the default config, platform linux/amd64. DefaultConfig() ManifestBuilder // CustomConfigBlob will set a custom blob as the image config without other checks. @@ -92,13 +96,17 @@ type Image struct { } func (img *Image) Digest() godigest.Digest { + if img.ManifestDescriptor.Digest != "" { + return img.ManifestDescriptor.Digest + } + + // when we'll migrate all code to the new format of creating images we can replace this with + // the value from manifestDescriptor blob, err := json.Marshal(img.Manifest) if err != nil { panic("unreachable: ispec.Manifest should always be marshable") } - // when we'll migrate all code to the new format of creating images we can replace this with - // the value from manifestDescriptor return godigest.FromBytes(blob) } @@ -106,6 +114,16 @@ func (img *Image) DigestStr() string { return img.Digest().String() } +func (img *Image) Size() int { + size := img.ConfigDescriptor.Size + img.ManifestDescriptor.Size + + for _, layer := range img.Manifest.Layers { + size += layer.Size + } + + return int(size) +} + func (img Image) Descriptor() ispec.Descriptor { return ispec.Descriptor{ MediaType: img.ManifestDescriptor.MediaType, @@ -122,6 +140,22 @@ func (img Image) DescriptorRef() *ispec.Descriptor { } } +func (img Image) AsImageMeta() mTypes.ImageMeta { + return mTypes.ImageMeta{ + MediaType: img.Manifest.MediaType, + Digest: img.ManifestDescriptor.Digest, + Size: img.ManifestDescriptor.Size, + Manifests: []mTypes.ManifestData{ + { + Size: img.ManifestDescriptor.Size, + Digest: img.ManifestDescriptor.Digest, + Manifest: img.Manifest, + Config: img.Config, + }, + }, + } +} + type Layer struct { Blob []byte MediaType string @@ -281,6 +315,16 @@ func (ib *BaseImageBuilder) DefaultConfig() ManifestBuilder { return ib.ImageConfig(GetDefaultConfig()) } +func (ib *BaseImageBuilder) PlatformConfig(arch, os string) ManifestBuilder { + conf := GetDefaultConfig() + + conf.Created = RandomDateRef(time.UTC) + conf.Author = getRandomAuthor() + conf.Platform = ispec.Platform{Architecture: arch, OS: os} + + return ib.ImageConfig(conf) +} + func (ib *BaseImageBuilder) EmptyConfig() ManifestBuilder { ib.configDescriptor = ispec.DescriptorEmptyJSON diff --git a/pkg/test/image-utils/images_test.go b/pkg/test/image-utils/images_test.go index 90b9692c..769497a9 100644 --- a/pkg/test/image-utils/images_test.go +++ b/pkg/test/image-utils/images_test.go @@ -166,16 +166,6 @@ func TestPredefinedImages(t *testing.T) { img = CreateRandomVulnerableImageWith().ArtifactType("art.type").Build() So(img.Manifest.ArtifactType, ShouldEqual, "art.type") }) - - Convey("Predefined Multiarch-Images", t, func() { - multiArch := CreateRandomMultiarch() - So(len(multiArch.Images), ShouldEqual, 3) - So(multiArch.Reference, ShouldResemble, multiArch.Digest().String()) - - multiArch = CreateVulnerableMultiarch() - So(len(multiArch.Images), ShouldEqual, 3) - So(multiArch.Reference, ShouldResemble, multiArch.Digest().String()) - }) } func TestImageMethods(t *testing.T) { diff --git a/pkg/test/image-utils/multiarch.go b/pkg/test/image-utils/multiarch.go index c8da8d67..a5926ba7 100644 --- a/pkg/test/image-utils/multiarch.go +++ b/pkg/test/image-utils/multiarch.go @@ -11,9 +11,8 @@ import ( ) type MultiarchImage struct { - Index ispec.Index - Images []Image - Reference string + Index ispec.Index + Images []Image IndexDescriptor ispec.Descriptor } @@ -31,17 +30,27 @@ func (mi *MultiarchImage) DigestStr() string { return mi.Digest().String() } -func (mi *MultiarchImage) IndexData() mTypes.IndexData { - indexBlob, err := json.Marshal(mi.Index) - if err != nil { - panic("unreachable: ispec.Index should always be marshable") +func (mi MultiarchImage) AsImageMeta() mTypes.ImageMeta { + index := mi.Index + + manifests := make([]mTypes.ManifestData, 0, len(index.Manifests)) + + for _, image := range mi.Images { + manifests = append(manifests, image.AsImageMeta().Manifests...) } - return mTypes.IndexData{IndexBlob: indexBlob} + return mTypes.ImageMeta{ + MediaType: ispec.MediaTypeImageIndex, + Digest: mi.IndexDescriptor.Digest, + Size: mi.IndexDescriptor.Size, + Index: &index, + Manifests: manifests, + } } type ImagesBuilder interface { Images(images []Image) MultiarchBuilder + RandomImages(count int) MultiarchBuilder } type MultiarchBuilder interface { @@ -88,6 +97,18 @@ func (mb *BaseMultiarchBuilder) Images(images []Image) MultiarchBuilder { return mb } +func (mb *BaseMultiarchBuilder) RandomImages(count int) MultiarchBuilder { + images := make([]Image, count) + + for i := range images { + images[i] = CreateRandomImage() + } + + mb.images = images + + return mb +} + func (mb *BaseMultiarchBuilder) Subject(subject *ispec.Descriptor) MultiarchBuilder { mb.subject = subject @@ -135,12 +156,9 @@ func (mb *BaseMultiarchBuilder) Build() MultiarchImage { indexDigest := godigest.FromBytes(indexBlob) - ref := indexDigest.String() - return MultiarchImage{ - Index: index, - Images: mb.images, - Reference: ref, + Index: index, + Images: mb.images, IndexDescriptor: ispec.Descriptor{ MediaType: ispec.MediaTypeImageIndex, diff --git a/pkg/test/image-utils/upload.go b/pkg/test/image-utils/upload.go index 2f5f5819..bda283f0 100644 --- a/pkg/test/image-utils/upload.go +++ b/pkg/test/image-utils/upload.go @@ -223,9 +223,15 @@ func UploadMultiarchImage(multiImage MultiarchImage, baseURL string, repo, ref s } // put manifest - indexBlob, err := json.Marshal(multiImage.Index) - if err = inject.Error(err); err != nil { - return err + indexBlob := multiImage.IndexDescriptor.Data + + if len(indexBlob) == 0 { + var err error + + indexBlob, err = json.Marshal(multiImage.Index) + if err = inject.Error(err); err != nil { + return err + } } resp, err := resty.R(). diff --git a/pkg/test/image-utils/utils.go b/pkg/test/image-utils/utils.go index 0b1289be..47c38d13 100644 --- a/pkg/test/image-utils/utils.go +++ b/pkg/test/image-utils/utils.go @@ -176,21 +176,3 @@ func GetRandomImageConfig() ([]byte, godigest.Digest) { return configBlobContent, configBlobDigestRaw } - -func GetIndexBlobWithManifests(manifestDigests []godigest.Digest) ([]byte, error) { - manifests := make([]ispec.Descriptor, 0, len(manifestDigests)) - - for _, manifestDigest := range manifestDigests { - manifests = append(manifests, ispec.Descriptor{ - Digest: manifestDigest, - MediaType: ispec.MediaTypeImageManifest, - }) - } - - indexContent := ispec.Index{ - MediaType: ispec.MediaTypeImageIndex, - Manifests: manifests, - } - - return json.Marshal(indexContent) -} diff --git a/pkg/test/mocks/repo_db_mock.go b/pkg/test/mocks/repo_db_mock.go index 9d40b4c1..32938298 100644 --- a/pkg/test/mocks/repo_db_mock.go +++ b/pkg/test/mocks/repo_db_mock.go @@ -9,74 +9,6 @@ import ( ) type MetaDBMock struct { - SetRepoDescriptionFn func(repo, description string) error - - IncrementRepoStarsFn func(repo string) error - - DecrementRepoStarsFn func(repo string) error - - GetRepoStarsFn func(repo string) (int, error) - - SetRepoLogoFn func(repo string, logoPath string) error - - SetRepoReferenceFn func(repo string, Reference string, manifestDigest godigest.Digest, mediaType string) error - - RemoveRepoReferenceFn func(repo, reference string, manifestDigest godigest.Digest) error - - DeleteRepoTagFn func(repo string, tag string) error - - GetRepoMetaFn func(repo string) (mTypes.RepoMetadata, error) - - GetUserRepoMetaFn func(ctx context.Context, repo string) (mTypes.RepoMetadata, error) - - SetRepoMetaFn func(repo string, repoMeta mTypes.RepoMetadata) error - - GetMultipleRepoMetaFn func(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool) ( - []mTypes.RepoMetadata, error) - - GetManifestDataFn func(manifestDigest godigest.Digest) (mTypes.ManifestData, error) - - SetManifestDataFn func(manifestDigest godigest.Digest, mm mTypes.ManifestData) error - - GetManifestMetaFn func(repo string, manifestDigest godigest.Digest) (mTypes.ManifestMetadata, error) - - SetManifestMetaFn func(repo string, manifestDigest godigest.Digest, mm mTypes.ManifestMetadata) error - - SetIndexDataFn func(digest godigest.Digest, indexData mTypes.IndexData) error - - GetIndexDataFn func(indexDigest godigest.Digest) (mTypes.IndexData, error) - - SetReferrerFn func(repo string, referredDigest godigest.Digest, referrer mTypes.ReferrerInfo) error - - DeleteReferrerFn func(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest) error - - GetReferrersFn func(repo string, referredDigest godigest.Digest) ([]mTypes.Descriptor, error) - - GetReferrersInfoFn func(repo string, referredDigest godigest.Digest, artifactTypes []string) ( - []mTypes.ReferrerInfo, error) - - IncrementImageDownloadsFn func(repo string, reference string) error - - UpdateSignaturesValidityFn func(repo string, manifestDigest godigest.Digest) error - - AddManifestSignatureFn func(repo string, signedManifestDigest godigest.Digest, sm mTypes.SignatureMetadata) error - - DeleteSignatureFn func(repo string, signedManifestDigest godigest.Digest, sm mTypes.SignatureMetadata) error - - SearchReposFn func(ctx context.Context, txt string) ( - []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error) - - SearchTagsFn func(ctx context.Context, txt string) ( - []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, - error) - - FilterReposFn func(ctx context.Context, filter mTypes.FilterRepoFunc) ( - []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) - - FilterTagsFn func(ctx context.Context, filterFunc mTypes.FilterFunc) ( - []mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) - GetStarredReposFn func(ctx context.Context) ([]string, error) GetBookmarkedReposFn func(ctx context.Context) ([]string, error) @@ -112,6 +44,69 @@ type MetaDBMock struct { ImageTrustStoreFn func() mTypes.ImageTrustStore SetImageTrustStoreFn func(mTypes.ImageTrustStore) + + SetRepoReferenceFn func(repo string, reference string, imageMeta mTypes.ImageMeta) error + + SearchReposFn func(ctx context.Context, searchText string, + ) ([]mTypes.RepoMeta, error) + + SearchTagsFn func(ctx context.Context, searchText string) ([]mTypes.FullImageMeta, error) + + FilterTagFn func(ctx context.Context, filterFunc mTypes.FilterFunc, + ) ([]mTypes.RepoMeta, map[string]mTypes.ImageMeta, error) + + GetImageMetaFn func(digest godigest.Digest) (mTypes.ImageMeta, error) + + GetMultipleRepoMetaFn func(ctx context.Context, filter func(repoMeta mTypes.RepoMeta) bool, + ) ([]mTypes.RepoMeta, error) + + FilterReposFn func(ctx context.Context, rankName mTypes.FilterRepoNameFunc, + filterFunc mTypes.FilterFullRepoFunc) ([]mTypes.RepoMeta, error) + + IncrementRepoStarsFn func(repo string) error + + DecrementRepoStarsFn func(repo string) error + + SetRepoMetaFn func(repo string, repoMeta mTypes.RepoMeta) error + + DeleteReferrerFn func(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest) error + + GetReferrersInfoFn func(repo string, referredDigest godigest.Digest, artifactTypes []string, + ) ([]mTypes.ReferrerInfo, error) + + IncrementImageDownloadsFn func(repo string, reference string) error + + UpdateSignaturesValidityFn func(repo string, manifestDigest godigest.Digest) error + + AddManifestSignatureFn func(repo string, signedManifestDigest godigest.Digest, sygMeta mTypes.SignatureMetadata, + ) error + + DeleteSignatureFn func(repo string, signedManifestDigest godigest.Digest, sigMeta mTypes.SignatureMetadata) error + + SetImageMetaFn func(digest godigest.Digest, imageMeta mTypes.ImageMeta) error + + FilterTagsFn func(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, + filterFunc mTypes.FilterFunc) ([]mTypes.FullImageMeta, error) + + GetRepoMetaFn func(ctx context.Context, repo string) (mTypes.RepoMeta, error) + + FilterImageMetaFn func(ctx context.Context, digests []string) (map[string]mTypes.ImageMeta, error) + + RemoveRepoReferenceFn func(repo, reference string, manifestDigest godigest.Digest) error + + GetFullImageMetaFn func(ctx context.Context, repo string, tag string) (mTypes.FullImageMeta, error) + + ResetRepoReferencesFn func(repo string) error + + ResetDBFn func() error +} + +func (sdm MetaDBMock) ResetDB() error { + if sdm.ResetDBFn != nil { + return sdm.ResetDBFn() + } + + return nil } func (sdm MetaDBMock) ImageTrustStore() mTypes.ImageTrustStore { @@ -128,221 +123,6 @@ func (sdm MetaDBMock) SetImageTrustStore(imgTrustStore mTypes.ImageTrustStore) { } } -func (sdm MetaDBMock) SetRepoDescription(repo, description string) error { - if sdm.SetRepoDescriptionFn != nil { - return sdm.SetRepoDescriptionFn(repo, description) - } - - return nil -} - -func (sdm MetaDBMock) IncrementRepoStars(repo string) error { - if sdm.IncrementRepoStarsFn != nil { - return sdm.IncrementRepoStarsFn(repo) - } - - return nil -} - -func (sdm MetaDBMock) DecrementRepoStars(repo string) error { - if sdm.DecrementRepoStarsFn != nil { - return sdm.DecrementRepoStarsFn(repo) - } - - return nil -} - -func (sdm MetaDBMock) GetRepoStars(repo string) (int, error) { - if sdm.GetRepoStarsFn != nil { - return sdm.GetRepoStarsFn(repo) - } - - return 0, nil -} - -func (sdm MetaDBMock) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest, - mediaType string, -) error { - if sdm.SetRepoReferenceFn != nil { - return sdm.SetRepoReferenceFn(repo, reference, manifestDigest, mediaType) - } - - return nil -} - -func (sdm MetaDBMock) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error { - if sdm.RemoveRepoReferenceFn != nil { - return sdm.RemoveRepoReferenceFn(repo, reference, manifestDigest) - } - - return nil -} - -func (sdm MetaDBMock) DeleteRepoTag(repo string, tag string) error { - if sdm.DeleteRepoTagFn != nil { - return sdm.DeleteRepoTagFn(repo, tag) - } - - return nil -} - -func (sdm MetaDBMock) GetRepoMeta(repo string) (mTypes.RepoMetadata, error) { - if sdm.GetRepoMetaFn != nil { - return sdm.GetRepoMetaFn(repo) - } - - return mTypes.RepoMetadata{}, nil -} - -func (sdm MetaDBMock) GetUserRepoMeta(ctx context.Context, repo string) (mTypes.RepoMetadata, error) { - if sdm.GetUserRepoMetaFn != nil { - return sdm.GetUserRepoMetaFn(ctx, repo) - } - - return mTypes.RepoMetadata{}, nil -} - -func (sdm MetaDBMock) SetRepoMeta(repo string, repoMeta mTypes.RepoMetadata) error { - if sdm.SetRepoMetaFn != nil { - return sdm.SetRepoMetaFn(repo, repoMeta) - } - - return nil -} - -func (sdm MetaDBMock) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta mTypes.RepoMetadata) bool, -) ([]mTypes.RepoMetadata, error) { - if sdm.GetMultipleRepoMetaFn != nil { - return sdm.GetMultipleRepoMetaFn(ctx, filter) - } - - return []mTypes.RepoMetadata{}, nil -} - -func (sdm MetaDBMock) GetManifestData(manifestDigest godigest.Digest) (mTypes.ManifestData, error) { - if sdm.GetManifestDataFn != nil { - return sdm.GetManifestDataFn(manifestDigest) - } - - return mTypes.ManifestData{}, nil -} - -func (sdm MetaDBMock) SetManifestData(manifestDigest godigest.Digest, md mTypes.ManifestData) error { - if sdm.SetManifestDataFn != nil { - return sdm.SetManifestDataFn(manifestDigest, md) - } - - return nil -} - -func (sdm MetaDBMock) GetManifestMeta(repo string, manifestDigest godigest.Digest) (mTypes.ManifestMetadata, error) { - if sdm.GetManifestMetaFn != nil { - return sdm.GetManifestMetaFn(repo, manifestDigest) - } - - return mTypes.ManifestMetadata{}, nil -} - -func (sdm MetaDBMock) SetManifestMeta(repo string, manifestDigest godigest.Digest, mm mTypes.ManifestMetadata) error { - if sdm.SetManifestMetaFn != nil { - return sdm.SetManifestMetaFn(repo, manifestDigest, mm) - } - - return nil -} - -func (sdm MetaDBMock) IncrementImageDownloads(repo string, reference string) error { - if sdm.IncrementImageDownloadsFn != nil { - return sdm.IncrementImageDownloadsFn(repo, reference) - } - - return nil -} - -func (sdm MetaDBMock) UpdateSignaturesValidity(repo string, manifestDigest godigest.Digest) error { - if sdm.UpdateSignaturesValidityFn != nil { - return sdm.UpdateSignaturesValidityFn(repo, manifestDigest) - } - - return nil -} - -func (sdm MetaDBMock) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, - sm mTypes.SignatureMetadata, -) error { - if sdm.AddManifestSignatureFn != nil { - return sdm.AddManifestSignatureFn(repo, signedManifestDigest, sm) - } - - return nil -} - -func (sdm MetaDBMock) DeleteSignature(repo string, signedManifestDigest godigest.Digest, - sm mTypes.SignatureMetadata, -) error { - if sdm.DeleteSignatureFn != nil { - return sdm.DeleteSignatureFn(repo, signedManifestDigest, sm) - } - - return nil -} - -func (sdm MetaDBMock) SearchRepos(ctx context.Context, searchText string, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { - if sdm.SearchReposFn != nil { - return sdm.SearchReposFn(ctx, searchText) - } - - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, - map[string]mTypes.IndexData{}, nil -} - -func (sdm MetaDBMock) SearchTags(ctx context.Context, searchText string, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { - if sdm.SearchTagsFn != nil { - return sdm.SearchTagsFn(ctx, searchText) - } - - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, - map[string]mTypes.IndexData{}, nil -} - -func (sdm MetaDBMock) FilterRepos(ctx context.Context, filter mTypes.FilterRepoFunc, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { - if sdm.FilterReposFn != nil { - return sdm.FilterReposFn(ctx, filter) - } - - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, - map[string]mTypes.IndexData{}, nil -} - -func (sdm MetaDBMock) FilterTags(ctx context.Context, filterFunc mTypes.FilterFunc, -) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, map[string]mTypes.IndexData, error) { - if sdm.FilterTagsFn != nil { - return sdm.FilterTagsFn(ctx, filterFunc) - } - - return []mTypes.RepoMetadata{}, map[string]mTypes.ManifestMetadata{}, - map[string]mTypes.IndexData{}, nil -} - -func (sdm MetaDBMock) SetIndexData(digest godigest.Digest, indexData mTypes.IndexData) error { - if sdm.SetIndexDataFn != nil { - return sdm.SetIndexDataFn(digest, indexData) - } - - return nil -} - -func (sdm MetaDBMock) GetIndexData(indexDigest godigest.Digest) (mTypes.IndexData, error) { - if sdm.GetIndexDataFn != nil { - return sdm.GetIndexDataFn(indexDigest) - } - - return mTypes.IndexData{}, nil -} - func (sdm MetaDBMock) PatchDB() error { if sdm.PatchDBFn != nil { return sdm.PatchDBFn() @@ -351,34 +131,6 @@ func (sdm MetaDBMock) PatchDB() error { return nil } -func (sdm MetaDBMock) SetReferrer(repo string, referredDigest godigest.Digest, referrer mTypes.ReferrerInfo) error { - if sdm.SetReferrerFn != nil { - return sdm.SetReferrerFn(repo, referredDigest, referrer) - } - - return nil -} - -func (sdm MetaDBMock) DeleteReferrer(repo string, referredDigest godigest.Digest, - referrerDigest godigest.Digest, -) error { - if sdm.DeleteReferrerFn != nil { - return sdm.DeleteReferrerFn(repo, referredDigest, referrerDigest) - } - - return nil -} - -func (sdm MetaDBMock) GetReferrersInfo(repo string, referredDigest godigest.Digest, - artifactTypes []string, -) ([]mTypes.ReferrerInfo, error) { - if sdm.GetReferrersInfoFn != nil { - return sdm.GetReferrersInfoFn(repo, referredDigest, artifactTypes) - } - - return []mTypes.ReferrerInfo{}, nil -} - func (sdm MetaDBMock) GetStarredRepos(ctx context.Context) ([]string, error) { if sdm.GetStarredReposFn != nil { return sdm.GetStarredReposFn(ctx) @@ -498,3 +250,184 @@ func (sdm MetaDBMock) DeleteUserAPIKey(ctx context.Context, id string) error { return nil } + +func (sdm MetaDBMock) SetImageMeta(digest godigest.Digest, imageMeta mTypes.ImageMeta) error { + if sdm.SetImageMetaFn != nil { + return sdm.SetImageMetaFn(digest, imageMeta) + } + + return nil +} + +func (sdm MetaDBMock) SetRepoReference(repo string, reference string, imageMeta mTypes.ImageMeta) error { + if sdm.SetRepoReferenceFn != nil { + return sdm.SetRepoReferenceFn(repo, reference, imageMeta) + } + + return nil +} + +func (sdm MetaDBMock) SearchRepos(ctx context.Context, searchText string) ([]mTypes.RepoMeta, error) { + if sdm.SearchReposFn != nil { + return sdm.SearchReposFn(ctx, searchText) + } + + return []mTypes.RepoMeta{}, nil +} + +func (sdm MetaDBMock) SearchTags(ctx context.Context, searchText string) ([]mTypes.FullImageMeta, error) { + if sdm.SearchTagsFn != nil { + return sdm.SearchTagsFn(ctx, searchText) + } + + return []mTypes.FullImageMeta{}, nil +} + +func (sdm MetaDBMock) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, + filterFunc mTypes.FilterFunc, +) ([]mTypes.FullImageMeta, error) { + if sdm.FilterTagsFn != nil { + return sdm.FilterTagsFn(ctx, filterRepoTag, filterFunc) + } + + return []mTypes.FullImageMeta{}, nil +} + +func (sdm MetaDBMock) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMeta, error) { + if sdm.GetRepoMetaFn != nil { + return sdm.GetRepoMetaFn(ctx, repo) + } + + return mTypes.RepoMeta{}, nil +} + +func (sdm MetaDBMock) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, error) { + if sdm.GetImageMetaFn != nil { + return sdm.GetImageMetaFn(digest) + } + + return mTypes.ImageMeta{}, nil +} + +func (sdm MetaDBMock) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta mTypes.RepoMeta) bool, +) ([]mTypes.RepoMeta, error) { + if sdm.GetMultipleRepoMetaFn != nil { + return sdm.GetMultipleRepoMetaFn(ctx, filter) + } + + return []mTypes.RepoMeta{}, nil +} + +func (sdm MetaDBMock) FilterRepos(ctx context.Context, rankName mTypes.FilterRepoNameFunc, + filterFunc mTypes.FilterFullRepoFunc, +) ([]mTypes.RepoMeta, error) { + if sdm.FilterReposFn != nil { + return sdm.FilterReposFn(ctx, rankName, filterFunc) + } + + return []mTypes.RepoMeta{}, nil +} + +func (sdm MetaDBMock) IncrementRepoStars(repo string) error { + if sdm.IncrementRepoStarsFn != nil { + return sdm.IncrementRepoStarsFn(repo) + } + + return nil +} + +func (sdm MetaDBMock) DecrementRepoStars(repo string) error { + if sdm.DecrementRepoStarsFn != nil { + return sdm.DecrementRepoStarsFn(repo) + } + + return nil +} + +func (sdm MetaDBMock) SetRepoMeta(repo string, repoMeta mTypes.RepoMeta) error { + if sdm.SetRepoMetaFn != nil { + return sdm.SetRepoMetaFn(repo, repoMeta) + } + + return nil +} + +func (sdm MetaDBMock) GetReferrersInfo(repo string, referredDigest godigest.Digest, + artifactTypes []string, +) ([]mTypes.ReferrerInfo, error) { + if sdm.GetReferrersInfoFn != nil { + return sdm.GetReferrersInfoFn(repo, referredDigest, artifactTypes) + } + + return []mTypes.ReferrerInfo{}, nil +} + +func (sdm MetaDBMock) IncrementImageDownloads(repo string, reference string) error { + if sdm.IncrementImageDownloadsFn != nil { + return sdm.IncrementImageDownloadsFn(repo, reference) + } + + return nil +} + +func (sdm MetaDBMock) UpdateSignaturesValidity(repo string, manifestDigest godigest.Digest) error { + if sdm.UpdateSignaturesValidityFn != nil { + return sdm.UpdateSignaturesValidityFn(repo, manifestDigest) + } + + return nil +} + +func (sdm MetaDBMock) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, + sygMeta mTypes.SignatureMetadata, +) error { + if sdm.AddManifestSignatureFn != nil { + return sdm.AddManifestSignatureFn(repo, signedManifestDigest, sygMeta) + } + + return nil +} + +func (sdm MetaDBMock) DeleteSignature(repo string, signedManifestDigest godigest.Digest, + sigMeta mTypes.SignatureMetadata, +) error { + if sdm.DeleteSignatureFn != nil { + return sdm.DeleteSignatureFn(repo, signedManifestDigest, sigMeta) + } + + return nil +} + +func (sdm MetaDBMock) FilterImageMeta(ctx context.Context, digests []string, +) (map[string]mTypes.ImageMeta, error) { + if sdm.FilterImageMetaFn != nil { + return sdm.FilterImageMetaFn(ctx, digests) + } + + return map[string]mTypes.ImageMeta{}, nil +} + +func (sdm MetaDBMock) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error { + if sdm.RemoveRepoReferenceFn != nil { + return sdm.RemoveRepoReferenceFn(repo, reference, manifestDigest) + } + + return nil +} + +func (sdm MetaDBMock) GetFullImageMeta(ctx context.Context, repo string, tag string, +) (mTypes.FullImageMeta, error) { + if sdm.GetFullImageMetaFn != nil { + return sdm.GetFullImageMetaFn(ctx, repo, tag) + } + + return mTypes.FullImageMeta{}, nil +} + +func (sdm MetaDBMock) ResetRepoReferences(repo string) error { + if sdm.ResetRepoReferencesFn != nil { + return sdm.ResetRepoReferencesFn(repo) + } + + return nil +} diff --git a/pkg/test/oci-utils/repo.go b/pkg/test/oci-utils/repo.go index dc14a3c5..6c726f22 100644 --- a/pkg/test/oci-utils/repo.go +++ b/pkg/test/oci-utils/repo.go @@ -1,110 +1,142 @@ package ociutils import ( - ispec "github.com/opencontainers/image-spec/specs-go/v1" + "context" + "fmt" + zerr "zotregistry.io/zot/errors" mTypes "zotregistry.io/zot/pkg/meta/types" + reqCtx "zotregistry.io/zot/pkg/requestcontext" imageUtil "zotregistry.io/zot/pkg/test/image-utils" ) type RepoImage struct { imageUtil.Image - Tag string + Reference string Statistics mTypes.DescriptorStatistics } type RepoMultiArchImage struct { imageUtil.MultiarchImage ImageStatistics map[string]mTypes.DescriptorStatistics - Tag string + Reference string } type Repo struct { Name string Images []RepoImage MultiArchImages []RepoMultiArchImage + Signatures map[string]mTypes.ManifestSignatures + Stars int IsBookmarked bool IsStarred bool } -func GetMetadataForRepos(repos ...Repo) ([]mTypes.RepoMetadata, map[string]mTypes.ManifestMetadata, - map[string]mTypes.IndexData, -) { - var ( - reposMetadata = []mTypes.RepoMetadata{} - manifestMetadataMap = map[string]mTypes.ManifestMetadata{} - indexDataMap = map[string]mTypes.IndexData{} - ) +func InitializeTestMetaDB(ctx context.Context, metaDB mTypes.MetaDB, repos ...Repo) (context.Context, error) { + uac := reqCtx.NewUserAccessControl() + uac.SetUsername("test") + uacContext := context.WithValue(ctx, reqCtx.GetContextKey(), *uac) + + err := validateRepos(repos) + if err != nil { + return uacContext, err + } for _, repo := range repos { - repoMeta := mTypes.RepoMetadata{ - Name: repo.Name, - Tags: map[string]mTypes.Descriptor{}, - Signatures: map[string]mTypes.ManifestSignatures{}, - Statistics: map[string]mTypes.DescriptorStatistics{}, - IsStarred: repo.IsStarred, - IsBookmarked: repo.IsBookmarked, - } + statistics := map[string]mTypes.DescriptorStatistics{"": {}} for _, image := range repo.Images { - addImageMetaToMetaDB(image, repoMeta, manifestMetadataMap) + err := metaDB.SetRepoReference(repo.Name, image.Reference, image.AsImageMeta()) + if err != nil { + return uacContext, err + } + + statistics[image.DigestStr()] = image.Statistics } for _, multiArch := range repo.MultiArchImages { - if multiArch.ImageStatistics == nil { - multiArch.ImageStatistics = map[string]mTypes.DescriptorStatistics{} - } - - repoMeta.Tags[multiArch.Tag] = mTypes.Descriptor{ - MediaType: ispec.MediaTypeImageIndex, - Digest: multiArch.DigestStr(), - } - - repoMeta.Statistics[multiArch.DigestStr()] = multiArch.ImageStatistics[multiArch.DigestStr()] - for _, image := range multiArch.Images { - addImageMetaToMetaDB(RepoImage{ - Image: image, - Statistics: multiArch.ImageStatistics[image.DigestStr()], - }, repoMeta, manifestMetadataMap) + err := metaDB.SetRepoReference(repo.Name, image.DigestStr(), image.AsImageMeta()) + if err != nil { + return uacContext, err + } + + statistics[image.DigestStr()] = multiArch.ImageStatistics[image.DigestStr()] } - indexDataMap[multiArch.IndexDescriptor.Digest.String()] = mTypes.IndexData{ - IndexBlob: multiArch.IndexDescriptor.Data, + err := metaDB.SetRepoReference(repo.Name, multiArch.Reference, multiArch.AsImageMeta()) + if err != nil { + return uacContext, err + } + + statistics[multiArch.DigestStr()] = multiArch.ImageStatistics[multiArch.DigestStr()] + } + + // Update repo metadata + repoMeta, err := metaDB.GetRepoMeta(ctx, repo.Name) + if err != nil { + return uacContext, err + } + + repoMeta.StarCount = repo.Stars + repoMeta.IsStarred = repo.IsStarred + repoMeta.IsBookmarked = repo.IsBookmarked + + // updateStatistics + for key, value := range statistics { + repoMeta.Statistics[key] = value + } + + // update signatures? + for key, value := range repo.Signatures { + repoMeta.Signatures[key] = value + } + + err = metaDB.SetRepoMeta(repo.Name, repoMeta) + if err != nil { + return uacContext, err + } + + // User data is set after we create the repo + if repo.IsBookmarked { + _, err := metaDB.ToggleBookmarkRepo(uacContext, repo.Name) + if err != nil { + return uacContext, err } } - reposMetadata = append(reposMetadata, repoMeta) - } - - return reposMetadata, manifestMetadataMap, indexDataMap -} - -func addImageMetaToMetaDB(image RepoImage, repoMeta mTypes.RepoMetadata, - manifestMetadataMap map[string]mTypes.ManifestMetadata, -) { - if image.Tag != "" { - repoMeta.Tags[image.Tag] = mTypes.Descriptor{ - MediaType: ispec.MediaTypeImageManifest, - Digest: image.DigestStr(), - } - } - // here we can do many more checks about the images like check for referrers, signatures but it's not needed yet - // I need just the tags for now and the fake signature. - - // This is done just to mark a manifest as signed in the resulted RepoMeta - if image.Manifest.ArtifactType == imageUtil.TestFakeSignatureArtType && image.Manifest.Subject != nil { - signedManifestDig := image.Manifest.Subject.Digest.String() - repoMeta.Signatures[signedManifestDig] = mTypes.ManifestSignatures{ - "fakeSignature": []mTypes.SignatureInfo{{SignatureManifestDigest: image.ManifestDescriptor.Digest.String()}}, + if repo.IsStarred { + _, err := metaDB.ToggleStarRepo(uacContext, repo.Name) + if err != nil { + return uacContext, err + } } } - repoMeta.Statistics[image.DigestStr()] = image.Statistics + return uacContext, nil +} - manifestMetadataMap[image.DigestStr()] = mTypes.ManifestMetadata{ - ManifestBlob: image.ManifestDescriptor.Data, - ConfigBlob: image.ConfigDescriptor.Data, - DownloadCount: image.Statistics.DownloadCount, +func validateRepos(repos []Repo) error { + repoNames := map[string]struct{}{} + + for _, repo := range repos { + if _, found := repoNames[repo.Name]; found { + return fmt.Errorf("%w '%s'", zerr.ErrMultipleReposSameName, repo.Name) + } + + repoNames[repo.Name] = struct{}{} + } + + return nil +} + +func GetFakeSignatureInfo(signatureDigest string) map[string][]mTypes.SignatureInfo { + return map[string][]mTypes.SignatureInfo{ + "fake-signature": { + { + SignatureManifestDigest: signatureDigest, + LayersInfo: []mTypes.LayerInfo{}, + }, + }, } } diff --git a/test/blackbox/cloud_only.bats b/test/blackbox/cloud_only.bats index 5d970ffc..2c8e38d4 100644 --- a/test/blackbox/cloud_only.bats +++ b/test/blackbox/cloud_only.bats @@ -41,8 +41,8 @@ function setup() { "region": "us-east-2", "cacheTablename": "BlobTable", "repoMetaTablename": "RepoMetadataTable", - "manifestDataTablename": "ManifestDataTable", - "indexDataTablename": "IndexDataTable", + "imageMetaTablename": "ImageMetaTable", + "repoBlobsInfoTablename": "RepoBlobsInfoTable", "userDataTablename": "UserDataTable", "apiKeyTablename":"ApiKeyTable", "versionTablename": "Version"