zot/pkg/meta/parse_test.go

package meta_test

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path"
	"testing"

	godigest "github.com/opencontainers/go-digest"
	ispec "github.com/opencontainers/image-spec/specs-go/v1"
	. "github.com/smartystreets/goconvey/convey"

	zerr "zotregistry.io/zot/errors"
	zcommon "zotregistry.io/zot/pkg/common"
	"zotregistry.io/zot/pkg/extensions/monitoring"
	"zotregistry.io/zot/pkg/log"
	"zotregistry.io/zot/pkg/meta"
	"zotregistry.io/zot/pkg/meta/boltdb"
	"zotregistry.io/zot/pkg/meta/dynamodb"
	mTypes "zotregistry.io/zot/pkg/meta/types"
	"zotregistry.io/zot/pkg/storage"
	"zotregistry.io/zot/pkg/storage/local"
	storageTypes "zotregistry.io/zot/pkg/storage/types"
	"zotregistry.io/zot/pkg/test"
	"zotregistry.io/zot/pkg/test/mocks"
)

const repo = "repo"

func TestParseStorageErrors(t *testing.T) {
	Convey("ParseStorage", t, func() {
		imageStore := mocks.MockedImageStore{
			GetIndexContentFn: func(repo string) ([]byte, error) {
				return nil, ErrTestError
			},
			GetRepositoriesFn: func() ([]string, error) {
				return []string{"repo1", "repo2"}, nil
			},
		}
		storeController := storage.StoreController{DefaultStore: imageStore}
		metaDB := mocks.MetaDBMock{}

		// sync repo fail
		err := meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
		So(err, ShouldNotBeNil)

		Convey("getAllRepos errors", func() {
			imageStore1 := mocks.MockedImageStore{
				GetRepositoriesFn: func() ([]string, error) {
					return []string{"repo1", "repo2"}, nil
				},
			}
			imageStore2 := mocks.MockedImageStore{
				GetRepositoriesFn: func() ([]string, error) {
					return nil, ErrTestError
				},
			}
			storeController := storage.StoreController{
				DefaultStore: imageStore1,
				SubStore: map[string]storageTypes.ImageStore{
					"a": imageStore2,
				},
			}

			err := meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
			So(err, ShouldNotBeNil)
		})
	})

	Convey("Parse Repo", t, func() {
		imageStore := mocks.MockedImageStore{}
		storeController := storage.StoreController{DefaultStore: &imageStore}
		metaDB := mocks.MetaDBMock{}
		log := log.NewLogger("debug", "")

		Convey("imageStore.GetIndexContent errors", func() {
			imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
				return nil, ErrTestError
			}

			err := meta.ParseRepo("repo", metaDB, storeController, log)
			So(err, ShouldNotBeNil)
		})

		Convey("json.Unmarshal errors", func() {
			imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
				return []byte("Invalid JSON"), nil
			}

			err := meta.ParseRepo("repo", metaDB, storeController, log)
			So(err, ShouldNotBeNil)
		})

		Convey("resetRepoMetaTags errors", func() {
			imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
				return []byte("{}"), nil
			}

			Convey("metaDB.GetRepoMeta errors", func() {
				metaDB.GetRepoMetaFn = func(repo string) (mTypes.RepoMetadata, error) {
					return mTypes.RepoMetadata{}, ErrTestError
				}

				err := meta.ParseRepo("repo", metaDB, storeController, log)
				So(err, ShouldNotBeNil)
			})
		})

		Convey("isManifestMetaPresent errors", func() {
			indexContent := ispec.Index{
				Manifests: []ispec.Descriptor{
					{
						Digest:    godigest.FromString("manifest1"),
						MediaType: ispec.MediaTypeImageManifest,
						Annotations: map[string]string{
							ispec.AnnotationRefName: "tag1",
						},
					},
				},
			}
			indexBlob, err := json.Marshal(indexContent)
			So(err, ShouldBeNil)

			imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
				return indexBlob, nil
			}

			Convey("metaDB.GetManifestMeta errors", func() {
				metaDB.GetManifestMetaFn = func(repo string, manifestDigest godigest.Digest) (mTypes.ManifestMetadata, error) {
					return mTypes.ManifestMetadata{}, ErrTestError
				}

				err = meta.ParseRepo("repo", metaDB, storeController, log)
				So(err, ShouldNotBeNil)
			})
		})

		Convey("manifestMetaIsPresent true", func() {
			indexContent := ispec.Index{
				Manifests: []ispec.Descriptor{
					{
						Digest:    godigest.FromString("manifest1"),
						MediaType: ispec.MediaTypeImageManifest,
						Annotations: map[string]string{
							ispec.AnnotationRefName: "tag1",
						},
					},
				},
			}
			indexBlob, err := json.Marshal(indexContent)
			So(err, ShouldBeNil)

			imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
				return indexBlob, nil
			}

			Convey("metaDB.SetRepoReference", func() {
				metaDB.SetRepoReferenceFn = func(repo, tag string, manifestDigest godigest.Digest, mediaType string) error {
					return ErrTestError
				}

				err = meta.ParseRepo("repo", metaDB, storeController, log)
				So(err, ShouldNotBeNil)
			})
		})

		Convey("manifestMetaIsPresent false", func() {
			indexContent := ispec.Index{
				Manifests: []ispec.Descriptor{
					{
						Digest:    godigest.FromString("manifest1"),
						MediaType: ispec.MediaTypeImageManifest,
						Annotations: map[string]string{
							ispec.AnnotationRefName: "tag1",
						},
					},
				},
			}
			indexBlob, err := json.Marshal(indexContent)
			So(err, ShouldBeNil)

			imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
				return indexBlob, nil
			}

			metaDB.GetManifestMetaFn = func(repo string, manifestDigest godigest.Digest) (mTypes.ManifestMetadata, error) {
				return mTypes.ManifestMetadata{}, zerr.ErrManifestMetaNotFound
			}

			Convey("GetImageManifest errors", func() {
				imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
					return nil, "", "", ErrTestError
				}
				err = meta.ParseRepo("repo", metaDB, storeController, log)
				So(err, ShouldNotBeNil)
			})

			Convey("CheckIsImageSignature errors", func() {
				// CheckIsImageSignature will fail because of a invalid json
				imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
					return []byte("Invalid JSON"), "", "", nil
				}
				err = meta.ParseRepo("repo", metaDB, storeController, log)
				So(err, ShouldNotBeNil)
			})
			Convey("CheckIsImageSignature -> not signature", func() {
				manifestContent := ispec.Manifest{}
				manifestBlob, err := json.Marshal(manifestContent)
				So(err, ShouldBeNil)

				imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
					return manifestBlob, "", "", nil
				}

				Convey("imgStore.GetBlobContent errors", func() {
					imageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) {
						return nil, ErrTestError
					}

					err = meta.ParseRepo("repo", metaDB, storeController, log)
					So(err, ShouldNotBeNil)
				})
			})

			Convey("CheckIsImageSignature -> is signature", func() {
				manifestContent := ispec.Manifest{
					Subject: &ispec.Descriptor{
						Digest: "123",
					},
					ArtifactType: "application/vnd.cncf.notary.signature",
					Layers:       []ispec.Descriptor{{MediaType: ispec.MediaTypeImageLayer}},
				}

				manifestBlob, err := json.Marshal(manifestContent)
				So(err, ShouldBeNil)

				imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
					return manifestBlob, "", "", nil
				}

				metaDB.AddManifestSignatureFn = func(repo string, signedManifestDigest godigest.Digest,
					sm mTypes.SignatureMetadata,
				) error {
					return ErrTestError
				}

				err = meta.ParseRepo("repo", metaDB, storeController, log)
				So(err, ShouldNotBeNil)

				metaDB.AddManifestSignatureFn = func(repo string, signedManifestDigest godigest.Digest,
					sm mTypes.SignatureMetadata,
				) error {
					return nil
				}

				metaDB.UpdateSignaturesValidityFn = func(repo string, signedManifestDigest godigest.Digest,
				) error {
					return ErrTestError
				}

				err = meta.ParseRepo("repo", metaDB, storeController, log)
				So(err, ShouldNotBeNil)
			})

			Convey("GetSignatureLayersInfo errors", func() {
				// get notation signature layers info
				badNotationManifestContent := ispec.Manifest{
					Subject: &ispec.Descriptor{
						Digest: "123",
					},
					ArtifactType: "application/vnd.cncf.notary.signature",
				}

				badNotationManifestBlob, err := json.Marshal(badNotationManifestContent)
				So(err, ShouldBeNil)

				imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
					return badNotationManifestBlob, "", "", nil
				}

				// wrong number of layers of notation manifest
				err = meta.ParseRepo("repo", metaDB, storeController, log)
				So(err, ShouldNotBeNil)

				notationManifestContent := ispec.Manifest{
					Subject: &ispec.Descriptor{
						Digest: "123",
					},
					ArtifactType: "application/vnd.cncf.notary.signature",
					Layers:       []ispec.Descriptor{{MediaType: ispec.MediaTypeImageLayer}},
				}

				notationManifestBlob, err := json.Marshal(notationManifestContent)
				So(err, ShouldBeNil)

				imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
					return notationManifestBlob, "", "", nil
				}

				imageStore.GetBlobContentFn = func(repo string, digest godigest.Digest) ([]byte, error) {
					return []byte{}, ErrTestError
				}

				// unable to get layer content
				err = meta.ParseRepo("repo", metaDB, storeController, log)
				So(err, ShouldNotBeNil)

				_, _, cosignManifestContent, _ := test.GetRandomImageComponents(10) //nolint:staticcheck
				_, _, signedManifest, _ := test.GetRandomImageComponents(10)        //nolint:staticcheck
				signatureTag, err := test.GetCosignSignatureTagForManifest(signedManifest)
				So(err, ShouldBeNil)

				cosignManifestContent.Annotations = map[string]string{ispec.AnnotationRefName: signatureTag}

				cosignManifestBlob, err := json.Marshal(cosignManifestContent)
				So(err, ShouldBeNil)

				imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
					return cosignManifestBlob, "", "", nil
				}

				indexContent := ispec.Index{
					Manifests: []ispec.Descriptor{
						{
							Digest:    godigest.FromString("cosignSig"),
							MediaType: ispec.MediaTypeImageManifest,
							Annotations: map[string]string{
								ispec.AnnotationRefName: signatureTag,
							},
						},
					},
				}
				indexBlob, err := json.Marshal(indexContent)
				So(err, ShouldBeNil)

				imageStore.GetIndexContentFn = func(repo string) ([]byte, error) {
					return indexBlob, nil
				}

				// unable to get layer content
				err = meta.ParseRepo("repo", metaDB, storeController, log)
				So(err, ShouldNotBeNil)
			})
		})
	})
}

func TestParseStorageWithBoltDB(t *testing.T) {
	Convey("Boltdb", t, func() {
		rootDir := t.TempDir()

		boltDB, err := boltdb.GetBoltDriver(boltdb.DBParameters{
			RootDir: rootDir,
		})
		So(err, ShouldBeNil)

		metaDB, err := boltdb.New(boltDB, log.NewLogger("debug", ""))
		So(err, ShouldBeNil)

		RunParseStorageTests(rootDir, metaDB)
	})
}

func TestParseStorageDynamoWrapper(t *testing.T) {
	skipIt(t)

	Convey("Dynamodb", t, func() {
		rootDir := t.TempDir()

		params := dynamodb.DBDriverParameters{
			Endpoint:              os.Getenv("DYNAMODBMOCK_ENDPOINT"),
			Region:                "us-east-2",
			RepoMetaTablename:     "RepoMetadataTable",
			ManifestDataTablename: "ManifestDataTable",
			IndexDataTablename:    "IndexDataTable",
			UserDataTablename:     "UserDataTable",
			APIKeyTablename:       "ApiKeyTable",
			VersionTablename:      "Version",
		}

		dynamoClient, err := dynamodb.GetDynamoClient(params)
		So(err, ShouldBeNil)

		dynamoWrapper, err := dynamodb.New(dynamoClient, params, log.NewLogger("debug", ""))
		So(err, ShouldBeNil)

		err = dynamoWrapper.ResetManifestDataTable()
		So(err, ShouldBeNil)

		err = dynamoWrapper.ResetRepoMetaTable()
		So(err, ShouldBeNil)

		RunParseStorageTests(rootDir, dynamoWrapper)
	})
}

func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) {
	Convey("Test with simple case", func() {
		imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false,
			log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)

		storeController := storage.StoreController{DefaultStore: imageStore}
		manifests := []ispec.Manifest{}
		for i := 0; i < 3; i++ {
			config, layers, manifest, err := test.GetRandomImageComponents(100) //nolint:staticcheck
			So(err, ShouldBeNil)

			manifests = append(manifests, manifest)

			err = test.WriteImageToFileSystem(
				test.Image{
					Config:   config,
					Layers:   layers,
					Manifest: manifest,
				}, repo, fmt.Sprintf("tag%d", i), storeController)
			So(err, ShouldBeNil)
		}

		// add fake signature for tag1
		signatureTag, err := test.GetCosignSignatureTagForManifest(manifests[1])
		So(err, ShouldBeNil)

		manifestBlob, err := json.Marshal(manifests[1])
		So(err, ShouldBeNil)

		signedManifestDigest := godigest.FromBytes(manifestBlob)

		config, layers, manifest, err := test.GetRandomImageComponents(100) //nolint:staticcheck
		So(err, ShouldBeNil)

		err = test.WriteImageToFileSystem(
			test.Image{
				Config:   config,
				Layers:   layers,
				Manifest: manifest,
			}, repo, signatureTag, storeController)
		So(err, ShouldBeNil)

		// remove tag2 from index.json
		indexPath := path.Join(rootDir, repo, "index.json")
		indexFile, err := os.Open(indexPath)
		So(err, ShouldBeNil)
		buf, err := io.ReadAll(indexFile)
		So(err, ShouldBeNil)

		var index ispec.Index
		if err = json.Unmarshal(buf, &index); err == nil {
			for _, manifest := range index.Manifests {
				if val, ok := manifest.Annotations[ispec.AnnotationRefName]; ok && val == "tag2" {
					delete(manifest.Annotations, ispec.AnnotationRefName)

					break
				}
			}
		}
		buf, err = json.Marshal(index)
		So(err, ShouldBeNil)

		err = os.WriteFile(indexPath, buf, 0o600)
		So(err, ShouldBeNil)

		err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
		So(err, ShouldBeNil)

		repos, err := metaDB.GetMultipleRepoMeta(context.Background(),
			func(repoMeta mTypes.RepoMetadata) bool { return true })
		So(err, ShouldBeNil)

		So(len(repos), ShouldEqual, 1)
		So(len(repos[0].Tags), ShouldEqual, 2)

		for _, descriptor := range repos[0].Tags {
			manifestMeta, err := metaDB.GetManifestMeta(repo, godigest.Digest(descriptor.Digest))
			So(err, ShouldBeNil)
			So(manifestMeta.ManifestBlob, ShouldNotBeNil)
			So(manifestMeta.ConfigBlob, ShouldNotBeNil)

			if descriptor.Digest == signedManifestDigest.String() {
				So(manifestMeta.Signatures, ShouldNotBeEmpty)
			}
		}
	})

	Convey("Accept orphan signatures", func() {
		imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false,
			log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)

		storeController := storage.StoreController{DefaultStore: imageStore}
		// add an image
		config, layers, manifest, err := test.GetRandomImageComponents(100) //nolint:staticcheck
		So(err, ShouldBeNil)

		err = test.WriteImageToFileSystem(
			test.Image{
				Config:   config,
				Layers:   layers,
				Manifest: manifest,
			}, repo, "tag1", storeController)
		So(err, ShouldBeNil)

		// add mock cosign signature without pushing the signed image
		image, err := test.GetRandomImage() //nolint:staticcheck
		So(err, ShouldBeNil)

		signatureTag, err := test.GetCosignSignatureTagForManifest(image.Manifest)
		So(err, ShouldBeNil)

		missingImageDigest := image.Digest()

		// get the body of the signature
		config, layers, manifest, err = test.GetRandomImageComponents(100) //nolint:staticcheck
		So(err, ShouldBeNil)

		err = test.WriteImageToFileSystem(
			test.Image{
				Config:   config,
				Layers:   layers,
				Manifest: manifest,
			}, repo, signatureTag, storeController)
		So(err, ShouldBeNil)

		err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
		So(err, ShouldBeNil)

		repos, err := metaDB.GetMultipleRepoMeta(
			context.Background(),
			func(repoMeta mTypes.RepoMetadata) bool { return true },
		)
		So(err, ShouldBeNil)

		for _, desc := range repos[0].Tags {
			So(desc.Digest, ShouldNotResemble, missingImageDigest.String())
		}

		So(len(repos), ShouldEqual, 1)
		So(repos[0].Tags, ShouldContainKey, "tag1")
		So(repos[0].Tags, ShouldNotContainKey, signatureTag)
		So(repos[0].Signatures, ShouldContainKey, missingImageDigest.String())
	})

	Convey("Check statistics after load", func() {
		imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false,
			log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil)

		storeController := storage.StoreController{DefaultStore: imageStore}
		// add an image
		image, err := test.GetRandomImage() //nolint:staticcheck
		So(err, ShouldBeNil)

		manifestDigest := image.Digest()

		err = test.WriteImageToFileSystem(image, repo, "tag", storeController)
		So(err, ShouldBeNil)

		err = metaDB.SetRepoReference(repo, "tag", manifestDigest, ispec.MediaTypeImageManifest)
		So(err, ShouldBeNil)

		err = metaDB.IncrementRepoStars(repo)
		So(err, ShouldBeNil)
		err = metaDB.IncrementImageDownloads(repo, "tag")
		So(err, ShouldBeNil)
		err = metaDB.IncrementImageDownloads(repo, "tag")
		So(err, ShouldBeNil)
		err = metaDB.IncrementImageDownloads(repo, "tag")
		So(err, ShouldBeNil)

		repoMeta, err := metaDB.GetRepoMeta(repo)
		So(err, ShouldBeNil)

		So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 3)
		So(repoMeta.Stars, ShouldEqual, 1)

		err = meta.ParseStorage(metaDB, storeController, log.NewLogger("debug", ""))
		So(err, ShouldBeNil)

		repoMeta, err = metaDB.GetRepoMeta(repo)
		So(err, ShouldBeNil)

		So(repoMeta.Statistics[manifestDigest.String()].DownloadCount, ShouldEqual, 3)
		So(repoMeta.Stars, ShouldEqual, 1)
	})
}

func TestGetReferredInfo(t *testing.T) {
	Convey("GetReferredInfo error", t, func() {
		_, _, _, err := meta.GetReferredInfo([]byte("bad json"), "digest", ispec.MediaTypeImageManifest)
		So(err, ShouldNotBeNil)

		_, _, _, err = meta.GetReferredInfo([]byte("bad json"), "digest", ispec.MediaTypeImageIndex)
		So(err, ShouldNotBeNil)
	})
}

func skipIt(t *testing.T) {
	t.Helper()

	if os.Getenv("S3MOCK_ENDPOINT") == "" {
		t.Skip("Skipping testing without AWS S3 mock server")
	}
}

func TestGetSignatureLayersInfo(t *testing.T) {
	Convey("wrong signature type", t, func() {
		layers, err := meta.GetSignatureLayersInfo("repo", "tag", "123", "wrong signature type", []byte{},
			nil, log.NewLogger("debug", ""))
		So(err, ShouldBeNil)
		So(layers, ShouldBeEmpty)
	})

	Convey("error while unmarshaling manifest content", t, func() {
		_, err := meta.GetSignatureLayersInfo("repo", "tag", "123", zcommon.CosignSignature, []byte("bad manifest"),
			nil, log.NewLogger("debug", ""))
		So(err, ShouldNotBeNil)

		_, err = meta.GetSignatureLayersInfo("repo", "tag", "123", zcommon.NotationSignature, []byte("bad manifest"),
			nil, log.NewLogger("debug", ""))
		So(err, ShouldNotBeNil)
	})
}