0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00

feat(repodb): update referrers api to use repodb (#1230)

Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae 2023-03-10 20:37:29 +02:00 committed by GitHub
parent c731acf6de
commit 5d1f91a79f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
37 changed files with 2011 additions and 201 deletions

View file

@ -89,11 +89,10 @@ jobs:
echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container
localstack wait -t 30 # to become ready before timing out
echo "Startup complete"
echo "Startup complete"
aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}"
aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name RepoMetadataTable --attribute-definitions AttributeName=RepoName,AttributeType=S --key-schema AttributeName=RepoName,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name ManifestDataTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
env:
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake

View file

@ -33,6 +33,9 @@ jobs:
make bin/skopeo
sudo cp bin/skopeo /usr/bin
skopeo -v
- name: Run referrers tests
run: |
make test-bats-referrers
- name: Run push-pull tests
run: |
make test-push-pull

View file

@ -298,6 +298,11 @@ test-push-pull: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM)
test-push-pull-verbose: binary check-skopeo $(BATS)
$(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull.bats
.PHONY: test-bats-referrers
test-bats-referrers: EXTENSIONS=search
test-bats-referrers: binary check-skopeo $(BATS) $(ORAS)
$(BATS) --trace --print-output-on-failure test/blackbox/referrers.bats
.PHONY: test-cloud-only
test-cloud-only: binary check-skopeo $(BATS)
$(BATS) --trace --print-output-on-failure test/blackbox/cloud-only.bats

View file

@ -63,6 +63,7 @@ var (
ErrManifestConflict = errors.New("manifest: multiple manifests found")
ErrManifestMetaNotFound = errors.New("repodb: image metadata not found for given manifest reference")
ErrManifestDataNotFound = errors.New("repodb: image data not found for given manifest digest")
ErrArtifactDataNotFound = errors.New("repodb: artifact data not found for given digest")
ErrIndexDataNotFount = errors.New("repodb: index data not found for given digest")
ErrRepoMetaNotFound = errors.New("repodb: repo metadata not found for given repo name")
ErrTagMetaNotFound = errors.New("repodb: tag metadata not found for given repo and tag names")

View file

@ -20,6 +20,7 @@
"cacheTablename": "ZotBlobTable",
"repoMetaTablename": "ZotRepoMetadataTable",
"manifestDataTablename": "ZotManifestDataTable",
"artifactDataTablename": "ZotArtifactDataTable",
"versionTablename": "ZotVersion"
}
},

View file

@ -512,7 +512,7 @@ func (c *Controller) InitRepoDB(reloadCtx context.Context) error {
return err
}
err = repodb.SyncRepoDB(driver, c.StoreController, c.Log)
err = repodb.ParseStorage(driver, c.StoreController, c.Log)
if err != nil {
return err
}
@ -554,6 +554,9 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d
indexDataTablename, ok := toStringIfOk(cacheDriverConfig, "indexdatatablename", log)
allParametersOk = allParametersOk && ok
artifactDataTablename, ok := toStringIfOk(cacheDriverConfig, "artifactdatatablename", log)
allParametersOk = allParametersOk && ok
versionTablename, ok := toStringIfOk(cacheDriverConfig, "versiontablename", log)
allParametersOk = allParametersOk && ok
@ -567,6 +570,7 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) d
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
VersionTablename: versionTablename,
}
}

View file

@ -170,6 +170,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
"cacheTablename": "BlobTable",
"repoMetaTablename": "RepoMetadataTable",
"manifestDataTablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"versionTablename": "Version",
}
@ -184,6 +185,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
"cacheTablename": "BlobTable",
"repoMetaTablename": "RepoMetadataTable",
"manifestDataTablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"versionTablename": "Version",
}
@ -197,6 +199,7 @@ func TestCreateCacheDatabaseDriver(t *testing.T) {
"cacheTablename": "BlobTable",
"repoMetaTablename": "RepoMetadataTable",
"manifestDataTablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"versionTablename": "Version",
}
@ -229,6 +232,7 @@ func TestCreateRepoDBDriver(t *testing.T) {
"cachetablename": "BlobTable",
"repometatablename": "RepoMetadataTable",
"manifestdatatablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
}
testFunc := func() { _, _ = api.CreateRepoDBDriver(conf.Storage.StorageConfig, log) }
@ -241,6 +245,7 @@ func TestCreateRepoDBDriver(t *testing.T) {
"cachetablename": "",
"repometatablename": "RepoMetadataTable",
"manifestdatatablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"versiontablename": 1,
}
@ -2574,6 +2579,7 @@ func TestAuthorizationWithBasicAuth(t *testing.T) {
manifestBlob := resp.Body()
var manifest ispec.Manifest
err = json.Unmarshal(manifestBlob, &manifest)
So(err, ShouldBeNil)

View file

@ -1008,6 +1008,7 @@ func TestGetReferrersGQL(t *testing.T) {
},
Subject: subjectDescriptor,
ArtifactType: artifactType,
MediaType: ispec.MediaTypeArtifactManifest,
Annotations: map[string]string{
"com.artifact.format": "test",
},
@ -1017,14 +1018,15 @@ func TestGetReferrersGQL(t *testing.T) {
So(err, ShouldBeNil)
artifactManifestDigest := godigest.FromBytes(artifactManifestBlob)
err = UploadArtifact(baseURL, repo, artifact)
err = UploadArtifactManifest(artifact, baseURL, repo)
So(err, ShouldBeNil)
gqlQuery := `
{Referrers(
repo: "%s",
digest: "%s",
type: ""
{
Referrers(
repo: "%s",
digest: "%s",
type: ""
){
ArtifactType,
Digest,
@ -5356,20 +5358,15 @@ func TestRepoDBWhenReadingImages(t *testing.T) {
func TestRepoDBWhenDeletingImages(t *testing.T) {
Convey("Setting up zot repo with test images", t, func() {
subpath := "/a"
dir := t.TempDir()
subDir := t.TempDir()
subRootDir := path.Join(subDir, subpath)
port := GetFreePort()
baseURL := GetBaseURL(port)
conf := config.New()
conf.HTTP.Port = port
conf.Storage.RootDirectory = dir
conf.Storage.SubPaths = make(map[string]config.StorageConfig)
conf.Storage.SubPaths[subpath] = config.StorageConfig{RootDirectory: subRootDir}
conf.Storage.GC = false
defaultVal := true
conf.Extensions = &extconf.ExtensionConfig{
Search: &extconf.SearchConfig{BaseConfig: extconf.BaseConfig{Enable: &defaultVal}},
@ -5384,46 +5381,32 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
defer ctlrManager.StopServer()
// push test images to repo 1 image 1
config1, layers1, manifest1, err := GetImageComponents(100)
image1, err := GetRandomImage("1.0.1")
So(err, ShouldBeNil)
layersSize1 := 0
for _, l := range layers1 {
layersSize1 += len(l)
}
err = UploadImage(
Image{
Manifest: manifest1,
Config: config1,
Layers: layers1,
Reference: "1.0.1",
},
image1,
baseURL,
"repo1",
)
So(err, ShouldBeNil)
// push test images to repo 1 image 2
config2, layers2, manifest2, err := GetImageComponents(200)
So(err, ShouldBeNil)
createdTime2 := time.Date(2009, 1, 1, 12, 0, 0, 0, time.UTC)
config2.History = append(config2.History, ispec.History{Created: &createdTime2})
manifest2, err = updateManifestConfig(manifest2, config2)
image2, err := GetImageWithConfig(ispec.Image{
Created: &createdTime2,
History: []ispec.History{
{
Created: &createdTime2,
},
},
})
So(err, ShouldBeNil)
layersSize2 := 0
for _, l := range layers2 {
layersSize2 += len(l)
}
image2.Reference = "1.0.2"
err = UploadImage(
Image{
Manifest: manifest2,
Config: config2,
Layers: layers2,
Reference: "1.0.2",
},
image2,
baseURL,
"repo1",
)
@ -5439,22 +5422,6 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
LastUpdated Size
}
}
Repos {
Name LastUpdated Size
Platforms { Os Arch }
Vendors Score
NewestImage {
RepoName Tag LastUpdated Size IsSigned
Manifests{
Platform { Os Arch }
LastUpdated Size
}
}
}
Layers {
Digest
Size
}
}
}`
@ -5627,7 +5594,7 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
So(sigManifestContent, ShouldNotBeZeroValue)
// check notation signature
manifest1Blob, err := json.Marshal(manifest1)
manifest1Blob, err := json.Marshal(image1.Manifest)
So(err, ShouldBeNil)
manifest1Digest := godigest.FromBytes(manifest1Blob)
So(sigManifestContent.Subject, ShouldNotBeNil)
@ -5653,6 +5620,65 @@ func TestRepoDBWhenDeletingImages(t *testing.T) {
So(responseStruct.GlobalSearchResult.GlobalSearch.Images[0].IsSigned, ShouldBeFalse)
})
Convey("Delete a referrer", func() {
referredImageDigest, err := image1.Digest()
So(err, ShouldBeNil)
referrerImage, err := GetImageWithSubject(referredImageDigest, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
err = UploadImage(
referrerImage,
baseURL,
"repo1",
)
So(err, ShouldBeNil)
// ------- check referrers for this image
query := fmt.Sprintf(`
{
Referrers(repo:"repo1", digest:"%s"){
MediaType
Digest
}
}`, referredImageDigest.String())
resp, err = resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query))
So(resp, ShouldNotBeNil)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct := &ReferrersResp{}
err = json.Unmarshal(resp.Body(), responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.ReferrersResult.Referrers), ShouldEqual, 1)
So(responseStruct.ReferrersResult.Referrers[0].Digest, ShouldResemble, referrerImage.Reference)
statusCode, err := DeleteImage("repo1", referrerImage.Reference, "badURL")
So(err, ShouldNotBeNil)
So(statusCode, ShouldEqual, -1)
// ------- Delete the referrer and see if it disappears from repoDB also
statusCode, err = DeleteImage("repo1", referrerImage.Reference, baseURL)
So(err, ShouldBeNil)
So(statusCode, ShouldEqual, http.StatusAccepted)
resp, err = resty.R().Get(baseURL + graphqlQueryPrefix + "?query=" + url.QueryEscape(query))
So(resp, ShouldNotBeNil)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
responseStruct = &ReferrersResp{}
err = json.Unmarshal(resp.Body(), responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.ReferrersResult.Referrers), ShouldEqual, 0)
})
Convey("Deleting causes errors", func() {
Convey("error while backing up the manifest", func() {
ctlr.StoreController.DefaultStore = mocks.MockedImageStore{

View file

@ -627,6 +627,22 @@ func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta repodb.RepoMetadata
return summary, imageSummaries
}
func StringMap2Annotations(strMap map[string]string) []*gql_generated.Annotation {
annotations := make([]*gql_generated.Annotation, 0, len(strMap))
for key, value := range strMap {
key := key
value := value
annotations = append(annotations, &gql_generated.Annotation{
Key: &key,
Value: &value,
})
}
return annotations
}
func GetPreloads(ctx context.Context) map[string]bool {
if !graphql.HasOperationContext(ctx) {
return map[string]bool{}

View file

@ -317,7 +317,7 @@ func TestImageFormat(t *testing.T) {
})
So(err, ShouldBeNil)
err = repodb.SyncRepoDB(repoDB, storeController, log)
err = repodb.ParseStorage(repoDB, storeController, log)
So(err, ShouldBeNil)
cveInfo := cveinfo.NewCVEInfo(storeController, repoDB, "", log)

View file

@ -88,7 +88,7 @@ func TestMultipleStoragePath(t *testing.T) {
})
So(err, ShouldBeNil)
err = repodb.SyncRepoDB(repoDB, storeController, log)
err = repodb.ParseStorage(repoDB, storeController, log)
So(err, ShouldBeNil)
scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log)
@ -178,7 +178,7 @@ func TestTrivyLibraryErrors(t *testing.T) {
})
So(err, ShouldBeNil)
err = repodb.SyncRepoDB(repoDB, storeController, log)
err = repodb.ParseStorage(repoDB, storeController, log)
So(err, ShouldBeNil)
scanner := NewScanner(storeController, repoDB, "ghcr.io/project-zot/trivy-db", log)
@ -424,7 +424,7 @@ func TestDefaultTrivyDBUrl(t *testing.T) {
})
So(err, ShouldBeNil)
err = repodb.SyncRepoDB(repoDB, storeController, log)
err = repodb.ParseStorage(repoDB, storeController, log)
So(err, ShouldBeNil)
// Use empty string for DB repository, the default url would be used internally

View file

@ -1193,41 +1193,33 @@ func getImageList(ctx context.Context, repo string, repoDB repodb.RepoDB, cveInf
}, nil
}
func getReferrers(store storage.ImageStore, repoName string, digest string, artifactTypes []string, log log.Logger) (
[]*gql_generated.Referrer, error,
) {
results := make([]*gql_generated.Referrer, 0)
func getReferrers(repoDB repodb.RepoDB, repo string, referredDigest string, artifactTypes []string,
log log.Logger,
) ([]*gql_generated.Referrer, error) {
refDigest := godigest.Digest(referredDigest)
if err := refDigest.Validate(); err != nil {
log.Error().Err(err).Msgf("graphql: bad digest string from request '%s'", referredDigest)
index, err := store.GetReferrers(repoName, godigest.Digest(digest), artifactTypes)
if err != nil {
log.Error().Err(err).Msg("error extracting referrers list")
return results, err
return []*gql_generated.Referrer{}, errors.Wrapf(err, "graphql: bad digest string from request '%s'",
referredDigest)
}
for _, manifest := range index.Manifests {
size := int(manifest.Size)
digest := manifest.Digest.String()
annotations := make([]*gql_generated.Annotation, 0)
artifactType := manifest.ArtifactType
mediaType := manifest.MediaType
referrers, err := repoDB.GetFilteredReferrersInfo(repo, refDigest, artifactTypes)
if err != nil {
return nil, err
}
for k, v := range manifest.Annotations {
key := k
value := v
results := make([]*gql_generated.Referrer, 0, len(referrers))
annotations = append(annotations, &gql_generated.Annotation{
Key: &key,
Value: &value,
})
}
for _, referrer := range referrers {
referrer := referrer
results = append(results, &gql_generated.Referrer{
MediaType: &mediaType,
ArtifactType: &artifactType,
Digest: &digest,
Size: &size,
Annotations: annotations,
MediaType: &referrer.MediaType,
ArtifactType: &referrer.ArtifactType,
Digest: &referrer.Digest,
Size: &referrer.Size,
Annotations: convert.StringMap2Annotations(referrer.Annotations),
})
}

View file

@ -1362,15 +1362,25 @@ func TestImageList(t *testing.T) {
func TestGetReferrers(t *testing.T) {
Convey("getReferrers", t, func() {
referredDigest := godigest.FromString("t").String()
Convey("referredDigest is empty", func() {
testLogger := log.NewLogger("debug", "")
_, err := getReferrers(mocks.RepoDBMock{}, "test", "", nil, testLogger)
So(err, ShouldNotBeNil)
})
Convey("GetReferrers returns error", func() {
testLogger := log.NewLogger("debug", "")
mockedStore := mocks.MockedImageStore{
GetReferrersFn: func(repo string, digest godigest.Digest, artifactType []string) (ispec.Index, error) {
return ispec.Index{}, ErrTestError
mockedStore := mocks.RepoDBMock{
GetFilteredReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string,
) ([]repodb.ReferrerInfo, error) {
return nil, ErrTestError
},
}
_, err := getReferrers(mockedStore, "test", "", nil, testLogger)
_, err := getReferrers(mockedStore, "test", referredDigest, nil, testLogger)
So(err, ShouldNotBeNil)
})
@ -1385,17 +1395,22 @@ func TestGetReferrers(t *testing.T) {
"key": "value",
},
}
mockedStore := mocks.MockedImageStore{
GetReferrersFn: func(repo string, digest godigest.Digest, artifactTypes []string) (ispec.Index, error) {
return ispec.Index{
Manifests: []ispec.Descriptor{
referrerDescriptor,
mockedStore := mocks.RepoDBMock{
GetFilteredReferrersInfoFn: func(repo string, referredDigest godigest.Digest, artifactTypes []string,
) ([]repodb.ReferrerInfo, error) {
return []repodb.ReferrerInfo{
{
Digest: referrerDescriptor.Digest.String(),
MediaType: referrerDescriptor.MediaType,
ArtifactType: referrerDescriptor.ArtifactType,
Size: int(referrerDescriptor.Size),
Annotations: referrerDescriptor.Annotations,
},
}, nil
},
}
referrers, err := getReferrers(mockedStore, "test", "", nil, testLogger)
referrers, err := getReferrers(mockedStore, "test", referredDigest, nil, testLogger)
So(err, ShouldBeNil)
So(*referrers[0].ArtifactType, ShouldEqual, referrerDescriptor.ArtifactType)
So(*referrers[0].MediaType, ShouldEqual, referrerDescriptor.MediaType)

View file

@ -130,9 +130,7 @@ func (r *queryResolver) Image(ctx context.Context, image string) (*gql_generated
// Referrers is the resolver for the Referrers field.
func (r *queryResolver) Referrers(ctx context.Context, repo string, digest string, typeArg []string) ([]*gql_generated.Referrer, error) {
store := r.storeController.GetImageStore(repo)
referrers, err := getReferrers(store, repo, digest, typeArg, r.log)
referrers, err := getReferrers(r.repoDB, repo, digest, typeArg, r.log)
if err != nil {
r.log.Error().Err(err).Msg("unable to get referrers from default store")

View file

@ -61,6 +61,11 @@ func NewBoltDBWrapper(params DBParameters) (*DBWrapper, error) {
return err
}
_, err = transaction.CreateBucketIfNotExists([]byte(repodb.ArtifactDataBucket))
if err != nil {
return err
}
_, err = transaction.CreateBucketIfNotExists([]byte(repodb.RepoMetadataBucket))
if err != nil {
return err
@ -133,6 +138,7 @@ func (bdw *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Diges
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{},
Signatures: map[string]repodb.ManifestSignatures{},
Referrers: map[string][]repodb.Descriptor{},
}
repoMetaBlob := repoBuck.Get([]byte(repo))
@ -259,6 +265,290 @@ func (bdw *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexDat
return indexMetadata, err
}
func (bdw DBWrapper) SetArtifactData(artifactDigest godigest.Digest, artifactData repodb.ArtifactData) error {
err := bdw.DB.Update(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.ArtifactDataBucket))
imBlob, err := json.Marshal(artifactData)
if err != nil {
return errors.Wrapf(err, "repodb: error while calculating blob for artifact with digest %s", artifactDigest)
}
err = buck.Put([]byte(artifactDigest), imBlob)
if err != nil {
return errors.Wrapf(err, "repodb: error while setting artifact blob for digest %s", artifactDigest)
}
return nil
})
return err
}
func (bdw DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) {
var artifactData repodb.ArtifactData
err := bdw.DB.View(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.ArtifactDataBucket))
blob := buck.Get([]byte(artifactDigest))
if len(blob) == 0 {
return zerr.ErrArtifactDataNotFound
}
err := json.Unmarshal(blob, &artifactData)
if err != nil {
return errors.Wrapf(err, "repodb: error while unmashaling artifact data for digest %s", artifactDigest)
}
return nil
})
return artifactData, err
}
func (bdw DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error {
err := bdw.DB.Update(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
repoMetaBlob := buck.Get([]byte(repo))
// object not found
if len(repoMetaBlob) == 0 {
var err error
// create a new object
repoMeta := repodb.RepoMetadata{
Name: repo,
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{
referredDigest.String(): {},
},
Signatures: map[string]repodb.ManifestSignatures{
referredDigest.String(): {},
},
Referrers: map[string][]repodb.Descriptor{
referredDigest.String(): {
{
Digest: referrer.Digest,
MediaType: referrer.MediaType,
},
},
},
}
repoMetaBlob, err = json.Marshal(repoMeta)
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
}
var repoMeta repodb.RepoMetadata
err := json.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil {
return err
}
refferers := repoMeta.Referrers[referredDigest.String()]
for i := range refferers {
if refferers[i].Digest == referrer.Digest {
return nil
}
}
refferers = append(refferers, repodb.Descriptor{
Digest: referrer.Digest,
MediaType: referrer.MediaType,
})
repoMeta.Referrers[referredDigest.String()] = refferers
repoMetaBlob, err = json.Marshal(repoMeta)
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
})
return err
}
func (bdw DBWrapper) DeleteReferrer(repo string, referredDigest godigest.Digest,
referrerDigest godigest.Digest,
) error {
return bdw.DB.Update(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
repoMetaBlob := buck.Get([]byte(repo))
if len(repoMetaBlob) == 0 {
return zerr.ErrRepoMetaNotFound
}
var repoMeta repodb.RepoMetadata
err := json.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil {
return err
}
referrers := repoMeta.Referrers[referredDigest.String()]
for i := range referrers {
if referrers[i].Digest == referrerDigest.String() {
referrers = append(referrers[:i], referrers[i+1:]...)
break
}
}
repoMeta.Referrers[referredDigest.String()] = referrers
repoMetaBlob, err = json.Marshal(repoMeta)
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
})
}
func (bdw DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) {
var referrers []repodb.Descriptor
err := bdw.DB.View(func(tx *bolt.Tx) error {
buck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
repoMetaBlob := buck.Get([]byte(repo))
if len(repoMetaBlob) == 0 {
return zerr.ErrRepoMetaNotFound
}
var repoMeta repodb.RepoMetadata
err := json.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil {
return err
}
referrers = repoMeta.Referrers[referredDigest.String()]
return nil
})
return referrers, err
}
func (bdw DBWrapper) GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest,
artifactTypes []string,
) ([]repodb.ReferrerInfo, error) {
referrersDescriptors, err := bdw.GetReferrers(repo, referredDigest)
if err != nil {
bdw.Log.Error().Msgf("repodb: failed to get referrers for '%s@%s'", repo, referredDigest.String())
return nil, err
}
referrersInfo := []repodb.ReferrerInfo{}
err = bdw.DB.View(func(tx *bolt.Tx) error {
artifactBuck := tx.Bucket([]byte(repodb.ArtifactDataBucket))
manifestBuck := tx.Bucket([]byte(repodb.ManifestDataBucket))
for _, descriptor := range referrersDescriptors {
referrerInfo := repodb.ReferrerInfo{}
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestDataBlob := manifestBuck.Get([]byte(descriptor.Digest))
if len(manifestDataBlob) == 0 {
bdw.Log.Error().Msgf("repodb: manifest data not found for digest %s", descriptor.Digest)
continue
}
var manifestData repodb.ManifestData
err = json.Unmarshal(manifestDataBlob, &manifestData)
if err != nil {
bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal manifest data for digest %s",
descriptor.Digest)
continue
}
var manifestContent ispec.Manifest
err := json.Unmarshal(manifestData.ManifestBlob, &manifestContent)
if err != nil {
bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal manifest for digest %s",
descriptor.Digest)
continue
}
referrerInfo = repodb.ReferrerInfo{
Digest: descriptor.Digest,
MediaType: ispec.MediaTypeImageManifest,
ArtifactType: manifestContent.Config.MediaType,
Size: len(manifestData.ManifestBlob),
Annotations: manifestContent.Annotations,
}
case ispec.MediaTypeArtifactManifest:
artifactDataBlob := artifactBuck.Get([]byte(descriptor.Digest))
if len(artifactDataBlob) == 0 {
bdw.Log.Error().Msgf("repodb: artifact data not found for digest %s", descriptor.Digest)
continue
}
var artifactData repodb.ArtifactData
err = json.Unmarshal(artifactDataBlob, &artifactData)
if err != nil {
bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal artifact data for digest %s", descriptor.Digest)
continue
}
manifestContent := ispec.Artifact{}
err := json.Unmarshal(artifactData.ManifestBlob, &manifestContent)
if err != nil {
bdw.Log.Error().Err(err).Msgf("repodb: can't unmarhsal artifact manifest for digest %s", descriptor.Digest)
continue
}
referrerInfo = repodb.ReferrerInfo{
Size: len(artifactData.ManifestBlob),
Digest: descriptor.Digest,
MediaType: manifestContent.MediaType,
Annotations: manifestContent.Annotations,
ArtifactType: manifestContent.ArtifactType,
}
}
if !common.MatchesArtifactTypes(referrerInfo.ArtifactType, artifactTypes) {
continue
}
referrersInfo = append(referrersInfo, referrerInfo)
}
return nil
})
return referrersInfo, err
}
func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest,
mediaType string,
) error {
@ -274,13 +564,16 @@ func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDi
// object not found
if len(repoMetaBlob) == 0 {
var err error
repoMetaBlob, err = json.Marshal(repodb.RepoMetadata{
// create a new object
repoMeta := repodb.RepoMetadata{
Name: repo,
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{},
Signatures: map[string]repodb.ManifestSignatures{},
})
Referrers: map[string][]repodb.Descriptor{},
}
repoMetaBlob, err = json.Marshal(repoMeta)
if err != nil {
return err
}
@ -303,6 +596,7 @@ func (bdw *DBWrapper) SetRepoReference(repo string, reference string, manifestDi
repoMeta.Statistics[manifestDigest.String()] = repodb.DescriptorStatistics{DownloadCount: 0}
repoMeta.Signatures[manifestDigest.String()] = repodb.ManifestSignatures{}
repoMeta.Referrers[manifestDigest.String()] = []repodb.Descriptor{}
repoMetaBlob, err = json.Marshal(repoMeta)
if err != nil {

View file

@ -70,6 +70,142 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("GetArtifactData", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
artifactBuck := tx.Bucket([]byte(repodb.ArtifactDataBucket))
return artifactBuck.Put([]byte("artifactDigest"), []byte("wrong json"))
})
So(err, ShouldBeNil)
_, err = boltdbWrapper.GetArtifactData("artifactDigest")
So(err, ShouldNotBeNil)
})
Convey("SetReferrer", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
return repoBuck.Put([]byte("repo"), []byte("wrong json"))
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetReferrer("repo", "ref", repodb.Descriptor{})
So(err, ShouldNotBeNil)
})
Convey("DeleteReferrer", func() {
Convey("RepoMeta not found", func() {
err := boltdbWrapper.DeleteReferrer("r", "dig", "dig")
So(err, ShouldNotBeNil)
})
Convey("bad repo meta blob", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
return repoBuck.Put([]byte("repo"), []byte("wrong json"))
})
So(err, ShouldBeNil)
err = boltdbWrapper.DeleteReferrer("repo", "dig", "dig")
So(err, ShouldNotBeNil)
})
})
Convey("GetReferrers", func() {
Convey("RepoMeta not found", func() {
_, err := boltdbWrapper.GetReferrers("repo", "dig")
So(err, ShouldNotBeNil)
})
Convey("bad repo meta blob", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
return repoBuck.Put([]byte("repo"), []byte("wrong json"))
})
So(err, ShouldBeNil)
_, err = boltdbWrapper.GetReferrers("repo", "dig")
So(err, ShouldNotBeNil)
})
})
Convey("GetFilteredReferrersInfo", func() {
Convey("getReferrers fails", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))
return repoBuck.Put([]byte("repo"), []byte("wrong json"))
})
So(err, ShouldBeNil)
_, err = boltdbWrapper.GetFilteredReferrersInfo("repo", "", nil)
So(err, ShouldNotBeNil)
})
Convey("unmarshal erorrs", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
manifestBuck := tx.Bucket([]byte(repodb.ManifestDataBucket))
artifactBuck := tx.Bucket([]byte(repodb.ArtifactDataBucket))
err = manifestBuck.Put([]byte("manifestDataRef"), []byte("bad json"))
So(err, ShouldBeNil)
err = artifactBuck.Put([]byte("artifactDataRef"), []byte("bad json"))
So(err, ShouldBeNil)
badBlob, err := json.Marshal(repodb.ArtifactData{
ManifestBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
err = artifactBuck.Put([]byte("artifactManifestRef"), badBlob)
So(err, ShouldBeNil)
badBlob, err = json.Marshal(repodb.ManifestData{
ManifestBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
err = manifestBuck.Put([]byte("badManifest"), badBlob)
So(err, ShouldBeNil)
return nil
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{
Digest: "manifestDataRef",
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{
Digest: "artifactDataRef",
MediaType: ispec.MediaTypeArtifactManifest,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{
Digest: "badManifest",
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
err = boltdbWrapper.SetReferrer("repo", "refDigest", repodb.Descriptor{
Digest: "artifactManifestRef",
MediaType: ispec.MediaTypeArtifactManifest,
})
So(err, ShouldBeNil)
refInfo, err := boltdbWrapper.GetFilteredReferrersInfo("repo", "refDigest", nil)
So(err, ShouldBeNil)
So(len(refInfo), ShouldEqual, 0)
})
})
Convey("SetRepoReference", func() {
err := boltdbWrapper.DB.Update(func(tx *bbolt.Tx) error {
repoBuck := tx.Bucket([]byte(repodb.RepoMetadataBucket))

View file

@ -1,6 +1,7 @@
package common
import (
"encoding/json"
"strings"
"time"
@ -197,3 +198,38 @@ func containsString(strSlice []string, str string) bool {
return false
}
func GetReferredSubject(descriptorBlob []byte) (godigest.Digest, bool) {
var manifest ispec.Manifest
err := json.Unmarshal(descriptorBlob, &manifest)
if err != nil {
return "", false
}
if manifest.Subject == nil || manifest.Subject.Digest.String() == "" {
return "", false
}
return manifest.Subject.Digest, true
}
func MatchesArtifactTypes(descriptorMediaType string, artifactTypes []string) bool {
if len(artifactTypes) == 0 {
return true
}
found := false
for _, artifactType := range artifactTypes {
if artifactType != "" && descriptorMediaType != artifactType {
continue
}
found = true
break
}
return found
}

View file

@ -0,0 +1,24 @@
package common_test
import (
"testing"
. "github.com/smartystreets/goconvey/convey"
"zotregistry.io/zot/pkg/meta/repodb/common"
)
func TestUtils(t *testing.T) {
Convey("GetReferredSubject", t, func() {
_, err := common.GetReferredSubject([]byte("bad json"))
So(err, ShouldNotBeNil)
})
Convey("MatchesArtifactTypes", t, func() {
res := common.MatchesArtifactTypes("", nil)
So(res, ShouldBeTrue)
res = common.MatchesArtifactTypes("type", []string{"someOtherType"})
So(res, ShouldBeFalse)
})
}

View file

@ -29,6 +29,9 @@ func TestWrapperErrors(t *testing.T) {
repoMetaTablename := "RepoMetadataTable" + uuid.String()
manifestDataTablename := "ManifestDataTable" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
artifactDataTablename := "ArtifactDataTable" + uuid.String()
versionTablename := "Version" + uuid.String()
Convey("Create table errors", t, func() {
@ -52,6 +55,8 @@ func TestWrapperErrors(t *testing.T) {
Client: dynamodb.NewFromConfig(cfg),
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
VersionTablename: versionTablename,
Patches: version.GetDynamoDBPatches(),
Log: log.Logger{Logger: zerolog.New(os.Stdout)},
@ -91,6 +96,8 @@ func TestWrapperErrors(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
VersionTablename: versionTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
Patches: version.GetDynamoDBPatches(),
Log: log.Logger{Logger: zerolog.New(os.Stdout)},
}

View file

@ -25,6 +25,8 @@ import (
"zotregistry.io/zot/pkg/test"
)
const badTablename = "bad tablename"
func TestIterator(t *testing.T) {
const (
endpoint = "http://localhost:4566"
@ -40,6 +42,7 @@ func TestIterator(t *testing.T) {
manifestDataTablename := "ManifestDataTable" + uuid.String()
versionTablename := "Version" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
artifactDataTablename := "ArtifactDataTable" + uuid.String()
Convey("TestIterator", t, func() {
dynamoWrapper, err := dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{
@ -48,6 +51,7 @@ func TestIterator(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
VersionTablename: versionTablename,
})
So(err, ShouldBeNil)
@ -133,6 +137,7 @@ func TestWrapperErrors(t *testing.T) {
manifestDataTablename := "ManifestDataTable" + uuid.String()
versionTablename := "Version" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
artifactDataTablename := "ArtifactData" + uuid.String()
ctx := context.Background()
@ -143,6 +148,7 @@ func TestWrapperErrors(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
VersionTablename: versionTablename,
})
So(err, ShouldBeNil)
@ -222,6 +228,114 @@ func TestWrapperErrors(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("GetArtifactData", func() {
dynamoWrapper.ArtifactDataTablename = badTablename
_, err = dynamoWrapper.GetArtifactData("dig")
So(err, ShouldNotBeNil)
})
Convey("GetArtifactData unmarhsal error", func() {
err = setBadArtifactData(dynamoWrapper.Client, artifactDataTablename, "dig")
So(err, ShouldBeNil)
_, err = dynamoWrapper.GetArtifactData("dig")
So(err, ShouldNotBeNil)
})
Convey("SetReferrer client error", func() {
dynamoWrapper.RepoMetaTablename = badTablename
err := dynamoWrapper.SetReferrer("repo", "", repodb.Descriptor{})
So(err, ShouldNotBeNil)
})
Convey("SetReferrer bad repoMeta", func() {
err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo")
So(err, ShouldBeNil)
err = dynamoWrapper.SetReferrer("repo", "", repodb.Descriptor{})
So(err, ShouldNotBeNil)
})
Convey("GetReferrers client error", func() {
dynamoWrapper.RepoMetaTablename = badTablename
_, err := dynamoWrapper.GetReferrers("repo", "")
So(err, ShouldNotBeNil)
})
Convey("GetReferrers bad repoMeta", func() {
err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo")
So(err, ShouldBeNil)
_, err = dynamoWrapper.GetReferrers("repo", "")
So(err, ShouldNotBeNil)
})
Convey("DeleteReferrer client error", func() {
dynamoWrapper.RepoMetaTablename = badTablename
err := dynamoWrapper.DeleteReferrer("repo", "", "")
So(err, ShouldNotBeNil)
})
Convey("DeleteReferrer bad repoMeta", func() {
err := setBadRepoMeta(dynamoWrapper.Client, repoMetaTablename, "repo")
So(err, ShouldBeNil)
err = dynamoWrapper.DeleteReferrer("repo", "", "")
So(err, ShouldNotBeNil)
})
Convey("GetFilteredReferrersInfo GetReferrers errors", func() {
dynamoWrapper.RepoMetaTablename = badTablename
_, err := dynamoWrapper.GetFilteredReferrersInfo("repo", "", nil)
So(err, ShouldNotBeNil)
})
Convey("GetFilteredReferrersInfo getData fails", func() {
dynamoWrapper.ManifestDataTablename = badTablename
dynamoWrapper.ArtifactDataTablename = badTablename
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{
Digest: "dig1",
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{
Digest: "dig2",
MediaType: ispec.MediaTypeArtifactManifest,
})
So(err, ShouldBeNil)
_, err := dynamoWrapper.GetFilteredReferrersInfo("repo", "rf", nil)
So(err, ShouldBeNil)
})
Convey("GetFilteredReferrersInfo bad descriptor blob", func() {
err = dynamoWrapper.SetArtifactData("dig2", repodb.ArtifactData{
ManifestBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetManifestData("dig3", repodb.ManifestData{
ManifestBlob: []byte("bad json"),
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{
Digest: "dig2",
MediaType: ispec.MediaTypeArtifactManifest,
})
So(err, ShouldBeNil)
err = dynamoWrapper.SetReferrer("repo", "rf", repodb.Descriptor{
Digest: "dig3",
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
_, err := dynamoWrapper.GetFilteredReferrersInfo("repo", "rf", nil)
So(err, ShouldBeNil)
})
Convey("IncrementRepoStars GetRepoMeta error", func() {
err = dynamoWrapper.IncrementRepoStars("repo")
So(err, ShouldNotBeNil)
@ -688,6 +802,7 @@ func TestWrapperErrors(t *testing.T) {
RepoMetaTablename: "",
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
VersionTablename: versionTablename,
})
So(err, ShouldNotBeNil)
@ -698,6 +813,7 @@ func TestWrapperErrors(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: "",
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
VersionTablename: versionTablename,
})
So(err, ShouldNotBeNil)
@ -708,6 +824,7 @@ func TestWrapperErrors(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: "",
ArtifactDataTablename: artifactDataTablename,
VersionTablename: versionTablename,
})
So(err, ShouldNotBeNil)
@ -718,6 +835,7 @@ func TestWrapperErrors(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
VersionTablename: "",
})
So(err, ShouldNotBeNil)
@ -728,8 +846,20 @@ func TestWrapperErrors(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: "",
VersionTablename: versionTablename,
})
So(err, ShouldNotBeNil)
_, err = dynamo.NewDynamoDBWrapper(dynamoParams.DBDriverParameters{ //nolint:contextcheck
Endpoint: endpoint,
Region: region,
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
VersionTablename: versionTablename,
ArtifactDataTablename: artifactDataTablename,
})
So(err, ShouldBeNil)
})
}
@ -759,6 +889,31 @@ func setBadManifestData(client *dynamodb.Client, manifestDataTableName, digest s
return err
}
func setBadArtifactData(client *dynamodb.Client, artifactDataTablename, digest string) error {
mdAttributeValue, err := attributevalue.Marshal("string")
if err != nil {
return err
}
_, err = client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
ExpressionAttributeNames: map[string]string{
"#AD": "ArtifactData",
},
ExpressionAttributeValues: map[string]types.AttributeValue{
":ArtifactData": mdAttributeValue,
},
Key: map[string]types.AttributeValue{
"ArtifactDigest": &types.AttributeValueMemberS{
Value: digest,
},
},
TableName: aws.String(artifactDataTablename),
UpdateExpression: aws.String("SET #AD = :ArtifactData"),
})
return err
}
func setBadIndexData(client *dynamodb.Client, indexDataTableName, digest string) error {
mdAttributeValue, err := attributevalue.Marshal("string")
if err != nil {

View file

@ -32,6 +32,7 @@ type DBWrapper struct {
RepoMetaTablename string
IndexDataTablename string
ManifestDataTablename string
ArtifactDataTablename string
VersionTablename string
Patches []func(client *dynamodb.Client, tableNames map[string]string) error
Log log.Logger
@ -62,6 +63,7 @@ func NewDynamoDBWrapper(params dynamoParams.DBDriverParameters) (*DBWrapper, err
RepoMetaTablename: params.RepoMetaTablename,
ManifestDataTablename: params.ManifestDataTablename,
IndexDataTablename: params.IndexDataTablename,
ArtifactDataTablename: params.ArtifactDataTablename,
VersionTablename: params.VersionTablename,
Patches: version.GetDynamoDBPatches(),
Log: log.Logger{Logger: zerolog.New(os.Stdout)},
@ -82,6 +84,11 @@ func NewDynamoDBWrapper(params dynamoParams.DBDriverParameters) (*DBWrapper, err
return nil, err
}
err = dynamoWrapper.createArtifactDataTable()
if err != nil {
return nil, err
}
err = dynamoWrapper.createIndexDataTable()
if err != nil {
return nil, err
@ -158,6 +165,7 @@ func (dwr *DBWrapper) SetManifestMeta(repo string, manifestDigest godigest.Diges
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{},
Signatures: map[string]repodb.ManifestSignatures{},
Referrers: map[string][]repodb.Descriptor{},
}
}
@ -307,6 +315,246 @@ func (dwr *DBWrapper) GetIndexData(indexDigest godigest.Digest) (repodb.IndexDat
return indexData, nil
}
func (dwr DBWrapper) SetArtifactData(artifactDigest godigest.Digest, artifactData repodb.ArtifactData) error {
artifactAttributeValue, err := attributevalue.Marshal(artifactData)
if err != nil {
return err
}
_, err = dwr.Client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{
ExpressionAttributeNames: map[string]string{
"#AD": "ArtifactData",
},
ExpressionAttributeValues: map[string]types.AttributeValue{
":ArtifactData": artifactAttributeValue,
},
Key: map[string]types.AttributeValue{
"ArtifactDigest": &types.AttributeValueMemberS{
Value: artifactDigest.String(),
},
},
TableName: aws.String(dwr.ArtifactDataTablename),
UpdateExpression: aws.String("SET #AD = :ArtifactData"),
})
return err
}
func (dwr DBWrapper) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) {
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
TableName: aws.String(dwr.ArtifactDataTablename),
Key: map[string]types.AttributeValue{
"ArtifactDigest": &types.AttributeValueMemberS{
Value: artifactDigest.String(),
},
},
})
if err != nil {
return repodb.ArtifactData{}, err
}
if resp.Item == nil {
return repodb.ArtifactData{}, zerr.ErrRepoMetaNotFound
}
var artifactData repodb.ArtifactData
err = attributevalue.Unmarshal(resp.Item["ArtifactData"], &artifactData)
if err != nil {
return repodb.ArtifactData{}, err
}
return artifactData, nil
}
func (dwr DBWrapper) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error {
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
TableName: aws.String(dwr.RepoMetaTablename),
Key: map[string]types.AttributeValue{
"RepoName": &types.AttributeValueMemberS{Value: repo},
},
})
if err != nil {
return err
}
repoMeta := repodb.RepoMetadata{
Name: repo,
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{},
Signatures: map[string]repodb.ManifestSignatures{},
Referrers: map[string][]repodb.Descriptor{},
}
if resp.Item != nil {
err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta)
if err != nil {
return err
}
}
refferers := repoMeta.Referrers[referredDigest.String()]
for i := range refferers {
if refferers[i].Digest == referrer.Digest {
return nil
}
}
refferers = append(refferers, referrer)
repoMeta.Referrers[referredDigest.String()] = refferers
return dwr.setRepoMeta(repo, repoMeta)
}
func (dwr DBWrapper) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) {
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
TableName: aws.String(dwr.RepoMetaTablename),
Key: map[string]types.AttributeValue{
"RepoName": &types.AttributeValueMemberS{Value: repo},
},
})
if err != nil {
return []repodb.Descriptor{}, err
}
repoMeta := repodb.RepoMetadata{
Name: repo,
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{},
Signatures: map[string]repodb.ManifestSignatures{},
Referrers: map[string][]repodb.Descriptor{},
}
if resp.Item != nil {
err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta)
if err != nil {
return []repodb.Descriptor{}, err
}
}
return repoMeta.Referrers[referredDigest.String()], nil
}
func (dwr DBWrapper) DeleteReferrer(repo string, referredDigest godigest.Digest,
referrerDigest godigest.Digest,
) error {
resp, err := dwr.Client.GetItem(context.TODO(), &dynamodb.GetItemInput{
TableName: aws.String(dwr.RepoMetaTablename),
Key: map[string]types.AttributeValue{
"RepoName": &types.AttributeValueMemberS{Value: repo},
},
})
if err != nil {
return err
}
repoMeta := repodb.RepoMetadata{
Name: repo,
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{},
Signatures: map[string]repodb.ManifestSignatures{},
Referrers: map[string][]repodb.Descriptor{},
}
if resp.Item != nil {
err := attributevalue.Unmarshal(resp.Item["RepoMetadata"], &repoMeta)
if err != nil {
return err
}
}
referrers := repoMeta.Referrers[referredDigest.String()]
for i := range referrers {
if referrers[i].Digest == referrerDigest.String() {
referrers = append(referrers[:i], referrers[i+1:]...)
break
}
}
repoMeta.Referrers[referredDigest.String()] = referrers
return dwr.setRepoMeta(repo, repoMeta)
}
func (dwr DBWrapper) GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest,
artifactTypes []string,
) ([]repodb.ReferrerInfo, error) {
referrersDescriptors, err := dwr.GetReferrers(repo, referredDigest)
if err != nil {
return nil, err
}
referrersInfo := []repodb.ReferrerInfo{}
for _, descriptor := range referrersDescriptors {
referrerInfo := repodb.ReferrerInfo{}
switch descriptor.MediaType {
case ispec.MediaTypeImageManifest:
manifestData, err := dwr.GetManifestData(godigest.Digest(descriptor.Digest))
if err != nil {
dwr.Log.Error().Msgf("repodb: manifest data not found for digest %s", descriptor.Digest)
continue
}
var manifestContent ispec.Manifest
err = json.Unmarshal(manifestData.ManifestBlob, &manifestContent)
if err != nil {
dwr.Log.Error().Err(err).Msgf("repodb: can't unmarhsal manifest for digest %s",
descriptor.Digest)
continue
}
referrerInfo = repodb.ReferrerInfo{
Digest: descriptor.Digest,
MediaType: ispec.MediaTypeImageManifest,
ArtifactType: manifestContent.Config.MediaType,
Size: len(manifestData.ManifestBlob),
Annotations: manifestContent.Annotations,
}
case ispec.MediaTypeArtifactManifest:
artifactData, err := dwr.GetArtifactData(godigest.Digest(descriptor.Digest))
if err != nil {
dwr.Log.Error().Msgf("repodb: artifact data not found for digest %s", descriptor.Digest)
continue
}
manifestContent := ispec.Artifact{}
err = json.Unmarshal(artifactData.ManifestBlob, &manifestContent)
if err != nil {
dwr.Log.Error().Err(err).Msgf("repodb: can't unmarhsal artifact manifest for digest %s", descriptor.Digest)
continue
}
referrerInfo = repodb.ReferrerInfo{
Digest: descriptor.Digest,
MediaType: manifestContent.MediaType,
ArtifactType: manifestContent.ArtifactType,
Size: len(artifactData.ManifestBlob),
Annotations: manifestContent.Annotations,
}
}
if !common.MatchesArtifactTypes(referrerInfo.ArtifactType, artifactTypes) {
continue
}
referrersInfo = append(referrersInfo, referrerInfo)
}
return referrersInfo, nil
}
func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest,
mediaType string,
) error {
@ -329,6 +577,7 @@ func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDi
Tags: map[string]repodb.Descriptor{},
Statistics: map[string]repodb.DescriptorStatistics{},
Signatures: map[string]repodb.ManifestSignatures{},
Referrers: map[string][]repodb.Descriptor{},
}
if resp.Item != nil {
@ -347,6 +596,7 @@ func (dwr *DBWrapper) SetRepoReference(repo string, reference string, manifestDi
repoMeta.Statistics[manifestDigest.String()] = repodb.DescriptorStatistics{DownloadCount: 0}
repoMeta.Signatures[manifestDigest.String()] = repodb.ManifestSignatures{}
repoMeta.Referrers[manifestDigest.String()] = []repodb.Descriptor{}
err = dwr.setRepoMeta(repo, repoMeta)
@ -1392,6 +1642,31 @@ func (dwr *DBWrapper) createIndexDataTable() error {
return dwr.waitTableToBeCreated(dwr.IndexDataTablename)
}
func (dwr DBWrapper) createArtifactDataTable() error {
_, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{
TableName: aws.String(dwr.ArtifactDataTablename),
AttributeDefinitions: []types.AttributeDefinition{
{
AttributeName: aws.String("ArtifactDigest"),
AttributeType: types.ScalarAttributeTypeS,
},
},
KeySchema: []types.KeySchemaElement{
{
AttributeName: aws.String("ArtifactDigest"),
KeyType: types.KeyTypeHash,
},
},
BillingMode: types.BillingModePayPerRequest,
})
if err != nil && !strings.Contains(err.Error(), "Table already exists") {
return err
}
return dwr.waitTableToBeCreated(dwr.ManifestDataTablename)
}
func (dwr *DBWrapper) createVersionTable() error {
_, err := dwr.Client.CreateTable(context.Background(), &dynamodb.CreateTableInput{
TableName: aws.String(dwr.VersionTablename),

View file

@ -2,5 +2,5 @@ package params
type DBDriverParameters struct {
Endpoint, Region, RepoMetaTablename, ManifestDataTablename, IndexDataTablename,
VersionTablename string
ArtifactDataTablename, VersionTablename string
}

View file

@ -12,20 +12,21 @@ import (
"zotregistry.io/zot/pkg/storage"
)
// SyncRepoDB will sync all repos found in the rootdirectory of the oci layout that zot was deployed on.
func SyncRepoDB(repoDB RepoDB, storeController storage.StoreController, log log.Logger) error {
// ParseStorage will sync all repos found in the rootdirectory of the oci layout that zot was deployed on with the
// ParseStorage database.
func ParseStorage(repoDB RepoDB, storeController storage.StoreController, log log.Logger) error {
allRepos, err := getAllRepos(storeController)
if err != nil {
rootDir := storeController.DefaultStore.RootDir()
log.Error().Err(err).Msgf("sync-repodb: failed to get all repo names present under %s", rootDir)
log.Error().Err(err).Msgf("load-local-layout: failed to get all repo names present under %s", rootDir)
return err
}
for _, repo := range allRepos {
err := SyncRepo(repo, repoDB, storeController, log)
err := ParseRepo(repo, repoDB, storeController, log)
if err != nil {
log.Error().Err(err).Msgf("sync-repodb: failed to sync repo %s", repo)
log.Error().Err(err).Msgf("load-local-layout: failed to sync repo %s", repo)
return err
}
@ -34,13 +35,13 @@ func SyncRepoDB(repoDB RepoDB, storeController storage.StoreController, log log.
return nil
}
// SyncRepo reads the contents of a repo and syncs all images signatures found.
func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreController, log log.Logger) error {
// ParseRepo reads the contents of a repo and syncs all images and signatures found.
func ParseRepo(repo string, repoDB RepoDB, storeController storage.StoreController, log log.Logger) error {
imageStore := storeController.GetImageStore(repo)
indexBlob, err := imageStore.GetIndexContent(repo)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to read index.json for repo %s", repo)
log.Error().Err(err).Msgf("load-repo: failed to read index.json for repo %s", repo)
return err
}
@ -49,14 +50,14 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
err = json.Unmarshal(indexBlob, &indexContent)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to unmarshal index.json for repo %s", repo)
log.Error().Err(err).Msgf("load-repo: failed to unmarshal index.json for repo %s", repo)
return err
}
err = resetRepoMetaTags(repo, repoDB, log)
if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
log.Error().Err(err).Msgf("sync-repo: failed to reset tag field in RepoMetadata for repo %s", repo)
log.Error().Err(err).Msgf("load-repo: failed to reset tag field in RepoMetadata for repo %s", repo)
return err
}
@ -76,7 +77,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
manifestMetaIsPresent, err := isManifestMetaPresent(repo, manifest, repoDB)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: error checking manifestMeta in RepoDB")
log.Error().Err(err).Msgf("load-repo: error checking manifestMeta in RepoDB")
return err
}
@ -84,7 +85,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
if manifestMetaIsPresent && hasTag {
err = repoDB.SetRepoReference(repo, tag, manifest.Digest, manifest.MediaType)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to set repo tag for %s:%s", repo, tag)
log.Error().Err(err).Msgf("load-repo: failed to set repo tag for %s:%s", repo, tag)
return err
}
@ -94,7 +95,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
manifestBlob, digest, _, err := imageStore.GetImageManifest(repo, manifest.Digest.String())
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to set repo tag for %s:%s", repo, tag)
log.Error().Err(err).Msgf("load-repo: failed to set repo tag for %s:%s", repo, tag)
return err
}
@ -105,7 +106,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
if errors.Is(err, zerr.ErrOrphanSignature) {
continue
} else {
log.Error().Err(err).Msgf("sync-repo: failed checking if image is signature for %s:%s", repo, tag)
log.Error().Err(err).Msgf("load-repo: failed checking if image is signature for %s:%s", repo, tag)
return err
}
@ -134,7 +135,7 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
err = SetMetadataFromInput(repo, reference, manifest.MediaType, manifest.Digest, manifestBlob,
imageStore, repoDB, log)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to set metadata for %s:%s", repo, tag)
log.Error().Err(err).Msgf("load-repo: failed to set metadata for %s:%s", repo, tag)
return err
}
@ -142,12 +143,13 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
// manage the signatures found
for _, sigData := range signaturesFound {
err := repoDB.AddManifestSignature(repo, godigest.Digest(sigData.signedManifestDigest), SignatureMetadata{
SignatureType: sigData.signatureType,
SignatureDigest: sigData.signatureDigest,
})
err := repoDB.AddManifestSignature(repo, godigest.Digest(sigData.signedManifestDigest),
SignatureMetadata{
SignatureType: sigData.signatureType,
SignatureDigest: sigData.signatureDigest,
})
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed set signature meta for signed image %s:%s manifest digest %s ",
log.Error().Err(err).Msgf("load-repo: failed set signature meta for signed image %s:%s manifest digest %s ",
sigData.repo, sigData.tag, sigData.signedManifestDigest)
return err
@ -161,13 +163,13 @@ func SyncRepo(repo string, repoDB RepoDB, storeController storage.StoreControlle
func resetRepoMetaTags(repo string, repoDB RepoDB, log log.Logger) error {
repoMeta, err := repoDB.GetRepoMeta(repo)
if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
log.Error().Err(err).Msgf("sync-repo: failed to get RepoMeta for repo %s", repo)
log.Error().Err(err).Msgf("load-repo: failed to get RepoMeta for repo %s", repo)
return err
}
if errors.Is(err, zerr.ErrRepoMetaNotFound) {
log.Info().Msgf("sync-repo: RepoMeta not found for repo %s, new RepoMeta will be created", repo)
log.Info().Msgf("load-repo: RepoMeta not found for repo %s, new RepoMeta will be created", repo)
return nil
}
@ -176,7 +178,7 @@ func resetRepoMetaTags(repo string, repoDB RepoDB, log log.Logger) error {
// We should have a way to delete all tags at once
err := repoDB.DeleteRepoTag(repo, tag)
if err != nil {
log.Error().Err(err).Msgf("sync-repo: failed to delete tag %s from RepoMeta for repo %s", tag, repo)
log.Error().Err(err).Msgf("load-repo: failed to delete tag %s from RepoMeta for repo %s", tag, repo)
return err
}
@ -220,7 +222,7 @@ func isManifestMetaPresent(repo string, manifest ispec.Descriptor, repoDB RepoDB
}
// NewManifestMeta takes raw data about an image and createa a new ManifestMetadate object.
func NewManifestData(repoName string, manifestBlob []byte, imgStore storage.ImageStore,
func NewManifestData(repoName string, manifestBlob []byte, imageStore storage.ImageStore,
) (ManifestData, error) {
var (
manifestContent ispec.Manifest
@ -233,7 +235,7 @@ func NewManifestData(repoName string, manifestBlob []byte, imgStore storage.Imag
return ManifestData{}, err
}
configBlob, err := imgStore.GetBlobContent(repoName, manifestContent.Config.Digest)
configBlob, err := imageStore.GetBlobContent(repoName, manifestContent.Config.Digest)
if err != nil {
return ManifestData{}, err
}
@ -249,7 +251,7 @@ func NewManifestData(repoName string, manifestBlob []byte, imgStore storage.Imag
return manifestData, nil
}
func NewIndexData(repoName string, indexBlob []byte,
func NewIndexData(repoName string, indexBlob []byte, imageStore storage.ImageStore,
) IndexData {
indexData := IndexData{}
@ -258,6 +260,13 @@ func NewIndexData(repoName string, indexBlob []byte,
return indexData
}
func NewArtifactData(repo string, descriptorBlob []byte, imageStore storage.ImageStore,
) ArtifactData {
return ArtifactData{
ManifestBlob: descriptorBlob,
}
}
// SetMetadataFromInput tries to set manifest metadata and update repo metadata by adding the current tag
// (in case the reference is a tag). The function expects image manifests and indexes (multi arch images).
func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Digest, descriptorBlob []byte,
@ -277,12 +286,33 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig
return err
}
case ispec.MediaTypeImageIndex:
indexData := NewIndexData(repo, descriptorBlob)
indexData := NewIndexData(repo, descriptorBlob, imageStore)
err := repoDB.SetIndexData(digest, indexData)
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting index data")
return err
}
case ispec.MediaTypeArtifactManifest:
artifactData := NewArtifactData(repo, descriptorBlob, imageStore)
err := repoDB.SetArtifactData(digest, artifactData)
if err != nil {
log.Error().Err(err).Msg("repodb: error while putting artifact data")
return err
}
}
if refferredDigest, hasSubject := GetReferredSubject(descriptorBlob); hasSubject {
err := repoDB.SetReferrer(repo, refferredDigest, Descriptor{
Digest: digest.String(),
MediaType: mediaType,
})
if err != nil {
log.Error().Err(err).Msg("repodb: error while settingg referrer")
return err
}
}
@ -296,3 +326,18 @@ func SetMetadataFromInput(repo, reference, mediaType string, digest godigest.Dig
return nil
}
func GetReferredSubject(descriptorBlob []byte) (godigest.Digest, bool) {
var manifest ispec.Manifest
err := json.Unmarshal(descriptorBlob, &manifest)
if err != nil {
return "", false
}
if manifest.Subject == nil || manifest.Subject.Digest.String() == "" {
return "", false
}
return manifest.Subject.Digest, true
}

View file

@ -12,7 +12,6 @@ import (
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
oras "github.com/oras-project/artifacts-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"
zerr "zotregistry.io/zot/errors"
@ -32,8 +31,8 @@ const repo = "repo"
var ErrTestError = errors.New("test error")
func TestSyncRepoDBErrors(t *testing.T) {
Convey("SyncRepoDB", t, func() {
func TestLoadOCILayoutErrors(t *testing.T) {
Convey("LoadOCILayout", t, func() {
imageStore := mocks.MockedImageStore{
GetIndexContentFn: func(repo string) ([]byte, error) {
return nil, ErrTestError
@ -46,7 +45,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
repoDB := mocks.RepoDBMock{}
// sync repo fail
err := repodb.SyncRepoDB(repoDB, storeController, log.NewLogger("debug", ""))
err := repodb.ParseStorage(repoDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
Convey("getAllRepos errors", func() {
@ -67,12 +66,12 @@ func TestSyncRepoDBErrors(t *testing.T) {
},
}
err := repodb.SyncRepoDB(repoDB, storeController, log.NewLogger("debug", ""))
err := repodb.ParseStorage(repoDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldNotBeNil)
})
})
Convey("SyncRepo", t, func() {
Convey("LoadRepo", t, func() {
imageStore := mocks.MockedImageStore{}
storeController := storage.StoreController{DefaultStore: &imageStore}
repoDB := mocks.RepoDBMock{}
@ -83,7 +82,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
return nil, ErrTestError
}
err := repodb.SyncRepo("repo", repoDB, storeController, log)
err := repodb.ParseRepo("repo", repoDB, storeController, log)
So(err, ShouldNotBeNil)
})
@ -92,7 +91,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
return []byte("Invalid JSON"), nil
}
err := repodb.SyncRepo("repo", repoDB, storeController, log)
err := repodb.ParseRepo("repo", repoDB, storeController, log)
So(err, ShouldNotBeNil)
})
@ -106,7 +105,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
return repodb.RepoMetadata{}, ErrTestError
}
err := repodb.SyncRepo("repo", repoDB, storeController, log)
err := repodb.ParseRepo("repo", repoDB, storeController, log)
So(err, ShouldNotBeNil)
})
@ -125,7 +124,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
return ErrTestError
}
err := repodb.SyncRepo("repo", repoDB, storeController, log)
err := repodb.ParseRepo("repo", repoDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
@ -154,7 +153,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
return repodb.ManifestMetadata{}, ErrTestError
}
err = repodb.SyncRepo("repo", repoDB, storeController, log)
err = repodb.ParseRepo("repo", repoDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
@ -183,7 +182,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
return ErrTestError
}
err = repodb.SyncRepo("repo", repoDB, storeController, log)
err = repodb.ParseRepo("repo", repoDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
@ -215,7 +214,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return nil, "", "", ErrTestError
}
err = repodb.SyncRepo("repo", repoDB, storeController, log)
err = repodb.ParseRepo("repo", repoDB, storeController, log)
So(err, ShouldNotBeNil)
})
@ -224,7 +223,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return []byte("Invalid JSON"), "", "", nil
}
err = repodb.SyncRepo("repo", repoDB, storeController, log)
err = repodb.ParseRepo("repo", repoDB, storeController, log)
So(err, ShouldNotBeNil)
})
Convey("CheckIsImageSignature -> not signature", func() {
@ -241,7 +240,7 @@ func TestSyncRepoDBErrors(t *testing.T) {
return nil, ErrTestError
}
err = repodb.SyncRepo("repo", repoDB, storeController, log)
err = repodb.ParseRepo("repo", repoDB, storeController, log)
So(err, ShouldNotBeNil)
})
@ -250,17 +249,19 @@ func TestSyncRepoDBErrors(t *testing.T) {
return []byte("invalid JSON"), nil
}
err = repodb.SyncRepo("repo", repoDB, storeController, log)
err = repodb.ParseRepo("repo", repoDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
Convey("CheckIsImageSignature -> is signature", func() {
manifestContent := oras.Manifest{
Subject: &oras.Descriptor{
manifestContent := ispec.Artifact{
Subject: &ispec.Descriptor{
Digest: "123",
},
ArtifactType: "application/vnd.cncf.notary.signature",
}
manifestBlob, err := json.Marshal(manifestContent)
So(err, ShouldBeNil)
@ -274,14 +275,14 @@ func TestSyncRepoDBErrors(t *testing.T) {
return ErrTestError
}
err = repodb.SyncRepo("repo", repoDB, storeController, log)
err = repodb.ParseRepo("repo", repoDB, storeController, log)
So(err, ShouldNotBeNil)
})
})
})
}
func TestSyncRepoDBWithStorage(t *testing.T) {
func TestLoadOCILayoutWithStorage(t *testing.T) {
Convey("Boltdb", t, func() {
rootDir := t.TempDir()
@ -359,7 +360,7 @@ func TestSyncRepoDBWithStorage(t *testing.T) {
})
So(err, ShouldBeNil)
err = repodb.SyncRepoDB(repoDB, storeController, log.NewLogger("debug", ""))
err = repodb.ParseStorage(repoDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
repos, err := repoDB.GetMultipleRepoMeta(
@ -435,7 +436,7 @@ func TestSyncRepoDBWithStorage(t *testing.T) {
})
So(err, ShouldBeNil)
err = repodb.SyncRepoDB(repoDB, storeController, log.NewLogger("debug", ""))
err = repodb.ParseStorage(repoDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
repos, err := repoDB.GetMultipleRepoMeta(
@ -451,7 +452,7 @@ func TestSyncRepoDBWithStorage(t *testing.T) {
})
}
func TestSyncRepoDBDynamoWrapper(t *testing.T) {
func TestLoadOCILayoutDynamoWrapper(t *testing.T) {
skipIt(t)
Convey("Dynamodb", t, func() {
@ -532,6 +533,7 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
ArtifactDataTablename: "ArtifactDataTable",
VersionTablename: "Version",
})
So(err, ShouldBeNil)
@ -542,7 +544,7 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
err = dynamoWrapper.ResetRepoMetaTable()
So(err, ShouldBeNil)
err = repodb.SyncRepoDB(dynamoWrapper, storeController, log.NewLogger("debug", ""))
err = repodb.ParseStorage(dynamoWrapper, storeController, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
repos, err := dynamoWrapper.GetMultipleRepoMeta(
@ -618,12 +620,13 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
Region: "us-east-2",
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
ArtifactDataTablename: "ArtifactDataTable",
IndexDataTablename: "IndexDataTable",
VersionTablename: "Version",
})
So(err, ShouldBeNil)
err = repodb.SyncRepoDB(repoDB, storeController, log.NewLogger("debug", ""))
err = repodb.ParseStorage(repoDB, storeController, log.NewLogger("debug", ""))
So(err, ShouldBeNil)
repos, err := repoDB.GetMultipleRepoMeta(
@ -640,6 +643,13 @@ func TestSyncRepoDBDynamoWrapper(t *testing.T) {
})
}
func TestGetReferredSubject(t *testing.T) {
Convey("GetReferredSubject error", t, func() {
_, err := repodb.GetReferredSubject([]byte("bad json"))
So(err, ShouldNotBeNil)
})
}
func skipIt(t *testing.T) {
t.Helper()

View file

@ -11,6 +11,7 @@ import (
const (
ManifestDataBucket = "ManifestData"
IndexDataBucket = "IndexData"
ArtifactDataBucket = "ArtifactData"
UserMetadataBucket = "UserMeta"
RepoMetadataBucket = "RepoMetadata"
VersionBucket = "Version"
@ -67,6 +68,26 @@ type RepoDB interface { //nolint:interfacebloat
// GetIndexData returns indexData for a given Index from the database
GetIndexData(indexDigest godigest.Digest) (IndexData, error)
// SetArtifactData sets artifactData for a given artifact in the database
SetArtifactData(artifactDigest godigest.Digest, artifactData ArtifactData) error
// GetArtifactData returns artifactData for a given artifact from the database
GetArtifactData(artifactDigest godigest.Digest) (ArtifactData, error)
// SetReferrer adds a referrer to the referrers list of a manifest inside a repo
SetReferrer(repo string, referredDigest godigest.Digest, referrer Descriptor) error
// SetReferrer delets a referrer to the referrers list of a manifest inside a repo
DeleteReferrer(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest) error
// GetReferrers returns the list of referrers for a referred manifest
GetReferrers(repo string, referredDigest godigest.Digest) ([]Descriptor, error)
// GetFilteredReferrersInfo returnes a list of for all referrers of the given digest that match one of the
// artifact types.
GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string) (
[]ReferrerInfo, error)
// IncrementManifestDownloads adds 1 to the download count of a manifest
IncrementImageDownloads(repo string, reference string) error
@ -107,6 +128,18 @@ type ManifestData struct {
ConfigBlob []byte
}
type ArtifactData struct {
ManifestBlob []byte
}
type ReferrerInfo struct {
Digest string
MediaType string
ArtifactType string
Size int
Annotations map[string]string
}
// Descriptor represents an image. Multiple images might have the same digests but different tags.
type Descriptor struct {
Digest string
@ -125,7 +158,9 @@ type RepoMetadata struct {
Statistics map[string]DescriptorStatistics
Signatures map[string]ManifestSignatures
Stars int
Referrers map[string][]Descriptor
Stars int
}
type LayerInfo struct {

View file

@ -76,6 +76,7 @@ func TestDynamoDBWrapper(t *testing.T) {
manifestDataTablename := "ManifestDataTable" + uuid.String()
versionTablename := "Version" + uuid.String()
indexDataTablename := "IndexDataTable" + uuid.String()
artifactDataTablename := "ArtifactDataTable" + uuid.String()
Convey("DynamoDB Wrapper", t, func() {
dynamoDBDriverParams := dynamoParams.DBDriverParameters{
@ -83,6 +84,7 @@ func TestDynamoDBWrapper(t *testing.T) {
RepoMetaTablename: repoMetaTablename,
ManifestDataTablename: manifestDataTablename,
IndexDataTablename: indexDataTablename,
ArtifactDataTablename: artifactDataTablename,
VersionTablename: versionTablename,
Region: "us-east-2",
}
@ -1757,6 +1759,202 @@ func RunRepoDBTests(repoDB repodb.RepoDB, preparationFuncs ...func() error) {
_, err = repoDB.GetIndexData(godigest.FromString("inexistent"))
So(err, ShouldNotBeNil)
})
Convey("Test artifact logic", func() {
artifact, err := test.GetRandomArtifact(nil)
So(err, ShouldBeNil)
artifactDigest, err := artifact.Digest()
So(err, ShouldBeNil)
artifactData, err := artifact.ArtifactData()
So(err, ShouldBeNil)
err = repoDB.SetArtifactData(artifactDigest, artifactData)
So(err, ShouldBeNil)
result, err := repoDB.GetArtifactData(artifactDigest)
So(err, ShouldBeNil)
So(result, ShouldResemble, artifactData)
_, err = repoDB.GetArtifactData(godigest.FromString("inexistent"))
So(err, ShouldNotBeNil)
})
Convey("Test Referrers", func() {
image, err := test.GetRandomImage("tag")
So(err, ShouldBeNil)
referredDigest, err := image.Digest()
So(err, ShouldBeNil)
manifestBlob, err := json.Marshal(image.Manifest)
So(err, ShouldBeNil)
configBlob, err := json.Marshal(image.Config)
So(err, ShouldBeNil)
manifestData := repodb.ManifestData{
ManifestBlob: manifestBlob,
ConfigBlob: configBlob,
}
err = repoDB.SetManifestData(referredDigest, manifestData)
So(err, ShouldBeNil)
err = repoDB.SetRepoReference("repo", "tag", referredDigest, ispec.MediaTypeImageManifest)
So(err, ShouldBeNil)
// ------- Add Artifact 1
artifact1, err := test.GetRandomArtifact(&ispec.Descriptor{
Digest: referredDigest,
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
artifactDigest1, err := artifact1.Digest()
So(err, ShouldBeNil)
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
Digest: artifactDigest1.String(),
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
// ------- Add Artifact 2
artifact2, err := test.GetRandomArtifact(&ispec.Descriptor{
Digest: referredDigest,
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
artifactDigest2, err := artifact2.Digest()
So(err, ShouldBeNil)
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
Digest: artifactDigest2.String(),
MediaType: ispec.MediaTypeArtifactManifest,
})
So(err, ShouldBeNil)
// ------ GetReferrers
referrers, err := repoDB.GetReferrers("repo", referredDigest)
So(len(referrers), ShouldEqual, 2)
So(referrers, ShouldContain, repodb.Descriptor{
Digest: artifactDigest1.String(),
MediaType: ispec.MediaTypeImageManifest,
})
So(referrers, ShouldContain, repodb.Descriptor{
Digest: artifactDigest2.String(),
MediaType: ispec.MediaTypeArtifactManifest,
})
So(err, ShouldBeNil)
// ------ DeleteReferrers
err = repoDB.DeleteReferrer("repo", referredDigest, artifactDigest1)
So(err, ShouldBeNil)
err = repoDB.DeleteReferrer("repo", referredDigest, artifactDigest2)
So(err, ShouldBeNil)
referrers, err = repoDB.GetReferrers("repo", referredDigest)
So(err, ShouldBeNil)
So(len(referrers), ShouldEqual, 0)
})
Convey("Test Referrers on empty Repo", func() {
repoMeta, err := repoDB.GetRepoMeta("repo")
So(err, ShouldNotBeNil)
So(repoMeta, ShouldResemble, repodb.RepoMetadata{})
referredDigest := godigest.FromString("referredDigest")
referrerDigest := godigest.FromString("referrerDigest")
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
Digest: referrerDigest.String(),
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
repoMeta, err = repoDB.GetRepoMeta("repo")
So(err, ShouldBeNil)
So(repoMeta.Referrers[referredDigest.String()][0].Digest, ShouldResemble, referrerDigest.String())
})
Convey("Test Referrers add same one twice", func() {
repoMeta, err := repoDB.GetRepoMeta("repo")
So(err, ShouldNotBeNil)
So(repoMeta, ShouldResemble, repodb.RepoMetadata{})
referredDigest := godigest.FromString("referredDigest")
referrerDigest := godigest.FromString("referrerDigest")
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
Digest: referrerDigest.String(),
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
Digest: referrerDigest.String(),
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
repoMeta, err = repoDB.GetRepoMeta("repo")
So(err, ShouldBeNil)
So(len(repoMeta.Referrers[referredDigest.String()]), ShouldEqual, 1)
})
Convey("GetFilteredReferrersInfo", func() {
referredDigest := godigest.FromString("referredDigest")
err := repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
Digest: "inexistendManifestDigest",
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
Digest: "inexistendArtifactManifestDigest",
MediaType: ispec.MediaTypeArtifactManifest,
})
So(err, ShouldBeNil)
// ------- Set existent manifest and artifact manifest
err = repoDB.SetManifestData("goodManifest", repodb.ManifestData{
ManifestBlob: []byte(`{"artifactType": "unwantedType"}`),
ConfigBlob: []byte("{}"),
})
So(err, ShouldBeNil)
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
Digest: "goodManifest",
MediaType: ispec.MediaTypeImageManifest,
})
So(err, ShouldBeNil)
err = repoDB.SetArtifactData("goodArtifact", repodb.ArtifactData{
ManifestBlob: []byte(`{"artifactType": "wantedType"}`),
})
So(err, ShouldBeNil)
err = repoDB.SetReferrer("repo", referredDigest, repodb.Descriptor{
Digest: "goodArtifact",
MediaType: ispec.MediaTypeArtifactManifest,
})
So(err, ShouldBeNil)
referrerInfo, err := repoDB.GetFilteredReferrersInfo("repo", referredDigest, []string{"wantedType"})
So(err, ShouldBeNil)
So(len(referrerInfo), ShouldEqual, 1)
So(referrerInfo[0].ArtifactType, ShouldResemble, "wantedType")
So(referrerInfo[0].Digest, ShouldResemble, "goodArtifact")
})
})
}

View file

@ -20,6 +20,7 @@ func TestCreateDynamo(t *testing.T) {
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
IndexDataTablename: "IndexDataTable",
ArtifactDataTablename: "ArtifactDataTable",
VersionTablename: "Version",
Region: "us-east-2",
}

View file

@ -7,19 +7,20 @@ import (
zerr "zotregistry.io/zot/errors"
"zotregistry.io/zot/pkg/log"
"zotregistry.io/zot/pkg/meta/repodb"
"zotregistry.io/zot/pkg/meta/repodb/common"
"zotregistry.io/zot/pkg/storage"
)
// OnUpdateManifest is called when a new manifest is added. It updates repodb according to the type
// of image pushed(normal images, signatues, etc.). In care of any errors, it makes sure to keep
// consistency between repodb and the image store.
func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest, body []byte,
func OnUpdateManifest(repo, reference, mediaType string, digest godigest.Digest, body []byte,
storeController storage.StoreController, repoDB repodb.RepoDB, log log.Logger,
) error {
imgStore := storeController.GetImageStore(name)
imgStore := storeController.GetImageStore(repo)
// check if image is a signature
isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(name, body, reference,
isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(repo, body, reference,
storeController)
if err != nil {
if errors.Is(err, zerr.ErrOrphanSignature) {
@ -30,8 +31,8 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest,
log.Error().Err(err).Msg("can't check if image is a signature or not")
if err := imgStore.DeleteImageManifest(name, reference, false); err != nil {
log.Error().Err(err).Msgf("couldn't remove image manifest %s in repo %s", reference, name)
if err := imgStore.DeleteImageManifest(repo, reference, false); err != nil {
log.Error().Err(err).Msgf("couldn't remove image manifest %s in repo %s", reference, repo)
return err
}
@ -42,7 +43,7 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest,
metadataSuccessfullySet := true
if isSignature {
err = repoDB.AddManifestSignature(name, signedManifestDigest, repodb.SignatureMetadata{
err = repoDB.AddManifestSignature(repo, signedManifestDigest, repodb.SignatureMetadata{
SignatureType: signatureType,
SignatureDigest: digest.String(),
})
@ -51,7 +52,7 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest,
metadataSuccessfullySet = false
}
} else {
err := repodb.SetMetadataFromInput(name, reference, mediaType, digest, body,
err := repodb.SetMetadataFromInput(repo, reference, mediaType, digest, body,
imgStore, repoDB, log)
if err != nil {
metadataSuccessfullySet = false
@ -59,10 +60,10 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest,
}
if !metadataSuccessfullySet {
log.Info().Msgf("uploding image meta was unsuccessful for tag %s in repo %s", reference, name)
log.Info().Msgf("uploding image meta was unsuccessful for tag %s in repo %s", reference, repo)
if err := imgStore.DeleteImageManifest(name, reference, false); err != nil {
log.Error().Err(err).Msgf("couldn't remove image manifest %s in repo %s", reference, name)
if err := imgStore.DeleteImageManifest(repo, reference, false); err != nil {
log.Error().Err(err).Msgf("couldn't remove image manifest %s in repo %s", reference, repo)
return err
}
@ -76,12 +77,12 @@ func OnUpdateManifest(name, reference, mediaType string, digest godigest.Digest,
// OnDeleteManifest is called when a manifest is deleted. It updates repodb according to the type
// of image pushed(normal images, signatues, etc.). In care of any errors, it makes sure to keep
// consistency between repodb and the image store.
func OnDeleteManifest(name, reference, mediaType string, digest godigest.Digest, manifestBlob []byte,
func OnDeleteManifest(repo, reference, mediaType string, digest godigest.Digest, manifestBlob []byte,
storeController storage.StoreController, repoDB repodb.RepoDB, log log.Logger,
) error {
imgStore := storeController.GetImageStore(name)
imgStore := storeController.GetImageStore(repo)
isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(name, manifestBlob,
isSignature, signatureType, signedManifestDigest, err := storage.CheckIsImageSignature(repo, manifestBlob,
reference, storeController)
if err != nil {
if errors.Is(err, zerr.ErrOrphanSignature) {
@ -98,7 +99,7 @@ func OnDeleteManifest(name, reference, mediaType string, digest godigest.Digest,
manageRepoMetaSuccessfully := true
if isSignature {
err = repoDB.DeleteSignature(name, signedManifestDigest, repodb.SignatureMetadata{
err = repoDB.DeleteSignature(repo, signedManifestDigest, repodb.SignatureMetadata{
SignatureDigest: digest.String(),
SignatureType: signatureType,
})
@ -107,22 +108,31 @@ func OnDeleteManifest(name, reference, mediaType string, digest godigest.Digest,
manageRepoMetaSuccessfully = false
}
} else {
err = repoDB.DeleteRepoTag(name, reference)
err = repoDB.DeleteRepoTag(repo, reference)
if err != nil {
log.Info().Msg("repodb: restoring image store")
// restore image store
_, err := imgStore.PutImageManifest(name, reference, mediaType, manifestBlob)
_, err := imgStore.PutImageManifest(repo, reference, mediaType, manifestBlob)
if err != nil {
log.Error().Err(err).Msg("repodb: error while restoring image store, database is not consistent")
}
manageRepoMetaSuccessfully = false
}
if refferredDigest, hasSubject := common.GetReferredSubject(manifestBlob); hasSubject {
err := repoDB.DeleteReferrer(repo, refferredDigest, digest)
if err != nil {
log.Error().Err(err).Msg("repodb: error while deleting referrer")
return err
}
}
}
if !manageRepoMetaSuccessfully {
log.Info().Msgf("repodb: deleting image meta was unsuccessful for tag %s in repo %s", reference, name)
log.Info().Msgf("repodb: deleting image meta was unsuccessful for tag %s in repo %s", reference, repo)
return err
}

View file

@ -6,9 +6,9 @@ import (
"testing"
"time"
notreg "github.com/notaryproject/notation-go/registry"
godigest "github.com/opencontainers/go-digest"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
oras "github.com/oras-project/artifacts-spec/specs-go/v1"
. "github.com/smartystreets/goconvey/convey"
zerr "zotregistry.io/zot/errors"
@ -95,10 +95,11 @@ func TestUpdateErrors(t *testing.T) {
log := log.NewLogger("debug", "")
Convey("zerr.ErrOrphanSignature", func() {
manifestContent := oras.Manifest{
Subject: &oras.Descriptor{
manifestContent := ispec.Artifact{
Subject: &ispec.Descriptor{
Digest: "123",
},
ArtifactType: notreg.ArtifactTypeNotation,
}
manifestBlob, err := json.Marshal(manifestContent)
So(err, ShouldBeNil)
@ -120,10 +121,11 @@ func TestUpdateErrors(t *testing.T) {
log := log.NewLogger("debug", "")
Convey("CheckIsImageSignature errors", func() {
manifestContent := oras.Manifest{
Subject: &oras.Descriptor{
manifestContent := ispec.Artifact{
Subject: &ispec.Descriptor{
Digest: "123",
},
ArtifactType: notreg.ArtifactTypeNotation,
}
manifestBlob, err := json.Marshal(manifestContent)
So(err, ShouldBeNil)
@ -143,6 +145,25 @@ func TestUpdateErrors(t *testing.T) {
err = repoDBUpdate.OnDeleteManifest("repo", "tag1", "digest", "media", manifestBlob,
storeController, repoDB, log)
So(err, ShouldNotBeNil)
imageStore.GetImageManifestFn = func(repo, reference string) ([]byte, godigest.Digest, string, error) {
return []byte{}, "", "", zerr.ErrManifestNotFound
}
err = repoDBUpdate.OnDeleteManifest("repo", "tag1", "digest", "media", manifestBlob,
storeController, repoDB, log)
So(err, ShouldNotBeNil)
})
Convey("DeleteReferrers errors", func() {
repoDB.DeleteReferrerFn = func(repo string, referredDigest, referrerDigest godigest.Digest) error {
return ErrTestError
}
err := repoDBUpdate.OnDeleteManifest("repo", "tag1", "digest", "media",
[]byte(`{"subject": {"digest": "dig"}}`),
storeController, repoDB, log)
So(err, ShouldNotBeNil)
})
})
@ -153,10 +174,11 @@ func TestUpdateErrors(t *testing.T) {
log := log.NewLogger("debug", "")
Convey("CheckIsImageSignature errors", func() {
manifestContent := oras.Manifest{
Subject: &oras.Descriptor{
manifestContent := ispec.Artifact{
Subject: &ispec.Descriptor{
Digest: "123",
},
ArtifactType: notreg.ArtifactTypeNotation,
}
manifestBlob, err := json.Marshal(manifestContent)
So(err, ShouldBeNil)
@ -205,5 +227,56 @@ func TestUpdateErrors(t *testing.T) {
manifestBlob, imageStore, repoDB, log)
So(err, ShouldBeNil)
})
Convey("SetMetadataFromInput SetData errors", func() {
imageStore := mocks.MockedImageStore{}
log := log.NewLogger("debug", "")
repoDB := mocks.RepoDBMock{
SetManifestDataFn: func(manifestDigest godigest.Digest, mm repodb.ManifestData) error {
return ErrTestError
},
}
err := repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest",
[]byte("{}"), imageStore, repoDB, log)
So(err, ShouldNotBeNil)
repoDB = mocks.RepoDBMock{
SetIndexDataFn: func(digest godigest.Digest, indexData repodb.IndexData) error {
return ErrTestError
},
}
err = repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageIndex, "digest",
[]byte("{}"), imageStore, repoDB, log)
So(err, ShouldNotBeNil)
repoDB = mocks.RepoDBMock{
SetArtifactDataFn: func(digest godigest.Digest, artifactData repodb.ArtifactData) error {
return ErrTestError
},
}
err = repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeArtifactManifest, "digest",
[]byte("{}"), imageStore, repoDB, log)
So(err, ShouldNotBeNil)
})
Convey("SetMetadataFromInput SetReferrer errors", func() {
imageStore := mocks.MockedImageStore{
GetBlobContentFn: func(repo string, digest godigest.Digest) ([]byte, error) {
return []byte("{}"), nil
},
}
log := log.NewLogger("debug", "")
repoDB := mocks.RepoDBMock{
SetReferrerFn: func(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error {
return ErrTestError
},
}
err := repodb.SetMetadataFromInput("repo", "ref", ispec.MediaTypeImageManifest, "digest",
[]byte(`{"subject": {"digest": "subjDigest"}}`), imageStore, repoDB, log)
So(err, ShouldNotBeNil)
})
})
}

View file

@ -119,6 +119,7 @@ func TestVersioningDynamoDB(t *testing.T) {
Region: region,
RepoMetaTablename: "RepoMetadataTable",
ManifestDataTablename: "ManifestDataTable",
ArtifactDataTablename: "ArtifactDataTable",
IndexDataTablename: "IndexDataTable",
VersionTablename: "Version",
})

View file

@ -7,6 +7,7 @@ import (
"strings"
"github.com/gobwas/glob"
notreg "github.com/notaryproject/notation-go/registry"
godigest "github.com/opencontainers/go-digest"
imeta "github.com/opencontainers/image-spec/specs-go"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
@ -18,6 +19,12 @@ import (
storageConstants "zotregistry.io/zot/pkg/storage/constants"
)
func SignatureMediaTypes() map[string]bool {
return map[string]bool{
notreg.ArtifactTypeNotation: true,
}
}
func GetTagsByIndex(index ispec.Index) []string {
tags := make([]string, 0)
@ -711,7 +718,7 @@ func IsSupportedMediaType(mediaType string) bool {
mediaType == oras.MediaTypeArtifactManifest
}
// imageIsSignature checks if the given image (repo:tag) represents a signature. The function
// CheckIsImageSignature checks if the given image (repo:tag) represents a signature. The function
// returns:
//
// - bool: if the image is a signature or not
@ -726,7 +733,7 @@ func CheckIsImageSignature(repoName string, manifestBlob []byte, reference strin
) (bool, string, godigest.Digest, error) {
const cosign = "cosign"
var manifestContent oras.Manifest
var manifestContent ispec.Artifact
err := json.Unmarshal(manifestBlob, &manifestContent)
if err != nil {
@ -734,7 +741,7 @@ func CheckIsImageSignature(repoName string, manifestBlob []byte, reference strin
}
// check notation signature
if manifestContent.Subject != nil {
if _, ok := SignatureMediaTypes()[manifestContent.ArtifactType]; ok && manifestContent.Subject != nil {
imgStore := storeController.GetImageStore(repoName)
_, signedImageManifestDigest, _, err := imgStore.GetImageManifest(repoName,

View file

@ -105,6 +105,37 @@ func (img Image) Digest() (godigest.Digest, error) {
return godigest.FromBytes(blob), nil
}
type Artifact struct {
Manifest ispec.Artifact
Blobs []ArtifactBlobs
Reference string
}
func (a Artifact) Digest() (godigest.Digest, error) {
blob, err := json.Marshal(a.Manifest)
if err != nil {
return "", err
}
return godigest.FromBytes(blob), nil
}
func (a Artifact) ArtifactData() (repodb.ArtifactData, error) {
blob, err := json.Marshal(a.Manifest)
if err != nil {
return repodb.ArtifactData{}, err
}
return repodb.ArtifactData{
ManifestBlob: blob,
}, nil
}
type ArtifactBlobs struct {
Blob []byte
MediaType string
}
type MultiarchImage struct {
Index ispec.Index
Images []Image
@ -623,6 +654,22 @@ func GetRandomImageComponents(layerSize int) (ispec.Image, [][]byte, ispec.Manif
return config, layers, manifest, nil
}
func GetRandomImage(reference string) (Image, error) {
const layerSize = 20
config, layers, manifest, err := GetRandomImageComponents(layerSize)
if err != nil {
return Image{}, err
}
return Image{
Manifest: manifest,
Layers: layers,
Config: config,
Reference: reference,
}, nil
}
func GetImageComponentsWithConfig(conf ispec.Image) (ispec.Image, [][]byte, ispec.Manifest, error) {
configBlob, err := json.Marshal(conf)
if err = Error(err); err != nil {
@ -728,6 +775,49 @@ func GetImageWithComponents(config ispec.Image, layers [][]byte) (Image, error)
}, nil
}
func GetRandomArtifact(subject *ispec.Descriptor) (Artifact, error) {
var randBlob [10]byte
_, err := rand.Read(randBlob[:])
if err != nil {
return Artifact{}, err
}
artifactBlobs := []ArtifactBlobs{
{
Blob: randBlob[:],
MediaType: "application/octet-stream",
},
}
blobsDescriptors := make([]ispec.Descriptor, 0, len(artifactBlobs))
for _, artifactBlob := range artifactBlobs {
blobsDescriptors = append(blobsDescriptors, ispec.Descriptor{
Digest: godigest.FromBytes(artifactBlob.Blob),
MediaType: artifactBlob.MediaType,
Size: int64(len(artifactBlob.Blob)),
})
}
artifactManifest := ispec.Artifact{
MediaType: ispec.MediaTypeArtifactManifest,
Blobs: blobsDescriptors,
Subject: subject,
}
artifactManifestBlob, err := json.Marshal(artifactManifest)
if err != nil {
return Artifact{}, err
}
return Artifact{
Manifest: artifactManifest,
Blobs: artifactBlobs,
Reference: godigest.FromBytes(artifactManifestBlob).String(),
}, nil
}
func GetCosignSignatureTagForManifest(manifest ispec.Manifest) (string, error) {
manifestBlob, err := json.Marshal(manifest)
if err != nil {
@ -743,6 +833,32 @@ func GetCosignSignatureTagForDigest(manifestDigest godigest.Digest) string {
return manifestDigest.Algorithm().String() + "-" + manifestDigest.Encoded() + ".sig"
}
func GetImageWithSubject(subjectDigest godigest.Digest, mediaType string) (Image, error) {
num := 100
conf, layers, manifest, err := GetRandomImageComponents(num)
if err != nil {
return Image{}, err
}
manifest.Subject = &ispec.Descriptor{
Digest: subjectDigest,
MediaType: mediaType,
}
manifestBlob, err := json.Marshal(manifest)
if err != nil {
return Image{}, err
}
return Image{
Manifest: manifest,
Config: conf,
Layers: layers,
Reference: godigest.FromBytes(manifestBlob).String(),
}, nil
}
func UploadImage(img Image, baseURL, repo string) error {
for _, blob := range img.Layers {
resp, err := resty.R().Post(baseURL + "/v2/" + repo + "/blobs/uploads/")
@ -830,7 +946,19 @@ func UploadImage(img Image, baseURL, repo string) error {
return err
}
func UploadArtifact(baseURL, repo string, artifactManifest *ispec.Artifact) error {
func DeleteImage(repo, reference, baseURL string) (int, error) {
resp, err := resty.R().Delete(
fmt.Sprintf(baseURL+"/v2/%s/manifests/%s", repo, reference),
)
if err != nil {
return -1, err
}
return resp.StatusCode(), err
}
// UploadArtifactManifest is used in tests where we don't need to upload the blobs of the artifact.
func UploadArtifactManifest(artifactManifest *ispec.Artifact, baseURL, repo string) error {
// put manifest
artifactManifestBlob, err := json.Marshal(artifactManifest)
if err != nil {

View file

@ -8,6 +8,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path"
"testing"
@ -21,10 +22,12 @@ import (
"zotregistry.io/zot/pkg/api"
"zotregistry.io/zot/pkg/api/config"
"zotregistry.io/zot/pkg/storage"
"zotregistry.io/zot/pkg/test"
"zotregistry.io/zot/pkg/test/mocks"
)
var ErrTestError = errors.New("test error")
var ErrTestError = errors.New("ErrTestError")
func TestCopyFiles(t *testing.T) {
Convey("sourceDir does not exist", t, func() {
@ -238,7 +241,7 @@ func TestUploadArtifact(t *testing.T) {
artifact := ispec.Artifact{}
err := test.UploadArtifact(baseURL, "test", &artifact)
err := test.UploadArtifactManifest(&artifact, baseURL, "test")
So(err, ShouldNotBeNil)
})
}
@ -1274,3 +1277,62 @@ func TestGenerateNotationCerts(t *testing.T) {
So(err, ShouldNotBeNil)
})
}
func TestWriteImageToFileSystem(t *testing.T) {
Convey("WriteImageToFileSystem errors", t, func() {
err := test.WriteImageToFileSystem(test.Image{}, "repo", storage.StoreController{
DefaultStore: mocks.MockedImageStore{
InitRepoFn: func(name string) error {
return ErrTestError
},
},
})
So(err, ShouldNotBeNil)
err = test.WriteImageToFileSystem(
test.Image{Layers: [][]byte{[]byte("testLayer")}},
"repo",
storage.StoreController{
DefaultStore: mocks.MockedImageStore{
FullBlobUploadFn: func(repo string, body io.Reader, digest godigest.Digest,
) (string, int64, error) {
return "", 0, ErrTestError
},
},
})
So(err, ShouldNotBeNil)
count := 0
err = test.WriteImageToFileSystem(
test.Image{Layers: [][]byte{[]byte("testLayer")}},
"repo",
storage.StoreController{
DefaultStore: mocks.MockedImageStore{
FullBlobUploadFn: func(repo string, body io.Reader, digest godigest.Digest,
) (string, int64, error) {
if count == 0 {
count++
return "", 0, nil
}
return "", 0, ErrTestError
},
},
})
So(err, ShouldNotBeNil)
err = test.WriteImageToFileSystem(
test.Image{Layers: [][]byte{[]byte("testLayer")}},
"repo",
storage.StoreController{
DefaultStore: mocks.MockedImageStore{
PutImageManifestFn: func(repo, reference, mediaType string, body []byte,
) (godigest.Digest, error) {
return "", ErrTestError
},
},
})
So(err, ShouldNotBeNil)
})
}

View file

@ -40,6 +40,19 @@ type RepoDBMock struct {
GetIndexDataFn func(indexDigest godigest.Digest) (repodb.IndexData, error)
SetArtifactDataFn func(digest godigest.Digest, artifactData repodb.ArtifactData) error
GetArtifactDataFn func(artifactDigest godigest.Digest) (repodb.ArtifactData, error)
SetReferrerFn func(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error
DeleteReferrerFn func(repo string, referredDigest godigest.Digest, referrerDigest godigest.Digest) error
GetReferrersFn func(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error)
GetFilteredReferrersInfoFn func(repo string, referredDigest godigest.Digest, artifactTypes []string) (
[]repodb.ReferrerInfo, error)
IncrementImageDownloadsFn func(repo string, reference string) error
AddManifestSignatureFn func(repo string, signedManifestDigest godigest.Digest, sm repodb.SignatureMetadata) error
@ -103,14 +116,6 @@ func (sdm RepoDBMock) GetRepoStars(repo string) (int, error) {
return 0, nil
}
func (sdm RepoDBMock) SetRepoLogo(repo string, logoPath string) error {
if sdm.SetRepoLogoFn != nil {
return sdm.SetRepoLogoFn(repo, logoPath)
}
return nil
}
func (sdm RepoDBMock) SetRepoReference(repo string, reference string, manifestDigest godigest.Digest,
mediaType string,
) error {
@ -300,3 +305,55 @@ func (sdm RepoDBMock) PatchDB() error {
return nil
}
func (sdm RepoDBMock) SetArtifactData(digest godigest.Digest, artifactData repodb.ArtifactData) error {
if sdm.SetArtifactDataFn != nil {
return sdm.SetArtifactDataFn(digest, artifactData)
}
return nil
}
func (sdm RepoDBMock) GetArtifactData(artifactDigest godigest.Digest) (repodb.ArtifactData, error) {
if sdm.GetArtifactDataFn != nil {
return sdm.GetArtifactDataFn(artifactDigest)
}
return repodb.ArtifactData{}, nil
}
func (sdm RepoDBMock) SetReferrer(repo string, referredDigest godigest.Digest, referrer repodb.Descriptor) error {
if sdm.SetReferrerFn != nil {
return sdm.SetReferrerFn(repo, referredDigest, referrer)
}
return nil
}
func (sdm RepoDBMock) DeleteReferrer(repo string, referredDigest godigest.Digest,
referrerDigest godigest.Digest,
) error {
if sdm.DeleteReferrerFn != nil {
return sdm.DeleteReferrerFn(repo, referredDigest, referrerDigest)
}
return nil
}
func (sdm RepoDBMock) GetReferrers(repo string, referredDigest godigest.Digest) ([]repodb.Descriptor, error) {
if sdm.GetReferrersFn != nil {
return sdm.GetReferrersFn(repo, referredDigest)
}
return []repodb.Descriptor{}, nil
}
func (sdm RepoDBMock) GetFilteredReferrersInfo(repo string, referredDigest godigest.Digest,
artifactTypes []string,
) ([]repodb.ReferrerInfo, error) {
if sdm.GetFilteredReferrersInfoFn != nil {
return sdm.GetFilteredReferrersInfoFn(repo, referredDigest, artifactTypes)
}
return []repodb.ReferrerInfo{}, nil
}

View file

@ -37,6 +37,7 @@ function setup() {
"cacheTablename": "BlobTable",
"repoMetaTablename": "RepoMetadataTable",
"manifestDataTablename": "ManifestDataTable",
"artifactDataTablename": "ArtifactDataTable",
"indexDataTablename": "IndexDataTable",
"versionTablename": "Version"
}

View file

@ -0,0 +1,80 @@
ROOT_DIR=$(git rev-parse --show-toplevel)
TEST_DATA_DIR=${ROOT_DIR}/test/data/
OS="${OS:-linux}"
ARCH="${ARCH:-amd64}"
ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}
mkdir -p ${TEST_DATA_DIR}
function verify_prerequisites {
if [ ! -f ${BATS_RUN_TMPDIR}/.firstrun ]; then
env | grep proxy >&3
touch ${BATS_RUN_TMPDIR}/.firstrun
fi
if [ ! -f ${ZOT_PATH} ]; then
echo "you need to build ${ZOT_PATH} before running the tests" >&3
return 1
fi
if [ ! -f ${ZOT_MINIMAL_PATH} ]; then
echo "you need to build ${ZOT_MINIMAL_PATH} before running tests" >&3
return 1
fi
if [ ! command -v curl ] &>/dev/null; then
echo "you need to install curl as a prerequisite to running the tests" >&3
return 1
fi
if [ ! command -v jq ] &>/dev/null; then
echo "you need to install jq as a prerequisite to running the tests" >&3
return 1
fi
if [ ! command -v skopeo ] &>/dev/null; then
echo "you need to install skopeo as a prerequisite to running the tests" >&3
return 1
fi
if [ ! command -v oras ] &>/dev/null; then
echo "you need to install oras as a prerequisite to running the tests" >&3
return 1
fi
return 0
}
function zot_serve() {
local zot_path=${1}
local config_file=${2}
local pid_dir=${3}
${zot_path} serve ${config_file} &
echo $! >>${pid_dir}/zot.pid
}
function zot_stop() {
local pid_dir=${1}
cat ${pid_dir}/zot.pid
kill $(cat ${pid_dir}/zot.pid)
rm ${pid_dir}/zot.pid
}
function setup_zot_file_level() {
local config_file=${1}
zot_serve ${ZOT_PATH} ${config_file} ${BATS_FILE_TMPDIR}
}
function teardown_zot_file_level() {
zot_stop ${BATS_FILE_TMPDIR}
}
function wait_zot_reachable() {
zot_url=${1}
curl --connect-timeout 3 \
--max-time 3 \
--retry 10 \
--retry-delay 0 \
--retry-max-time 60 \
--retry-connrefused \
${zot_url}
}

View file

@ -0,0 +1,109 @@
load helpers_referrers
function setup() {
# Verify prerequisites are available
if ! verify_prerequisites; then
exit 1
fi
# Download test data to folder common for the entire suite, not just this file
skopeo --insecure-policy copy --format=oci docker://alpine:latest oci:${TEST_DATA_DIR}alpine:latest
# Setup zot server
ZOT_ROOT_DIR=${BATS_FILE_TMPDIR}/zot
echo ${ZOT_ROOT_DIR}
ZOT_LOG_FILE=${ZOT_ROOT_DIR}/zot-log.json
ZOT_CONFIG_FILE=${BATS_FILE_TMPDIR}/zot_config.json
mkdir -p ${ZOT_ROOT_DIR}
touch ${ZOT_LOG_FILE}
cat >${ZOT_CONFIG_FILE} <<EOF
{
"distSpecVersion": "1.1.0",
"storage": {
"rootDirectory": "${ZOT_ROOT_DIR}"
},
"http": {
"address": "0.0.0.0",
"port": "8080"
},
"log": {
"level": "debug",
"output": "${ZOT_LOG_FILE}"
},
"extensions": {
"search": {
"enable": true
}
}
}
EOF
# Add artifact contents to files
ARTIFACT_BLOBS_DIR=${BATS_FILE_TMPDIR}/artifact-blobs
mkdir -p ${ARTIFACT_BLOBS_DIR}
IMAGE_MANIFEST_REFERRER=${ARTIFACT_BLOBS_DIR}/image-manifest-ref-blob
echo IMAGE_MANIFEST_REFERRER=${IMAGE_MANIFEST_REFERRER}
touch ${IMAGE_MANIFEST_REFERRER}
cat >${IMAGE_MANIFEST_REFERRER} <<EOF
This artifact is represented as an ispec image manifest, this is the layer inside the manifest.
EOF
ARTIFACT_MANIFEST_REFERRER=${ARTIFACT_BLOBS_DIR}/artifact-manifest-ref-blob
touch ${ARTIFACT_MANIFEST_REFERRER}
cat >${ARTIFACT_MANIFEST_REFERRER} <<EOF
This artifact is represented as an ispec artifact manifest, this is the blob inside the manifest.
EOF
setup_zot_file_level ${ZOT_CONFIG_FILE}
echo "yes"
wait_zot_reachable "http://127.0.0.1:8080/v2/_catalog"
run skopeo --insecure-policy copy --dest-tls-verify=false \
oci:${TEST_DATA_DIR}/alpine:latest \
docker://127.0.0.1:8080/alpine:latest
[ "$status" -eq 0 ]
run curl http://127.0.0.1:8080/v2/_catalog
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.repositories[]') = '"alpine"' ]
run oras attach --plain-http --image-spec v1.1-image --artifact-type image.type 127.0.0.1:8080/alpine:latest ${IMAGE_MANIFEST_REFERRER}
[ "$status" -eq 0 ]
run oras attach --plain-http --image-spec v1.1-artifact --artifact-type artifact.type 127.0.0.1:8080/alpine:latest ${ARTIFACT_MANIFEST_REFERRER}
[ "$status" -eq 0 ]
MANIFEST_DIGEST=$(skopeo inspect --tls-verify=false docker://localhost:8080/alpine:latest | jq -r '.Digest')
echo ${MANIFEST_DIGEST}
curl -X GET http://127.0.0.1:8080/v2/alpine/referrers/${MANIFEST_DIGEST}?artifactType=image.type
}
function teardown() {
local ZOT_ROOT_DIR=${BATS_FILE_TMPDIR}/zot
zot_stop ${BATS_FILE_TMPDIR}
rm -rf ${ZOT_ROOT_DIR}
}
@test "add referrers, one artifact and one image" {
# Check referrers API using the normal REST endpoint
run curl -X GET http://127.0.0.1:8080/v2/alpine/referrers/${MANIFEST_DIGEST}?artifactType=image.type
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.manifests[].artifactType') = '"image.type"' ]
run curl -X GET http://127.0.0.1:8080/v2/alpine/referrers/${MANIFEST_DIGEST}?artifactType=artifact.type
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.manifests[].artifactType') = '"artifact.type"' ]
# Check referrers API using the GQL endpoint
REFERRER_QUERY_DATA="{ \"query\": \"{ Referrers(repo:\\\"alpine\\\", digest:\\\"${MANIFEST_DIGEST}\\\", type:[\\\"image.type\\\"]) { MediaType ArtifactType Digest Size} }\"}"
run curl -X POST -H "Content-Type: application/json" --data "${REFERRER_QUERY_DATA}" http://localhost:8080/v2/_zot/ext/search
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.data.Referrers[].ArtifactType') = '"image.type"' ]
REFERRER_QUERY_DATA="{ \"query\": \"{ Referrers(repo:\\\"alpine\\\", digest:\\\"${MANIFEST_DIGEST}\\\", type:[\\\"artifact.type\\\"]) { MediaType ArtifactType Digest Size} }\"}"
run curl -X POST -H "Content-Type: application/json" --data "${REFERRER_QUERY_DATA}" http://localhost:8080/v2/_zot/ext/search
[ "$status" -eq 0 ]
[ $(echo "${lines[-1]}" | jq '.data.Referrers[].ArtifactType') = '"artifact.type"' ]
}