Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2025-02-17 23:45:36 -05:00
Code Issues Activity

fix(metadb): fix unexpected panic when dereferencing map fields (#1993)

Browse source 
Signed-off-by: Laurentiu Niculae <niculae.laurentiu1@gmail.com>
This commit is contained in:
LaurentiuNiculae 2023-11-02 23:35:49 +02:00 • committed by GitHub
parent 9074f8483b
commit 6a66a9b9b4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 248 additions and 260 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

398
pkg/meta/boltdb/boltdb.go
View file

@ -148,21 +148,9 @@ func (bdw *BoltDB) SetRepoReference(ctx context.Context, repo string, reference
return err return err
} }
repoMetaBlob := repoBuck.Get([]byte(repo)) protoRepoMeta, err := getProtoRepoMeta(repo, repoBuck)
if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
protoRepoMeta := &proto_go.RepoMeta{ return err
Name: repo,
Tags: map[string]*proto_go.TagDescriptor{"": {}}, // This is done so Protobuf can initialize a non-nil map
Statistics: map[string]*proto_go.DescriptorStatistics{"": {}},
Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}},
Referrers: map[string]*proto_go.ReferrersInfo{"": {}},
}
if len(repoMetaBlob) > 0 {
err := proto.Unmarshal(repoMetaBlob, protoRepoMeta)
if err != nil {
return err
}
} }
// 2. Referrers // 2. Referrers
@ -227,17 +215,11 @@ func (bdw *BoltDB) SetRepoReference(ctx context.Context, repo string, reference
} }
// 4. Blobs // 4. Blobs
repoBlobsBytes := repoBlobsBuck.Get([]byte(protoRepoMeta.Name)) repoBlobsBytes := repoBlobsBuck.Get([]byte(repo))
repoBlobs := &proto_go.RepoBlobs{} repoBlobs, err := unmarshalProtoRepoBlobs(repo, repoBlobsBytes)
if err != nil {
if len(repoBlobsBytes) == 0 { return err
repoBlobs.Blobs = make(map[string]*proto_go.BlobInfo)
} else {
err := proto.Unmarshal(repoBlobsBytes, repoBlobs)
if err != nil {
return err
}
} }
protoRepoMeta, repoBlobs, err = common.AddImageMetaToRepoMeta(protoRepoMeta, repoBlobs, reference, imageMeta) protoRepoMeta, repoBlobs, err = common.AddImageMetaToRepoMeta(protoRepoMeta, repoBlobs, reference, imageMeta)
@ -245,27 +227,97 @@ func (bdw *BoltDB) SetRepoReference(ctx context.Context, repo string, reference
return err return err
} }
repoBlobsBytes, err = proto.Marshal(repoBlobs) err = setProtoRepoBlobs(repoBlobs, repoBlobsBuck)
if err != nil { if err != nil {
return err return err
} }
err = repoBlobsBuck.Put([]byte(protoRepoMeta.Name), repoBlobsBytes) return setProtoRepoMeta(protoRepoMeta, repoBuck)
if err != nil {
return err
}
repoMetaBlob, err = proto.Marshal(protoRepoMeta)
if err != nil {
return err
}
return repoBuck.Put([]byte(repo), repoMetaBlob)
}) })
return err return err
} }
func unmarshalProtoRepoBlobs(repo string, repoBlobsBytes []byte) (*proto_go.RepoBlobs, error) {
repoBlobs := &proto_go.RepoBlobs{
Name: repo,
}
if len(repoBlobsBytes) > 0 {
err := proto.Unmarshal(repoBlobsBytes, repoBlobs)
if err != nil {
return nil, err
}
}
if repoBlobs.Blobs == nil {
repoBlobs.Blobs = map[string]*proto_go.BlobInfo{"": {}}
}
return repoBlobs, nil
}
func setProtoRepoBlobs(repoBlobs *proto_go.RepoBlobs, repoBlobsBuck *bbolt.Bucket) error {
repoBlobsBytes, err := proto.Marshal(repoBlobs)
if err != nil {
return err
}
return repoBlobsBuck.Put([]byte(repoBlobs.Name), repoBlobsBytes)
}
func getProtoRepoMeta(repo string, repoMetaBuck *bbolt.Bucket) (*proto_go.RepoMeta, error) {
repoMetaBlob := repoMetaBuck.Get([]byte(repo))
return unmarshalProtoRepoMeta(repo, repoMetaBlob)
}
// unmarshalProtoRepoMeta will unmarshal the repoMeta blob and initialize nil maps. If the blob is empty
// an empty initialized object is returned.
func unmarshalProtoRepoMeta(repo string, repoMetaBlob []byte) (*proto_go.RepoMeta, error) {
protoRepoMeta := &proto_go.RepoMeta{
Name: repo,
}
if len(repoMetaBlob) > 0 {
err := proto.Unmarshal(repoMetaBlob, protoRepoMeta)
if err != nil {
return nil, err
}
}
if protoRepoMeta.Tags == nil {
protoRepoMeta.Tags = map[string]*proto_go.TagDescriptor{"": {}}
}
if protoRepoMeta.Statistics == nil {
protoRepoMeta.Statistics = map[string]*proto_go.DescriptorStatistics{"": {}}
}
if protoRepoMeta.Signatures == nil {
protoRepoMeta.Signatures = map[string]*proto_go.ManifestSignatures{"": {}}
}
if protoRepoMeta.Referrers == nil {
protoRepoMeta.Referrers = map[string]*proto_go.ReferrersInfo{"": {}}
}
if len(repoMetaBlob) == 0 {
return protoRepoMeta, zerr.ErrRepoMetaNotFound
}
return protoRepoMeta, nil
}
func setProtoRepoMeta(repoMeta *proto_go.RepoMeta, repoBuck *bbolt.Bucket) error {
repoMetaBlob, err := proto.Marshal(repoMeta)
if err != nil {
return err
}
return repoBuck.Put([]byte(repoMeta.Name), repoMetaBlob)
}
func (bdw *BoltDB) FilterImageMeta(ctx context.Context, digests []string, func (bdw *BoltDB) FilterImageMeta(ctx context.Context, digests []string,
) (map[string]mTypes.ImageMeta, error) { ) (map[string]mTypes.ImageMeta, error) {
imageMetaMap := map[string]mTypes.ImageMeta{} imageMetaMap := map[string]mTypes.ImageMeta{}
@ -274,7 +326,7 @@ func (bdw *BoltDB) FilterImageMeta(ctx context.Context, digests []string,
imageBuck := transaction.Bucket([]byte(ImageMetaBuck)) imageBuck := transaction.Bucket([]byte(ImageMetaBuck))
for _, digest := range digests { for _, digest := range digests {
protoImageMeta, err := fetchProtoImageMeta(imageBuck, digest) protoImageMeta, err := getProtoImageMeta(imageBuck, digest)
if err != nil { if err != nil {
return err return err
} }
@ -283,7 +335,7 @@ func (bdw *BoltDB) FilterImageMeta(ctx context.Context, digests []string,
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
for _, manifest := range protoImageMeta.Index.Index.Manifests { for _, manifest := range protoImageMeta.Index.Index.Manifests {
imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest) imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
if err != nil { if err != nil {
return err return err
} }
@ -326,9 +378,7 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string,
continue continue
} }
var protoRepoMeta proto_go.RepoMeta protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob)
err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta)
if err != nil { if err != nil {
return err return err
} }
@ -339,7 +389,7 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string,
protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name)
protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name)
repos = append(repos, mConvert.GetRepoMeta(&protoRepoMeta)) repos = append(repos, mConvert.GetRepoMeta(protoRepoMeta))
} }
return nil return nil
@ -348,7 +398,7 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string,
return repos, err return repos, err
} }
func fetchProtoImageMeta(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageMeta, error) { func getProtoImageMeta(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageMeta, error) {
imageMetaBlob := imageBuck.Get([]byte(digest)) imageMetaBlob := imageBuck.Get([]byte(digest))
if len(imageMetaBlob) == 0 { if len(imageMetaBlob) == 0 {
@ -393,9 +443,7 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string,
return err return err
} }
protoRepoMeta := &proto_go.RepoMeta{} protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob)
err := proto.Unmarshal(repoMetaBlob, protoRepoMeta)
if err != nil { if err != nil {
return err return err
} }
@ -416,7 +464,7 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string,
case ispec.MediaTypeImageManifest: case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest manifestDigest := descriptor.Digest
imageManifestData, err := fetchProtoImageMeta(imageBuck, manifestDigest) imageManifestData, err := getProtoImageMeta(imageBuck, manifestDigest)
if err != nil { if err != nil {
return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w",
manifestDigest, err) manifestDigest, err)
@ -426,7 +474,7 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string,
case ispec.MediaTypeImageIndex: case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest indexDigest := descriptor.Digest
imageIndexData, err := fetchProtoImageMeta(imageBuck, indexDigest) imageIndexData, err := getProtoImageMeta(imageBuck, indexDigest)
if err != nil { if err != nil {
return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w",
indexDigest, err) indexDigest, err)
@ -435,7 +483,7 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string,
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests)) manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests))
for _, manifest := range imageIndexData.Index.Index.Manifests { for _, manifest := range imageIndexData.Index.Index.Manifests {
imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest) imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
if err != nil { if err != nil {
return err return err
} }
@ -483,9 +531,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRe
continue continue
} }
protoRepoMeta := &proto_go.RepoMeta{} protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob)
err := proto.Unmarshal(repoMetaBlob, protoRepoMeta)
if err != nil { if err != nil {
viewError = errors.Join(viewError, err) viewError = errors.Join(viewError, err)
@ -506,7 +552,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRe
case ispec.MediaTypeImageManifest: case ispec.MediaTypeImageManifest:
manifestDigest := descriptor.Digest manifestDigest := descriptor.Digest
imageManifestData, err := fetchProtoImageMeta(imageMetaBuck, manifestDigest) imageManifestData, err := getProtoImageMeta(imageMetaBuck, manifestDigest)
if err != nil { if err != nil {
viewError = errors.Join(viewError, err) viewError = errors.Join(viewError, err)
@ -521,7 +567,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRe
case ispec.MediaTypeImageIndex: case ispec.MediaTypeImageIndex:
indexDigest := descriptor.Digest indexDigest := descriptor.Digest
imageIndexData, err := fetchProtoImageMeta(imageMetaBuck, indexDigest) imageIndexData, err := getProtoImageMeta(imageMetaBuck, indexDigest)
if err != nil { if err != nil {
viewError = errors.Join(viewError, err) viewError = errors.Join(viewError, err)
@ -533,7 +579,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRe
for _, manifest := range imageIndexData.Index.Index.Manifests { for _, manifest := range imageIndexData.Index.Index.Manifests {
manifestDigest := manifest.Digest manifestDigest := manifest.Digest
imageManifestData, err := fetchProtoImageMeta(imageMetaBuck, manifestDigest) imageManifestData, err := getProtoImageMeta(imageMetaBuck, manifestDigest)
if err != nil { if err != nil {
viewError = errors.Join(viewError, err) viewError = errors.Join(viewError, err)
@ -588,9 +634,7 @@ func (bdw *BoltDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRepo
continue continue
} }
repoMeta := proto_go.RepoMeta{} repoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob)
err := proto.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil { if err != nil {
return err return err
} }
@ -598,7 +642,7 @@ func (bdw *BoltDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRepo
repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name)
repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name)
fullRepoMeta := mConvert.GetRepoMeta(&repoMeta) fullRepoMeta := mConvert.GetRepoMeta(repoMeta)
if filter(fullRepoMeta) { if filter(fullRepoMeta) {
repos = append(repos, fullRepoMeta) repos = append(repos, fullRepoMeta)
@ -615,7 +659,7 @@ func (bdw *BoltDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRepo
} }
func (bdw *BoltDB) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMeta, error) { func (bdw *BoltDB) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMeta, error) {
var protoRepoMeta proto_go.RepoMeta var protoRepoMeta *proto_go.RepoMeta
err := bdw.DB.View(func(tx *bbolt.Tx) error { err := bdw.DB.View(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetaBuck)) buck := tx.Bucket([]byte(RepoMetaBuck))
@ -624,13 +668,9 @@ func (bdw *BoltDB) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMet
repoMetaBlob := buck.Get([]byte(repo)) repoMetaBlob := buck.Get([]byte(repo))
// object not found var err error
if repoMetaBlob == nil {
return zerr.ErrRepoMetaNotFound
}
// object found protoRepoMeta, err = unmarshalProtoRepoMeta(repo, repoMetaBlob)
err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta)
if err != nil { if err != nil {
return err return err
} }
@ -642,7 +682,7 @@ func (bdw *BoltDB) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMet
return nil return nil
}) })
return mConvert.GetRepoMeta(&protoRepoMeta), err return mConvert.GetRepoMeta(protoRepoMeta), err
} }
func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string) (mTypes.FullImageMeta, error) { func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string) (mTypes.FullImageMeta, error) {
@ -662,8 +702,9 @@ func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string
return zerr.ErrRepoMetaNotFound return zerr.ErrRepoMetaNotFound
} }
// object found var err error
err := proto.Unmarshal(repoMetaBlob, protoRepoMeta)
protoRepoMeta, err = unmarshalProtoRepoMeta(repo, repoMetaBlob)
if err != nil { if err != nil {
return err return err
} }
@ -677,7 +718,7 @@ func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string
return zerr.ErrImageMetaNotFound return zerr.ErrImageMetaNotFound
} }
protoImageMeta, err = fetchProtoImageMeta(imageBuck, descriptor.Digest) protoImageMeta, err = getProtoImageMeta(imageBuck, descriptor.Digest)
if err != nil { if err != nil {
return err return err
} }
@ -686,7 +727,7 @@ func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
for _, manifest := range protoImageMeta.Index.Index.Manifests { for _, manifest := range protoImageMeta.Index.Index.Manifests {
imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest) imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
if err != nil { if err != nil {
return err return err
} }
@ -709,7 +750,7 @@ func (bdw *BoltDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, error
err := bdw.DB.View(func(tx *bbolt.Tx) error { err := bdw.DB.View(func(tx *bbolt.Tx) error {
imageBuck := tx.Bucket([]byte(ImageMetaBuck)) imageBuck := tx.Bucket([]byte(ImageMetaBuck))
protoImageMeta, err := fetchProtoImageMeta(imageBuck, digest.String()) protoImageMeta, err := getProtoImageMeta(imageBuck, digest.String())
if err != nil { if err != nil {
return err return err
} }
@ -718,7 +759,7 @@ func (bdw *BoltDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, error
manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests))
for _, manifest := range protoImageMeta.Index.Index.Manifests { for _, manifest := range protoImageMeta.Index.Index.Manifests {
imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest) imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest)
if err != nil { if err != nil {
return err return err
} }
@ -751,16 +792,14 @@ func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta
continue continue
} }
protoRepoMeta := proto_go.RepoMeta{} protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob)
err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta)
if err != nil { if err != nil {
return err return err
} }
delete(protoRepoMeta.Tags, "") delete(protoRepoMeta.Tags, "")
repoMeta := mConvert.GetRepoMeta(&protoRepoMeta) repoMeta := mConvert.GetRepoMeta(protoRepoMeta)
if filter(repoMeta) { if filter(repoMeta) {
foundRepos = append(foundRepos, repoMeta) foundRepos = append(foundRepos, repoMeta)
@ -777,9 +816,9 @@ func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godige
sygMeta mTypes.SignatureMetadata, sygMeta mTypes.SignatureMetadata,
) error { ) error {
err := bdw.DB.Update(func(tx *bbolt.Tx) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBlob := buck.Get([]byte(repo)) repoMetaBlob := repoMetaBuck.Get([]byte(repo))
if len(repoMetaBlob) == 0 { if len(repoMetaBlob) == 0 {
var err error var err error
@ -810,12 +849,10 @@ func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godige
return err return err
} }
return buck.Put([]byte(repo), repoMetaBlob) return repoMetaBuck.Put([]byte(repo), repoMetaBlob)
} }
protoRepoMeta := &proto_go.RepoMeta{} protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
err := proto.Unmarshal(repoMetaBlob, protoRepoMeta)
if err != nil { if err != nil {
return err return err
} }
@ -852,12 +889,7 @@ func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godige
manifestSignatures.Map[sygMeta.SignatureType] = signatureSlice manifestSignatures.Map[sygMeta.SignatureType] = signatureSlice
protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures
repoMetaBlob, err = proto.Marshal(protoRepoMeta) return setProtoRepoMeta(protoRepoMeta, repoMetaBuck)
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
}) })
return err return err
@ -867,16 +899,14 @@ func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Di
sigMeta mTypes.SignatureMetadata, sigMeta mTypes.SignatureMetadata,
) error { ) error {
err := bdw.DB.Update(func(tx *bbolt.Tx) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBlob := buck.Get([]byte(repo)) repoMetaBlob := repoMetaBuck.Get([]byte(repo))
if repoMetaBlob == nil { if repoMetaBlob == nil {
return zerr.ErrManifestMetaNotFound return zerr.ErrManifestMetaNotFound
} }
protoRepoMeta := proto_go.RepoMeta{} protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta)
if err != nil { if err != nil {
return err return err
} }
@ -900,12 +930,7 @@ func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Di
protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures
repoMetaBlob, err = proto.Marshal(&protoRepoMeta) return setProtoRepoMeta(protoRepoMeta, repoMetaBuck)
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
}) })
return err return err
@ -913,28 +938,21 @@ func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Di
func (bdw *BoltDB) IncrementRepoStars(repo string) error { func (bdw *BoltDB) IncrementRepoStars(repo string) error {
err := bdw.DB.Update(func(tx *bbolt.Tx) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBlob := buck.Get([]byte(repo)) repoMetaBlob := repoMetaBuck.Get([]byte(repo))
if repoMetaBlob == nil { if repoMetaBlob == nil {
return zerr.ErrRepoMetaNotFound return zerr.ErrRepoMetaNotFound
} }
var repoMeta proto_go.RepoMeta protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
err := proto.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil { if err != nil {
return err return err
} }
repoMeta.Stars++ protoRepoMeta.Stars++
repoMetaBlob, err = proto.Marshal(&repoMeta) return setProtoRepoMeta(protoRepoMeta, repoMetaBuck)
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
}) })
return err return err
@ -942,32 +960,25 @@ func (bdw *BoltDB) IncrementRepoStars(repo string) error {
func (bdw *BoltDB) DecrementRepoStars(repo string) error { func (bdw *BoltDB) DecrementRepoStars(repo string) error {
err := bdw.DB.Update(func(tx *bbolt.Tx) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBlob := buck.Get([]byte(repo)) repoMetaBlob := repoMetaBuck.Get([]byte(repo))
if repoMetaBlob == nil { if repoMetaBlob == nil {
return zerr.ErrRepoMetaNotFound return zerr.ErrRepoMetaNotFound
} }
var repoMeta proto_go.RepoMeta protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
err := proto.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil { if err != nil {
return err return err
} }
if repoMeta.Stars == 0 { if protoRepoMeta.Stars == 0 {
return nil return nil
} }
repoMeta.Stars-- protoRepoMeta.Stars--
repoMetaBlob, err = proto.Marshal(&repoMeta) return setProtoRepoMeta(protoRepoMeta, repoMetaBuck)
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
}) })
return err return err
@ -996,14 +1007,8 @@ func (bdw *BoltDB) ResetRepoReferences(repo string) error {
repoMetaBlob := buck.Get([]byte(repo)) repoMetaBlob := buck.Get([]byte(repo))
if repoMetaBlob == nil { protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
return nil if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
}
protoRepoMeta := &proto_go.RepoMeta{}
err := proto.Unmarshal(repoMetaBlob, protoRepoMeta)
if err != nil {
return err return err
} }
@ -1033,18 +1038,13 @@ func (bdw *BoltDB) GetReferrersInfo(repo string, referredDigest godigest.Digest,
buck := tx.Bucket([]byte(RepoMetaBuck)) buck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBlob := buck.Get([]byte(repo)) repoMetaBlob := buck.Get([]byte(repo))
if len(repoMetaBlob) == 0 {
return zerr.ErrRepoMetaNotFound
}
var repoMeta proto_go.RepoMeta protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
err := proto.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil { if err != nil {
return err return err
} }
referrersInfo := repoMeta.Referrers[referredDigest.String()].List referrersInfo := protoRepoMeta.Referrers[referredDigest.String()].List
for i := range referrersInfo { for i := range referrersInfo {
if !common.MatchesArtifactTypes(referrersInfo[i].ArtifactType, artifactTypes) { if !common.MatchesArtifactTypes(referrersInfo[i].ArtifactType, artifactTypes) {
@ -1068,16 +1068,14 @@ func (bdw *BoltDB) GetReferrersInfo(repo string, referredDigest godigest.Digest,
func (bdw *BoltDB) UpdateStatsOnDownload(repo string, reference string) error { func (bdw *BoltDB) UpdateStatsOnDownload(repo string, reference string) error {
err := bdw.DB.Update(func(tx *bbolt.Tx) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck))
repoMetaBlob := buck.Get([]byte(repo)) repoMetaBlob := repoMetaBuck.Get([]byte(repo))
if repoMetaBlob == nil { if repoMetaBlob == nil {
return zerr.ErrRepoMetaNotFound return zerr.ErrRepoMetaNotFound
} }
var repoMeta proto_go.RepoMeta protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
err := proto.Unmarshal(repoMetaBlob, &repoMeta)
if err != nil { if err != nil {
return err return err
} }
@ -1086,7 +1084,7 @@ func (bdw *BoltDB) UpdateStatsOnDownload(repo string, reference string) error {
if !common.ReferenceIsDigest(reference) { if !common.ReferenceIsDigest(reference) {
// search digest for tag // search digest for tag
descriptor, found := repoMeta.Tags[reference] descriptor, found := protoRepoMeta.Tags[reference]
if !found { if !found {
return zerr.ErrManifestMetaNotFound return zerr.ErrManifestMetaNotFound
@ -1095,21 +1093,16 @@ func (bdw *BoltDB) UpdateStatsOnDownload(repo string, reference string) error {
manifestDigest = descriptor.Digest manifestDigest = descriptor.Digest
} }
manifestStatistics, ok := repoMeta.Statistics[manifestDigest] manifestStatistics, ok := protoRepoMeta.Statistics[manifestDigest]
if !ok { if !ok {
return zerr.ErrManifestMetaNotFound return zerr.ErrManifestMetaNotFound
} }
manifestStatistics.DownloadCount++ manifestStatistics.DownloadCount++
manifestStatistics.LastPullTimestamp = timestamppb.Now() manifestStatistics.LastPullTimestamp = timestamppb.Now()
repoMeta.Statistics[manifestDigest] = manifestStatistics protoRepoMeta.Statistics[manifestDigest] = manifestStatistics
repoMetaBlob, err = proto.Marshal(&repoMeta) return setProtoRepoMeta(protoRepoMeta, repoMetaBuck)
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
}) })
return err return err
@ -1147,9 +1140,7 @@ func (bdw *BoltDB) UpdateSignaturesValidity(repo string, manifestDigest godigest
return zerr.ErrRepoMetaNotFound return zerr.ErrRepoMetaNotFound
} }
protoRepoMeta := proto_go.RepoMeta{} protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
err = proto.Unmarshal(repoMetaBlob, &protoRepoMeta)
if err != nil { if err != nil {
return err return err
} }
@ -1188,12 +1179,7 @@ func (bdw *BoltDB) UpdateSignaturesValidity(repo string, manifestDigest godigest
protoRepoMeta.Signatures[manifestDigest.String()] = &manifestSignatures protoRepoMeta.Signatures[manifestDigest.String()] = &manifestSignatures
repoMetaBlob, err = proto.Marshal(&protoRepoMeta) return setProtoRepoMeta(protoRepoMeta, repoBuck)
if err != nil {
return err
}
return repoBuck.Put([]byte(repo), repoMetaBlob)
}) })
return err return err
@ -1201,23 +1187,20 @@ func (bdw *BoltDB) UpdateSignaturesValidity(repo string, manifestDigest godigest
func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error { func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error {
err := bdw.DB.Update(func(tx *bbolt.Tx) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error {
buck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck))
imageMetaBuck := tx.Bucket([]byte(ImageMetaBuck)) imageMetaBuck := tx.Bucket([]byte(ImageMetaBuck))
repoBlobsBuck := tx.Bucket([]byte(RepoBlobsBuck)) repoBlobsBuck := tx.Bucket([]byte(RepoBlobsBuck))
repoMetaBlob := buck.Get([]byte(repo)) protoRepoMeta, err := getProtoRepoMeta(repo, repoMetaBuck)
if repoMetaBlob == nil {
return nil
}
repoMeta := &proto_go.RepoMeta{}
err := proto.Unmarshal(repoMetaBlob, repoMeta)
if err != nil { if err != nil {
if errors.Is(err, zerr.ErrRepoMetaNotFound) {
return nil
}
return err return err
} }
imageMeta, err := fetchProtoImageMeta(imageMetaBuck, manifestDigest.String()) protoImageMeta, err := getProtoImageMeta(imageMetaBuck, manifestDigest.String())
if err != nil { if err != nil {
if errors.Is(err, zerr.ErrImageMetaNotFound) { if errors.Is(err, zerr.ErrImageMetaNotFound) {
return nil return nil
@ -1227,12 +1210,12 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go
} }
// Remove Referrers // Remove Referrers
if subject := mConvert.GetImageSubject(imageMeta); subject != nil { if subject := mConvert.GetImageSubject(protoImageMeta); subject != nil {
referredDigest := subject.Digest.String() referredDigest := subject.Digest.String()
refInfo := &proto_go.ReferrersInfo{} refInfo := &proto_go.ReferrersInfo{}
if repoMeta.Referrers[referredDigest] != nil { if protoRepoMeta.Referrers[referredDigest] != nil {
refInfo = repoMeta.Referrers[referredDigest] refInfo = protoRepoMeta.Referrers[referredDigest]
} }
referrers := refInfo.List referrers := refInfo.List
@ -1251,16 +1234,16 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go
refInfo.List = referrers refInfo.List = referrers
repoMeta.Referrers[referredDigest] = refInfo protoRepoMeta.Referrers[referredDigest] = refInfo
} }
if !common.ReferenceIsDigest(reference) { if !common.ReferenceIsDigest(reference) {
delete(repoMeta.Tags, reference) delete(protoRepoMeta.Tags, reference)
} else { } else {
// remove all tags pointing to this digest // remove all tags pointing to this digest
for tag, desc := range repoMeta.Tags { for tag, desc := range protoRepoMeta.Tags {
if desc.Digest == reference { if desc.Digest == reference {
delete(repoMeta.Tags, tag) delete(protoRepoMeta.Tags, tag)
} }
} }
} }
@ -1268,32 +1251,26 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go
/* try to find at least one tag pointing to manifestDigest /* try to find at least one tag pointing to manifestDigest
if not found then we can also remove everything related to this digest */ if not found then we can also remove everything related to this digest */
var foundTag bool var foundTag bool
for _, desc := range repoMeta.Tags { for _, desc := range protoRepoMeta.Tags {
if desc.Digest == manifestDigest.String() { if desc.Digest == manifestDigest.String() {
foundTag = true foundTag = true
} }
} }
if !foundTag { if !foundTag {
delete(repoMeta.Statistics, manifestDigest.String()) delete(protoRepoMeta.Statistics, manifestDigest.String())
delete(repoMeta.Signatures, manifestDigest.String()) delete(protoRepoMeta.Signatures, manifestDigest.String())
delete(repoMeta.Referrers, manifestDigest.String()) delete(protoRepoMeta.Referrers, manifestDigest.String())
} }
repoBlobsBytes := repoBlobsBuck.Get([]byte(repoMeta.Name)) repoBlobsBytes := repoBlobsBuck.Get([]byte(protoRepoMeta.Name))
repoBlobs := &proto_go.RepoBlobs{} repoBlobs, err := unmarshalProtoRepoBlobs(repo, repoBlobsBytes)
if err != nil {
if len(repoBlobsBytes) == 0 { return err
repoBlobs.Blobs = make(map[string]*proto_go.BlobInfo)
} else {
err := proto.Unmarshal(repoBlobsBytes, repoBlobs)
if err != nil {
return err
}
} }
repoMeta, repoBlobs, err = common.RemoveImageFromRepoMeta(repoMeta, repoBlobs, reference) protoRepoMeta, repoBlobs, err = common.RemoveImageFromRepoMeta(protoRepoMeta, repoBlobs, reference)
if err != nil { if err != nil {
return err return err
} }
@ -1303,17 +1280,12 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go
return err return err
} }
err = repoBlobsBuck.Put([]byte(repoMeta.Name), repoBlobsBytes) err = repoBlobsBuck.Put([]byte(protoRepoMeta.Name), repoBlobsBytes)
if err != nil { if err != nil {
return err return err
} }
repoMetaBlob, err = proto.Marshal(repoMeta) return setProtoRepoMeta(protoRepoMeta, repoMetaBuck)
if err != nil {
return err
}
return buck.Put([]byte(repo), repoMetaBlob)
}) })
return err return err
@ -1371,31 +1343,19 @@ func (bdw *BoltDB) ToggleStarRepo(ctx context.Context, repo string) (mTypes.Togg
return zerr.ErrRepoMetaNotFound return zerr.ErrRepoMetaNotFound
} }
repoMeta := &proto_go.RepoMeta{} protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob)
err = proto.Unmarshal(repoMetaBlob, repoMeta)
if err != nil { if err != nil {
return err return err
} }
switch res { switch res {
case mTypes.Added: case mTypes.Added:
repoMeta.Stars++ protoRepoMeta.Stars++
case mTypes.Removed: case mTypes.Removed:
repoMeta.Stars-- protoRepoMeta.Stars--
} }
repoMetaBlob, err = proto.Marshal(repoMeta) return setProtoRepoMeta(protoRepoMeta, repoBuck)
if err != nil {
return err
}
err = repoBuck.Put([]byte(repo), repoMetaBlob)
if err != nil {
return err
}
return nil
}); err != nil { }); err != nil {
return mTypes.NotChanged, err return mTypes.NotChanged, err
} }

8
pkg/meta/convert/convert.go
View file

@ -406,6 +406,10 @@ func GetFullImageMetaFromProto(tag string, protoRepoMeta *proto_go.RepoMeta, pro
func GetFullManifestData(protoRepoMeta *proto_go.RepoMeta, manifestData []mTypes.ManifestData, func GetFullManifestData(protoRepoMeta *proto_go.RepoMeta, manifestData []mTypes.ManifestData,
) []mTypes.FullManifestMeta { ) []mTypes.FullManifestMeta {
if protoRepoMeta == nil {
return []mTypes.FullManifestMeta{}
}
results := []mTypes.FullManifestMeta{} results := []mTypes.FullManifestMeta{}
for i := range manifestData { for i := range manifestData {
@ -421,6 +425,10 @@ func GetFullManifestData(protoRepoMeta *proto_go.RepoMeta, manifestData []mTypes
} }
func GetRepoMeta(protoRepoMeta *proto_go.RepoMeta) mTypes.RepoMeta { func GetRepoMeta(protoRepoMeta *proto_go.RepoMeta) mTypes.RepoMeta {
if protoRepoMeta == nil {
return mTypes.RepoMeta{}
}
repoDownloads := int32(0) repoDownloads := int32(0)
for _, descriptor := range protoRepoMeta.Tags { for _, descriptor := range protoRepoMeta.Tags {

91
pkg/meta/dynamodb/dynamodb.go
View file

@ -208,25 +208,45 @@ func (dwr *DynamoDB) getProtoRepoMeta(ctx context.Context, repo string) (*proto_
return nil, err return nil, err
} }
if resp.Item == nil { protoRepoMeta := &proto_go.RepoMeta{
return nil, zerr.ErrRepoMetaNotFound Name: repo,
} }
blob := []byte{} blob := []byte{}
err = attributevalue.Unmarshal(resp.Item["RepoMetadata"], &blob) if resp.Item != nil {
if err != nil { err = attributevalue.Unmarshal(resp.Item["RepoMetadata"], &blob)
return nil, err if err != nil {
return nil, err
}
err = proto.Unmarshal(blob, protoRepoMeta)
if err != nil {
return nil, err
}
} }
repoMeta := &proto_go.RepoMeta{} if protoRepoMeta.Tags == nil {
protoRepoMeta.Tags = map[string]*proto_go.TagDescriptor{"": {}}
err = proto.Unmarshal(blob, repoMeta)
if err != nil {
return nil, err
} }
return repoMeta, nil if protoRepoMeta.Statistics == nil {
protoRepoMeta.Statistics = map[string]*proto_go.DescriptorStatistics{"": {}}
}
if protoRepoMeta.Signatures == nil {
protoRepoMeta.Signatures = map[string]*proto_go.ManifestSignatures{"": {}}
}
if protoRepoMeta.Referrers == nil {
protoRepoMeta.Referrers = map[string]*proto_go.ReferrersInfo{"": {}}
}
if len(blob) == 0 || resp.Item == nil {
return protoRepoMeta, zerr.ErrRepoMetaNotFound
}
return protoRepoMeta, nil
} }
func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, reference string, func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, reference string,
@ -252,18 +272,8 @@ func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, referenc
} }
repoMeta, err := dwr.getProtoRepoMeta(ctx, repo) repoMeta, err := dwr.getProtoRepoMeta(ctx, repo)
if err != nil { if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) {
if !errors.Is(err, zerr.ErrRepoMetaNotFound) { return err
return err
}
repoMeta = &proto_go.RepoMeta{
Name: repo,
Tags: map[string]*proto_go.TagDescriptor{"": {}},
Statistics: map[string]*proto_go.DescriptorStatistics{"": {}},
Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}},
Referrers: map[string]*proto_go.ReferrersInfo{"": {}},
}
} }
// 2. Referrers // 2. Referrers
@ -328,7 +338,7 @@ func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, referenc
} }
// 4. Blobs // 4. Blobs
repoBlobs, err := dwr.getRepoBlobsInfo(repo) //nolint: contextcheck repoBlobs, err := dwr.getProtoRepoBlobs(ctx, repo)
if err != nil { if err != nil {
return err return err
} }
@ -346,8 +356,8 @@ func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, referenc
return dwr.setProtoRepoMeta(repo, repoMeta) //nolint: contextcheck return dwr.setProtoRepoMeta(repo, repoMeta) //nolint: contextcheck
} }
func (dwr *DynamoDB) getRepoBlobsInfo(repo string) (*proto_go.RepoBlobs, error) { func (dwr *DynamoDB) getProtoRepoBlobs(ctx context.Context, repo string) (*proto_go.RepoBlobs, error) {
resp, err := dwr.Client.GetItem(context.Background(), &dynamodb.GetItemInput{ resp, err := dwr.Client.GetItem(ctx, &dynamodb.GetItemInput{
TableName: aws.String(dwr.RepoBlobsTablename), TableName: aws.String(dwr.RepoBlobsTablename),
Key: map[string]types.AttributeValue{ Key: map[string]types.AttributeValue{
"Key": &types.AttributeValueMemberS{Value: repo}, "Key": &types.AttributeValueMemberS{Value: repo},
@ -357,29 +367,28 @@ func (dwr *DynamoDB) getRepoBlobsInfo(repo string) (*proto_go.RepoBlobs, error)
return nil, err return nil, err
} }
if resp.Item == nil { repoBlobs := &proto_go.RepoBlobs{
return &proto_go.RepoBlobs{Name: repo, Blobs: make(map[string]*proto_go.BlobInfo)}, nil Name: repo,
} }
repoBlobsBytes := []byte{} repoBlobsBytes := []byte{}
err = attributevalue.Unmarshal(resp.Item["RepoBlobsInfo"], &repoBlobsBytes) if resp.Item != nil {
if err != nil { err = attributevalue.Unmarshal(resp.Item["RepoBlobsInfo"], &repoBlobsBytes)
return nil, err
}
repoBlobs := &proto_go.RepoBlobs{}
if len(repoBlobsBytes) == 0 {
repoBlobs.Blobs = make(map[string]*proto_go.BlobInfo)
} else {
err := proto.Unmarshal(repoBlobsBytes, repoBlobs)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if len(repoBlobsBytes) > 0 {
err := proto.Unmarshal(repoBlobsBytes, repoBlobs)
if err != nil {
return nil, err
}
}
} }
if len(repoBlobs.Blobs) == 0 { if repoBlobs.Blobs == nil {
repoBlobs.Blobs = make(map[string]*proto_go.BlobInfo) repoBlobs.Blobs = map[string]*proto_go.BlobInfo{"": {}}
} }
return repoBlobs, nil return repoBlobs, nil
@ -1262,7 +1271,7 @@ func (dwr *DynamoDB) RemoveRepoReference(repo, reference string, manifestDigest
delete(protoRepoMeta.Referrers, manifestDigest.String()) delete(protoRepoMeta.Referrers, manifestDigest.String())
} }
repoBlobsInfo, err := dwr.getRepoBlobsInfo(repo) repoBlobsInfo, err := dwr.getProtoRepoBlobs(context.Background(), repo)
if err != nil { if err != nil {
return err return err
} }

11
pkg/meta/meta_test.go
View file

@ -612,6 +612,17 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func
So(repoMeta2.Size, ShouldEqual, img2Size) So(repoMeta2.Size, ShouldEqual, img2Size)
}) })
Convey("Set, delete and set again", func() {
err := metaDB.SetRepoReference(ctx, repo1, tag1, imgData1)
So(err, ShouldBeNil)
err = metaDB.RemoveRepoReference(repo1, tag1, imgData1.Digest)
So(err, ShouldBeNil)
err = metaDB.SetRepoReference(ctx, repo1, tag1, imgData1)
So(err, ShouldBeNil)
})
Convey("Check repo blobs info for manifest image", func() { Convey("Check repo blobs info for manifest image", func() {
image1 := CreateImageWith().RandomLayers(2, 10). image1 := CreateImageWith().RandomLayers(2, 10).
ImageConfig(ispec.Image{Platform: ispec.Platform{OS: "os1", Architecture: "arch1"}}). ImageConfig(ispec.Image{Platform: ispec.Platform{OS: "os1", Architecture: "arch1"}}).