diff --git a/pkg/meta/boltdb/boltdb.go b/pkg/meta/boltdb/boltdb.go index 0cb25b0b..45e9d9bb 100644 --- a/pkg/meta/boltdb/boltdb.go +++ b/pkg/meta/boltdb/boltdb.go @@ -148,21 +148,9 @@ func (bdw *BoltDB) SetRepoReference(ctx context.Context, repo string, reference return err } - repoMetaBlob := repoBuck.Get([]byte(repo)) - - protoRepoMeta := &proto_go.RepoMeta{ - Name: repo, - Tags: map[string]*proto_go.TagDescriptor{"": {}}, // This is done so Protobuf can initialize a non-nil map - Statistics: map[string]*proto_go.DescriptorStatistics{"": {}}, - Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}}, - Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, - } - - if len(repoMetaBlob) > 0 { - err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) - if err != nil { - return err - } + protoRepoMeta, err := getProtoRepoMeta(repo, repoBuck) + if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { + return err } // 2. Referrers @@ -227,17 +215,11 @@ func (bdw *BoltDB) SetRepoReference(ctx context.Context, repo string, reference } // 4. Blobs - repoBlobsBytes := repoBlobsBuck.Get([]byte(protoRepoMeta.Name)) + repoBlobsBytes := repoBlobsBuck.Get([]byte(repo)) - repoBlobs := &proto_go.RepoBlobs{} - - if len(repoBlobsBytes) == 0 { - repoBlobs.Blobs = make(map[string]*proto_go.BlobInfo) - } else { - err := proto.Unmarshal(repoBlobsBytes, repoBlobs) - if err != nil { - return err - } + repoBlobs, err := unmarshalProtoRepoBlobs(repo, repoBlobsBytes) + if err != nil { + return err } protoRepoMeta, repoBlobs, err = common.AddImageMetaToRepoMeta(protoRepoMeta, repoBlobs, reference, imageMeta) @@ -245,27 +227,97 @@ func (bdw *BoltDB) SetRepoReference(ctx context.Context, repo string, reference return err } - repoBlobsBytes, err = proto.Marshal(repoBlobs) + err = setProtoRepoBlobs(repoBlobs, repoBlobsBuck) if err != nil { return err } - err = repoBlobsBuck.Put([]byte(protoRepoMeta.Name), repoBlobsBytes) - if err != nil { - return err - } - - repoMetaBlob, err = proto.Marshal(protoRepoMeta) - if err != nil { - return err - } - - return repoBuck.Put([]byte(repo), repoMetaBlob) + return setProtoRepoMeta(protoRepoMeta, repoBuck) }) return err } +func unmarshalProtoRepoBlobs(repo string, repoBlobsBytes []byte) (*proto_go.RepoBlobs, error) { + repoBlobs := &proto_go.RepoBlobs{ + Name: repo, + } + + if len(repoBlobsBytes) > 0 { + err := proto.Unmarshal(repoBlobsBytes, repoBlobs) + if err != nil { + return nil, err + } + } + + if repoBlobs.Blobs == nil { + repoBlobs.Blobs = map[string]*proto_go.BlobInfo{"": {}} + } + + return repoBlobs, nil +} + +func setProtoRepoBlobs(repoBlobs *proto_go.RepoBlobs, repoBlobsBuck *bbolt.Bucket) error { + repoBlobsBytes, err := proto.Marshal(repoBlobs) + if err != nil { + return err + } + + return repoBlobsBuck.Put([]byte(repoBlobs.Name), repoBlobsBytes) +} + +func getProtoRepoMeta(repo string, repoMetaBuck *bbolt.Bucket) (*proto_go.RepoMeta, error) { + repoMetaBlob := repoMetaBuck.Get([]byte(repo)) + + return unmarshalProtoRepoMeta(repo, repoMetaBlob) +} + +// unmarshalProtoRepoMeta will unmarshal the repoMeta blob and initialize nil maps. If the blob is empty +// an empty initialized object is returned. +func unmarshalProtoRepoMeta(repo string, repoMetaBlob []byte) (*proto_go.RepoMeta, error) { + protoRepoMeta := &proto_go.RepoMeta{ + Name: repo, + } + + if len(repoMetaBlob) > 0 { + err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + if err != nil { + return nil, err + } + } + + if protoRepoMeta.Tags == nil { + protoRepoMeta.Tags = map[string]*proto_go.TagDescriptor{"": {}} + } + + if protoRepoMeta.Statistics == nil { + protoRepoMeta.Statistics = map[string]*proto_go.DescriptorStatistics{"": {}} + } + + if protoRepoMeta.Signatures == nil { + protoRepoMeta.Signatures = map[string]*proto_go.ManifestSignatures{"": {}} + } + + if protoRepoMeta.Referrers == nil { + protoRepoMeta.Referrers = map[string]*proto_go.ReferrersInfo{"": {}} + } + + if len(repoMetaBlob) == 0 { + return protoRepoMeta, zerr.ErrRepoMetaNotFound + } + + return protoRepoMeta, nil +} + +func setProtoRepoMeta(repoMeta *proto_go.RepoMeta, repoBuck *bbolt.Bucket) error { + repoMetaBlob, err := proto.Marshal(repoMeta) + if err != nil { + return err + } + + return repoBuck.Put([]byte(repoMeta.Name), repoMetaBlob) +} + func (bdw *BoltDB) FilterImageMeta(ctx context.Context, digests []string, ) (map[string]mTypes.ImageMeta, error) { imageMetaMap := map[string]mTypes.ImageMeta{} @@ -274,7 +326,7 @@ func (bdw *BoltDB) FilterImageMeta(ctx context.Context, digests []string, imageBuck := transaction.Bucket([]byte(ImageMetaBuck)) for _, digest := range digests { - protoImageMeta, err := fetchProtoImageMeta(imageBuck, digest) + protoImageMeta, err := getProtoImageMeta(imageBuck, digest) if err != nil { return err } @@ -283,7 +335,7 @@ func (bdw *BoltDB) FilterImageMeta(ctx context.Context, digests []string, manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) for _, manifest := range protoImageMeta.Index.Index.Manifests { - imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest) + imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest) if err != nil { return err } @@ -326,9 +378,7 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, continue } - var protoRepoMeta proto_go.RepoMeta - - err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) + protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob) if err != nil { return err } @@ -339,7 +389,7 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) - repos = append(repos, mConvert.GetRepoMeta(&protoRepoMeta)) + repos = append(repos, mConvert.GetRepoMeta(protoRepoMeta)) } return nil @@ -348,7 +398,7 @@ func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, return repos, err } -func fetchProtoImageMeta(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageMeta, error) { +func getProtoImageMeta(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageMeta, error) { imageMetaBlob := imageBuck.Get([]byte(digest)) if len(imageMetaBlob) == 0 { @@ -393,9 +443,7 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, return err } - protoRepoMeta := &proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob) if err != nil { return err } @@ -416,7 +464,7 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, case ispec.MediaTypeImageManifest: manifestDigest := descriptor.Digest - imageManifestData, err := fetchProtoImageMeta(imageBuck, manifestDigest) + imageManifestData, err := getProtoImageMeta(imageBuck, manifestDigest) if err != nil { return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", manifestDigest, err) @@ -426,7 +474,7 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, case ispec.MediaTypeImageIndex: indexDigest := descriptor.Digest - imageIndexData, err := fetchProtoImageMeta(imageBuck, indexDigest) + imageIndexData, err := getProtoImageMeta(imageBuck, indexDigest) if err != nil { return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", indexDigest, err) @@ -435,7 +483,7 @@ func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests)) for _, manifest := range imageIndexData.Index.Index.Manifests { - imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest) + imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest) if err != nil { return err } @@ -483,9 +531,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRe continue } - protoRepoMeta := &proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob) if err != nil { viewError = errors.Join(viewError, err) @@ -506,7 +552,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRe case ispec.MediaTypeImageManifest: manifestDigest := descriptor.Digest - imageManifestData, err := fetchProtoImageMeta(imageMetaBuck, manifestDigest) + imageManifestData, err := getProtoImageMeta(imageMetaBuck, manifestDigest) if err != nil { viewError = errors.Join(viewError, err) @@ -521,7 +567,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRe case ispec.MediaTypeImageIndex: indexDigest := descriptor.Digest - imageIndexData, err := fetchProtoImageMeta(imageMetaBuck, indexDigest) + imageIndexData, err := getProtoImageMeta(imageMetaBuck, indexDigest) if err != nil { viewError = errors.Join(viewError, err) @@ -533,7 +579,7 @@ func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRe for _, manifest := range imageIndexData.Index.Index.Manifests { manifestDigest := manifest.Digest - imageManifestData, err := fetchProtoImageMeta(imageMetaBuck, manifestDigest) + imageManifestData, err := getProtoImageMeta(imageMetaBuck, manifestDigest) if err != nil { viewError = errors.Join(viewError, err) @@ -588,9 +634,7 @@ func (bdw *BoltDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRepo continue } - repoMeta := proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, &repoMeta) + repoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob) if err != nil { return err } @@ -598,7 +642,7 @@ func (bdw *BoltDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRepo repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) - fullRepoMeta := mConvert.GetRepoMeta(&repoMeta) + fullRepoMeta := mConvert.GetRepoMeta(repoMeta) if filter(fullRepoMeta) { repos = append(repos, fullRepoMeta) @@ -615,7 +659,7 @@ func (bdw *BoltDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRepo } func (bdw *BoltDB) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMeta, error) { - var protoRepoMeta proto_go.RepoMeta + var protoRepoMeta *proto_go.RepoMeta err := bdw.DB.View(func(tx *bbolt.Tx) error { buck := tx.Bucket([]byte(RepoMetaBuck)) @@ -624,13 +668,9 @@ func (bdw *BoltDB) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMet repoMetaBlob := buck.Get([]byte(repo)) - // object not found - if repoMetaBlob == nil { - return zerr.ErrRepoMetaNotFound - } + var err error - // object found - err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) + protoRepoMeta, err = unmarshalProtoRepoMeta(repo, repoMetaBlob) if err != nil { return err } @@ -642,7 +682,7 @@ func (bdw *BoltDB) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMet return nil }) - return mConvert.GetRepoMeta(&protoRepoMeta), err + return mConvert.GetRepoMeta(protoRepoMeta), err } func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string) (mTypes.FullImageMeta, error) { @@ -662,8 +702,9 @@ func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string return zerr.ErrRepoMetaNotFound } - // object found - err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + var err error + + protoRepoMeta, err = unmarshalProtoRepoMeta(repo, repoMetaBlob) if err != nil { return err } @@ -677,7 +718,7 @@ func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string return zerr.ErrImageMetaNotFound } - protoImageMeta, err = fetchProtoImageMeta(imageBuck, descriptor.Digest) + protoImageMeta, err = getProtoImageMeta(imageBuck, descriptor.Digest) if err != nil { return err } @@ -686,7 +727,7 @@ func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) for _, manifest := range protoImageMeta.Index.Index.Manifests { - imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest) + imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest) if err != nil { return err } @@ -709,7 +750,7 @@ func (bdw *BoltDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, error err := bdw.DB.View(func(tx *bbolt.Tx) error { imageBuck := tx.Bucket([]byte(ImageMetaBuck)) - protoImageMeta, err := fetchProtoImageMeta(imageBuck, digest.String()) + protoImageMeta, err := getProtoImageMeta(imageBuck, digest.String()) if err != nil { return err } @@ -718,7 +759,7 @@ func (bdw *BoltDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, error manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) for _, manifest := range protoImageMeta.Index.Index.Manifests { - imageManifestData, err := fetchProtoImageMeta(imageBuck, manifest.Digest) + imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest) if err != nil { return err } @@ -751,16 +792,14 @@ func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta continue } - protoRepoMeta := proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) + protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob) if err != nil { return err } delete(protoRepoMeta.Tags, "") - repoMeta := mConvert.GetRepoMeta(&protoRepoMeta) + repoMeta := mConvert.GetRepoMeta(protoRepoMeta) if filter(repoMeta) { foundRepos = append(foundRepos, repoMeta) @@ -777,9 +816,9 @@ func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godige sygMeta mTypes.SignatureMetadata, ) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) + repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) - repoMetaBlob := buck.Get([]byte(repo)) + repoMetaBlob := repoMetaBuck.Get([]byte(repo)) if len(repoMetaBlob) == 0 { var err error @@ -810,12 +849,10 @@ func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godige return err } - return buck.Put([]byte(repo), repoMetaBlob) + return repoMetaBuck.Put([]byte(repo), repoMetaBlob) } - protoRepoMeta := &proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) + protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) if err != nil { return err } @@ -852,12 +889,7 @@ func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godige manifestSignatures.Map[sygMeta.SignatureType] = signatureSlice protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures - repoMetaBlob, err = proto.Marshal(protoRepoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) + return setProtoRepoMeta(protoRepoMeta, repoMetaBuck) }) return err @@ -867,16 +899,14 @@ func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Di sigMeta mTypes.SignatureMetadata, ) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) + repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) - repoMetaBlob := buck.Get([]byte(repo)) + repoMetaBlob := repoMetaBuck.Get([]byte(repo)) if repoMetaBlob == nil { return zerr.ErrManifestMetaNotFound } - protoRepoMeta := proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, &protoRepoMeta) + protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) if err != nil { return err } @@ -900,12 +930,7 @@ func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Di protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures - repoMetaBlob, err = proto.Marshal(&protoRepoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) + return setProtoRepoMeta(protoRepoMeta, repoMetaBuck) }) return err @@ -913,28 +938,21 @@ func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Di func (bdw *BoltDB) IncrementRepoStars(repo string) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) + repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) - repoMetaBlob := buck.Get([]byte(repo)) + repoMetaBlob := repoMetaBuck.Get([]byte(repo)) if repoMetaBlob == nil { return zerr.ErrRepoMetaNotFound } - var repoMeta proto_go.RepoMeta - - err := proto.Unmarshal(repoMetaBlob, &repoMeta) + protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) if err != nil { return err } - repoMeta.Stars++ + protoRepoMeta.Stars++ - repoMetaBlob, err = proto.Marshal(&repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) + return setProtoRepoMeta(protoRepoMeta, repoMetaBuck) }) return err @@ -942,32 +960,25 @@ func (bdw *BoltDB) IncrementRepoStars(repo string) error { func (bdw *BoltDB) DecrementRepoStars(repo string) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) + repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) - repoMetaBlob := buck.Get([]byte(repo)) + repoMetaBlob := repoMetaBuck.Get([]byte(repo)) if repoMetaBlob == nil { return zerr.ErrRepoMetaNotFound } - var repoMeta proto_go.RepoMeta - - err := proto.Unmarshal(repoMetaBlob, &repoMeta) + protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) if err != nil { return err } - if repoMeta.Stars == 0 { + if protoRepoMeta.Stars == 0 { return nil } - repoMeta.Stars-- + protoRepoMeta.Stars-- - repoMetaBlob, err = proto.Marshal(&repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) + return setProtoRepoMeta(protoRepoMeta, repoMetaBuck) }) return err @@ -996,14 +1007,8 @@ func (bdw *BoltDB) ResetRepoReferences(repo string) error { repoMetaBlob := buck.Get([]byte(repo)) - if repoMetaBlob == nil { - return nil - } - - protoRepoMeta := &proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) - if err != nil { + protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) + if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { return err } @@ -1033,18 +1038,13 @@ func (bdw *BoltDB) GetReferrersInfo(repo string, referredDigest godigest.Digest, buck := tx.Bucket([]byte(RepoMetaBuck)) repoMetaBlob := buck.Get([]byte(repo)) - if len(repoMetaBlob) == 0 { - return zerr.ErrRepoMetaNotFound - } - var repoMeta proto_go.RepoMeta - - err := proto.Unmarshal(repoMetaBlob, &repoMeta) + protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) if err != nil { return err } - referrersInfo := repoMeta.Referrers[referredDigest.String()].List + referrersInfo := protoRepoMeta.Referrers[referredDigest.String()].List for i := range referrersInfo { if !common.MatchesArtifactTypes(referrersInfo[i].ArtifactType, artifactTypes) { @@ -1068,16 +1068,14 @@ func (bdw *BoltDB) GetReferrersInfo(repo string, referredDigest godigest.Digest, func (bdw *BoltDB) UpdateStatsOnDownload(repo string, reference string) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) + repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) - repoMetaBlob := buck.Get([]byte(repo)) + repoMetaBlob := repoMetaBuck.Get([]byte(repo)) if repoMetaBlob == nil { return zerr.ErrRepoMetaNotFound } - var repoMeta proto_go.RepoMeta - - err := proto.Unmarshal(repoMetaBlob, &repoMeta) + protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) if err != nil { return err } @@ -1086,7 +1084,7 @@ func (bdw *BoltDB) UpdateStatsOnDownload(repo string, reference string) error { if !common.ReferenceIsDigest(reference) { // search digest for tag - descriptor, found := repoMeta.Tags[reference] + descriptor, found := protoRepoMeta.Tags[reference] if !found { return zerr.ErrManifestMetaNotFound @@ -1095,21 +1093,16 @@ func (bdw *BoltDB) UpdateStatsOnDownload(repo string, reference string) error { manifestDigest = descriptor.Digest } - manifestStatistics, ok := repoMeta.Statistics[manifestDigest] + manifestStatistics, ok := protoRepoMeta.Statistics[manifestDigest] if !ok { return zerr.ErrManifestMetaNotFound } manifestStatistics.DownloadCount++ manifestStatistics.LastPullTimestamp = timestamppb.Now() - repoMeta.Statistics[manifestDigest] = manifestStatistics + protoRepoMeta.Statistics[manifestDigest] = manifestStatistics - repoMetaBlob, err = proto.Marshal(&repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) + return setProtoRepoMeta(protoRepoMeta, repoMetaBuck) }) return err @@ -1147,9 +1140,7 @@ func (bdw *BoltDB) UpdateSignaturesValidity(repo string, manifestDigest godigest return zerr.ErrRepoMetaNotFound } - protoRepoMeta := proto_go.RepoMeta{} - - err = proto.Unmarshal(repoMetaBlob, &protoRepoMeta) + protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) if err != nil { return err } @@ -1188,12 +1179,7 @@ func (bdw *BoltDB) UpdateSignaturesValidity(repo string, manifestDigest godigest protoRepoMeta.Signatures[manifestDigest.String()] = &manifestSignatures - repoMetaBlob, err = proto.Marshal(&protoRepoMeta) - if err != nil { - return err - } - - return repoBuck.Put([]byte(repo), repoMetaBlob) + return setProtoRepoMeta(protoRepoMeta, repoBuck) }) return err @@ -1201,23 +1187,20 @@ func (bdw *BoltDB) UpdateSignaturesValidity(repo string, manifestDigest godigest func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error { err := bdw.DB.Update(func(tx *bbolt.Tx) error { - buck := tx.Bucket([]byte(RepoMetaBuck)) + repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) imageMetaBuck := tx.Bucket([]byte(ImageMetaBuck)) repoBlobsBuck := tx.Bucket([]byte(RepoBlobsBuck)) - repoMetaBlob := buck.Get([]byte(repo)) - if repoMetaBlob == nil { - return nil - } - - repoMeta := &proto_go.RepoMeta{} - - err := proto.Unmarshal(repoMetaBlob, repoMeta) + protoRepoMeta, err := getProtoRepoMeta(repo, repoMetaBuck) if err != nil { + if errors.Is(err, zerr.ErrRepoMetaNotFound) { + return nil + } + return err } - imageMeta, err := fetchProtoImageMeta(imageMetaBuck, manifestDigest.String()) + protoImageMeta, err := getProtoImageMeta(imageMetaBuck, manifestDigest.String()) if err != nil { if errors.Is(err, zerr.ErrImageMetaNotFound) { return nil @@ -1227,12 +1210,12 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go } // Remove Referrers - if subject := mConvert.GetImageSubject(imageMeta); subject != nil { + if subject := mConvert.GetImageSubject(protoImageMeta); subject != nil { referredDigest := subject.Digest.String() refInfo := &proto_go.ReferrersInfo{} - if repoMeta.Referrers[referredDigest] != nil { - refInfo = repoMeta.Referrers[referredDigest] + if protoRepoMeta.Referrers[referredDigest] != nil { + refInfo = protoRepoMeta.Referrers[referredDigest] } referrers := refInfo.List @@ -1251,16 +1234,16 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go refInfo.List = referrers - repoMeta.Referrers[referredDigest] = refInfo + protoRepoMeta.Referrers[referredDigest] = refInfo } if !common.ReferenceIsDigest(reference) { - delete(repoMeta.Tags, reference) + delete(protoRepoMeta.Tags, reference) } else { // remove all tags pointing to this digest - for tag, desc := range repoMeta.Tags { + for tag, desc := range protoRepoMeta.Tags { if desc.Digest == reference { - delete(repoMeta.Tags, tag) + delete(protoRepoMeta.Tags, tag) } } } @@ -1268,32 +1251,26 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go /* try to find at least one tag pointing to manifestDigest if not found then we can also remove everything related to this digest */ var foundTag bool - for _, desc := range repoMeta.Tags { + for _, desc := range protoRepoMeta.Tags { if desc.Digest == manifestDigest.String() { foundTag = true } } if !foundTag { - delete(repoMeta.Statistics, manifestDigest.String()) - delete(repoMeta.Signatures, manifestDigest.String()) - delete(repoMeta.Referrers, manifestDigest.String()) + delete(protoRepoMeta.Statistics, manifestDigest.String()) + delete(protoRepoMeta.Signatures, manifestDigest.String()) + delete(protoRepoMeta.Referrers, manifestDigest.String()) } - repoBlobsBytes := repoBlobsBuck.Get([]byte(repoMeta.Name)) + repoBlobsBytes := repoBlobsBuck.Get([]byte(protoRepoMeta.Name)) - repoBlobs := &proto_go.RepoBlobs{} - - if len(repoBlobsBytes) == 0 { - repoBlobs.Blobs = make(map[string]*proto_go.BlobInfo) - } else { - err := proto.Unmarshal(repoBlobsBytes, repoBlobs) - if err != nil { - return err - } + repoBlobs, err := unmarshalProtoRepoBlobs(repo, repoBlobsBytes) + if err != nil { + return err } - repoMeta, repoBlobs, err = common.RemoveImageFromRepoMeta(repoMeta, repoBlobs, reference) + protoRepoMeta, repoBlobs, err = common.RemoveImageFromRepoMeta(protoRepoMeta, repoBlobs, reference) if err != nil { return err } @@ -1303,17 +1280,12 @@ func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest go return err } - err = repoBlobsBuck.Put([]byte(repoMeta.Name), repoBlobsBytes) + err = repoBlobsBuck.Put([]byte(protoRepoMeta.Name), repoBlobsBytes) if err != nil { return err } - repoMetaBlob, err = proto.Marshal(repoMeta) - if err != nil { - return err - } - - return buck.Put([]byte(repo), repoMetaBlob) + return setProtoRepoMeta(protoRepoMeta, repoMetaBuck) }) return err @@ -1371,31 +1343,19 @@ func (bdw *BoltDB) ToggleStarRepo(ctx context.Context, repo string) (mTypes.Togg return zerr.ErrRepoMetaNotFound } - repoMeta := &proto_go.RepoMeta{} - - err = proto.Unmarshal(repoMetaBlob, repoMeta) + protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) if err != nil { return err } switch res { case mTypes.Added: - repoMeta.Stars++ + protoRepoMeta.Stars++ case mTypes.Removed: - repoMeta.Stars-- + protoRepoMeta.Stars-- } - repoMetaBlob, err = proto.Marshal(repoMeta) - if err != nil { - return err - } - - err = repoBuck.Put([]byte(repo), repoMetaBlob) - if err != nil { - return err - } - - return nil + return setProtoRepoMeta(protoRepoMeta, repoBuck) }); err != nil { return mTypes.NotChanged, err } diff --git a/pkg/meta/convert/convert.go b/pkg/meta/convert/convert.go index 50cd38df..6ce297fd 100644 --- a/pkg/meta/convert/convert.go +++ b/pkg/meta/convert/convert.go @@ -406,6 +406,10 @@ func GetFullImageMetaFromProto(tag string, protoRepoMeta *proto_go.RepoMeta, pro func GetFullManifestData(protoRepoMeta *proto_go.RepoMeta, manifestData []mTypes.ManifestData, ) []mTypes.FullManifestMeta { + if protoRepoMeta == nil { + return []mTypes.FullManifestMeta{} + } + results := []mTypes.FullManifestMeta{} for i := range manifestData { @@ -421,6 +425,10 @@ func GetFullManifestData(protoRepoMeta *proto_go.RepoMeta, manifestData []mTypes } func GetRepoMeta(protoRepoMeta *proto_go.RepoMeta) mTypes.RepoMeta { + if protoRepoMeta == nil { + return mTypes.RepoMeta{} + } + repoDownloads := int32(0) for _, descriptor := range protoRepoMeta.Tags { diff --git a/pkg/meta/dynamodb/dynamodb.go b/pkg/meta/dynamodb/dynamodb.go index d29d78cf..5d3c2fcb 100644 --- a/pkg/meta/dynamodb/dynamodb.go +++ b/pkg/meta/dynamodb/dynamodb.go @@ -208,25 +208,45 @@ func (dwr *DynamoDB) getProtoRepoMeta(ctx context.Context, repo string) (*proto_ return nil, err } - if resp.Item == nil { - return nil, zerr.ErrRepoMetaNotFound + protoRepoMeta := &proto_go.RepoMeta{ + Name: repo, } blob := []byte{} - err = attributevalue.Unmarshal(resp.Item["RepoMetadata"], &blob) - if err != nil { - return nil, err + if resp.Item != nil { + err = attributevalue.Unmarshal(resp.Item["RepoMetadata"], &blob) + if err != nil { + return nil, err + } + + err = proto.Unmarshal(blob, protoRepoMeta) + if err != nil { + return nil, err + } } - repoMeta := &proto_go.RepoMeta{} - - err = proto.Unmarshal(blob, repoMeta) - if err != nil { - return nil, err + if protoRepoMeta.Tags == nil { + protoRepoMeta.Tags = map[string]*proto_go.TagDescriptor{"": {}} } - return repoMeta, nil + if protoRepoMeta.Statistics == nil { + protoRepoMeta.Statistics = map[string]*proto_go.DescriptorStatistics{"": {}} + } + + if protoRepoMeta.Signatures == nil { + protoRepoMeta.Signatures = map[string]*proto_go.ManifestSignatures{"": {}} + } + + if protoRepoMeta.Referrers == nil { + protoRepoMeta.Referrers = map[string]*proto_go.ReferrersInfo{"": {}} + } + + if len(blob) == 0 || resp.Item == nil { + return protoRepoMeta, zerr.ErrRepoMetaNotFound + } + + return protoRepoMeta, nil } func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, reference string, @@ -252,18 +272,8 @@ func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, referenc } repoMeta, err := dwr.getProtoRepoMeta(ctx, repo) - if err != nil { - if !errors.Is(err, zerr.ErrRepoMetaNotFound) { - return err - } - - repoMeta = &proto_go.RepoMeta{ - Name: repo, - Tags: map[string]*proto_go.TagDescriptor{"": {}}, - Statistics: map[string]*proto_go.DescriptorStatistics{"": {}}, - Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}}, - Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, - } + if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { + return err } // 2. Referrers @@ -328,7 +338,7 @@ func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, referenc } // 4. Blobs - repoBlobs, err := dwr.getRepoBlobsInfo(repo) //nolint: contextcheck + repoBlobs, err := dwr.getProtoRepoBlobs(ctx, repo) if err != nil { return err } @@ -346,8 +356,8 @@ func (dwr *DynamoDB) SetRepoReference(ctx context.Context, repo string, referenc return dwr.setProtoRepoMeta(repo, repoMeta) //nolint: contextcheck } -func (dwr *DynamoDB) getRepoBlobsInfo(repo string) (*proto_go.RepoBlobs, error) { - resp, err := dwr.Client.GetItem(context.Background(), &dynamodb.GetItemInput{ +func (dwr *DynamoDB) getProtoRepoBlobs(ctx context.Context, repo string) (*proto_go.RepoBlobs, error) { + resp, err := dwr.Client.GetItem(ctx, &dynamodb.GetItemInput{ TableName: aws.String(dwr.RepoBlobsTablename), Key: map[string]types.AttributeValue{ "Key": &types.AttributeValueMemberS{Value: repo}, @@ -357,29 +367,28 @@ func (dwr *DynamoDB) getRepoBlobsInfo(repo string) (*proto_go.RepoBlobs, error) return nil, err } - if resp.Item == nil { - return &proto_go.RepoBlobs{Name: repo, Blobs: make(map[string]*proto_go.BlobInfo)}, nil + repoBlobs := &proto_go.RepoBlobs{ + Name: repo, } repoBlobsBytes := []byte{} - err = attributevalue.Unmarshal(resp.Item["RepoBlobsInfo"], &repoBlobsBytes) - if err != nil { - return nil, err - } - - repoBlobs := &proto_go.RepoBlobs{} - if len(repoBlobsBytes) == 0 { - repoBlobs.Blobs = make(map[string]*proto_go.BlobInfo) - } else { - err := proto.Unmarshal(repoBlobsBytes, repoBlobs) + if resp.Item != nil { + err = attributevalue.Unmarshal(resp.Item["RepoBlobsInfo"], &repoBlobsBytes) if err != nil { return nil, err } + + if len(repoBlobsBytes) > 0 { + err := proto.Unmarshal(repoBlobsBytes, repoBlobs) + if err != nil { + return nil, err + } + } } - if len(repoBlobs.Blobs) == 0 { - repoBlobs.Blobs = make(map[string]*proto_go.BlobInfo) + if repoBlobs.Blobs == nil { + repoBlobs.Blobs = map[string]*proto_go.BlobInfo{"": {}} } return repoBlobs, nil @@ -1262,7 +1271,7 @@ func (dwr *DynamoDB) RemoveRepoReference(repo, reference string, manifestDigest delete(protoRepoMeta.Referrers, manifestDigest.String()) } - repoBlobsInfo, err := dwr.getRepoBlobsInfo(repo) + repoBlobsInfo, err := dwr.getProtoRepoBlobs(context.Background(), repo) if err != nil { return err } diff --git a/pkg/meta/meta_test.go b/pkg/meta/meta_test.go index 68b65816..c4e79872 100644 --- a/pkg/meta/meta_test.go +++ b/pkg/meta/meta_test.go @@ -612,6 +612,17 @@ func RunMetaDBTests(t *testing.T, metaDB mTypes.MetaDB, preparationFuncs ...func So(repoMeta2.Size, ShouldEqual, img2Size) }) + Convey("Set, delete and set again", func() { + err := metaDB.SetRepoReference(ctx, repo1, tag1, imgData1) + So(err, ShouldBeNil) + + err = metaDB.RemoveRepoReference(repo1, tag1, imgData1.Digest) + So(err, ShouldBeNil) + + err = metaDB.SetRepoReference(ctx, repo1, tag1, imgData1) + So(err, ShouldBeNil) + }) + Convey("Check repo blobs info for manifest image", func() { image1 := CreateImageWith().RandomLayers(2, 10). ImageConfig(ispec.Image{Platform: ispec.Platform{OS: "os1", Architecture: "arch1"}}).