From 49c3d057062ada77925552a8567fda665ab2286c Mon Sep 17 00:00:00 2001 From: peusebiu Date: Tue, 22 Nov 2022 20:13:08 +0200 Subject: [PATCH] fix(storage): sanitize storage locks (#1003) removed all locks from common code removed locks from GetBlobContent Signed-off-by: Petu Eusebiu --- pkg/extensions/search/common/oci_layout.go | 21 ++++ pkg/extensions/sync/sync_internal_test.go | 7 +- pkg/extensions/sync/utils.go | 6 ++ pkg/storage/common.go | 31 +----- pkg/storage/local/local.go | 75 ++++++------- pkg/storage/s3/s3.go | 117 ++++++++++++++------- pkg/storage/s3/s3_test.go | 51 +++++++++ pkg/storage/storage_test.go | 13 +++ 8 files changed, 220 insertions(+), 101 deletions(-) diff --git a/pkg/extensions/search/common/oci_layout.go b/pkg/extensions/search/common/oci_layout.go index f15e341a..2ebc2bfa 100644 --- a/pkg/extensions/search/common/oci_layout.go +++ b/pkg/extensions/search/common/oci_layout.go @@ -7,6 +7,7 @@ import ( "fmt" "path" "strconv" + "time" notreg "github.com/notaryproject/notation-go/registry" godigest "github.com/opencontainers/go-digest" @@ -89,8 +90,13 @@ func (olu BaseOciLayoutUtils) GetRepositories() ([]string, error) { // Below method will return image path including root dir, root dir is determined by splitting. func (olu BaseOciLayoutUtils) GetImageManifests(repo string) ([]ispec.Descriptor, error) { + var lockLatency time.Time + imageStore := olu.StoreController.GetImageStore(repo) + imageStore.RLock(&lockLatency) + defer imageStore.RUnlock(&lockLatency) + buf, err := imageStore.GetIndexContent(repo) if err != nil { if goerrors.Is(errors.ErrRepoNotFound, err) { @@ -118,8 +124,13 @@ func (olu BaseOciLayoutUtils) GetImageManifests(repo string) ([]ispec.Descriptor func (olu BaseOciLayoutUtils) GetImageBlobManifest(repo string, digest godigest.Digest) (ispec.Manifest, error) { var blobIndex ispec.Manifest + var lockLatency time.Time + imageStore := olu.StoreController.GetImageStore(repo) + imageStore.RLock(&lockLatency) + defer imageStore.RUnlock(&lockLatency) + blobBuf, err := imageStore.GetBlobContent(repo, digest) if err != nil { olu.Log.Error().Err(err).Msg("unable to open image metadata file") @@ -139,8 +150,13 @@ func (olu BaseOciLayoutUtils) GetImageBlobManifest(repo string, digest godigest. func (olu BaseOciLayoutUtils) GetImageInfo(repo string, digest godigest.Digest) (ispec.Image, error) { var imageInfo ispec.Image + var lockLatency time.Time + imageStore := olu.StoreController.GetImageStore(repo) + imageStore.RLock(&lockLatency) + defer imageStore.RUnlock(&lockLatency) + blobBuf, err := imageStore.GetBlobContent(repo, digest) if err != nil { olu.Log.Error().Err(err).Msg("unable to open image layers file") @@ -265,6 +281,11 @@ func (olu BaseOciLayoutUtils) GetImageConfigInfo(repo string, manifestDigest god func (olu BaseOciLayoutUtils) GetImageManifestSize(repo string, manifestDigest godigest.Digest) int64 { imageStore := olu.StoreController.GetImageStore(repo) + var lockLatency time.Time + + imageStore.RLock(&lockLatency) + defer imageStore.RUnlock(&lockLatency) + manifestBlob, err := imageStore.GetBlobContent(repo, manifestDigest) if err != nil { olu.Log.Error().Err(err).Msg("error when getting manifest blob content") diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index 4ae54626..d44e7b05 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -322,7 +322,12 @@ func TestSyncInternal(t *testing.T) { Layers: []ispec.Descriptor{desc}, } - sig := newSignaturesCopier(client, *regURL, storage.StoreController{DefaultStore: &local.ImageStoreLocal{}}, log) + metrics := monitoring.NewMetricsServer(false, log) + imageStore := local.NewImageStore(t.TempDir(), false, storage.DefaultGCDelay, + false, false, log, metrics, nil, nil, + ) + + sig := newSignaturesCopier(client, *regURL, storage.StoreController{DefaultStore: imageStore}, log) err = sig.syncCosignSignature(testImage, testImage, testImageTag, &ispec.Manifest{}) So(err, ShouldNotBeNil) diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go index 9e02e0cb..bf637795 100644 --- a/pkg/extensions/sync/utils.go +++ b/pkg/extensions/sync/utils.go @@ -12,6 +12,7 @@ import ( "path" "regexp" "strings" + "time" "github.com/Masterminds/semver" glob "github.com/bmatcuk/doublestar/v4" @@ -333,6 +334,8 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string, ) error { log.Info().Msgf("pushing synced local image %s/%s:%s to local registry", localCachePath, localRepo, reference) + var lockLatency time.Time + metrics := monitoring.NewMetricsServer(false, log) cacheImageStore := local.NewImageStore(localCachePath, false, @@ -373,7 +376,10 @@ func pushSyncedLocalImage(localRepo, reference, localCachePath string, } for _, manifest := range indexManifest.Manifests { + cacheImageStore.RLock(&lockLatency) manifestBuf, err := cacheImageStore.GetBlobContent(localRepo, manifest.Digest) + cacheImageStore.RUnlock(&lockLatency) + if err != nil { log.Error().Str("errorType", TypeOf(err)). Err(err).Str("dir", path.Join(cacheImageStore.RootDir(), localRepo)).Str("digest", manifest.Digest.String()). diff --git a/pkg/storage/common.go b/pkg/storage/common.go index df654596..1e8487bb 100644 --- a/pkg/storage/common.go +++ b/pkg/storage/common.go @@ -6,7 +6,6 @@ import ( "os" "path" "strings" - "time" "github.com/docker/distribution/registry/storage/driver" "github.com/notaryproject/notation-go" @@ -134,28 +133,24 @@ func validateOCIManifest(imgStore ImageStore, repo, reference string, manifest * // validate image config config := manifest.Config - blobFile, _, err := imgStore.GetBlob(repo, config.Digest, "") + blobBuf, err := imgStore.GetBlobContent(repo, config.Digest) if err != nil { return config.Digest, zerr.ErrBlobNotFound } - defer blobFile.Close() - - dec := json.NewDecoder(blobFile) - var cspec ispec.Image - if err := dec.Decode(&cspec); err != nil { + + err = json.Unmarshal(blobBuf, &cspec) + if err != nil { return "", zerr.ErrBadManifest } // validate the layers for _, l := range manifest.Layers { - blobFile, _, err := imgStore.GetBlob(repo, l.Digest, "") + _, err := imgStore.GetBlobContent(repo, l.Digest) if err != nil { return l.Digest, zerr.ErrBlobNotFound } - - defer blobFile.Close() } return "", nil @@ -461,8 +456,6 @@ func ApplyLinter(imgStore ImageStore, linter Lint, repo string, manifestDesc isp func GetOrasReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, artifactType string, log zerolog.Logger, ) ([]oras.Descriptor, error) { - var lockLatency time.Time - if err := gdigest.Validate(); err != nil { return nil, err } @@ -477,9 +470,6 @@ func GetOrasReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, return nil, err } - imgStore.RLock(&lockLatency) - defer imgStore.RUnlock(&lockLatency) - found := false result := []oras.Descriptor{} @@ -489,10 +479,7 @@ func GetOrasReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, continue } - imgStore.RUnlock(&lockLatency) buf, err := imgStore.GetBlobContent(repo, manifest.Digest) - imgStore.RLock(&lockLatency) - if err != nil { log.Error().Err(err).Str("blob", imgStore.BlobPath(repo, manifest.Digest)).Msg("failed to read manifest") @@ -540,8 +527,6 @@ func GetOrasReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, func GetReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, artifactType string, log zerolog.Logger, ) (ispec.Index, error) { - var lockLatency time.Time - nilIndex := ispec.Index{} if err := gdigest.Validate(); err != nil { @@ -558,9 +543,6 @@ func GetReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, art return nilIndex, err } - imgStore.RLock(&lockLatency) - defer imgStore.RUnlock(&lockLatency) - found := false result := []ispec.Descriptor{} @@ -570,10 +552,7 @@ func GetReferrers(imgStore ImageStore, repo string, gdigest godigest.Digest, art continue } - imgStore.RUnlock(&lockLatency) buf, err := imgStore.GetBlobContent(repo, manifest.Digest) - imgStore.RLock(&lockLatency) - if err != nil { log.Error().Err(err).Str("blob", imgStore.BlobPath(repo, manifest.Digest)).Msg("failed to read manifest") diff --git a/pkg/storage/local/local.go b/pkg/storage/local/local.go index 97e25d42..0e04b18e 100644 --- a/pkg/storage/local/local.go +++ b/pkg/storage/local/local.go @@ -1,7 +1,6 @@ package local import ( - "bytes" "context" "encoding/json" "errors" @@ -378,11 +377,16 @@ func (is *ImageStoreLocal) GetNextRepository(repo string) (string, error) { // GetImageTags returns a list of image tags available in the specified repository. func (is *ImageStoreLocal) GetImageTags(repo string) ([]string, error) { + var lockLatency time.Time + dir := path.Join(is.rootDir, repo) if !is.DirExists(dir) { return nil, zerr.ErrRepoNotFound } + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + index, err := storage.GetIndex(is, repo, is.log) if err != nil { return nil, err @@ -393,11 +397,16 @@ func (is *ImageStoreLocal) GetImageTags(repo string) ([]string, error) { // GetImageManifest returns the image manifest of an image in the specific repository. func (is *ImageStoreLocal) GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) { + var lockLatency time.Time + dir := path.Join(is.rootDir, repo) if !is.DirExists(dir) { return nil, "", "", zerr.ErrRepoNotFound } + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + index, err := storage.GetIndex(is, repo, is.log) if err != nil { return nil, "", "", err @@ -439,6 +448,11 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, / return "", err } + var lockLatency time.Time + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + digest, err := storage.ValidateManifest(is, repo, reference, mediaType, body, is.log) if err != nil { return digest, err @@ -478,11 +492,6 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, / return desc.Digest, nil } - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - // write manifest to "blobs" dir := path.Join(is.rootDir, repo, "blobs", mDigest.Algorithm().String()) _ = ensureDir(dir, is.log) @@ -495,15 +504,11 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, / return "", err } - is.Unlock(&lockLatency) - err = storage.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) if err != nil { return "", err } - is.Lock(&lockLatency) - // now update "index.json" index.Manifests = append(index.Manifests, desc) dir = path.Join(is.rootDir, repo) @@ -516,13 +521,8 @@ func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, / return "", err } - is.Unlock(&lockLatency) - // apply linter only on images, not signatures pass, err := storage.ApplyLinter(is, is.linter, repo, desc) - - is.Lock(&lockLatency) - if !pass { is.log.Error().Err(err).Str("repo", repo).Str("reference", reference).Msg("linter didn't pass") @@ -557,6 +557,9 @@ func (is *ImageStoreLocal) DeleteImageManifest(repo, reference string, detectCol return zerr.ErrRepoNotFound } + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + index, err := storage.GetIndex(is, repo, is.log) if err != nil { return err @@ -576,9 +579,6 @@ func (is *ImageStoreLocal) DeleteImageManifest(repo, reference string, detectCol return err } - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - // now update "index.json" dir = path.Join(is.rootDir, repo) file := path.Join(dir, "index.json") @@ -1258,37 +1258,30 @@ func (is *ImageStoreLocal) GetBlob(repo string, digest godigest.Digest, mediaTyp return blobReadCloser, binfo.Size(), nil } +// GetBlobContent returns blob contents, SHOULD lock from outside. func (is *ImageStoreLocal) GetBlobContent(repo string, digest godigest.Digest) ([]byte, error) { if err := digest.Validate(); err != nil { return []byte{}, err } - blob, _, err := is.GetBlob(repo, digest, ispec.MediaTypeImageManifest) - if err != nil { - return []byte{}, err - } - defer blob.Close() + blobPath := is.BlobPath(repo, digest) - buf := new(bytes.Buffer) + blob, err := os.ReadFile(blobPath) + if os.IsNotExist(err) { + is.log.Error().Err(err).Str("blob", blobPath).Msg("blob doesn't exist") - _, err = buf.ReadFrom(blob) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("failed to read blob") - - return []byte{}, err + return []byte{}, zerr.ErrBlobNotFound } - return buf.Bytes(), nil + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to read blob") + + return blob, nil } +// GetIndexContent returns index.json contents, SHOULD lock from outside. func (is *ImageStoreLocal) GetIndexContent(repo string) ([]byte, error) { - var lockLatency time.Time - dir := path.Join(is.rootDir, repo) - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - buf, err := os.ReadFile(path.Join(dir, "index.json")) if err != nil { if os.IsNotExist(err) { @@ -1345,11 +1338,21 @@ func (is *ImageStoreLocal) DeleteBlob(repo string, digest godigest.Digest) error func (is *ImageStoreLocal) GetReferrers(repo string, gdigest godigest.Digest, artifactType string, ) (ispec.Index, error) { + var lockLatency time.Time + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + return storage.GetReferrers(is, repo, gdigest, artifactType, is.log) } func (is *ImageStoreLocal) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string, ) ([]oras.Descriptor, error) { + var lockLatency time.Time + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + return storage.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) } @@ -1483,9 +1486,7 @@ func (is *ImageStoreLocal) gcRepo(repo string) error { var lockLatency time.Time is.Lock(&lockLatency) - err := is.garbageCollect(dir, repo) - is.Unlock(&lockLatency) if err != nil { diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index 8fe6c0c7..0d7037a9 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -284,11 +284,16 @@ func (is *ObjectStorage) GetNextRepository(repo string) (string, error) { // GetImageTags returns a list of image tags available in the specified repository. func (is *ObjectStorage) GetImageTags(repo string) ([]string, error) { + var lockLatency time.Time + dir := path.Join(is.rootDir, repo) if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() { return nil, zerr.ErrRepoNotFound } + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + index, err := storage.GetIndex(is, repo, is.log) if err != nil { return nil, err @@ -299,11 +304,16 @@ func (is *ObjectStorage) GetImageTags(repo string) ([]string, error) { // GetImageManifest returns the image manifest of an image in the specific repository. func (is *ObjectStorage) GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) { + var lockLatency time.Time + dir := path.Join(is.rootDir, repo) if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() { return nil, "", "", zerr.ErrRepoNotFound } + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + index, err := storage.GetIndex(is, repo, is.log) if err != nil { return nil, "", "", zerr.ErrRepoNotFound @@ -345,6 +355,11 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n return "", err } + var lockLatency time.Time + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + dig, err := storage.ValidateManifest(is, repo, reference, mediaType, body, is.log) if err != nil { return dig, err @@ -384,11 +399,6 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n return desc.Digest, nil } - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - // write manifest to "blobs" dir := path.Join(is.rootDir, repo, "blobs", mDigest.Algorithm().String()) manifestPath := path.Join(dir, mDigest.Encoded()) @@ -399,10 +409,7 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n return "", err } - is.Unlock(&lockLatency) - err = storage.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) - is.Lock(&lockLatency) if err != nil { return "", err } @@ -419,10 +426,8 @@ func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //n return "", err } - is.Unlock(&lockLatency) // apply linter only on images, not signatures pass, err := storage.ApplyLinter(is, is.linter, repo, desc) - is.Lock(&lockLatency) if !pass { is.log.Error().Err(err).Str("repo", repo).Str("reference", reference).Msg("linter didn't pass") @@ -450,6 +455,9 @@ func (is *ObjectStorage) DeleteImageManifest(repo, reference string, detectColli return zerr.ErrRepoNotFound } + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + index, err := storage.GetIndex(is, repo, is.log) if err != nil { return err @@ -469,9 +477,6 @@ func (is *ObjectStorage) DeleteImageManifest(repo, reference string, detectColli return err } - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - // now update "index.json" dir = path.Join(is.rootDir, repo) file := path.Join(dir, "index.json") @@ -686,8 +691,6 @@ func (is *ObjectStorage) FinishBlobUpload(repo, uuid string, body io.Reader, dst return zerr.ErrBadBlobDigest } - defer fileWriter.Close() - if err := fileWriter.Commit(); err != nil { is.log.Error().Err(err).Msg("failed to commit file") @@ -696,6 +699,8 @@ func (is *ObjectStorage) FinishBlobUpload(repo, uuid string, body io.Reader, dst if err := fileWriter.Close(); err != nil { is.log.Error().Err(err).Msg("failed to close file") + + return err } fileReader, err := is.store.Reader(context.Background(), src, 0) @@ -705,6 +710,8 @@ func (is *ObjectStorage) FinishBlobUpload(repo, uuid string, body io.Reader, dst return zerr.ErrUploadNotFound } + defer fileReader.Close() + srcDigest, err := godigest.FromReader(fileReader) if err != nil { is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") @@ -719,8 +726,6 @@ func (is *ObjectStorage) FinishBlobUpload(repo, uuid string, body io.Reader, dst return zerr.ErrBadBlobDigest } - fileReader.Close() - dst := is.BlobPath(repo, dstDigest) var lockLatency time.Time @@ -1209,48 +1214,86 @@ func (is *ObjectStorage) GetBlob(repo string, digest godigest.Digest, mediaType return blobReadCloser, binfo.Size(), nil } +// GetBlobContent returns blob contents, SHOULD lock from outside. func (is *ObjectStorage) GetBlobContent(repo string, digest godigest.Digest) ([]byte, error) { - blob, _, err := is.GetBlob(repo, digest, ispec.MediaTypeImageManifest) - if err != nil { - return []byte{}, err - } - defer blob.Close() - - buf := new(bytes.Buffer) - - _, err = buf.ReadFrom(blob) - if err != nil { - is.log.Error().Err(err).Msg("failed to read blob") - + if err := digest.Validate(); err != nil { return []byte{}, err } - return buf.Bytes(), nil + blobPath := is.BlobPath(repo, digest) + + binfo, err := is.store.Stat(context.Background(), blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return []byte{}, zerr.ErrBlobNotFound + } + + blobBuf, err := is.store.GetContent(context.Background(), blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") + + return nil, err + } + + // is a 'deduped' blob? + if binfo.Size() == 0 && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { + // Check blobs in cache + dstRecord, err := is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return nil, zerr.ErrBlobNotFound + } + + blobBuf, err := is.store.GetContent(context.Background(), dstRecord) + if err != nil { + is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") + + return nil, err + } + + return blobBuf, nil + } + + return blobBuf, nil } func (is *ObjectStorage) GetReferrers(repo string, gdigest godigest.Digest, artifactType string, ) (ispec.Index, error) { + var lockLatency time.Time + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + return storage.GetReferrers(is, repo, gdigest, artifactType, is.log) } func (is *ObjectStorage) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string, ) ([]artifactspec.Descriptor, error) { - return storage.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) -} - -func (is *ObjectStorage) GetIndexContent(repo string) ([]byte, error) { var lockLatency time.Time - dir := path.Join(is.rootDir, repo) - is.RLock(&lockLatency) defer is.RUnlock(&lockLatency) + return storage.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) +} + +// GetIndexContent returns index.json contents, SHOULD lock from outside. +func (is *ObjectStorage) GetIndexContent(repo string) ([]byte, error) { + dir := path.Join(is.rootDir, repo) + buf, err := is.store.GetContent(context.Background(), path.Join(dir, "index.json")) if err != nil { + if errors.Is(err, driver.PathNotFoundError{}) { + is.log.Error().Err(err).Str("dir", dir).Msg("index.json doesn't exist") + + return []byte{}, zerr.ErrRepoNotFound + } + is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") - return []byte{}, zerr.ErrRepoNotFound + return []byte{}, err } return buf, nil diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index aef2a83e..e3f106b8 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -891,6 +891,16 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("Test GetIndexContent", func(c C) { + imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + GetContentFn: func(ctx context.Context, path string) ([]byte, error) { + return []byte{}, driver.PathNotFoundError{} + }, + }) + _, err := imgStore.GetIndexContent(testImage) + So(err, ShouldNotBeNil) + }) + Convey("Test DeleteImageManifest2", func(c C) { imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{}) err := imgStore.DeleteImageManifest(testImage, "1.0", false) @@ -1092,6 +1102,18 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { So(err, ShouldNotBeNil) }) + Convey("Test GetBlobContent", func(c C) { + imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ + GetContentFn: func(ctx context.Context, path string) ([]byte, error) { + return []byte{}, errS3 + }, + }) + + d := godigest.FromBytes([]byte("")) + _, err := imgStore.GetBlobContent(testImage, d) + So(err, ShouldNotBeNil) + }) + Convey("Test DeleteBlob", func(c C) { imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{ DeleteFn: func(ctx context.Context, path string) error { @@ -1233,6 +1255,14 @@ func TestS3Dedupe(t *testing.T) { err = blobReadCloser.Close() So(err, ShouldBeNil) + blobContent, err := imgStore.GetBlobContent("dedupe2", digest) + So(err, ShouldBeNil) + So(len(blobContent), ShouldBeGreaterThan, 0) + So(checkBlobSize1, ShouldEqual, len(blobContent)) + So(getBlobSize1, ShouldEqual, len(blobContent)) + err = blobReadCloser.Close() + So(err, ShouldBeNil) + cblob, cdigest = test.GetRandomImageConfig() _, clen, err = imgStore.FullBlobUpload("dedupe2", bytes.NewReader(cblob), cdigest) So(err, ShouldBeNil) @@ -1359,6 +1389,10 @@ func TestS3Dedupe(t *testing.T) { So(err, ShouldBeNil) So(getBlobSize1, ShouldEqual, getBlobSize3) + blobContent, err := imgStore.GetBlobContent("dedupe3", digest) + So(err, ShouldBeNil) + So(getBlobSize1, ShouldEqual, len(blobContent)) + _, checkBlobSize3, err := imgStore.CheckBlob("dedupe3", digest) So(err, ShouldBeNil) So(checkBlobSize3, ShouldBeGreaterThan, 0) @@ -2261,6 +2295,9 @@ func TestS3DedupeErr(t *testing.T) { _, _, err = imgStore.GetBlob("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip") So(err, ShouldNotBeNil) + _, err = imgStore.GetBlobContent("repo2", digest) + So(err, ShouldNotBeNil) + _, _, _, err = imgStore.GetBlobPartial("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip", 0, 1) So(err, ShouldNotBeNil) }) @@ -2303,11 +2340,22 @@ func TestS3DedupeErr(t *testing.T) { return io.NopCloser(strings.NewReader("")), nil }, + + GetContentFn: func(ctx context.Context, path string) ([]byte, error) { + if strings.Contains(path, "repo1/dst1") { + return []byte{}, errS3 + } + + return []byte{}, nil + }, }) _, _, err = imgStore.GetBlob("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip") So(err, ShouldNotBeNil) + _, err = imgStore.GetBlobContent("repo2", digest) + So(err, ShouldNotBeNil) + _, _, _, err = imgStore.GetBlobPartial("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip", 0, 1) So(err, ShouldNotBeNil) }) @@ -2331,6 +2379,9 @@ func TestS3DedupeErr(t *testing.T) { _, _, err = imgStore.GetBlob("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip") So(err, ShouldNotBeNil) + _, err = imgStore.GetBlobContent("repo2", digest) + So(err, ShouldNotBeNil) + _, _, _, err = imgStore.GetBlobPartial("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip", 0, 1) So(err, ShouldNotBeNil) }) diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index ab6015c8..ad983929 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "path" "strings" @@ -257,6 +258,18 @@ func TestStorageAPIs(t *testing.T) { blob, _, err := imgStore.GetBlob("test", digest, "application/vnd.oci.image.layer.v1.tar+gzip") So(err, ShouldBeNil) + + blobBuf := new(strings.Builder) + n, err := io.Copy(blobBuf, blob) + // check errors + So(n, ShouldEqual, buflen) + So(err, ShouldBeNil) + So(blobBuf.String(), ShouldEqual, buf.String()) + + blobContent, err := imgStore.GetBlobContent("test", digest) + So(err, ShouldBeNil) + So(blobContent, ShouldResemble, content) + err = blob.Close() So(err, ShouldBeNil)