Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2025-03-11 02:17:43 -05:00
Code Issues Activity

Added storage interface

Browse source
This commit is contained in:
Petu Eusebiu 2021-09-30 16:27:13 +03:00 • committed by Ramkumar Chinchani
parent 20f4051446
commit 7d077eaf5a
12 changed files with 1754 additions and 1389 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
pkg/api/controller.go
View file

@ -115,7 +115,7 @@ func (c *Controller) Run() error {
if len(c.Config.Storage.SubPaths) > 0 {
subPaths := c.Config.Storage.SubPaths
subImageStore := make(map[string]*storage.ImageStore)
subImageStore := make(map[string]storage.ImageStore)
// creating image store per subpaths
for route, storageConfig := range subPaths {

2
pkg/api/routes.go
View file

@ -1237,6 +1237,6 @@ func WriteDataFromReader(w http.ResponseWriter, status int, length int64, mediaT
}
// will return image storage corresponding to subpath provided in config.
func (rh *RouteHandler) getImageStore(name string) *storage.ImageStore {
func (rh *RouteHandler) getImageStore(name string) storage.ImageStore {
return rh.c.StoreController.GetImageStore(name)
}

30
pkg/extensions/search/common/common_test.go
View file

@ -157,48 +157,52 @@ func TestImageFormat(t *testing.T) {
Convey("Test valid image", t, func() {
log := log.NewLogger("debug", "")
dbDir := "../../../../test/data"
olu := common.NewOciLayoutUtils(log)
isValidImage, err := olu.IsValidImageFormat(path.Join(dbDir, "zot-test"))
defaultStore := storage.NewImageStore(dbDir, false, false, log)
storeController := storage.StoreController{DefaultStore: defaultStore}
olu := common.NewOciLayoutUtils(storeController, log)
isValidImage, err := olu.IsValidImageFormat("zot-test")
So(err, ShouldBeNil)
So(isValidImage, ShouldEqual, true)
isValidImage, err = olu.IsValidImageFormat(path.Join(dbDir, "zot-test:0.0.1"))
isValidImage, err = olu.IsValidImageFormat("zot-test:0.0.1")
So(err, ShouldBeNil)
So(isValidImage, ShouldEqual, true)
isValidImage, err = olu.IsValidImageFormat(path.Join(dbDir, "zot-test:0.0."))
isValidImage, err = olu.IsValidImageFormat("zot-test:0.0.")
So(err, ShouldBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = olu.IsValidImageFormat(path.Join(dbDir, "zot-noindex-test"))
isValidImage, err = olu.IsValidImageFormat("zot-noindex-test")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = olu.IsValidImageFormat(path.Join(dbDir, "zot--tet"))
isValidImage, err = olu.IsValidImageFormat("zot--tet")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = olu.IsValidImageFormat(path.Join(dbDir, "zot-noindex-test"))
isValidImage, err = olu.IsValidImageFormat("zot-noindex-test")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = olu.IsValidImageFormat(path.Join(dbDir, "zot-squashfs-noblobs"))
isValidImage, err = olu.IsValidImageFormat("zot-squashfs-noblobs")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = olu.IsValidImageFormat(path.Join(dbDir, "zot-squashfs-invalid-index"))
isValidImage, err = olu.IsValidImageFormat("zot-squashfs-invalid-index")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = olu.IsValidImageFormat(path.Join(dbDir, "zot-squashfs-invalid-blob"))
isValidImage, err = olu.IsValidImageFormat("zot-squashfs-invalid-blob")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = olu.IsValidImageFormat(path.Join(dbDir, "zot-squashfs-test:0.3.22-squashfs"))
isValidImage, err = olu.IsValidImageFormat("zot-squashfs-test:0.3.22-squashfs")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
isValidImage, err = olu.IsValidImageFormat(path.Join(dbDir, "zot-nonreadable-test"))
isValidImage, err = olu.IsValidImageFormat("zot-nonreadable-test")
So(err, ShouldNotBeNil)
So(isValidImage, ShouldEqual, false)
})
@ -443,7 +447,7 @@ func TestUtilsMethod(t *testing.T) {
subStore := storage.NewImageStore(subRootDir, false, false, log)
subStoreMap := make(map[string]*storage.ImageStore)
subStoreMap := make(map[string]storage.ImageStore)
subStoreMap["/b"] = subStore

52
pkg/extensions/search/common/oci_layout.go
View file

@ -3,14 +3,15 @@ package common
import (
"encoding/json"
"io/ioutil"
"os"
"path"
"strings"
"time"
goerrors "errors"
"github.com/anuvu/zot/errors"
"github.com/anuvu/zot/pkg/log"
"github.com/anuvu/zot/pkg/storage"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/google/go-containerregistry/pkg/v1/types"
godigest "github.com/opencontainers/go-digest"
@ -19,21 +20,22 @@ import (
// OciLayoutInfo ...
type OciLayoutUtils struct {
Log log.Logger
Log log.Logger
StoreController storage.StoreController
}
// NewOciLayoutUtils initializes a new OciLayoutUtils object.
func NewOciLayoutUtils(log log.Logger) *OciLayoutUtils {
return &OciLayoutUtils{Log: log}
func NewOciLayoutUtils(storeController storage.StoreController, log log.Logger) *OciLayoutUtils {
return &OciLayoutUtils{Log: log, StoreController: storeController}
}
// Below method will return image path including root dir, root dir is determined by splitting.
func (olu OciLayoutUtils) GetImageManifests(imagePath string) ([]ispec.Descriptor, error) {
buf, err := ioutil.ReadFile(path.Join(imagePath, "index.json"))
func (olu OciLayoutUtils) GetImageManifests(image string) ([]ispec.Descriptor, error) {
imageStore := olu.StoreController.GetImageStore(image)
buf, err := imageStore.GetIndexContent(image)
if err != nil {
if os.IsNotExist(err) {
if goerrors.Is(errors.ErrRepoNotFound, err) {
olu.Log.Error().Err(err).Msg("index.json doesn't exist")
return nil, errors.ErrRepoNotFound
@ -47,17 +49,20 @@ func (olu OciLayoutUtils) GetImageManifests(imagePath string) ([]ispec.Descripto
var index ispec.Index
if err := json.Unmarshal(buf, &index); err != nil {
olu.Log.Error().Err(err).Str("dir", imagePath).Msg("invalid JSON")
olu.Log.Error().Err(err).Str("dir", path.Join(imageStore.RootDir(), image)).Msg("invalid JSON")
return nil, errors.ErrRepoNotFound
}
return index.Manifests, nil
}
//nolint: interfacer
func (olu OciLayoutUtils) GetImageBlobManifest(imageDir string, digest godigest.Digest) (v1.Manifest, error) {
var blobIndex v1.Manifest
blobBuf, err := ioutil.ReadFile(path.Join(imageDir, "blobs", digest.Algorithm().String(), digest.Encoded()))
imageStore := olu.StoreController.GetImageStore(imageDir)
blobBuf, err := imageStore.GetBlobContent(imageDir, digest.String())
if err != nil {
olu.Log.Error().Err(err).Msg("unable to open image metadata file")
@ -73,10 +78,13 @@ func (olu OciLayoutUtils) GetImageBlobManifest(imageDir string, digest godigest.
return blobIndex, nil
}
//nolint: interfacer
func (olu OciLayoutUtils) GetImageInfo(imageDir string, hash v1.Hash) (ispec.Image, error) {
var imageInfo ispec.Image
blobBuf, err := ioutil.ReadFile(path.Join(imageDir, "blobs", hash.Algorithm, hash.Hex))
imageStore := olu.StoreController.GetImageStore(imageDir)
blobBuf, err := imageStore.GetBlobContent(imageDir, hash.String())
if err != nil {
olu.Log.Error().Err(err).Msg("unable to open image layers file")
@ -92,17 +100,10 @@ func (olu OciLayoutUtils) GetImageInfo(imageDir string, hash v1.Hash) (ispec.Ima
return imageInfo, err
}
func (olu OciLayoutUtils) IsValidImageFormat(imagePath string) (bool, error) {
imageDir, inputTag := GetImageDirAndTag(imagePath)
if !DirExists(imageDir) {
olu.Log.Error().Msg("image directory doesn't exist")
return false, errors.ErrRepoNotFound
}
func (olu OciLayoutUtils) IsValidImageFormat(image string) (bool, error) {
imageDir, inputTag := GetImageDirAndTag(image)
manifests, err := olu.GetImageManifests(imageDir)
if err != nil {
return false, err
}
@ -181,15 +182,6 @@ func (olu OciLayoutUtils) GetImageTagsWithTimestamp(repo string) ([]TagInfo, err
return tagsInfo, nil
}
func DirExists(d string) bool {
fi, err := os.Stat(d)
if err != nil && os.IsNotExist(err) {
return false
}
return fi.IsDir()
}
func GetImageDirAndTag(imageName string) (string, string) {
var imageDir string

6
pkg/extensions/search/cve/cve.go
View file

@ -40,7 +40,7 @@ func ScanImage(config *config.Config) (report.Results, error) {
func GetCVEInfo(storeController storage.StoreController, log log.Logger) (*CveInfo, error) {
cveController := CveTrivyController{}
layoutUtils := common.NewOciLayoutUtils(log)
layoutUtils := common.NewOciLayoutUtils(storeController, log)
subCveConfig := make(map[string]*config.Config)
@ -118,7 +118,7 @@ func (cveinfo CveInfo) GetTrivyConfig(image string) *config.Config {
return trivyConfig
}
func (cveinfo CveInfo) GetImageListForCVE(repo string, id string, imgStore *storage.ImageStore,
func (cveinfo CveInfo) GetImageListForCVE(repo string, id string, imgStore storage.ImageStore,
trivyConfig *config.Config) ([]*string, error) {
tags := make([]*string, 0)
@ -134,7 +134,7 @@ func (cveinfo CveInfo) GetImageListForCVE(repo string, id string, imgStore *stor
for _, tag := range tagList {
trivyConfig.TrivyConfig.Input = fmt.Sprintf("%s:%s", path.Join(rootDir, repo), tag)
isValidImage, _ := cveinfo.LayoutUtils.IsValidImageFormat(trivyConfig.TrivyConfig.Input)
isValidImage, _ := cveinfo.LayoutUtils.IsValidImageFormat(fmt.Sprintf("%s:%s", repo, tag))
if !isValidImage {
cveinfo.Log.Debug().Str("image", repo+":"+tag).Msg("image media type not supported for scanning")

4
pkg/extensions/search/cve/cve_test.go
View file

@ -96,7 +96,7 @@ func testSetup() error {
storeController := storage.StoreController{DefaultStore: storage.NewImageStore(dir, false, false, log)}
layoutUtils := common.NewOciLayoutUtils(log)
layoutUtils := common.NewOciLayoutUtils(storeController, log)
cve = &cveinfo.CveInfo{Log: log, StoreController: storeController, LayoutUtils: layoutUtils}
@ -421,7 +421,7 @@ func TestMultipleStoragePath(t *testing.T) {
storeController.DefaultStore = firstStore
subStore := make(map[string]*storage.ImageStore)
subStore := make(map[string]storage.ImageStore)
subStore["/a"] = secondStore
subStore["/b"] = thirdStore

10
pkg/extensions/search/digest/digest.go
View file

@ -3,9 +3,9 @@ package digestinfo
import (
"strings"
"github.com/anuvu/zot/errors"
"github.com/anuvu/zot/pkg/extensions/search/common"
"github.com/anuvu/zot/pkg/log"
"github.com/anuvu/zot/pkg/storage"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
)
@ -16,8 +16,8 @@ type DigestInfo struct {
}
// NewDigestInfo initializes a new DigestInfo object.
func NewDigestInfo(log log.Logger) *DigestInfo {
layoutUtils := common.NewOciLayoutUtils(log)
func NewDigestInfo(storeController storage.StoreController, log log.Logger) *DigestInfo {
layoutUtils := common.NewOciLayoutUtils(storeController, log)
return &DigestInfo{Log: log, LayoutUtils: layoutUtils}
}
@ -26,10 +26,6 @@ func NewDigestInfo(log log.Logger) *DigestInfo {
func (digestinfo DigestInfo) GetImageTagsByDigest(repo string, digest string) ([]*string, error) {
uniqueTags := []*string{}
if !common.DirExists(repo) {
return nil, errors.ErrRepoNotFound
}
manifests, err := digestinfo.LayoutUtils.GetImageManifests(repo)
if err != nil {

102
pkg/extensions/search/digest/digest_test.go
View file

@ -15,6 +15,7 @@ import (
ext "github.com/anuvu/zot/pkg/extensions"
digestinfo "github.com/anuvu/zot/pkg/extensions/search/digest"
"github.com/anuvu/zot/pkg/log"
"github.com/anuvu/zot/pkg/storage"
. "github.com/smartystreets/goconvey/convey"
"gopkg.in/resty.v1"
)
@ -23,6 +24,7 @@ import (
var (
digestInfo *digestinfo.DigestInfo
rootDir string
subRootDir string
)
const (
@ -62,8 +64,15 @@ func testSetup() error {
return err
}
subDir, err := ioutil.TempDir("", "sub_digest_test")
if err != nil {
return err
}
rootDir = dir
subRootDir = subDir
// Test images used/copied:
// IMAGE NAME TAG DIGEST CONFIG LAYERS SIZE
// zot-test 0.0.1 2bacca16 adf3bb6c 76MB
@ -71,14 +80,26 @@ func testSetup() error {
// zot-cve-test 0.0.1 63a795ca 8dd57e17 75MB
// 7a0437f0 75MB
err = os.Mkdir(subDir+"/a", 0700)
if err != nil {
return err
}
err = copyFiles("../../../../test/data", rootDir)
if err != nil {
return err
}
err = copyFiles("../../../../test/data", subDir+"/a/")
if err != nil {
return err
}
log := log.NewLogger("debug", "")
digestInfo = digestinfo.NewDigestInfo(log)
storeController := storage.StoreController{DefaultStore: storage.NewImageStore(rootDir, false, false, log)}
digestInfo = digestinfo.NewDigestInfo(storeController, log)
return nil
}
@ -131,30 +152,30 @@ func copyFiles(sourceDir string, destDir string) error {
func TestDigestInfo(t *testing.T) {
Convey("Test image tag", t, func() {
// Search by manifest digest
imageTags, err := digestInfo.GetImageTagsByDigest(path.Join(rootDir, "zot-cve-test"), "63a795ca")
imageTags, err := digestInfo.GetImageTagsByDigest("zot-cve-test", "63a795ca")
So(err, ShouldBeNil)
So(len(imageTags), ShouldEqual, 1)
So(*imageTags[0], ShouldEqual, "0.0.1")
// Search by config digest
imageTags, err = digestInfo.GetImageTagsByDigest(path.Join(rootDir, "zot-test"), "adf3bb6c")
imageTags, err = digestInfo.GetImageTagsByDigest("zot-test", "adf3bb6c")
So(err, ShouldBeNil)
So(len(imageTags), ShouldEqual, 1)
So(*imageTags[0], ShouldEqual, "0.0.1")
// Search by layer digest
imageTags, err = digestInfo.GetImageTagsByDigest(path.Join(rootDir, "zot-cve-test"), "7a0437f0")
imageTags, err = digestInfo.GetImageTagsByDigest("zot-cve-test", "7a0437f0")
So(err, ShouldBeNil)
So(len(imageTags), ShouldEqual, 1)
So(*imageTags[0], ShouldEqual, "0.0.1")
// Search by non-existent image
imageTags, err = digestInfo.GetImageTagsByDigest(path.Join(rootDir, "zot-tes"), "63a795ca")
imageTags, err = digestInfo.GetImageTagsByDigest("zot-tes", "63a795ca")
So(err, ShouldNotBeNil)
So(len(imageTags), ShouldEqual, 0)
// Search by non-existent digest
imageTags, err = digestInfo.GetImageTagsByDigest(path.Join(rootDir, "zot-test"), "111")
imageTags, err = digestInfo.GetImageTagsByDigest("zot-test", "111")
So(err, ShouldBeNil)
So(len(imageTags), ShouldEqual, 0)
})
@ -286,6 +307,75 @@ func TestDigestSearchHTTP(t *testing.T) {
})
}
func TestDigestSearchHTTPSubPaths(t *testing.T) {
Convey("Test image search by digest scanning using storage subpaths", t, func() {
config := api.NewConfig()
config.HTTP.Port = Port1
config.Extensions = &ext.ExtensionConfig{
Search: &ext.SearchConfig{Enable: true},
}
c := api.NewController(config)
globalDir, err := ioutil.TempDir("", "digest_test")
if err != nil {
panic(err)
}
defer os.RemoveAll(globalDir)
c.Config.Storage.RootDirectory = globalDir
subPathMap := make(map[string]api.StorageConfig)
subPathMap["/a"] = api.StorageConfig{RootDirectory: subRootDir}
c.Config.Storage.SubPaths = subPathMap
go func() {
// this blocks
if err := c.Run(); err != nil {
return
}
}()
// wait till ready
for {
_, err := resty.R().Get(BaseURL1)
if err == nil {
break
}
time.Sleep(100 * time.Millisecond)
}
// shut down server
defer func() {
ctx := context.Background()
_ = c.Server.Shutdown(ctx)
}()
resp, err := resty.R().Get(BaseURL1 + "/v2/")
So(resp, ShouldNotBeNil)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
resp, err = resty.R().Get(BaseURL1 + "/query")
So(resp, ShouldNotBeNil)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
resp, err = resty.R().Get(BaseURL1 + "/query?query={ImageListForDigest(id:\"sha\"){Name%20Tags}}")
So(resp, ShouldNotBeNil)
So(err, ShouldBeNil)
So(resp.StatusCode(), ShouldEqual, 200)
var responseStruct ImgResponseForDigest
err = json.Unmarshal(resp.Body(), &responseStruct)
So(err, ShouldBeNil)
So(len(responseStruct.Errors), ShouldEqual, 0)
So(len(responseStruct.ImgListForDigest.Images), ShouldEqual, 2)
})
}
func TestDigestSearchDisabled(t *testing.T) {
Convey("Test disabling image search", t, func() {
dir, err := ioutil.TempDir("", "digest_test")

35
pkg/extensions/search/resolver.go
View file

@ -5,7 +5,6 @@ package search
import (
"context"
"fmt"
"path"
"strconv"
"strings"
@ -54,7 +53,7 @@ func GetResolverConfig(log log.Logger, storeController storage.StoreController,
}
}
digestInfo := digestinfo.NewDigestInfo(log)
digestInfo := digestinfo.NewDigestInfo(storeController, log)
resConfig := &Resolver{cveInfo: cveInfo, storeController: storeController, digestInfo: digestInfo, log: log}
@ -69,7 +68,7 @@ func (r *queryResolver) CVEListForImage(ctx context.Context, image string) (*CVE
r.log.Info().Str("image", image).Msg("scanning image")
isValidImage, err := r.cveInfo.LayoutUtils.IsValidImageFormat(trivyConfig.TrivyConfig.Input)
isValidImage, err := r.cveInfo.LayoutUtils.IsValidImageFormat(image)
if !isValidImage {
r.log.Debug().Str("image", image).Msg("image media type not supported for scanning")
@ -201,7 +200,7 @@ func (r *queryResolver) ImageListForCve(ctx context.Context, id string) ([]*ImgR
return finalCveResult, nil
}
func (r *queryResolver) getImageListForCVE(repoList []string, id string, imgStore *storage.ImageStore,
func (r *queryResolver) getImageListForCVE(repoList []string, id string, imgStore storage.ImageStore,
trivyConfig *config.Config) ([]*ImgResultForCve, error) {
cveResult := []*ImgResultForCve{}
@ -238,7 +237,7 @@ func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, im
r.log.Info().Str("image", image).Msg("extracting list of tags available in image")
tagsInfo, err := r.cveInfo.LayoutUtils.GetImageTagsWithTimestamp(imagePath)
tagsInfo, err := r.cveInfo.LayoutUtils.GetImageTagsWithTimestamp(image)
if err != nil {
r.log.Error().Err(err).Msg("unable to read image tags")
@ -252,7 +251,7 @@ func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, im
for _, tag := range tagsInfo {
trivyConfig.TrivyConfig.Input = fmt.Sprintf("%s:%s", imagePath, tag.Name)
isValidImage, _ := r.cveInfo.LayoutUtils.IsValidImageFormat(trivyConfig.TrivyConfig.Input)
isValidImage, _ := r.cveInfo.LayoutUtils.IsValidImageFormat(fmt.Sprintf("%s:%s", image, tag.Name))
if !isValidImage {
r.log.Debug().Str("image",
fmt.Sprintf("%s:%s", image, tag.Name)).
@ -325,9 +324,7 @@ func (r *queryResolver) ImageListForDigest(ctx context.Context, id string) ([]*I
r.log.Info().Msg("scanning each global repository")
rootDir := defaultStore.RootDir()
partialImgResultForDigest, err := r.getImageListForDigest(rootDir, repoList, id)
partialImgResultForDigest, err := r.getImageListForDigest(repoList, id)
if err != nil {
r.log.Error().Err(err).Msg("unable to get image and tag list for global repositories")
@ -338,8 +335,6 @@ func (r *queryResolver) ImageListForDigest(ctx context.Context, id string) ([]*I
subStore := r.storeController.SubStore
for _, store := range subStore {
rootDir := store.RootDir()
subRepoList, err := store.GetRepositories()
if err != nil {
r.log.Error().Err(err).Msg("unable to search sub-repositories")
@ -347,7 +342,7 @@ func (r *queryResolver) ImageListForDigest(ctx context.Context, id string) ([]*I
return imgResultForDigest, err
}
partialImgResultForDigest, err = r.getImageListForDigest(rootDir, subRepoList, id)
partialImgResultForDigest, err = r.getImageListForDigest(subRepoList, id)
if err != nil {
r.log.Error().Err(err).Msg("unable to get image and tag list for sub-repositories")
@ -360,7 +355,7 @@ func (r *queryResolver) ImageListForDigest(ctx context.Context, id string) ([]*I
return imgResultForDigest, nil
}
func (r *queryResolver) getImageListForDigest(rootDir string, repoList []string,
func (r *queryResolver) getImageListForDigest(repoList []string,
digest string) ([]*ImgResultForDigest, error) {
imgResultForDigest := []*ImgResultForDigest{}
@ -369,7 +364,7 @@ func (r *queryResolver) getImageListForDigest(rootDir string, repoList []string,
for _, repo := range repoList {
r.log.Info().Str("repo", repo).Msg("filtering list of tags in image repo by digest")
tags, err := r.digestInfo.GetImageTagsByDigest(path.Join(rootDir, repo), digest)
tags, err := r.digestInfo.GetImageTagsByDigest(repo, digest)
if err != nil {
r.log.Error().Err(err).Msg("unable to get filtered list of image tags")
@ -424,7 +419,7 @@ func (r *queryResolver) ImageListWithLatestTag(ctx context.Context) ([]*ImageInf
return imageList, nil
}
func (r *queryResolver) getImageListWithLatestTag(store *storage.ImageStore) ([]*ImageInfo, error) {
func (r *queryResolver) getImageListWithLatestTag(store storage.ImageStore) ([]*ImageInfo, error) {
results := make([]*ImageInfo, 0)
repoList, err := store.GetRepositories()
@ -438,12 +433,10 @@ func (r *queryResolver) getImageListWithLatestTag(store *storage.ImageStore) ([]
r.log.Info().Msg("no repositories found")
}
dir := store.RootDir()
layoutUtils := common.NewOciLayoutUtils(r.log)
layoutUtils := common.NewOciLayoutUtils(r.storeController, r.log)
for _, repo := range repoList {
tagsInfo, err := layoutUtils.GetImageTagsWithTimestamp(path.Join(dir, repo))
tagsInfo, err := layoutUtils.GetImageTagsWithTimestamp(repo)
if err != nil {
r.log.Error().Err(err).Msg("extension api: error getting tag timestamp info")
@ -460,7 +453,7 @@ func (r *queryResolver) getImageListWithLatestTag(store *storage.ImageStore) ([]
digest := godigest.Digest(latestTag.Digest)
manifest, err := layoutUtils.GetImageBlobManifest(path.Join(dir, repo), digest)
manifest, err := layoutUtils.GetImageBlobManifest(repo, digest)
if err != nil {
r.log.Error().Err(err).Msg("extension api: error reading manifest")
@ -471,7 +464,7 @@ func (r *queryResolver) getImageListWithLatestTag(store *storage.ImageStore) ([]
name := repo
imageConfig, err := layoutUtils.GetImageInfo(path.Join(dir, repo), manifest.Config.Digest)
imageConfig, err := layoutUtils.GetImageInfo(repo, manifest.Config.Digest)
if err != nil {
r.log.Error().Err(err).Msg("extension api: error reading image config")

1321
pkg/storage/storage.go
View file

File diff suppressed because it is too large Load diff

1333
pkg/storage/storage_fs.go Normal file
View file

File diff suppressed because it is too large Load diff

246
pkg/storage/storage_test.go
View file

@ -38,11 +38,18 @@ func TestAPIs(t *testing.T) {
v, err := il.ValidateRepo(repoName)
So(v, ShouldEqual, false)
So(err, ShouldNotBeNil)
ok := il.DirExists(path.Join(il.RootDir(), repoName))
So(ok, ShouldBeFalse)
})
Convey("Initialize repo", func() {
err := il.InitRepo(repoName)
So(err, ShouldBeNil)
ok := il.DirExists(path.Join(il.RootDir(), repoName))
So(ok, ShouldBeTrue)
storeController := storage.StoreController{}
storeController.DefaultStore = il
So(storeController.GetImageStore("test"), ShouldResemble, il)
})
Convey("Validate repo", func() {
@ -78,6 +85,13 @@ func TestAPIs(t *testing.T) {
So(err, ShouldBeNil)
So(v, ShouldNotBeEmpty)
err = il.DeleteBlobUpload("test", v)
So(err, ShouldBeNil)
v, err = il.NewBlobUpload("test")
So(err, ShouldBeNil)
So(v, ShouldNotBeEmpty)
Convey("Get blob upload", func() {
b, err := il.GetBlobUpload("test", "invalid")
So(err, ShouldNotBeNil)
@ -92,17 +106,40 @@ func TestAPIs(t *testing.T) {
So(b, ShouldBeGreaterThanOrEqualTo, 0)
content := []byte("test-data1")
firstChunkContent := []byte("test")
firstChunkBuf := bytes.NewBuffer(firstChunkContent)
secondChunkContent := []byte("-data1")
secondChunkBuf := bytes.NewBuffer(secondChunkContent)
firstChunkLen := firstChunkBuf.Len()
secondChunkLen := secondChunkBuf.Len()
buf := bytes.NewBuffer(content)
l := buf.Len()
d := godigest.FromBytes(content)
b, err = il.PutBlobChunk("test", v, 0, int64(l), buf)
So(err, ShouldBeNil)
So(b, ShouldEqual, l)
blobDigest := d
// invalid chunk range
_, err = il.PutBlobChunk("test", v, 10, int64(l), buf)
So(err, ShouldNotBeNil)
b, err = il.PutBlobChunk("test", v, 0, int64(firstChunkLen), firstChunkBuf)
So(err, ShouldBeNil)
So(b, ShouldEqual, firstChunkLen)
b, err = il.GetBlobUpload("test", v)
So(err, ShouldBeNil)
So(b, ShouldEqual, int64(firstChunkLen))
b, err = il.BlobUploadInfo("test", v)
So(err, ShouldBeNil)
So(b, ShouldEqual, int64(firstChunkLen))
b, err = il.PutBlobChunk("test", v, int64(firstChunkLen), int64(l), secondChunkBuf)
So(err, ShouldBeNil)
So(b, ShouldEqual, secondChunkLen)
err = il.FinishBlobUpload("test", v, buf, d.String())
So(err, ShouldBeNil)
So(b, ShouldEqual, l)
_, _, err = il.CheckBlob("test", d.String())
So(err, ShouldBeNil)
@ -121,11 +158,62 @@ func TestAPIs(t *testing.T) {
_, err = il.PutImageManifest("test", d.String(), ispec.MediaTypeImageManifest, []byte{})
So(err, ShouldNotBeNil)
_, err = il.PutImageManifest("test", d.String(), ispec.MediaTypeImageManifest, []byte(`{"test":true}`))
So(err, ShouldNotBeNil)
_, err = il.PutImageManifest("test", d.String(), ispec.MediaTypeImageManifest, mb)
So(err, ShouldNotBeNil)
_, _, _, err = il.GetImageManifest("test", d.String())
So(err, ShouldNotBeNil)
_, _, _, err = il.GetImageManifest("inexistent", d.String())
So(err, ShouldNotBeNil)
annotationsMap := make(map[string]string)
annotationsMap[ispec.AnnotationRefName] = "1.0"
m := ispec.Manifest{
Config: ispec.Descriptor{
Digest: d,
Size: int64(l),
},
Layers: []ispec.Descriptor{
{
MediaType: "application/vnd.oci.image.layer.v1.tar",
Digest: d,
Size: int64(l),
},
},
Annotations: annotationsMap,
}
m.SchemaVersion = 2
mb, _ = json.Marshal(m)
d := godigest.FromBytes(mb)
So(os.Chmod(path.Join(il.RootDir(), "test", "index.json"), 0000), ShouldBeNil)
_, err = il.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, mb)
So(err, ShouldNotBeNil)
So(os.Chmod(path.Join(il.RootDir(), "test", "index.json"), 0755), ShouldBeNil)
_, err = il.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, mb)
So(err, ShouldBeNil)
manifestPath := path.Join(il.RootDir(), "test", "blobs", d.Algorithm().String(), d.Encoded())
So(os.Chmod(manifestPath, 0000), ShouldBeNil)
_, _, _, err = il.GetImageManifest("test", d.String())
So(err, ShouldNotBeNil)
So(os.Remove(manifestPath), ShouldBeNil)
_, _, _, err = il.GetImageManifest("test", d.String())
So(err, ShouldNotBeNil)
So(os.Chmod(path.Join(il.RootDir(), "test"), 0000), ShouldBeNil)
_, err = il.PutImageManifest("test", "2.0", ispec.MediaTypeImageManifest, mb)
So(err, ShouldNotBeNil)
So(os.Chmod(path.Join(il.RootDir(), "test"), 0755), ShouldBeNil)
So(os.RemoveAll(path.Join(il.RootDir(), "test")), ShouldBeNil)
})
Convey("Good image manifest", func() {
@ -147,8 +235,20 @@ func TestAPIs(t *testing.T) {
}
m.SchemaVersion = 2
mb, _ := json.Marshal(m)
mb, _ = json.Marshal(m)
d := godigest.FromBytes(mb)
// bad manifest
m.Layers[0].Digest = godigest.FromBytes([]byte("inexistent"))
badMb, _ := json.Marshal(m)
_, err = il.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, badMb)
So(err, ShouldNotBeNil)
_, err = il.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, mb)
So(err, ShouldBeNil)
// same manifest for coverage
_, err = il.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, mb)
So(err, ShouldBeNil)
@ -158,6 +258,9 @@ func TestAPIs(t *testing.T) {
_, err = il.PutImageManifest("test", "3.0", ispec.MediaTypeImageManifest, mb)
So(err, ShouldBeNil)
_, err = il.GetImageTags("inexistent")
So(err, ShouldNotBeNil)
// total tags should be 3 but they have same reference.
tags, err := il.GetImageTags("test")
So(err, ShouldBeNil)
@ -166,6 +269,9 @@ func TestAPIs(t *testing.T) {
_, _, _, err = il.GetImageManifest("test", d.String())
So(err, ShouldBeNil)
_, _, _, err = il.GetImageManifest("test", "3.0")
So(err, ShouldBeNil)
err = il.DeleteImageManifest("test", "1.0")
So(err, ShouldBeNil)
@ -178,6 +284,15 @@ func TestAPIs(t *testing.T) {
So(err, ShouldBeNil)
So(hasBlob, ShouldEqual, true)
// invalid DeleteImageManifest
indexPath := path.Join(il.RootDir(), "test", "index.json")
So(os.Chmod(indexPath, 0000), ShouldBeNil)
err = il.DeleteImageManifest("test", d.String())
So(err, ShouldNotBeNil)
So(os.Chmod(indexPath, 0755), ShouldBeNil)
// If we pass reference all manifest with input reference should be deleted.
err = il.DeleteImageManifest("test", d.String())
So(err, ShouldBeNil)
@ -191,6 +306,12 @@ func TestAPIs(t *testing.T) {
So(err, ShouldNotBeNil)
So(hasBlob, ShouldEqual, false)
err = il.DeleteBlob("test", "inexistent")
So(err, ShouldNotBeNil)
err = il.DeleteBlob("test", godigest.FromBytes([]byte("inexistent")).String())
So(err, ShouldNotBeNil)
err = il.DeleteBlob("test", blobDigest.String())
So(err, ShouldBeNil)
@ -209,6 +330,9 @@ func TestAPIs(t *testing.T) {
So(v, ShouldNotBeEmpty)
Convey("Get blob upload", func() {
err = il.FinishBlobUpload("test", v, bytes.NewBuffer([]byte{}), "inexistent")
So(err, ShouldNotBeNil)
b, err := il.GetBlobUpload("test", "invalid")
So(err, ShouldNotBeNil)
So(b, ShouldEqual, -1)
@ -217,6 +341,9 @@ func TestAPIs(t *testing.T) {
So(err, ShouldBeNil)
So(b, ShouldBeGreaterThanOrEqualTo, 0)
_, err = il.BlobUploadInfo("test", "inexistent")
So(err, ShouldNotBeNil)
b, err = il.BlobUploadInfo("test", v)
So(err, ShouldBeNil)
So(b, ShouldBeGreaterThanOrEqualTo, 0)
@ -229,24 +356,50 @@ func TestAPIs(t *testing.T) {
So(err, ShouldBeNil)
So(b, ShouldEqual, l)
_, err = il.PutBlobChunkStreamed("test", "inexistent", buf)
So(err, ShouldNotBeNil)
err = il.FinishBlobUpload("test", "inexistent", buf, d.String())
So(err, ShouldNotBeNil)
err = il.FinishBlobUpload("test", v, buf, d.String())
So(err, ShouldBeNil)
So(b, ShouldEqual, l)
_, _, err = il.CheckBlob("test", d.String())
So(err, ShouldBeNil)
_, _, err = il.GetBlob("test", "inexistent", "application/vnd.oci.image.layer.v1.tar+gzip")
So(err, ShouldNotBeNil)
_, _, err = il.GetBlob("test", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip")
So(err, ShouldBeNil)
blobContent, err := il.GetBlobContent("test", d.String())
So(err, ShouldBeNil)
So(content, ShouldResemble, blobContent)
_, err = il.GetBlobContent("inexistent", d.String())
So(err, ShouldNotBeNil)
m := ispec.Manifest{}
m.SchemaVersion = 2
mb, _ := json.Marshal(m)
Convey("Bad digests", func() {
_, _, err := il.FullBlobUpload("test", bytes.NewBuffer([]byte{}), "inexistent")
So(err, ShouldNotBeNil)
_, _, err = il.CheckBlob("test", "inexistent")
So(err, ShouldNotBeNil)
})
Convey("Bad image manifest", func() {
_, err = il.PutImageManifest("test", d.String(), ispec.MediaTypeImageManifest, mb)
So(err, ShouldNotBeNil)
_, err = il.PutImageManifest("test", d.String(), ispec.MediaTypeImageManifest, []byte("bad json"))
So(err, ShouldNotBeNil)
_, _, _, err = il.GetImageManifest("test", d.String())
So(err, ShouldNotBeNil)
})
@ -265,18 +418,38 @@ func TestAPIs(t *testing.T) {
},
},
}
m.SchemaVersion = 2
mb, _ = json.Marshal(m)
d := godigest.FromBytes(mb)
_, err = il.PutImageManifest("test", d.String(), ispec.MediaTypeImageManifest, mb)
So(err, ShouldBeNil)
// same manifest for coverage
_, err = il.PutImageManifest("test", d.String(), ispec.MediaTypeImageManifest, mb)
So(err, ShouldBeNil)
_, _, _, err = il.GetImageManifest("test", d.String())
So(err, ShouldBeNil)
_, err = il.GetIndexContent("inexistent")
So(err, ShouldNotBeNil)
indexContent, err := il.GetIndexContent("test")
So(err, ShouldBeNil)
var index ispec.Index
err = json.Unmarshal(indexContent, &index)
So(err, ShouldBeNil)
So(len(index.Manifests), ShouldEqual, 1)
err = il.DeleteImageManifest("test", "1.0")
So(err, ShouldNotBeNil)
err = il.DeleteImageManifest("inexistent", "1.0")
So(err, ShouldNotBeNil)
err = il.DeleteImageManifest("test", d.String())
So(err, ShouldBeNil)
@ -508,7 +681,7 @@ func TestAPIs(t *testing.T) {
func TestDedupe(t *testing.T) {
Convey("Dedupe", t, func(c C) {
Convey("Nil ImageStore", func() {
is := &storage.ImageStore{}
var is storage.ImageStore
So(func() { _ = is.DedupeBlob("", "", "") }, ShouldPanic)
})
@ -645,8 +818,8 @@ func TestNegativeCases(t *testing.T) {
})
Convey("Invalid get image tags", t, func(c C) {
il := &storage.ImageStore{}
_, err := il.GetImageTags("test")
var ilfs storage.ImageStoreFS
_, err := ilfs.GetImageTags("test")
So(err, ShouldNotBeNil)
dir, err := ioutil.TempDir("", "oci-repo-test")
@ -654,7 +827,7 @@ func TestNegativeCases(t *testing.T) {
panic(err)
}
defer os.RemoveAll(dir)
il = storage.NewImageStore(dir, true, true, log.Logger{Logger: zerolog.New(os.Stdout)})
il := storage.NewImageStore(dir, true, true, log.Logger{Logger: zerolog.New(os.Stdout)})
So(il, ShouldNotBeNil)
So(il.InitRepo("test"), ShouldBeNil)
So(os.Remove(path.Join(dir, "test", "index.json")), ShouldBeNil)
@ -668,8 +841,8 @@ func TestNegativeCases(t *testing.T) {
})
Convey("Invalid get image manifest", t, func(c C) {
il := &storage.ImageStore{}
_, _, _, err := il.GetImageManifest("test", "")
var ilfs storage.ImageStoreFS
_, _, _, err := ilfs.GetImageManifest("test", "")
So(err, ShouldNotBeNil)
dir, err := ioutil.TempDir("", "oci-repo-test")
@ -677,9 +850,12 @@ func TestNegativeCases(t *testing.T) {
panic(err)
}
defer os.RemoveAll(dir)
il = storage.NewImageStore(dir, true, true, log.Logger{Logger: zerolog.New(os.Stdout)})
il := storage.NewImageStore(dir, true, true, log.Logger{Logger: zerolog.New(os.Stdout)})
So(il, ShouldNotBeNil)
So(il.InitRepo("test"), ShouldBeNil)
So(os.Chmod(path.Join(dir, "test", "index.json"), 0000), ShouldBeNil)
_, _, _, err = il.GetImageManifest("test", "")
So(err, ShouldNotBeNil)
So(os.Remove(path.Join(dir, "test", "index.json")), ShouldBeNil)
_, _, _, err = il.GetImageManifest("test", "")
So(err, ShouldNotBeNil)
@ -690,7 +866,47 @@ func TestNegativeCases(t *testing.T) {
So(err, ShouldNotBeNil)
})
Convey("Invalid dedupe sceanrios", t, func() {
Convey("Invalid new blob upload", t, func(c C) {
dir, err := ioutil.TempDir("", "oci-repo-test")
if err != nil {
panic(err)
}
defer os.RemoveAll(dir)
il := storage.NewImageStore(dir, true, true, log.Logger{Logger: zerolog.New(os.Stdout)})
So(il, ShouldNotBeNil)
So(il.InitRepo("test"), ShouldBeNil)
So(os.Chmod(path.Join(dir, "test", ".uploads"), 0000), ShouldBeNil)
_, err = il.NewBlobUpload("test")
So(err, ShouldNotBeNil)
So(os.Chmod(path.Join(dir, "test"), 0000), ShouldBeNil)
_, err = il.NewBlobUpload("test")
So(err, ShouldNotBeNil)
So(os.Chmod(path.Join(dir, "test"), 0755), ShouldBeNil)
So(il.InitRepo("test"), ShouldBeNil)
_, err = il.NewBlobUpload("test")
So(err, ShouldNotBeNil)
So(os.Chmod(path.Join(dir, "test", ".uploads"), 0755), ShouldBeNil)
v, err := il.NewBlobUpload("test")
So(err, ShouldBeNil)
So(os.Chmod(path.Join(dir, "test", ".uploads"), 0000), ShouldBeNil)
content := []byte("test-data3")
buf := bytes.NewBuffer(content)
l := buf.Len()
_, err = il.PutBlobChunkStreamed("test", v, buf)
So(err, ShouldNotBeNil)
_, err = il.PutBlobChunk("test", v, 0, int64(l), buf)
So(err, ShouldNotBeNil)
})
Convey("Invalid dedupe scenarios", t, func() {
dir, err := ioutil.TempDir("", "oci-repo-test")
if err != nil {
panic(err)
@ -826,7 +1042,7 @@ func TestStorageHandler(t *testing.T) {
storeController.DefaultStore = firstStore
subStore := make(map[string]*storage.ImageStore)
subStore := make(map[string]storage.ImageStore)
subStore["/a"] = secondStore
subStore["/b"] = thirdStore