diff --git a/README.md b/README.md index 12cbf3da..4cf23e4f 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,14 @@ * Uses [OCI storage layout](https://github.com/opencontainers/image-spec/blob/master/image-layout.md) for storage layout * Currently suitable for on-prem deployments (e.g. colocated with Kubernetes) * TLS support -* Authentication via TLS mutual authentication and HTTP *BASIC* (local _htpasswd_ and LDAP) +* Authentication via: + * TLS mutual authentication + * HTTP *Basic* (local _htpasswd_ and LDAP) + * HTTP *Bearer* token * Doesn't require _root_ privileges +* Storage optimizations: + * Automatic garbage collection of orphaned blobs + * Layer deduplication using hard links when content is identical * Swagger based documentation * Released under Apache 2.0 License * ```go get -u github.com/anuvu/zot/cmd/zot``` diff --git a/WORKSPACE b/WORKSPACE index c26fbd20..c93fc632 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -872,8 +872,8 @@ go_repository( go_repository( name = "io_etcd_go_bbolt", importpath = "go.etcd.io/bbolt", - sum = "h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=", - version = "v1.3.2", + sum = "h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=", + version = "v1.3.4", ) go_repository( @@ -935,8 +935,8 @@ go_repository( go_repository( name = "org_golang_x_sys", importpath = "golang.org/x/sys", - sum = "h1:wYqz/tQaWUgGKyx+B/rssSE6wkIKdY5Ee6ryOmzarIg=", - version = "v0.0.0-20190913121621-c3b328c6e5a7", + sum = "h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=", + version = "v0.0.0-20200223170610-d5e6a3e2c0ae", ) go_repository( @@ -1198,6 +1198,20 @@ go_repository( version = "v1.51.0", ) +go_repository( + name = "com_github_boltdb_bolt", + importpath = "github.com/boltdb/bolt", + sum = "h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=", + version = "v1.3.1", +) + +go_repository( + name = "com_github_etcd_io_bbolt", + importpath = "github.com/etcd-io/bbolt", + sum = "h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM=", + version = "v1.3.3", +) + go_repository( name = "com_github_apex_log", importpath = "github.com/apex/log", diff --git a/errors/errors.go b/errors/errors.go index 08be4653..3291073a 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -21,4 +21,7 @@ var ( ErrLDAPEmptyPassphrase = errors.New("ldap: empty passphrase") ErrLDAPBadConn = errors.New("ldap: bad connection") ErrLDAPConfig = errors.New("config: invalid LDAP configuration") + ErrCacheRootBucket = errors.New("cache: unable to create/update root bucket") + ErrCacheNoBucket = errors.New("cache: unable to find bucket") + ErrCacheMiss = errors.New("cache: miss") ) diff --git a/go.mod b/go.mod index b4c144fb..97f05980 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,8 @@ require ( github.com/spf13/viper v1.6.1 github.com/swaggo/http-swagger v0.0.0-20190614090009-c2865af9083e github.com/swaggo/swag v1.6.3 + go.etcd.io/bbolt v1.3.4 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 + golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae // indirect gopkg.in/resty.v1 v1.12.0 ) diff --git a/go.sum b/go.sum index 7b7c2fb4..df263d62 100644 --- a/go.sum +++ b/go.sum @@ -160,6 +160,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nmcclain/asn1-ber v0.0.0-20170104154839-2661553a0484 h1:D9EvfGQvlkKaDr2CRKN++7HbSXbefUNDrPq60T+g24s= @@ -171,8 +172,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/openSUSE/umoci v0.4.5 h1:MZgFLy5Jl3VKe5uEOU9c25FoySbx3vUXeXLw4Jf6aRs= -github.com/openSUSE/umoci v0.4.5/go.mod h1:3p4KA5nwyY65lVmQZxv7tm0YEylJ+t1fY91ORsVXv58= github.com/openSUSE/umoci v0.4.6-0.20200320140503-9aa268eeb258 h1:/8Yu54FufyHHQgIZ/wLy+BLQyzk0gbOG24xf5suWOOI= github.com/openSUSE/umoci v0.4.6-0.20200320140503-9aa268eeb258/go.mod h1:3p4KA5nwyY65lVmQZxv7tm0YEylJ+t1fY91ORsVXv58= github.com/opencontainers/distribution-spec v1.0.0-rc0 h1:xMzwhweo1gjvEo74mQjGTLau0TD3ACyTEC1310NbuSQ= @@ -283,6 +282,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= +go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -329,8 +330,10 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae h1:xiXzMMEQdQcric9hXtr1QU98MHunKK7OTtsoU6bYWs4= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190913121621-c3b328c6e5a7 h1:wYqz/tQaWUgGKyx+B/rssSE6wkIKdY5Ee6ryOmzarIg= golang.org/x/sys v0.0.0-20190913121621-c3b328c6e5a7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -355,6 +358,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= diff --git a/pkg/api/controller.go b/pkg/api/controller.go index 6098ed92..04bf570d 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "net" "net/http" + "os" "github.com/anuvu/zot/errors" "github.com/anuvu/zot/pkg/log" @@ -41,12 +42,16 @@ func (c *Controller) Run() error { engine.Use(log.SessionLogger(c.Log), handlers.RecoveryHandler(handlers.RecoveryLogger(c.Log), handlers.PrintRecoveryStack(false))) + c.ImageStore = storage.NewImageStore(c.Config.Storage.RootDirectory, c.Log) + if c.ImageStore == nil { + // we can't proceed without at least a image store + os.Exit(1) + } + c.Router = engine c.Router.UseEncodedPath() _ = NewRouteHandler(c) - c.ImageStore = storage.NewImageStore(c.Config.Storage.RootDirectory, c.Log) - addr := fmt.Sprintf("%s:%s", c.Config.HTTP.Address, c.Config.HTTP.Port) server := &http.Server{Addr: addr, Handler: c.Router} c.Server = server diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 718b21d4..3f79607c 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -20,7 +20,6 @@ import ( "sort" "strconv" "strings" - "sync" _ "github.com/anuvu/zot/docs" // nolint (golint) - as required by swaggo "github.com/anuvu/zot/errors" @@ -41,12 +40,11 @@ const ( ) type RouteHandler struct { - c *Controller - blobLock sync.RWMutex + c *Controller } func NewRouteHandler(c *Controller) *RouteHandler { - rh := &RouteHandler{c: c, blobLock: sync.RWMutex{}} + rh := &RouteHandler{c: c} rh.SetupRoutes() return rh @@ -56,9 +54,9 @@ func NewRouteHandler(c *Controller) *RouteHandler { func (rh *RouteHandler) blobRLockWrapper(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { - rh.blobLock.RLock() + rh.c.ImageStore.RLock() f(w, r) - rh.blobLock.RUnlock() + rh.c.ImageStore.RUnlock() } } @@ -66,9 +64,9 @@ func (rh *RouteHandler) blobRLockWrapper(f func(w http.ResponseWriter, func (rh *RouteHandler) blobLockWrapper(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { - rh.blobLock.Lock() + rh.c.ImageStore.Lock() f(w, r) - rh.blobLock.Unlock() + rh.c.ImageStore.Unlock() } } diff --git a/pkg/compliance/v1_0_0/check.go b/pkg/compliance/v1_0_0/check.go index c5f0dea2..3b1df400 100644 --- a/pkg/compliance/v1_0_0/check.go +++ b/pkg/compliance/v1_0_0/check.go @@ -135,7 +135,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { } // without a "?digest=<>" should fail - content := []byte("this is a blob") + content := []byte("this is a blob1") digest := godigest.FromBytes(content) So(digest, ShouldNotBeNil) resp, err = resty.R().Put(loc) @@ -172,7 +172,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { Convey("Monolithic blob upload with body", func() { Print("\nMonolithic blob upload") // create content - content := []byte("this is a blob") + content := []byte("this is a blob2") digest := godigest.FromBytes(content) So(digest, ShouldNotBeNil) // setting invalid URL params should fail @@ -228,7 +228,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { } // without a "?digest=<>" should fail - content := []byte("this is a blob") + content := []byte("this is a blob3") digest := godigest.FromBytes(content) So(digest, ShouldNotBeNil) resp, err = resty.R().Put(loc) @@ -271,7 +271,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(loc, ShouldNotBeEmpty) var buf bytes.Buffer - chunk1 := []byte("this is the first chunk") + chunk1 := []byte("this is the first chunk1") n, err := buf.Write(chunk1) So(n, ShouldEqual, len(chunk1)) So(err, ShouldBeNil) @@ -299,7 +299,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(resp.StatusCode(), ShouldEqual, 416) So(resp.String(), ShouldNotBeEmpty) - chunk2 := []byte("this is the second chunk") + chunk2 := []byte("this is the second chunk1") n, err = buf.Write(chunk2) So(n, ShouldEqual, len(chunk2)) So(err, ShouldBeNil) @@ -339,7 +339,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(loc, ShouldNotBeEmpty) var buf bytes.Buffer - chunk1 := []byte("this is the first chunk") + chunk1 := []byte("this is the first chunk2") n, err := buf.Write(chunk1) So(n, ShouldEqual, len(chunk1)) So(err, ShouldBeNil) @@ -367,7 +367,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(resp.StatusCode(), ShouldEqual, 416) So(resp.String(), ShouldNotBeEmpty) - chunk2 := []byte("this is the second chunk") + chunk2 := []byte("this is the second chunk2") n, err = buf.Write(chunk2) So(n, ShouldEqual, len(chunk2)) So(err, ShouldBeNil) @@ -422,7 +422,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { loc := Location(baseURL, resp) So(loc, ShouldNotBeEmpty) - content := []byte("this is a blob") + content := []byte("this is a blob4") digest := godigest.FromBytes(content) So(digest, ShouldNotBeNil) // monolithic blob upload @@ -461,7 +461,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { resp, err = resty.R().Get(loc) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 204) - content := []byte("this is a blob") + content := []byte("this is a blob5") digest := godigest.FromBytes(content) So(digest, ShouldNotBeNil) // monolithic blob upload: success @@ -507,7 +507,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(d, ShouldNotBeEmpty) So(d, ShouldEqual, digest.String()) - content = []byte("this is a blob") + content = []byte("this is a blob5") digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) // create a manifest with same blob but a different tag @@ -611,7 +611,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { resp, err = resty.R().Get(loc) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 204) - content := []byte("this is a blob") + content := []byte("this is a blob7") digest := godigest.FromBytes(content) So(digest, ShouldNotBeNil) // monolithic blob upload: success diff --git a/pkg/storage/BUILD.bazel b/pkg/storage/BUILD.bazel index 3efaa617..3ca8b38a 100644 --- a/pkg/storage/BUILD.bazel +++ b/pkg/storage/BUILD.bazel @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["storage.go"], + srcs = [ + "cache.go", + "storage.go", + ], importpath = "github.com/anuvu/zot/pkg/storage", visibility = ["//visibility:public"], deps = [ @@ -13,16 +16,21 @@ go_library( "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library", "@com_github_opensuse_umoci//:go_default_library", "@com_github_rs_zerolog//:go_default_library", + "@io_etcd_go_bbolt//:go_default_library", ], ) go_test( name = "go_default_test", timeout = "short", - srcs = ["storage_test.go"], + srcs = [ + "cache_test.go", + "storage_test.go", + ], embed = [":go_default_library"], race = "on", deps = [ + "//errors:go_default_library", "//pkg/log:go_default_library", "@com_github_opencontainers_go_digest//:go_default_library", "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library", diff --git a/pkg/storage/cache.go b/pkg/storage/cache.go new file mode 100644 index 00000000..5c7a0758 --- /dev/null +++ b/pkg/storage/cache.go @@ -0,0 +1,163 @@ +package storage + +import ( + "path" + "strings" + + "github.com/anuvu/zot/errors" + zlog "github.com/anuvu/zot/pkg/log" + "go.etcd.io/bbolt" +) + +const ( + BlobsCache = "blobs" +) + +type Cache struct { + db *bbolt.DB + log zlog.Logger +} + +// Blob is a blob record +type Blob struct { + Path string +} + +func NewCache(rootDir string, name string, log zlog.Logger) *Cache { + dbPath := path.Join(rootDir, name+".db") + db, err := bbolt.Open(dbPath, 0600, nil) + + if err != nil { + log.Error().Err(err).Str("dbPath", dbPath).Msg("unable to create cache db") + return nil + } + + if err := db.Update(func(tx *bbolt.Tx) error { + if _, err := tx.CreateBucketIfNotExists([]byte(BlobsCache)); err != nil { + // this is a serious failure + log.Error().Err(err).Str("dbPath", dbPath).Msg("unable to create a root bucket") + return err + } + return nil + }); err != nil { + // something went wrong + log.Error().Err(err).Msg("unable to create a cache") + return nil + } + + return &Cache{db: db, log: log} +} + +func (c *Cache) PutBlob(digest string, path string) error { + if err := c.db.Update(func(tx *bbolt.Tx) error { + root := tx.Bucket([]byte(BlobsCache)) + if root == nil { + // this is a serious failure + err := errors.ErrCacheRootBucket + c.log.Error().Err(err).Msg("unable to access root bucket") + return err + } + b, err := root.CreateBucketIfNotExists([]byte(digest)) + if err != nil { + // this is a serious failure + c.log.Error().Err(err).Str("bucket", digest).Msg("unable to create a bucket") + return err + } + if err := b.Put([]byte(path), nil); err != nil { + c.log.Error().Err(err).Str("bucket", digest).Str("value", path).Msg("unable to put record") + return err + } + return nil + }); err != nil { + return err + } + + return nil +} + +func (c *Cache) GetBlob(digest string) (string, error) { + var blobPath strings.Builder + + if err := c.db.View(func(tx *bbolt.Tx) error { + root := tx.Bucket([]byte(BlobsCache)) + if root == nil { + // this is a serious failure + err := errors.ErrCacheRootBucket + c.log.Error().Err(err).Msg("unable to access root bucket") + return err + } + + b := root.Bucket([]byte(digest)) + if b != nil { + // get first key + c := b.Cursor() + k, _ := c.First() + blobPath.WriteString(string(k)) + return nil + } + + return errors.ErrCacheMiss + }); err != nil { + return "", err + } + + if len(blobPath.String()) == 0 { + return "", nil + } + + return blobPath.String(), nil +} + +func (c *Cache) HasBlob(digest string, blob string) bool { + if err := c.db.View(func(tx *bbolt.Tx) error { + root := tx.Bucket([]byte(BlobsCache)) + if root == nil { + // this is a serious failure + err := errors.ErrCacheRootBucket + c.log.Error().Err(err).Msg("unable to access root bucket") + return err + } + + b := root.Bucket([]byte(digest)) + if b == nil { + return errors.ErrCacheMiss + } + if b.Get([]byte(blob)) == nil { + return errors.ErrCacheMiss + } + + return nil + }); err != nil { + return false + } + + return true +} + +func (c *Cache) DeleteBlob(digest string, path string) error { + if err := c.db.Update(func(tx *bbolt.Tx) error { + root := tx.Bucket([]byte(BlobsCache)) + if root == nil { + // this is a serious failure + err := errors.ErrCacheRootBucket + c.log.Error().Err(err).Msg("unable to access root bucket") + return err + } + + b := root.Bucket([]byte(digest)) + if b == nil { + return errors.ErrCacheMiss + } + + if err := b.Delete([]byte(path)); err != nil { + c.log.Error().Err(err).Str("digest", digest).Str("path", path).Msg("unable to delete") + return err + } + + return nil + }); err != nil { + return err + } + + return nil +} diff --git a/pkg/storage/cache_test.go b/pkg/storage/cache_test.go new file mode 100644 index 00000000..05ea6c61 --- /dev/null +++ b/pkg/storage/cache_test.go @@ -0,0 +1,52 @@ +package storage_test + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/anuvu/zot/errors" + "github.com/anuvu/zot/pkg/log" + "github.com/anuvu/zot/pkg/storage" + . "github.com/smartystreets/goconvey/convey" +) + +func TestCache(t *testing.T) { + Convey("Make a new cache", t, func() { + dir, err := ioutil.TempDir("", "cache_test") + So(err, ShouldBeNil) + So(dir, ShouldNotBeEmpty) + defer os.RemoveAll(dir) + + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + So(storage.NewCache("/deadBEEF", "cache_test", log), ShouldBeNil) + + c := storage.NewCache(dir, "cache_test", log) + So(c, ShouldNotBeNil) + + v, err := c.GetBlob("key") + So(err, ShouldEqual, errors.ErrCacheMiss) + So(v, ShouldBeEmpty) + + b := c.HasBlob("key", "value") + So(b, ShouldBeFalse) + + err = c.PutBlob("key", "value") + So(err, ShouldBeNil) + + b = c.HasBlob("key", "value") + So(b, ShouldBeTrue) + + v, err = c.GetBlob("key") + So(err, ShouldBeNil) + So(v, ShouldNotBeEmpty) + + err = c.DeleteBlob("bogusKey", "bogusValue") + So(err, ShouldEqual, errors.ErrCacheMiss) + + err = c.DeleteBlob("key", "bogusValue") + So(err, ShouldBeNil) + }) +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 8e486565..6f64da95 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -37,24 +37,50 @@ type ImageStore struct { rootDir string lock *sync.RWMutex blobUploads map[string]BlobUpload + cache *Cache log zerolog.Logger } // NewImageStore returns a new image store backed by a file storage. func NewImageStore(rootDir string, log zlog.Logger) *ImageStore { - is := &ImageStore{rootDir: rootDir, + if _, err := os.Stat(rootDir); os.IsNotExist(err) { + if err := os.MkdirAll(rootDir, 0700); err != nil { + log.Error().Err(err).Str("rootDir", rootDir).Msg("unable to create root dir") + return nil + } + } + + is := &ImageStore{ + rootDir: rootDir, lock: &sync.RWMutex{}, blobUploads: make(map[string]BlobUpload), + cache: NewCache(rootDir, "cache", log), log: log.With().Caller().Logger(), } - if _, err := os.Stat(rootDir); os.IsNotExist(err) { - _ = os.MkdirAll(rootDir, 0700) - } - return is } +// RLock read-lock +func (is *ImageStore) RLock() { + is.lock.RLock() +} + +// RUnlock read-unlock +func (is *ImageStore) RUnlock() { + is.lock.RUnlock() +} + +// Lock write-lock +func (is *ImageStore) Lock() { + is.lock.Lock() +} + +// Unlock write-unlock +func (is *ImageStore) Unlock() { + is.lock.Unlock() +} + // InitRepo creates an image repository under this store. func (is *ImageStore) InitRepo(name string) error { repoDir := path.Join(is.rootDir, name) @@ -63,16 +89,10 @@ func (is *ImageStore) InitRepo(name string) error { return nil } - // create repo dir - ensureDir(repoDir) - // create "blobs" subdir - dir := path.Join(repoDir, "blobs") - ensureDir(dir) - + ensureDir(path.Join(repoDir, "blobs"), is.log) // create BlobUploadDir subdir - dir = path.Join(repoDir, BlobUploadDir) - ensureDir(dir) + ensureDir(path.Join(repoDir, BlobUploadDir), is.log) // "oci-layout" file - create if it doesn't exist ilPath := path.Join(repoDir, ispec.ImageLayoutFile) @@ -85,7 +105,8 @@ func (is *ImageStore) InitRepo(name string) error { } if err := ioutil.WriteFile(ilPath, buf, 0644); err != nil { - is.log.Panic().Err(err).Str("file", ilPath).Msg("unable to write file") + is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") + return err } } @@ -101,7 +122,8 @@ func (is *ImageStore) InitRepo(name string) error { } if err := ioutil.WriteFile(indexPath, buf, 0644); err != nil { - is.log.Panic().Err(err).Str("file", indexPath).Msg("unable to write file") + is.log.Error().Err(err).Str("file", indexPath).Msg("unable to write file") + return err } } @@ -111,8 +133,8 @@ func (is *ImageStore) InitRepo(name string) error { // ValidateRepo validates that the repository layout is complaint with the OCI repo layout. func (is *ImageStore) ValidateRepo(name string) (bool, error) { // https://github.com/opencontainers/image-spec/blob/master/image-layout.md#content - // at least, expect exactly 4 entries - ["blobs", "oci-layout", "index.json"] and BlobUploadDir - // in each image store + // at least, expect at least 3 entries - ["blobs", "oci-layout", "index.json"] + // and an additional/optional BlobUploadDir in each image store dir := path.Join(is.rootDir, name) if !dirExists(dir) { return false, errors.ErrRepoNotFound @@ -124,15 +146,14 @@ func (is *ImageStore) ValidateRepo(name string) (bool, error) { return false, errors.ErrRepoNotFound } - if len(files) != 4 { - return false, nil + if len(files) < 3 { + return false, errors.ErrRepoBadVersion } found := map[string]bool{ "blobs": false, ispec.ImageLayoutFile: false, "index.json": false, - BlobUploadDir: false, } for _, file := range files { @@ -195,7 +216,7 @@ func (is *ImageStore) GetRepositories() ([]string, error) { return nil } - is.log.Debug().Str("dir", path).Str("name", info.Name()).Msg("found image store") + //is.log.Debug().Str("dir", path).Str("name", info.Name()).Msg("found image store") stores = append(stores, rel) return nil @@ -212,7 +233,6 @@ func (is *ImageStore) GetImageTags(repo string) ([]string, error) { } buf, err := ioutil.ReadFile(path.Join(dir, "index.json")) - if err != nil { is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") return nil, errors.ErrRepoNotFound @@ -290,9 +310,7 @@ func (is *ImageStore) GetImageManifest(repo string, reference string) ([]byte, s return nil, "", "", errors.ErrManifestNotFound } - p := path.Join(dir, "blobs") - p = path.Join(p, digest.Algorithm().String()) - p = path.Join(p, digest.Encoded()) + p := path.Join(dir, "blobs", digest.Algorithm().String(), digest.Encoded()) buf, err = ioutil.ReadFile(p) @@ -319,19 +337,24 @@ func (is *ImageStore) GetImageManifest(repo string, reference string) ([]byte, s func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType string, body []byte) (string, error) { if err := is.InitRepo(repo); err != nil { + is.log.Debug().Err(err).Msg("init repo") return "", err } if mediaType != ispec.MediaTypeImageManifest { + is.log.Debug().Interface("actual", mediaType). + Interface("expected", ispec.MediaTypeImageManifest).Msg("bad manifest media type") return "", errors.ErrBadManifest } if len(body) == 0 { + is.log.Debug().Int("len", len(body)).Msg("invalid body length") return "", errors.ErrBadManifest } var m ispec.Manifest if err := json.Unmarshal(body, &m); err != nil { + is.log.Error().Err(err).Msg("unable to unmarshal JSON") return "", errors.ErrBadManifest } @@ -345,6 +368,7 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType blobPath := is.BlobPath(repo, digest) if _, err := os.Stat(blobPath); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to find blob") return digest.String(), errors.ErrBlobNotFound } } @@ -418,13 +442,12 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType } // write manifest to "blobs" - dir = path.Join(is.rootDir, repo) - dir = path.Join(dir, "blobs") - dir = path.Join(dir, mDigest.Algorithm().String()) - _ = os.MkdirAll(dir, 0755) + dir = path.Join(is.rootDir, repo, "blobs", mDigest.Algorithm().String()) + ensureDir(dir, is.log) file := path.Join(dir, mDigest.Encoded()) if err := ioutil.WriteFile(file, body, 0644); err != nil { + is.log.Error().Err(err).Str("file", file).Msg("unable to write") return "", err } @@ -435,10 +458,12 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType buf, err = json.Marshal(index) if err != nil { + is.log.Error().Err(err).Str("file", file).Msg("unable to marshal JSON") return "", err } if err := ioutil.WriteFile(file, buf, 0644); err != nil { + is.log.Error().Err(err).Str("file", file).Msg("unable to write") return "", err } @@ -530,9 +555,7 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error { return err } - p := path.Join(dir, "blobs") - p = path.Join(p, digest.Algorithm().String()) - p = path.Join(p, digest.Encoded()) + p := path.Join(dir, "blobs", digest.Algorithm().String(), digest.Encoded()) _ = os.Remove(p) @@ -542,8 +565,7 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error { // BlobUploadPath returns the upload path for a blob in this store. func (is *ImageStore) BlobUploadPath(repo string, uuid string) string { dir := path.Join(is.rootDir, repo) - blobUploadPath := path.Join(dir, BlobUploadDir) - blobUploadPath = path.Join(blobUploadPath, uuid) + blobUploadPath := path.Join(dir, BlobUploadDir, uuid) return blobUploadPath } @@ -711,14 +733,17 @@ func (is *ImageStore) FinishBlobUpload(repo string, uuid string, body io.Reader, return errors.ErrBadBlobDigest } - dir := path.Join(is.rootDir, repo) - dir = path.Join(dir, "blobs") - dir = path.Join(dir, dstDigest.Algorithm().String()) - _ = os.MkdirAll(dir, 0755) + dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) + ensureDir(dir, is.log) dst := is.BlobPath(repo, dstDigest) - // move the blob from uploads to final dest - _ = os.Rename(src, dst) + if is.cache != nil { + if err := is.DedupeBlob(src, dstDigest, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to dedupe blob") + return err + } + } return err } @@ -767,34 +792,80 @@ func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, digest string) return "", -1, errors.ErrBadBlobDigest } - dir := path.Join(is.rootDir, repo) - dir = path.Join(dir, "blobs") - dir = path.Join(dir, dstDigest.Algorithm().String()) - _ = os.MkdirAll(dir, 0755) + dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) + ensureDir(dir, is.log) dst := is.BlobPath(repo, dstDigest) - // move the blob from uploads to final dest - _ = os.Rename(src, dst) + if is.cache != nil { + if err := is.DedupeBlob(src, dstDigest, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to dedupe blob") + return "", -1, err + } + } return uuid, n, err } +// nolint (interfacer) +func (is *ImageStore) DedupeBlob(src string, dstDigest godigest.Digest, dst string) error { + dstRecord, err := is.cache.GetBlob(dstDigest.String()) + if err != nil && err != errors.ErrCacheMiss { + is.log.Error().Err(err).Str("blobPath", dst).Msg("unable to lookup blob record") + return err + } + + if dstRecord == "" { + if err := is.cache.PutBlob(dstDigest.String(), dst); err != nil { + is.log.Error().Err(err).Str("blobPath", dst).Msg("unable to insert blob record") + return err + } + + // move the blob from uploads to final dest + if err := os.Rename(src, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dst", dst).Msg("unable to rename blob") + return err + } + } else { + dstRecordFi, err := os.Stat(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("unable to stat") + return err + } + dstFi, err := os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("unable to stat") + return err + } + if !os.SameFile(dstFi, dstRecordFi) { + if err := os.Link(dstRecord, dst); err != nil { + is.log.Error().Err(err).Str("blobPath", dst).Str("link", dstRecord).Msg("unable to hard link") + return err + } + } + if err := os.Remove(src); err != nil { + is.log.Error().Err(err).Str("src", src).Msg("uname to remove blob") + return err + } + } + + return nil +} + // DeleteBlobUpload deletes an existing blob upload that is currently in progress. func (is *ImageStore) DeleteBlobUpload(repo string, uuid string) error { blobUploadPath := is.BlobUploadPath(repo, uuid) - _ = os.Remove(blobUploadPath) + if err := os.Remove(blobUploadPath); err != nil { + is.log.Error().Err(err).Str("blobUploadPath", blobUploadPath).Msg("error deleting blob upload") + return err + } return nil } // BlobPath returns the repository path of a blob. func (is *ImageStore) BlobPath(repo string, digest godigest.Digest) string { - dir := path.Join(is.rootDir, repo) - blobPath := path.Join(dir, "blobs") - blobPath = path.Join(blobPath, digest.Algorithm().String()) - blobPath = path.Join(blobPath, digest.Encoded()) - - return blobPath + return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded()) } // CheckBlob verifies a blob and returns true if the blob is correct. @@ -860,7 +931,17 @@ func (is *ImageStore) DeleteBlob(repo string, digest string) error { return errors.ErrBlobNotFound } - _ = os.Remove(blobPath) + if is.cache != nil { + if err := is.cache.DeleteBlob(digest, blobPath); err != nil { + is.log.Error().Err(err).Str("digest", digest).Str("blobPath", blobPath).Msg("unable to remove blob path from cache") + return err + } + } + + if err := os.Remove(blobPath); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") + return err + } return nil } @@ -888,8 +969,8 @@ func dirExists(d string) bool { return true } -func ensureDir(dir string) { +func ensureDir(dir string, log zerolog.Logger) { if err := os.MkdirAll(dir, 0755); err != nil { - panic(err) + log.Panic().Err(err).Str("dir", dir).Msg("unable to create dir") } } diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 6d017fc5..4fcb5453 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -6,6 +6,9 @@ import ( "encoding/json" "io/ioutil" "os" + "path" + "strings" + "sync" "testing" "github.com/anuvu/zot/pkg/log" @@ -86,13 +89,103 @@ func TestAPIs(t *testing.T) { So(err, ShouldBeNil) So(b, ShouldBeGreaterThanOrEqualTo, 0) - content := []byte("test-data") + content := []byte("test-data1") buf := bytes.NewBuffer(content) l := buf.Len() d := godigest.FromBytes(content) b, err = il.PutBlobChunk("test", v, 0, int64(l), buf) So(err, ShouldBeNil) So(b, ShouldEqual, l) + blobDigest := d + + err = il.FinishBlobUpload("test", v, buf, d.String()) + So(err, ShouldBeNil) + So(b, ShouldEqual, l) + + _, _, err = il.CheckBlob("test", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + + _, _, err = il.GetBlob("test", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + + m := ispec.Manifest{} + m.SchemaVersion = 2 + mb, _ := json.Marshal(m) + + Convey("Bad image manifest", func() { + _, err = il.PutImageManifest("test", d.String(), ispec.MediaTypeImageManifest, mb) + So(err, ShouldNotBeNil) + + _, _, _, err = il.GetImageManifest("test", d.String()) + So(err, ShouldNotBeNil) + }) + + Convey("Good image manifest", func() { + m := ispec.Manifest{ + Config: ispec.Descriptor{ + Digest: d, + Size: int64(l), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: d, + Size: int64(l), + }, + }, + } + m.SchemaVersion = 2 + mb, _ = json.Marshal(m) + d := godigest.FromBytes(mb) + _, err = il.PutImageManifest("test", d.String(), ispec.MediaTypeImageManifest, mb) + So(err, ShouldBeNil) + + _, _, _, err = il.GetImageManifest("test", d.String()) + So(err, ShouldBeNil) + + err = il.DeleteImageManifest("test", "1.0") + So(err, ShouldNotBeNil) + + err = il.DeleteBlob("test", blobDigest.String()) + So(err, ShouldBeNil) + + err = il.DeleteImageManifest("test", d.String()) + So(err, ShouldBeNil) + + _, _, _, err = il.GetImageManifest("test", d.String()) + So(err, ShouldNotBeNil) + }) + }) + + err = il.DeleteBlobUpload("test", v) + So(err, ShouldNotBeNil) + }) + + Convey("New blob upload streamed", func() { + v, err := il.NewBlobUpload("test") + So(err, ShouldBeNil) + So(v, ShouldNotBeEmpty) + + Convey("Get blob upload", func() { + b, err := il.GetBlobUpload("test", "invalid") + So(err, ShouldNotBeNil) + So(b, ShouldEqual, -1) + + b, err = il.GetBlobUpload("test", v) + So(err, ShouldBeNil) + So(b, ShouldBeGreaterThanOrEqualTo, 0) + + b, err = il.BlobUploadInfo("test", v) + So(err, ShouldBeNil) + So(b, ShouldBeGreaterThanOrEqualTo, 0) + + content := []byte("test-data2") + buf := bytes.NewBuffer(content) + l := buf.Len() + d := godigest.FromBytes(content) + b, err = il.PutBlobChunkStreamed("test", v, buf) + So(err, ShouldBeNil) + So(b, ShouldEqual, l) err = il.FinishBlobUpload("test", v, buf, d.String()) So(err, ShouldBeNil) @@ -151,7 +244,257 @@ func TestAPIs(t *testing.T) { }) err = il.DeleteBlobUpload("test", v) + So(err, ShouldNotBeNil) + }) + + Convey("Dedupe", func() { + blobDigest1 := "" + blobDigest2 := "" + + // manifest1 + v, err := il.NewBlobUpload("dedupe1") So(err, ShouldBeNil) + So(v, ShouldNotBeEmpty) + + content := []byte("test-data3") + buf := bytes.NewBuffer(content) + l := buf.Len() + d := godigest.FromBytes(content) + b, err := il.PutBlobChunkStreamed("dedupe1", v, buf) + So(err, ShouldBeNil) + So(b, ShouldEqual, l) + blobDigest1 = strings.Split(d.String(), ":")[1] + So(blobDigest1, ShouldNotBeEmpty) + + err = il.FinishBlobUpload("dedupe1", v, buf, d.String()) + So(err, ShouldBeNil) + So(b, ShouldEqual, l) + + _, _, err = il.CheckBlob("dedupe1", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + + _, _, err = il.GetBlob("dedupe1", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + + m := ispec.Manifest{} + m.SchemaVersion = 2 + m = ispec.Manifest{ + Config: ispec.Descriptor{ + Digest: d, + Size: int64(l), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: d, + Size: int64(l), + }, + }, + } + m.SchemaVersion = 2 + mb, _ := json.Marshal(m) + d = godigest.FromBytes(mb) + _, err = il.PutImageManifest("dedupe1", d.String(), ispec.MediaTypeImageManifest, mb) + So(err, ShouldBeNil) + + _, _, _, err = il.GetImageManifest("dedupe1", d.String()) + So(err, ShouldBeNil) + + // manifest2 + v, err = il.NewBlobUpload("dedupe2") + So(err, ShouldBeNil) + So(v, ShouldNotBeEmpty) + + content = []byte("test-data3") + buf = bytes.NewBuffer(content) + l = buf.Len() + d = godigest.FromBytes(content) + b, err = il.PutBlobChunkStreamed("dedupe2", v, buf) + So(err, ShouldBeNil) + So(b, ShouldEqual, l) + blobDigest2 = strings.Split(d.String(), ":")[1] + So(blobDigest2, ShouldNotBeEmpty) + + err = il.FinishBlobUpload("dedupe2", v, buf, d.String()) + So(err, ShouldBeNil) + So(b, ShouldEqual, l) + + _, _, err = il.CheckBlob("dedupe2", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + + _, _, err = il.GetBlob("dedupe2", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + + m = ispec.Manifest{} + m.SchemaVersion = 2 + m = ispec.Manifest{ + Config: ispec.Descriptor{ + Digest: d, + Size: int64(l), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: d, + Size: int64(l), + }, + }, + } + m.SchemaVersion = 2 + mb, _ = json.Marshal(m) + d = godigest.FromBytes(mb) + _, err = il.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest, mb) + So(err, ShouldBeNil) + + _, _, _, err = il.GetImageManifest("dedupe2", d.String()) + So(err, ShouldBeNil) + + // verify that dedupe with hard links happened + fi1, err := os.Stat(path.Join(dir, "dedupe2", "blobs", "sha256", blobDigest1)) + So(err, ShouldBeNil) + fi2, err := os.Stat(path.Join(dir, "dedupe2", "blobs", "sha256", blobDigest2)) + So(err, ShouldBeNil) + So(os.SameFile(fi1, fi2), ShouldBeTrue) + }) + + Convey("Locks", func() { + // in parallel, a mix of read and write locks - mainly for coverage + var wg sync.WaitGroup + for i := 0; i < 1000; i++ { + wg.Add(2) + go func() { + defer wg.Done() + il.Lock() + func() {}() + il.Unlock() + }() + go func() { + defer wg.Done() + il.RLock() + func() {}() + il.RUnlock() + }() + } + wg.Wait() }) }) } + +func TestDedupe(t *testing.T) { + Convey("Dedupe", t, func(c C) { + Convey("Nil ImageStore", func() { + is := &storage.ImageStore{} + So(func() { _ = is.DedupeBlob("", "", "") }, ShouldPanic) + }) + + Convey("Valid ImageStore", func() { + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + + is := storage.NewImageStore(dir, log.Logger{Logger: zerolog.New(os.Stdout)}) + + So(is.DedupeBlob("", "", ""), ShouldNotBeNil) + }) + }) +} + +func TestNegativeCases(t *testing.T) { + Convey("Invalid root dir", t, func(c C) { + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + os.RemoveAll(dir) + + So(storage.NewImageStore(dir, log.Logger{Logger: zerolog.New(os.Stdout)}), ShouldNotBeNil) + So(storage.NewImageStore("/deadBEEF", log.Logger{Logger: zerolog.New(os.Stdout)}), ShouldBeNil) + }) + + Convey("Invalid init repo", t, func(c C) { + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + il := storage.NewImageStore(dir, log.Logger{Logger: zerolog.New(os.Stdout)}) + err = os.Chmod(dir, 0000) // remove all perms + So(err, ShouldBeNil) + So(func() { _ = il.InitRepo("test") }, ShouldPanic) + }) + + Convey("Invalid validate repo", t, func(c C) { + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + il := storage.NewImageStore(dir, log.Logger{Logger: zerolog.New(os.Stdout)}) + So(il, ShouldNotBeNil) + So(il.InitRepo("test"), ShouldBeNil) + files, err := ioutil.ReadDir(path.Join(dir, "test")) + So(err, ShouldBeNil) + for _, f := range files { + os.Remove(path.Join(dir, "test", f.Name())) + } + _, err = il.ValidateRepo("test") + So(err, ShouldNotBeNil) + os.RemoveAll(path.Join(dir, "test")) + _, err = il.ValidateRepo("test") + So(err, ShouldNotBeNil) + err = os.Chmod(dir, 0000) // remove all perms + So(err, ShouldBeNil) + So(func() { _, _ = il.ValidateRepo("test") }, ShouldPanic) + os.RemoveAll(dir) + _, err = il.GetRepositories() + So(err, ShouldNotBeNil) + }) + + Convey("Invalid get image tags", t, func(c C) { + il := &storage.ImageStore{} + _, err := il.GetImageTags("test") + So(err, ShouldNotBeNil) + + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + il = storage.NewImageStore(dir, log.Logger{Logger: zerolog.New(os.Stdout)}) + So(il, ShouldNotBeNil) + So(il.InitRepo("test"), ShouldBeNil) + So(os.Remove(path.Join(dir, "test", "index.json")), ShouldBeNil) + _, err = il.GetImageTags("test") + So(err, ShouldNotBeNil) + So(os.RemoveAll(path.Join(dir, "test")), ShouldBeNil) + So(il.InitRepo("test"), ShouldBeNil) + So(ioutil.WriteFile(path.Join(dir, "test", "index.json"), []byte{}, 0755), ShouldBeNil) + _, err = il.GetImageTags("test") + So(err, ShouldNotBeNil) + }) + + Convey("Invalid get image manifest", t, func(c C) { + il := &storage.ImageStore{} + _, _, _, err := il.GetImageManifest("test", "") + So(err, ShouldNotBeNil) + + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + il = storage.NewImageStore(dir, log.Logger{Logger: zerolog.New(os.Stdout)}) + So(il, ShouldNotBeNil) + So(il.InitRepo("test"), ShouldBeNil) + So(os.Remove(path.Join(dir, "test", "index.json")), ShouldBeNil) + _, _, _, err = il.GetImageManifest("test", "") + So(err, ShouldNotBeNil) + So(os.RemoveAll(path.Join(dir, "test")), ShouldBeNil) + So(il.InitRepo("test"), ShouldBeNil) + So(ioutil.WriteFile(path.Join(dir, "test", "index.json"), []byte{}, 0755), ShouldBeNil) + _, _, _, err = il.GetImageManifest("test", "") + So(err, ShouldNotBeNil) + }) +}