From 25f5a45296a02d6c80bbae445b2266fa609754c0 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Mon, 17 Feb 2020 13:57:15 -0800 Subject: [PATCH] dedupe: use hard links to dedupe blobs As the number of repos and layers increases, the greater the probability that layers are duplicated. We dedupe using hard links when content is the same. This is intended to be purely a storage layer optimization. Access control when available is orthogonal this optimization. Add a durable cache to help speed up layer lookups. Update README. Add more unit tests. --- README.md | 8 +- WORKSPACE | 22 ++- errors/errors.go | 3 + go.mod | 2 + go.sum | 10 +- pkg/api/controller.go | 9 +- pkg/api/routes.go | 14 +- pkg/compliance/v1_0_0/check.go | 22 +-- pkg/storage/BUILD.bazel | 12 +- pkg/storage/cache.go | 163 ++++++++++++++++ pkg/storage/cache_test.go | 52 +++++ pkg/storage/storage.go | 193 ++++++++++++------ pkg/storage/storage_test.go | 345 ++++++++++++++++++++++++++++++++- 13 files changed, 767 insertions(+), 88 deletions(-) create mode 100644 pkg/storage/cache.go create mode 100644 pkg/storage/cache_test.go diff --git a/README.md b/README.md index 12cbf3da..4cf23e4f 100644 --- a/README.md +++ b/README.md @@ -7,8 +7,14 @@ * Uses [OCI storage layout](https://github.com/opencontainers/image-spec/blob/master/image-layout.md) for storage layout * Currently suitable for on-prem deployments (e.g. colocated with Kubernetes) * TLS support -* Authentication via TLS mutual authentication and HTTP *BASIC* (local _htpasswd_ and LDAP) +* Authentication via: + * TLS mutual authentication + * HTTP *Basic* (local _htpasswd_ and LDAP) + * HTTP *Bearer* token * Doesn't require _root_ privileges +* Storage optimizations: + * Automatic garbage collection of orphaned blobs + * Layer deduplication using hard links when content is identical * Swagger based documentation * Released under Apache 2.0 License * ```go get -u github.com/anuvu/zot/cmd/zot``` diff --git a/WORKSPACE b/WORKSPACE index c26fbd20..c93fc632 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -872,8 +872,8 @@ go_repository( go_repository( name = "io_etcd_go_bbolt", importpath = "go.etcd.io/bbolt", - sum = "h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=", - version = "v1.3.2", + sum = "h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg=", + version = "v1.3.4", ) go_repository( @@ -935,8 +935,8 @@ go_repository( go_repository( name = "org_golang_x_sys", importpath = "golang.org/x/sys", - sum = "h1:wYqz/tQaWUgGKyx+B/rssSE6wkIKdY5Ee6ryOmzarIg=", - version = "v0.0.0-20190913121621-c3b328c6e5a7", + sum = "h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=", + version = "v0.0.0-20200223170610-d5e6a3e2c0ae", ) go_repository( @@ -1198,6 +1198,20 @@ go_repository( version = "v1.51.0", ) +go_repository( + name = "com_github_boltdb_bolt", + importpath = "github.com/boltdb/bolt", + sum = "h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=", + version = "v1.3.1", +) + +go_repository( + name = "com_github_etcd_io_bbolt", + importpath = "github.com/etcd-io/bbolt", + sum = "h1:gSJmxrs37LgTqR/oyJBWok6k6SvXEUerFTbltIhXkBM=", + version = "v1.3.3", +) + go_repository( name = "com_github_apex_log", importpath = "github.com/apex/log", diff --git a/errors/errors.go b/errors/errors.go index 08be4653..3291073a 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -21,4 +21,7 @@ var ( ErrLDAPEmptyPassphrase = errors.New("ldap: empty passphrase") ErrLDAPBadConn = errors.New("ldap: bad connection") ErrLDAPConfig = errors.New("config: invalid LDAP configuration") + ErrCacheRootBucket = errors.New("cache: unable to create/update root bucket") + ErrCacheNoBucket = errors.New("cache: unable to find bucket") + ErrCacheMiss = errors.New("cache: miss") ) diff --git a/go.mod b/go.mod index b4c144fb..97f05980 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,8 @@ require ( github.com/spf13/viper v1.6.1 github.com/swaggo/http-swagger v0.0.0-20190614090009-c2865af9083e github.com/swaggo/swag v1.6.3 + go.etcd.io/bbolt v1.3.4 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 + golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae // indirect gopkg.in/resty.v1 v1.12.0 ) diff --git a/go.sum b/go.sum index 7b7c2fb4..df263d62 100644 --- a/go.sum +++ b/go.sum @@ -160,6 +160,7 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nmcclain/asn1-ber v0.0.0-20170104154839-2661553a0484 h1:D9EvfGQvlkKaDr2CRKN++7HbSXbefUNDrPq60T+g24s= @@ -171,8 +172,6 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/openSUSE/umoci v0.4.5 h1:MZgFLy5Jl3VKe5uEOU9c25FoySbx3vUXeXLw4Jf6aRs= -github.com/openSUSE/umoci v0.4.5/go.mod h1:3p4KA5nwyY65lVmQZxv7tm0YEylJ+t1fY91ORsVXv58= github.com/openSUSE/umoci v0.4.6-0.20200320140503-9aa268eeb258 h1:/8Yu54FufyHHQgIZ/wLy+BLQyzk0gbOG24xf5suWOOI= github.com/openSUSE/umoci v0.4.6-0.20200320140503-9aa268eeb258/go.mod h1:3p4KA5nwyY65lVmQZxv7tm0YEylJ+t1fY91ORsVXv58= github.com/opencontainers/distribution-spec v1.0.0-rc0 h1:xMzwhweo1gjvEo74mQjGTLau0TD3ACyTEC1310NbuSQ= @@ -283,6 +282,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= +go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -329,8 +330,10 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae h1:xiXzMMEQdQcric9hXtr1QU98MHunKK7OTtsoU6bYWs4= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190913121621-c3b328c6e5a7 h1:wYqz/tQaWUgGKyx+B/rssSE6wkIKdY5Ee6ryOmzarIg= golang.org/x/sys v0.0.0-20190913121621-c3b328c6e5a7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -355,6 +358,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= diff --git a/pkg/api/controller.go b/pkg/api/controller.go index 6098ed92..04bf570d 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "net" "net/http" + "os" "github.com/anuvu/zot/errors" "github.com/anuvu/zot/pkg/log" @@ -41,12 +42,16 @@ func (c *Controller) Run() error { engine.Use(log.SessionLogger(c.Log), handlers.RecoveryHandler(handlers.RecoveryLogger(c.Log), handlers.PrintRecoveryStack(false))) + c.ImageStore = storage.NewImageStore(c.Config.Storage.RootDirectory, c.Log) + if c.ImageStore == nil { + // we can't proceed without at least a image store + os.Exit(1) + } + c.Router = engine c.Router.UseEncodedPath() _ = NewRouteHandler(c) - c.ImageStore = storage.NewImageStore(c.Config.Storage.RootDirectory, c.Log) - addr := fmt.Sprintf("%s:%s", c.Config.HTTP.Address, c.Config.HTTP.Port) server := &http.Server{Addr: addr, Handler: c.Router} c.Server = server diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 718b21d4..3f79607c 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -20,7 +20,6 @@ import ( "sort" "strconv" "strings" - "sync" _ "github.com/anuvu/zot/docs" // nolint (golint) - as required by swaggo "github.com/anuvu/zot/errors" @@ -41,12 +40,11 @@ const ( ) type RouteHandler struct { - c *Controller - blobLock sync.RWMutex + c *Controller } func NewRouteHandler(c *Controller) *RouteHandler { - rh := &RouteHandler{c: c, blobLock: sync.RWMutex{}} + rh := &RouteHandler{c: c} rh.SetupRoutes() return rh @@ -56,9 +54,9 @@ func NewRouteHandler(c *Controller) *RouteHandler { func (rh *RouteHandler) blobRLockWrapper(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { - rh.blobLock.RLock() + rh.c.ImageStore.RLock() f(w, r) - rh.blobLock.RUnlock() + rh.c.ImageStore.RUnlock() } } @@ -66,9 +64,9 @@ func (rh *RouteHandler) blobRLockWrapper(f func(w http.ResponseWriter, func (rh *RouteHandler) blobLockWrapper(f func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) { - rh.blobLock.Lock() + rh.c.ImageStore.Lock() f(w, r) - rh.blobLock.Unlock() + rh.c.ImageStore.Unlock() } } diff --git a/pkg/compliance/v1_0_0/check.go b/pkg/compliance/v1_0_0/check.go index c5f0dea2..3b1df400 100644 --- a/pkg/compliance/v1_0_0/check.go +++ b/pkg/compliance/v1_0_0/check.go @@ -135,7 +135,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { } // without a "?digest=<>" should fail - content := []byte("this is a blob") + content := []byte("this is a blob1") digest := godigest.FromBytes(content) So(digest, ShouldNotBeNil) resp, err = resty.R().Put(loc) @@ -172,7 +172,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { Convey("Monolithic blob upload with body", func() { Print("\nMonolithic blob upload") // create content - content := []byte("this is a blob") + content := []byte("this is a blob2") digest := godigest.FromBytes(content) So(digest, ShouldNotBeNil) // setting invalid URL params should fail @@ -228,7 +228,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { } // without a "?digest=<>" should fail - content := []byte("this is a blob") + content := []byte("this is a blob3") digest := godigest.FromBytes(content) So(digest, ShouldNotBeNil) resp, err = resty.R().Put(loc) @@ -271,7 +271,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(loc, ShouldNotBeEmpty) var buf bytes.Buffer - chunk1 := []byte("this is the first chunk") + chunk1 := []byte("this is the first chunk1") n, err := buf.Write(chunk1) So(n, ShouldEqual, len(chunk1)) So(err, ShouldBeNil) @@ -299,7 +299,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(resp.StatusCode(), ShouldEqual, 416) So(resp.String(), ShouldNotBeEmpty) - chunk2 := []byte("this is the second chunk") + chunk2 := []byte("this is the second chunk1") n, err = buf.Write(chunk2) So(n, ShouldEqual, len(chunk2)) So(err, ShouldBeNil) @@ -339,7 +339,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(loc, ShouldNotBeEmpty) var buf bytes.Buffer - chunk1 := []byte("this is the first chunk") + chunk1 := []byte("this is the first chunk2") n, err := buf.Write(chunk1) So(n, ShouldEqual, len(chunk1)) So(err, ShouldBeNil) @@ -367,7 +367,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(resp.StatusCode(), ShouldEqual, 416) So(resp.String(), ShouldNotBeEmpty) - chunk2 := []byte("this is the second chunk") + chunk2 := []byte("this is the second chunk2") n, err = buf.Write(chunk2) So(n, ShouldEqual, len(chunk2)) So(err, ShouldBeNil) @@ -422,7 +422,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { loc := Location(baseURL, resp) So(loc, ShouldNotBeEmpty) - content := []byte("this is a blob") + content := []byte("this is a blob4") digest := godigest.FromBytes(content) So(digest, ShouldNotBeNil) // monolithic blob upload @@ -461,7 +461,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { resp, err = resty.R().Get(loc) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 204) - content := []byte("this is a blob") + content := []byte("this is a blob5") digest := godigest.FromBytes(content) So(digest, ShouldNotBeNil) // monolithic blob upload: success @@ -507,7 +507,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { So(d, ShouldNotBeEmpty) So(d, ShouldEqual, digest.String()) - content = []byte("this is a blob") + content = []byte("this is a blob5") digest = godigest.FromBytes(content) So(digest, ShouldNotBeNil) // create a manifest with same blob but a different tag @@ -611,7 +611,7 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) { resp, err = resty.R().Get(loc) So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, 204) - content := []byte("this is a blob") + content := []byte("this is a blob7") digest := godigest.FromBytes(content) So(digest, ShouldNotBeNil) // monolithic blob upload: success diff --git a/pkg/storage/BUILD.bazel b/pkg/storage/BUILD.bazel index 3efaa617..3ca8b38a 100644 --- a/pkg/storage/BUILD.bazel +++ b/pkg/storage/BUILD.bazel @@ -2,7 +2,10 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "go_default_library", - srcs = ["storage.go"], + srcs = [ + "cache.go", + "storage.go", + ], importpath = "github.com/anuvu/zot/pkg/storage", visibility = ["//visibility:public"], deps = [ @@ -13,16 +16,21 @@ go_library( "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library", "@com_github_opensuse_umoci//:go_default_library", "@com_github_rs_zerolog//:go_default_library", + "@io_etcd_go_bbolt//:go_default_library", ], ) go_test( name = "go_default_test", timeout = "short", - srcs = ["storage_test.go"], + srcs = [ + "cache_test.go", + "storage_test.go", + ], embed = [":go_default_library"], race = "on", deps = [ + "//errors:go_default_library", "//pkg/log:go_default_library", "@com_github_opencontainers_go_digest//:go_default_library", "@com_github_opencontainers_image_spec//specs-go/v1:go_default_library", diff --git a/pkg/storage/cache.go b/pkg/storage/cache.go new file mode 100644 index 00000000..5c7a0758 --- /dev/null +++ b/pkg/storage/cache.go @@ -0,0 +1,163 @@ +package storage + +import ( + "path" + "strings" + + "github.com/anuvu/zot/errors" + zlog "github.com/anuvu/zot/pkg/log" + "go.etcd.io/bbolt" +) + +const ( + BlobsCache = "blobs" +) + +type Cache struct { + db *bbolt.DB + log zlog.Logger +} + +// Blob is a blob record +type Blob struct { + Path string +} + +func NewCache(rootDir string, name string, log zlog.Logger) *Cache { + dbPath := path.Join(rootDir, name+".db") + db, err := bbolt.Open(dbPath, 0600, nil) + + if err != nil { + log.Error().Err(err).Str("dbPath", dbPath).Msg("unable to create cache db") + return nil + } + + if err := db.Update(func(tx *bbolt.Tx) error { + if _, err := tx.CreateBucketIfNotExists([]byte(BlobsCache)); err != nil { + // this is a serious failure + log.Error().Err(err).Str("dbPath", dbPath).Msg("unable to create a root bucket") + return err + } + return nil + }); err != nil { + // something went wrong + log.Error().Err(err).Msg("unable to create a cache") + return nil + } + + return &Cache{db: db, log: log} +} + +func (c *Cache) PutBlob(digest string, path string) error { + if err := c.db.Update(func(tx *bbolt.Tx) error { + root := tx.Bucket([]byte(BlobsCache)) + if root == nil { + // this is a serious failure + err := errors.ErrCacheRootBucket + c.log.Error().Err(err).Msg("unable to access root bucket") + return err + } + b, err := root.CreateBucketIfNotExists([]byte(digest)) + if err != nil { + // this is a serious failure + c.log.Error().Err(err).Str("bucket", digest).Msg("unable to create a bucket") + return err + } + if err := b.Put([]byte(path), nil); err != nil { + c.log.Error().Err(err).Str("bucket", digest).Str("value", path).Msg("unable to put record") + return err + } + return nil + }); err != nil { + return err + } + + return nil +} + +func (c *Cache) GetBlob(digest string) (string, error) { + var blobPath strings.Builder + + if err := c.db.View(func(tx *bbolt.Tx) error { + root := tx.Bucket([]byte(BlobsCache)) + if root == nil { + // this is a serious failure + err := errors.ErrCacheRootBucket + c.log.Error().Err(err).Msg("unable to access root bucket") + return err + } + + b := root.Bucket([]byte(digest)) + if b != nil { + // get first key + c := b.Cursor() + k, _ := c.First() + blobPath.WriteString(string(k)) + return nil + } + + return errors.ErrCacheMiss + }); err != nil { + return "", err + } + + if len(blobPath.String()) == 0 { + return "", nil + } + + return blobPath.String(), nil +} + +func (c *Cache) HasBlob(digest string, blob string) bool { + if err := c.db.View(func(tx *bbolt.Tx) error { + root := tx.Bucket([]byte(BlobsCache)) + if root == nil { + // this is a serious failure + err := errors.ErrCacheRootBucket + c.log.Error().Err(err).Msg("unable to access root bucket") + return err + } + + b := root.Bucket([]byte(digest)) + if b == nil { + return errors.ErrCacheMiss + } + if b.Get([]byte(blob)) == nil { + return errors.ErrCacheMiss + } + + return nil + }); err != nil { + return false + } + + return true +} + +func (c *Cache) DeleteBlob(digest string, path string) error { + if err := c.db.Update(func(tx *bbolt.Tx) error { + root := tx.Bucket([]byte(BlobsCache)) + if root == nil { + // this is a serious failure + err := errors.ErrCacheRootBucket + c.log.Error().Err(err).Msg("unable to access root bucket") + return err + } + + b := root.Bucket([]byte(digest)) + if b == nil { + return errors.ErrCacheMiss + } + + if err := b.Delete([]byte(path)); err != nil { + c.log.Error().Err(err).Str("digest", digest).Str("path", path).Msg("unable to delete") + return err + } + + return nil + }); err != nil { + return err + } + + return nil +} diff --git a/pkg/storage/cache_test.go b/pkg/storage/cache_test.go new file mode 100644 index 00000000..05ea6c61 --- /dev/null +++ b/pkg/storage/cache_test.go @@ -0,0 +1,52 @@ +package storage_test + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/anuvu/zot/errors" + "github.com/anuvu/zot/pkg/log" + "github.com/anuvu/zot/pkg/storage" + . "github.com/smartystreets/goconvey/convey" +) + +func TestCache(t *testing.T) { + Convey("Make a new cache", t, func() { + dir, err := ioutil.TempDir("", "cache_test") + So(err, ShouldBeNil) + So(dir, ShouldNotBeEmpty) + defer os.RemoveAll(dir) + + log := log.NewLogger("debug", "") + So(log, ShouldNotBeNil) + + So(storage.NewCache("/deadBEEF", "cache_test", log), ShouldBeNil) + + c := storage.NewCache(dir, "cache_test", log) + So(c, ShouldNotBeNil) + + v, err := c.GetBlob("key") + So(err, ShouldEqual, errors.ErrCacheMiss) + So(v, ShouldBeEmpty) + + b := c.HasBlob("key", "value") + So(b, ShouldBeFalse) + + err = c.PutBlob("key", "value") + So(err, ShouldBeNil) + + b = c.HasBlob("key", "value") + So(b, ShouldBeTrue) + + v, err = c.GetBlob("key") + So(err, ShouldBeNil) + So(v, ShouldNotBeEmpty) + + err = c.DeleteBlob("bogusKey", "bogusValue") + So(err, ShouldEqual, errors.ErrCacheMiss) + + err = c.DeleteBlob("key", "bogusValue") + So(err, ShouldBeNil) + }) +} diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 8e486565..6f64da95 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -37,24 +37,50 @@ type ImageStore struct { rootDir string lock *sync.RWMutex blobUploads map[string]BlobUpload + cache *Cache log zerolog.Logger } // NewImageStore returns a new image store backed by a file storage. func NewImageStore(rootDir string, log zlog.Logger) *ImageStore { - is := &ImageStore{rootDir: rootDir, + if _, err := os.Stat(rootDir); os.IsNotExist(err) { + if err := os.MkdirAll(rootDir, 0700); err != nil { + log.Error().Err(err).Str("rootDir", rootDir).Msg("unable to create root dir") + return nil + } + } + + is := &ImageStore{ + rootDir: rootDir, lock: &sync.RWMutex{}, blobUploads: make(map[string]BlobUpload), + cache: NewCache(rootDir, "cache", log), log: log.With().Caller().Logger(), } - if _, err := os.Stat(rootDir); os.IsNotExist(err) { - _ = os.MkdirAll(rootDir, 0700) - } - return is } +// RLock read-lock +func (is *ImageStore) RLock() { + is.lock.RLock() +} + +// RUnlock read-unlock +func (is *ImageStore) RUnlock() { + is.lock.RUnlock() +} + +// Lock write-lock +func (is *ImageStore) Lock() { + is.lock.Lock() +} + +// Unlock write-unlock +func (is *ImageStore) Unlock() { + is.lock.Unlock() +} + // InitRepo creates an image repository under this store. func (is *ImageStore) InitRepo(name string) error { repoDir := path.Join(is.rootDir, name) @@ -63,16 +89,10 @@ func (is *ImageStore) InitRepo(name string) error { return nil } - // create repo dir - ensureDir(repoDir) - // create "blobs" subdir - dir := path.Join(repoDir, "blobs") - ensureDir(dir) - + ensureDir(path.Join(repoDir, "blobs"), is.log) // create BlobUploadDir subdir - dir = path.Join(repoDir, BlobUploadDir) - ensureDir(dir) + ensureDir(path.Join(repoDir, BlobUploadDir), is.log) // "oci-layout" file - create if it doesn't exist ilPath := path.Join(repoDir, ispec.ImageLayoutFile) @@ -85,7 +105,8 @@ func (is *ImageStore) InitRepo(name string) error { } if err := ioutil.WriteFile(ilPath, buf, 0644); err != nil { - is.log.Panic().Err(err).Str("file", ilPath).Msg("unable to write file") + is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") + return err } } @@ -101,7 +122,8 @@ func (is *ImageStore) InitRepo(name string) error { } if err := ioutil.WriteFile(indexPath, buf, 0644); err != nil { - is.log.Panic().Err(err).Str("file", indexPath).Msg("unable to write file") + is.log.Error().Err(err).Str("file", indexPath).Msg("unable to write file") + return err } } @@ -111,8 +133,8 @@ func (is *ImageStore) InitRepo(name string) error { // ValidateRepo validates that the repository layout is complaint with the OCI repo layout. func (is *ImageStore) ValidateRepo(name string) (bool, error) { // https://github.com/opencontainers/image-spec/blob/master/image-layout.md#content - // at least, expect exactly 4 entries - ["blobs", "oci-layout", "index.json"] and BlobUploadDir - // in each image store + // at least, expect at least 3 entries - ["blobs", "oci-layout", "index.json"] + // and an additional/optional BlobUploadDir in each image store dir := path.Join(is.rootDir, name) if !dirExists(dir) { return false, errors.ErrRepoNotFound @@ -124,15 +146,14 @@ func (is *ImageStore) ValidateRepo(name string) (bool, error) { return false, errors.ErrRepoNotFound } - if len(files) != 4 { - return false, nil + if len(files) < 3 { + return false, errors.ErrRepoBadVersion } found := map[string]bool{ "blobs": false, ispec.ImageLayoutFile: false, "index.json": false, - BlobUploadDir: false, } for _, file := range files { @@ -195,7 +216,7 @@ func (is *ImageStore) GetRepositories() ([]string, error) { return nil } - is.log.Debug().Str("dir", path).Str("name", info.Name()).Msg("found image store") + //is.log.Debug().Str("dir", path).Str("name", info.Name()).Msg("found image store") stores = append(stores, rel) return nil @@ -212,7 +233,6 @@ func (is *ImageStore) GetImageTags(repo string) ([]string, error) { } buf, err := ioutil.ReadFile(path.Join(dir, "index.json")) - if err != nil { is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") return nil, errors.ErrRepoNotFound @@ -290,9 +310,7 @@ func (is *ImageStore) GetImageManifest(repo string, reference string) ([]byte, s return nil, "", "", errors.ErrManifestNotFound } - p := path.Join(dir, "blobs") - p = path.Join(p, digest.Algorithm().String()) - p = path.Join(p, digest.Encoded()) + p := path.Join(dir, "blobs", digest.Algorithm().String(), digest.Encoded()) buf, err = ioutil.ReadFile(p) @@ -319,19 +337,24 @@ func (is *ImageStore) GetImageManifest(repo string, reference string) ([]byte, s func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType string, body []byte) (string, error) { if err := is.InitRepo(repo); err != nil { + is.log.Debug().Err(err).Msg("init repo") return "", err } if mediaType != ispec.MediaTypeImageManifest { + is.log.Debug().Interface("actual", mediaType). + Interface("expected", ispec.MediaTypeImageManifest).Msg("bad manifest media type") return "", errors.ErrBadManifest } if len(body) == 0 { + is.log.Debug().Int("len", len(body)).Msg("invalid body length") return "", errors.ErrBadManifest } var m ispec.Manifest if err := json.Unmarshal(body, &m); err != nil { + is.log.Error().Err(err).Msg("unable to unmarshal JSON") return "", errors.ErrBadManifest } @@ -345,6 +368,7 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType blobPath := is.BlobPath(repo, digest) if _, err := os.Stat(blobPath); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to find blob") return digest.String(), errors.ErrBlobNotFound } } @@ -418,13 +442,12 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType } // write manifest to "blobs" - dir = path.Join(is.rootDir, repo) - dir = path.Join(dir, "blobs") - dir = path.Join(dir, mDigest.Algorithm().String()) - _ = os.MkdirAll(dir, 0755) + dir = path.Join(is.rootDir, repo, "blobs", mDigest.Algorithm().String()) + ensureDir(dir, is.log) file := path.Join(dir, mDigest.Encoded()) if err := ioutil.WriteFile(file, body, 0644); err != nil { + is.log.Error().Err(err).Str("file", file).Msg("unable to write") return "", err } @@ -435,10 +458,12 @@ func (is *ImageStore) PutImageManifest(repo string, reference string, mediaType buf, err = json.Marshal(index) if err != nil { + is.log.Error().Err(err).Str("file", file).Msg("unable to marshal JSON") return "", err } if err := ioutil.WriteFile(file, buf, 0644); err != nil { + is.log.Error().Err(err).Str("file", file).Msg("unable to write") return "", err } @@ -530,9 +555,7 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error { return err } - p := path.Join(dir, "blobs") - p = path.Join(p, digest.Algorithm().String()) - p = path.Join(p, digest.Encoded()) + p := path.Join(dir, "blobs", digest.Algorithm().String(), digest.Encoded()) _ = os.Remove(p) @@ -542,8 +565,7 @@ func (is *ImageStore) DeleteImageManifest(repo string, reference string) error { // BlobUploadPath returns the upload path for a blob in this store. func (is *ImageStore) BlobUploadPath(repo string, uuid string) string { dir := path.Join(is.rootDir, repo) - blobUploadPath := path.Join(dir, BlobUploadDir) - blobUploadPath = path.Join(blobUploadPath, uuid) + blobUploadPath := path.Join(dir, BlobUploadDir, uuid) return blobUploadPath } @@ -711,14 +733,17 @@ func (is *ImageStore) FinishBlobUpload(repo string, uuid string, body io.Reader, return errors.ErrBadBlobDigest } - dir := path.Join(is.rootDir, repo) - dir = path.Join(dir, "blobs") - dir = path.Join(dir, dstDigest.Algorithm().String()) - _ = os.MkdirAll(dir, 0755) + dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) + ensureDir(dir, is.log) dst := is.BlobPath(repo, dstDigest) - // move the blob from uploads to final dest - _ = os.Rename(src, dst) + if is.cache != nil { + if err := is.DedupeBlob(src, dstDigest, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to dedupe blob") + return err + } + } return err } @@ -767,34 +792,80 @@ func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, digest string) return "", -1, errors.ErrBadBlobDigest } - dir := path.Join(is.rootDir, repo) - dir = path.Join(dir, "blobs") - dir = path.Join(dir, dstDigest.Algorithm().String()) - _ = os.MkdirAll(dir, 0755) + dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) + ensureDir(dir, is.log) dst := is.BlobPath(repo, dstDigest) - // move the blob from uploads to final dest - _ = os.Rename(src, dst) + if is.cache != nil { + if err := is.DedupeBlob(src, dstDigest, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to dedupe blob") + return "", -1, err + } + } return uuid, n, err } +// nolint (interfacer) +func (is *ImageStore) DedupeBlob(src string, dstDigest godigest.Digest, dst string) error { + dstRecord, err := is.cache.GetBlob(dstDigest.String()) + if err != nil && err != errors.ErrCacheMiss { + is.log.Error().Err(err).Str("blobPath", dst).Msg("unable to lookup blob record") + return err + } + + if dstRecord == "" { + if err := is.cache.PutBlob(dstDigest.String(), dst); err != nil { + is.log.Error().Err(err).Str("blobPath", dst).Msg("unable to insert blob record") + return err + } + + // move the blob from uploads to final dest + if err := os.Rename(src, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dst", dst).Msg("unable to rename blob") + return err + } + } else { + dstRecordFi, err := os.Stat(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("unable to stat") + return err + } + dstFi, err := os.Stat(dst) + if err != nil && !os.IsNotExist(err) { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("unable to stat") + return err + } + if !os.SameFile(dstFi, dstRecordFi) { + if err := os.Link(dstRecord, dst); err != nil { + is.log.Error().Err(err).Str("blobPath", dst).Str("link", dstRecord).Msg("unable to hard link") + return err + } + } + if err := os.Remove(src); err != nil { + is.log.Error().Err(err).Str("src", src).Msg("uname to remove blob") + return err + } + } + + return nil +} + // DeleteBlobUpload deletes an existing blob upload that is currently in progress. func (is *ImageStore) DeleteBlobUpload(repo string, uuid string) error { blobUploadPath := is.BlobUploadPath(repo, uuid) - _ = os.Remove(blobUploadPath) + if err := os.Remove(blobUploadPath); err != nil { + is.log.Error().Err(err).Str("blobUploadPath", blobUploadPath).Msg("error deleting blob upload") + return err + } return nil } // BlobPath returns the repository path of a blob. func (is *ImageStore) BlobPath(repo string, digest godigest.Digest) string { - dir := path.Join(is.rootDir, repo) - blobPath := path.Join(dir, "blobs") - blobPath = path.Join(blobPath, digest.Algorithm().String()) - blobPath = path.Join(blobPath, digest.Encoded()) - - return blobPath + return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded()) } // CheckBlob verifies a blob and returns true if the blob is correct. @@ -860,7 +931,17 @@ func (is *ImageStore) DeleteBlob(repo string, digest string) error { return errors.ErrBlobNotFound } - _ = os.Remove(blobPath) + if is.cache != nil { + if err := is.cache.DeleteBlob(digest, blobPath); err != nil { + is.log.Error().Err(err).Str("digest", digest).Str("blobPath", blobPath).Msg("unable to remove blob path from cache") + return err + } + } + + if err := os.Remove(blobPath); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") + return err + } return nil } @@ -888,8 +969,8 @@ func dirExists(d string) bool { return true } -func ensureDir(dir string) { +func ensureDir(dir string, log zerolog.Logger) { if err := os.MkdirAll(dir, 0755); err != nil { - panic(err) + log.Panic().Err(err).Str("dir", dir).Msg("unable to create dir") } } diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 6d017fc5..4fcb5453 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -6,6 +6,9 @@ import ( "encoding/json" "io/ioutil" "os" + "path" + "strings" + "sync" "testing" "github.com/anuvu/zot/pkg/log" @@ -86,13 +89,103 @@ func TestAPIs(t *testing.T) { So(err, ShouldBeNil) So(b, ShouldBeGreaterThanOrEqualTo, 0) - content := []byte("test-data") + content := []byte("test-data1") buf := bytes.NewBuffer(content) l := buf.Len() d := godigest.FromBytes(content) b, err = il.PutBlobChunk("test", v, 0, int64(l), buf) So(err, ShouldBeNil) So(b, ShouldEqual, l) + blobDigest := d + + err = il.FinishBlobUpload("test", v, buf, d.String()) + So(err, ShouldBeNil) + So(b, ShouldEqual, l) + + _, _, err = il.CheckBlob("test", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + + _, _, err = il.GetBlob("test", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + + m := ispec.Manifest{} + m.SchemaVersion = 2 + mb, _ := json.Marshal(m) + + Convey("Bad image manifest", func() { + _, err = il.PutImageManifest("test", d.String(), ispec.MediaTypeImageManifest, mb) + So(err, ShouldNotBeNil) + + _, _, _, err = il.GetImageManifest("test", d.String()) + So(err, ShouldNotBeNil) + }) + + Convey("Good image manifest", func() { + m := ispec.Manifest{ + Config: ispec.Descriptor{ + Digest: d, + Size: int64(l), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: d, + Size: int64(l), + }, + }, + } + m.SchemaVersion = 2 + mb, _ = json.Marshal(m) + d := godigest.FromBytes(mb) + _, err = il.PutImageManifest("test", d.String(), ispec.MediaTypeImageManifest, mb) + So(err, ShouldBeNil) + + _, _, _, err = il.GetImageManifest("test", d.String()) + So(err, ShouldBeNil) + + err = il.DeleteImageManifest("test", "1.0") + So(err, ShouldNotBeNil) + + err = il.DeleteBlob("test", blobDigest.String()) + So(err, ShouldBeNil) + + err = il.DeleteImageManifest("test", d.String()) + So(err, ShouldBeNil) + + _, _, _, err = il.GetImageManifest("test", d.String()) + So(err, ShouldNotBeNil) + }) + }) + + err = il.DeleteBlobUpload("test", v) + So(err, ShouldNotBeNil) + }) + + Convey("New blob upload streamed", func() { + v, err := il.NewBlobUpload("test") + So(err, ShouldBeNil) + So(v, ShouldNotBeEmpty) + + Convey("Get blob upload", func() { + b, err := il.GetBlobUpload("test", "invalid") + So(err, ShouldNotBeNil) + So(b, ShouldEqual, -1) + + b, err = il.GetBlobUpload("test", v) + So(err, ShouldBeNil) + So(b, ShouldBeGreaterThanOrEqualTo, 0) + + b, err = il.BlobUploadInfo("test", v) + So(err, ShouldBeNil) + So(b, ShouldBeGreaterThanOrEqualTo, 0) + + content := []byte("test-data2") + buf := bytes.NewBuffer(content) + l := buf.Len() + d := godigest.FromBytes(content) + b, err = il.PutBlobChunkStreamed("test", v, buf) + So(err, ShouldBeNil) + So(b, ShouldEqual, l) err = il.FinishBlobUpload("test", v, buf, d.String()) So(err, ShouldBeNil) @@ -151,7 +244,257 @@ func TestAPIs(t *testing.T) { }) err = il.DeleteBlobUpload("test", v) + So(err, ShouldNotBeNil) + }) + + Convey("Dedupe", func() { + blobDigest1 := "" + blobDigest2 := "" + + // manifest1 + v, err := il.NewBlobUpload("dedupe1") So(err, ShouldBeNil) + So(v, ShouldNotBeEmpty) + + content := []byte("test-data3") + buf := bytes.NewBuffer(content) + l := buf.Len() + d := godigest.FromBytes(content) + b, err := il.PutBlobChunkStreamed("dedupe1", v, buf) + So(err, ShouldBeNil) + So(b, ShouldEqual, l) + blobDigest1 = strings.Split(d.String(), ":")[1] + So(blobDigest1, ShouldNotBeEmpty) + + err = il.FinishBlobUpload("dedupe1", v, buf, d.String()) + So(err, ShouldBeNil) + So(b, ShouldEqual, l) + + _, _, err = il.CheckBlob("dedupe1", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + + _, _, err = il.GetBlob("dedupe1", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + + m := ispec.Manifest{} + m.SchemaVersion = 2 + m = ispec.Manifest{ + Config: ispec.Descriptor{ + Digest: d, + Size: int64(l), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: d, + Size: int64(l), + }, + }, + } + m.SchemaVersion = 2 + mb, _ := json.Marshal(m) + d = godigest.FromBytes(mb) + _, err = il.PutImageManifest("dedupe1", d.String(), ispec.MediaTypeImageManifest, mb) + So(err, ShouldBeNil) + + _, _, _, err = il.GetImageManifest("dedupe1", d.String()) + So(err, ShouldBeNil) + + // manifest2 + v, err = il.NewBlobUpload("dedupe2") + So(err, ShouldBeNil) + So(v, ShouldNotBeEmpty) + + content = []byte("test-data3") + buf = bytes.NewBuffer(content) + l = buf.Len() + d = godigest.FromBytes(content) + b, err = il.PutBlobChunkStreamed("dedupe2", v, buf) + So(err, ShouldBeNil) + So(b, ShouldEqual, l) + blobDigest2 = strings.Split(d.String(), ":")[1] + So(blobDigest2, ShouldNotBeEmpty) + + err = il.FinishBlobUpload("dedupe2", v, buf, d.String()) + So(err, ShouldBeNil) + So(b, ShouldEqual, l) + + _, _, err = il.CheckBlob("dedupe2", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + + _, _, err = il.GetBlob("dedupe2", d.String(), "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + + m = ispec.Manifest{} + m.SchemaVersion = 2 + m = ispec.Manifest{ + Config: ispec.Descriptor{ + Digest: d, + Size: int64(l), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: d, + Size: int64(l), + }, + }, + } + m.SchemaVersion = 2 + mb, _ = json.Marshal(m) + d = godigest.FromBytes(mb) + _, err = il.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest, mb) + So(err, ShouldBeNil) + + _, _, _, err = il.GetImageManifest("dedupe2", d.String()) + So(err, ShouldBeNil) + + // verify that dedupe with hard links happened + fi1, err := os.Stat(path.Join(dir, "dedupe2", "blobs", "sha256", blobDigest1)) + So(err, ShouldBeNil) + fi2, err := os.Stat(path.Join(dir, "dedupe2", "blobs", "sha256", blobDigest2)) + So(err, ShouldBeNil) + So(os.SameFile(fi1, fi2), ShouldBeTrue) + }) + + Convey("Locks", func() { + // in parallel, a mix of read and write locks - mainly for coverage + var wg sync.WaitGroup + for i := 0; i < 1000; i++ { + wg.Add(2) + go func() { + defer wg.Done() + il.Lock() + func() {}() + il.Unlock() + }() + go func() { + defer wg.Done() + il.RLock() + func() {}() + il.RUnlock() + }() + } + wg.Wait() }) }) } + +func TestDedupe(t *testing.T) { + Convey("Dedupe", t, func(c C) { + Convey("Nil ImageStore", func() { + is := &storage.ImageStore{} + So(func() { _ = is.DedupeBlob("", "", "") }, ShouldPanic) + }) + + Convey("Valid ImageStore", func() { + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + + is := storage.NewImageStore(dir, log.Logger{Logger: zerolog.New(os.Stdout)}) + + So(is.DedupeBlob("", "", ""), ShouldNotBeNil) + }) + }) +} + +func TestNegativeCases(t *testing.T) { + Convey("Invalid root dir", t, func(c C) { + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + os.RemoveAll(dir) + + So(storage.NewImageStore(dir, log.Logger{Logger: zerolog.New(os.Stdout)}), ShouldNotBeNil) + So(storage.NewImageStore("/deadBEEF", log.Logger{Logger: zerolog.New(os.Stdout)}), ShouldBeNil) + }) + + Convey("Invalid init repo", t, func(c C) { + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + il := storage.NewImageStore(dir, log.Logger{Logger: zerolog.New(os.Stdout)}) + err = os.Chmod(dir, 0000) // remove all perms + So(err, ShouldBeNil) + So(func() { _ = il.InitRepo("test") }, ShouldPanic) + }) + + Convey("Invalid validate repo", t, func(c C) { + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + il := storage.NewImageStore(dir, log.Logger{Logger: zerolog.New(os.Stdout)}) + So(il, ShouldNotBeNil) + So(il.InitRepo("test"), ShouldBeNil) + files, err := ioutil.ReadDir(path.Join(dir, "test")) + So(err, ShouldBeNil) + for _, f := range files { + os.Remove(path.Join(dir, "test", f.Name())) + } + _, err = il.ValidateRepo("test") + So(err, ShouldNotBeNil) + os.RemoveAll(path.Join(dir, "test")) + _, err = il.ValidateRepo("test") + So(err, ShouldNotBeNil) + err = os.Chmod(dir, 0000) // remove all perms + So(err, ShouldBeNil) + So(func() { _, _ = il.ValidateRepo("test") }, ShouldPanic) + os.RemoveAll(dir) + _, err = il.GetRepositories() + So(err, ShouldNotBeNil) + }) + + Convey("Invalid get image tags", t, func(c C) { + il := &storage.ImageStore{} + _, err := il.GetImageTags("test") + So(err, ShouldNotBeNil) + + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + il = storage.NewImageStore(dir, log.Logger{Logger: zerolog.New(os.Stdout)}) + So(il, ShouldNotBeNil) + So(il.InitRepo("test"), ShouldBeNil) + So(os.Remove(path.Join(dir, "test", "index.json")), ShouldBeNil) + _, err = il.GetImageTags("test") + So(err, ShouldNotBeNil) + So(os.RemoveAll(path.Join(dir, "test")), ShouldBeNil) + So(il.InitRepo("test"), ShouldBeNil) + So(ioutil.WriteFile(path.Join(dir, "test", "index.json"), []byte{}, 0755), ShouldBeNil) + _, err = il.GetImageTags("test") + So(err, ShouldNotBeNil) + }) + + Convey("Invalid get image manifest", t, func(c C) { + il := &storage.ImageStore{} + _, _, _, err := il.GetImageManifest("test", "") + So(err, ShouldNotBeNil) + + dir, err := ioutil.TempDir("", "oci-repo-test") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + il = storage.NewImageStore(dir, log.Logger{Logger: zerolog.New(os.Stdout)}) + So(il, ShouldNotBeNil) + So(il.InitRepo("test"), ShouldBeNil) + So(os.Remove(path.Join(dir, "test", "index.json")), ShouldBeNil) + _, _, _, err = il.GetImageManifest("test", "") + So(err, ShouldNotBeNil) + So(os.RemoveAll(path.Join(dir, "test")), ShouldBeNil) + So(il.InitRepo("test"), ShouldBeNil) + So(ioutil.WriteFile(path.Join(dir, "test", "index.json"), []byte{}, 0755), ShouldBeNil) + _, _, _, err = il.GetImageManifest("test", "") + So(err, ShouldNotBeNil) + }) +}