config: support multiple storage locations

added support to point multiple storage locations in zot by running multiple instance of zot in background.

see examples/config-multiple.json for more info about config.

Closes #181

README.md

@@ -23,6 +23,7 @@ https://anuvu.github.io/zot/
 * Storage optimizations:
   * Automatic garbage collection of orphaned blobs
   * Layer deduplication using hard links when content is identical
+* Serve [multiple storage paths (and backends)](./examples/config-multiple.json) using a single zot server
 * Swagger based documentation
 * Single binary for _all_ the above features
 * Released under Apache 2.0 License

errors/errors.go

@@ -37,4 +37,6 @@ var (
 	ErrScanNotSupported        = errors.New("search: scanning of image media type not supported")
 	ErrCLITimeout              = errors.New("cli: Query timed out while waiting for results")
 	ErrDuplicateConfigName     = errors.New("cli: cli config name already added")
+	ErrInvalidRoute            = errors.New("routes: invalid route prefix")
+	ErrImgStoreNotFound        = errors.New("routes: image store not found corresponding to given route")
 )

examples/config-multiple-cve.json

@@ -0,0 +1,37 @@
+{
+    "version": "0.1.0-dev",
+    "storage": {
+        "rootDirectory": "/tmp/zot",
+        "dedupe": true,
+        "gc": true,
+        "subPaths": {
+            "/infra": {
+                "rootDirectory": "/tmp/zot1",
+                "dedupe": true
+            },
+            "/b": {
+                "rootDirectory": "/tmp/zot2",
+                "dedupe": true
+            },
+            "/c": {
+                "rootDirectory": "/tmp/zot3",
+                "dedupe": false
+            }
+        }
+    },
+    "http": {
+        "address": "127.0.0.1",
+        "port": "5000",
+        "ReadOnly": false
+    },
+    "log": {
+        "level": "debug"
+    },
+    "extensions": {
+        "search": {
+            "cve": {
+                "updateInterval": "24h"
+            }
+        }
+    }
+}

examples/config-multiple.json

@@ -0,0 +1,30 @@
+{
+    "version": "0.1.0-dev",
+    "storage": {
+        "rootDirectory": "/tmp/zot",
+        "dedupe": true,
+        "gc": true,
+        "subPaths": {
+            "/a": {
+                "rootDirectory": "/tmp/zot1",
+                "dedupe": true
+            },
+            "/b": {
+                "rootDirectory": "/tmp/zot2",
+                "dedupe": true
+            },
+            "/c": {
+                "rootDirectory": "/tmp/zot3",
+                "dedupe": false
+            }
+        }
+    },
+    "http": {
+        "address": "127.0.0.1",
+        "port": "5000",
+        "ReadOnly": false
+    },
+    "log": {
+        "level": "debug"
+    }
+}

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/aquasecurity/trivy v0.0.0-00010101000000-000000000000
 	github.com/briandowns/spinner v1.11.1
 	github.com/chartmuseum/auth v0.4.0
+	github.com/containers/storage v1.29.0
 	github.com/dustin/go-humanize v1.0.0
 	github.com/getlantern/deepcopy v0.0.0-20160317154340-7f45deb8130a
 	github.com/go-chi/chi v4.0.2+incompatible // indirect
@@ -17,7 +18,7 @@ require (
 	github.com/google/go-containerregistry v0.0.0-20200331213917-3d03ed9b1ca2
 	github.com/gorilla/handlers v1.4.2
 	github.com/gorilla/mux v1.7.4
-	github.com/json-iterator/go v1.1.9
+	github.com/json-iterator/go v1.1.10
 	github.com/mitchellh/mapstructure v1.1.2
 	github.com/nmcclain/asn1-ber v0.0.0-20170104154839-2661553a0484 // indirect
 	github.com/nmcclain/ldap v0.0.0-20191021200707-3b3b69a7e9e3
@@ -31,14 +32,14 @@ require (
 	github.com/smartystreets/goconvey v1.6.4
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/viper v1.6.1
-	github.com/stretchr/testify v1.6.1
+	github.com/stretchr/testify v1.7.0
 	github.com/swaggo/http-swagger v0.0.0-20190614090009-c2865af9083e
 	github.com/swaggo/swag v1.6.3
 	github.com/vektah/gqlparser/v2 v2.0.1
-	go.etcd.io/bbolt v1.3.4
+	go.etcd.io/bbolt v1.3.5
-	golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9
+	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0
 	gopkg.in/resty.v1 v1.12.0
-	gopkg.in/yaml.v2 v2.2.8
+	gopkg.in/yaml.v2 v2.4.0
 )
 
 replace github.com/aquasecurity/trivy => github.com/anuvu/trivy v0.9.2-0.20200731014147-c5f97b59c172

pkg/api/config.go

@@ -72,11 +72,18 @@ type LogConfig struct {
 	Output string
 }
 
+type GlobalStorageConfig struct {
+	RootDirectory string
+	Dedupe        bool
+	GC            bool
+	SubPaths      map[string]StorageConfig
+}
+
 type Config struct {
 	Version    string
 	Commit     string
 	BinaryType string
-	Storage    StorageConfig
+	Storage    GlobalStorageConfig
 	HTTP       HTTPConfig
 	Log        *LogConfig
 	Extensions *ext.ExtensionConfig
@@ -87,7 +94,7 @@ func NewConfig() *Config {
 		Version:    dspec.Version,
 		Commit:     Commit,
 		BinaryType: BinaryType,
-		Storage:    StorageConfig{GC: true, Dedupe: true},
+		Storage:    GlobalStorageConfig{GC: true, Dedupe: true},
 		HTTP:       HTTPConfig{Address: "127.0.0.1", Port: "8080"},
 		Log:        &LogConfig{Level: "debug"},
 	}

pkg/api/controller.go

@@ -7,7 +7,6 @@ import (
 	"io/ioutil"
 	"net"
 	"net/http"
-	"os"
 	"time"
 
 	"github.com/anuvu/zot/errors"
@@ -23,11 +22,11 @@ const (
 )
 
 type Controller struct {
-	Config     *Config
+	Config          *Config
-	Router     *mux.Router
+	Router          *mux.Router
-	ImageStore *storage.ImageStore
+	StoreController storage.StoreController
-	Log        log.Logger
+	Log             log.Logger
-	Server     *http.Server
+	Server          *http.Server
 }
 
 func NewController(config *Config) *Controller {
@@ -63,20 +62,69 @@ func (c *Controller) Run() error {
 		handlers.RecoveryHandler(handlers.RecoveryLogger(c.Log),
 			handlers.PrintRecoveryStack(false)))
 
-	c.ImageStore = storage.NewImageStore(c.Config.Storage.RootDirectory, c.Config.Storage.GC,
-		c.Config.Storage.Dedupe, c.Log)
-	if c.ImageStore == nil {
-		// we can't proceed without at least a image store
-		os.Exit(1)
-	}
-
-	// Enable extensions if extension config is provided
-	if c.Config != nil && c.Config.Extensions != nil {
-		ext.EnableExtensions(c.Config.Extensions, c.Log, c.Config.Storage.RootDirectory)
-	}
-
 	c.Router = engine
 	c.Router.UseEncodedPath()
+
+	c.StoreController = storage.StoreController{}
+
+	if c.Config.Storage.RootDirectory != "" {
+		if c.Config.Storage.Dedupe {
+			err := storage.ValidateHardLink(c.Config.Storage.RootDirectory)
+			if err != nil {
+				c.Log.Warn().Msg("input storage root directory filesystem does not supports hardlinking," +
+					"disabling dedupe functionality")
+
+				c.Config.Storage.Dedupe = false
+			}
+		}
+
+		defaultStore := storage.NewImageStore(c.Config.Storage.RootDirectory,
+			c.Config.Storage.GC, c.Config.Storage.Dedupe, c.Log)
+
+		c.StoreController.DefaultStore = defaultStore
+
+		// Enable extensions if extension config is provided
+		if c.Config != nil && c.Config.Extensions != nil {
+			ext.EnableExtensions(c.Config.Extensions, c.Log, c.Config.Storage.RootDirectory)
+		}
+	} else {
+		// we can't proceed without global storage
+		c.Log.Error().Err(errors.ErrImgStoreNotFound).Msg("controller: no storage config provided")
+
+		return errors.ErrImgStoreNotFound
+	}
+
+	if c.Config.Storage.SubPaths != nil {
+		if len(c.Config.Storage.SubPaths) > 0 {
+			subPaths := c.Config.Storage.SubPaths
+
+			subImageStore := make(map[string]*storage.ImageStore)
+
+			// creating image store per subpaths
+			for route, storageConfig := range subPaths {
+				if storageConfig.Dedupe {
+					err := storage.ValidateHardLink(storageConfig.RootDirectory)
+					if err != nil {
+						c.Log.Warn().Msg("input storage root directory filesystem does not supports hardlinking, " +
+							"disabling dedupe functionality")
+
+						storageConfig.Dedupe = false
+					}
+				}
+
+				subImageStore[route] = storage.NewImageStore(storageConfig.RootDirectory,
+					storageConfig.GC, storageConfig.Dedupe, c.Log)
+
+				// Enable extensions if extension config is provided
+				if c.Config != nil && c.Config.Extensions != nil {
+					ext.EnableExtensions(c.Config.Extensions, c.Log, storageConfig.RootDirectory)
+				}
+			}
+
+			c.StoreController.SubStore = subImageStore
+		}
+	}
+
 	_ = NewRouteHandler(c)
 
 	addr := fmt.Sprintf("%s:%s", c.Config.HTTP.Address, c.Config.HTTP.Port)

pkg/api/controller_test.go

@@ -371,6 +371,136 @@ func TestBasicAuth(t *testing.T) {
 	})
 }
 
+func TestMultipleInstance(t *testing.T) {
+	Convey("Negative test zot multiple instance", t, func() {
+		config := api.NewConfig()
+		config.HTTP.Port = SecurePort1
+		htpasswdPath := makeHtpasswdFile()
+		defer os.Remove(htpasswdPath)
+
+		config.HTTP.Auth = &api.AuthConfig{
+			HTPasswd: api.AuthHTPasswd{
+				Path: htpasswdPath,
+			},
+		}
+		c := api.NewController(config)
+		err := c.Run()
+		So(err, ShouldEqual, errors.ErrImgStoreNotFound)
+
+		globalDir, err := ioutil.TempDir("", "oci-repo-test")
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(globalDir)
+
+		subDir, err := ioutil.TempDir("/tmp", "oci-sub-test")
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(subDir)
+
+		c.Config.Storage.RootDirectory = globalDir
+		subPathMap := make(map[string]api.StorageConfig)
+
+		subPathMap["/a"] = api.StorageConfig{RootDirectory: subDir}
+
+		go func() {
+			if err := c.Run(); err != nil {
+				return
+			}
+		}()
+
+		// wait till ready
+		for {
+			_, err := resty.R().Get(BaseURL1)
+			if err == nil {
+				break
+			}
+			time.Sleep(100 * time.Millisecond)
+		}
+
+		defer func() {
+			ctx := context.Background()
+			_ = c.Server.Shutdown(ctx)
+		}()
+
+		client := resty.New()
+
+		tagResponse, err := client.R().SetBasicAuth(username, passphrase).
+			Get(BaseURL1 + "/v2/zot-test/tags/list")
+		So(err, ShouldBeNil)
+		So(tagResponse.StatusCode(), ShouldEqual, 404)
+	})
+
+	Convey("Test zot multiple instance", t, func() {
+		config := api.NewConfig()
+		config.HTTP.Port = SecurePort1
+		htpasswdPath := makeHtpasswdFile()
+		defer os.Remove(htpasswdPath)
+
+		config.HTTP.Auth = &api.AuthConfig{
+			HTPasswd: api.AuthHTPasswd{
+				Path: htpasswdPath,
+			},
+		}
+		c := api.NewController(config)
+		globalDir, err := ioutil.TempDir("", "oci-repo-test")
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(globalDir)
+
+		subDir, err := ioutil.TempDir("/tmp", "oci-sub-test")
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(subDir)
+
+		c.Config.Storage.RootDirectory = globalDir
+		subPathMap := make(map[string]api.StorageConfig)
+
+		subPathMap["/a"] = api.StorageConfig{RootDirectory: subDir}
+		go func() {
+			// this blocks
+			if err := c.Run(); err != nil {
+				return
+			}
+		}()
+
+		// wait till ready
+		for {
+			_, err := resty.R().Get(BaseURL1)
+			if err == nil {
+				break
+			}
+			time.Sleep(100 * time.Millisecond)
+		}
+
+		defer func() {
+			ctx := context.Background()
+			_ = c.Server.Shutdown(ctx)
+		}()
+
+		// without creds, should get access error
+		resp, err := resty.R().Get(BaseURL1 + "/v2/")
+		So(err, ShouldBeNil)
+		So(resp, ShouldNotBeNil)
+		So(resp.StatusCode(), ShouldEqual, 401)
+		var e api.Error
+		err = json.Unmarshal(resp.Body(), &e)
+		So(err, ShouldBeNil)
+
+		// with creds, should get expected status code
+		resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1)
+		So(resp, ShouldNotBeNil)
+		So(resp.StatusCode(), ShouldEqual, 404)
+
+		resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/v2/")
+		So(resp, ShouldNotBeNil)
+		So(resp.StatusCode(), ShouldEqual, 200)
+	})
+}
+
 func TestTLSWithBasicAuth(t *testing.T) {
 	Convey("Make a new controller", t, func() {
 		caCert, err := ioutil.ReadFile(CACert)
@@ -1820,13 +1950,13 @@ func TestParallelRequests(t *testing.T) {
 		{
 			srcImageName:  "zot-cve-test",
 			srcImageTag:   "0.0.1",
-			destImageName: "zot-3-test",
+			destImageName: "a/zot-3-test",
 			testCaseName:  "Request-3",
 		},
 		{
 			srcImageName:  "zot-cve-test",
 			srcImageTag:   "0.0.1",
-			destImageName: "zot-4-test",
+			destImageName: "b/zot-4-test",
 			testCaseName:  "Request-4",
 		},
 		{
@@ -1881,13 +2011,13 @@ func TestParallelRequests(t *testing.T) {
 		{
 			srcImageName:  "zot-cve-test",
 			srcImageTag:   "0.0.1",
-			destImageName: "zot-3-test",
+			destImageName: "a/zot-3-test",
 			testCaseName:  "Request-13",
 		},
 		{
 			srcImageName:  "zot-cve-test",
 			srcImageTag:   "0.0.1",
-			destImageName: "zot-4-test",
+			destImageName: "b/zot-4-test",
 			testCaseName:  "Request-14",
 		},
 	}
@@ -1896,8 +2026,6 @@ func TestParallelRequests(t *testing.T) {
 	config.HTTP.Port = SecurePort1
 	htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase))
 
-	// defer os.Remove(htpasswdPath)
-
 	config.HTTP.Auth = &api.AuthConfig{
 		HTPasswd: api.AuthHTPasswd{
 			Path: htpasswdPath,
@@ -1911,11 +2039,23 @@ func TestParallelRequests(t *testing.T) {
 		panic(err)
 	}
 
-	err = copyFiles("../../test/data", dir)
+	firstSubDir, err := ioutil.TempDir("", "oci-sub-dir")
 	if err != nil {
 		panic(err)
 	}
-	//defer os.RemoveAll(dir)
+
+	secondSubDir, err := ioutil.TempDir("", "oci-sub-dir")
+	if err != nil {
+		panic(err)
+	}
+
+	subPaths := make(map[string]api.StorageConfig)
+
+	subPaths["/a"] = api.StorageConfig{RootDirectory: firstSubDir}
+	subPaths["/b"] = api.StorageConfig{RootDirectory: secondSubDir}
+
+	c.Config.Storage.SubPaths = subPaths
+
 	c.Config.Storage.RootDirectory = dir
 
 	go func() {
@@ -1939,7 +2079,7 @@ func TestParallelRequests(t *testing.T) {
 	for i, testcase := range testCases {
 		testcase := testcase
 		j := i
-		//println(i)
+
 		t.Run(testcase.testCaseName, func(t *testing.T) {
 			t.Parallel()
 			client := resty.New()
@@ -1949,7 +2089,7 @@ func TestParallelRequests(t *testing.T) {
 			assert.Equal(t, err, nil, "Error should be nil")
 			assert.NotEqual(t, tagResponse.StatusCode(), 400, "bad request")
 
-			manifestList := getAllManifests(path.Join(c.Config.Storage.RootDirectory, testcase.srcImageName))
+			manifestList := getAllManifests(path.Join("../../test/data", testcase.srcImageName))
 
 			for _, manifest := range manifestList {
 				headResponse, err := client.R().SetBasicAuth(username, passphrase).
@@ -1963,7 +2103,7 @@ func TestParallelRequests(t *testing.T) {
 				assert.Equal(t, getResponse.StatusCode(), 404, "response status code should return 404")
 			}
 
-			blobList := getAllBlobs(path.Join(c.Config.Storage.RootDirectory, testcase.srcImageName))
+			blobList := getAllBlobs(path.Join("../../test/data", testcase.srcImageName))
 
 			for _, blob := range blobList {
 				// Get request of blob
@@ -1981,7 +2121,7 @@ func TestParallelRequests(t *testing.T) {
 				assert.Equal(t, err, nil, "Should not be nil")
 				assert.NotEqual(t, getResponse.StatusCode(), 500, "internal server error should not occurred")
 
-				blobPath := path.Join(c.Config.Storage.RootDirectory, testcase.srcImageName, "blobs/sha256", blob)
+				blobPath := path.Join("../../test/data", testcase.srcImageName, "blobs/sha256", blob)
 
 				buf, err := ioutil.ReadFile(blobPath)
 				if err != nil {
@@ -2050,9 +2190,6 @@ func TestParallelRequests(t *testing.T) {
 								SetHeader("Content-Range", fmt.Sprintf("%d", readContent)+"-"+fmt.Sprintf("%d", readContent+n-1)).
 								SetBasicAuth(username, passphrase).
 								Patch(BaseURL1 + "/v2/" + testcase.destImageName + "/blobs/uploads/" + sessionID)
-							if err != nil {
-								panic(err)
-							}
 
 							assert.Equal(t, err, nil, "Error should be nil")
 							assert.NotEqual(t, patchResponse.StatusCode(), 500, "response status code should not return 500")
@@ -2273,3 +2410,79 @@ func stopServer(ctrl *api.Controller) {
 		panic(err)
 	}
 }
+
+func TestHardLink(t *testing.T) {
+	Convey("Validate hard link", t, func() {
+		config := api.NewConfig()
+		config.HTTP.Port = SecurePort1
+		htpasswdPath := makeHtpasswdFileFromString(getCredString(username, passphrase))
+
+		config.HTTP.Auth = &api.AuthConfig{
+			HTPasswd: api.AuthHTPasswd{
+				Path: htpasswdPath,
+			},
+		}
+
+		c := api.NewController(config)
+
+		dir, err := ioutil.TempDir("", "hard-link-test")
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(dir)
+
+		err = os.Chmod(dir, 0400)
+		if err != nil {
+			panic(err)
+		}
+
+		subDir, err := ioutil.TempDir("", "sub-hardlink-test")
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(subDir)
+
+		err = os.Chmod(subDir, 0400)
+		if err != nil {
+			panic(err)
+		}
+
+		c.Config.Storage.RootDirectory = dir
+		subPaths := make(map[string]api.StorageConfig)
+
+		subPaths["/a"] = api.StorageConfig{RootDirectory: subDir, Dedupe: true}
+
+		c.Config.Storage.SubPaths = subPaths
+
+		go func() {
+			// this blocks
+			if err := c.Run(); err != nil {
+				return
+			}
+		}()
+
+		time.Sleep(5 * time.Second)
+
+		// wait till ready
+		for {
+			_, err := resty.R().Get(BaseURL1)
+			if err == nil {
+				break
+			}
+
+			time.Sleep(100 * time.Millisecond)
+		}
+
+		err = os.Chmod(dir, 0644)
+		if err != nil {
+			panic(err)
+		}
+
+		err = os.Chmod(subDir, 0644)
+		if err != nil {
+			panic(err)
+		}
+
+		So(c.Config.Storage.Dedupe, ShouldEqual, false)
+	})
+}

pkg/api/routes.go

@@ -25,6 +25,7 @@ import (
 	"github.com/anuvu/zot/errors"
 	ext "github.com/anuvu/zot/pkg/extensions"
 	"github.com/anuvu/zot/pkg/log"
+	"github.com/anuvu/zot/pkg/storage"
 	"github.com/gorilla/mux"
 	jsoniter "github.com/json-iterator/go"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
@@ -90,7 +91,7 @@ func (rh *RouteHandler) SetupRoutes() {
 	rh.c.Router.PathPrefix("/swagger/v2/").Methods("GET").Handler(httpSwagger.WrapHandler)
 	// Setup Extensions Routes
 	if rh.c.Config != nil && rh.c.Config.Extensions != nil {
-		ext.SetupRoutes(rh.c.Router, rh.c.Config.Storage.RootDirectory, rh.c.ImageStore, rh.c.Log)
+		ext.SetupRoutes(rh.c.Router, rh.c.StoreController, rh.c.Log)
 	}
 }
 
@@ -139,6 +140,7 @@ type ImageTags struct {
 // @Failure 400 {string} 	string 				"bad request".
 func (rh *RouteHandler) ListTags(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
+
 	name, ok := vars["name"]
 
 	if !ok || name == "" {
@@ -146,11 +148,11 @@ func (rh *RouteHandler) ListTags(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	is := rh.getImageStore(name)
+
 	paginate := false
 	n := -1
 
-	var err error
-
 	nQuery, ok := r.URL.Query()["n"]
 
 	if ok {
@@ -161,6 +163,8 @@ func (rh *RouteHandler) ListTags(w http.ResponseWriter, r *http.Request) {
 
 		var n1 int64
 
+		var err error
+
 		if n1, err = strconv.ParseInt(nQuery[0], 10, 0); err != nil {
 			w.WriteHeader(http.StatusBadRequest)
 			return
@@ -182,7 +186,7 @@ func (rh *RouteHandler) ListTags(w http.ResponseWriter, r *http.Request) {
 		last = lastQuery[0]
 	}
 
-	tags, err := rh.c.ImageStore.GetImageTags(name)
+	tags, err := is.GetImageTags(name)
 	if err != nil {
 		WriteJSON(w, http.StatusNotFound, NewErrorList(NewError(NAME_UNKNOWN, map[string]string{"name": name})))
 		return
@@ -255,13 +259,15 @@ func (rh *RouteHandler) CheckManifest(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	is := rh.getImageStore(name)
+
 	reference, ok := vars["reference"]
 	if !ok || reference == "" {
 		WriteJSON(w, http.StatusNotFound, NewErrorList(NewError(MANIFEST_INVALID, map[string]string{"reference": reference})))
 		return
 	}
 
-	_, digest, _, err := rh.c.ImageStore.GetImageManifest(name, reference)
+	_, digest, _, err := is.GetImageManifest(name, reference)
 	if err != nil {
 		switch err {
 		case errors.ErrRepoNotFound:
@@ -310,13 +316,15 @@ func (rh *RouteHandler) GetManifest(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	is := rh.getImageStore(name)
+
 	reference, ok := vars["reference"]
 	if !ok || reference == "" {
 		WriteJSON(w, http.StatusNotFound, NewErrorList(NewError(MANIFEST_UNKNOWN, map[string]string{"reference": reference})))
 		return
 	}
 
-	content, digest, mediaType, err := rh.c.ImageStore.GetImageManifest(name, reference)
+	content, digest, mediaType, err := is.GetImageManifest(name, reference)
 	if err != nil {
 		switch err {
 		case errors.ErrRepoNotFound:
@@ -362,6 +370,8 @@ func (rh *RouteHandler) UpdateManifest(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	is := rh.getImageStore(name)
+
 	reference, ok := vars["reference"]
 	if !ok || reference == "" {
 		WriteJSON(w, http.StatusNotFound, NewErrorList(NewError(MANIFEST_INVALID, map[string]string{"reference": reference})))
@@ -382,7 +392,7 @@ func (rh *RouteHandler) UpdateManifest(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	digest, err := rh.c.ImageStore.PutImageManifest(name, reference, mediaType, body)
+	digest, err := is.PutImageManifest(name, reference, mediaType, body)
 	if err != nil {
 		switch err {
 		case errors.ErrRepoNotFound:
@@ -428,13 +438,16 @@ func (rh *RouteHandler) DeleteManifest(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	is := rh.getImageStore(name)
+
 	reference, ok := vars["reference"]
 	if !ok || reference == "" {
 		w.WriteHeader(http.StatusNotFound)
+
 		return
 	}
 
-	err := rh.c.ImageStore.DeleteImageManifest(name, reference)
+	err := is.DeleteImageManifest(name, reference)
 	if err != nil {
 		switch err {
 		case errors.ErrRepoNotFound:
@@ -476,6 +489,8 @@ func (rh *RouteHandler) CheckBlob(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	is := rh.getImageStore(name)
+
 	digest, ok := vars["digest"]
 	if !ok || digest == "" {
 		w.WriteHeader(http.StatusNotFound)
@@ -484,7 +499,7 @@ func (rh *RouteHandler) CheckBlob(w http.ResponseWriter, r *http.Request) {
 
 	mediaType := r.Header.Get("Accept")
 
-	ok, blen, err := rh.c.ImageStore.CheckBlob(name, digest, mediaType)
+	ok, blen, err := is.CheckBlob(name, digest, mediaType)
 	if err != nil {
 		switch err {
 		case errors.ErrBadBlobDigest:
@@ -530,6 +545,8 @@ func (rh *RouteHandler) GetBlob(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	is := rh.getImageStore(name)
+
 	digest, ok := vars["digest"]
 	if !ok || digest == "" {
 		w.WriteHeader(http.StatusNotFound)
@@ -538,7 +555,7 @@ func (rh *RouteHandler) GetBlob(w http.ResponseWriter, r *http.Request) {
 
 	mediaType := r.Header.Get("Accept")
 
-	br, blen, err := rh.c.ImageStore.GetBlob(name, digest, mediaType)
+	br, blen, err := is.GetBlob(name, digest, mediaType)
 	if err != nil {
 		switch err {
 		case errors.ErrBadBlobDigest:
@@ -576,16 +593,20 @@ func (rh *RouteHandler) DeleteBlob(w http.ResponseWriter, r *http.Request) {
 
 	if !ok || name == "" {
 		w.WriteHeader(http.StatusNotFound)
+
 		return
 	}
 
 	digest, ok := vars["digest"]
 	if !ok || digest == "" {
 		w.WriteHeader(http.StatusNotFound)
+
 		return
 	}
 
-	err := rh.c.ImageStore.DeleteBlob(name, digest)
+	is := rh.getImageStore(name)
+
+	err := is.DeleteBlob(name, digest)
 	if err != nil {
 		switch err {
 		case errors.ErrBadBlobDigest:
@@ -626,6 +647,9 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
+	is := rh.getImageStore(name)
+
+	// currently zot does not support cross-repository mounting, following dist-spec and returning 202
 	if mountDigests, ok := r.URL.Query()["mount"]; ok {
 		if len(mountDigests) != 1 {
 			w.WriteHeader(http.StatusBadRequest)
@@ -639,9 +663,9 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request)
 		}
 
 		// zot does not support cross mounting directly and do a workaround by copying blob using hard link
-		err := rh.c.ImageStore.MountBlob(name, from[0], mountDigests[0])
+		err := is.MountBlob(name, from[0], mountDigests[0])
 		if err != nil {
-			u, err := rh.c.ImageStore.NewBlobUpload(name)
+			u, err := is.NewBlobUpload(name)
 			if err != nil {
 				switch err {
 				case errors.ErrRepoNotFound:
@@ -703,7 +727,7 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request)
 			return
 		}
 
-		sessionID, size, err := rh.c.ImageStore.FullBlobUpload(name, r.Body, digest)
+		sessionID, size, err := is.FullBlobUpload(name, r.Body, digest)
 		if err != nil {
 			rh.c.Log.Error().Err(err).Int64("actual", size).Int64("expected", contentLength).Msg("failed full upload")
 			w.WriteHeader(http.StatusInternalServerError)
@@ -725,7 +749,7 @@ func (rh *RouteHandler) CreateBlobUpload(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	u, err := rh.c.ImageStore.NewBlobUpload(name)
+	u, err := is.NewBlobUpload(name)
 	if err != nil {
 		switch err {
 		case errors.ErrRepoNotFound:
@@ -765,13 +789,16 @@ func (rh *RouteHandler) GetBlobUpload(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	is := rh.getImageStore(name)
+
 	sessionID, ok := vars["session_id"]
 	if !ok || sessionID == "" {
 		w.WriteHeader(http.StatusNotFound)
+
 		return
 	}
 
-	size, err := rh.c.ImageStore.GetBlobUpload(name, sessionID)
+	size, err := is.GetBlobUpload(name, sessionID)
 	if err != nil {
 		switch err {
 		case errors.ErrBadUploadRange:
@@ -824,19 +851,21 @@ func (rh *RouteHandler) PatchBlobUpload(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
+	is := rh.getImageStore(name)
+
 	sessionID, ok := vars["session_id"]
 	if !ok || sessionID == "" {
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
 
-	var err error
-
 	var clen int64
 
+	var err error
+
 	if r.Header.Get("Content-Length") == "" || r.Header.Get("Content-Range") == "" {
 		// streamed blob upload
-		clen, err = rh.c.ImageStore.PutBlobChunkStreamed(name, sessionID, r.Body)
+		clen, err = is.PutBlobChunkStreamed(name, sessionID, r.Body)
 	} else {
 		// chunked blob upload
 
@@ -863,7 +892,7 @@ func (rh *RouteHandler) PatchBlobUpload(w http.ResponseWriter, r *http.Request)
 			return
 		}
 
-		clen, err = rh.c.ImageStore.PutBlobChunk(name, sessionID, from, to, r.Body)
+		clen, err = is.PutBlobChunk(name, sessionID, from, to, r.Body)
 	}
 
 	if err != nil {
@@ -916,6 +945,8 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
+	is := rh.getImageStore(name)
+
 	sessionID, ok := vars["session_id"]
 	if !ok || sessionID == "" {
 		w.WriteHeader(http.StatusNotFound)
@@ -969,7 +1000,7 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request)
 			return
 		}
 
-		_, err = rh.c.ImageStore.PutBlobChunk(name, sessionID, from, to, r.Body)
+		_, err = is.PutBlobChunk(name, sessionID, from, to, r.Body)
 		if err != nil {
 			switch err {
 			case errors.ErrBadUploadRange:
@@ -992,7 +1023,7 @@ func (rh *RouteHandler) UpdateBlobUpload(w http.ResponseWriter, r *http.Request)
 
 finish:
 	// blob chunks already transferred, just finish
-	if err := rh.c.ImageStore.FinishBlobUpload(name, sessionID, r.Body, digest); err != nil {
+	if err := is.FinishBlobUpload(name, sessionID, r.Body, digest); err != nil {
 		switch err {
 		case errors.ErrBadBlobDigest:
 			WriteJSON(w, http.StatusBadRequest,
@@ -1040,13 +1071,15 @@ func (rh *RouteHandler) DeleteBlobUpload(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
+	is := rh.getImageStore(name)
+
 	sessionID, ok := vars["session_id"]
 	if !ok || sessionID == "" {
 		w.WriteHeader(http.StatusNotFound)
 		return
 	}
 
-	if err := rh.c.ImageStore.DeleteBlobUpload(name, sessionID); err != nil {
+	if err := is.DeleteBlobUpload(name, sessionID); err != nil {
 		switch err {
 		case errors.ErrRepoNotFound:
 			WriteJSON(w, http.StatusNotFound,
@@ -1078,13 +1111,32 @@ type RepositoryList struct {
 // @Failure 500 {string} string "internal server error"
 // @Router /v2/_catalog [get].
 func (rh *RouteHandler) ListRepositories(w http.ResponseWriter, r *http.Request) {
-	repos, err := rh.c.ImageStore.GetRepositories()
+	combineRepoList := make([]string, 0)
-	if err != nil {
+
-		w.WriteHeader(http.StatusInternalServerError)
+	subStore := rh.c.StoreController.SubStore
-		return
+
+	for _, imgStore := range subStore {
+		repos, err := imgStore.GetRepositories()
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+
+		combineRepoList = append(combineRepoList, repos...)
 	}
 
-	is := RepositoryList{Repositories: repos}
+	singleStore := rh.c.StoreController.DefaultStore
+	if singleStore != nil {
+		repos, err := singleStore.GetRepositories()
+		if err != nil {
+			w.WriteHeader(http.StatusInternalServerError)
+			return
+		}
+
+		combineRepoList = append(combineRepoList, repos...)
+	}
+
+	is := RepositoryList{Repositories: combineRepoList}
 
 	WriteJSON(w, http.StatusOK, is)
 }
@@ -1148,3 +1200,8 @@ func WriteDataFromReader(w http.ResponseWriter, status int, length int64, mediaT
 		}
 	}
 }
+
+// will return image storage corresponding to subpath provided in config.
+func (rh *RouteHandler) getImageStore(name string) *storage.ImageStore {
+	return rh.c.StoreController.GetImageStore(name)
+}

pkg/compliance/config.go

@@ -1,11 +1,12 @@
 package compliance
 
 type Config struct {
-	Address    string
+	Address     string
-	Port       string
+	Port        string
-	Version    string
+	Version     string
-	OutputJSON bool
+	StorageInfo []string
-	Compliance bool
+	OutputJSON  bool
+	Compliance  bool
 }
 
 func NewConfig() *Config {

pkg/compliance/v1_0_0/check.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"strings"
 	"testing"
 
@@ -46,6 +47,8 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
 
 	baseURL := fmt.Sprintf("http://%s:%s", config.Address, config.Port)
 
+	storageInfo := config.StorageInfo
+
 	fmt.Println("------------------------------")
 	fmt.Println("Checking for v1.0.0 compliance")
 	fmt.Println("------------------------------")
@@ -459,6 +462,11 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
 			loc := Location(baseURL, resp)
 			So(loc, ShouldNotBeEmpty)
 
+			// since we are not specifying any prefix i.e provided in config while starting server,
+			// so it should store repo7 to global root dir
+			_, err = os.Stat(path.Join(storageInfo[0], "repo7"))
+			So(err, ShouldBeNil)
+
 			resp, err = resty.R().Get(loc)
 			So(err, ShouldBeNil)
 			So(resp.StatusCode(), ShouldEqual, 204)
@@ -686,6 +694,276 @@ func CheckWorkflows(t *testing.T, config *compliance.Config) {
 			So(err, ShouldBeNil)
 			So(resp.StatusCode(), ShouldEqual, 202)
 		})
+
+		Convey("Multiple Storage", func() {
+			// test APIS on subpath routes, default storage already tested above
+			// subpath route firsttest
+			resp, err := resty.R().Post(baseURL + "/v2/firsttest/first/blobs/uploads/")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 202)
+			firstloc := Location(baseURL, resp)
+			So(firstloc, ShouldNotBeEmpty)
+
+			resp, err = resty.R().Get(firstloc)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 204)
+
+			// if firsttest route is used as prefix in url that means repo should be stored in subpaths["firsttest"] rootdir
+			_, err = os.Stat(path.Join(storageInfo[1], "firsttest/first"))
+			So(err, ShouldBeNil)
+
+			// subpath route secondtest
+			resp, err = resty.R().Post(baseURL + "/v2/secondtest/second/blobs/uploads/")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 202)
+			secondloc := Location(baseURL, resp)
+			So(secondloc, ShouldNotBeEmpty)
+
+			resp, err = resty.R().Get(secondloc)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 204)
+
+			// if secondtest route is used as prefix in url that means repo should be stored in subpaths["secondtest"] rootdir
+			_, err = os.Stat(path.Join(storageInfo[2], "secondtest/second"))
+			So(err, ShouldBeNil)
+
+			content := []byte("this is a blob5")
+			digest := godigest.FromBytes(content)
+			So(digest, ShouldNotBeNil)
+			// monolithic blob upload: success
+			// first test
+			resp, err = resty.R().SetQueryParam("digest", digest.String()).
+				SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(firstloc)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 201)
+			firstblobLoc := resp.Header().Get("Location")
+			So(firstblobLoc, ShouldNotBeEmpty)
+			So(resp.Header().Get("Content-Length"), ShouldEqual, "0")
+			So(resp.Header().Get(api.DistContentDigestKey), ShouldNotBeEmpty)
+
+			// second test
+			resp, err = resty.R().SetQueryParam("digest", digest.String()).
+				SetHeader("Content-Type", "application/octet-stream").SetBody(content).Put(secondloc)
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 201)
+			secondblobLoc := resp.Header().Get("Location")
+			So(secondblobLoc, ShouldNotBeEmpty)
+			So(resp.Header().Get("Content-Length"), ShouldEqual, "0")
+			So(resp.Header().Get(api.DistContentDigestKey), ShouldNotBeEmpty)
+
+			// check a non-existent manifest
+			resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
+				SetBody(content).Head(baseURL + "/v2/unknown/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+
+			resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
+				SetBody(content).Head(baseURL + "/v2/firsttest/unknown/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+
+			resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
+				SetBody(content).Head(baseURL + "/v2/secondtest/unknown/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+
+			// create a manifest
+			m := ispec.Manifest{
+				Config: ispec.Descriptor{
+					Digest: digest,
+					Size:   int64(len(content)),
+				},
+				Layers: []ispec.Descriptor{
+					{
+						MediaType: "application/vnd.oci.image.layer.v1.tar",
+						Digest:    digest,
+						Size:      int64(len(content)),
+					},
+				},
+			}
+			m.SchemaVersion = 2
+			content, err = json.Marshal(m)
+			So(err, ShouldBeNil)
+			digest = godigest.FromBytes(content)
+			So(digest, ShouldNotBeNil)
+			// subpath firsttest
+			resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
+				SetBody(content).Put(baseURL + "/v2/firsttest/first/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 201)
+			d := resp.Header().Get(api.DistContentDigestKey)
+			So(d, ShouldNotBeEmpty)
+			So(d, ShouldEqual, digest.String())
+
+			// subpath secondtest
+			resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
+				SetBody(content).Put(baseURL + "/v2/secondtest/second/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 201)
+			d = resp.Header().Get(api.DistContentDigestKey)
+			So(d, ShouldNotBeEmpty)
+			So(d, ShouldEqual, digest.String())
+
+			content = []byte("this is a blob5")
+			digest = godigest.FromBytes(content)
+			So(digest, ShouldNotBeNil)
+			// create a manifest with same blob but a different tag
+			m = ispec.Manifest{
+				Config: ispec.Descriptor{
+					Digest: digest,
+					Size:   int64(len(content)),
+				},
+				Layers: []ispec.Descriptor{
+					{
+						MediaType: "application/vnd.oci.image.layer.v1.tar",
+						Digest:    digest,
+						Size:      int64(len(content)),
+					},
+				},
+			}
+			m.SchemaVersion = 2
+			content, err = json.Marshal(m)
+			So(err, ShouldBeNil)
+			digest = godigest.FromBytes(content)
+			So(digest, ShouldNotBeNil)
+
+			// subpath firsttest
+			resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
+				SetBody(content).Put(baseURL + "/v2/firsttest/first/manifests/test:2.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 201)
+			d = resp.Header().Get(api.DistContentDigestKey)
+			So(d, ShouldNotBeEmpty)
+			So(d, ShouldEqual, digest.String())
+
+			// subpath secondtest
+			resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json").
+				SetBody(content).Put(baseURL + "/v2/secondtest/second/manifests/test:2.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 201)
+			d = resp.Header().Get(api.DistContentDigestKey)
+			So(d, ShouldNotBeEmpty)
+			So(d, ShouldEqual, digest.String())
+
+			// check/get by tag
+			resp, err = resty.R().Head(baseURL + "/v2/firsttest/first/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+			resp, err = resty.R().Get(baseURL + "/v2/firsttest/first/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+			So(resp.Body(), ShouldNotBeEmpty)
+			resp, err = resty.R().Head(baseURL + "/v2/secondtest/second/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+			resp, err = resty.R().Get(baseURL + "/v2/secondtest/second/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+			So(resp.Body(), ShouldNotBeEmpty)
+
+			// check/get by reference
+			resp, err = resty.R().Head(baseURL + "/v2/firsttest/first/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+			resp, err = resty.R().Get(baseURL + "/v2/firsttest/first/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+			So(resp.Body(), ShouldNotBeEmpty)
+
+			resp, err = resty.R().Head(baseURL + "/v2/secondtest/second/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+			resp, err = resty.R().Get(baseURL + "/v2/secondtest/second/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 200)
+			So(resp.Body(), ShouldNotBeEmpty)
+
+			// delete manifest by tag should fail
+			resp, err = resty.R().Delete(baseURL + "/v2/firsttest/first/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 400)
+
+			resp, err = resty.R().Delete(baseURL + "/v2/secondtest/second/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 400)
+
+			// delete manifest by digest
+			resp, err = resty.R().Delete(baseURL + "/v2/firsttest/first/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 202)
+
+			resp, err = resty.R().Delete(baseURL + "/v2/secondtest/second/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 202)
+
+			// delete manifest by digest
+			resp, err = resty.R().Delete(baseURL + "/v2/firsttest/first/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+
+			resp, err = resty.R().Delete(baseURL + "/v2/secondtest/second/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+
+			// delete again should fail
+			resp, err = resty.R().Delete(baseURL + "/v2/firsttest/first/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+
+			resp, err = resty.R().Delete(baseURL + "/v2/secondtest/second/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+
+			// check/get by tag
+			resp, err = resty.R().Head(baseURL + "/v2/firsttest/first/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+			resp, err = resty.R().Get(baseURL + "/v2/firsttest/first/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+			So(resp.Body(), ShouldNotBeEmpty)
+
+			resp, err = resty.R().Head(baseURL + "/v2/secondtest/second/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+			resp, err = resty.R().Get(baseURL + "/v2/secondtest/second/manifests/test:1.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+			So(resp.Body(), ShouldNotBeEmpty)
+
+			resp, err = resty.R().Head(baseURL + "/v2/firsttest/first/repo7/manifests/test:2.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+			resp, err = resty.R().Get(baseURL + "/v2/firsttest/first/manifests/test:2.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+			So(resp.Body(), ShouldNotBeEmpty)
+
+			resp, err = resty.R().Head(baseURL + "/v2/secondtest/second/manifests/test:2.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+			resp, err = resty.R().Get(baseURL + "/v2/secondtest/second/manifests/test:2.0")
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+			So(resp.Body(), ShouldNotBeEmpty)
+
+			// check/get by reference
+			resp, err = resty.R().Head(baseURL + "/v2/firsttest/first/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+			resp, err = resty.R().Get(baseURL + "/v2/firsttest/first/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+			So(resp.Body(), ShouldNotBeEmpty)
+
+			resp, err = resty.R().Head(baseURL + "/v2/secondtest/second/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+			resp, err = resty.R().Get(baseURL + "/v2/secondtest/second/manifests/" + digest.String())
+			So(err, ShouldBeNil)
+			So(resp.StatusCode(), ShouldEqual, 404)
+			So(resp.Body(), ShouldNotBeEmpty)
+		})
 	})
 }
 

pkg/compliance/v1_0_0/check_test.go

@@ -18,24 +18,35 @@ import (
 // nolint: gochecknoglobals
 var (
 	listenAddress = "127.0.0.1"
+	defaultDir    = ""
+	firstDir      = ""
+	secondDir     = ""
 )
 
 func TestWorkflows(t *testing.T) {
 	ctrl, randomPort := startServer()
 	defer stopServer(ctrl)
+
+	storageInfo := []string{defaultDir, firstDir, secondDir}
+
 	v1_0_0.CheckWorkflows(t, &compliance.Config{
-		Address: listenAddress,
+		Address:     listenAddress,
-		Port:    randomPort,
+		Port:        randomPort,
+		StorageInfo: storageInfo,
 	})
 }
 
 func TestWorkflowsOutputJSON(t *testing.T) {
 	ctrl, randomPort := startServer()
 	defer stopServer(ctrl)
+
+	storageInfo := []string{defaultDir, firstDir, secondDir}
+
 	v1_0_0.CheckWorkflows(t, &compliance.Config{
-		Address:    listenAddress,
+		Address:     listenAddress,
-		Port:       randomPort,
+		Port:        randomPort,
-		OutputJSON: true,
+		OutputJSON:  true,
+		StorageInfo: storageInfo,
 	})
 }
 
@@ -59,8 +70,31 @@ func startServer() (*api.Controller, string) {
 		panic(err)
 	}
 
+	defaultDir = dir
+
+	firstSubDir, err := ioutil.TempDir("", "oci-repo-test")
+	if err != nil {
+		panic(err)
+	}
+
+	firstDir = firstSubDir
+
+	secondSubDir, err := ioutil.TempDir("", "oci-repo-test")
+	if err != nil {
+		panic(err)
+	}
+
+	secondDir = secondSubDir
+
+	subPaths := make(map[string]api.StorageConfig)
+
+	subPaths["/firsttest"] = api.StorageConfig{RootDirectory: firstSubDir}
+	subPaths["/secondtest"] = api.StorageConfig{RootDirectory: secondSubDir}
+
 	ctrl.Config.Storage.RootDirectory = dir
 
+	ctrl.Config.Storage.SubPaths = subPaths
+
 	go func() {
 		// this blocks
 		if err := ctrl.Run(); err != nil {

pkg/extensions/extensions.go

@@ -55,9 +55,9 @@ func EnableExtensions(extension *ExtensionConfig, log log.Logger, rootDir string
 }
 
 // SetupRoutes ...
-func SetupRoutes(router *mux.Router, rootDir string, imgStore *storage.ImageStore, log log.Logger) {
+func SetupRoutes(router *mux.Router, storeController storage.StoreController, log log.Logger) {
 	log.Info().Msg("setting up extensions routes")
-	resConfig := search.GetResolverConfig(rootDir, log, imgStore)
+	resConfig := search.GetResolverConfig(log, storeController)
 	router.PathPrefix("/query").Methods("GET", "POST").
 		Handler(gqlHandler.NewDefaultServer(search.NewExecutableSchema(resConfig)))
 }

pkg/extensions/minimal.go

@@ -21,6 +21,6 @@ func EnableExtensions(extension *ExtensionConfig, log log.Logger, rootDir string
 }
 
 // SetupRoutes ...
-func SetupRoutes(router *mux.Router, rootDir string, imgStore *storage.ImageStore, log log.Logger) {
+func SetupRoutes(router *mux.Router, storeController storage.StoreController, log log.Logger) {
 	log.Warn().Msg("skipping setting up extensions routes because given zot binary doesn't support any extensions, please build zot full binary for this feature")
 }

pkg/extensions/search/cve/cve.go

@@ -2,6 +2,7 @@ package cveinfo
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
@@ -10,6 +11,7 @@ import (
 
 	"github.com/anuvu/zot/errors"
 	"github.com/anuvu/zot/pkg/log"
+	"github.com/anuvu/zot/pkg/storage"
 	integration "github.com/aquasecurity/trivy/integration"
 	config "github.com/aquasecurity/trivy/integration/config"
 	"github.com/aquasecurity/trivy/pkg/report"
@@ -44,6 +46,84 @@ func ScanImage(config *config.Config) (report.Results, error) {
 	return integration.ScanTrivyImage(config.TrivyConfig)
 }
 
+func GetCVEInfo(storeController storage.StoreController, log log.Logger) (*CveInfo, error) {
+	cveController := CveTrivyController{}
+
+	subCveConfig := make(map[string]*config.Config)
+
+	if storeController.DefaultStore != nil {
+		imageStore := storeController.DefaultStore
+
+		rootDir := imageStore.RootDir()
+
+		config, err := NewTrivyConfig(rootDir)
+		if err != nil {
+			return nil, err
+		}
+
+		cveController.DefaultCveConfig = config
+	}
+
+	if storeController.SubStore != nil {
+		for route, storage := range storeController.SubStore {
+			rootDir := storage.RootDir()
+
+			config, err := NewTrivyConfig(rootDir)
+			if err != nil {
+				return nil, err
+			}
+
+			subCveConfig[route] = config
+		}
+	}
+
+	cveController.SubCveConfig = subCveConfig
+
+	return &CveInfo{Log: log, CveTrivyController: cveController, StoreController: storeController}, nil
+}
+
+func getRoutePrefix(name string) string {
+	names := strings.SplitN(name, "/", 2)
+
+	if len(names) != 2 { // nolint: gomnd
+		// it means route is of global storage e.g "centos:latest"
+		if len(names) == 1 {
+			return "/"
+		}
+	}
+
+	return fmt.Sprintf("/%s", names[0])
+}
+
+func (cveinfo CveInfo) GetTrivyConfig(image string) *config.Config {
+	// Split image to get route prefix
+	prefixName := getRoutePrefix(image)
+
+	var trivyConfig *config.Config
+
+	var ok bool
+
+	var rootDir string
+
+	// Get corresponding CVE trivy config, if no sub cve config present that means its default
+	trivyConfig, ok = cveinfo.CveTrivyController.SubCveConfig[prefixName]
+	if ok {
+		imgStore := cveinfo.StoreController.SubStore[prefixName]
+
+		rootDir = imgStore.RootDir()
+	} else {
+		trivyConfig = cveinfo.CveTrivyController.DefaultCveConfig
+
+		imgStore := cveinfo.StoreController.DefaultStore
+
+		rootDir = imgStore.RootDir()
+	}
+
+	trivyConfig.TrivyConfig.Input = path.Join(rootDir, image)
+
+	return trivyConfig
+}
+
 func (cveinfo CveInfo) IsValidImageFormat(imagePath string) (bool, error) {
 	imageDir, inputTag := getImageDirAndTag(imagePath)
 
@@ -113,16 +193,85 @@ func getImageDirAndTag(imageName string) (string, string) {
 	return imageDir, imageTag
 }
 
+// Below method will return image path including root dir, root dir is determined by splitting.
+func (cveinfo CveInfo) GetImageRepoPath(image string) string {
+	var rootDir string
+
+	prefixName := getRoutePrefix(image)
+
+	subStore := cveinfo.StoreController.SubStore
+
+	if subStore != nil {
+		imgStore, ok := cveinfo.StoreController.SubStore[prefixName]
+		if ok {
+			rootDir = imgStore.RootDir()
+		} else {
+			rootDir = cveinfo.StoreController.DefaultStore.RootDir()
+		}
+	} else {
+		rootDir = cveinfo.StoreController.DefaultStore.RootDir()
+	}
+
+	return path.Join(rootDir, image)
+}
+
+func (cveinfo CveInfo) GetImageListForCVE(repo string, id string, imgStore *storage.ImageStore,
+	trivyConfig *config.Config) ([]*string, error) {
+	tags := make([]*string, 0)
+
+	tagList, err := imgStore.GetImageTags(repo)
+	if err != nil {
+		cveinfo.Log.Error().Err(err).Msg("unable to get list of image tag")
+
+		return tags, err
+	}
+
+	rootDir := imgStore.RootDir()
+
+	for _, tag := range tagList {
+		trivyConfig.TrivyConfig.Input = fmt.Sprintf("%s:%s", path.Join(rootDir, repo), tag)
+
+		isValidImage, _ := cveinfo.IsValidImageFormat(trivyConfig.TrivyConfig.Input)
+		if !isValidImage {
+			cveinfo.Log.Debug().Str("image", repo+":"+tag).Msg("image media type not supported for scanning")
+
+			continue
+		}
+
+		cveinfo.Log.Info().Str("image", repo+":"+tag).Msg("scanning image")
+
+		results, err := ScanImage(trivyConfig)
+		if err != nil {
+			cveinfo.Log.Error().Err(err).Str("image", repo+":"+tag).Msg("unable to scan image")
+
+			continue
+		}
+
+		for _, result := range results {
+			for _, vulnerability := range result.Vulnerabilities {
+				if vulnerability.VulnerabilityID == id {
+					copyImgTag := tag
+					tags = append(tags, &copyImgTag)
+
+					break
+				}
+			}
+		}
+	}
+
+	return tags, nil
+}
+
 // GetImageTagsWithTimestamp returns a list of image tags with timestamp available in the specified repository.
-func (cveinfo CveInfo) GetImageTagsWithTimestamp(rootDir string, repo string) ([]TagInfo, error) {
+func (cveinfo CveInfo) GetImageTagsWithTimestamp(repo string) ([]TagInfo, error) {
 	tagsInfo := make([]TagInfo, 0)
 
-	dir := path.Join(rootDir, repo)
+	imagePath := cveinfo.GetImageRepoPath(repo)
-	if !dirExists(dir) {
+	if !dirExists(imagePath) {
 		return nil, errors.ErrRepoNotFound
 	}
 
-	manifests, err := cveinfo.getImageManifests(dir)
+	manifests, err := cveinfo.getImageManifests(imagePath)
 
 	if err != nil {
 		cveinfo.Log.Error().Err(err).Msg("unable to read image manifests")
@@ -135,7 +284,7 @@ func (cveinfo CveInfo) GetImageTagsWithTimestamp(rootDir string, repo string) ([
 
 		v, ok := manifest.Annotations[ispec.AnnotationRefName]
 		if ok {
-			imageBlobManifest, err := cveinfo.getImageBlobManifest(dir, digest)
+			imageBlobManifest, err := cveinfo.getImageBlobManifest(imagePath, digest)
 
 			if err != nil {
 				cveinfo.Log.Error().Err(err).Msg("unable to read image blob manifest")
@@ -143,7 +292,7 @@ func (cveinfo CveInfo) GetImageTagsWithTimestamp(rootDir string, repo string) ([
 				return tagsInfo, err
 			}
 
-			imageInfo, err := cveinfo.getImageInfo(dir, imageBlobManifest.Config.Digest)
+			imageInfo, err := cveinfo.getImageInfo(imagePath, imageBlobManifest.Config.Digest)
 			if err != nil {
 				cveinfo.Log.Error().Err(err).Msg("unable to read image info")
 

pkg/extensions/search/cve/cve_test.go

@@ -16,6 +16,7 @@ import (
 	ext "github.com/anuvu/zot/pkg/extensions"
 	cveinfo "github.com/anuvu/zot/pkg/extensions/search/cve"
 	"github.com/anuvu/zot/pkg/log"
+	"github.com/anuvu/zot/pkg/storage"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	. "github.com/smartystreets/goconvey/convey"
 	"gopkg.in/resty.v1"
@@ -23,8 +24,9 @@ import (
 
 // nolint:gochecknoglobals
 var (
-	cve   *cveinfo.CveInfo
+	cve            *cveinfo.CveInfo
-	dbDir string
+	dbDir          string
+	updateDuration time.Duration
 )
 
 const (
@@ -76,7 +78,11 @@ func testSetup() error {
 		return err
 	}
 
-	cve = &cveinfo.CveInfo{Log: log.NewLogger("debug", "")}
+	log := log.NewLogger("debug", "")
+
+	cve = &cveinfo.CveInfo{Log: log}
+
+	cve.StoreController = storage.StoreController{DefaultStore: storage.NewImageStore(dir, false, false, log)}
 
 	dbDir = dir
 
@@ -365,6 +371,67 @@ func makeHtpasswdFile() string {
 	return f.Name()
 }
 
+func TestMultipleStoragePath(t *testing.T) {
+	Convey("Test multiple storage path", t, func() {
+		// Create temporary directory
+		firstRootDir, err := ioutil.TempDir("", "util_test")
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(firstRootDir)
+
+		secondRootDir, err := ioutil.TempDir("", "util_test")
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(secondRootDir)
+
+		thirdRootDir, err := ioutil.TempDir("", "util_test")
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(thirdRootDir)
+
+		log := log.NewLogger("debug", "")
+
+		// Create ImageStore
+		firstStore := storage.NewImageStore(firstRootDir, false, false, log)
+
+		secondStore := storage.NewImageStore(secondRootDir, false, false, log)
+
+		thirdStore := storage.NewImageStore(thirdRootDir, false, false, log)
+
+		storeController := storage.StoreController{}
+
+		storeController.DefaultStore = firstStore
+
+		subStore := make(map[string]*storage.ImageStore)
+
+		subStore["/a"] = secondStore
+		subStore["/b"] = thirdStore
+
+		storeController.SubStore = subStore
+
+		cveInfo, err := cveinfo.GetCVEInfo(storeController, log)
+
+		So(err, ShouldBeNil)
+		So(cveInfo.StoreController.DefaultStore, ShouldNotBeNil)
+		So(cveInfo.StoreController.SubStore, ShouldNotBeNil)
+
+		imagePath := cveInfo.GetImageRepoPath("zot-test")
+		So(imagePath, ShouldEqual, path.Join(firstRootDir, "zot-test"))
+
+		imagePath = cveInfo.GetImageRepoPath("a/zot-a-test")
+		So(imagePath, ShouldEqual, path.Join(secondRootDir, "a/zot-a-test"))
+
+		imagePath = cveInfo.GetImageRepoPath("b/zot-b-test")
+		So(imagePath, ShouldEqual, path.Join(thirdRootDir, "b/zot-b-test"))
+
+		imagePath = cveInfo.GetImageRepoPath("c/zot-c-test")
+		So(imagePath, ShouldEqual, path.Join(firstRootDir, "c/zot-c-test"))
+	})
+}
+
 func TestDownloadDB(t *testing.T) {
 	Convey("Download DB passing invalid dir", t, func() {
 		err := testSetup()
@@ -425,35 +492,35 @@ func TestImageFormat(t *testing.T) {
 
 func TestImageTag(t *testing.T) {
 	Convey("Test image tag", t, func() {
-		imageTags, err := cve.GetImageTagsWithTimestamp(dbDir, "zot-test")
+		imageTags, err := cve.GetImageTagsWithTimestamp("zot-test")
 		So(err, ShouldBeNil)
 		So(len(imageTags), ShouldNotEqual, 0)
 
-		imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-tes")
+		imageTags, err = cve.GetImageTagsWithTimestamp("zot-tes")
 		So(err, ShouldNotBeNil)
 		So(imageTags, ShouldBeNil)
 
-		imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-noindex-test")
+		imageTags, err = cve.GetImageTagsWithTimestamp("zot-noindex-test")
 		So(err, ShouldNotBeNil)
 		So(len(imageTags), ShouldEqual, 0)
 
-		imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-squashfs-noblobs")
+		imageTags, err = cve.GetImageTagsWithTimestamp("zot-squashfs-noblobs")
 		So(err, ShouldNotBeNil)
 		So(len(imageTags), ShouldEqual, 0)
 
-		imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-squashfs-invalid-index")
+		imageTags, err = cve.GetImageTagsWithTimestamp("zot-squashfs-invalid-index")
 		So(err, ShouldNotBeNil)
 		So(len(imageTags), ShouldEqual, 0)
 
-		imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-squashfs-invalid-blob")
+		imageTags, err = cve.GetImageTagsWithTimestamp("zot-squashfs-invalid-blob")
 		So(err, ShouldNotBeNil)
 		So(len(imageTags), ShouldEqual, 0)
 
-		imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-invalid-layer")
+		imageTags, err = cve.GetImageTagsWithTimestamp("zot-invalid-layer")
 		So(err, ShouldNotBeNil)
 		So(len(imageTags), ShouldEqual, 0)
 
-		imageTags, err = cve.GetImageTagsWithTimestamp(dbDir, "zot-no-layer")
+		imageTags, err = cve.GetImageTagsWithTimestamp("zot-no-layer")
 		So(err, ShouldNotBeNil)
 		So(len(imageTags), ShouldEqual, 0)
 	})
@@ -461,7 +528,7 @@ func TestImageTag(t *testing.T) {
 
 func TestCVESearch(t *testing.T) {
 	Convey("Test image vulenrability scanning", t, func() {
-		updateDuration, _ := time.ParseDuration("1h")
+		updateDuration, _ = time.ParseDuration("1h")
 		config := api.NewConfig()
 		config.HTTP.Port = SecurePort1
 		htpasswdPath := makeHtpasswdFile()
@@ -473,7 +540,6 @@ func TestCVESearch(t *testing.T) {
 			},
 		}
 		c := api.NewController(config)
-		defer os.RemoveAll(dbDir)
 		c.Config.Storage.RootDirectory = dbDir
 		cveConfig := &ext.CVEConfig{
 			UpdateInterval: updateDuration,
@@ -698,13 +764,30 @@ func TestCVEConfig(t *testing.T) {
 			},
 		}
 		c := api.NewController(config)
-		dir, err := ioutil.TempDir("", "oci-repo-test")
+		firstDir, err := ioutil.TempDir("", "oci-repo-test")
 		if err != nil {
 			panic(err)
 		}
-		defer os.RemoveAll(dir)
+
-		c.Config.Storage.RootDirectory = dir
+		secondDir, err := ioutil.TempDir("", "oci-repo-test")
-		c.Config.Extensions = &ext.ExtensionConfig{}
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(firstDir)
+		defer os.RemoveAll(secondDir)
+
+		err = copyFiles("../../../../test/data", path.Join(secondDir, "a"))
+		if err != nil {
+			panic(err)
+		}
+
+		c.Config.Storage.RootDirectory = firstDir
+		subPaths := make(map[string]api.StorageConfig)
+		subPaths["/a"] = api.StorageConfig{
+			RootDirectory: secondDir,
+		}
+		c.Config.Storage.SubPaths = subPaths
+
 		go func() {
 			// this blocks
 			if err := c.Run(); err != nil {
@@ -721,6 +804,22 @@ func TestCVEConfig(t *testing.T) {
 			time.Sleep(100 * time.Millisecond)
 		}
 
+		resp, _ := resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/v2/")
+		So(resp, ShouldNotBeNil)
+		So(resp.StatusCode(), ShouldEqual, 200)
+
+		resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/v2/_catalog")
+		So(resp, ShouldNotBeNil)
+		So(resp.StatusCode(), ShouldEqual, 200)
+
+		resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/v2/a/zot-test/tags/list")
+		So(resp, ShouldNotBeNil)
+		So(resp.StatusCode(), ShouldEqual, 200)
+
+		resp, _ = resty.R().SetBasicAuth(username, passphrase).Get(BaseURL1 + "/v2/zot-test/tags/list")
+		So(resp, ShouldNotBeNil)
+		So(resp.StatusCode(), ShouldEqual, 404)
+
 		defer func() {
 			ctx := context.Background()
 			_ = c.Server.Shutdown(ctx)

pkg/extensions/search/cve/models.go

@@ -5,13 +5,20 @@ import (
 	"time"
 
 	"github.com/anuvu/zot/pkg/log"
+	"github.com/anuvu/zot/pkg/storage"
 	config "github.com/aquasecurity/trivy/integration/config"
 )
 
 // CveInfo ...
 type CveInfo struct {
-	Log            log.Logger
+	Log                log.Logger
-	CveTrivyConfig *config.Config
+	CveTrivyController CveTrivyController
+	StoreController    storage.StoreController
+}
+
+type CveTrivyController struct {
+	DefaultCveConfig *config.Config
+	SubCveConfig     map[string]*config.Config
 }
 
 type TagInfo struct {

pkg/extensions/search/resolver.go

@@ -4,10 +4,11 @@ package search
 
 import (
 	"context"
-	"path"
+	"fmt"
 	"strings"
 
 	"github.com/anuvu/zot/pkg/log"
+	"github.com/aquasecurity/trivy/integration/config"
 
 	cveinfo "github.com/anuvu/zot/pkg/extensions/search/cve"
 	"github.com/anuvu/zot/pkg/storage"
@@ -15,9 +16,8 @@ import (
 
 // Resolver ...
 type Resolver struct {
-	cveInfo  *cveinfo.CveInfo
+	cveInfo         *cveinfo.CveInfo
-	imgStore *storage.ImageStore
+	storeController storage.StoreController
-	dir      string
 }
 
 // Query ...
@@ -35,33 +35,31 @@ type cveDetail struct {
 }
 
 // GetResolverConfig ...
-func GetResolverConfig(dir string, log log.Logger, imgstorage *storage.ImageStore) Config {
+func GetResolverConfig(log log.Logger, storeController storage.StoreController) Config {
-	config, err := cveinfo.NewTrivyConfig(dir)
+	cveInfo, err := cveinfo.GetCVEInfo(storeController, log)
 	if err != nil {
 		panic(err)
 	}
 
-	cve := &cveinfo.CveInfo{Log: log, CveTrivyConfig: config}
+	resConfig := &Resolver{cveInfo: cveInfo, storeController: storeController}
-
-	resConfig := &Resolver{cveInfo: cve, imgStore: imgstorage, dir: dir}
 
 	return Config{Resolvers: resConfig, Directives: DirectiveRoot{},
 		Complexity: ComplexityRoot{}}
 }
 
 func (r *queryResolver) CVEListForImage(ctx context.Context, image string) (*CVEResultForImage, error) {
-	r.cveInfo.CveTrivyConfig.TrivyConfig.Input = path.Join(r.dir, image)
+	trivyConfig := r.cveInfo.GetTrivyConfig(image)
 
 	r.cveInfo.Log.Info().Str("image", image).Msg("scanning image")
 
-	isValidImage, err := r.cveInfo.IsValidImageFormat(r.cveInfo.CveTrivyConfig.TrivyConfig.Input)
+	isValidImage, err := r.cveInfo.IsValidImageFormat(trivyConfig.TrivyConfig.Input)
 	if !isValidImage {
 		r.cveInfo.Log.Debug().Str("image", image).Msg("image media type not supported for scanning")
 
 		return &CVEResultForImage{}, err
 	}
 
-	cveResults, err := cveinfo.ScanImage(r.cveInfo.CveTrivyConfig)
+	cveResults, err := cveinfo.ScanImage(trivyConfig)
 	if err != nil {
 		r.cveInfo.Log.Error().Err(err).Msg("unable to scan image repository")
 
@@ -134,62 +132,70 @@ func (r *queryResolver) CVEListForImage(ctx context.Context, image string) (*CVE
 }
 
 func (r *queryResolver) ImageListForCve(ctx context.Context, id string) ([]*ImgResultForCve, error) {
-	cveResult := []*ImgResultForCve{}
+	finalCveResult := []*ImgResultForCve{}
 
 	r.cveInfo.Log.Info().Msg("extracting repositories")
 
-	repoList, err := r.imgStore.GetRepositories()
+	defaultStore := r.storeController.DefaultStore
+
+	defaultTrivyConfig := r.cveInfo.CveTrivyController.DefaultCveConfig
+
+	repoList, err := defaultStore.GetRepositories()
 	if err != nil {
 		r.cveInfo.Log.Error().Err(err).Msg("unable to search repositories")
 
-		return cveResult, err
+		return finalCveResult, err
 	}
 
-	r.cveInfo.Log.Info().Msg("scanning each repository")
+	r.cveInfo.Log.Info().Msg("scanning each global repository")
+
+	cveResult, err := r.getImageListForCVE(repoList, id, defaultStore, defaultTrivyConfig)
+	if err != nil {
+		r.cveInfo.Log.Error().Err(err).Msg("error getting cve list for global repositories")
+
+		return finalCveResult, err
+	}
+
+	finalCveResult = append(finalCveResult, cveResult...)
+
+	subStore := r.storeController.SubStore
+	for route, store := range subStore {
+		subRepoList, err := store.GetRepositories()
+		if err != nil {
+			r.cveInfo.Log.Error().Err(err).Msg("unable to search repositories")
+
+			return cveResult, err
+		}
+
+		subTrivyConfig := r.cveInfo.CveTrivyController.SubCveConfig[route]
+
+		subCveResult, err := r.getImageListForCVE(subRepoList, id, store, subTrivyConfig)
+		if err != nil {
+			r.cveInfo.Log.Error().Err(err).Msg("unable to get cve result for sub repositories")
+
+			return finalCveResult, err
+		}
+
+		finalCveResult = append(finalCveResult, subCveResult...)
+	}
+
+	return finalCveResult, nil
+}
+
+func (r *queryResolver) getImageListForCVE(repoList []string, id string, imgStore *storage.ImageStore,
+	trivyConfig *config.Config) ([]*ImgResultForCve, error) {
+	cveResult := []*ImgResultForCve{}
 
 	for _, repo := range repoList {
 		r.cveInfo.Log.Info().Str("repo", repo).Msg("extracting list of tags available in image repo")
 
-		tagList, err := r.imgStore.GetImageTags(repo)
+		name := repo
+
+		tags, err := r.cveInfo.GetImageListForCVE(repo, id, imgStore, trivyConfig)
 		if err != nil {
-			r.cveInfo.Log.Error().Err(err).Msg("unable to get list of image tag")
+			r.cveInfo.Log.Error().Err(err).Msg("error getting tag")
-		}
 
-		var name string
+			return cveResult, err
-
-		tags := make([]*string, 0)
-
-		for _, tag := range tagList {
-			r.cveInfo.CveTrivyConfig.TrivyConfig.Input = path.Join(r.dir, repo+":"+tag)
-
-			isValidImage, _ := r.cveInfo.IsValidImageFormat(r.cveInfo.CveTrivyConfig.TrivyConfig.Input)
-			if !isValidImage {
-				r.cveInfo.Log.Debug().Str("image", repo+":"+tag).Msg("image media type not supported for scanning")
-
-				continue
-			}
-
-			r.cveInfo.Log.Info().Str("image", repo+":"+tag).Msg("scanning image")
-
-			results, err := cveinfo.ScanImage(r.cveInfo.CveTrivyConfig)
-			if err != nil {
-				r.cveInfo.Log.Error().Err(err).Str("image", repo+":"+tag).Msg("unable to scan image")
-
-				continue
-			}
-
-			name = repo
-
-			for _, result := range results {
-				for _, vulnerability := range result.Vulnerabilities {
-					if vulnerability.VulnerabilityID == id {
-						copyImgTag := tag
-						tags = append(tags, &copyImgTag)
-
-						break
-					}
-				}
-			}
 		}
 
 		if len(tags) != 0 {
@@ -203,9 +209,17 @@ func (r *queryResolver) ImageListForCve(ctx context.Context, id string) ([]*ImgR
 func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, image string) (*ImgResultForFixedCve, error) { // nolint: lll
 	imgResultForFixedCVE := &ImgResultForFixedCve{}
 
+	r.cveInfo.Log.Info().Str("image", image).Msg("retrieving image path")
+
+	imagePath := r.cveInfo.GetImageRepoPath(image)
+
+	r.cveInfo.Log.Info().Str("image", image).Msg("retrieving trivy config")
+
+	trivyConfig := r.cveInfo.GetTrivyConfig(image)
+
 	r.cveInfo.Log.Info().Str("image", image).Msg("extracting list of tags available in image")
 
-	tagsInfo, err := r.cveInfo.GetImageTagsWithTimestamp(r.dir, image)
+	tagsInfo, err := r.cveInfo.GetImageTagsWithTimestamp(image)
 	if err != nil {
 		r.cveInfo.Log.Error().Err(err).Msg("unable to read image tags")
 
@@ -217,9 +231,9 @@ func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, im
 	var hasCVE bool
 
 	for _, tag := range tagsInfo {
-		r.cveInfo.CveTrivyConfig.TrivyConfig.Input = path.Join(r.dir, image+":"+tag.Name)
+		trivyConfig.TrivyConfig.Input = fmt.Sprintf("%s:%s", imagePath, tag.Name)
 
-		isValidImage, _ := r.cveInfo.IsValidImageFormat(r.cveInfo.CveTrivyConfig.TrivyConfig.Input)
+		isValidImage, _ := r.cveInfo.IsValidImageFormat(trivyConfig.TrivyConfig.Input)
 		if !isValidImage {
 			r.cveInfo.Log.Debug().Str("image",
 				image+":"+tag.Name).Msg("image media type not supported for scanning, adding as an infected image")
@@ -231,7 +245,7 @@ func (r *queryResolver) ImageListWithCVEFixed(ctx context.Context, id string, im
 
 		r.cveInfo.Log.Info().Str("image", image+":"+tag.Name).Msg("scanning image")
 
-		results, err := cveinfo.ScanImage(r.cveInfo.CveTrivyConfig)
+		results, err := cveinfo.ScanImage(trivyConfig)
 		if err != nil {
 			r.cveInfo.Log.Error().Err(err).Str("image", image+":"+tag.Name).Msg("unable to scan image")
 

pkg/storage/storage.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strings"
 	"sync"
 	"time"
 
@@ -37,6 +38,11 @@ type BlobUpload struct {
 	ID        string
 }
 
+type StoreController struct {
+	DefaultStore *ImageStore
+	SubStore     map[string]*ImageStore
+}
+
 // ImageStore provides the image storage operations.
 type ImageStore struct {
 	rootDir     string
@@ -48,6 +54,39 @@ type ImageStore struct {
 	log         zerolog.Logger
 }
 
+func (is *ImageStore) RootDir() string {
+	return is.rootDir
+}
+
+func getRoutePrefix(name string) string {
+	names := strings.SplitN(name, "/", 2)
+
+	if len(names) != 2 { // nolint: gomnd
+		// it means route is of global storage e.g "centos:latest"
+		if len(names) == 1 {
+			return "/"
+		}
+	}
+
+	return fmt.Sprintf("/%s", names[0])
+}
+
+func (sc StoreController) GetImageStore(name string) *ImageStore {
+	if sc.SubStore != nil {
+		// SubStore is being provided, now we need to find equivalent image store and this will be found by splitting name
+		prefixName := getRoutePrefix(name)
+
+		imgStore, ok := sc.SubStore[prefixName]
+		if !ok {
+			imgStore = sc.DefaultStore
+		}
+
+		return imgStore
+	}
+
+	return sc.DefaultStore
+}
+
 // NewImageStore returns a new image store backed by a file storage.
 func NewImageStore(rootDir string, gc bool, dedupe bool, log zlog.Logger) *ImageStore {
 	if _, err := os.Stat(rootDir); os.IsNotExist(err) {
@@ -1174,6 +1213,36 @@ func Scrub(dir string, fix bool) error {
 
 // utility routines
 
+func CheckHardLink(srcFileName string, destFileName string) error {
+	return os.Link(srcFileName, destFileName)
+}
+
+func ValidateHardLink(rootDir string) error {
+	err := ioutil.WriteFile(path.Join(rootDir, "hardlinkcheck.txt"), //nolint: gosec
+		[]byte("check whether hardlinks work on filesystem"), 0644)
+	if err != nil {
+		return err
+	}
+
+	err = CheckHardLink(path.Join(rootDir, "hardlinkcheck.txt"), path.Join(rootDir, "duphardlinkcheck.txt"))
+	if err != nil {
+		// Remove hardlinkcheck.txt if hardlink fails
+		zerr := os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt"))
+		if zerr != nil {
+			return zerr
+		}
+
+		return err
+	}
+
+	err = os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt"))
+	if err != nil {
+		return err
+	}
+
+	return os.RemoveAll(path.Join(rootDir, "duphardlinkcheck.txt"))
+}
+
 func dirExists(d string) bool {
 	fi, err := os.Stat(d)
 	if err != nil && os.IsNotExist(err) {

pkg/storage/storage_test.go

@@ -598,3 +598,89 @@ func TestNegativeCases(t *testing.T) {
 		So(err, ShouldNotBeNil)
 	})
 }
+
+func TestHardLink(t *testing.T) {
+	Convey("Test if filesystem supports hardlink", t, func() {
+		dir, err := ioutil.TempDir("", "storage-hard-test")
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(dir)
+
+		err = storage.ValidateHardLink(dir)
+		So(err, ShouldBeNil)
+
+		err = ioutil.WriteFile(path.Join(dir, "hardtest.txt"), []byte("testing hard link code"), 0644) //nolint: gosec
+		if err != nil {
+			panic(err)
+		}
+
+		err = os.Chmod(dir, 0400)
+		if err != nil {
+			panic(err)
+		}
+
+		err = storage.CheckHardLink(path.Join(dir, "hardtest.txt"), path.Join(dir, "duphardtest.txt"))
+		So(err, ShouldNotBeNil)
+
+		err = os.Chmod(dir, 0644)
+		if err != nil {
+			panic(err)
+		}
+	})
+}
+
+func TestStorageHandler(t *testing.T) {
+	Convey("Test storage handler", t, func() {
+		// Create temporary directory
+		firstRootDir, err := ioutil.TempDir("", "util_test")
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(firstRootDir)
+
+		secondRootDir, err := ioutil.TempDir("", "util_test")
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(secondRootDir)
+
+		thirdRootDir, err := ioutil.TempDir("", "util_test")
+		if err != nil {
+			panic(err)
+		}
+		defer os.RemoveAll(thirdRootDir)
+
+		log := log.NewLogger("debug", "")
+
+		// Create ImageStore
+		firstStore := storage.NewImageStore(firstRootDir, false, false, log)
+
+		secondStore := storage.NewImageStore(secondRootDir, false, false, log)
+
+		thirdStore := storage.NewImageStore(thirdRootDir, false, false, log)
+
+		storeController := storage.StoreController{}
+
+		storeController.DefaultStore = firstStore
+
+		subStore := make(map[string]*storage.ImageStore)
+
+		subStore["/a"] = secondStore
+		subStore["/b"] = thirdStore
+
+		storeController.SubStore = subStore
+
+		is := storeController.GetImageStore("zot-x-test")
+		So(is.RootDir(), ShouldEqual, firstRootDir)
+
+		is = storeController.GetImageStore("a/zot-a-test")
+		So(is.RootDir(), ShouldEqual, secondRootDir)
+
+		is = storeController.GetImageStore("b/zot-b-test")
+		So(is.RootDir(), ShouldEqual, thirdRootDir)
+
+		is = storeController.GetImageStore("c/zot-c-test")
+		So(is.RootDir(), ShouldEqual, firstRootDir)
+	})
+}