feat(redis): dummy implementation of MetaDB interface for redis cache

Signed-off-by: Alexei Dodon <adodon@cisco.com>

Makefile

@@ -510,6 +510,7 @@ run-blackbox-cloud-ci: check-blackbox-prerequisites check-awslocal binary $(BATS
 	echo running cloud CI bats tests; \
 	$(BATS) $(BATS_FLAGS) test/blackbox/cloud_only.bats
 	$(BATS) $(BATS_FLAGS) test/blackbox/sync_cloud.bats
+	$(BATS) $(BATS_FLAGS) test/blackbox/redis_s3.bats
 
 .PHONY: run-blackbox-dedupe-nightly
 run-blackbox-dedupe-nightly: check-blackbox-prerequisites check-awslocal binary binary-minimal

examples/config-redis.json

@@ -0,0 +1,37 @@
+{
+    "distSpecVersion": "1.1.0",
+    "storage": {
+        "dedupe": true,
+        "gc": true,
+        "rootDirectory": "/tmp/zot",
+        "cacheDriver": {
+            "name": "redis",
+            "rootDir": "/tmp/zot/_redis",
+            "url": "redis://localhost:6379"
+        },
+        "storageDriver": {
+            "name": "s3",
+            "rootdirectory": "/zot",
+            "region": "us-east-2",
+            "regionendpoint": "localhost:4566",
+            "bucket": "zot-storage",
+            "secure": false,
+            "skipverify": false
+        }
+    },
+    "http": {
+        "address": "0.0.0.0",
+        "port": "8484"
+    },
+    "log": {
+        "level": "debug"
+    },
+    "extensions": {
+      "ui": {
+        "enable": true
+      },
+      "search": {
+        "enable": true
+      }
+    }
+}

pkg/cli/server/root.go

@@ -262,7 +262,8 @@ func validateCacheConfig(cfg *config.Config, log zlog.Logger) error {
 		}
 
 		// unsupported cache driver
-		if cfg.Storage.CacheDriver["name"] != storageConstants.DynamoDBDriverName {
+		if cfg.Storage.CacheDriver["name"] != storageConstants.DynamoDBDriverName &&
+			cfg.Storage.CacheDriver["name"] != storageConstants.RedisDriverName {
 			log.Error().Err(zerr.ErrBadConfig).
 				Interface("cacheDriver", cfg.Storage.CacheDriver["name"]).Msg("invalid cache config, unsupported cache driver")
 
@@ -272,8 +273,8 @@ func validateCacheConfig(cfg *config.Config, log zlog.Logger) error {
 
 	if !cfg.Storage.RemoteCache && cfg.Storage.CacheDriver != nil {
 		log.Warn().Err(zerr.ErrBadConfig).Str("directory", cfg.Storage.RootDirectory).
-			Msg("invalid cache config, remoteCache set to false but cacheDriver config (remote caching) provided for directory" +
-				"will ignore and use local caching")
+			Msg("invalid cache config, remoteCache set to false but cacheDriver config (remote caching) provided for " +
+				"directory will ignore and use local caching")
 	}
 
 	// subpaths

pkg/meta/meta.go

@@ -2,6 +2,7 @@ package meta
 
 import (
 	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
+	"github.com/redis/go-redis/v9"
 	"go.etcd.io/bbolt"
 
 	"zotregistry.dev/zot/errors"
@@ -9,19 +10,31 @@ import (
 	"zotregistry.dev/zot/pkg/log"
 	"zotregistry.dev/zot/pkg/meta/boltdb"
 	mdynamodb "zotregistry.dev/zot/pkg/meta/dynamodb"
+	"zotregistry.dev/zot/pkg/meta/redisdb"
 	mTypes "zotregistry.dev/zot/pkg/meta/types"
+	sconstants "zotregistry.dev/zot/pkg/storage/constants"
 )
 
 func New(storageConfig config.StorageConfig, log log.Logger) (mTypes.MetaDB, error) {
 	if storageConfig.RemoteCache {
-		dynamoParams := getDynamoParams(storageConfig.CacheDriver, log)
+		if storageConfig.CacheDriver["name"] == sconstants.DynamoDBDriverName {
+			dynamoParams := getDynamoParams(storageConfig.CacheDriver, log)
 
-		client, err := mdynamodb.GetDynamoClient(dynamoParams)
-		if err != nil {
+			client, err := mdynamodb.GetDynamoClient(dynamoParams)
+			if err != nil {
+				return nil, err
+			}
+
+			return Create(sconstants.DynamoDBDriverName, client, dynamoParams, log) //nolint:contextcheck
+		}
+		// go-redis supports connecting via the redis uri specification (more convenient than parameter parsing)
+		redisURL := getRedisURL(storageConfig.CacheDriver, log)
+		client, err := redisdb.GetRedisClient(redisURL)
+		if err != nil { //nolint:wsl
 			return nil, err
 		}
 
-		return Create("dynamodb", client, dynamoParams, log) //nolint:contextcheck
+		return Create(sconstants.RedisDriverName, client, &redisdb.RedisDB{Client: client}, log) //nolint:contextcheck
 	}
 
 	params := boltdb.DBParameters{}
@@ -51,6 +64,18 @@ func Create(dbtype string, dbDriver, parameters interface{}, log log.Logger, //n
 
 			return boltdb.New(properDriver, log)
 		}
+	case "redis":
+		{
+			properDriver, ok := dbDriver.(*redis.Client)
+			if !ok {
+				log.Error().Err(errors.ErrTypeAssertionFailed).
+					Msgf("failed to cast type, expected type '%T' but got '%T'", &redis.Client{}, dbDriver)
+
+				return nil, errors.ErrTypeAssertionFailed
+			}
+
+			return redisdb.New(properDriver, log)
+		}
 	case "dynamodb":
 		{
 			properDriver, ok := dbDriver.(*dynamodb.Client)
@@ -122,6 +147,16 @@ func getDynamoParams(cacheDriverConfig map[string]interface{}, log log.Logger) m
 	}
 }
 
+func getRedisURL(cacheDriverConfig map[string]interface{}, log log.Logger) string {
+	url, ok := toStringIfOk(cacheDriverConfig, "url", log)
+
+	if !ok {
+		log.Panic().Msg("redis parameters are not specified correctly, can't proceed")
+	}
+
+	return url
+}
+
 func toStringIfOk(cacheDriverConfig map[string]interface{}, param string, log log.Logger) (string, bool) {
 	val, ok := cacheDriverConfig[param]
 

pkg/meta/redisdb/redis.go

@@ -0,0 +1,262 @@
+package redisdb
+
+import (
+	"context"
+	"time"
+
+	godigest "github.com/opencontainers/go-digest"
+	"github.com/redis/go-redis/v9"
+
+	"zotregistry.dev/zot/pkg/log"
+	mTypes "zotregistry.dev/zot/pkg/meta/types"
+)
+
+type RedisDB struct {
+	Client        *redis.Client
+	imgTrustStore mTypes.ImageTrustStore
+	Log           log.Logger
+}
+
+func New(client *redis.Client, log log.Logger) (*RedisDB, error) {
+	redisWrapper := RedisDB{
+		Client:        client,
+		imgTrustStore: nil,
+		Log:           log,
+	}
+
+	// Using the Config value, create the DynamoDB client
+	return &redisWrapper, nil
+}
+
+func GetRedisClient(url string) (*redis.Client, error) {
+	opts, err := redis.ParseURL(url)
+	if err != nil {
+		return nil, err
+	}
+
+	return redis.NewClient(opts), nil
+}
+
+// GetStarredRepos returns starred repos and takes current user in consideration.
+func (rc *RedisDB) GetStarredRepos(ctx context.Context) ([]string, error) {
+	return []string{}, nil
+}
+
+// GetBookmarkedRepos returns bookmarked repos and takes current user in consideration.
+func (rc *RedisDB) GetBookmarkedRepos(ctx context.Context) ([]string, error) {
+	return []string{}, nil
+}
+
+// ToggleStarRepo adds/removes stars on repos.
+func (rc *RedisDB) ToggleStarRepo(ctx context.Context, reponame string) (mTypes.ToggleState, error) {
+	return 0, nil
+}
+
+// ToggleBookmarkRepo adds/removes bookmarks on repos.
+func (rc *RedisDB) ToggleBookmarkRepo(ctx context.Context, reponame string) (mTypes.ToggleState, error) {
+	return 0, nil
+}
+
+// UserDB profile/api key CRUD.
+func (rc *RedisDB) GetUserData(ctx context.Context) (mTypes.UserData, error) {
+	return mTypes.UserData{}, nil
+}
+
+func (rc *RedisDB) SetUserData(ctx context.Context, userData mTypes.UserData) error {
+	return nil
+}
+
+func (rc *RedisDB) SetUserGroups(ctx context.Context, groups []string) error {
+	return nil
+}
+
+func (rc *RedisDB) GetUserGroups(ctx context.Context) ([]string, error) {
+	return []string{}, nil
+}
+
+func (rc *RedisDB) DeleteUserData(ctx context.Context) error {
+	return nil
+}
+
+func (rc *RedisDB) GetUserAPIKeyInfo(hashedKey string) (string, error) {
+	return "", nil
+}
+
+func (rc *RedisDB) GetUserAPIKeys(ctx context.Context) ([]mTypes.APIKeyDetails, error) {
+	return []mTypes.APIKeyDetails{}, nil
+}
+
+func (rc *RedisDB) AddUserAPIKey(ctx context.Context, hashedKey string, apiKeyDetails *mTypes.APIKeyDetails) error {
+	return nil
+}
+
+func (rc *RedisDB) IsAPIKeyExpired(ctx context.Context, hashedKey string) (bool, error) {
+	return false, nil
+}
+
+func (rc *RedisDB) UpdateUserAPIKeyLastUsed(ctx context.Context, hashedKey string) error {
+	return nil
+}
+
+func (rc *RedisDB) DeleteUserAPIKey(ctx context.Context, id string) error {
+	return nil
+}
+
+func (rc *RedisDB) SetImageMeta(digest godigest.Digest, imageMeta mTypes.ImageMeta) error {
+	return nil
+}
+
+// SetRepoReference sets the given image data to the repo metadata.
+func (rc *RedisDB) SetRepoReference(ctx context.Context, repo string,
+	reference string, imageMeta mTypes.ImageMeta,
+) error {
+	return nil
+}
+
+// SearchRepos searches for repos given a search string.
+func (rc *RedisDB) SearchRepos(ctx context.Context, searchText string) ([]mTypes.RepoMeta, error) {
+	return []mTypes.RepoMeta{}, nil
+}
+
+// SearchTags searches for images(repo:tag) given a search string.
+func (rc *RedisDB) SearchTags(ctx context.Context, searchText string) ([]mTypes.FullImageMeta, error) {
+	return []mTypes.FullImageMeta{}, nil
+}
+
+// FilterTags filters for images given a filter function.
+func (rc *RedisDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc,
+	filterFunc mTypes.FilterFunc,
+) ([]mTypes.FullImageMeta, error) {
+	return []mTypes.FullImageMeta{}, nil
+}
+
+// FilterRepos filters for repos given a filter function.
+func (rc *RedisDB) FilterRepos(ctx context.Context, rankName mTypes.FilterRepoNameFunc,
+	filterFunc mTypes.FilterFullRepoFunc,
+) ([]mTypes.RepoMeta, error) {
+	return []mTypes.RepoMeta{}, nil
+}
+
+// GetRepoMeta returns the full information about a repo.
+func (rc *RedisDB) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMeta, error) {
+	return mTypes.RepoMeta{}, nil
+}
+
+// GetFullImageMeta returns the full information about an image.
+func (rc *RedisDB) GetFullImageMeta(ctx context.Context, repo string, tag string) (mTypes.FullImageMeta, error) {
+	return mTypes.FullImageMeta{}, nil
+}
+
+// GetImageMeta returns the raw information about an image.
+func (rc *RedisDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, error) {
+	return mTypes.ImageMeta{}, nil
+}
+
+// GetMultipleRepoMeta returns a list of all repos that match the given filter function.
+func (rc *RedisDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta mTypes.RepoMeta) bool) (
+	[]mTypes.RepoMeta, error,
+) {
+	return []mTypes.RepoMeta{}, nil
+}
+
+// AddManifestSignature adds signature metadata to a given manifest in the database.
+func (rc *RedisDB) AddManifestSignature(repo string, signedManifestDigest godigest.Digest,
+	sm mTypes.SignatureMetadata,
+) error {
+	return nil
+}
+
+// DeleteSignature deletes signature metadata to a given manifest from the database.
+func (rc *RedisDB) DeleteSignature(repo string, signedManifestDigest godigest.Digest,
+	sigMeta mTypes.SignatureMetadata,
+) error {
+	return nil
+}
+
+// UpdateSignaturesValidity checks and updates signatures validity of a given manifest.
+func (rc *RedisDB) UpdateSignaturesValidity(ctx context.Context, repo string, manifestDigest godigest.Digest) error {
+	return nil
+}
+
+// IncrementRepoStars adds 1 to the star count of an image.
+func (rc *RedisDB) IncrementRepoStars(repo string) error {
+	return nil
+}
+
+// DecrementRepoStars subtracts 1 from the star count of an image.
+func (rc *RedisDB) DecrementRepoStars(repo string) error {
+	return nil
+}
+
+// SetRepoMeta returns RepoMetadata of a repo from the database.
+func (rc *RedisDB) SetRepoMeta(repo string, repoMeta mTypes.RepoMeta) error {
+	return nil
+}
+
+// DeleteRepoMeta.
+func (rc *RedisDB) DeleteRepoMeta(repo string) error {
+	return nil
+}
+
+// GetReferrersInfo returns a list of  for all referrers of the given digest that match one of the
+// artifact types.
+func (rc *RedisDB) GetReferrersInfo(repo string, referredDigest godigest.Digest,
+	artifactTypes []string,
+) ([]mTypes.ReferrerInfo, error) {
+	return []mTypes.ReferrerInfo{}, nil
+}
+
+// UpdateStatsOnDownload adds 1 to the download count of an image and sets the timestamp of download.
+func (rc *RedisDB) UpdateStatsOnDownload(repo string, reference string) error {
+	return nil
+}
+
+// FilterImageMeta returns the image data for the given digests.
+func (rc *RedisDB) FilterImageMeta(ctx context.Context,
+	digests []string,
+) (map[mTypes.ImageDigest]mTypes.ImageMeta, error) {
+	return map[mTypes.ImageDigest]mTypes.ImageMeta{}, nil
+}
+
+/*
+	RemoveRepoReference removes the tag from RepoMetadata if the reference is a tag,
+
+it also removes its corresponding digest from Statistics, Signatures and Referrers if there are no tags
+pointing to it.
+If the reference is a digest then it will remove the digest from Statistics, Signatures and Referrers only
+if there are no tags pointing to the digest, otherwise it's noop.
+*/
+func (rc *RedisDB) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error {
+	return nil
+}
+
+// ResetRepoReferences resets all layout specific data (tags, signatures, referrers, etc.) but keep user and image
+// specific metadata such as star count, downloads other statistics.
+func (rc *RedisDB) ResetRepoReferences(repo string) error {
+	return nil
+}
+
+func (rc *RedisDB) GetRepoLastUpdated(repo string) time.Time {
+	return time.Now()
+}
+
+func (rc *RedisDB) GetAllRepoNames() ([]string, error) {
+	return []string{}, nil
+}
+
+// ResetDB will delete all data in the DB.
+func (rc *RedisDB) ResetDB() error {
+	return nil
+}
+
+func (rc *RedisDB) PatchDB() error {
+	return nil
+}
+
+func (rc *RedisDB) ImageTrustStore() mTypes.ImageTrustStore {
+	return rc.imgTrustStore
+}
+
+func (rc *RedisDB) SetImageTrustStore(imgTrustStore mTypes.ImageTrustStore) {
+	rc.imgTrustStore = imgTrustStore
+}

pkg/storage/cache.go

@@ -32,19 +32,32 @@ func CreateCacheDatabaseDriver(storageConfig config.StorageConfig, log zlog.Logg
 			return nil, nil
 		}
 
-		if name != constants.DynamoDBDriverName {
+		if name != constants.DynamoDBDriverName &&
+			name != constants.RedisDriverName {
 			log.Warn().Str("driver", name).Msg("remote cache driver unsupported!")
 
 			return nil, nil
 		}
 
-		// dynamodb
-		dynamoParams := cache.DynamoDBDriverParameters{}
-		dynamoParams.Endpoint, _ = storageConfig.CacheDriver["endpoint"].(string)
-		dynamoParams.Region, _ = storageConfig.CacheDriver["region"].(string)
-		dynamoParams.TableName, _ = storageConfig.CacheDriver["cachetablename"].(string)
+		if name == constants.DynamoDBDriverName {
+			// dynamodb
+			dynamoParams := cache.DynamoDBDriverParameters{}
+			dynamoParams.Endpoint, _ = storageConfig.CacheDriver["endpoint"].(string)
+			dynamoParams.Region, _ = storageConfig.CacheDriver["region"].(string)
+			dynamoParams.TableName, _ = storageConfig.CacheDriver["cachetablename"].(string)
 
-		return Create("dynamodb", dynamoParams, log)
+			return Create(name, dynamoParams, log)
+		}
+
+		if name == constants.RedisDriverName {
+			// redis
+			redisParams := cache.RedisDriverParameters{}
+			redisParams.RootDir, _ = storageConfig.CacheDriver["rootDir"].(string)
+			redisParams.URL, _ = storageConfig.CacheDriver["url"].(string)
+			redisParams.UseRelPaths = getUseRelPaths(&storageConfig)
+
+			return Create(name, redisParams, log)
+		}
 	}
 
 	return nil, nil

pkg/storage/cache/redis.go

@@ -31,7 +31,7 @@ func NewRedisCache(parameters interface{}, log zlog.Logger) (*RedisDriver, error
 	properParameters, ok := parameters.(RedisDriverParameters)
 	if !ok {
 		log.Error().Err(zerr.ErrTypeAssertionFailed).Msgf("failed to cast type, expected type '%T' but got '%T'",
-			BoltDBDriverParameters{}, parameters)
+			RedisDriverParameters{}, parameters)
 
 		return nil, zerr.ErrTypeAssertionFailed
 	}

pkg/storage/constants/constants.go

@@ -19,6 +19,7 @@ const (
 	DBCacheLockCheckTimeout = 10 * time.Second
 	BoltdbName              = "cache"
 	DynamoDBDriverName      = "dynamodb"
+	RedisDriverName         = "redis"
 	DefaultGCDelay          = 1 * time.Hour
 	DefaultRetentionDelay   = 24 * time.Hour
 	DefaultGCInterval       = 1 * time.Hour

test/blackbox/ci.sh

@@ -9,7 +9,7 @@ PATH=$PATH:${SCRIPTPATH}/../../hack/tools/bin
 
 tests=("pushpull" "pushpull_authn" "delete_images" "referrers" "metadata" "anonymous_policy"
       "annotations" "detect_manifest_collision" "cve" "sync" "sync_docker" "sync_replica_cluster"
-      "scrub" "garbage_collect" "metrics" "metrics_minimal" "multiarch_index" "redis_local" "redis_s3")
+      "scrub" "garbage_collect" "metrics" "metrics_minimal" "multiarch_index" "redis_local")
 
 for test in ${tests[*]}; do
     ${BATS} ${BATS_FLAGS} ${SCRIPTPATH}/${test}.bats > ${test}.log & pids+=($!)

test/blackbox/helpers_redis.bash

@@ -1,15 +1,12 @@
-ROOT_DIR=$(git rev-parse --show-toplevel)
-OS=$(go env GOOS)
-ARCH=$(go env GOARCH)
-ZOT_MINIMAL_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH}-minimal
-ZB_PATH=${ROOT_DIR}/bin/zb-${OS}-${ARCH}
-TEST_DATA_DIR=${BATS_FILE_TMPDIR}/test/data
 
 function redis_start() {
-  docker run -d --name redis_server -p 6379:6379 redis
+  local cname="$1" # container name
+  local free_port="$2"
+  docker run -d --name ${cname} -p ${free_port}:6379 redis
 }
 
 function redis_stop() {
-  docker stop redis_server
-  docker rm -f redis_server
+  local cname="$1"
+  docker stop ${cname}
+  docker rm -f ${cname}
 }

test/blackbox/redis_local.bats

@@ -30,8 +30,12 @@ function setup_file() {
         exit 1
     fi
 
+    # Download test data to folder common for the entire suite, not just this file
+    skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20
+
     # Setup redis server
-    redis_start
+    redis_port=$(get_free_port)
+    redis_start redis_server_local ${redis_port}
 
     # Setup zot server
     local zot_root_dir=${BATS_FILE_TMPDIR}/zot
@@ -49,7 +53,7 @@ function setup_file() {
         "cacheDriver": {
             "name": "redis",
             "rootDir": "${zot_root_dir}/_redis",
-            "url": "redis://localhost:6379"
+            "url": "redis://localhost:${redis_port}"
         }
     },
     "http": {
@@ -63,6 +67,9 @@ function setup_file() {
     "extensions": {
       "ui": {
         "enable": true
+      },
+      "search": {
+        "enable": true
       }
     }
 }
@@ -86,7 +93,8 @@ EOF
 
     run curl http://127.0.0.1:${zot_port}/v2/_catalog
     [ "$status" -eq 0 ]
-    [ $(echo "${lines[-1]}" | jq '.repositories[]') = '"golang"' ]
+    [ $(echo "${lines[-1]}" | jq '.repositories[0]') = '"golang"' ]
+    [ $(echo "${lines[-1]}" | jq '.repositories[1]') = '"golang2"' ]
     run curl http://127.0.0.1:${zot_port}/v2/golang/tags/list
     [ "$status" -eq 0 ]
     [ $(echo "${lines[-1]}" | jq '.tags[]') = '"1.20"' ]
@@ -95,6 +103,8 @@ EOF
 @test "pull both images" {
     local oci_data_dir=${BATS_FILE_TMPDIR}/oci
     zot_port=`cat ${BATS_FILE_TMPDIR}/zot.port`
+
+    mkdir -p ${oci_data_dir}
     run skopeo --insecure-policy copy --src-tls-verify=false \
         docker://127.0.0.1:${zot_port}/golang:1.20 \
         oci:${oci_data_dir}/golang:1.20
@@ -111,5 +121,5 @@ EOF
 
 function teardown_file() {
     zot_stop_all
-    redis_stop
+    redis_stop redis_server_local
 }

test/blackbox/redis_s3.bats

@@ -7,16 +7,6 @@ load helpers_redis
 load helpers_cloud
 
 function verify_prerequisites() {
-    if [ ! $(command -v curl) ]; then
-        echo "you need to install curl as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
-    if [ ! $(command -v jq) ]; then
-        echo "you need to install jq as a prerequisite to running the tests" >&3
-        return 1
-    fi
-
     if [ ! $(command -v docker) ]; then
         echo "you need to install docker as a prerequisite to running the tests" >&3
         return 1
@@ -32,7 +22,8 @@ function setup_file() {
     fi
 
     # Setup redis server
-    redis_start
+    redis_port=$(get_free_port)
+    redis_start redis_server ${redis_port}
 
     # Setup zot server
     local zot_root_dir=${BATS_FILE_TMPDIR}/zot
@@ -52,7 +43,7 @@ function setup_file() {
         "cacheDriver": {
             "name": "redis",
             "rootDir": "${zot_root_dir}/_redis",
-            "url": "redis://localhost:6379"
+            "url": "redis://localhost:${redis_port}"
         },
         "storageDriver": {
             "name": "s3",
@@ -75,12 +66,15 @@ function setup_file() {
     "extensions": {
       "ui": {
         "enable": true
+      },
+      "search": {
+        "enable": true
       }
     }
 }
 EOF
 
-    awslocal s3 --region "us-east-2" mb s3://zot-storage
+    awslocal s3 ls s3://zot-storage || awslocal s3 --region "us-east-2" mb s3://zot-storage
 
     zot_serve ${ZOT_PATH} ${zot_sync_ondemand_config_file}
     wait_zot_reachable ${zot_port}
@@ -123,5 +117,5 @@ EOF
 
 function teardown_file() {
     zot_stop_all
-    redis_stop
+    redis_stop redis_server
 }