diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index be40bbdf..d7703f32 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -38,7 +38,7 @@ jobs: s3mock: image: ghcr.io/project-zot/localstack/localstack:1.2.0 env: - SERVICES: s3 + SERVICES: s3,dynamodb ports: - 4563-4599:4563-4599 - 9090:8080 @@ -79,6 +79,7 @@ jobs: - name: Run build and test timeout-minutes: 60 run: | + aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 echo "Building for $OS:$ARCH" cd $GITHUB_WORKSPACE if [[ $OS == "linux" && $ARCH == "amd64" ]]; then @@ -87,8 +88,10 @@ jobs: else make OS=$OS ARCH=$ARCH binary binary-minimal binary-debug cli bench exporter-minimal fi + aws dynamodb --endpoint-url "http://localhost:4566" --region "us-east-2" delete-table --table-name "BlobTable" env: S3MOCK_ENDPOINT: localhost:4566 + DYNAMODBMOCK_ENDPOINT: http://localhost:4566 AWS_ACCESS_KEY_ID: fake AWS_SECRET_ACCESS_KEY: fake OS: ${{ matrix.os }} diff --git a/.github/workflows/ecosystem-tools.yaml b/.github/workflows/ecosystem-tools.yaml index dc782e7a..6c55cb12 100644 --- a/.github/workflows/ecosystem-tools.yaml +++ b/.github/workflows/ecosystem-tools.yaml @@ -54,3 +54,20 @@ jobs: - name: Run annotations tests run: | make test-annotations + - name: Install localstack + run: | + pip install --upgrade pyopenssl + pip install localstack awscli-local[ver1] # install LocalStack cli and awslocal + docker pull localstack/localstack # Make sure to pull the latest version of the image + localstack start -d # Start LocalStack in the background + + echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container + localstack wait -t 30 # to become ready before timing out + echo "Startup complete" + - name: Run cloud-only tests + run: | + sudo mkdir /zot + make test-cloud-only + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake diff --git a/Makefile b/Makefile index 321c3e82..e70c857c 100644 --- a/Makefile +++ b/Makefile @@ -274,6 +274,14 @@ test-push-pull: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) test-push-pull-verbose: binary check-skopeo $(BATS) $(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull.bats +.PHONY: test-cloud-only +test-cloud-only: binary check-skopeo $(BATS) + $(BATS) --trace --print-output-on-failure test/blackbox/cloud-only.bats + +.PHONY: test-cloud-only-verbose +test-cloud-only-verbose: binary check-skopeo $(BATS) + $(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/cloud-only.bats + .PHONY: test-bats-sync test-bats-sync: EXTENSIONS=sync test-bats-sync: binary binary-minimal check-skopeo $(BATS) $(NOTATION) $(COSIGN) diff --git a/examples/config-dynamodb.json b/examples/config-dynamodb.json new file mode 100644 index 00000000..7123630b --- /dev/null +++ b/examples/config-dynamodb.json @@ -0,0 +1,30 @@ +{ + "distSpecVersion": "1.0.1-dev", + "storage": { + "rootDirectory": "/tmp/zot", + "dedupe": true, + "remoteCache": true, + "storageDriver": { + "name": "s3", + "rootdirectory": "/zot", + "region": "us-east-2", + "regionendpoint": "localhost:4566", + "bucket": "zot-storage", + "secure": false, + "skipverify": false + }, + "cacheDriver": { + "name": "dynamodb", + "endpoint": "http://localhost:4566", + "region": "us-east-2", + "tableName": "BlobTable" + } + }, + "http": { + "address": "127.0.0.1", + "port": "8080" + }, + "log": { + "level": "debug" + } +} diff --git a/examples/config-s3.json b/examples/config-s3.json index e345fdca..afe1b061 100644 --- a/examples/config-s3.json +++ b/examples/config-s3.json @@ -11,6 +11,12 @@ "secure": true, "skipverify": false }, + "cacheDriver": { + "name": "dynamodb", + "endpoint": "http://localhost:4566", + "region": "us-east-2", + "tableName": "MainTable" + }, "subPaths": { "/a": { "rootDirectory": "/tmp/zot1", @@ -27,6 +33,7 @@ "/b": { "rootDirectory": "/tmp/zot2", "dedupe": true, + "remoteCache": false, "storageDriver": { "name": "s3", "rootdirectory": "/zot-b", @@ -39,6 +46,7 @@ "/c": { "rootDirectory": "/tmp/zot3", "dedupe": true, + "remoteCache": true, "storageDriver": { "name": "s3", "rootdirectory": "/zot-c", @@ -46,6 +54,12 @@ "bucket": "zot-storage", "secure": false, "skipverify": false + }, + "cacheDriver": { + "name": "dynamodb", + "endpoint": "http://localhost:4566", + "region": "us-east-2", + "tableName": "cTable" } } } diff --git a/go.mod b/go.mod index 16936275..218bc43a 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,7 @@ require ( require ( github.com/aquasecurity/trivy v0.0.0-00010101000000-000000000000 + github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.3 github.com/containers/image/v5 v5.23.0 github.com/notaryproject/notation-go v0.12.0-beta.1 github.com/opencontainers/distribution-spec/specs-go v0.0.0-20220620172159-4ab4752c3b86 @@ -59,6 +60,12 @@ require ( github.com/swaggo/http-swagger v1.3.3 ) +require ( + github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.19 // indirect +) + require ( bitbucket.org/creachadair/shell v0.0.7 // indirect cloud.google.com/go v0.104.0 // indirect @@ -115,12 +122,13 @@ require ( github.com/aquasecurity/tfsec v0.58.11 // indirect github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect github.com/aws/aws-sdk-go v1.44.114 // indirect - github.com/aws/aws-sdk-go-v2 v1.16.16 // indirect - github.com/aws/aws-sdk-go-v2/config v1.17.8 // indirect + github.com/aws/aws-sdk-go-v2 v1.17.1 + github.com/aws/aws-sdk-go-v2/config v1.17.8 github.com/aws/aws-sdk-go-v2/credentials v1.12.21 // indirect + github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.2 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 // indirect github.com/aws/aws-sdk-go-v2/service/ecr v1.15.0 // indirect github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.12.0 // indirect @@ -128,7 +136,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.11.23 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.6 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 // indirect - github.com/aws/smithy-go v1.13.3 // indirect + github.com/aws/smithy-go v1.13.4 // indirect github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20220228164355-396b2034c795 // indirect github.com/benbjohnson/clock v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index b35a2508..3ebac230 100644 --- a/go.sum +++ b/go.sum @@ -428,32 +428,45 @@ github.com/aws/aws-sdk-go v1.44.114/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.7.1/go.mod h1:L5LuPC1ZgDr2xQS7AmIec/Jlc7O/Y1u2KxJyNVab250= github.com/aws/aws-sdk-go-v2 v1.14.0/go.mod h1:ZA3Y8V0LrlWj63MQAnRHgKf/5QB//LSZCPNWlWrNGLU= -github.com/aws/aws-sdk-go-v2 v1.16.16 h1:M1fj4FE2lB4NzRb9Y0xdWsn2P0+2UHVxwKyOa4YJNjk= github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= +github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk= +github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= github.com/aws/aws-sdk-go-v2/config v1.5.0/go.mod h1:RWlPOAW3E3tbtNAqTwvSW54Of/yP3oiZXMI0xfUdjyA= github.com/aws/aws-sdk-go-v2/config v1.17.8 h1:b9LGqNnOdg9vR4Q43tBTVWk4J6F+W774MSchvKJsqnE= github.com/aws/aws-sdk-go-v2/config v1.17.8/go.mod h1:UkCI3kb0sCdvtjiXYiU4Zx5h07BOpgBTtkPu/49r+kA= github.com/aws/aws-sdk-go-v2/credentials v1.3.1/go.mod h1:r0n73xwsIVagq8RsxmZbGSRQFj9As3je72C2WzUIToc= github.com/aws/aws-sdk-go-v2/credentials v1.12.21 h1:4tjlyCD0hRGNQivh5dN8hbP30qQhMLBE/FgQR1vHHWM= github.com/aws/aws-sdk-go-v2/credentials v1.12.21/go.mod h1:O+4XyAt4e+oBAoIwNUYkRg3CVMscaIJdmZBOcPgJ8D8= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.2 h1:UBAIkLzejHf9CDlzKKe28k7xYTYleA2LIC5DUbNx+50= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.10.2/go.mod h1:4CTiMSedeR1/yn5WoD1q9tQAN6aZadfY5rsXad/LiVQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.3.0/go.mod h1:2LAuqPx1I6jNfaGDucWfA2zqQCYCOMCDHiCOciALyNw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 h1:r08j4sbZu/RVi+BNxkBJwPMUYY3P8mgSDuKkZ/ZN1lE= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17/go.mod h1:yIkQcCDYNsZfXpd5UX2Cy+sWA1jPgIhGTw9cOBzfVnQ= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.5/go.mod h1:2hXc8ooJqF2nAznsbJQIn+7h851/bu8GVC80OVTTqf8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 h1:s4g/wnzMf+qepSNgTvaQQHNxyMLKSawNhKCPNy++2xY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 h1:nBO/RFxeq/IS5G9Of+ZrgucRciie2qpLy++3UGZ+q2E= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25/go.mod h1:Zb29PYkf42vVYQY6pvSyJCJcFHlPIiY+YKdPtwnvMkY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.3.0/go.mod h1:miRSv9l093jX/t/j+mBCaLqFHo9xKYzJ7DGm1BsGoJM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 h1:/K482T5A3623WJgWT8w1yRAFK4RzGzEl7y39yhtn9eA= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 h1:oRHDrwCTVT8ZXi4sr9Ld+EXk7N/KGssOr2ygNeojEhw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19/go.mod h1:6Q0546uHDp421okhmmGfbxzq2hBqbXFNpi4k+Q1JnQA= github.com/aws/aws-sdk-go-v2/internal/ini v1.1.1/go.mod h1:Zy8smImhTdOETZqfyn01iNOe0CNggVbPjCajyaz6Gvg= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24 h1:wj5Rwc05hvUSvKuOF29IYb9QrCLjU+rHAy/x/o0DK2c= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24/go.mod h1:jULHjqqjDlbyTa7pfM7WICATnOv+iOhjletM3N0Xbu8= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.3 h1:2oB4ikNEMLaPtu6lbNFJyTSayBILvrOfa2VfOffcuvU= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.17.3/go.mod h1:BiglbKCG56L8tmMnUEyEQo422BO9xnNR8vVHnOsByf8= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.22 h1:vSUuWw6gsDfLEqZr1qHKV2uKW3rc6tND2DoGUk34iHs= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.13.22/go.mod h1:5lIdkQbMmEblCTEAyFAsLduBtMPD9Bqt9fwPjBK1KWU= github.com/aws/aws-sdk-go-v2/service/ecr v1.4.1/go.mod h1:FglZcyeiBqcbvyinl+n14aT/EWC7S1MIH+Gan2iizt0= github.com/aws/aws-sdk-go-v2/service/ecr v1.15.0 h1:lY2Z2sBP+zSbJ6CvvmnFgPcgknoQ0OJV88AwVetRRFk= github.com/aws/aws-sdk-go-v2/service/ecr v1.15.0/go.mod h1:4zYI85WiYDhFaU1jPFVfkD7HlBcdnITDE3QxDwy4Kus= github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.4.1/go.mod h1:eD5Eo4drVP2FLTw0G+SMIPWNWvQRGGTtIZR2XeAagoA= github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.12.0 h1:LsqBpyRofMG6eDs6YGud6FhdGyIyXelAasPOZ6wWLro= github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.12.0/go.mod h1:IArQ3IBR00FkuraKwudKZZU32OxJfdTdwV+W5iZh3Y4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10 h1:dpiPHgmFstgkLG07KaYAewvuptq5kvo52xn7tVSrtrQ= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.10/go.mod h1:9cBNUHI2aW4ho0A5T87O294iPDuuUOSIEDjnd1Lq/z0= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.19 h1:V03dAtcAN4Qtly7H3/0B6m3t/cyl4FgyKFqK738fyJw= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.19/go.mod h1:2WpVWFC5n4DYhjNXzObtge8xfgId9UP6GWca46KJFLo= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.1/go.mod h1:zceowr5Z1Nh2WVP8bf/3ikB41IZW59E4yIYbg+pC6mw= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 h1:Jrd/oMh0PKQc6+BowB+pLEwLIgaQF29eYbe7E1Av9Ug= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI= @@ -468,8 +481,9 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.16.19 h1:9pPi0PsFNAGILFfPCk8Y0iyEBGc github.com/aws/aws-sdk-go-v2/service/sts v1.16.19/go.mod h1:h4J3oPZQbxLhzGnk+j9dfYHi5qIOVJ5kczZd658/ydM= github.com/aws/smithy-go v1.6.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/aws/smithy-go v1.11.0/go.mod h1:3xHYmszWVx2c0kIwQeEVf9uSm4fYZt67FBJnwub1bgM= -github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA= github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.13.4 h1:/RN2z1txIJWeXeOkzX+Hk/4Uuvv7dWtCjbmVJcrskyk= +github.com/aws/smithy-go v1.13.4/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20220228164355-396b2034c795 h1:IWeCJzU+IYaO2rVEBlGPTBfe90cmGXFTLdhUFlzDGsY= github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20220228164355-396b2034c795/go.mod h1:8vJsEZ4iRqG+Vx6pKhWK6U00qcj0KC37IsfszMkY6UE= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= diff --git a/pkg/api/config/config.go b/pkg/api/config/config.go index 765e315f..393115f8 100644 --- a/pkg/api/config/config.go +++ b/pkg/api/config/config.go @@ -29,6 +29,7 @@ type StorageConfig struct { GCDelay time.Duration GCInterval time.Duration StorageDriver map[string]interface{} `mapstructure:",omitempty"` + CacheDriver map[string]interface{} `mapstructure:",omitempty"` } type TLSConfig struct { diff --git a/pkg/api/controller.go b/pkg/api/controller.go index a02df139..d7c1c1c1 100644 --- a/pkg/api/controller.go +++ b/pkg/api/controller.go @@ -268,7 +268,7 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error { var defaultStore storage.ImageStore if c.Config.Storage.StorageDriver == nil { // false positive lint - linter does not implement Lint method - //nolint:typecheck + //nolint:typecheck,contextcheck defaultStore = local.NewImageStore(c.Config.Storage.RootDirectory, c.Config.Storage.GC, c.Config.Storage.GCDelay, c.Config.Storage.Dedupe, c.Config.Storage.Commit, c.Log, c.Metrics, linter, @@ -296,7 +296,7 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error { } // false positive lint - linter does not implement Lint method - //nolint: typecheck + //nolint: typecheck,contextcheck defaultStore = s3.NewImageStore(rootDir, c.Config.Storage.RootDirectory, c.Config.Storage.GC, c.Config.Storage.GCDelay, c.Config.Storage.Dedupe, c.Config.Storage.Commit, c.Log, c.Metrics, linter, store, @@ -315,6 +315,7 @@ func (c *Controller) InitImageStore(reloadCtx context.Context) error { if len(c.Config.Storage.SubPaths) > 0 { subPaths := c.Config.Storage.SubPaths + //nolint: contextcheck subImageStore, err := c.getSubStore(subPaths, linter) if err != nil { c.Log.Error().Err(err).Msg("controller: error getting sub image store") @@ -443,7 +444,18 @@ func CreateCacheDatabaseDriver(storageConfig config.StorageConfig, log log.Logge return driver } - // used for tests, dynamodb when it comes + // dynamodb + if storageConfig.CacheDriver != nil { + dynamoParams := cache.DynamoDBDriverParameters{} + dynamoParams.Endpoint, _ = storageConfig.CacheDriver["endpoint"].(string) + dynamoParams.Region, _ = storageConfig.CacheDriver["region"].(string) + dynamoParams.TableName, _ = storageConfig.CacheDriver["tablename"].(string) + + driver, _ := storage.Create("dynamodb", dynamoParams, log) + + return driver + } + return nil } diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index 7dc1705f..4540002c 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -99,6 +99,14 @@ func skipIt(t *testing.T) { } } +func skipDynamo(t *testing.T) { + t.Helper() + + if os.Getenv("DYNAMODBMOCK_ENDPOINT") == "" { + t.Skip("Skipping testing without AWS DynamoDB mock server") + } +} + func TestNew(t *testing.T) { Convey("Make a new controller", t, func() { conf := config.New() @@ -108,7 +116,7 @@ func TestNew(t *testing.T) { } func TestCreateCacheDatabaseDriver(t *testing.T) { - Convey("Test CreateCacheDatabaseDriver", t, func() { + Convey("Test CreateCacheDatabaseDriver boltdb", t, func() { log := log.NewLogger("debug", "") // fail create db, no perm @@ -132,6 +140,35 @@ func TestCreateCacheDatabaseDriver(t *testing.T) { driver = api.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) So(driver, ShouldBeNil) }) + skipDynamo(t) + skipIt(t) + Convey("Test CreateCacheDatabaseDriver dynamodb", t, func() { + log := log.NewLogger("debug", "") + dir := t.TempDir() + // good config + conf := config.New() + conf.Storage.RootDirectory = dir + conf.Storage.Dedupe = true + conf.Storage.RemoteCache = true + conf.Storage.StorageDriver = map[string]interface{}{ + "name": "s3", + "rootdirectory": "/zot", + "region": "us-east-2", + "bucket": "zot-storage", + "secure": true, + "skipverify": false, + } + + conf.Storage.CacheDriver = map[string]interface{}{ + "name": "dynamodb", + "endpoint": "http://localhost:4566", + "region": "us-east-2", + "tableName": "BlobTable", + } + + driver := api.CreateCacheDatabaseDriver(conf.Storage.StorageConfig, log) + So(driver, ShouldNotBeNil) + }) } func TestRunAlreadyRunningServer(t *testing.T) { diff --git a/pkg/cli/root.go b/pkg/cli/root.go index 244c979f..7629170f 100644 --- a/pkg/cli/root.go +++ b/pkg/cli/root.go @@ -237,6 +237,75 @@ func validateStorageConfig(cfg *config.Config) error { return nil } +func validateCacheConfig(cfg *config.Config) error { + // global + // dedupe true, remote storage, remoteCache true, but no cacheDriver (remote) + //nolint: lll + if cfg.Storage.Dedupe && cfg.Storage.StorageDriver != nil && cfg.Storage.RemoteCache && cfg.Storage.CacheDriver == nil { + log.Error().Err(errors.ErrBadConfig).Msg( + "dedupe set to true with remote storage and caching, but no remote cache configured!") + + return errors.ErrBadConfig + } + + if cfg.Storage.CacheDriver != nil && cfg.Storage.RemoteCache { + // local storage with remote caching + if cfg.Storage.StorageDriver == nil { + log.Error().Err(errors.ErrBadConfig).Msg("cannot have local storage driver with remote caching!") + + return errors.ErrBadConfig + } + + // unsupported cache driver + if cfg.Storage.CacheDriver["name"] != storageConstants.DynamoDBDriverName { + log.Error().Err(errors.ErrBadConfig).Msgf("unsupported cache driver: %s", cfg.Storage.CacheDriver["name"]) + + return errors.ErrBadConfig + } + } + + if !cfg.Storage.RemoteCache && cfg.Storage.CacheDriver != nil { + log.Warn().Err(errors.ErrBadConfig).Msgf( + "remoteCache set to false but cacheDriver config (remote caching) provided for %s,"+ + "will ignore and use local caching", cfg.Storage.RootDirectory) + } + + // subpaths + for _, subPath := range cfg.Storage.SubPaths { + // dedupe true, remote storage, remoteCache true, but no cacheDriver (remote) + //nolint: lll + if subPath.Dedupe && subPath.StorageDriver != nil && subPath.RemoteCache && subPath.CacheDriver == nil { + log.Error().Err(errors.ErrBadConfig).Msg("dedupe set to true with remote storage and caching, but no remote cache configured!") + + return errors.ErrBadConfig + } + + if subPath.CacheDriver != nil && subPath.RemoteCache { + // local storage with remote caching + if subPath.StorageDriver == nil { + log.Error().Err(errors.ErrBadConfig).Msg("cannot have local storage driver with remote caching!") + + return errors.ErrBadConfig + } + + // unsupported cache driver + if subPath.CacheDriver["name"] != storageConstants.DynamoDBDriverName { + log.Error().Err(errors.ErrBadConfig).Msgf("unsupported cache driver: %s", subPath.CacheDriver["name"]) + + return errors.ErrBadConfig + } + } + + if !subPath.RemoteCache && subPath.CacheDriver != nil { + log.Warn().Err(errors.ErrBadConfig).Msgf( + "remoteCache set to false but cacheDriver config (remote caching) provided for %s,"+ + "will ignore and use local caching", subPath.RootDirectory) + } + } + + return nil +} + func validateConfiguration(config *config.Config) error { if err := validateHTTP(config); err != nil { return err @@ -258,6 +327,10 @@ func validateConfiguration(config *config.Config) error { return err } + if err := validateCacheConfig(config); err != nil { + return err + } + // check authorization config, it should have basic auth enabled or ldap if config.HTTP.RawAccessControl != nil { // checking for anonymous policy only authorization config: no users, no policies but anonymous policy diff --git a/pkg/cli/root_test.go b/pkg/cli/root_test.go index 2bb19115..5b13e0db 100644 --- a/pkg/cli/root_test.go +++ b/pkg/cli/root_test.go @@ -117,52 +117,301 @@ func TestVerify(t *testing.T) { So(err, ShouldBeNil) defer os.Remove(tmpfile.Name()) // clean up - // dedup true, can't parse database type - content := []byte(`{"storage":{"rootDirectory":"/tmp/zot", "dedupe": true, - "cache": {"type": 123}}, - "http":{"address":"127.0.0.1","port":"8080","realm":"zot", - "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`) + // dedupe true, remote storage, remoteCache true, but no cacheDriver (remote) + content := []byte(`{ + "storage":{ + "rootDirectory":"/tmp/zot", + "dedupe":true, + "remoteCache":true, + "storageDriver":{ + "name":"s3", + "rootdirectory":"/zot", + "region":"us-east-2", + "bucket":"zot-storage", + "secure":true, + "skipverify":false + } + }, + "http":{ + "address":"127.0.0.1", + "port":"8080", + "realm":"zot", + "auth":{ + "htpasswd":{ + "path":"test/data/htpasswd" + }, + "failDelay":1 + } + } + }`) err = os.WriteFile(tmpfile.Name(), content, 0o0600) So(err, ShouldBeNil) os.Args = []string{"cli_test", "verify", tmpfile.Name()} So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic) - // dedup true, wrong database type - content = []byte(`{"storage":{"rootDirectory":"/tmp/zot", "dedupe": true, - "cache": {"type": "wrong"}}, - "http":{"address":"127.0.0.1","port":"8080","realm":"zot", - "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`) + // local storage with remote caching + content = []byte(`{ + "storage":{ + "rootDirectory":"/tmp/zot", + "dedupe":true, + "remoteCache":true, + "cacheDriver":{ + "name":"dynamodb", + "endpoint":"http://localhost:4566", + "region":"us-east-2", + "tableName":"BlobTable" + } + }, + "http":{ + "address":"127.0.0.1", + "port":"8080", + "realm":"zot", + "auth":{ + "htpasswd":{ + "path":"test/data/htpasswd" + }, + "failDelay":1 + } + } + }`) err = os.WriteFile(tmpfile.Name(), content, 0o0600) So(err, ShouldBeNil) os.Args = []string{"cli_test", "verify", tmpfile.Name()} So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic) + // unsupported cache driver + content = []byte(`{ + "storage":{ + "rootDirectory":"/tmp/zot", + "dedupe":true, + "remoteCache":true, + "cacheDriver":{ + "name":"unsupportedDriver" + }, + "storageDriver":{ + "name":"s3", + "rootdirectory":"/zot", + "region":"us-east-2", + "bucket":"zot-storage", + "secure":true, + "skipverify":false + } + }, + "http":{ + "address":"127.0.0.1", + "port":"8080", + "realm":"zot", + "auth":{ + "htpasswd":{ + "path":"test/data/htpasswd" + }, + "failDelay":1 + } + } + }`) + err = os.WriteFile(tmpfile.Name(), content, 0o0600) + So(err, ShouldBeNil) + + os.Args = []string{"cli_test", "verify", tmpfile.Name()} + So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic) + + // remoteCache false but provided cacheDriver config, ignored + content = []byte(`{ + "storage":{ + "rootDirectory":"/tmp/zot", + "dedupe":true, + "remoteCache":false, + "cacheDriver":{ + "name":"dynamodb", + "endpoint":"http://localhost:4566", + "region":"us-east-2", + "tableName":"BlobTable" + }, + "storageDriver":{ + "name":"s3", + "rootdirectory":"/zot", + "region":"us-east-2", + "bucket":"zot-storage", + "secure":true, + "skipverify":false + } + }, + "http":{ + "address":"127.0.0.1", + "port":"8080", + "realm":"zot", + "auth":{ + "htpasswd":{ + "path":"test/data/htpasswd" + }, + "failDelay":1 + } + } + }`) + + err = os.WriteFile(tmpfile.Name(), content, 0o0600) + So(err, ShouldBeNil) + + os.Args = []string{"cli_test", "verify", tmpfile.Name()} + So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldNotPanic) + // SubPaths - // dedup true, wrong database type - content = []byte(`{"storage":{"rootDirectory":"/tmp/zot", "dedupe": false, - "subPaths": {"/a": {"rootDirectory": "/zot-a", "dedupe": true, - "cache": {"type": "wrong"}}}}, - "http":{"address":"127.0.0.1","port":"8080","realm":"zot", - "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`) + // dedupe true, remote storage, remoteCache true, but no cacheDriver (remote) + content = []byte(`{ + "storage":{ + "rootDirectory":"/tmp/zot", + "dedupe":false, + "subPaths":{ + "/a":{ + "rootDirectory":"/zot-a", + "dedupe":true, + "remoteCache":true, + "storageDriver":{ + "name":"s3", + "rootdirectory":"/zot", + "region":"us-east-2", + "bucket":"zot-storage", + "secure":true, + "skipverify":false + } + } + } + }, + "http":{ + "address":"127.0.0.1", + "port":"8080", + "realm":"zot", + "auth":{ + "htpasswd":{ + "path":"test/data/htpasswd" + }, + "failDelay":1 + } + } + }`) err = os.WriteFile(tmpfile.Name(), content, 0o0600) So(err, ShouldBeNil) os.Args = []string{"cli_test", "verify", tmpfile.Name()} So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic) - // dedup true, can't parse database type - content = []byte(`{"storage":{"rootDirectory":"/tmp/zot", "dedupe": false, - "subPaths": {"/a": {"rootDirectory": "/zot-a", "dedupe": true, - "cache": {"type": 123}}}}, - "http":{"address":"127.0.0.1","port":"8080","realm":"zot", - "auth":{"htpasswd":{"path":"test/data/htpasswd"},"failDelay":1}}}`) + // local storage with remote caching + content = []byte(`{ + "storage":{ + "rootDirectory":"/tmp/zot", + "dedupe":false, + "subPaths":{ + "/a":{ + "rootDirectory":"/zot-a", + "dedupe":true, + "remoteCache":true, + "cacheDriver":{ + "name":"dynamodb", + "endpoint":"http://localhost:4566", + "region":"us-east-2", + "tableName":"BlobTable" + } + } + } + }, + "http":{ + "address":"127.0.0.1", + "port":"8080", + "realm":"zot", + "auth":{ + "htpasswd":{ + "path":"test/data/htpasswd" + }, + "failDelay":1 + } + } + }`) err = os.WriteFile(tmpfile.Name(), content, 0o0600) So(err, ShouldBeNil) os.Args = []string{"cli_test", "verify", tmpfile.Name()} So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic) + + // unsupported cache driver + content = []byte(`{ + "storage":{ + "rootDirectory":"/tmp/zot", + "dedupe":false, + "subPaths":{ + "/a":{ + "rootDirectory":"/zot-a", + "dedupe":true, + "remoteCache":true, + "cacheDriver":{ + "name":"badDriverName" + }, + "storageDriver":{ + "name":"s3", + "rootdirectory":"/zot", + "region":"us-east-2", + "bucket":"zot-storage", + "secure":true, + "skipverify":false + } + } + } + }, + "http":{ + "address":"127.0.0.1", + "port":"8080", + "realm":"zot", + "auth":{ + "htpasswd":{ + "path":"test/data/htpasswd" + }, + "failDelay":1 + } + } + }`) + err = os.WriteFile(tmpfile.Name(), content, 0o0600) + So(err, ShouldBeNil) + + os.Args = []string{"cli_test", "verify", tmpfile.Name()} + So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldPanic) + + // remoteCache false but provided cacheDriver config, ignored + content = []byte(`{ + "storage":{ + "rootDirectory":"/tmp/zot", + "dedupe":false, + "subPaths":{ + "/a":{ + "rootDirectory":"/zot-a", + "dedupe":true, + "remoteCache":false, + "cacheDriver":{ + "name":"dynamodb", + "endpoint":"http://localhost:4566", + "region":"us-east-2", + "tableName":"BlobTable" + } + } + } + }, + "http":{ + "address":"127.0.0.1", + "port":"8080", + "realm":"zot", + "auth":{ + "htpasswd":{ + "path":"test/data/htpasswd" + }, + "failDelay":1 + } + } + }`) + err = os.WriteFile(tmpfile.Name(), content, 0o0600) + So(err, ShouldBeNil) + + os.Args = []string{"cli_test", "verify", tmpfile.Name()} + So(func() { _ = cli.NewServerRootCmd().Execute() }, ShouldNotPanic) }) Convey("Test apply defaults cache db", t, func(c C) { diff --git a/pkg/extensions/sync/on_demand.go b/pkg/extensions/sync/on_demand.go index d87ea06a..6ad1f57a 100644 --- a/pkg/extensions/sync/on_demand.go +++ b/pkg/extensions/sync/on_demand.go @@ -185,7 +185,7 @@ func syncOneImage(ctx context.Context, imageChannel chan error, cfg Config, stor upstreamAddr: upstreamAddr, copyOptions: options, } - + //nolint:contextcheck skipped, copyErr := syncRun(regCfg, localRepo, upstreamRepo, reference, syncContextUtils, sig, log) if skipped { continue diff --git a/pkg/storage/cache.go b/pkg/storage/cache.go index 19e47e7d..2ddb072c 100644 --- a/pkg/storage/cache.go +++ b/pkg/storage/cache.go @@ -12,6 +12,10 @@ func Create(dbtype string, parameters interface{}, log zlog.Logger) (cache.Cache { return cache.NewBoltDBCache(parameters, log), nil } + case "dynamodb": + { + return cache.NewDynamoDBCache(parameters, log), nil + } default: { return nil, errors.ErrBadConfig diff --git a/pkg/storage/cache/boltdb.go b/pkg/storage/cache/boltdb.go index d496575b..b9c68a86 100644 --- a/pkg/storage/cache/boltdb.go +++ b/pkg/storage/cache/boltdb.go @@ -32,18 +32,14 @@ func NewBoltDBCache(parameters interface{}, log zlog.Logger) Cache { panic("Failed type assertion") } - return NewCache(properParameters, log) -} - -func NewCache(parameters BoltDBDriverParameters, log zlog.Logger) *BoltDBDriver { - err := os.MkdirAll(parameters.RootDir, constants.DefaultDirPerms) + err := os.MkdirAll(properParameters.RootDir, constants.DefaultDirPerms) if err != nil { - log.Error().Err(err).Msgf("unable to create directory for cache db: %v", parameters.RootDir) + log.Error().Err(err).Msgf("unable to create directory for cache db: %v", properParameters.RootDir) return nil } - dbPath := path.Join(parameters.RootDir, parameters.Name+constants.DBExtensionName) + dbPath := path.Join(properParameters.RootDir, properParameters.Name+constants.DBExtensionName) dbOpts := &bbolt.Options{ Timeout: constants.DBCacheLockCheckTimeout, FreelistType: bbolt.FreelistArrayType, @@ -72,7 +68,12 @@ func NewCache(parameters BoltDBDriverParameters, log zlog.Logger) *BoltDBDriver return nil } - return &BoltDBDriver{rootDir: parameters.RootDir, db: cacheDB, useRelPaths: parameters.UseRelPaths, log: log} + return &BoltDBDriver{ + rootDir: properParameters.RootDir, + db: cacheDB, + useRelPaths: properParameters.UseRelPaths, + log: log, + } } func (d *BoltDBDriver) Name() string { diff --git a/pkg/storage/cache/dynamodb.go b/pkg/storage/cache/dynamodb.go new file mode 100644 index 00000000..5d135759 --- /dev/null +++ b/pkg/storage/cache/dynamodb.go @@ -0,0 +1,216 @@ +package cache + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + godigest "github.com/opencontainers/go-digest" + + zerr "zotregistry.io/zot/errors" + zlog "zotregistry.io/zot/pkg/log" +) + +type DynamoDBDriver struct { + client *dynamodb.Client + log zlog.Logger + tableName string +} + +type DynamoDBDriverParameters struct { + Endpoint, Region, TableName string +} + +type Blob struct { + Digest string `dynamodbav:"Digest,string"` + BlobPath []string `dynamodbav:"BlobPath,stringset"` +} + +// Use ONLY for tests. +func (d *DynamoDBDriver) NewTable(tableName string) error { + //nolint:gomnd + _, err := d.client.CreateTable(context.TODO(), &dynamodb.CreateTableInput{ + TableName: &tableName, + AttributeDefinitions: []types.AttributeDefinition{ + { + AttributeName: aws.String("Digest"), + AttributeType: types.ScalarAttributeTypeS, + }, + }, + KeySchema: []types.KeySchemaElement{ + { + AttributeName: aws.String("Digest"), + KeyType: types.KeyTypeHash, + }, + }, + ProvisionedThroughput: &types.ProvisionedThroughput{ + ReadCapacityUnits: aws.Int64(10), + WriteCapacityUnits: aws.Int64(5), + }, + }) + if err != nil { + return err + } + + d.tableName = tableName + + return nil +} + +func NewDynamoDBCache(parameters interface{}, log zlog.Logger) Cache { + properParameters, ok := parameters.(DynamoDBDriverParameters) + if !ok { + panic("Failed type assertion!") + } + + // custom endpoint resolver to point to localhost + customResolver := aws.EndpointResolverWithOptionsFunc( + func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + PartitionID: "aws", + URL: properParameters.Endpoint, + SigningRegion: region, + }, nil + }) + + // Using the SDK's default configuration, loading additional config + // and credentials values from the environment variables, shared + // credentials, and shared configuration files + cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithRegion(properParameters.Region), + config.WithEndpointResolverWithOptions(customResolver)) + if err != nil { + log.Error().Msgf("unable to load AWS SDK config for dynamodb, %v", err) + + return nil + } + + // Using the Config value, create the DynamoDB client + return &DynamoDBDriver{client: dynamodb.NewFromConfig(cfg), tableName: properParameters.TableName, log: log} +} + +func (d *DynamoDBDriver) Name() string { + return "dynamodb" +} + +// Returns the first path of the blob if it exists. +func (d *DynamoDBDriver) GetBlob(digest godigest.Digest) (string, error) { + resp, err := d.client.GetItem(context.TODO(), &dynamodb.GetItemInput{ + TableName: aws.String(d.tableName), + Key: map[string]types.AttributeValue{ + "Digest": &types.AttributeValueMemberS{Value: digest.String()}, + }, + }) + if err != nil { + d.log.Error().Msgf("failed to get blob %v, %v", d.tableName, err) + + return "", err + } + + out := Blob{} + + if resp.Item == nil { + return "", zerr.ErrCacheMiss + } + + _ = attributevalue.UnmarshalMap(resp.Item, &out) + + if len(out.BlobPath) == 0 { + return "", nil + } + + return out.BlobPath[0], nil +} + +func (d *DynamoDBDriver) PutBlob(digest godigest.Digest, path string) error { + if path == "" { + d.log.Error().Err(zerr.ErrEmptyValue).Str("digest", digest.String()).Msg("empty path provided") + + return zerr.ErrEmptyValue + } + + marshaledKey, _ := attributevalue.MarshalMap(map[string]interface{}{"Digest": digest.String()}) + expression := "ADD BlobPath :i" + attrPath := types.AttributeValueMemberSS{Value: []string{path}} + + if _, err := d.client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ + Key: marshaledKey, + TableName: &d.tableName, + UpdateExpression: &expression, + ExpressionAttributeValues: map[string]types.AttributeValue{":i": &attrPath}, + }); err != nil { + d.log.Error().Err(err) + + return err + } + + return nil +} + +func (d *DynamoDBDriver) HasBlob(digest godigest.Digest, path string) bool { + resp, err := d.client.GetItem(context.TODO(), &dynamodb.GetItemInput{ + TableName: aws.String(d.tableName), + Key: map[string]types.AttributeValue{ + "Digest": &types.AttributeValueMemberS{Value: digest.String()}, + }, + }) + if err != nil { + d.log.Error().Msgf("failed to get blob %v, %v", d.tableName, err) + + return false + } + + out := Blob{} + + if resp.Item == nil { + d.log.Error().Err(zerr.ErrCacheMiss) + + return false + } + + _ = attributevalue.UnmarshalMap(resp.Item, &out) + + for _, item := range out.BlobPath { + if item == path { + return true + } + } + + d.log.Error().Err(zerr.ErrCacheMiss) + + return false +} + +func (d *DynamoDBDriver) DeleteBlob(digest godigest.Digest, path string) error { + marshaledKey, _ := attributevalue.MarshalMap(map[string]interface{}{"Digest": digest.String()}) + + expression := "DELETE BlobPath :i" + attrPath := types.AttributeValueMemberSS{Value: []string{path}} + + _, err := d.client.UpdateItem(context.TODO(), &dynamodb.UpdateItemInput{ + Key: marshaledKey, + TableName: &d.tableName, + UpdateExpression: &expression, + ExpressionAttributeValues: map[string]types.AttributeValue{":i": &attrPath}, + }) + if err != nil { + d.log.Error().Err(err).Str("digest", digest.String()).Str("path", path).Msg("unable to delete") + + return err + } + + result, _ := d.GetBlob(digest) + + if result == "" { + d.log.Debug().Str("digest", digest.String()).Str("path", path).Msg("deleting empty bucket") + + _, _ = d.client.DeleteItem(context.TODO(), &dynamodb.DeleteItemInput{ + Key: marshaledKey, + TableName: &d.tableName, + }) + } + + return nil +} diff --git a/pkg/storage/cache/dynamodb_test.go b/pkg/storage/cache/dynamodb_test.go new file mode 100644 index 00000000..64a67f4d --- /dev/null +++ b/pkg/storage/cache/dynamodb_test.go @@ -0,0 +1,111 @@ +package cache_test + +import ( + "os" + "path" + "testing" + + godigest "github.com/opencontainers/go-digest" + . "github.com/smartystreets/goconvey/convey" + + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/storage" + "zotregistry.io/zot/pkg/storage/cache" +) + +func skipIt(t *testing.T) { + t.Helper() + + if os.Getenv("DYNAMODBMOCK_ENDPOINT") == "" { + t.Skip("Skipping testing without AWS DynamoDB mock server") + } +} + +func TestDynamoDB(t *testing.T) { + skipIt(t) + Convey("Test dynamoDB", t, func(c C) { + log := log.NewLogger("debug", "") + dir := t.TempDir() + + // bad params + + So(func() { + _ = cache.NewDynamoDBCache("bad params", log) + }, ShouldPanic) + + keyDigest := godigest.FromString("key") + + cacheDriver, err := storage.Create("dynamodb", cache.DynamoDBDriverParameters{ + Endpoint: "http://brokenlink", + TableName: "BlobTable", + Region: "us-east-2", + }, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + val, err := cacheDriver.GetBlob(keyDigest) + So(err, ShouldNotBeNil) + So(val, ShouldBeEmpty) + + err = cacheDriver.PutBlob(keyDigest, path.Join(dir, "value")) + So(err, ShouldNotBeNil) + + exists := cacheDriver.HasBlob(keyDigest, path.Join(dir, "value")) + So(exists, ShouldBeFalse) + + err = cacheDriver.DeleteBlob(keyDigest, path.Join(dir, "value")) + So(err, ShouldNotBeNil) + + cacheDriver, err = storage.Create("dynamodb", cache.DynamoDBDriverParameters{ + Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"), + TableName: "BlobTable", + Region: "us-east-2", + }, log) + So(cacheDriver, ShouldNotBeNil) + So(err, ShouldBeNil) + + returnedName := cacheDriver.Name() + So(returnedName, ShouldEqual, "dynamodb") + + val, err = cacheDriver.GetBlob(keyDigest) + So(err, ShouldNotBeNil) + So(val, ShouldBeEmpty) + + err = cacheDriver.PutBlob(keyDigest, "") + So(err, ShouldNotBeNil) + + err = cacheDriver.PutBlob(keyDigest, path.Join(dir, "value")) + So(err, ShouldBeNil) + + val, err = cacheDriver.GetBlob(keyDigest) + So(err, ShouldBeNil) + So(val, ShouldNotBeEmpty) + + exists = cacheDriver.HasBlob(keyDigest, path.Join(dir, "value")) + So(exists, ShouldBeTrue) + + err = cacheDriver.DeleteBlob(keyDigest, path.Join(dir, "value")) + So(err, ShouldBeNil) + + exists = cacheDriver.HasBlob(keyDigest, path.Join(dir, "value")) + So(exists, ShouldBeFalse) + + err = cacheDriver.PutBlob(keyDigest, path.Join(dir, "value1")) + So(err, ShouldBeNil) + + err = cacheDriver.PutBlob(keyDigest, path.Join(dir, "value2")) + So(err, ShouldBeNil) + + err = cacheDriver.DeleteBlob(keyDigest, path.Join(dir, "value1")) + So(err, ShouldBeNil) + + exists = cacheDriver.HasBlob(keyDigest, path.Join(dir, "value2")) + So(exists, ShouldBeTrue) + + exists = cacheDriver.HasBlob(keyDigest, path.Join(dir, "value1")) + So(exists, ShouldBeFalse) + + err = cacheDriver.DeleteBlob(keyDigest, path.Join(dir, "value2")) + So(err, ShouldBeNil) + }) +} diff --git a/pkg/storage/cache_benchmark_test.go b/pkg/storage/cache_benchmark_test.go new file mode 100644 index 00000000..962eab9e --- /dev/null +++ b/pkg/storage/cache_benchmark_test.go @@ -0,0 +1,520 @@ +package storage_test + +import ( + "math/rand" + "os/exec" + "testing" + "time" + + godigest "github.com/opencontainers/go-digest" + + "zotregistry.io/zot/pkg/log" + "zotregistry.io/zot/pkg/storage" + "zotregistry.io/zot/pkg/storage/cache" +) + +const ( + region string = "us-east-2" + localEndpoint string = "http://localhost:4566" + awsEndpoint string = "https://dynamodb.us-east-2.amazonaws.com" + datasetSize int = 5000 +) + +func generateRandomString() string { + //nolint: gosec + seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) + charset := "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + + randomBytes := make([]byte, 10) + for i := range randomBytes { + randomBytes[i] = charset[seededRand.Intn(len(charset))] + } + + return string(randomBytes) +} + +func generateData() map[godigest.Digest]string { + dataMap := make(map[godigest.Digest]string, datasetSize) + //nolint: gosec + seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) + + for i := 0; i < datasetSize; i++ { + randomString := generateRandomString() + counter := 0 + + for seededRand.Float32() < 0.5 && counter < 5 { + counter++ + randomString += "/" + randomString += generateRandomString() + } + digest := godigest.FromString(randomString) + dataMap[digest] = randomString + } + + return dataMap +} + +func helperPutAll(cache cache.Cache, testData map[godigest.Digest]string) { + for digest, path := range testData { + _ = cache.PutBlob(digest, path) + } +} + +func helperDeleteAll(cache cache.Cache, testData map[godigest.Digest]string) { + for digest, path := range testData { + _ = cache.DeleteBlob(digest, path) + } +} + +func helperHasAll(cache cache.Cache, testData map[godigest.Digest]string) { + for digest, path := range testData { + _ = cache.HasBlob(digest, path) + } +} + +func helperGetAll(cache cache.Cache, testData map[godigest.Digest]string) { + for digest := range testData { + _, _ = cache.GetBlob(digest) + } +} + +func helperMix(cache cache.Cache, testData map[godigest.Digest]string, digestSlice []godigest.Digest) { + // The test data contains datasetSize entries by default, and each set of operations uses 5 entries + for i := 0; i < 1000; i++ { + _ = cache.PutBlob(digestSlice[i*5], testData[digestSlice[i*5]]) + _ = cache.PutBlob(digestSlice[i*5+1], testData[digestSlice[i*5+1]]) + _ = cache.PutBlob(digestSlice[i*5+2], testData[digestSlice[i*5+2]]) + _ = cache.PutBlob(digestSlice[i*5+2], testData[digestSlice[i*5+3]]) + _ = cache.DeleteBlob(digestSlice[i*5+1], testData[digestSlice[i*5+1]]) + _ = cache.DeleteBlob(digestSlice[i*5+2], testData[digestSlice[i*5+3]]) + _ = cache.DeleteBlob(digestSlice[i*5+2], testData[digestSlice[i*5+2]]) + _ = cache.HasBlob(digestSlice[i*5], testData[digestSlice[i*5]]) + _ = cache.HasBlob(digestSlice[i*5+1], testData[digestSlice[i*5+1]]) + _, _ = cache.GetBlob(digestSlice[i*5]) + _, _ = cache.GetBlob(digestSlice[i*5+1]) + _ = cache.PutBlob(digestSlice[i*5], testData[digestSlice[i*5+4]]) + _, _ = cache.GetBlob(digestSlice[i*5+4]) + _ = cache.DeleteBlob(digestSlice[i*5], testData[digestSlice[i*5+4]]) + _ = cache.DeleteBlob(digestSlice[i*5], testData[digestSlice[i*5]]) + } +} + +// BoltDB tests + +func BenchmarkPutLocal(b *testing.B) { + dir := b.TempDir() + log := log.NewLogger("error", "") + cache, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache_test", + UseRelPaths: false, + }, log) + testData := generateData() + + b.ResetTimer() + + helperPutAll(cache, testData) +} + +func BenchmarkDeleteLocal(b *testing.B) { + dir := b.TempDir() + log := log.NewLogger("error", "") + cache, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache_test", + UseRelPaths: false, + }, log) + testData := generateData() + + for digest, path := range testData { + _ = cache.PutBlob(digest, path) + } + + b.ResetTimer() + + helperDeleteAll(cache, testData) +} + +func BenchmarkHasLocal(b *testing.B) { + dir := b.TempDir() + log := log.NewLogger("error", "") + cache, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache_test", + UseRelPaths: false, + }, log) + testData := generateData() + + for digest, path := range testData { + _ = cache.PutBlob(digest, path) + } + + b.ResetTimer() + + helperHasAll(cache, testData) +} + +func BenchmarkGetLocal(b *testing.B) { + dir := b.TempDir() + log := log.NewLogger("error", "") + cache, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache_test", + UseRelPaths: false, + }, log) + testData := generateData() + counter := 1 + + var previousDigest godigest.Digest + + for digest, path := range testData { + if counter != 10 { + _ = cache.PutBlob(digest, path) + previousDigest = digest + counter++ + } else { + _ = cache.PutBlob(previousDigest, path) + counter = 1 + } + } + + b.ResetTimer() + + helperGetAll(cache, testData) +} + +func BenchmarkMixLocal(b *testing.B) { + dir := b.TempDir() + log := log.NewLogger("error", "") + cache, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache_test", + UseRelPaths: false, + }, log) + testData := generateData() + digestSlice := make([]godigest.Digest, datasetSize) + counter := 0 + + for key := range testData { + digestSlice[counter] = key + counter++ + } + + b.ResetTimer() + + helperMix(cache, testData, digestSlice) +} + +// DynamoDB Local tests + +func BenchmarkPutLocalstack(b *testing.B) { + log := log.NewLogger("error", "") + tableName := generateRandomString() + + // Create Table + _, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", localEndpoint, "create-table", + "--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S", + "--key-schema", "AttributeName=Digest,KeyType=HASH", + "--billing-mode", "PAY_PER_REQUEST").Output() + if err != nil { + panic(err) + } + + cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{ + Endpoint: localEndpoint, + Region: region, + TableName: tableName, + }, log) + testData := generateData() + + b.ResetTimer() + + helperPutAll(cache, testData) +} + +func BenchmarkDeleteLocalstack(b *testing.B) { + log := log.NewLogger("error", "") + tableName := generateRandomString() + + // Create Table + _, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", localEndpoint, "create-table", + "--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S", + "--key-schema", "AttributeName=Digest,KeyType=HASH", + "--billing-mode", "PAY_PER_REQUEST").Output() + if err != nil { + panic(err) + } + + cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{ + Endpoint: localEndpoint, + Region: region, + TableName: tableName, + }, log) + testData := generateData() + + for digest, path := range testData { + _ = cache.PutBlob(digest, path) + } + + b.ResetTimer() + + helperDeleteAll(cache, testData) +} + +func BenchmarkHasLocalstack(b *testing.B) { + log := log.NewLogger("error", "") + tableName := generateRandomString() + + // Create Table + _, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", localEndpoint, "create-table", + "--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S", + "--key-schema", "AttributeName=Digest,KeyType=HASH", + "--billing-mode", "PAY_PER_REQUEST").Output() + if err != nil { + panic(err) + } + + cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{ + Endpoint: localEndpoint, + Region: region, + TableName: tableName, + }, log) + testData := generateData() + + for digest, path := range testData { + _ = cache.PutBlob(digest, path) + } + + b.ResetTimer() + + helperHasAll(cache, testData) +} + +func BenchmarkGetLocalstack(b *testing.B) { + log := log.NewLogger("error", "") + tableName := generateRandomString() + + // Create Table + _, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", localEndpoint, "create-table", + "--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S", + "--key-schema", "AttributeName=Digest,KeyType=HASH", + "--billing-mode", "PAY_PER_REQUEST").Output() + if err != nil { + panic(err) + } + + cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{ + Endpoint: localEndpoint, + Region: region, + TableName: tableName, + }, log) + testData := generateData() + counter := 1 + + var previousDigest godigest.Digest + + for digest, path := range testData { + if counter != 10 { + _ = cache.PutBlob(digest, path) + previousDigest = digest + counter++ + } else { + _ = cache.PutBlob(previousDigest, path) + counter = 1 + } + } + + b.ResetTimer() + + helperGetAll(cache, testData) +} + +func BenchmarkMixLocalstack(b *testing.B) { + log := log.NewLogger("error", "") + tableName := generateRandomString() + + // Create Table + _, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", localEndpoint, "create-table", + "--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S", + "--key-schema", "AttributeName=Digest,KeyType=HASH", + "--billing-mode", "PAY_PER_REQUEST").Output() + if err != nil { + panic(err) + } + + cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{ + Endpoint: localEndpoint, + Region: region, + TableName: tableName, + }, log) + testData := generateData() + digestSlice := make([]godigest.Digest, datasetSize) + counter := 0 + + for key := range testData { + digestSlice[counter] = key + counter++ + } + + b.ResetTimer() + + helperMix(cache, testData, digestSlice) +} + +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +// DANGER ZONE: tests with true AWS endpoint +// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +func BenchmarkPutAWS(b *testing.B) { + log := log.NewLogger("error", "") + tableName := generateRandomString() + + // Create Table + _, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", awsEndpoint, "create-table", + "--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S", + "--key-schema", "AttributeName=Digest,KeyType=HASH", + "--billing-mode", "PAY_PER_REQUEST").Output() + if err != nil { + panic(err) + } + + cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{ + Endpoint: awsEndpoint, + Region: region, + TableName: tableName, + }, log) + testData := generateData() + + b.ResetTimer() + + helperPutAll(cache, testData) +} + +func BenchmarkDeleteAWS(b *testing.B) { + log := log.NewLogger("error", "") + tableName := generateRandomString() + + // Create Table + _, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", awsEndpoint, "create-table", + "--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S", + "--key-schema", "AttributeName=Digest,KeyType=HASH", + "--billing-mode", "PAY_PER_REQUEST").Output() + if err != nil { + panic(err) + } + + cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{ + Endpoint: awsEndpoint, + Region: region, + TableName: tableName, + }, log) + testData := generateData() + + for digest, path := range testData { + _ = cache.PutBlob(digest, path) + } + + b.ResetTimer() + + helperDeleteAll(cache, testData) +} + +func BenchmarkHasAWS(b *testing.B) { + log := log.NewLogger("error", "") + tableName := generateRandomString() + + // Create Table + _, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", awsEndpoint, "create-table", + "--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S", + "--key-schema", "AttributeName=Digest,KeyType=HASH", + "--billing-mode", "PAY_PER_REQUEST").Output() + if err != nil { + panic(err) + } + + cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{ + Endpoint: awsEndpoint, + Region: region, + TableName: tableName, + }, log) + testData := generateData() + + for digest, path := range testData { + _ = cache.PutBlob(digest, path) + } + + b.ResetTimer() + + helperHasAll(cache, testData) +} + +func BenchmarkGetAWS(b *testing.B) { + log := log.NewLogger("error", "") + tableName := generateRandomString() + + // Create Table + _, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", awsEndpoint, "create-table", + "--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S", + "--key-schema", "AttributeName=Digest,KeyType=HASH", + "--billing-mode", "PAY_PER_REQUEST").Output() + if err != nil { + panic(err) + } + + cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{ + Endpoint: awsEndpoint, + Region: region, + TableName: tableName, + }, log) + testData := generateData() + counter := 1 + + var previousDigest godigest.Digest + + for digest, path := range testData { + if counter != 10 { + _ = cache.PutBlob(digest, path) + previousDigest = digest + counter++ + } else { + _ = cache.PutBlob(previousDigest, path) + counter = 1 + } + } + + b.ResetTimer() + + helperGetAll(cache, testData) +} + +func BenchmarkMixAWS(b *testing.B) { + log := log.NewLogger("error", "") + tableName := generateRandomString() + + // Create Table + _, err := exec.Command("aws", "dynamodb", "--region", region, "--endpoint-url", awsEndpoint, "create-table", + "--table-name", tableName, "--attribute-definitions", "AttributeName=Digest,AttributeType=S", + "--key-schema", "AttributeName=Digest,KeyType=HASH", + "--billing-mode", "PAY_PER_REQUEST").Output() + if err != nil { + panic(err) + } + + cache, _ := storage.Create("dynamodb", cache.DynamoDBDriverParameters{ + Endpoint: awsEndpoint, + Region: region, + TableName: tableName, + }, log) + testData := generateData() + digestSlice := make([]godigest.Digest, datasetSize) + counter := 0 + + for key := range testData { + digestSlice[counter] = key + counter++ + } + + b.ResetTimer() + + helperMix(cache, testData, digestSlice) +} diff --git a/pkg/storage/constants/constants.go b/pkg/storage/constants/constants.go index f4355a3e..933f8eab 100644 --- a/pkg/storage/constants/constants.go +++ b/pkg/storage/constants/constants.go @@ -19,5 +19,5 @@ const ( DBCacheLockCheckTimeout = 10 * time.Second BoltdbName = "cache" ReferrerFilterAnnotation = "org.opencontainers.references.filtersApplied" - // + DynamoDBDriverName = "dynamodb" ) diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index e3f106b8..c1a14850 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -44,6 +44,7 @@ var ( fileInfoSize = 10 errorText = "new s3 error" errS3 = errors.New(errorText) + zotStorageTest = "zot-storage-test" ) func cleanupStorage(store driver.StorageDriver, name string) { @@ -56,6 +57,10 @@ func skipIt(t *testing.T) { if os.Getenv("S3MOCK_ENDPOINT") == "" { t.Skip("Skipping testing without AWS S3 mock server") } + + if os.Getenv("DYNAMODBMOCK_ENDPOINT") == "" { + t.Skip("Skipping testing without AWS DynamoDB mock server") + } } func createMockStorage(rootDir string, cacheDir string, dedupe bool, store driver.StorageDriver) storage.ImageStore { @@ -84,7 +89,7 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) ( storage.ImageStore, error, ) { - bucket := "zot-storage-test" + bucket := zotStorageTest endpoint := os.Getenv("S3MOCK_ENDPOINT") storageDriverParams := map[string]interface{}{ "rootDir": rootDir, @@ -130,6 +135,66 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) ( return store, il, err } +func createObjectsStoreDynamo(rootDir string, cacheDir string, dedupe bool, tableName string) ( + driver.StorageDriver, + storage.ImageStore, + error, +) { + bucket := zotStorageTest + endpoint := os.Getenv("S3MOCK_ENDPOINT") + storageDriverParams := map[string]interface{}{ + "rootDir": rootDir, + "name": "s3", + "region": "us-east-2", + "bucket": bucket, + "regionendpoint": endpoint, + "accesskey": "minioadmin", + "secretkey": "minioadmin", + "secure": false, + "skipverify": false, + } + + storeName := fmt.Sprintf("%v", storageDriverParams["name"]) + + store, err := factory.Create(storeName, storageDriverParams) + if err != nil { + panic(err) + } + + // create bucket if it doesn't exists + _, err = resty.R().Put("http://" + endpoint + "/" + bucket) + if err != nil { + panic(err) + } + + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + + var cacheDriver cache.Cache + + // from pkg/cli/root.go/applyDefaultValues, s3 magic + + cacheDriver, _ = storage.Create("dynamodb", cache.DynamoDBDriverParameters{ + Endpoint: os.Getenv("DYNAMODBMOCK_ENDPOINT"), + Region: os.Getenv("us-east-2"), + TableName: tableName, + }, log) + + tableName = strings.ReplaceAll(tableName, "/", "") + //nolint:errcheck + cacheDriverDynamo, _ := cacheDriver.(*cache.DynamoDBDriver) + + err = cacheDriverDynamo.NewTable(tableName) + if err != nil { + panic(err) + } + + il := s3.NewImageStore(rootDir, cacheDir, false, storage.DefaultGCDelay, + dedupe, false, log, metrics, nil, store, cacheDriver) + + return store, il, err +} + type FileInfoMock struct { IsDirFn func() bool SizeFn func() int64 @@ -607,7 +672,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { }) Convey("Unable to create subpath cache db", func(c C) { - bucket := "zot-storage-test" + bucket := zotStorageTest endpoint := os.Getenv("S3MOCK_ENDPOINT") storageDriverParams := config.GlobalStorageConfig{ @@ -1448,6 +1513,189 @@ func TestS3Dedupe(t *testing.T) { So(fi1.Size(), ShouldEqual, fi3.Size()) }) }) + + Convey("Dedupe with dynamodb", t, func(c C) { + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + + tdir := t.TempDir() + + storeDriver, imgStore, _ := createObjectsStoreDynamo(testDir, tdir, true, tdir) + defer cleanupStorage(storeDriver, testDir) + + // manifest1 + upload, err := imgStore.NewBlobUpload("dedupe1") + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data3") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + digest := godigest.FromBytes(content) + blob, err := imgStore.PutBlobChunkStreamed("dedupe1", upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + blobDigest1 := digest + So(blobDigest1, ShouldNotBeEmpty) + + err = imgStore.FinishBlobUpload("dedupe1", upload, buf, digest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + _, checkBlobSize1, err := imgStore.CheckBlob("dedupe1", digest) + So(checkBlobSize1, ShouldBeGreaterThan, 0) + So(err, ShouldBeNil) + + blobReadCloser, getBlobSize1, err := imgStore.GetBlob("dedupe1", digest, + "application/vnd.oci.image.layer.v1.tar+gzip") + So(getBlobSize1, ShouldBeGreaterThan, 0) + So(err, ShouldBeNil) + err = blobReadCloser.Close() + So(err, ShouldBeNil) + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload("dedupe1", bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob("dedupe1", cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest, + Size: int64(buflen), + }, + }, + } + + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(manifestBuf) + _, err = imgStore.PutImageManifest("dedupe1", digest.String(), + ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("dedupe1", digest.String()) + So(err, ShouldBeNil) + + // manifest2 + upload, err = imgStore.NewBlobUpload("dedupe2") + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content = []byte("test-data3") + buf = bytes.NewBuffer(content) + buflen = buf.Len() + digest = godigest.FromBytes(content) + + blob, err = imgStore.PutBlobChunkStreamed("dedupe2", upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + blobDigest2 := digest + So(blobDigest2, ShouldNotBeEmpty) + + err = imgStore.FinishBlobUpload("dedupe2", upload, buf, digest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + _, checkBlobSize2, err := imgStore.CheckBlob("dedupe2", digest) + So(err, ShouldBeNil) + So(checkBlobSize2, ShouldBeGreaterThan, 0) + + blobReadCloser, getBlobSize2, err := imgStore.GetBlob("dedupe2", digest, + "application/vnd.oci.image.layer.v1.tar+gzip") + So(err, ShouldBeNil) + So(getBlobSize2, ShouldBeGreaterThan, 0) + So(checkBlobSize1, ShouldEqual, checkBlobSize2) + So(getBlobSize1, ShouldEqual, getBlobSize2) + err = blobReadCloser.Close() + So(err, ShouldBeNil) + + cblob, cdigest = test.GetRandomImageConfig() + _, clen, err = imgStore.FullBlobUpload("dedupe2", bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err = imgStore.CheckBlob("dedupe2", cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: digest, + Size: int64(buflen), + }, + }, + } + manifest.SchemaVersion = 2 + manifestBuf, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(manifestBuf) + _, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest, + manifestBuf) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest("dedupe2", digest.String()) + So(err, ShouldBeNil) + + fi1, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256", + blobDigest1.Encoded())) + So(err, ShouldBeNil) + + fi2, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256", + blobDigest2.Encoded())) + So(err, ShouldBeNil) + + // original blob should have the real content of blob + So(fi1.Size(), ShouldNotEqual, fi2.Size()) + So(fi1.Size(), ShouldBeGreaterThan, 0) + // deduped blob should be of size 0 + So(fi2.Size(), ShouldEqual, 0) + + Convey("Check that delete blobs moves the real content to the next contenders", func() { + // if we delete blob1, the content should be moved to blob2 + err = imgStore.DeleteBlob("dedupe1", blobDigest1) + So(err, ShouldBeNil) + + _, err = storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256", + blobDigest1.Encoded())) + So(err, ShouldNotBeNil) + + fi2, err = storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256", + blobDigest2.Encoded())) + So(err, ShouldBeNil) + + So(fi2.Size(), ShouldBeGreaterThan, 0) + // the second blob should now be equal to the deleted blob. + So(fi2.Size(), ShouldEqual, fi1.Size()) + + err = imgStore.DeleteBlob("dedupe2", blobDigest2) + So(err, ShouldBeNil) + + _, err = storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256", + blobDigest2.Encoded())) + So(err, ShouldNotBeNil) + }) + }) } func TestS3PullRange(t *testing.T) { diff --git a/test/blackbox/cloud-only.bats b/test/blackbox/cloud-only.bats new file mode 100644 index 00000000..571f07b1 --- /dev/null +++ b/test/blackbox/cloud-only.bats @@ -0,0 +1,88 @@ +load helpers_cloud + +function setup() { + # Verify prerequisites are available + if ! verify_prerequisites; then + exit 1 + fi + + # Setup zot server + local zot_root_dir=${BATS_FILE_TMPDIR}/zot + local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json + + echo ${zot_root_dir} >&3 + + mkdir -p ${zot_root_dir} + + cat > ${zot_config_file}<&3 + [[ "$line" =~ .*metadata.* || "$line" =~ .*trivy.* ]] + done +} diff --git a/test/blackbox/helpers_cloud.bash b/test/blackbox/helpers_cloud.bash new file mode 100644 index 00000000..5daaff40 --- /dev/null +++ b/test/blackbox/helpers_cloud.bash @@ -0,0 +1,44 @@ +ROOT_DIR=$(git rev-parse --show-toplevel) +OS="${OS:-linux}" +ARCH="${ARCH:-amd64}" +ZOT_PATH=${ROOT_DIR}/bin/zot-${OS}-${ARCH} + + +function verify_prerequisites { + if [ ! -f ${ZOT_PATH} ]; then + echo "you need to build ${ZOT_PATH} before running the tests" >&3 + return 1 + fi + + if [ ! command -v skopeo &> /dev/null ]; then + echo "you need to install skopeo as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! command -v awslocal ] &>/dev/null; then + echo "you need to install aws cli as a prerequisite to running the tests" >&3 + return 1 + fi + + return 0 +} + +function zot_serve_strace() { + local config_file=${1} + strace -o "strace.txt" -f -e trace=openat ${ZOT_PATH} serve ${config_file} & +} + +function zot_stop() { + pkill zot +} + +function wait_zot_reachable() { + zot_url=${1} + curl --connect-timeout 3 \ + --max-time 3 \ + --retry 10 \ + --retry-delay 0 \ + --retry-max-time 60 \ + --retry-connrefused \ + ${zot_url} +}