diff --git a/.github/workflows/ecosystem-tools.yaml b/.github/workflows/ecosystem-tools.yaml index 4963336d..27a81054 100644 --- a/.github/workflows/ecosystem-tools.yaml +++ b/.github/workflows/ecosystem-tools.yaml @@ -17,6 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/clean-runner - uses: actions/setup-go@v4 with: cache: false @@ -98,6 +99,9 @@ jobs: - name: Run annotations tests run: | make test-annotations + - name: Run garbage collect tests + run: | + make test-garbage-collect - name: Install localstack run: | pip install --upgrade pyopenssl @@ -129,4 +133,4 @@ jobs: sudo du -sh /var/ sudo du -sh /var/lib/docker/ du -sh /home/runner/work/ - set +x \ No newline at end of file + set +x diff --git a/.github/workflows/gc-stress-test.yaml b/.github/workflows/gc-stress-test.yaml index fe80e987..c99889cd 100644 --- a/.github/workflows/gc-stress-test.yaml +++ b/.github/workflows/gc-stress-test.yaml @@ -12,8 +12,8 @@ on: permissions: read-all jobs: - client-tools: - name: GC with short interval + gc-referrers-stress-local: + name: GC(with referrers) on filesystem with short interval runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -27,7 +27,31 @@ jobs: run: | make binary make bench - ./bin/zot-linux-amd64 serve examples/config-gc-bench.json & + ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-local.json & + sleep 10 + bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 + + killall -r zot-* + + # clean zot storage + sudo rm -rf /tmp/zot + + gc-stress-local: + name: GC(without referrers) on filesystem with short interval + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/clean-runner + - uses: actions/setup-go@v4 + with: + cache: false + go-version: 1.20.x + + - name: Run zb + run: | + make binary + make bench + ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-local.json & sleep 10 bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 55458ac2..e497e46d 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -74,3 +74,84 @@ jobs: - name: Run sync harness run: | make test-sync-harness + gc-referrers-stress-s3: + name: GC(with referrers) on S3 with short interval + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/clean-runner + - uses: actions/setup-go@v4 + with: + cache: false + go-version: 1.20.x + - name: Setup localstack service + run: | + pip install localstack # Install LocalStack cli + docker pull localstack/localstack:1.3 # Make sure to pull the latest version of the image + localstack start -d # Start LocalStack in the background + + echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container + localstack wait -t 30 # to become ready before timing out + echo "Startup complete" + + aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}" + aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake + - name: Run zb + run: | + make binary + make bench + ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3.json & + sleep 10 + bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 + + killall -r zot-* + + # clean zot storage + sudo rm -rf /tmp/zot + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake + + gc-stress-s3: + name: GC(without referrers) on S3 with short interval + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/clean-runner + - uses: actions/setup-go@v4 + with: + cache: false + go-version: 1.20.x + - name: Setup localstack service + run: | + pip install localstack # Install LocalStack cli + docker pull localstack/localstack:1.3 # Make sure to pull the latest version of the image + localstack start -d # Start LocalStack in the background + + echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container + localstack wait -t 30 # to become ready before timing out + echo "Startup complete" + + aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}" + aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake + - name: Run zb + run: | + make binary + make bench + ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3.json & + sleep 10 + bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 + + killall -r zot-* + + # clean zot storage + sudo rm -rf /tmp/zot + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake diff --git a/Makefile b/Makefile index ef298ae3..4b6b4ae7 100644 --- a/Makefile +++ b/Makefile @@ -357,6 +357,14 @@ test-push-pull: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(H test-push-pull-verbose: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(CRICTL) $(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/pushpull.bats +.PHONY: test-garbage-collect +test-garbage-collect: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) + $(BATS) --trace --print-output-on-failure test/blackbox/garbage_collect.bats + +.PHONY: test-garbage-collect-verbose +test-garbage-collect-verbose: binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) + $(BATS) --trace --verbose-run --print-output-on-failure --show-output-of-passing-tests test/blackbox/garbage_collect.bats + .PHONY: test-push-pull-running-dedupe test-push-pull-running-dedupe: check-linux binary check-skopeo $(BATS) $(REGCLIENT) $(ORAS) $(HELM) $(BATS) --trace --print-output-on-failure test/blackbox/pushpull_running_dedupe.bats diff --git a/errors/errors.go b/errors/errors.go index 41e19590..47985604 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -58,6 +58,7 @@ var ( ErrBadBlob = errors.New("blob: bad blob") ErrBadBlobDigest = errors.New("blob: bad blob digest") ErrBlobReferenced = errors.New("blob: referenced by manifest") + ErrManifestReferenced = errors.New("manifest: referenced by index image") ErrUnknownCode = errors.New("error: unknown error code") ErrBadCACert = errors.New("tls: invalid ca cert") ErrBadUser = errors.New("auth: non-existent user") @@ -155,4 +156,7 @@ var ( ErrGQLEndpointNotFound = errors.New("cli: the server doesn't have a gql endpoint") ErrGQLQueryNotSupported = errors.New("cli: query is not supported or has different arguments") ErrBadHTTPStatusCode = errors.New("cli: the response doesn't contain the expected status code") + ErrFileAlreadyCancelled = errors.New("storageDriver: file already cancelled") + ErrFileAlreadyClosed = errors.New("storageDriver: file already closed") + ErrFileAlreadyCommitted = errors.New("storageDriver: file already committed") ) diff --git a/examples/config-gc.json b/examples/config-gc.json index c4923946..cad60105 100644 --- a/examples/config-gc.json +++ b/examples/config-gc.json @@ -3,7 +3,10 @@ "storage": { "rootDirectory": "/tmp/zot", "gc": true, - "gcDelay": "1s" + "gcReferrers": true, + "gcDelay": "2h", + "untaggedImageRetentionDelay": "4h", + "gcInterval": "1h" }, "http": { "address": "127.0.0.1", diff --git a/go.mod b/go.mod index ab1d934f..0089e9d6 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/99designs/gqlgen v0.17.35 github.com/Masterminds/semver v1.5.0 - github.com/apex/log v1.9.0 + github.com/apex/log v1.9.0 // indirect github.com/aquasecurity/trivy-db v0.0.0-20230726112157-167ba4f2faeb github.com/bmatcuk/doublestar/v4 v4.6.0 github.com/briandowns/spinner v1.23.0 @@ -23,7 +23,6 @@ require ( github.com/gorilla/mux v1.8.0 github.com/hashicorp/golang-lru/v2 v2.0.5 github.com/json-iterator/go v1.1.12 - github.com/minio/sha256-simd v1.0.1 github.com/mitchellh/mapstructure v1.5.0 github.com/nmcclain/ldap v0.0.0-20210720162743-7f8d1e44eeba github.com/olekukonko/tablewriter v0.0.5 @@ -367,7 +366,6 @@ require ( github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/klauspost/compress v1.16.6 // indirect - github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/klauspost/pgzip v1.2.6 // indirect github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f // indirect github.com/knqyf263/go-deb-version v0.0.0-20230223133812-3ed183d23422 // indirect diff --git a/go.sum b/go.sum index ec290448..cc5c4c80 100644 --- a/go.sum +++ b/go.sum @@ -1130,8 +1130,6 @@ github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= -github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -1258,8 +1256,6 @@ github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/migueleliasweb/go-github-mock v0.0.19 h1:z/88f6wPqZVFnE7s9DbwXMhCtmV/0FofNxc4M7FuSdU= github.com/migueleliasweb/go-github-mock v0.0.19/go.mod h1:dBoCB3W9NjzyABhoGkfI0iSlFpzulAXhI7M+9A4ONYI= -github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= -github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.5/go.mod h1:v8+iFts2sPIKUV1ltktPXMCC8fumSKFItNcD2cLtRR4= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= @@ -2059,7 +2055,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/api/config/config.go b/pkg/api/config/config.go index acf9925d..b22eadac 100644 --- a/pkg/api/config/config.go +++ b/pkg/api/config/config.go @@ -23,15 +23,17 @@ var ( ) type StorageConfig struct { - RootDirectory string - Dedupe bool - RemoteCache bool - GC bool - Commit bool - GCDelay time.Duration - GCInterval time.Duration - StorageDriver map[string]interface{} `mapstructure:",omitempty"` - CacheDriver map[string]interface{} `mapstructure:",omitempty"` + RootDirectory string + Dedupe bool + RemoteCache bool + GC bool + Commit bool + GCDelay time.Duration + GCInterval time.Duration + GCReferrers bool + UntaggedImageRetentionDelay time.Duration + StorageDriver map[string]interface{} `mapstructure:",omitempty"` + CacheDriver map[string]interface{} `mapstructure:",omitempty"` } type TLSConfig struct { @@ -188,8 +190,9 @@ func New() *Config { BinaryType: BinaryType, Storage: GlobalStorageConfig{ StorageConfig: StorageConfig{ - GC: true, GCDelay: storageConstants.DefaultGCDelay, - GCInterval: storageConstants.DefaultGCInterval, Dedupe: true, + GC: true, GCReferrers: true, GCDelay: storageConstants.DefaultGCDelay, + UntaggedImageRetentionDelay: storageConstants.DefaultUntaggedImgeRetentionDelay, + GCInterval: storageConstants.DefaultGCInterval, Dedupe: true, }, }, HTTP: HTTPConfig{Address: "127.0.0.1", Port: "8080", Auth: &AuthConfig{FailDelay: 0}}, diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index f71ce857..c717759c 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -971,7 +971,7 @@ func TestInterruptedBlobUpload(t *testing.T) { defer cm.StopServer() client := resty.New() - blob := make([]byte, 50*1024*1024) + blob := make([]byte, 200*1024*1024) digest := godigest.FromBytes(blob).String() //nolint: dupl @@ -1024,6 +1024,7 @@ func TestInterruptedBlobUpload(t *testing.T) { So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) }) + //nolint: dupl Convey("Test negative interrupt PATCH blob upload", func() { resp, err := client.R().Post(baseURL + "/v2/" + AuthorizedNamespace + "/blobs/uploads/") So(err, ShouldBeNil) @@ -1126,6 +1127,7 @@ func TestInterruptedBlobUpload(t *testing.T) { So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) }) + //nolint: dupl Convey("Test negative interrupt PUT blob upload", func() { resp, err := client.R().Post(baseURL + "/v2/" + AuthorizedNamespace + "/blobs/uploads/") So(err, ShouldBeNil) @@ -6746,6 +6748,12 @@ func TestManifestImageIndex(t *testing.T) { So(digestHdr, ShouldEqual, digest.String()) }) + Convey("Deleting manifest contained by a multiarch image should not be allowed", func() { + resp, err = resty.R().Delete(baseURL + fmt.Sprintf("/v2/index/manifests/%s", m2dgst.String())) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusMethodNotAllowed) + }) + Convey("Deleting an image index", func() { // delete manifest by tag should pass resp, err = resty.R().Delete(baseURL + "/v2/index/manifests/test:index3") @@ -7296,7 +7304,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) { So(digest, ShouldNotBeNil) // monolithic blob upload - injected := inject.InjectFailure(0) + injected := inject.InjectFailure(2) if injected { request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, loc, bytes.NewReader(content)) tokens := strings.Split(loc, "/") @@ -7369,7 +7377,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) { // Testing router path: @Router /v2/{name}/manifests/{reference} [put] //nolint:lll // gofumpt conflicts with lll Convey("Uploading an image manifest blob (when injected simulates that PutImageManifest failed due to 'too many open files' error)", func() { - injected := inject.InjectFailure(1) + injected := inject.InjectFailure(2) request, _ := http.NewRequestWithContext(context.TODO(), http.MethodPut, baseURL, bytes.NewReader(content)) request = mux.SetURLVars(request, map[string]string{"name": "repotest", "reference": "1.0"}) @@ -7430,6 +7438,7 @@ func TestInjectTooManyOpenFiles(t *testing.T) { So(resp.StatusCode, ShouldEqual, http.StatusCreated) } }) + Convey("when index.json is not in json format", func() { resp, err = resty.R().SetHeader("Content-Type", "application/vnd.oci.image.manifest.v1+json"). SetBody(content).Put(baseURL + "/v2/repotest/manifests/v1.0") @@ -7456,21 +7465,22 @@ func TestInjectTooManyOpenFiles(t *testing.T) { func TestGCSignaturesAndUntaggedManifests(t *testing.T) { Convey("Make controller", t, func() { - repoName := "testrepo" //nolint:goconst - tag := "0.0.1" - - port := test.GetFreePort() - baseURL := test.GetBaseURL(port) - conf := config.New() - conf.HTTP.Port = port - - ctlr := makeController(conf, t.TempDir()) - Convey("Garbage collect signatures without subject and manifests without tags", func(c C) { + repoName := "testrepo" //nolint:goconst + tag := "0.0.1" + + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + + ctlr := makeController(conf, t.TempDir()) + dir := t.TempDir() ctlr.Config.Storage.RootDirectory = dir ctlr.Config.Storage.GC = true ctlr.Config.Storage.GCDelay = 1 * time.Millisecond + ctlr.Config.Storage.UntaggedImageRetentionDelay = 1 * time.Millisecond ctlr.Config.Storage.Dedupe = false @@ -7582,75 +7592,88 @@ func TestGCSignaturesAndUntaggedManifests(t *testing.T) { So(err, ShouldBeNil) }) - // push an image without tag - cfg, layers, manifest, err := test.GetImageComponents(2) //nolint:staticcheck - So(err, ShouldBeNil) + Convey("Overwrite original image, signatures should be garbage-collected", func() { + // push an image without tag + cfg, layers, manifest, err := test.GetImageComponents(2) //nolint:staticcheck + So(err, ShouldBeNil) - manifestBuf, err := json.Marshal(manifest) - So(err, ShouldBeNil) - untaggedManifestDigest := godigest.FromBytes(manifestBuf) + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + untaggedManifestDigest := godigest.FromBytes(manifestBuf) - err = test.UploadImage( - test.Image{ - Config: cfg, - Layers: layers, - Manifest: manifest, - }, baseURL, repoName, untaggedManifestDigest.String()) - So(err, ShouldBeNil) + err = test.UploadImage( + test.Image{ + Config: cfg, + Layers: layers, + Manifest: manifest, + }, baseURL, repoName, untaggedManifestDigest.String()) + So(err, ShouldBeNil) - // overwrite image so that signatures will get invalidated and gc'ed - cfg, layers, manifest, err = test.GetImageComponents(3) //nolint:staticcheck - So(err, ShouldBeNil) + // overwrite image so that signatures will get invalidated and gc'ed + cfg, layers, manifest, err = test.GetImageComponents(3) //nolint:staticcheck + So(err, ShouldBeNil) - err = test.UploadImage( - test.Image{ - Config: cfg, - Layers: layers, - Manifest: manifest, - }, baseURL, repoName, tag) - So(err, ShouldBeNil) + err = test.UploadImage( + test.Image{ + Config: cfg, + Layers: layers, + Manifest: manifest, + }, baseURL, repoName, tag) + So(err, ShouldBeNil) - manifestBuf, err = json.Marshal(manifest) - So(err, ShouldBeNil) - newManifestDigest := godigest.FromBytes(manifestBuf) + manifestBuf, err = json.Marshal(manifest) + So(err, ShouldBeNil) + newManifestDigest := godigest.FromBytes(manifestBuf) - err = ctlr.StoreController.DefaultStore.RunGCRepo(repoName) - So(err, ShouldBeNil) + err = ctlr.StoreController.DefaultStore.RunGCRepo(repoName) + So(err, ShouldBeNil) - // both signatures should be gc'ed - resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, cosignTag)) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + // both signatures should be gc'ed + resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, cosignTag)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) - resp, err = resty.R().SetQueryParam("artifactType", notreg.ArtifactTypeNotation).Get( - fmt.Sprintf("%s/v2/%s/referrers/%s", baseURL, repoName, digest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) + resp, err = resty.R().SetQueryParam("artifactType", notreg.ArtifactTypeNotation).Get( + fmt.Sprintf("%s/v2/%s/referrers/%s", baseURL, repoName, digest.String())) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) - err = json.Unmarshal(resp.Body(), &index) - So(err, ShouldBeNil) - So(len(index.Manifests), ShouldEqual, 0) + err = json.Unmarshal(resp.Body(), &index) + So(err, ShouldBeNil) + So(len(index.Manifests), ShouldEqual, 0) - resp, err = resty.R().SetQueryParam("artifactType", notreg.ArtifactTypeNotation).Get( - fmt.Sprintf("%s/v2/%s/referrers/%s", baseURL, repoName, newManifestDigest.String())) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusOK) + resp, err = resty.R().SetQueryParam("artifactType", notreg.ArtifactTypeNotation).Get( + fmt.Sprintf("%s/v2/%s/referrers/%s", baseURL, repoName, newManifestDigest.String())) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) - err = json.Unmarshal(resp.Body(), &index) - So(err, ShouldBeNil) - So(len(index.Manifests), ShouldEqual, 0) + err = json.Unmarshal(resp.Body(), &index) + So(err, ShouldBeNil) + So(len(index.Manifests), ShouldEqual, 0) - // untagged image should also be gc'ed - resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, untaggedManifestDigest)) - So(err, ShouldBeNil) - So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + // untagged image should also be gc'ed + resp, err = resty.R().Get(baseURL + fmt.Sprintf("/v2/%s/manifests/%s", repoName, untaggedManifestDigest)) + So(err, ShouldBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) + }) }) Convey("Do not gc manifests which are part of a multiarch image", func(c C) { + repoName := "testrepo" //nolint:goconst + tag := "0.0.1" + + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + + ctlr := makeController(conf, t.TempDir()) + dir := t.TempDir() ctlr.Config.Storage.RootDirectory = dir ctlr.Config.Storage.GC = true - ctlr.Config.Storage.GCDelay = 500 * time.Millisecond + ctlr.Config.Storage.GCDelay = 1 * time.Second + ctlr.Config.Storage.UntaggedImageRetentionDelay = 1 * time.Second err := test.WriteImageToFileSystem(test.CreateDefaultImage(), repoName, tag, test.GetDefaultStoreController(dir, ctlr.Log)) @@ -7787,7 +7810,10 @@ func TestPeriodicGC(t *testing.T) { subPaths := make(map[string]config.StorageConfig) - subPaths["/a"] = config.StorageConfig{RootDirectory: subDir, GC: true, GCDelay: 1 * time.Second, GCInterval: 24 * time.Hour, RemoteCache: false, Dedupe: false} //nolint:lll // gofumpt conflicts with lll + subPaths["/a"] = config.StorageConfig{ + RootDirectory: subDir, GC: true, GCDelay: 1 * time.Second, + UntaggedImageRetentionDelay: 1 * time.Second, GCInterval: 24 * time.Hour, RemoteCache: false, Dedupe: false, + } //nolint:lll // gofumpt conflicts with lll ctlr.Config.Storage.Dedupe = false ctlr.Config.Storage.SubPaths = subPaths diff --git a/pkg/api/routes.go b/pkg/api/routes.go index fcccc1e9..0ccafcdf 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -832,6 +832,11 @@ func (rh *RouteHandler) DeleteManifest(response http.ResponseWriter, request *ht details["reference"] = reference e := apiErr.NewError(apiErr.UNSUPPORTED).AddDetail(details) zcommon.WriteJSON(response, http.StatusBadRequest, apiErr.NewErrorList(e)) + } else if errors.Is(err, zerr.ErrManifestReferenced) { + // manifest is part of an index image, don't allow index manipulations. + details["reference"] = reference + e := apiErr.NewError(apiErr.DENIED).AddDetail(details) + zcommon.WriteJSON(response, http.StatusMethodNotAllowed, apiErr.NewErrorList(e)) } else { rh.c.Log.Error().Err(err).Msg("unexpected error") response.WriteHeader(http.StatusInternalServerError) diff --git a/pkg/cli/cve_cmd_test.go b/pkg/cli/cve_cmd_test.go index 7860543b..bc9057cf 100644 --- a/pkg/cli/cve_cmd_test.go +++ b/pkg/cli/cve_cmd_test.go @@ -468,7 +468,7 @@ func TestNegativeServerResponse(t *testing.T) { dir := t.TempDir() - imageStore := local.NewImageStore(dir, false, 0, false, false, + imageStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) storeController := storage.StoreController{ diff --git a/pkg/cli/root.go b/pkg/cli/root.go index 2d4876a8..ce00cbe8 100644 --- a/pkg/cli/root.go +++ b/pkg/cli/root.go @@ -27,7 +27,6 @@ import ( "zotregistry.io/zot/pkg/extensions/monitoring" zlog "zotregistry.io/zot/pkg/log" storageConstants "zotregistry.io/zot/pkg/storage/constants" - "zotregistry.io/zot/pkg/storage/s3" ) // metadataConfig reports metadata after parsing, which we use to track @@ -631,6 +630,10 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper, log z config.Storage.GCDelay = 0 } + if viperInstance.Get("storage::gcdelay") == nil { + config.Storage.UntaggedImageRetentionDelay = 0 + } + if viperInstance.Get("storage::gcinterval") == nil { config.Storage.GCInterval = 0 } @@ -649,7 +652,7 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper, log z // s3 dedup=false, check for previous dedup usage and set to true if cachedb found if !config.Storage.Dedupe && config.Storage.StorageDriver != nil { cacheDir, _ := config.Storage.StorageDriver["rootdirectory"].(string) - cachePath := path.Join(cacheDir, s3.CacheDBName+storageConstants.DBExtensionName) + cachePath := path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) if _, err := os.Stat(cachePath); err == nil { log.Info().Msg("Config: dedupe set to false for s3 driver but used to be true.") @@ -667,10 +670,10 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper, log z storageConfig.RemoteCache = true } - // s3 dedup=false, check for previous dedup usage and set to true if cachedb found + // s3 dedup=false, check for previous dedupe usage and set to true if cachedb found if !storageConfig.Dedupe && storageConfig.StorageDriver != nil { subpathCacheDir, _ := storageConfig.StorageDriver["rootdirectory"].(string) - subpathCachePath := path.Join(subpathCacheDir, s3.CacheDBName+storageConstants.DBExtensionName) + subpathCachePath := path.Join(subpathCacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) if _, err := os.Stat(subpathCachePath); err == nil { log.Info().Msg("Config: dedupe set to false for s3 driver but used to be true. ") @@ -682,11 +685,21 @@ func applyDefaultValues(config *config.Config, viperInstance *viper.Viper, log z // if gc is enabled if storageConfig.GC { + // and gcReferrers is not set, it is set to default value + if !viperInstance.IsSet("storage::subpaths::" + name + "::gcreferrers") { + storageConfig.GCReferrers = true + } + // and gcDelay is not set, it is set to default value if !viperInstance.IsSet("storage::subpaths::" + name + "::gcdelay") { storageConfig.GCDelay = storageConstants.DefaultGCDelay } + // and retentionDelay is not set, it is set to default value + if !viperInstance.IsSet("storage::subpaths::" + name + "::retentiondelay") { + storageConfig.UntaggedImageRetentionDelay = storageConstants.DefaultUntaggedImgeRetentionDelay + } + // and gcInterval is not set, it is set to default value if !viperInstance.IsSet("storage::subpaths::" + name + "::gcinterval") { storageConfig.GCInterval = storageConstants.DefaultGCInterval diff --git a/pkg/cli/root_test.go b/pkg/cli/root_test.go index d0359276..f6859e4f 100644 --- a/pkg/cli/root_test.go +++ b/pkg/cli/root_test.go @@ -14,7 +14,6 @@ import ( "zotregistry.io/zot/pkg/api/config" "zotregistry.io/zot/pkg/cli" storageConstants "zotregistry.io/zot/pkg/storage/constants" - "zotregistry.io/zot/pkg/storage/s3" . "zotregistry.io/zot/pkg/test" ) @@ -521,7 +520,7 @@ func TestVerify(t *testing.T) { // s3 dedup=false, check for previous dedup usage and set to true if cachedb found cacheDir := t.TempDir() - existingDBPath := path.Join(cacheDir, s3.CacheDBName+storageConstants.DBExtensionName) + existingDBPath := path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) _, err = os.Create(existingDBPath) So(err, ShouldBeNil) @@ -537,7 +536,7 @@ func TestVerify(t *testing.T) { // subpath s3 dedup=false, check for previous dedup usage and set to true if cachedb found cacheDir = t.TempDir() - existingDBPath = path.Join(cacheDir, s3.CacheDBName+storageConstants.DBExtensionName) + existingDBPath = path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) _, err = os.Create(existingDBPath) So(err, ShouldBeNil) diff --git a/pkg/extensions/extension_image_trust_test.go b/pkg/extensions/extension_image_trust_test.go index 2c81c352..e55adb8f 100644 --- a/pkg/extensions/extension_image_trust_test.go +++ b/pkg/extensions/extension_image_trust_test.go @@ -149,7 +149,7 @@ func TestSignatureUploadAndVerification(t *testing.T) { writers := io.MultiWriter(os.Stdout, logFile) logger.Logger = logger.Output(writers) - imageStore := local.NewImageStore(globalDir, false, 0, false, false, + imageStore := local.NewImageStore(globalDir, false, false, 0, 0, false, false, logger, monitoring.NewMetricsServer(false, logger), nil, nil) storeController := storage.StoreController{ @@ -267,7 +267,7 @@ func TestSignatureUploadAndVerification(t *testing.T) { writers := io.MultiWriter(os.Stdout, logFile) logger.Logger = logger.Output(writers) - imageStore := local.NewImageStore(globalDir, false, 0, false, false, + imageStore := local.NewImageStore(globalDir, false, false, 0, 0, false, false, logger, monitoring.NewMetricsServer(false, logger), nil, nil) storeController := storage.StoreController{ @@ -385,7 +385,7 @@ func TestSignatureUploadAndVerification(t *testing.T) { writers := io.MultiWriter(os.Stdout, logFile) logger.Logger = logger.Output(writers) - imageStore := local.NewImageStore(globalDir, false, 0, false, false, + imageStore := local.NewImageStore(globalDir, false, false, 0, 0, false, false, logger, monitoring.NewMetricsServer(false, logger), nil, nil) storeController := storage.StoreController{ @@ -558,7 +558,7 @@ func TestSignatureUploadAndVerification(t *testing.T) { writers := io.MultiWriter(os.Stdout, logFile) logger.Logger = logger.Output(writers) - imageStore := local.NewImageStore(globalDir, false, 0, false, false, + imageStore := local.NewImageStore(globalDir, false, false, 0, 0, false, false, logger, monitoring.NewMetricsServer(false, logger), nil, nil) storeController := storage.StoreController{ @@ -813,7 +813,7 @@ func TestSignatureUploadAndVerification(t *testing.T) { writers := io.MultiWriter(os.Stdout, logFile) logger.Logger = logger.Output(writers) - imageStore := local.NewImageStore(globalDir, false, 0, false, false, + imageStore := local.NewImageStore(globalDir, false, false, 0, 0, false, false, logger, monitoring.NewMetricsServer(false, logger), nil, nil) storeController := storage.StoreController{ diff --git a/pkg/extensions/extension_scrub.go b/pkg/extensions/extension_scrub.go index 2997a23d..1d67acde 100644 --- a/pkg/extensions/extension_scrub.go +++ b/pkg/extensions/extension_scrub.go @@ -4,8 +4,6 @@ package extensions import ( - "errors" - "io" "time" "zotregistry.io/zot/pkg/api/config" @@ -30,19 +28,25 @@ func EnableScrubExtension(config *config.Config, log log.Logger, storeController log.Warn().Msg("Scrub interval set to too-short interval < 2h, changing scrub duration to 2 hours and continuing.") //nolint:lll // gofumpt conflicts with lll } - generator := &taskGenerator{ - imgStore: storeController.DefaultStore, - log: log, + // is local imagestore (because of umoci dependency which works only locally) + if config.Storage.StorageDriver == nil { + generator := &taskGenerator{ + imgStore: storeController.DefaultStore, + log: log, + } + sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority) } - sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority) if config.Storage.SubPaths != nil { for route := range config.Storage.SubPaths { - generator := &taskGenerator{ - imgStore: storeController.SubStore[route], - log: log, + // is local imagestore (because of umoci dependency which works only locally) + if config.Storage.SubPaths[route].StorageDriver == nil { + generator := &taskGenerator{ + imgStore: storeController.SubStore[route], + log: log, + } + sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority) } - sch.SubmitGenerator(generator, config.Extensions.Scrub.Interval, scheduler.LowPriority) } } } else { @@ -59,8 +63,7 @@ type taskGenerator struct { func (gen *taskGenerator) Next() (scheduler.Task, error) { repo, err := gen.imgStore.GetNextRepository(gen.lastRepo) - - if err != nil && !errors.Is(err, io.EOF) { + if err != nil { return nil, err } diff --git a/pkg/extensions/lint/lint_test.go b/pkg/extensions/lint/lint_test.go index 8db278a8..240cafaf 100644 --- a/pkg/extensions/lint/lint_test.go +++ b/pkg/extensions/lint/lint_test.go @@ -490,7 +490,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { var index ispec.Index linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) - imgStore := local.NewImageStore(dir, false, 0, false, false, + imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) indexContent, err := imgStore.GetIndexContent("zot-test") @@ -522,7 +522,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { var index ispec.Index linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) - imgStore := local.NewImageStore(dir, false, 0, false, false, + imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) indexContent, err := imgStore.GetIndexContent("zot-test") @@ -592,7 +592,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { index.Manifests = append(index.Manifests, manifestDesc) linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) - imgStore := local.NewImageStore(dir, false, 0, false, false, + imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) @@ -654,7 +654,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { index.Manifests = append(index.Manifests, manifestDesc) linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) - imgStore := local.NewImageStore(dir, false, 0, false, false, + imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) @@ -718,7 +718,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { index.Manifests = append(index.Manifests, manifestDesc) linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) - imgStore := local.NewImageStore(dir, false, 0, false, false, + imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) pass, err := linter.CheckMandatoryAnnotations("zot-test", digest, imgStore) @@ -781,7 +781,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { index.Manifests = append(index.Manifests, manifestDesc) linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) - imgStore := local.NewImageStore(dir, false, 0, false, false, + imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) err = os.Chmod(path.Join(dir, "zot-test", "blobs"), 0o000) @@ -879,7 +879,7 @@ func TestVerifyMandatoryAnnotationsFunction(t *testing.T) { index.Manifests = append(index.Manifests, manifestDesc) linter := lint.NewLinter(lintConfig, log.NewLogger("debug", "")) - imgStore := local.NewImageStore(dir, false, 0, false, false, + imgStore := local.NewImageStore(dir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), linter, nil) err = os.Chmod(path.Join(dir, "zot-test", "blobs", "sha256", manifest.Config.Digest.Encoded()), 0o000) diff --git a/pkg/extensions/scrub/scrub_test.go b/pkg/extensions/scrub/scrub_test.go index 1fd3f95a..29814850 100644 --- a/pkg/extensions/scrub/scrub_test.go +++ b/pkg/extensions/scrub/scrub_test.go @@ -198,7 +198,7 @@ func TestRunScrubRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, 1*time.Second, true, + imgStore := local.NewImageStore(dir, true, true, 1*time.Second, 1*time.Second, true, true, log, metrics, nil, cacheDriver) srcStorageCtlr := test.GetDefaultStoreController(dir, log) @@ -234,7 +234,7 @@ func TestRunScrubRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, 1*time.Second, true, + imgStore := local.NewImageStore(dir, true, true, 1*time.Second, 1*time.Second, true, true, log, metrics, nil, cacheDriver) srcStorageCtlr := test.GetDefaultStoreController(dir, log) @@ -276,8 +276,8 @@ func TestRunScrubRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, 1*time.Second, - true, true, log, metrics, nil, cacheDriver, + imgStore := local.NewImageStore(dir, true, true, 1*time.Second, + 1*time.Second, true, true, log, metrics, nil, cacheDriver, ) srcStorageCtlr := test.GetDefaultStoreController(dir, log) diff --git a/pkg/extensions/search/cve/cve_test.go b/pkg/extensions/search/cve/cve_test.go index 9dc8c855..b93f56b4 100644 --- a/pkg/extensions/search/cve/cve_test.go +++ b/pkg/extensions/search/cve/cve_test.go @@ -322,8 +322,8 @@ func TestImageFormat(t *testing.T) { dbDir := t.TempDir() metrics := monitoring.NewMetricsServer(false, log) - defaultStore := local.NewImageStore(imgDir, false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil) + defaultStore := local.NewImageStore(imgDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) storeController := storage.StoreController{DefaultStore: defaultStore} params := boltdb.DBParameters{ diff --git a/pkg/extensions/search/cve/trivy/scanner_internal_test.go b/pkg/extensions/search/cve/trivy/scanner_internal_test.go index c867b747..6f791a55 100644 --- a/pkg/extensions/search/cve/trivy/scanner_internal_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_internal_test.go @@ -25,6 +25,7 @@ import ( mTypes "zotregistry.io/zot/pkg/meta/types" "zotregistry.io/zot/pkg/storage" storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/storage/imagestore" "zotregistry.io/zot/pkg/storage/local" storageTypes "zotregistry.io/zot/pkg/storage/types" "zotregistry.io/zot/pkg/test" @@ -73,14 +74,14 @@ func TestMultipleStoragePath(t *testing.T) { // Create ImageStore - firstStore := local.NewImageStore(firstRootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, - nil, nil) + firstStore := local.NewImageStore(firstRootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) - secondStore := local.NewImageStore(secondRootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, - nil, nil) + secondStore := local.NewImageStore(secondRootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) - thirdStore := local.NewImageStore(thirdRootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, - nil, nil) + thirdStore := local.NewImageStore(thirdRootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) storeController := storage.StoreController{} @@ -188,7 +189,8 @@ func TestTrivyLibraryErrors(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) // Create ImageStore - store := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) + store := local.NewImageStore(rootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = store @@ -405,7 +407,8 @@ func TestImageScannable(t *testing.T) { // Continue with initializing the objects the scanner depends on metrics := monitoring.NewMetricsServer(false, log) - store := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) + store := local.NewImageStore(rootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = store @@ -471,7 +474,8 @@ func TestDefaultTrivyDBUrl(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) // Create ImageStore - store := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) + store := local.NewImageStore(rootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) storeController := storage.StoreController{} storeController.DefaultStore = store @@ -515,7 +519,7 @@ func TestDefaultTrivyDBUrl(t *testing.T) { func TestIsIndexScanable(t *testing.T) { Convey("IsIndexScanable", t, func() { storeController := storage.StoreController{} - storeController.DefaultStore = &local.ImageStoreLocal{} + storeController.DefaultStore = &imagestore.ImageStore{} metaDB := &boltdb.BoltDB{} log := log.NewLogger("debug", "") diff --git a/pkg/extensions/search/cve/trivy/scanner_test.go b/pkg/extensions/search/cve/trivy/scanner_test.go index 5e55537a..1ff033f8 100644 --- a/pkg/extensions/search/cve/trivy/scanner_test.go +++ b/pkg/extensions/search/cve/trivy/scanner_test.go @@ -182,7 +182,7 @@ func TestVulnerableLayer(t *testing.T) { tempDir := t.TempDir() log := log.NewLogger("debug", "") - imageStore := local.NewImageStore(tempDir, false, 0, false, false, + imageStore := local.NewImageStore(tempDir, false, false, 0, 0, false, false, log, monitoring.NewMetricsServer(false, log), nil, nil) storeController := storage.StoreController{ diff --git a/pkg/extensions/search/search_test.go b/pkg/extensions/search/search_test.go index 4af69feb..f1feec88 100644 --- a/pkg/extensions/search/search_test.go +++ b/pkg/extensions/search/search_test.go @@ -1193,7 +1193,7 @@ func TestExpandedRepoInfo(t *testing.T) { ctlr := api.NewController(conf) - imageStore := local.NewImageStore(tempDir, false, 0, false, false, + imageStore := local.NewImageStore(tempDir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) storeController := storage.StoreController{ @@ -1325,8 +1325,8 @@ func TestExpandedRepoInfo(t *testing.T) { log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - testStorage := local.NewImageStore(rootDir, false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil) + testStorage := local.NewImageStore(rootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) resp, err := resty.R().Get(baseURL + "/v2/") So(resp, ShouldNotBeNil) @@ -1671,7 +1671,7 @@ func TestExpandedRepoInfo(t *testing.T) { conf.Extensions.Search.CVE = nil ctlr := api.NewController(conf) - imageStore := local.NewImageStore(conf.Storage.RootDirectory, false, 0, false, false, + imageStore := local.NewImageStore(conf.Storage.RootDirectory, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) storeController := storage.StoreController{ @@ -5420,8 +5420,8 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { // get signatur digest log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storage := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil) + storage := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) indexBlob, err := storage.GetIndexContent(repo) So(err, ShouldBeNil) @@ -5497,8 +5497,8 @@ func TestMetaDBWhenDeletingImages(t *testing.T) { // get signatur digest log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storage := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil) + storage := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil) indexBlob, err := storage.GetIndexContent(repo) So(err, ShouldBeNil) diff --git a/pkg/extensions/search/userprefs_test.go b/pkg/extensions/search/userprefs_test.go index a7760076..4e7315f6 100644 --- a/pkg/extensions/search/userprefs_test.go +++ b/pkg/extensions/search/userprefs_test.go @@ -543,7 +543,7 @@ func TestChangingRepoState(t *testing.T) { } // ------ Create the test repos - defaultStore := local.NewImageStore(conf.Storage.RootDirectory, false, 0, false, false, + defaultStore := local.NewImageStore(conf.Storage.RootDirectory, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) err = WriteImageToFileSystem(img, accesibleRepo, "tag", storage.StoreController{ diff --git a/pkg/extensions/sync/constants/consts.go b/pkg/extensions/sync/constants/consts.go index 9dff8e95..69484d24 100644 --- a/pkg/extensions/sync/constants/consts.go +++ b/pkg/extensions/sync/constants/consts.go @@ -2,7 +2,8 @@ package constants // references type. const ( - Oras = "OrasReference" - Cosign = "CosignSignature" - OCI = "OCIReference" + Oras = "OrasReference" + Cosign = "CosignSignature" + OCI = "OCIReference" + SyncBlobUploadDir = ".sync" ) diff --git a/pkg/extensions/sync/local.go b/pkg/extensions/sync/local.go index 2a7108dc..ca79f608 100644 --- a/pkg/extensions/sync/local.go +++ b/pkg/extensions/sync/local.go @@ -281,8 +281,9 @@ func getImageStoreFromImageReference(imageReference types.ImageReference, repo, metrics := monitoring.NewMetricsServer(false, log.Logger{}) - tempImageStore := local.NewImageStore(tempRootDir, false, - storageConstants.DefaultGCDelay, false, false, log.Logger{}, metrics, nil, nil) + tempImageStore := local.NewImageStore(tempRootDir, false, false, + storageConstants.DefaultGCDelay, storageConstants.DefaultUntaggedImgeRetentionDelay, + false, false, log.Logger{}, metrics, nil, nil) return tempImageStore } diff --git a/pkg/extensions/sync/oci_layout.go b/pkg/extensions/sync/oci_layout.go index 1d137719..5806d96b 100644 --- a/pkg/extensions/sync/oci_layout.go +++ b/pkg/extensions/sync/oci_layout.go @@ -12,6 +12,7 @@ import ( "github.com/containers/image/v5/types" "github.com/gofrs/uuid" + "zotregistry.io/zot/pkg/extensions/sync/constants" "zotregistry.io/zot/pkg/storage" storageConstants "zotregistry.io/zot/pkg/storage/constants" "zotregistry.io/zot/pkg/test/inject" @@ -39,7 +40,7 @@ func (oci OciLayoutStorageImpl) GetContext() *types.SystemContext { func (oci OciLayoutStorageImpl) GetImageReference(repo string, reference string) (types.ImageReference, error) { localImageStore := oci.storeController.GetImageStore(repo) - tempSyncPath := path.Join(localImageStore.RootDir(), repo, SyncBlobUploadDir) + tempSyncPath := path.Join(localImageStore.RootDir(), repo, constants.SyncBlobUploadDir) // create session folder uuid, err := uuid.NewV4() diff --git a/pkg/extensions/sync/sync_internal_test.go b/pkg/extensions/sync/sync_internal_test.go index 80415a5b..43a77902 100644 --- a/pkg/extensions/sync/sync_internal_test.go +++ b/pkg/extensions/sync/sync_internal_test.go @@ -68,8 +68,8 @@ func TestInjectSyncUtils(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - imageStore := local.NewImageStore(t.TempDir(), false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil, + imageStore := local.NewImageStore(t.TempDir(), false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, nil, ) injected = inject.InjectFailure(0) @@ -182,8 +182,8 @@ func TestLocalRegistry(t *testing.T) { UseRelPaths: true, }, log) - syncImgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + syncImgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) repoName := "repo" registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, nil, log) @@ -300,8 +300,8 @@ func TestLocalRegistry(t *testing.T) { MandatoryAnnotations: []string{"annot1"}, }, log) - syncImgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, linter, cacheDriver) + syncImgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, linter, cacheDriver) repoName := "repo" registry := NewLocalRegistry(storage.StoreController{DefaultStore: syncImgStore}, nil, log) diff --git a/pkg/extensions/sync/sync_test.go b/pkg/extensions/sync/sync_test.go index 1a7c0191..644ab0ca 100644 --- a/pkg/extensions/sync/sync_test.go +++ b/pkg/extensions/sync/sync_test.go @@ -43,6 +43,7 @@ import ( extconf "zotregistry.io/zot/pkg/extensions/config" syncconf "zotregistry.io/zot/pkg/extensions/config/sync" "zotregistry.io/zot/pkg/extensions/sync" + syncConstants "zotregistry.io/zot/pkg/extensions/sync/constants" "zotregistry.io/zot/pkg/log" mTypes "zotregistry.io/zot/pkg/meta/types" storageConstants "zotregistry.io/zot/pkg/storage/constants" @@ -591,7 +592,7 @@ func TestOnDemand(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) - err = os.MkdirAll(path.Join(destDir, testImage, sync.SyncBlobUploadDir), 0o000) + err = os.MkdirAll(path.Join(destDir, testImage, syncConstants.SyncBlobUploadDir), 0o000) if err != nil { panic(err) } @@ -604,7 +605,7 @@ func TestOnDemand(t *testing.T) { So(err, ShouldBeNil) So(resp.StatusCode(), ShouldEqual, http.StatusNotFound) - err = os.Chmod(path.Join(destDir, testImage, sync.SyncBlobUploadDir), 0o755) + err = os.Chmod(path.Join(destDir, testImage, syncConstants.SyncBlobUploadDir), 0o755) if err != nil { panic(err) } @@ -1687,7 +1688,7 @@ func TestPermsDenied(t *testing.T) { defer dcm.StopServer() - syncSubDir := path.Join(destDir, testImage, sync.SyncBlobUploadDir) + syncSubDir := path.Join(destDir, testImage, syncConstants.SyncBlobUploadDir) err := os.MkdirAll(syncSubDir, 0o755) So(err, ShouldBeNil) @@ -1698,7 +1699,7 @@ func TestPermsDenied(t *testing.T) { dcm.StartAndWait(destPort) found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "couldn't get a local image reference", 20*time.Second) + "couldn't get a local image reference", 50*time.Second) if err != nil { panic(err) } @@ -4911,7 +4912,7 @@ func TestOnDemandPullsOnce(t *testing.T) { done := make(chan bool) var maxLen int - syncBlobUploadDir := path.Join(destDir, testImage, sync.SyncBlobUploadDir) + syncBlobUploadDir := path.Join(destDir, testImage, syncConstants.SyncBlobUploadDir) go func() { for { @@ -4994,7 +4995,7 @@ func TestError(t *testing.T) { }() found, err := test.ReadLogFileAndSearchString(dctlr.Config.Log.Output, - "finished syncing all repos", 15*time.Second) + "couldn't commit image to local image store", 30*time.Second) if err != nil { panic(err) } @@ -6531,7 +6532,7 @@ func pushRepo(url, repoName string) godigest.Digest { func waitSync(rootDir, repoName string) { // wait for .sync subdirs to be removed for { - dirs, err := os.ReadDir(path.Join(rootDir, repoName, sync.SyncBlobUploadDir)) + dirs, err := os.ReadDir(path.Join(rootDir, repoName, syncConstants.SyncBlobUploadDir)) if err == nil && len(dirs) == 0 { // stop watching /.sync/ subdirs return diff --git a/pkg/extensions/sync/utils.go b/pkg/extensions/sync/utils.go index 5eb21006..7513cb56 100644 --- a/pkg/extensions/sync/utils.go +++ b/pkg/extensions/sync/utils.go @@ -29,10 +29,6 @@ import ( "zotregistry.io/zot/pkg/test/inject" ) -const ( - SyncBlobUploadDir = ".sync" -) - // Get sync.FileCredentials from file. func getFileCredentials(filepath string) (syncconf.CredentialsFile, error) { credsFile, err := os.ReadFile(filepath) diff --git a/pkg/meta/hooks_test.go b/pkg/meta/hooks_test.go index 733966bd..f52a25e5 100644 --- a/pkg/meta/hooks_test.go +++ b/pkg/meta/hooks_test.go @@ -31,8 +31,8 @@ func TestOnUpdateManifest(t *testing.T) { storeController := storage.StoreController{} log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storeController.DefaultStore = local.NewImageStore(rootDir, true, 1*time.Second, - true, true, log, metrics, nil, nil, + storeController.DefaultStore = local.NewImageStore(rootDir, true, true, 1*time.Second, + 1*time.Second, true, true, log, metrics, nil, nil, ) params := boltdb.DBParameters{ @@ -72,8 +72,8 @@ func TestOnUpdateManifest(t *testing.T) { storeController := storage.StoreController{} log := log.NewLogger("debug", "") metrics := monitoring.NewMetricsServer(false, log) - storeController.DefaultStore = local.NewImageStore(rootDir, true, 1*time.Second, - true, true, log, metrics, nil, nil, + storeController.DefaultStore = local.NewImageStore(rootDir, true, true, 1*time.Second, + 1*time.Second, true, true, log, metrics, nil, nil, ) metaDB := mocks.MetaDBMock{ diff --git a/pkg/meta/parse_test.go b/pkg/meta/parse_test.go index 13f016fb..67a351b0 100644 --- a/pkg/meta/parse_test.go +++ b/pkg/meta/parse_test.go @@ -399,7 +399,7 @@ func TestParseStorageDynamoWrapper(t *testing.T) { func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { Convey("Test with simple case", func() { - imageStore := local.NewImageStore(rootDir, false, 0, false, false, + imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} @@ -485,7 +485,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { }) Convey("Accept orphan signatures", func() { - imageStore := local.NewImageStore(rootDir, false, 0, false, false, + imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} @@ -542,7 +542,7 @@ func RunParseStorageTests(rootDir string, metaDB mTypes.MetaDB) { }) Convey("Check statistics after load", func() { - imageStore := local.NewImageStore(rootDir, false, 0, false, false, + imageStore := local.NewImageStore(rootDir, false, false, 0, 0, false, false, log.NewLogger("debug", ""), monitoring.NewMetricsServer(false, log.NewLogger("debug", "")), nil, nil) storeController := storage.StoreController{DefaultStore: imageStore} diff --git a/pkg/storage/cache/boltdb.go b/pkg/storage/cache/boltdb.go index 89d06891..f146e0dd 100644 --- a/pkg/storage/cache/boltdb.go +++ b/pkg/storage/cache/boltdb.go @@ -77,6 +77,10 @@ func NewBoltDBCache(parameters interface{}, log zlog.Logger) Cache { } } +func (d *BoltDBDriver) UsesRelativePaths() bool { + return d.useRelPaths +} + func (d *BoltDBDriver) Name() string { return "boltdb" } diff --git a/pkg/storage/cache/cacheinterface.go b/pkg/storage/cache/cacheinterface.go index d02fe95c..de1df6b2 100644 --- a/pkg/storage/cache/cacheinterface.go +++ b/pkg/storage/cache/cacheinterface.go @@ -19,4 +19,7 @@ type Cache interface { // Delete a blob from the cachedb. DeleteBlob(digest godigest.Digest, path string) error + + // UsesRelativePaths returns if cache is storing blobs relative to cache rootDir + UsesRelativePaths() bool } diff --git a/pkg/storage/cache/dynamodb.go b/pkg/storage/cache/dynamodb.go index 890d5177..4aafed09 100644 --- a/pkg/storage/cache/dynamodb.go +++ b/pkg/storage/cache/dynamodb.go @@ -99,6 +99,10 @@ func NewDynamoDBCache(parameters interface{}, log zlog.Logger) Cache { return driver } +func (d *DynamoDBDriver) UsesRelativePaths() bool { + return false +} + func (d *DynamoDBDriver) Name() string { return "dynamodb" } diff --git a/pkg/storage/cache_test.go b/pkg/storage/cache_test.go index 01b31b77..6158f25e 100644 --- a/pkg/storage/cache_test.go +++ b/pkg/storage/cache_test.go @@ -35,6 +35,8 @@ func TestCache(t *testing.T) { }, log) So(cacheDriver, ShouldNotBeNil) + So(cacheDriver.UsesRelativePaths(), ShouldBeTrue) + name := cacheDriver.Name() So(name, ShouldEqual, "boltdb") diff --git a/pkg/storage/common/common.go b/pkg/storage/common/common.go index cdb44ece..af1e9975 100644 --- a/pkg/storage/common/common.go +++ b/pkg/storage/common/common.go @@ -5,22 +5,22 @@ import ( "encoding/json" "errors" "fmt" - "io" "math/rand" "path" "strings" "time" + "github.com/docker/distribution/registry/storage/driver" notreg "github.com/notaryproject/notation-go/registry" godigest "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/schema" imeta "github.com/opencontainers/image-spec/specs-go" ispec "github.com/opencontainers/image-spec/specs-go/v1" oras "github.com/oras-project/artifacts-spec/specs-go/v1" - "github.com/rs/zerolog" zerr "zotregistry.io/zot/errors" zcommon "zotregistry.io/zot/pkg/common" + zlog "zotregistry.io/zot/pkg/log" "zotregistry.io/zot/pkg/scheduler" storageConstants "zotregistry.io/zot/pkg/storage/constants" storageTypes "zotregistry.io/zot/pkg/storage/types" @@ -62,7 +62,7 @@ func GetManifestDescByReference(index ispec.Index, reference string) (ispec.Desc } func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaType string, body []byte, - log zerolog.Logger, + log zlog.Logger, ) (godigest.Digest, error) { // validate the manifest if !IsSupportedMediaType(mediaType) { @@ -105,7 +105,7 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy continue } - ok, _, err := imgStore.StatBlob(repo, layer.Digest) + ok, _, _, err := imgStore.StatBlob(repo, layer.Digest) if !ok || err != nil { log.Error().Err(err).Str("digest", layer.Digest.String()).Msg("missing layer blob") @@ -136,7 +136,7 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy } for _, manifest := range indexManifest.Manifests { - if ok, _, err := imgStore.StatBlob(repo, manifest.Digest); !ok || err != nil { + if ok, _, _, err := imgStore.StatBlob(repo, manifest.Digest); !ok || err != nil { log.Error().Err(err).Str("digest", manifest.Digest.String()).Msg("missing manifest blob") return "", zerr.ErrBadManifest @@ -147,7 +147,7 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy return "", nil } -func GetAndValidateRequestDigest(body []byte, digestStr string, log zerolog.Logger) (godigest.Digest, error) { +func GetAndValidateRequestDigest(body []byte, digestStr string, log zlog.Logger) (godigest.Digest, error) { bodyDigest := godigest.FromBytes(body) d, err := godigest.Parse(digestStr) @@ -169,7 +169,7 @@ CheckIfIndexNeedsUpdate verifies if an index needs to be updated given a new man Returns whether or not index needs update, in the latter case it will also return the previous digest. */ func CheckIfIndexNeedsUpdate(index *ispec.Index, desc *ispec.Descriptor, - log zerolog.Logger, + log zlog.Logger, ) (bool, godigest.Digest, error) { var oldDgst godigest.Digest @@ -242,11 +242,15 @@ func CheckIfIndexNeedsUpdate(index *ispec.Index, desc *ispec.Descriptor, } // GetIndex returns the contents of index.json. -func GetIndex(imgStore storageTypes.ImageStore, repo string, log zerolog.Logger) (ispec.Index, error) { +func GetIndex(imgStore storageTypes.ImageStore, repo string, log zlog.Logger) (ispec.Index, error) { var index ispec.Index buf, err := imgStore.GetIndexContent(repo) if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return index, zerr.ErrRepoNotFound + } + return index, err } @@ -260,7 +264,7 @@ func GetIndex(imgStore storageTypes.ImageStore, repo string, log zerolog.Logger) } // GetImageIndex returns a multiarch type image. -func GetImageIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger, +func GetImageIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zlog.Logger, ) (ispec.Index, error) { var imageIndex ispec.Index @@ -285,7 +289,7 @@ func GetImageIndex(imgStore storageTypes.ImageStore, repo string, digest godiges return imageIndex, nil } -func GetImageManifest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger, +func GetImageManifest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zlog.Logger, ) (ispec.Manifest, error) { var manifestContent ispec.Manifest @@ -352,7 +356,7 @@ index, ensure that they do not have a name or they are not in other manifest indexes else GC can never clean them. */ func UpdateIndexWithPrunedImageManifests(imgStore storageTypes.ImageStore, index *ispec.Index, repo string, - desc ispec.Descriptor, oldDgst godigest.Digest, log zerolog.Logger, + desc ispec.Descriptor, oldDgst godigest.Digest, log zlog.Logger, ) error { if (desc.MediaType == ispec.MediaTypeImageIndex) && (oldDgst != "") { otherImgIndexes := []ispec.Descriptor{} @@ -385,7 +389,7 @@ same constitutent manifests so that they can be garbage-collected correctly PruneImageManifestsFromIndex is a helper routine to achieve this. */ func PruneImageManifestsFromIndex(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, //nolint:gocyclo,lll - outIndex ispec.Index, otherImgIndexes []ispec.Descriptor, log zerolog.Logger, + outIndex ispec.Index, otherImgIndexes []ispec.Descriptor, log zlog.Logger, ) ([]ispec.Descriptor, error) { dir := path.Join(imgStore.RootDir(), repo) @@ -459,8 +463,8 @@ func PruneImageManifestsFromIndex(imgStore storageTypes.ImageStore, repo string, return prunedManifests, nil } -func isBlobReferencedInManifest(imgStore storageTypes.ImageStore, repo string, - bdigest, mdigest godigest.Digest, log zerolog.Logger, +func isBlobReferencedInImageManifest(imgStore storageTypes.ImageStore, repo string, + bdigest, mdigest godigest.Digest, log zlog.Logger, ) (bool, error) { if bdigest == mdigest { return true, nil @@ -487,16 +491,14 @@ func isBlobReferencedInManifest(imgStore storageTypes.ImageStore, repo string, return false, nil } -func isBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string, - digest godigest.Digest, index ispec.Index, log zerolog.Logger, +func IsBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string, + digest godigest.Digest, index ispec.Index, log zlog.Logger, ) (bool, error) { for _, desc := range index.Manifests { var found bool switch desc.MediaType { case ispec.MediaTypeImageIndex: - /* this branch is not needed, because every manifests in index is already checked - when this one is hit, all manifests are referenced in index.json */ indexImage, err := GetImageIndex(imgStore, repo, desc.Digest, log) if err != nil { log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). @@ -505,9 +507,9 @@ func isBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string, return false, err } - found, _ = isBlobReferencedInImageIndex(imgStore, repo, digest, indexImage, log) + found, _ = IsBlobReferencedInImageIndex(imgStore, repo, digest, indexImage, log) case ispec.MediaTypeImageManifest: - found, _ = isBlobReferencedInManifest(imgStore, repo, digest, desc.Digest, log) + found, _ = isBlobReferencedInImageManifest(imgStore, repo, digest, desc.Digest, log) } if found { @@ -519,7 +521,7 @@ func isBlobReferencedInImageIndex(imgStore storageTypes.ImageStore, repo string, } func IsBlobReferenced(imgStore storageTypes.ImageStore, repo string, - digest godigest.Digest, log zerolog.Logger, + digest godigest.Digest, log zlog.Logger, ) (bool, error) { dir := path.Join(imgStore.RootDir(), repo) if !imgStore.DirExists(dir) { @@ -531,7 +533,133 @@ func IsBlobReferenced(imgStore storageTypes.ImageStore, repo string, return false, err } - return isBlobReferencedInImageIndex(imgStore, repo, digest, index, log) + return IsBlobReferencedInImageIndex(imgStore, repo, digest, index, log) +} + +/* Garbage Collection */ + +func AddImageManifestBlobsToReferences(imgStore storageTypes.ImageStore, + repo string, mdigest godigest.Digest, refBlobs map[string]bool, log zlog.Logger, +) error { + manifestContent, err := GetImageManifest(imgStore, repo, mdigest, log) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", mdigest.String()). + Msg("gc: failed to read manifest image") + + return err + } + + refBlobs[mdigest.String()] = true + refBlobs[manifestContent.Config.Digest.String()] = true + + // if there is a Subject, it may not exist yet and that is ok + if manifestContent.Subject != nil { + refBlobs[manifestContent.Subject.Digest.String()] = true + } + + for _, layer := range manifestContent.Layers { + refBlobs[layer.Digest.String()] = true + } + + return nil +} + +func AddORASImageManifestBlobsToReferences(imgStore storageTypes.ImageStore, + repo string, mdigest godigest.Digest, refBlobs map[string]bool, log zlog.Logger, +) error { + manifestContent, err := GetOrasManifestByDigest(imgStore, repo, mdigest, log) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", mdigest.String()). + Msg("gc: failed to read manifest image") + + return err + } + + refBlobs[mdigest.String()] = true + + // if there is a Subject, it may not exist yet and that is ok + if manifestContent.Subject != nil { + refBlobs[manifestContent.Subject.Digest.String()] = true + } + + for _, blob := range manifestContent.Blobs { + refBlobs[blob.Digest.String()] = true + } + + return nil +} + +func AddImageIndexBlobsToReferences(imgStore storageTypes.ImageStore, + repo string, mdigest godigest.Digest, refBlobs map[string]bool, log zlog.Logger, +) error { + index, err := GetImageIndex(imgStore, repo, mdigest, log) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", mdigest.String()). + Msg("gc: failed to read manifest image") + + return err + } + + refBlobs[mdigest.String()] = true + + // if there is a Subject, it may not exist yet and that is ok + if index.Subject != nil { + refBlobs[index.Subject.Digest.String()] = true + } + + for _, manifest := range index.Manifests { + refBlobs[manifest.Digest.String()] = true + } + + return nil +} + +func AddIndexBlobToReferences(imgStore storageTypes.ImageStore, + repo string, index ispec.Index, refBlobs map[string]bool, log zlog.Logger, +) error { + for _, desc := range index.Manifests { + switch desc.MediaType { + case ispec.MediaTypeImageIndex: + if err := AddImageIndexBlobsToReferences(imgStore, repo, desc.Digest, refBlobs, log); err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("failed to read blobs in multiarch(index) image") + + return err + } + case ispec.MediaTypeImageManifest: + if err := AddImageManifestBlobsToReferences(imgStore, repo, desc.Digest, refBlobs, log); err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("failed to read blobs in image manifest") + + return err + } + case oras.MediaTypeArtifactManifest: + if err := AddORASImageManifestBlobsToReferences(imgStore, repo, desc.Digest, refBlobs, log); err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("failed to read blobs in image manifest") + + return err + } + } + } + + return nil +} + +func AddRepoBlobsToReferences(imgStore storageTypes.ImageStore, + repo string, refBlobs map[string]bool, log zlog.Logger, +) error { + dir := path.Join(imgStore.RootDir(), repo) + if !imgStore.DirExists(dir) { + return zerr.ErrRepoNotFound + } + + index, err := GetIndex(imgStore, repo, log) + if err != nil { + return err + } + + return AddIndexBlobToReferences(imgStore, repo, index, refBlobs, log) } func ApplyLinter(imgStore storageTypes.ImageStore, linter Lint, repo string, descriptor ispec.Descriptor, @@ -580,7 +708,7 @@ func IsSignature(descriptor ispec.Descriptor) bool { } func GetOrasReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godigest.Digest, artifactType string, - log zerolog.Logger, + log zlog.Logger, ) ([]oras.Descriptor, error) { if err := gdigest.Validate(); err != nil { return nil, err @@ -638,7 +766,7 @@ func GetOrasReferrers(imgStore storageTypes.ImageStore, repo string, gdigest god } func GetReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godigest.Digest, artifactTypes []string, - log zerolog.Logger, + log zlog.Logger, ) (ispec.Index, error) { nilIndex := ispec.Index{} @@ -741,7 +869,7 @@ func GetReferrers(imgStore storageTypes.ImageStore, repo string, gdigest godiges return index, nil } -func GetOrasManifestByDigest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zerolog.Logger, +func GetOrasManifestByDigest(imgStore storageTypes.ImageStore, repo string, digest godigest.Digest, log zlog.Logger, ) (oras.Manifest, error) { var artManifest oras.Manifest @@ -827,7 +955,7 @@ type DedupeTaskGenerator struct { and generating a task for each unprocessed one*/ lastDigests []godigest.Digest done bool - Log zerolog.Logger + Log zlog.Logger } func (gen *DedupeTaskGenerator) Next() (scheduler.Task, error) { @@ -879,11 +1007,11 @@ type dedupeTask struct { // blobs paths with the same digest ^ duplicateBlobs []string dedupe bool - log zerolog.Logger + log zlog.Logger } func newDedupeTask(imgStore storageTypes.ImageStore, digest godigest.Digest, dedupe bool, - duplicateBlobs []string, log zerolog.Logger, + duplicateBlobs []string, log zlog.Logger, ) *dedupeTask { return &dedupeTask{imgStore, digest, duplicateBlobs, dedupe, log} } @@ -929,8 +1057,7 @@ func (gen *GCTaskGenerator) Next() (scheduler.Task, error) { gen.nextRun = time.Now().Add(time.Duration(delay) * time.Second) repo, err := gen.ImgStore.GetNextRepository(gen.lastRepo) - - if err != nil && !errors.Is(err, io.EOF) { + if err != nil { return nil, err } diff --git a/pkg/storage/common/common_test.go b/pkg/storage/common/common_test.go index 315d9640..782bde5f 100644 --- a/pkg/storage/common/common_test.go +++ b/pkg/storage/common/common_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "os" + "path" "testing" godigest "github.com/opencontainers/go-digest" @@ -36,8 +37,8 @@ func TestValidateManifest(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, - true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) content := []byte("this is a blob") digest := godigest.FromBytes(content) @@ -81,6 +82,37 @@ func TestValidateManifest(t *testing.T) { So(internalErr.GetDetails()["jsonSchemaValidation"], ShouldEqual, "[schemaVersion: Must be less than or equal to 2]") }) + Convey("bad config blob", func() { + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: digest, + Size: int64(len(content)), + }, + }, + } + + manifest.SchemaVersion = 2 + + configBlobPath := imgStore.BlobPath("test", cdigest) + + err := os.WriteFile(configBlobPath, []byte("bad config blob"), 0o000) + So(err, ShouldBeNil) + + body, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + // this was actually an umoci error on config blob + _, _, err = imgStore.PutImageManifest("test", "1.0", ispec.MediaTypeImageManifest, body) + So(err, ShouldBeNil) + }) + Convey("manifest with non-distributable layers", func() { content := []byte("this blob doesn't exist") digest := godigest.FromBytes(content) @@ -124,29 +156,29 @@ func TestGetReferrersErrors(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, false, - true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, true, log, metrics, nil, cacheDriver) artifactType := "application/vnd.example.icecream.v1" validDigest := godigest.FromBytes([]byte("blob")) Convey("Trigger invalid digest error", func(c C) { _, err := common.GetReferrers(imgStore, "zot-test", "invalidDigest", - []string{artifactType}, log.With().Caller().Logger()) + []string{artifactType}, log) So(err, ShouldNotBeNil) _, err = common.GetOrasReferrers(imgStore, "zot-test", "invalidDigest", - artifactType, log.With().Caller().Logger()) + artifactType, log) So(err, ShouldNotBeNil) }) Convey("Trigger repo not found error", func(c C) { _, err := common.GetReferrers(imgStore, "zot-test", validDigest, - []string{artifactType}, log.With().Caller().Logger()) + []string{artifactType}, log) So(err, ShouldNotBeNil) _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, - artifactType, log.With().Caller().Logger()) + artifactType, log) So(err, ShouldNotBeNil) }) @@ -179,11 +211,11 @@ func TestGetReferrersErrors(t *testing.T) { } _, err = common.GetReferrers(imgStore, "zot-test", validDigest, - []string{artifactType}, log.With().Caller().Logger()) + []string{artifactType}, log) So(err, ShouldNotBeNil) _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, - artifactType, log.With().Caller().Logger()) + artifactType, log) So(err, ShouldNotBeNil) }) @@ -198,11 +230,11 @@ func TestGetReferrersErrors(t *testing.T) { } _, err = common.GetReferrers(imgStore, "zot-test", validDigest, - []string{artifactType}, log.With().Caller().Logger()) + []string{artifactType}, log) So(err, ShouldNotBeNil) _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, - artifactType, log.With().Caller().Logger()) + artifactType, log) So(err, ShouldNotBeNil) }) @@ -227,11 +259,11 @@ func TestGetReferrersErrors(t *testing.T) { } _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, - artifactType, log.With().Caller().Logger()) + artifactType, log) So(err, ShouldNotBeNil) _, err = common.GetOrasReferrers(imgStore, "zot-test", digest, - artifactType, log.With().Caller().Logger()) + artifactType, log) So(err, ShouldNotBeNil) }) @@ -245,7 +277,7 @@ func TestGetReferrersErrors(t *testing.T) { }, } - _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log.With().Caller().Logger()) + _, err = common.GetOrasReferrers(imgStore, "zot-test", validDigest, artifactType, log) So(err, ShouldNotBeNil) }) @@ -272,7 +304,7 @@ func TestGetReferrersErrors(t *testing.T) { } _, err = common.GetReferrers(imgStore, "zot-test", validDigest, - []string{artifactType}, log.With().Caller().Logger()) + []string{artifactType}, log) So(err, ShouldNotBeNil) }) @@ -306,7 +338,7 @@ func TestGetReferrersErrors(t *testing.T) { } _, err = common.GetReferrers(imgStore, "zot-test", validDigest, - []string{artifactType}, log.With().Caller().Logger()) + []string{artifactType}, log) So(err, ShouldBeNil) }) @@ -326,7 +358,7 @@ func TestGetReferrersErrors(t *testing.T) { } _, err = common.GetReferrers(imgStore, "zot-test", validDigest, - []string{}, log.With().Caller().Logger()) + []string{}, log) So(err, ShouldNotBeNil) }) @@ -348,7 +380,7 @@ func TestGetReferrersErrors(t *testing.T) { } ref, err := common.GetReferrers(imgStore, "zot-test", validDigest, - []string{"art.type"}, log.With().Caller().Logger()) + []string{"art.type"}, log) So(err, ShouldBeNil) So(len(ref.Manifests), ShouldEqual, 0) }) @@ -356,7 +388,7 @@ func TestGetReferrersErrors(t *testing.T) { } func TestGetImageIndexErrors(t *testing.T) { - log := zerolog.New(os.Stdout) + log := log.Logger{Logger: zerolog.New(os.Stdout)} Convey("Trigger invalid digest error", t, func(c C) { imgStore := &mocks.MockedImageStore{} @@ -400,3 +432,193 @@ func TestIsSignature(t *testing.T) { So(isSingature, ShouldBeFalse) }) } + +func TestGarbageCollectManifestErrors(t *testing.T) { + Convey("Make imagestore and upload manifest", t, func(c C) { + dir := t.TempDir() + + repoName := "test" + + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) + + Convey("trigger repo not found in GetReferencedBlobs()", func() { + err := common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log) + So(err, ShouldNotBeNil) + }) + + content := []byte("this is a blob") + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + + _, blen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), digest) + So(err, ShouldBeNil) + So(blen, ShouldEqual, len(content)) + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: digest, + Size: int64(len(content)), + }, + }, + } + + manifest.SchemaVersion = 2 + + body, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + manifestDigest := godigest.FromBytes(body) + + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageManifest, body) + So(err, ShouldBeNil) + + Convey("trigger GetIndex error in GetReferencedBlobs", func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName), 0o000) + So(err, ShouldBeNil) + + defer func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName), 0o755) + So(err, ShouldBeNil) + }() + + err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log) + So(err, ShouldNotBeNil) + }) + + Convey("trigger GetImageManifest error in GetReferencedBlobsInImageManifest", func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", manifestDigest.Encoded()), 0o000) + So(err, ShouldBeNil) + + defer func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", manifestDigest.Encoded()), 0o755) + So(err, ShouldBeNil) + }() + + err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log) + So(err, ShouldNotBeNil) + }) + }) +} + +func TestGarbageCollectIndexErrors(t *testing.T) { + Convey("Make imagestore and upload manifest", t, func(c C) { + dir := t.TempDir() + + repoName := "test" + + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) + + content := []byte("this is a blob") + bdgst := godigest.FromBytes(content) + So(bdgst, ShouldNotBeNil) + + _, bsize, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), bdgst) + So(err, ShouldBeNil) + So(bsize, ShouldEqual, len(content)) + + var index ispec.Index + index.SchemaVersion = 2 + index.MediaType = ispec.MediaTypeImageIndex + + var digest godigest.Digest + for i := 0; i < 4; i++ { + // upload image config blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest := test.GetRandomImageConfig() + buf := bytes.NewBuffer(cblob) + buflen := buf.Len() + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst, + Size: bsize, + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: digest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(content)), + }) + } + + // upload index image + indexContent, err := json.Marshal(index) + So(err, ShouldBeNil) + indexDigest := godigest.FromBytes(indexContent) + So(indexDigest, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + So(err, ShouldBeNil) + + err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log) + So(err, ShouldBeNil) + + Convey("trigger GetImageIndex error in GetReferencedBlobsInImageIndex", func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", indexDigest.Encoded()), 0o000) + So(err, ShouldBeNil) + + defer func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", indexDigest.Encoded()), 0o755) + So(err, ShouldBeNil) + }() + + err = common.AddRepoBlobsToReferences(imgStore, repoName, map[string]bool{}, log) + So(err, ShouldNotBeNil) + }) + }) +} diff --git a/pkg/storage/constants/constants.go b/pkg/storage/constants/constants.go index 55ea6b03..7b9b49f8 100644 --- a/pkg/storage/constants/constants.go +++ b/pkg/storage/constants/constants.go @@ -6,20 +6,22 @@ import ( const ( // BlobUploadDir defines the upload directory for blob uploads. - BlobUploadDir = ".uploads" - SchemaVersion = 2 - DefaultFilePerms = 0o600 - DefaultDirPerms = 0o700 - RLOCK = "RLock" - RWLOCK = "RWLock" - BlobsCache = "blobs" - DuplicatesBucket = "duplicates" - OriginalBucket = "original" - DBExtensionName = ".db" - DBCacheLockCheckTimeout = 10 * time.Second - BoltdbName = "cache" - DynamoDBDriverName = "dynamodb" - DefaultGCDelay = 1 * time.Hour - DefaultGCInterval = 1 * time.Hour - S3StorageDriverName = "s3" + BlobUploadDir = ".uploads" + SchemaVersion = 2 + DefaultFilePerms = 0o600 + DefaultDirPerms = 0o700 + RLOCK = "RLock" + RWLOCK = "RWLock" + BlobsCache = "blobs" + DuplicatesBucket = "duplicates" + OriginalBucket = "original" + DBExtensionName = ".db" + DBCacheLockCheckTimeout = 10 * time.Second + BoltdbName = "cache" + DynamoDBDriverName = "dynamodb" + DefaultGCDelay = 1 * time.Hour + DefaultUntaggedImgeRetentionDelay = 24 * time.Hour + DefaultGCInterval = 1 * time.Hour + S3StorageDriverName = "s3" + LocalStorageDriverName = "local" ) diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go new file mode 100644 index 00000000..3b611d2d --- /dev/null +++ b/pkg/storage/imagestore/imagestore.go @@ -0,0 +1,2252 @@ +package imagestore + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "path/filepath" + "strings" + "sync" + "time" + "unicode/utf8" + + "github.com/docker/distribution/registry/storage/driver" + guuid "github.com/gofrs/uuid" + godigest "github.com/opencontainers/go-digest" + ispec "github.com/opencontainers/image-spec/specs-go/v1" + artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" + + zerr "zotregistry.io/zot/errors" + zcommon "zotregistry.io/zot/pkg/common" + "zotregistry.io/zot/pkg/extensions/monitoring" + syncConstants "zotregistry.io/zot/pkg/extensions/sync/constants" + zlog "zotregistry.io/zot/pkg/log" + zreg "zotregistry.io/zot/pkg/regexp" + "zotregistry.io/zot/pkg/scheduler" + "zotregistry.io/zot/pkg/storage/cache" + common "zotregistry.io/zot/pkg/storage/common" + storageConstants "zotregistry.io/zot/pkg/storage/constants" + storageTypes "zotregistry.io/zot/pkg/storage/types" + "zotregistry.io/zot/pkg/test/inject" +) + +const ( + cosignSignatureTagSuffix = "sig" + SBOMTagSuffix = "sbom" +) + +// ImageStore provides the image storage operations. +type ImageStore struct { + rootDir string + storeDriver storageTypes.Driver + lock *sync.RWMutex + log zlog.Logger + metrics monitoring.MetricServer + cache cache.Cache + dedupe bool + linter common.Lint + commit bool + gc bool + gcReferrers bool + gcDelay time.Duration + retentionDelay time.Duration +} + +func (is *ImageStore) RootDir() string { + return is.rootDir +} + +func (is *ImageStore) DirExists(d string) bool { + return is.storeDriver.DirExists(d) +} + +// NewImageStore returns a new image store backed by cloud storages. +// see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers +// Use the last argument to properly set a cache database, or it will default to boltDB local storage. +func NewImageStore(rootDir string, cacheDir string, gc bool, gcReferrers bool, gcDelay time.Duration, + untaggedImageRetentionDelay time.Duration, dedupe, commit bool, log zlog.Logger, metrics monitoring.MetricServer, + linter common.Lint, storeDriver storageTypes.Driver, cacheDriver cache.Cache, +) storageTypes.ImageStore { + if err := storeDriver.EnsureDir(rootDir); err != nil { + log.Error().Err(err).Str("rootDir", rootDir).Msg("unable to create root dir") + + return nil + } + + imgStore := &ImageStore{ + rootDir: rootDir, + storeDriver: storeDriver, + lock: &sync.RWMutex{}, + log: log, + metrics: metrics, + dedupe: dedupe, + linter: linter, + commit: commit, + gc: gc, + gcReferrers: gcReferrers, + gcDelay: gcDelay, + retentionDelay: untaggedImageRetentionDelay, + cache: cacheDriver, + } + + return imgStore +} + +// RLock read-lock. +func (is *ImageStore) RLock(lockStart *time.Time) { + *lockStart = time.Now() + + is.lock.RLock() +} + +// RUnlock read-unlock. +func (is *ImageStore) RUnlock(lockStart *time.Time) { + is.lock.RUnlock() + + lockEnd := time.Now() + // includes time spent in acquiring and holding a lock + latency := lockEnd.Sub(*lockStart) + monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RLOCK) // histogram +} + +// Lock write-lock. +func (is *ImageStore) Lock(lockStart *time.Time) { + *lockStart = time.Now() + + is.lock.Lock() +} + +// Unlock write-unlock. +func (is *ImageStore) Unlock(lockStart *time.Time) { + is.lock.Unlock() + + lockEnd := time.Now() + // includes time spent in acquiring and holding a lock + latency := lockEnd.Sub(*lockStart) + monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RWLOCK) // histogram +} + +func (is *ImageStore) initRepo(name string) error { + repoDir := path.Join(is.rootDir, name) + + if !utf8.ValidString(name) { + is.log.Error().Msg("input is not valid UTF-8") + + return zerr.ErrInvalidRepositoryName + } + + if !zreg.FullNameRegexp.MatchString(name) { + is.log.Error().Str("repository", name).Msg("invalid repository name") + + return zerr.ErrInvalidRepositoryName + } + + // create "blobs" subdir + err := is.storeDriver.EnsureDir(path.Join(repoDir, "blobs")) + if err != nil { + is.log.Error().Err(err).Msg("error creating blobs subdir") + + return err + } + // create BlobUploadDir subdir + err = is.storeDriver.EnsureDir(path.Join(repoDir, storageConstants.BlobUploadDir)) + if err != nil { + is.log.Error().Err(err).Msg("error creating blob upload subdir") + + return err + } + + // "oci-layout" file - create if it doesn't exist + ilPath := path.Join(repoDir, ispec.ImageLayoutFile) + if _, err := is.storeDriver.Stat(ilPath); err != nil { + il := ispec.ImageLayout{Version: ispec.ImageLayoutVersion} + + buf, err := json.Marshal(il) + if err != nil { + is.log.Error().Err(err).Msg("unable to marshal JSON") + + return err + } + + if _, err := is.storeDriver.WriteFile(ilPath, buf); err != nil { + is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") + + return err + } + } + + // "index.json" file - create if it doesn't exist + indexPath := path.Join(repoDir, "index.json") + if _, err := is.storeDriver.Stat(indexPath); err != nil { + index := ispec.Index{} + index.SchemaVersion = 2 + + buf, err := json.Marshal(index) + if err != nil { + is.log.Error().Err(err).Msg("unable to marshal JSON") + + return err + } + + if _, err := is.storeDriver.WriteFile(indexPath, buf); err != nil { + is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") + + return err + } + } + + return nil +} + +// InitRepo creates an image repository under this store. +func (is *ImageStore) InitRepo(name string) error { + var lockLatency time.Time + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + + return is.initRepo(name) +} + +// ValidateRepo validates that the repository layout is complaint with the OCI repo layout. +func (is *ImageStore) ValidateRepo(name string) (bool, error) { + if !zreg.FullNameRegexp.MatchString(name) { + return false, zerr.ErrInvalidRepositoryName + } + + // https://github.com/opencontainers/image-spec/blob/master/image-layout.md#content + // at least, expect at least 3 entries - ["blobs", "oci-layout", "index.json"] + // and an additional/optional BlobUploadDir in each image store + // for s3 we can not create empty dirs, so we check only against index.json and oci-layout + dir := path.Join(is.rootDir, name) + if fi, err := is.storeDriver.Stat(dir); err != nil || !fi.IsDir() { + return false, zerr.ErrRepoNotFound + } + + files, err := is.storeDriver.List(dir) + if err != nil { + is.log.Error().Err(err).Str("dir", dir).Msg("unable to read directory") + + return false, zerr.ErrRepoNotFound + } + + //nolint:gomnd + if len(files) < 2 { + return false, zerr.ErrRepoBadVersion + } + + found := map[string]bool{ + ispec.ImageLayoutFile: false, + "index.json": false, + } + + for _, file := range files { + fileInfo, err := is.storeDriver.Stat(file) + if err != nil { + return false, err + } + + filename, err := filepath.Rel(dir, file) + if err != nil { + return false, err + } + + if filename == "blobs" && !fileInfo.IsDir() { + return false, nil + } + + found[filename] = true + } + + // check blobs dir exists only for filesystem, in s3 we can't have empty dirs + if is.storeDriver.Name() == storageConstants.LocalStorageDriverName { + if !is.storeDriver.DirExists(path.Join(dir, "blobs")) { + return false, nil + } + } + + for k, v := range found { + if !v && k != storageConstants.BlobUploadDir { + return false, nil + } + } + + buf, err := is.storeDriver.ReadFile(path.Join(dir, ispec.ImageLayoutFile)) + if err != nil { + return false, err + } + + var il ispec.ImageLayout + if err := json.Unmarshal(buf, &il); err != nil { + return false, err + } + + if il.Version != ispec.ImageLayoutVersion { + return false, zerr.ErrRepoBadVersion + } + + return true, nil +} + +// GetRepositories returns a list of all the repositories under this store. +func (is *ImageStore) GetRepositories() ([]string, error) { + var lockLatency time.Time + + dir := is.rootDir + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + stores := make([]string, 0) + + err := is.storeDriver.Walk(dir, func(fileInfo driver.FileInfo) error { + if !fileInfo.IsDir() { + return nil + } + + rel, err := filepath.Rel(is.rootDir, fileInfo.Path()) + if err != nil { + return nil //nolint:nilerr // ignore paths that are not under root dir + } + + if ok, err := is.ValidateRepo(rel); !ok || err != nil { + return nil //nolint:nilerr // ignore invalid repos + } + + stores = append(stores, rel) + + return nil + }) + + // if the root directory is not yet created then return an empty slice of repositories + var perr driver.PathNotFoundError + if errors.As(err, &perr) { + return stores, nil + } + + return stores, err +} + +// GetNextRepository returns next repository under this store. +func (is *ImageStore) GetNextRepository(repo string) (string, error) { + var lockLatency time.Time + + dir := is.rootDir + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + _, err := is.storeDriver.List(dir) + if err != nil { + is.log.Error().Err(err).Msg("failure walking storage root-dir") + + return "", err + } + + found := false + store := "" + err = is.storeDriver.Walk(dir, func(fileInfo driver.FileInfo) error { + if !fileInfo.IsDir() { + return nil + } + + rel, err := filepath.Rel(is.rootDir, fileInfo.Path()) + if err != nil { + return nil //nolint:nilerr // ignore paths not relative to root dir + } + + ok, err := is.ValidateRepo(rel) + if !ok || err != nil { + return nil //nolint:nilerr // ignore invalid repos + } + + if repo == "" && ok && err == nil { + store = rel + + return io.EOF + } + + if found { + store = rel + + return io.EOF + } + + if rel == repo { + found = true + } + + return nil + }) + + driverErr := &driver.Error{} + + if errors.Is(err, io.EOF) || + (errors.As(err, driverErr) && errors.Is(driverErr.Enclosed, io.EOF)) { + return store, nil + } + + return store, err +} + +// GetImageTags returns a list of image tags available in the specified repository. +func (is *ImageStore) GetImageTags(repo string) ([]string, error) { + var lockLatency time.Time + + dir := path.Join(is.rootDir, repo) + if fi, err := is.storeDriver.Stat(dir); err != nil || !fi.IsDir() { + return nil, zerr.ErrRepoNotFound + } + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return nil, err + } + + return common.GetTagsByIndex(index), nil +} + +// GetImageManifest returns the image manifest of an image in the specific repository. +func (is *ImageStore) GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) { + dir := path.Join(is.rootDir, repo) + if fi, err := is.storeDriver.Stat(dir); err != nil || !fi.IsDir() { + return nil, "", "", zerr.ErrRepoNotFound + } + + var lockLatency time.Time + + var err error + + is.RLock(&lockLatency) + defer func() { + is.RUnlock(&lockLatency) + + if err == nil { + monitoring.IncDownloadCounter(is.metrics, repo) + } + }() + + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return nil, "", "", err + } + + manifestDesc, found := common.GetManifestDescByReference(index, reference) + if !found { + return nil, "", "", zerr.ErrManifestNotFound + } + + buf, err := is.GetBlobContent(repo, manifestDesc.Digest) + if err != nil { + if errors.Is(err, zerr.ErrBlobNotFound) { + return nil, "", "", zerr.ErrManifestNotFound + } + + return nil, "", "", err + } + + var manifest ispec.Manifest + if err := json.Unmarshal(buf, &manifest); err != nil { + is.log.Error().Err(err).Str("dir", dir).Msg("invalid JSON") + + return nil, "", "", err + } + + return buf, manifestDesc.Digest, manifestDesc.MediaType, nil +} + +// PutImageManifest adds an image manifest to the repository. +func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //nolint: gocyclo + body []byte, +) (godigest.Digest, godigest.Digest, error) { + if err := is.InitRepo(repo); err != nil { + is.log.Debug().Err(err).Msg("init repo") + + return "", "", err + } + + var lockLatency time.Time + + var err error + + is.Lock(&lockLatency) + defer func() { + is.Unlock(&lockLatency) + + if err == nil { + monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) + monitoring.IncUploadCounter(is.metrics, repo) + } + }() + + refIsDigest := true + + mDigest, err := common.GetAndValidateRequestDigest(body, reference, is.log) + if err != nil { + if errors.Is(err, zerr.ErrBadManifest) { + return mDigest, "", err + } + + refIsDigest = false + } + + dig, err := common.ValidateManifest(is, repo, reference, mediaType, body, is.log) + if err != nil { + return dig, "", err + } + + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return "", "", err + } + + // create a new descriptor + desc := ispec.Descriptor{ + MediaType: mediaType, Size: int64(len(body)), Digest: mDigest, + } + + if !refIsDigest { + desc.Annotations = map[string]string{ispec.AnnotationRefName: reference} + } + + var subjectDigest godigest.Digest + + artifactType := "" + + if mediaType == ispec.MediaTypeImageManifest { + var manifest ispec.Manifest + + err := json.Unmarshal(body, &manifest) + if err != nil { + return "", "", err + } + + if manifest.Subject != nil { + subjectDigest = manifest.Subject.Digest + } + + artifactType = zcommon.GetManifestArtifactType(manifest) + } else if mediaType == ispec.MediaTypeImageIndex { + var index ispec.Index + + err := json.Unmarshal(body, &index) + if err != nil { + return "", "", err + } + + if index.Subject != nil { + subjectDigest = index.Subject.Digest + } + + artifactType = zcommon.GetIndexArtifactType(index) + } + + updateIndex, oldDgst, err := common.CheckIfIndexNeedsUpdate(&index, &desc, is.log) + if err != nil { + return "", "", err + } + + if !updateIndex { + return desc.Digest, subjectDigest, nil + } + + // write manifest to "blobs" + dir := path.Join(is.rootDir, repo, "blobs", mDigest.Algorithm().String()) + manifestPath := path.Join(dir, mDigest.Encoded()) + + if _, err = is.storeDriver.WriteFile(manifestPath, body); err != nil { + is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write") + + return "", "", err + } + + err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) + if err != nil { + return "", "", err + } + + // now update "index.json" + index.Manifests = append(index.Manifests, desc) + dir = path.Join(is.rootDir, repo) + indexPath := path.Join(dir, "index.json") + + buf, err := json.Marshal(index) + if err != nil { + is.log.Error().Err(err).Str("file", indexPath).Msg("unable to marshal JSON") + + return "", "", err + } + + // update the descriptors artifact type in order to check for signatures when applying the linter + desc.ArtifactType = artifactType + + // apply linter only on images, not signatures + pass, err := common.ApplyLinter(is, is.linter, repo, desc) + if !pass { + is.log.Error().Err(err).Str("repository", repo).Str("reference", reference).Msg("linter didn't pass") + + return "", "", err + } + + if _, err = is.storeDriver.WriteFile(indexPath, buf); err != nil { + is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write") + + return "", "", err + } + + return desc.Digest, subjectDigest, nil +} + +// DeleteImageManifest deletes the image manifest from the repository. +func (is *ImageStore) DeleteImageManifest(repo, reference string, detectCollisions bool) error { + dir := path.Join(is.rootDir, repo) + if fi, err := is.storeDriver.Stat(dir); err != nil || !fi.IsDir() { + return zerr.ErrRepoNotFound + } + + var lockLatency time.Time + + var err error + + is.Lock(&lockLatency) + defer func() { + is.Unlock(&lockLatency) + + if err == nil { + monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) + } + }() + + err = is.deleteImageManifest(repo, reference, detectCollisions) + if err != nil { + return err + } + + return nil +} + +func (is *ImageStore) deleteImageManifest(repo, reference string, detectCollisions bool) error { + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return err + } + + manifestDesc, err := common.RemoveManifestDescByReference(&index, reference, detectCollisions) + if err != nil { + return err + } + + /* check if manifest is referenced in image indexes, do not allow index images manipulations + (ie. remove manifest being part of an image index) */ + if manifestDesc.MediaType == ispec.MediaTypeImageManifest { + for _, mDesc := range index.Manifests { + if mDesc.MediaType == ispec.MediaTypeImageIndex { + if ok, _ := common.IsBlobReferencedInImageIndex(is, repo, manifestDesc.Digest, ispec.Index{ + Manifests: []ispec.Descriptor{mDesc}, + }, is.log); ok { + return zerr.ErrManifestReferenced + } + } + } + } + + err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log) + if err != nil { + return err + } + + // now update "index.json" + dir := path.Join(is.rootDir, repo) + file := path.Join(dir, "index.json") + + buf, err := json.Marshal(index) + if err != nil { + return err + } + + if _, err := is.storeDriver.WriteFile(file, buf); err != nil { + is.log.Debug().Str("deleting reference", reference).Msg("") + + return err + } + + // Delete blob only when blob digest not present in manifest entry. + // e.g. 1.0.1 & 1.0.2 have same blob digest so if we delete 1.0.1, blob should not be removed. + toDelete := true + + for _, manifest := range index.Manifests { + if manifestDesc.Digest.String() == manifest.Digest.String() { + toDelete = false + + break + } + } + + if toDelete { + p := path.Join(dir, "blobs", manifestDesc.Digest.Algorithm().String(), manifestDesc.Digest.Encoded()) + + err = is.storeDriver.Delete(p) + if err != nil { + return err + } + } + + return nil +} + +// BlobUploadPath returns the upload path for a blob in this store. +func (is *ImageStore) BlobUploadPath(repo, uuid string) string { + dir := path.Join(is.rootDir, repo) + blobUploadPath := path.Join(dir, storageConstants.BlobUploadDir, uuid) + + return blobUploadPath +} + +// NewBlobUpload returns the unique ID for an upload in progress. +func (is *ImageStore) NewBlobUpload(repo string) (string, error) { + if err := is.InitRepo(repo); err != nil { + is.log.Error().Err(err).Msg("error initializing repo") + + return "", err + } + + uuid, err := guuid.NewV4() + if err != nil { + return "", err + } + + uid := uuid.String() + + blobUploadPath := is.BlobUploadPath(repo, uid) + + // create multipart upload (append false) + writer, err := is.storeDriver.Writer(blobUploadPath, false) + if err != nil { + is.log.Debug().Err(err).Str("blob", blobUploadPath).Msg("failed to start multipart writer") + + return "", zerr.ErrRepoNotFound + } + + defer writer.Close() + + return uid, nil +} + +// GetBlobUpload returns the current size of a blob upload. +func (is *ImageStore) GetBlobUpload(repo, uuid string) (int64, error) { + blobUploadPath := is.BlobUploadPath(repo, uuid) + + if !utf8.ValidString(blobUploadPath) { + is.log.Error().Msg("input is not valid UTF-8") + + return -1, zerr.ErrInvalidRepositoryName + } + + writer, err := is.storeDriver.Writer(blobUploadPath, true) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return -1, zerr.ErrUploadNotFound + } + + return -1, err + } + + defer writer.Close() + + return writer.Size(), nil +} + +// PutBlobChunkStreamed appends another chunk of data to the specified blob. It returns +// the number of actual bytes to the blob. +func (is *ImageStore) PutBlobChunkStreamed(repo, uuid string, body io.Reader) (int64, error) { + if err := is.InitRepo(repo); err != nil { + return -1, err + } + + blobUploadPath := is.BlobUploadPath(repo, uuid) + + file, err := is.storeDriver.Writer(blobUploadPath, true) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return -1, zerr.ErrUploadNotFound + } + + is.log.Error().Err(err).Msg("failed to continue multipart upload") + + return -1, err + } + + var n int64 //nolint: varnamelen + + defer func() { + err = file.Close() + }() + + n, err = io.Copy(file, body) + + return n, err +} + +// PutBlobChunk writes another chunk of data to the specified blob. It returns +// the number of actual bytes to the blob. +func (is *ImageStore) PutBlobChunk(repo, uuid string, from, to int64, + body io.Reader, +) (int64, error) { + if err := is.InitRepo(repo); err != nil { + return -1, err + } + + blobUploadPath := is.BlobUploadPath(repo, uuid) + + file, err := is.storeDriver.Writer(blobUploadPath, true) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return -1, zerr.ErrUploadNotFound + } + + is.log.Error().Err(err).Msg("failed to continue multipart upload") + + return -1, err + } + + defer file.Close() + + if from != file.Size() { + is.log.Error().Int64("expected", from).Int64("actual", file.Size()). + Msg("invalid range start for blob upload") + + return -1, zerr.ErrBadUploadRange + } + + n, err := io.Copy(file, body) + + return n, err +} + +// BlobUploadInfo returns the current blob size in bytes. +func (is *ImageStore) BlobUploadInfo(repo, uuid string) (int64, error) { + blobUploadPath := is.BlobUploadPath(repo, uuid) + + writer, err := is.storeDriver.Writer(blobUploadPath, true) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return -1, zerr.ErrUploadNotFound + } + + return -1, err + } + + defer writer.Close() + + return writer.Size(), nil +} + +// FinishBlobUpload finalizes the blob upload and moves blob the repository. +func (is *ImageStore) FinishBlobUpload(repo, uuid string, body io.Reader, dstDigest godigest.Digest) error { + if err := dstDigest.Validate(); err != nil { + return err + } + + src := is.BlobUploadPath(repo, uuid) + + // complete multiUploadPart + fileWriter, err := is.storeDriver.Writer(src, true) + if err != nil { + is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") + + return zerr.ErrUploadNotFound + } + + if err := fileWriter.Commit(); err != nil { + is.log.Error().Err(err).Msg("failed to commit file") + + return err + } + + if err := fileWriter.Close(); err != nil { + is.log.Error().Err(err).Msg("failed to close file") + + return err + } + + fileReader, err := is.storeDriver.Reader(src, 0) + if err != nil { + is.log.Error().Err(err).Str("blob", src).Msg("failed to open file") + + return zerr.ErrUploadNotFound + } + + defer fileReader.Close() + + srcDigest, err := godigest.FromReader(fileReader) + if err != nil { + is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") + + return zerr.ErrBadBlobDigest + } + + if srcDigest != dstDigest { + is.log.Error().Str("srcDigest", srcDigest.String()). + Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") + + return zerr.ErrBadBlobDigest + } + + dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) + + err = is.storeDriver.EnsureDir(dir) + if err != nil { + is.log.Error().Err(err).Msg("error creating blobs/sha256 dir") + + return err + } + + dst := is.BlobPath(repo, dstDigest) + + var lockLatency time.Time + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + + if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { + err = is.DedupeBlob(src, dstDigest, dst) + if err := inject.Error(err); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to dedupe blob") + + return err + } + } else { + if err := is.storeDriver.Move(src, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to finish blob") + + return err + } + } + + return nil +} + +// FullBlobUpload handles a full blob upload, and no partial session is created. +func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, dstDigest godigest.Digest) (string, int64, error) { + if err := dstDigest.Validate(); err != nil { + return "", -1, err + } + + if err := is.InitRepo(repo); err != nil { + return "", -1, err + } + + u, err := guuid.NewV4() + if err != nil { + return "", -1, err + } + + uuid := u.String() + src := is.BlobUploadPath(repo, uuid) + digester := sha256.New() + buf := new(bytes.Buffer) + + _, err = buf.ReadFrom(body) + if err != nil { + is.log.Error().Err(err).Msg("failed to read blob") + + return "", -1, err + } + + nbytes, err := is.storeDriver.WriteFile(src, buf.Bytes()) + if err != nil { + is.log.Error().Err(err).Msg("failed to write blob") + + return "", -1, err + } + + _, err = digester.Write(buf.Bytes()) + if err != nil { + is.log.Error().Err(err).Msg("digester failed to write") + + return "", -1, err + } + + srcDigest := godigest.NewDigestFromEncoded(godigest.SHA256, fmt.Sprintf("%x", digester.Sum(nil))) + if srcDigest != dstDigest { + is.log.Error().Str("srcDigest", srcDigest.String()). + Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") + + return "", -1, zerr.ErrBadBlobDigest + } + + dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) + _ = is.storeDriver.EnsureDir(dir) + + var lockLatency time.Time + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + + dst := is.BlobPath(repo, dstDigest) + + if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { + if err := is.DedupeBlob(src, dstDigest, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to dedupe blob") + + return "", -1, err + } + } else { + if err := is.storeDriver.Move(src, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). + Str("dst", dst).Msg("unable to finish blob") + + return "", -1, err + } + } + + return uuid, int64(nbytes), nil +} + +func (is *ImageStore) DedupeBlob(src string, dstDigest godigest.Digest, dst string) error { +retry: + is.log.Debug().Str("src", src).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: enter") + + dstRecord, err := is.cache.GetBlob(dstDigest) + if err := inject.Error(err); err != nil && !errors.Is(err, zerr.ErrCacheMiss) { + is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to lookup blob record") + + return err + } + + if dstRecord == "" { + // cache record doesn't exist, so first disk and cache entry for this digest + if err := is.cache.PutBlob(dstDigest, dst); err != nil { + is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") + + return err + } + + // move the blob from uploads to final dest + if err := is.storeDriver.Move(src, dst); err != nil { + is.log.Error().Err(err).Str("src", src).Str("dst", dst).Msg("dedupe: unable to rename blob") + + return err + } + + is.log.Debug().Str("src", src).Str("dst", dst).Msg("dedupe: rename") + } else { + // cache record exists, but due to GC and upgrades from older versions, + // disk content and cache records may go out of sync + + if is.cache.UsesRelativePaths() { + dstRecord = path.Join(is.rootDir, dstRecord) + } + + _, err := is.storeDriver.Stat(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat") + // the actual blob on disk may have been removed by GC, so sync the cache + err := is.cache.DeleteBlob(dstDigest, dstRecord) + if err = inject.Error(err); err != nil { + //nolint:lll + is.log.Error().Err(err).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: unable to delete blob record") + + return err + } + + goto retry + } + + // prevent overwrite original blob + if !is.storeDriver.SameFile(dst, dstRecord) { + if err := is.storeDriver.Link(dstRecord, dst); err != nil { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to link blobs") + + return err + } + + if err := is.cache.PutBlob(dstDigest, dst); err != nil { + is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") + + return err + } + } + + // remove temp blobupload + if err := is.storeDriver.Delete(src); err != nil { + is.log.Error().Err(err).Str("src", src).Msg("dedupe: unable to remove blob") + + return err + } + + is.log.Debug().Str("src", src).Msg("dedupe: remove") + } + + return nil +} + +// DeleteBlobUpload deletes an existing blob upload that is currently in progress. +func (is *ImageStore) DeleteBlobUpload(repo, uuid string) error { + blobUploadPath := is.BlobUploadPath(repo, uuid) + + writer, err := is.storeDriver.Writer(blobUploadPath, true) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + return zerr.ErrUploadNotFound + } + + return err + } + + defer writer.Close() + + if err := writer.Cancel(); err != nil { + is.log.Error().Err(err).Str("blobUploadPath", blobUploadPath).Msg("error deleting blob upload") + + return err + } + + return nil +} + +// BlobPath returns the repository path of a blob. +func (is *ImageStore) BlobPath(repo string, digest godigest.Digest) string { + return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded()) +} + +/* + CheckBlob verifies a blob and returns true if the blob is correct + +If the blob is not found but it's found in cache then it will be copied over. +*/ +func (is *ImageStore) CheckBlob(repo string, digest godigest.Digest) (bool, int64, error) { + var lockLatency time.Time + + if err := digest.Validate(); err != nil { + return false, -1, err + } + + blobPath := is.BlobPath(repo, digest) + + if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + } else { + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + } + + binfo, err := is.storeDriver.Stat(blobPath) + if err == nil && binfo.Size() > 0 { + is.log.Debug().Str("blob path", blobPath).Msg("blob path found") + + return true, binfo.Size(), nil + } + // otherwise is a 'deduped' blob (empty file) + + // Check blobs in cache + dstRecord, err := is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return false, -1, zerr.ErrBlobNotFound + } + + blobSize, err := is.copyBlob(repo, blobPath, dstRecord) + if err != nil { + return false, -1, zerr.ErrBlobNotFound + } + + // put deduped blob in cache + if err := is.cache.PutBlob(digest, blobPath); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("dedupe: unable to insert blob record") + + return false, -1, err + } + + return true, blobSize, nil +} + +// StatBlob verifies if a blob is present inside a repository. The caller function SHOULD lock from outside. +func (is *ImageStore) StatBlob(repo string, digest godigest.Digest) (bool, int64, time.Time, error) { + if err := digest.Validate(); err != nil { + return false, -1, time.Time{}, err + } + + blobPath := is.BlobPath(repo, digest) + + binfo, err := is.storeDriver.Stat(blobPath) + if err == nil && binfo.Size() > 0 { + is.log.Debug().Str("blob path", blobPath).Msg("blob path found") + + return true, binfo.Size(), binfo.ModTime(), nil + } + + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return false, -1, time.Time{}, zerr.ErrBlobNotFound + } + + // then it's a 'deduped' blob + + // Check blobs in cache + dstRecord, err := is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return false, -1, time.Time{}, zerr.ErrBlobNotFound + } + + binfo, err = is.storeDriver.Stat(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return false, -1, time.Time{}, zerr.ErrBlobNotFound + } + + return true, binfo.Size(), binfo.ModTime(), nil +} + +func (is *ImageStore) checkCacheBlob(digest godigest.Digest) (string, error) { + if err := digest.Validate(); err != nil { + return "", err + } + + if fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { + return "", zerr.ErrBlobNotFound + } + + dstRecord, err := is.cache.GetBlob(digest) + if err != nil { + return "", err + } + + if is.cache.UsesRelativePaths() { + dstRecord = path.Join(is.rootDir, dstRecord) + } + + if _, err := is.storeDriver.Stat(dstRecord); err != nil { + is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to stat blob") + + // the actual blob on disk may have been removed by GC, so sync the cache + if err := is.cache.DeleteBlob(digest, dstRecord); err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", dstRecord). + Msg("unable to remove blob path from cache") + + return "", err + } + + return "", zerr.ErrBlobNotFound + } + + is.log.Debug().Str("digest", digest.String()).Str("dstRecord", dstRecord).Msg("cache: found dedupe record") + + return dstRecord, nil +} + +func (is *ImageStore) copyBlob(repo string, blobPath, dstRecord string) (int64, error) { + if err := is.initRepo(repo); err != nil { + is.log.Error().Err(err).Str("repository", repo).Msg("unable to initialize an empty repo") + + return -1, err + } + + _ = is.storeDriver.EnsureDir(filepath.Dir(blobPath)) + + if err := is.storeDriver.Link(dstRecord, blobPath); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Str("link", dstRecord).Msg("dedupe: unable to hard link") + + return -1, zerr.ErrBlobNotFound + } + + // return original blob with content instead of the deduped one (blobPath) + binfo, err := is.storeDriver.Stat(dstRecord) + if err == nil { + return binfo.Size(), nil + } + + return -1, zerr.ErrBlobNotFound +} + +// GetBlobPartial returns a partial stream to read the blob. +// blob selector instead of directly downloading the blob. +func (is *ImageStore) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, +) (io.ReadCloser, int64, int64, error) { + var lockLatency time.Time + + if err := digest.Validate(); err != nil { + return nil, -1, -1, err + } + + blobPath := is.BlobPath(repo, digest) + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return nil, -1, -1, zerr.ErrBlobNotFound + } + + // is a deduped blob + if binfo.Size() == 0 { + // Check blobs in cache + blobPath, err = is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return nil, -1, -1, zerr.ErrBlobNotFound + } + + binfo, err = is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return nil, -1, -1, zerr.ErrBlobNotFound + } + } + + end := to + + if to < 0 || to >= binfo.Size() { + end = binfo.Size() - 1 + } + + blobHandle, err := is.storeDriver.Reader(blobPath, from) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") + + return nil, -1, -1, err + } + + blobReadCloser, err := newBlobStream(blobHandle, from, end) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob stream") + + return nil, -1, -1, err + } + + // The caller function is responsible for calling Close() + return blobReadCloser, end - from + 1, binfo.Size(), nil +} + +// GetBlob returns a stream to read the blob. +// blob selector instead of directly downloading the blob. +func (is *ImageStore) GetBlob(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) { + var lockLatency time.Time + + if err := digest.Validate(); err != nil { + return nil, -1, err + } + + blobPath := is.BlobPath(repo, digest) + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return nil, -1, zerr.ErrBlobNotFound + } + + blobReadCloser, err := is.storeDriver.Reader(blobPath, 0) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") + + return nil, -1, err + } + + // is a 'deduped' blob? + if binfo.Size() == 0 { + // Check blobs in cache + dstRecord, err := is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return nil, -1, zerr.ErrBlobNotFound + } + + binfo, err := is.storeDriver.Stat(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to stat blob") + + return nil, -1, zerr.ErrBlobNotFound + } + + blobReadCloser, err := is.storeDriver.Reader(dstRecord, 0) + if err != nil { + is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") + + return nil, -1, err + } + + return blobReadCloser, binfo.Size(), nil + } + + // The caller function is responsible for calling Close() + return blobReadCloser, binfo.Size(), nil +} + +// GetBlobContent returns blob contents, the caller function SHOULD lock from outside. +func (is *ImageStore) GetBlobContent(repo string, digest godigest.Digest) ([]byte, error) { + if err := digest.Validate(); err != nil { + return []byte{}, err + } + + blobPath := is.BlobPath(repo, digest) + + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return []byte{}, zerr.ErrBlobNotFound + } + + blobBuf, err := is.storeDriver.ReadFile(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") + + return nil, err + } + + // is a 'deduped' blob? + if binfo.Size() == 0 { + // Check blobs in cache + dstRecord, err := is.checkCacheBlob(digest) + if err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") + + return nil, zerr.ErrBlobNotFound + } + + blobBuf, err := is.storeDriver.ReadFile(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") + + return nil, err + } + + return blobBuf, nil + } + + return blobBuf, nil +} + +func (is *ImageStore) GetReferrers(repo string, gdigest godigest.Digest, artifactTypes []string, +) (ispec.Index, error) { + var lockLatency time.Time + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + return common.GetReferrers(is, repo, gdigest, artifactTypes, is.log) +} + +func (is *ImageStore) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string, +) ([]artifactspec.Descriptor, error) { + var lockLatency time.Time + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + return common.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) +} + +// GetIndexContent returns index.json contents, the caller function SHOULD lock from outside. +func (is *ImageStore) GetIndexContent(repo string) ([]byte, error) { + dir := path.Join(is.rootDir, repo) + + buf, err := is.storeDriver.ReadFile(path.Join(dir, "index.json")) + if err != nil { + if errors.Is(err, driver.PathNotFoundError{}) { + is.log.Error().Err(err).Str("dir", dir).Msg("index.json doesn't exist") + + return []byte{}, zerr.ErrRepoNotFound + } + + is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") + + return []byte{}, err + } + + return buf, nil +} + +// DeleteBlob removes the blob from the repository. +func (is *ImageStore) DeleteBlob(repo string, digest godigest.Digest) error { + var lockLatency time.Time + + if err := digest.Validate(); err != nil { + return err + } + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + + return is.deleteBlob(repo, digest) +} + +func (is *ImageStore) deleteBlob(repo string, digest godigest.Digest) error { + blobPath := is.BlobPath(repo, digest) + + _, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") + + return zerr.ErrBlobNotFound + } + + // first check if this blob is not currently in use + if ok, _ := common.IsBlobReferenced(is, repo, digest, is.log); ok { + return zerr.ErrBlobReferenced + } + + if fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { + dstRecord, err := is.cache.GetBlob(digest) + if err != nil && !errors.Is(err, zerr.ErrCacheMiss) { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to lookup blob record") + + return err + } + + // remove cache entry and move blob contents to the next candidate if there is any + if ok := is.cache.HasBlob(digest, blobPath); ok { + if err := is.cache.DeleteBlob(digest, blobPath); err != nil { + is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", blobPath). + Msg("unable to remove blob path from cache") + + return err + } + } + + // if the deleted blob is one with content + if dstRecord == blobPath { + // get next candidate + dstRecord, err := is.cache.GetBlob(digest) + if err != nil && !errors.Is(err, zerr.ErrCacheMiss) { + is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to lookup blob record") + + return err + } + + // if we have a new candidate move the blob content to it + if dstRecord != "" { + /* check to see if we need to move the content from original blob to duplicate one + (in case of filesystem, this should not be needed */ + binfo, err := is.storeDriver.Stat(dstRecord) + if err != nil { + is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") + + return err + } + + if binfo.Size() == 0 { + if err := is.storeDriver.Move(blobPath, dstRecord); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") + + return err + } + } + + return nil + } + } + } + + if err := is.storeDriver.Delete(blobPath); err != nil { + is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") + + return err + } + + return nil +} + +func (is *ImageStore) garbageCollect(repo string) error { + if is.gcReferrers { + is.log.Info().Msg("gc: manifests with missing referrers") + + // gc all manifests that have a missing subject, stop when no gc happened in a full loop over index.json. + stop := false + for !stop { + // because we gc manifests in the loop, need to get latest index.json content + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return err + } + + gced, err := is.garbageCollectIndexReferrers(repo, index, index) + if err != nil { + return err + } + + /* if we delete any manifest then loop again and gc manifests with + a subject pointing to the last ones which were gc'ed. */ + stop = !gced + } + } + + index, err := common.GetIndex(is, repo, is.log) + if err != nil { + return err + } + + is.log.Info().Msg("gc: manifests without tags") + + // apply image retention policy + if err := is.garbageCollectUntaggedManifests(index, repo); err != nil { + return err + } + + is.log.Info().Msg("gc: blobs") + + if err := is.garbageCollectBlobs(is, repo, is.gcDelay, is.log); err != nil { + return err + } + + return nil +} + +/* +garbageCollectIndexReferrers will gc all referrers with a missing subject recursively + +rootIndex is indexJson, need to pass it down to garbageCollectReferrer() +rootIndex is the place we look for referrers. +*/ +func (is *ImageStore) garbageCollectIndexReferrers(repo string, rootIndex ispec.Index, index ispec.Index, +) (bool, error) { + var count int + + var err error + + for _, desc := range index.Manifests { + switch desc.MediaType { + case ispec.MediaTypeImageIndex: + indexImage, err := common.GetImageIndex(is, repo, desc.Digest, is.log) + if err != nil { + is.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("gc: failed to read multiarch(index) image") + + return false, err + } + + gced, err := is.garbageCollectReferrer(repo, rootIndex, desc, indexImage.Subject) + if err != nil { + return false, err + } + + /* if we gc index then no need to continue searching for referrers inside it. + they will be gced when the next garbage collect is executed(if they are older than retentionDelay), + because manifests part of indexes will still be referenced in index.json */ + if gced { + return true, nil + } + + if gced, err = is.garbageCollectIndexReferrers(repo, rootIndex, indexImage); err != nil { + return false, err + } + + if gced { + count++ + } + + case ispec.MediaTypeImageManifest, artifactspec.MediaTypeArtifactManifest: + image, err := common.GetImageManifest(is, repo, desc.Digest, is.log) + if err != nil { + is.log.Error().Err(err).Str("repo", repo).Str("digest", desc.Digest.String()). + Msg("gc: failed to read manifest image") + + return false, err + } + + gced, err := is.garbageCollectReferrer(repo, rootIndex, desc, image.Subject) + if err != nil { + return false, err + } + + if gced { + count++ + } + } + } + + return count > 0, err +} + +func (is *ImageStore) garbageCollectReferrer(repo string, index ispec.Index, manifestDesc ispec.Descriptor, + subject *ispec.Descriptor, +) (bool, error) { + var gced bool + + var err error + + if subject != nil { + // try to find subject in index.json + if ok := isManifestReferencedInIndex(index, subject.Digest); !ok { + gced, err = garbageCollectManifest(is, repo, manifestDesc.Digest, is.gcDelay) + if err != nil { + return false, err + } + } + } + + tag, ok := manifestDesc.Annotations[ispec.AnnotationRefName] + if ok { + if strings.HasPrefix(tag, "sha256-") && (strings.HasSuffix(tag, cosignSignatureTagSuffix) || + strings.HasSuffix(tag, SBOMTagSuffix)) { + if ok := isManifestReferencedInIndex(index, getSubjectFromCosignTag(tag)); !ok { + gced, err = garbageCollectManifest(is, repo, manifestDesc.Digest, is.gcDelay) + if err != nil { + return false, err + } + } + } + } + + return gced, err +} + +func (is *ImageStore) garbageCollectUntaggedManifests(index ispec.Index, repo string) error { + referencedByImageIndex := make([]string, 0) + + if err := identifyManifestsReferencedInIndex(is, index, repo, &referencedByImageIndex); err != nil { + return err + } + + // first gather manifests part of image indexes and referrers, we want to skip checking them + for _, desc := range index.Manifests { + // skip manifests referenced in image indexes + if zcommon.Contains(referencedByImageIndex, desc.Digest.String()) { + continue + } + + // remove untagged images + if desc.MediaType == ispec.MediaTypeImageManifest || desc.MediaType == ispec.MediaTypeImageIndex { + _, ok := desc.Annotations[ispec.AnnotationRefName] + if !ok { + _, err := garbageCollectManifest(is, repo, desc.Digest, is.retentionDelay) + if err != nil { + return err + } + } + } + } + + return nil +} + +// Adds both referenced manifests and referrers from an index. +func identifyManifestsReferencedInIndex(imgStore *ImageStore, index ispec.Index, repo string, referenced *[]string, +) error { + for _, desc := range index.Manifests { + switch desc.MediaType { + case ispec.MediaTypeImageIndex: + indexImage, err := common.GetImageIndex(imgStore, repo, desc.Digest, imgStore.log) + if err != nil { + imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). + Msg("gc: failed to read multiarch(index) image") + + return err + } + + if indexImage.Subject != nil { + *referenced = append(*referenced, desc.Digest.String()) + } + + for _, indexDesc := range indexImage.Manifests { + *referenced = append(*referenced, indexDesc.Digest.String()) + } + + if err := identifyManifestsReferencedInIndex(imgStore, indexImage, repo, referenced); err != nil { + return err + } + case ispec.MediaTypeImageManifest, artifactspec.MediaTypeArtifactManifest: + image, err := common.GetImageManifest(imgStore, repo, desc.Digest, imgStore.log) + if err != nil { + imgStore.log.Error().Err(err).Str("repo", repo).Str("digest", desc.Digest.String()). + Msg("gc: failed to read manifest image") + + return err + } + + if image.Subject != nil { + *referenced = append(*referenced, desc.Digest.String()) + } + } + } + + return nil +} + +func garbageCollectManifest(imgStore *ImageStore, repo string, digest godigest.Digest, delay time.Duration, +) (bool, error) { + canGC, err := isBlobOlderThan(imgStore, repo, digest, delay, imgStore.log) + if err != nil { + imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", digest.String()). + Str("delay", imgStore.gcDelay.String()).Msg("gc: failed to check if blob is older than delay") + + return false, err + } + + if canGC { + imgStore.log.Info().Str("repository", repo).Str("digest", digest.String()). + Msg("gc: removing unreferenced manifest") + + if err := imgStore.deleteImageManifest(repo, digest.String(), true); err != nil { + if errors.Is(err, zerr.ErrManifestConflict) { + imgStore.log.Info().Str("repository", repo).Str("digest", digest.String()). + Msg("gc: skipping removing manifest due to conflict") + + return false, nil + } + + return false, err + } + + return true, nil + } + + return false, nil +} + +func (is *ImageStore) garbageCollectBlobs(imgStore *ImageStore, repo string, + delay time.Duration, log zlog.Logger, +) error { + refBlobs := map[string]bool{} + + err := common.AddRepoBlobsToReferences(imgStore, repo, refBlobs, log) + if err != nil { + log.Error().Err(err).Str("repository", repo).Msg("unable to get referenced blobs in repo") + + return err + } + + allBlobs, err := imgStore.GetAllBlobs(repo) + if err != nil { + // /blobs/sha256/ may be empty in the case of s3, no need to return err, we want to skip + if errors.As(err, &driver.PathNotFoundError{}) { + return nil + } + + log.Error().Err(err).Str("repository", repo).Msg("unable to get all blobs") + + return err + } + + reaped := 0 + + for _, blob := range allBlobs { + digest := godigest.NewDigestFromEncoded(godigest.SHA256, blob) + if err = digest.Validate(); err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", blob).Msg("unable to parse digest") + + return err + } + + if _, ok := refBlobs[digest.String()]; !ok { + ok, err := isBlobOlderThan(imgStore, repo, digest, delay, log) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", blob).Msg("unable to determine GC delay") + + return err + } + + if !ok { + continue + } + + if err := imgStore.deleteBlob(repo, digest); err != nil { + if errors.Is(err, zerr.ErrBlobReferenced) { + if err := imgStore.deleteImageManifest(repo, digest.String(), true); err != nil { + if errors.Is(err, zerr.ErrManifestConflict) { + continue + } + + log.Error().Err(err).Str("repository", repo).Str("digest", blob).Msg("unable to delete blob") + + return err + } + } else { + log.Error().Err(err).Str("repository", repo).Str("digest", blob).Msg("unable to delete blob") + + return err + } + } + + log.Info().Str("repository", repo).Str("digest", blob).Msg("garbage collected blob") + + reaped++ + } + } + + // if we cleaned all blobs let's also remove the repo so that it won't be returned by catalog + if reaped == len(allBlobs) { + log.Info().Str("repository", repo).Msg("garbage collected all blobs, cleaning repo...") + + if err := is.storeDriver.Delete(path.Join(is.rootDir, repo)); err != nil { + log.Error().Err(err).Str("repository", repo).Msg("unable to delete repo") + + return err + } + } + + log.Info().Str("repository", repo).Int("count", reaped).Msg("garbage collected blobs") + + return nil +} + +func (is *ImageStore) gcRepo(repo string) error { + var lockLatency time.Time + + is.Lock(&lockLatency) + err := is.garbageCollect(repo) + is.Unlock(&lockLatency) + + if err != nil { + return err + } + + return nil +} + +func (is *ImageStore) GetAllBlobs(repo string) ([]string, error) { + dir := path.Join(is.rootDir, repo, "blobs", "sha256") + + files, err := is.storeDriver.List(dir) + if err != nil { + return []string{}, err + } + + ret := []string{} + + for _, file := range files { + ret = append(ret, filepath.Base(file)) + } + + return ret, nil +} + +func (is *ImageStore) RunGCRepo(repo string) error { + is.log.Info().Msg(fmt.Sprintf("executing GC of orphaned blobs for %s", path.Join(is.RootDir(), repo))) + + if err := is.gcRepo(repo); err != nil { + errMessage := fmt.Sprintf("error while running GC for %s", path.Join(is.RootDir(), repo)) + is.log.Error().Err(err).Msg(errMessage) + is.log.Info().Msg(fmt.Sprintf("GC unsuccessfully completed for %s", path.Join(is.RootDir(), repo))) + + return err + } + + is.log.Info().Msg(fmt.Sprintf("GC successfully completed for %s", path.Join(is.RootDir(), repo))) + + return nil +} + +func (is *ImageStore) RunGCPeriodically(interval time.Duration, sch *scheduler.Scheduler) { + generator := &common.GCTaskGenerator{ + ImgStore: is, + } + + sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) +} + +func (is *ImageStore) GetNextDigestWithBlobPaths(lastDigests []godigest.Digest) (godigest.Digest, []string, error) { + var lockLatency time.Time + + dir := is.rootDir + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + var duplicateBlobs []string + + var digest godigest.Digest + + err := is.storeDriver.Walk(dir, func(fileInfo driver.FileInfo) error { + // skip blobs under .sync + if strings.HasSuffix(fileInfo.Path(), syncConstants.SyncBlobUploadDir) { + return driver.ErrSkipDir + } + + if fileInfo.IsDir() { + return nil + } + + blobDigest := godigest.NewDigestFromEncoded("sha256", path.Base(fileInfo.Path())) + if err := blobDigest.Validate(); err != nil { //nolint: nilerr + return nil //nolint: nilerr // ignore files which are not blobs + } + + if digest == "" && !zcommon.Contains(lastDigests, blobDigest) { + digest = blobDigest + } + + if blobDigest == digest { + duplicateBlobs = append(duplicateBlobs, fileInfo.Path()) + } + + return nil + }) + + // if the root directory is not yet created + var perr driver.PathNotFoundError + + if errors.As(err, &perr) { + return digest, duplicateBlobs, nil + } + + return digest, duplicateBlobs, err +} + +func (is *ImageStore) getOriginalBlobFromDisk(duplicateBlobs []string) (string, error) { + for _, blobPath := range duplicateBlobs { + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") + + return "", zerr.ErrBlobNotFound + } + + if binfo.Size() > 0 { + return blobPath, nil + } + } + + return "", zerr.ErrBlobNotFound +} + +func (is *ImageStore) getOriginalBlob(digest godigest.Digest, duplicateBlobs []string) (string, error) { + var originalBlob string + + var err error + + originalBlob, err = is.checkCacheBlob(digest) + if err != nil && !errors.Is(err, zerr.ErrBlobNotFound) && !errors.Is(err, zerr.ErrCacheMiss) { + is.log.Error().Err(err).Msg("rebuild dedupe: unable to find blob in cache") + + return originalBlob, err + } + + // if we still don't have, search it + if originalBlob == "" { + is.log.Warn().Msg("rebuild dedupe: failed to find blob in cache, searching it in s3...") + // a rebuild dedupe was attempted in the past + // get original blob, should be found otherwise exit with error + + originalBlob, err = is.getOriginalBlobFromDisk(duplicateBlobs) + if err != nil { + return originalBlob, err + } + } + + is.log.Info().Str("originalBlob", originalBlob).Msg("rebuild dedupe: found original blob") + + return originalBlob, nil +} + +func (is *ImageStore) dedupeBlobs(digest godigest.Digest, duplicateBlobs []string) error { + if fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { + is.log.Error().Err(zerr.ErrDedupeRebuild).Msg("no cache driver found, can not dedupe blobs") + + return zerr.ErrDedupeRebuild + } + + is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest") + + var originalBlob string + + // rebuild from dedupe false to true + for _, blobPath := range duplicateBlobs { + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") + + return err + } + + if binfo.Size() == 0 { + is.log.Warn().Msg("rebuild dedupe: found file without content, trying to find the original blob") + // a rebuild dedupe was attempted in the past + // get original blob, should be found otherwise exit with error + if originalBlob == "" { + originalBlob, err = is.getOriginalBlob(digest, duplicateBlobs) + if err != nil { + is.log.Error().Err(err).Msg("rebuild dedupe: unable to find original blob") + + return zerr.ErrDedupeRebuild + } + + // cache original blob + if ok := is.cache.HasBlob(digest, originalBlob); !ok { + if err := is.cache.PutBlob(digest, originalBlob); err != nil { + return err + } + } + } + + // cache dedupe blob + if ok := is.cache.HasBlob(digest, blobPath); !ok { + if err := is.cache.PutBlob(digest, blobPath); err != nil { + return err + } + } + } else { + // if we have an original blob cached then we can safely dedupe the rest of them + if originalBlob != "" { + if err := is.storeDriver.Link(originalBlob, blobPath); err != nil { + is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: unable to dedupe blob") + + return err + } + } + + // cache it + if ok := is.cache.HasBlob(digest, blobPath); !ok { + if err := is.cache.PutBlob(digest, blobPath); err != nil { + return err + } + } + + // mark blob as preserved + originalBlob = blobPath + } + } + + is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest finished successfully") + + return nil +} + +func (is *ImageStore) restoreDedupedBlobs(digest godigest.Digest, duplicateBlobs []string) error { + is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: restoring deduped blobs for digest") + + // first we need to find the original blob, either in cache or by checking each blob size + originalBlob, err := is.getOriginalBlob(digest, duplicateBlobs) + if err != nil { + is.log.Error().Err(err).Msg("rebuild dedupe: unable to find original blob") + + return zerr.ErrDedupeRebuild + } + + for _, blobPath := range duplicateBlobs { + binfo, err := is.storeDriver.Stat(blobPath) + if err != nil { + is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") + + return err + } + + // if we find a deduped blob, then copy original blob content to deduped one + if binfo.Size() == 0 { + // move content from original blob to deduped one + buf, err := is.storeDriver.ReadFile(originalBlob) + if err != nil { + is.log.Error().Err(err).Str("path", originalBlob).Msg("rebuild dedupe: failed to get original blob content") + + return err + } + + _, err = is.storeDriver.WriteFile(blobPath, buf) + if err != nil { + return err + } + } + } + + is.log.Info().Str("digest", digest.String()). + Msg("rebuild dedupe: restoring deduped blobs for digest finished successfully") + + return nil +} + +func (is *ImageStore) RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error { + var lockLatency time.Time + + is.Lock(&lockLatency) + defer is.Unlock(&lockLatency) + + if dedupe { + return is.dedupeBlobs(digest, duplicateBlobs) + } + + return is.restoreDedupedBlobs(digest, duplicateBlobs) +} + +func (is *ImageStore) RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) { + generator := &common.DedupeTaskGenerator{ + ImgStore: is, + Dedupe: is.dedupe, + Log: is.log, + } + + sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) +} + +type blobStream struct { + reader io.Reader + closer io.Closer +} + +func newBlobStream(readCloser io.ReadCloser, from, to int64) (io.ReadCloser, error) { + if from < 0 || to < from { + return nil, zerr.ErrBadRange + } + + return &blobStream{reader: io.LimitReader(readCloser, to-from+1), closer: readCloser}, nil +} + +func (bs *blobStream) Read(buf []byte) (int, error) { + return bs.reader.Read(buf) +} + +func (bs *blobStream) Close() error { + return bs.closer.Close() +} + +func isBlobOlderThan(imgStore storageTypes.ImageStore, repo string, + digest godigest.Digest, delay time.Duration, log zlog.Logger, +) (bool, error) { + _, _, modtime, err := imgStore.StatBlob(repo, digest) + if err != nil { + log.Error().Err(err).Str("repository", repo).Str("digest", digest.String()). + Msg("gc: failed to stat blob") + + return false, err + } + + if modtime.Add(delay).After(time.Now()) { + return false, nil + } + + log.Info().Str("repository", repo).Str("digest", digest.String()).Msg("perform GC on blob") + + return true, nil +} + +func getSubjectFromCosignTag(tag string) godigest.Digest { + alg := strings.Split(tag, "-")[0] + encoded := strings.Split(strings.Split(tag, "-")[1], ".sig")[0] + + return godigest.NewDigestFromEncoded(godigest.Algorithm(alg), encoded) +} + +func isManifestReferencedInIndex(index ispec.Index, digest godigest.Digest) bool { + for _, manifest := range index.Manifests { + if manifest.Digest == digest { + return true + } + } + + return false +} diff --git a/pkg/storage/local/driver.go b/pkg/storage/local/driver.go new file mode 100644 index 00000000..2e12ac9a --- /dev/null +++ b/pkg/storage/local/driver.go @@ -0,0 +1,481 @@ +package local + +import ( + "bufio" + "bytes" + "errors" + "io" + "io/fs" + "os" + "path" + "sort" + "syscall" + "time" + "unicode/utf8" + + storagedriver "github.com/docker/distribution/registry/storage/driver" + + zerr "zotregistry.io/zot/errors" + storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/test/inject" +) + +type Driver struct { + commit bool +} + +func New(commit bool) *Driver { + return &Driver{commit: commit} +} + +func (driver *Driver) Name() string { + return storageConstants.LocalStorageDriverName +} + +func (driver *Driver) EnsureDir(path string) error { + err := os.MkdirAll(path, storageConstants.DefaultDirPerms) + + return driver.formatErr(err) +} + +func (driver *Driver) DirExists(path string) bool { + if !utf8.ValidString(path) { + return false + } + + fileInfo, err := os.Stat(path) + if err != nil { + if e, ok := err.(*fs.PathError); ok && errors.Is(e.Err, syscall.ENAMETOOLONG) || //nolint: errorlint + errors.Is(e.Err, syscall.EINVAL) { + return false + } + } + + if err != nil && os.IsNotExist(err) { + return false + } + + if !fileInfo.IsDir() { + return false + } + + return true +} + +func (driver *Driver) Reader(path string, offset int64) (io.ReadCloser, error) { + file, err := os.OpenFile(path, os.O_RDONLY, storageConstants.DefaultFilePerms) + if err != nil { + if os.IsNotExist(err) { + return nil, storagedriver.PathNotFoundError{Path: path} + } + + return nil, driver.formatErr(err) + } + + seekPos, err := file.Seek(offset, io.SeekStart) + if err != nil { + file.Close() + + return nil, driver.formatErr(err) + } else if seekPos < offset { + file.Close() + + return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset} + } + + return file, nil +} + +func (driver *Driver) ReadFile(path string) ([]byte, error) { + reader, err := driver.Reader(path, 0) + if err != nil { + return nil, err + } + + defer reader.Close() + + buf, err := io.ReadAll(reader) + if err != nil { + return nil, driver.formatErr(err) + } + + return buf, nil +} + +func (driver *Driver) Delete(path string) error { + _, err := os.Stat(path) + if err != nil && !os.IsNotExist(err) { + return driver.formatErr(err) + } else if err != nil { + return storagedriver.PathNotFoundError{Path: path} + } + + return os.RemoveAll(path) +} + +func (driver *Driver) Stat(path string) (storagedriver.FileInfo, error) { + fi, err := os.Stat(path) //nolint: varnamelen + if err != nil { + if os.IsNotExist(err) { + return nil, storagedriver.PathNotFoundError{Path: path} + } + + return nil, driver.formatErr(err) + } + + return fileInfo{ + path: path, + FileInfo: fi, + }, nil +} + +func (driver *Driver) Writer(filepath string, append bool) (storagedriver.FileWriter, error) { //nolint:predeclared + if append { + _, err := os.Stat(filepath) + if err != nil { + if os.IsNotExist(err) { + return nil, storagedriver.PathNotFoundError{Path: filepath} + } + + return nil, driver.formatErr(err) + } + } + + parentDir := path.Dir(filepath) + if err := os.MkdirAll(parentDir, storageConstants.DefaultDirPerms); err != nil { + return nil, driver.formatErr(err) + } + + file, err := os.OpenFile(filepath, os.O_WRONLY|os.O_CREATE, storageConstants.DefaultFilePerms) + if err != nil { + return nil, driver.formatErr(err) + } + + var offset int64 + + if !append { + err := file.Truncate(0) + if err != nil { + file.Close() + + return nil, driver.formatErr(err) + } + } else { + n, err := file.Seek(0, io.SeekEnd) //nolint: varnamelen + if err != nil { + file.Close() + + return nil, driver.formatErr(err) + } + offset = n + } + + return newFileWriter(file, offset, driver.commit), nil +} + +func (driver *Driver) WriteFile(filepath string, content []byte) (int, error) { + writer, err := driver.Writer(filepath, false) + if err != nil { + return -1, err + } + + nbytes, err := io.Copy(writer, bytes.NewReader(content)) + if err != nil { + _ = writer.Cancel() + + return -1, driver.formatErr(err) + } + + return int(nbytes), writer.Close() +} + +func (driver *Driver) Walk(path string, walkFn storagedriver.WalkFn) error { + children, err := driver.List(path) + if err != nil { + return err + } + + sort.Stable(sort.StringSlice(children)) + + for _, child := range children { + // Calling driver.Stat for every entry is quite + // expensive when running against backends with a slow Stat + // implementation, such as s3. This is very likely a serious + // performance bottleneck. + fileInfo, err := driver.Stat(child) + if err != nil { + switch errors.As(err, &storagedriver.PathNotFoundError{}) { + case true: + // repository was removed in between listing and enumeration. Ignore it. + continue + default: + return err + } + } + err = walkFn(fileInfo) + //nolint: gocritic + if err == nil && fileInfo.IsDir() { + if err := driver.Walk(child, walkFn); err != nil { + return err + } + } else if errors.Is(err, storagedriver.ErrSkipDir) { + // Stop iteration if it's a file, otherwise noop if it's a directory + if !fileInfo.IsDir() { + return nil + } + } else if err != nil { + return driver.formatErr(err) + } + } + + return nil +} + +func (driver *Driver) List(fullpath string) ([]string, error) { + dir, err := os.Open(fullpath) + if err != nil { + if os.IsNotExist(err) { + return nil, storagedriver.PathNotFoundError{Path: fullpath} + } + + return nil, driver.formatErr(err) + } + + defer dir.Close() + + fileNames, err := dir.Readdirnames(0) + if err != nil { + return nil, driver.formatErr(err) + } + + keys := make([]string, 0, len(fileNames)) + for _, fileName := range fileNames { + keys = append(keys, path.Join(fullpath, fileName)) + } + + return keys, nil +} + +func (driver *Driver) Move(sourcePath string, destPath string) error { + if _, err := os.Stat(sourcePath); os.IsNotExist(err) { + return storagedriver.PathNotFoundError{Path: sourcePath} + } + + if err := os.MkdirAll(path.Dir(destPath), storageConstants.DefaultDirPerms); err != nil { + return driver.formatErr(err) + } + + return driver.formatErr(os.Rename(sourcePath, destPath)) +} + +func (driver *Driver) SameFile(path1, path2 string) bool { + file1, err := os.Stat(path1) + if err != nil { + return false + } + + file2, err := os.Stat(path2) + if err != nil { + return false + } + + return os.SameFile(file1, file2) +} + +func (driver *Driver) Link(src, dest string) error { + if err := os.Remove(dest); err != nil && !os.IsNotExist(err) { + return err + } + + return driver.formatErr(os.Link(src, dest)) +} + +func (driver *Driver) formatErr(err error) error { + switch actual := err.(type) { //nolint: errorlint + case nil: + return nil + case storagedriver.PathNotFoundError: + actual.DriverName = driver.Name() + + return actual + case storagedriver.InvalidPathError: + actual.DriverName = driver.Name() + + return actual + case storagedriver.InvalidOffsetError: + actual.DriverName = driver.Name() + + return actual + default: + storageError := storagedriver.Error{ + DriverName: driver.Name(), + Enclosed: err, + } + + return storageError + } +} + +type fileInfo struct { + os.FileInfo + path string +} + +// asserts fileInfo implements storagedriver.FileInfo. +var _ storagedriver.FileInfo = fileInfo{} + +// Path provides the full path of the target of this file info. +func (fi fileInfo) Path() string { + return fi.path +} + +// Size returns current length in bytes of the file. The return value can +// be used to write to the end of the file at path. The value is +// meaningless if IsDir returns true. +func (fi fileInfo) Size() int64 { + if fi.IsDir() { + return 0 + } + + return fi.FileInfo.Size() +} + +// ModTime returns the modification time for the file. For backends that +// don't have a modification time, the creation time should be returned. +func (fi fileInfo) ModTime() time.Time { + return fi.FileInfo.ModTime() +} + +// IsDir returns true if the path is a directory. +func (fi fileInfo) IsDir() bool { + return fi.FileInfo.IsDir() +} + +type fileWriter struct { + file *os.File + size int64 + bw *bufio.Writer + closed bool + committed bool + cancelled bool + commit bool +} + +func newFileWriter(file *os.File, size int64, commit bool) *fileWriter { + return &fileWriter{ + file: file, + size: size, + commit: commit, + bw: bufio.NewWriter(file), + } +} + +func (fw *fileWriter) Write(buf []byte) (int, error) { + //nolint: gocritic + if fw.closed { + return 0, zerr.ErrFileAlreadyClosed + } else if fw.committed { + return 0, zerr.ErrFileAlreadyCommitted + } else if fw.cancelled { + return 0, zerr.ErrFileAlreadyCancelled + } + + n, err := fw.bw.Write(buf) + fw.size += int64(n) + + return n, err +} + +func (fw *fileWriter) Size() int64 { + return fw.size +} + +func (fw *fileWriter) Close() error { + if fw.closed { + return zerr.ErrFileAlreadyClosed + } + + if err := fw.bw.Flush(); err != nil { + return err + } + + if fw.commit { + if err := inject.Error(fw.file.Sync()); err != nil { + return err + } + } + + if err := inject.Error(fw.file.Close()); err != nil { + return err + } + + fw.closed = true + + return nil +} + +func (fw *fileWriter) Cancel() error { + if fw.closed { + return zerr.ErrFileAlreadyClosed + } + + fw.cancelled = true + fw.file.Close() + + return os.Remove(fw.file.Name()) +} + +func (fw *fileWriter) Commit() error { + //nolint: gocritic + if fw.closed { + return zerr.ErrFileAlreadyClosed + } else if fw.committed { + return zerr.ErrFileAlreadyCommitted + } else if fw.cancelled { + return zerr.ErrFileAlreadyCancelled + } + + if err := fw.bw.Flush(); err != nil { + return err + } + + if fw.commit { + if err := fw.file.Sync(); err != nil { + return err + } + } + + fw.committed = true + + return nil +} + +func ValidateHardLink(rootDir string) error { + if err := os.MkdirAll(rootDir, storageConstants.DefaultDirPerms); err != nil { + return err + } + + err := os.WriteFile(path.Join(rootDir, "hardlinkcheck.txt"), + []byte("check whether hardlinks work on filesystem"), storageConstants.DefaultFilePerms) + if err != nil { + return err + } + + err = os.Link(path.Join(rootDir, "hardlinkcheck.txt"), path.Join(rootDir, "duphardlinkcheck.txt")) + if err != nil { + // Remove hardlinkcheck.txt if hardlink fails + zerr := os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt")) + if zerr != nil { + return zerr + } + + return err + } + + err = os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt")) + if err != nil { + return err + } + + return os.RemoveAll(path.Join(rootDir, "duphardlinkcheck.txt")) +} diff --git a/pkg/storage/local/local.go b/pkg/storage/local/local.go index 6f8abde6..0492b727 100644 --- a/pkg/storage/local/local.go +++ b/pkg/storage/local/local.go @@ -1,1974 +1,35 @@ package local import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/fs" - "os" - "path" - "path/filepath" - "strings" - "sync" "time" - "unicode/utf8" - apexlog "github.com/apex/log" - guuid "github.com/gofrs/uuid" - "github.com/minio/sha256-simd" - notreg "github.com/notaryproject/notation-go/registry" - godigest "github.com/opencontainers/go-digest" - imeta "github.com/opencontainers/image-spec/specs-go" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/opencontainers/umoci" - "github.com/opencontainers/umoci/oci/casext" - oras "github.com/oras-project/artifacts-spec/specs-go/v1" - "github.com/rs/zerolog" - - zerr "zotregistry.io/zot/errors" - zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/monitoring" zlog "zotregistry.io/zot/pkg/log" - zreg "zotregistry.io/zot/pkg/regexp" - "zotregistry.io/zot/pkg/scheduler" "zotregistry.io/zot/pkg/storage/cache" common "zotregistry.io/zot/pkg/storage/common" - storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/storage/imagestore" storageTypes "zotregistry.io/zot/pkg/storage/types" - "zotregistry.io/zot/pkg/test/inject" ) -const ( - cosignSignatureTagSuffix = "sig" - SBOMTagSuffix = "sbom" -) - -// ImageStoreLocal provides the image storage operations. -type ImageStoreLocal struct { - rootDir string - lock *sync.RWMutex - cache cache.Cache - gc bool - dedupe bool - commit bool - gcDelay time.Duration - log zerolog.Logger - metrics monitoring.MetricServer - linter common.Lint -} - -func (is *ImageStoreLocal) RootDir() string { - return is.rootDir -} - -func (is *ImageStoreLocal) DirExists(d string) bool { - return zcommon.DirExists(d) -} - // NewImageStore returns a new image store backed by a file storage. // Use the last argument to properly set a cache database, or it will default to boltDB local storage. -func NewImageStore(rootDir string, gc bool, gcDelay time.Duration, dedupe, commit bool, +func NewImageStore(rootDir string, gc bool, gcReferrers bool, gcDelay time.Duration, + untaggedImageRetentionDelay time.Duration, dedupe, commit bool, log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, cacheDriver cache.Cache, ) storageTypes.ImageStore { - if _, err := os.Stat(rootDir); os.IsNotExist(err) { - if err := os.MkdirAll(rootDir, storageConstants.DefaultDirPerms); err != nil { - log.Error().Err(err).Str("rootDir", rootDir).Msg("unable to create root dir") - - return nil - } - } - - imgStore := &ImageStoreLocal{ - rootDir: rootDir, - lock: &sync.RWMutex{}, - gc: gc, - gcDelay: gcDelay, - dedupe: dedupe, - commit: commit, - log: log.With().Caller().Logger(), - metrics: metrics, - linter: linter, - } - - imgStore.cache = cacheDriver - - if gc { - // we use umoci GC to perform garbage-collection, but it uses its own logger - // - so capture those logs, could be useful - apexlog.SetLevel(apexlog.DebugLevel) - apexlog.SetHandler(apexlog.HandlerFunc(func(entry *apexlog.Entry) error { - e := log.Debug() - for k, v := range entry.Fields { - e = e.Interface(k, v) - } - e.Msg(entry.Message) - - return nil - })) - } - - return imgStore -} - -// RLock read-lock. -func (is *ImageStoreLocal) RLock(lockStart *time.Time) { - *lockStart = time.Now() - - is.lock.RLock() -} - -// RUnlock read-unlock. -func (is *ImageStoreLocal) RUnlock(lockStart *time.Time) { - is.lock.RUnlock() - - lockEnd := time.Now() - latency := lockEnd.Sub(*lockStart) - monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RLOCK) // histogram -} - -// Lock write-lock. -func (is *ImageStoreLocal) Lock(lockStart *time.Time) { - *lockStart = time.Now() - - is.lock.Lock() -} - -// Unlock write-unlock. -func (is *ImageStoreLocal) Unlock(lockStart *time.Time) { - is.lock.Unlock() - - lockEnd := time.Now() - latency := lockEnd.Sub(*lockStart) - monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RWLOCK) // histogram -} - -func (is *ImageStoreLocal) initRepo(name string) error { - repoDir := path.Join(is.rootDir, name) - - if !utf8.ValidString(name) { - is.log.Error().Msg("input is not valid UTF-8") - - return zerr.ErrInvalidRepositoryName - } - - if !zreg.FullNameRegexp.MatchString(name) { - is.log.Error().Str("repository", name).Msg("invalid repository name") - - return zerr.ErrInvalidRepositoryName - } - - // create "blobs" subdir - err := ensureDir(path.Join(repoDir, "blobs"), is.log) - if err != nil { - is.log.Error().Err(err).Msg("error creating blobs subdir") - - return err - } - // create BlobUploadDir subdir - err = ensureDir(path.Join(repoDir, storageConstants.BlobUploadDir), is.log) - if err != nil { - is.log.Error().Err(err).Msg("error creating blob upload subdir") - - return err - } - - // "oci-layout" file - create if it doesn't exist - ilPath := path.Join(repoDir, ispec.ImageLayoutFile) - if _, err := os.Stat(ilPath); err != nil { - il := ispec.ImageLayout{Version: ispec.ImageLayoutVersion} - - buf, err := json.Marshal(il) - if err != nil { - is.log.Panic().Err(err).Msg("unable to marshal JSON") - } - - if err := is.writeFile(ilPath, buf); err != nil { - is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") - - return err - } - } - - // "index.json" file - create if it doesn't exist - indexPath := path.Join(repoDir, "index.json") - if _, err := os.Stat(indexPath); err != nil { - index := ispec.Index{Versioned: imeta.Versioned{SchemaVersion: storageConstants.SchemaVersion}} - - buf, err := json.Marshal(index) - if err != nil { - is.log.Panic().Err(err).Msg("unable to marshal JSON") - } - - if err := is.writeFile(indexPath, buf); err != nil { - is.log.Error().Err(err).Str("file", indexPath).Msg("unable to write file") - - return err - } - } - - return nil -} - -// InitRepo creates an image repository under this store. -func (is *ImageStoreLocal) InitRepo(name string) error { - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - return is.initRepo(name) -} - -// ValidateRepo validates that the repository layout is complaint with the OCI repo layout. -func (is *ImageStoreLocal) ValidateRepo(name string) (bool, error) { - // https://github.com/opencontainers/image-spec/blob/master/image-layout.md#content - // at least, expect at least 3 entries - ["blobs", "oci-layout", "index.json"] - // and an additional/optional BlobUploadDir in each image store - if !zreg.FullNameRegexp.MatchString(name) { - return false, zerr.ErrInvalidRepositoryName - } - - dir := path.Join(is.rootDir, name) - if !is.DirExists(dir) { - return false, zerr.ErrRepoNotFound - } - - files, err := os.ReadDir(dir) - if err != nil { - is.log.Error().Err(err).Str("dir", dir).Msg("unable to read directory") - - return false, zerr.ErrRepoNotFound - } - - if len(files) < 3 { //nolint:gomnd - return false, zerr.ErrRepoBadVersion - } - - found := map[string]bool{ - "blobs": false, - ispec.ImageLayoutFile: false, - "index.json": false, - } - - for _, file := range files { - if file.Name() == "blobs" && !file.IsDir() { - return false, nil - } - - found[file.Name()] = true - } - - for k, v := range found { - if !v && k != storageConstants.BlobUploadDir { - return false, nil - } - } - - buf, err := os.ReadFile(path.Join(dir, ispec.ImageLayoutFile)) - if err != nil { - return false, err - } - - var il ispec.ImageLayout - if err := json.Unmarshal(buf, &il); err != nil { - return false, err - } - - if il.Version != ispec.ImageLayoutVersion { - return false, zerr.ErrRepoBadVersion - } - - return true, nil -} - -// GetRepositories returns a list of all the repositories under this store. -func (is *ImageStoreLocal) GetRepositories() ([]string, error) { - var lockLatency time.Time - - dir := is.rootDir - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - _, err := os.ReadDir(dir) - if err != nil { - is.log.Error().Err(err).Msg("failure walking storage root-dir") - - return nil, err - } - - stores := make([]string, 0) - err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if !info.IsDir() { - return nil - } - - rel, err := filepath.Rel(is.rootDir, path) - if err != nil { - return nil //nolint:nilerr // ignore paths not relative to root dir - } - - if ok, err := is.ValidateRepo(rel); !ok || err != nil { - return nil //nolint:nilerr // ignore invalid repos - } - - // is.log.Debug().Str("dir", path).Str("name", info.Name()).Msg("found image store") - stores = append(stores, rel) - - return nil - }) - - return stores, err -} - -// GetNextRepository returns next repository under this store. -func (is *ImageStoreLocal) GetNextRepository(repo string) (string, error) { - var lockLatency time.Time - - dir := is.rootDir - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - _, err := os.ReadDir(dir) - if err != nil { - is.log.Error().Err(err).Msg("failure walking storage root-dir") - - return "", err - } - - found := false - store := "" - err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - - if !info.IsDir() { - return nil - } - - rel, err := filepath.Rel(is.rootDir, path) - if err != nil { - return nil //nolint:nilerr // ignore paths not relative to root dir - } - - ok, err := is.ValidateRepo(rel) - if !ok || err != nil { - return nil //nolint:nilerr // ignore invalid repos - } - - if repo == "" && ok && err == nil { - store = rel - - return io.EOF - } - - if found { - store = rel - - return io.EOF - } - - if rel == repo { - found = true - } - - return nil - }) - - return store, err -} - -// GetImageTags returns a list of image tags available in the specified repository. -func (is *ImageStoreLocal) GetImageTags(repo string) ([]string, error) { - var lockLatency time.Time - - dir := path.Join(is.rootDir, repo) - if !is.DirExists(dir) { - return nil, zerr.ErrRepoNotFound - } - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return nil, err - } - - return common.GetTagsByIndex(index), nil -} - -// GetImageManifest returns the image manifest of an image in the specific repository. -func (is *ImageStoreLocal) GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) { - dir := path.Join(is.rootDir, repo) - if !is.DirExists(dir) { - return nil, "", "", zerr.ErrRepoNotFound - } - - var lockLatency time.Time - - var err error - - is.RLock(&lockLatency) - defer func() { - is.RUnlock(&lockLatency) - - if err == nil { - monitoring.IncDownloadCounter(is.metrics, repo) - } - }() - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return nil, "", "", err - } - - manifestDesc, found := common.GetManifestDescByReference(index, reference) - if !found { - return nil, "", "", zerr.ErrManifestNotFound - } - - buf, err := is.GetBlobContent(repo, manifestDesc.Digest) - if err != nil { - if errors.Is(err, zerr.ErrBlobNotFound) { - return nil, "", "", zerr.ErrManifestNotFound - } - - return nil, "", "", err - } - - var manifest ispec.Manifest - if err := json.Unmarshal(buf, &manifest); err != nil { - is.log.Error().Err(err).Str("dir", dir).Msg("invalid JSON") - - return nil, "", "", err - } - - return buf, manifestDesc.Digest, manifestDesc.MediaType, nil -} - -// PutImageManifest adds an image manifest to the repository. -func (is *ImageStoreLocal) PutImageManifest(repo, reference, mediaType string, //nolint: gocyclo - body []byte, -) (godigest.Digest, godigest.Digest, error) { - if err := is.InitRepo(repo); err != nil { - is.log.Debug().Err(err).Msg("init repo") - - return "", "", err - } - - var lockLatency time.Time - - var err error - - is.Lock(&lockLatency) - defer func() { - is.Unlock(&lockLatency) - - if err == nil { - monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) - monitoring.IncUploadCounter(is.metrics, repo) - } - }() - - refIsDigest := true - - mDigest, err := common.GetAndValidateRequestDigest(body, reference, is.log) - if err != nil { - if errors.Is(err, zerr.ErrBadManifest) { - return mDigest, "", err - } - - refIsDigest = false - } - - digest, err := common.ValidateManifest(is, repo, reference, mediaType, body, is.log) - if err != nil { - return digest, "", err - } - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return "", "", err - } - - // create a new descriptor - desc := ispec.Descriptor{ - MediaType: mediaType, Size: int64(len(body)), Digest: mDigest, - } - - if !refIsDigest { - desc.Annotations = map[string]string{ispec.AnnotationRefName: reference} - } - - var subjectDigest godigest.Digest - - artifactType := "" - - if mediaType == ispec.MediaTypeImageManifest { - var manifest ispec.Manifest - - err := json.Unmarshal(body, &manifest) - if err != nil { - return "", "", err - } - - if manifest.Subject != nil { - subjectDigest = manifest.Subject.Digest - } - - artifactType = zcommon.GetManifestArtifactType(manifest) - } else if mediaType == ispec.MediaTypeImageIndex { - var index ispec.Index - - err := json.Unmarshal(body, &index) - if err != nil { - return "", "", err - } - - if index.Subject != nil { - subjectDigest = index.Subject.Digest - } - - artifactType = zcommon.GetIndexArtifactType(index) - } - - updateIndex, oldDgst, err := common.CheckIfIndexNeedsUpdate(&index, &desc, is.log) - if err != nil { - return "", "", err - } - - if !updateIndex { - return desc.Digest, subjectDigest, nil - } - - // write manifest to "blobs" - dir := path.Join(is.rootDir, repo, "blobs", mDigest.Algorithm().String()) - _ = ensureDir(dir, is.log) - file := path.Join(dir, mDigest.Encoded()) - - // in case the linter will not pass, it will be garbage collected - if err := is.writeFile(file, body); err != nil { - is.log.Error().Err(err).Str("file", file).Msg("unable to write") - - return "", "", err - } - - err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) - if err != nil { - return "", "", err - } - - // now update "index.json" - index.Manifests = append(index.Manifests, desc) - dir = path.Join(is.rootDir, repo) - file = path.Join(dir, "index.json") - - buf, err := json.Marshal(index) - if err := inject.Error(err); err != nil { - is.log.Error().Err(err).Str("file", file).Msg("unable to marshal JSON") - - return "", "", err - } - - // update the descriptors artifact type in order to check for signatures when applying the linter - desc.ArtifactType = artifactType - - // apply linter only on images, not signatures or indexes - pass, err := common.ApplyLinter(is, is.linter, repo, desc) - if !pass { - is.log.Error().Err(err).Str("repository", repo).Str("reference", reference).Msg("linter didn't pass") - - return "", "", err - } - - err = is.writeFile(file, buf) - if err := inject.Error(err); err != nil { - is.log.Error().Err(err).Str("file", file).Msg("unable to write") - - return "", "", err - } - - return desc.Digest, subjectDigest, nil -} - -// DeleteImageManifest deletes the image manifest from the repository. -func (is *ImageStoreLocal) DeleteImageManifest(repo, reference string, detectCollision bool) error { - dir := path.Join(is.rootDir, repo) - if !is.DirExists(dir) { - return zerr.ErrRepoNotFound - } - - var lockLatency time.Time - - var err error - - is.Lock(&lockLatency) - defer func() { - is.Unlock(&lockLatency) - - if err == nil { - monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) - } - }() - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return err - } - - manifestDesc, err := common.RemoveManifestDescByReference(&index, reference, detectCollision) - if err != nil { - return err - } - - err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log) - if err != nil { - return err - } - - // now update "index.json" - dir = path.Join(is.rootDir, repo) - file := path.Join(dir, "index.json") - - buf, err := json.Marshal(index) - if err != nil { - return err - } - - if err := is.writeFile(file, buf); err != nil { - return err - } - - // Delete blob only when blob digest not present in manifest entry. - // e.g. 1.0.1 & 1.0.2 have same blob digest so if we delete 1.0.1, blob should not be removed. - toDelete := true - - for _, manifest := range index.Manifests { - if manifestDesc.Digest.String() == manifest.Digest.String() { - toDelete = false - - break - } - } - - if toDelete { - p := path.Join(dir, "blobs", manifestDesc.Digest.Algorithm().String(), manifestDesc.Digest.Encoded()) - - _ = os.Remove(p) - } - - return nil -} - -// BlobUploadPath returns the upload path for a blob in this store. -func (is *ImageStoreLocal) BlobUploadPath(repo, uuid string) string { - dir := path.Join(is.rootDir, repo) - blobUploadPath := path.Join(dir, storageConstants.BlobUploadDir, uuid) - - return blobUploadPath -} - -// NewBlobUpload returns the unique ID for an upload in progress. -func (is *ImageStoreLocal) NewBlobUpload(repo string) (string, error) { - if err := is.InitRepo(repo); err != nil { - is.log.Error().Err(err).Msg("error initializing repo") - - return "", err - } - - uuid, err := guuid.NewV4() - if err != nil { - return "", err - } - - uid := uuid.String() - - blobUploadPath := is.BlobUploadPath(repo, uid) - - file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, storageConstants.DefaultFilePerms) - if err != nil { - return "", zerr.ErrRepoNotFound - } - - defer file.Close() - - return uid, nil -} - -// GetBlobUpload returns the current size of a blob upload. -func (is *ImageStoreLocal) GetBlobUpload(repo, uuid string) (int64, error) { - blobUploadPath := is.BlobUploadPath(repo, uuid) - - if !utf8.ValidString(blobUploadPath) { - is.log.Error().Msg("input is not valid UTF-8") - - return -1, zerr.ErrInvalidRepositoryName - } - - binfo, err := os.Stat(blobUploadPath) - if err != nil { - if os.IsNotExist(err) { - return -1, zerr.ErrUploadNotFound - } - - return -1, err - } - - return binfo.Size(), nil -} - -// PutBlobChunkStreamed appends another chunk of data to the specified blob. It returns -// the number of actual bytes to the blob. -func (is *ImageStoreLocal) PutBlobChunkStreamed(repo, uuid string, body io.Reader) (int64, error) { - if err := is.InitRepo(repo); err != nil { - return -1, err - } - - blobUploadPath := is.BlobUploadPath(repo, uuid) - - _, err := os.Stat(blobUploadPath) - if err != nil { - return -1, zerr.ErrUploadNotFound - } - - file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, storageConstants.DefaultFilePerms) - if err != nil { - is.log.Error().Err(err).Msg("failed to open file") - - return -1, err - } - - defer func() { - if is.commit { - _ = file.Sync() - } - - _ = file.Close() - }() - - if _, err := file.Seek(0, io.SeekEnd); err != nil { - is.log.Error().Err(err).Msg("failed to seek file") - - return -1, err - } - - n, err := io.Copy(file, body) - - return n, err -} - -// PutBlobChunk writes another chunk of data to the specified blob. It returns -// the number of actual bytes to the blob. -func (is *ImageStoreLocal) PutBlobChunk(repo, uuid string, from, to int64, - body io.Reader, -) (int64, error) { - if err := is.InitRepo(repo); err != nil { - return -1, err - } - - blobUploadPath := is.BlobUploadPath(repo, uuid) - - binfo, err := os.Stat(blobUploadPath) - if err != nil { - return -1, zerr.ErrUploadNotFound - } - - if from != binfo.Size() { - is.log.Error().Int64("expected", from).Int64("actual", binfo.Size()). - Msg("invalid range start for blob upload") - - return -1, zerr.ErrBadUploadRange - } - - file, err := os.OpenFile(blobUploadPath, os.O_WRONLY|os.O_CREATE, storageConstants.DefaultFilePerms) - if err != nil { - is.log.Error().Err(err).Msg("failed to open file") - - return -1, err - } - - defer func() { - if is.commit { - _ = file.Sync() - } - - _ = file.Close() - }() - - if _, err := file.Seek(from, io.SeekStart); err != nil { - is.log.Error().Err(err).Msg("failed to seek file") - - return -1, err - } - - n, err := io.Copy(file, body) - - return n, err -} - -// BlobUploadInfo returns the current blob size in bytes. -func (is *ImageStoreLocal) BlobUploadInfo(repo, uuid string) (int64, error) { - blobUploadPath := is.BlobUploadPath(repo, uuid) - - binfo, err := os.Stat(blobUploadPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobUploadPath).Msg("failed to stat blob") - - return -1, err - } - - size := binfo.Size() - - return size, nil -} - -// FinishBlobUpload finalizes the blob upload and moves blob the repository. -func (is *ImageStoreLocal) FinishBlobUpload(repo, uuid string, body io.Reader, dstDigest godigest.Digest) error { - if err := dstDigest.Validate(); err != nil { - return err - } - - src := is.BlobUploadPath(repo, uuid) - - _, err := os.Stat(src) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to stat blob") - - return zerr.ErrUploadNotFound - } - - blobFile, err := os.Open(src) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") - - return zerr.ErrUploadNotFound - } - - defer blobFile.Close() - - digester := sha256.New() - - _, err = io.Copy(digester, blobFile) - if err != nil { - is.log.Error().Err(err).Str("repository", repo).Str("blob", src).Str("digest", dstDigest.String()). - Msg("unable to compute hash") - - return err - } - - srcDigest := godigest.NewDigestFromEncoded(godigest.SHA256, fmt.Sprintf("%x", digester.Sum(nil))) - - if srcDigest != dstDigest { - is.log.Error().Str("srcDigest", srcDigest.String()). - Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") - - return zerr.ErrBadBlobDigest - } - - dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) - - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - err = ensureDir(dir, is.log) - if err != nil { - is.log.Error().Err(err).Msg("error creating blobs/sha256 dir") - - return err - } - - dst := is.BlobPath(repo, dstDigest) - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - err = is.DedupeBlob(src, dstDigest, dst) - if err := inject.Error(err); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to dedupe blob") - - return err - } - } else { - if err := os.Rename(src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to finish blob") - - return err - } - } - - return nil -} - -// FullBlobUpload handles a full blob upload, and no partial session is created. -func (is *ImageStoreLocal) FullBlobUpload(repo string, body io.Reader, dstDigest godigest.Digest, -) (string, int64, error) { - if err := dstDigest.Validate(); err != nil { - return "", -1, err - } - - if err := is.InitRepo(repo); err != nil { - return "", -1, err - } - - u, err := guuid.NewV4() - if err != nil { - return "", -1, err - } - - uuid := u.String() - - src := is.BlobUploadPath(repo, uuid) - - blobFile, err := os.Create(src) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") - - return "", -1, zerr.ErrUploadNotFound - } - - defer func() { - if is.commit { - _ = blobFile.Sync() - } - - _ = blobFile.Close() - }() - - digester := sha256.New() - mw := io.MultiWriter(blobFile, digester) - - nbytes, err := io.Copy(mw, body) - if err != nil { - return "", -1, err - } - - srcDigest := godigest.NewDigestFromEncoded(godigest.SHA256, fmt.Sprintf("%x", digester.Sum(nil))) - if srcDigest != dstDigest { - is.log.Error().Str("srcDigest", srcDigest.String()). - Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") - - return "", -1, zerr.ErrBadBlobDigest - } - - dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) - - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - _ = ensureDir(dir, is.log) - dst := is.BlobPath(repo, dstDigest) - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - if err := is.DedupeBlob(src, dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to dedupe blob") - - return "", -1, err - } - } else { - if err := os.Rename(src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to finish blob") - - return "", -1, err - } - } - - return uuid, nbytes, nil -} - -func (is *ImageStoreLocal) DedupeBlob(src string, dstDigest godigest.Digest, dst string) error { -retry: - is.log.Debug().Str("src", src).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: enter") - - dstRecord, err := is.cache.GetBlob(dstDigest) - - if err != nil && !errors.Is(err, zerr.ErrCacheMiss) { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to lookup blob record") - - return err - } - - if dstRecord == "" { - // cache record doesn't exist, so first disk and cache entry for this diges - if err := is.cache.PutBlob(dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") - - return err - } - - // move the blob from uploads to final dest - if err := os.Rename(src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dst", dst).Msg("dedupe: unable to rename blob") - - return err - } - - is.log.Debug().Str("src", src).Str("dst", dst).Msg("dedupe: rename") - } else { - // cache record exists, but due to GC and upgrades from older versions, - // disk content and cache records may go out of sync - dstRecord = path.Join(is.rootDir, dstRecord) - - dstRecordFi, err := os.Stat(dstRecord) - if err != nil { - is.log.Warn().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat cache record, removing it") - // the actual blob on disk may have been removed by GC, so sync the cache - if err := is.cache.DeleteBlob(dstDigest, dstRecord); err != nil { - //nolint:lll // gofumpt conflicts with lll - is.log.Error().Err(err).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: unable to delete blob record") - - return err - } - - goto retry - } - - dstFi, err := os.Stat(dst) - if err != nil && !os.IsNotExist(err) { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat") - - return err - } - - if !os.SameFile(dstFi, dstRecordFi) { - // blob lookup cache out of sync with actual disk contents - if err := os.Remove(dst); err != nil && !os.IsNotExist(err) { - is.log.Error().Err(err).Str("dst", dst).Msg("dedupe: unable to remove blob") - - return err - } - - is.log.Debug().Str("blobPath", dst).Str("dstRecord", dstRecord).Msg("dedupe: creating hard link") - - if err := os.Link(dstRecord, dst); err != nil { - is.log.Error().Err(err).Str("blobPath", dst).Str("link", dstRecord).Msg("dedupe: unable to hard link") - - return err - } - } - - // also put dedupe blob in cache - if err := is.cache.PutBlob(dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") - - return err - } - - if err := os.Remove(src); err != nil { - is.log.Error().Err(err).Str("src", src).Msg("dedupe: uname to remove blob") - - return err - } - - is.log.Debug().Str("src", src).Msg("dedupe: remove") - } - - return nil -} - -// DeleteBlobUpload deletes an existing blob upload that is currently in progress. -func (is *ImageStoreLocal) DeleteBlobUpload(repo, uuid string) error { - blobUploadPath := is.BlobUploadPath(repo, uuid) - if err := os.Remove(blobUploadPath); err != nil { - is.log.Error().Err(err).Str("blobUploadPath", blobUploadPath).Msg("error deleting blob upload") - - return err - } - - return nil -} - -// BlobPath returns the repository path of a blob. -func (is *ImageStoreLocal) BlobPath(repo string, digest godigest.Digest) string { - return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded()) -} - -/* - CheckBlob verifies a blob and returns true if the blob is correct - -If the blob is not found but it's found in cache then it will be copied over. -*/ -func (is *ImageStoreLocal) CheckBlob(repo string, digest godigest.Digest) (bool, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return false, -1, err - } - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - } else { - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - } - - if ok, size, err := is.StatBlob(repo, digest); err == nil || ok { - return true, size, nil - } - - blobPath := is.BlobPath(repo, digest) - - is.log.Debug().Str("blob", blobPath).Msg("failed to find blob, searching it in cache") - - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - return false, -1, zerr.ErrBlobNotFound - } - - // If found copy to location - blobSize, err := is.copyBlob(repo, blobPath, dstRecord) - if err != nil { - return false, -1, zerr.ErrBlobNotFound - } - - if err := is.cache.PutBlob(digest, blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("dedupe: unable to insert blob record") - - return false, -1, err - } - - return true, blobSize, nil -} - -// StatBlob verifies if a blob is present inside a repository. The caller function SHOULD lock from outside. -func (is *ImageStoreLocal) StatBlob(repo string, digest godigest.Digest) (bool, int64, error) { - if err := digest.Validate(); err != nil { - return false, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - binfo, err := os.Stat(blobPath) - if err != nil { - is.log.Debug().Str("blob path", blobPath).Msg("failed to find blob") - - return false, -1, zerr.ErrBlobNotFound - } - - return true, binfo.Size(), nil -} - -func (is *ImageStoreLocal) checkCacheBlob(digest godigest.Digest) (string, error) { - if err := digest.Validate(); err != nil { - return "", err - } - - if !is.dedupe || fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { - return "", zerr.ErrBlobNotFound - } - - dstRecord, err := is.cache.GetBlob(digest) - if err != nil { - if errors.Is(err, zerr.ErrCacheMiss) { - is.log.Debug().Err(err).Str("digest", string(digest)).Msg("unable to find blob in cache") - } else { - is.log.Error().Err(err).Str("digest", string(digest)).Msg("unable to search blob in cache") - } - - return "", err - } - - dstRecord = path.Join(is.rootDir, dstRecord) - - if _, err := os.Stat(dstRecord); err != nil { - is.log.Warn().Err(err).Str("blob", dstRecord).Msg("unable to stat cache record, removing it") - - // the actual blob on disk may have been removed by GC, so sync the cache - if err := is.cache.DeleteBlob(digest, dstRecord); err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", dstRecord). - Msg("unable to remove blob path from cache") - - return "", err - } - - return "", zerr.ErrBlobNotFound - } - - is.log.Debug().Str("digest", digest.String()).Str("dstRecord", dstRecord).Msg("cache: found dedupe record") - - return dstRecord, nil -} - -func (is *ImageStoreLocal) copyBlob(repo, blobPath, dstRecord string) (int64, error) { - if err := is.initRepo(repo); err != nil { - is.log.Error().Err(err).Str("repository", repo).Msg("unable to initialize an empty repo") - - return -1, err - } - - _ = ensureDir(filepath.Dir(blobPath), is.log) - - if err := os.Link(dstRecord, blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Str("link", dstRecord).Msg("dedupe: unable to hard link") - - return -1, zerr.ErrBlobNotFound - } - - binfo, err := os.Stat(blobPath) - if err == nil { - return binfo.Size(), nil - } - - return -1, zerr.ErrBlobNotFound -} - -// blobStream is using to serve blob range requests. -type blobStream struct { - reader io.Reader - closer io.Closer -} - -func newBlobStream(blobPath string, from, to int64) (io.ReadCloser, error) { - blobFile, err := os.Open(blobPath) - if err != nil { - return nil, err - } - - if from > 0 { - _, err = blobFile.Seek(from, io.SeekStart) - if err != nil { - return nil, err - } - } - - if from < 0 || to < from { - return nil, zerr.ErrBadRange - } - - blobstrm := blobStream{reader: blobFile, closer: blobFile} - - blobstrm.reader = io.LimitReader(blobFile, to-from+1) - - return &blobstrm, nil -} - -func (bs *blobStream) Read(buf []byte) (int, error) { - return bs.reader.Read(buf) -} - -func (bs *blobStream) Close() error { - return bs.closer.Close() -} - -// GetBlobPartial returns a partial stream to read the blob. -// blob selector instead of directly downloading the blob. -func (is *ImageStoreLocal) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, -) (io.ReadCloser, int64, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return nil, -1, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - binfo, err := os.Stat(blobPath) - if err != nil { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return nil, -1, -1, zerr.ErrBlobNotFound - } - - if to < 0 || to >= binfo.Size() { - to = binfo.Size() - 1 - } - - blobReadCloser, err := newBlobStream(blobPath, from, to) - if err != nil { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("failed to open blob") - - return nil, -1, -1, err - } - - // The caller function is responsible for calling Close() - return blobReadCloser, to - from + 1, binfo.Size(), nil -} - -// GetBlob returns a stream to read the blob. -// blob selector instead of directly downloading the blob. -func (is *ImageStoreLocal) GetBlob(repo string, digest godigest.Digest, mediaType string, -) (io.ReadCloser, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return nil, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - binfo, err := os.Stat(blobPath) - if err != nil { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return nil, -1, zerr.ErrBlobNotFound - } - - blobReadCloser, err := os.Open(blobPath) - if err != nil { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("failed to open blob") - - return nil, -1, err - } - - // The caller function is responsible for calling Close() - return blobReadCloser, binfo.Size(), nil -} - -// GetBlobContent returns blob contents, SHOULD lock from outside. -func (is *ImageStoreLocal) GetBlobContent(repo string, digest godigest.Digest) ([]byte, error) { - if err := digest.Validate(); err != nil { - return []byte{}, err - } - - blobPath := is.BlobPath(repo, digest) - - blob, err := os.ReadFile(blobPath) - if err != nil { - if os.IsNotExist(err) { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("blob doesn't exist") - - return []byte{}, zerr.ErrBlobNotFound - } - - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to read blob") - - return []byte{}, err - } - - return blob, nil -} - -// GetIndexContent returns index.json contents, SHOULD lock from outside. -func (is *ImageStoreLocal) GetIndexContent(repo string) ([]byte, error) { - dir := path.Join(is.rootDir, repo) - - buf, err := os.ReadFile(path.Join(dir, "index.json")) - if err != nil { - if os.IsNotExist(err) { - is.log.Debug().Err(err).Str("dir", dir).Msg("index.json doesn't exist") - - return []byte{}, zerr.ErrRepoNotFound - } - - is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") - - return []byte{}, err - } - - return buf, nil -} - -// DeleteBlob removes the blob from the repository. -func (is *ImageStoreLocal) DeleteBlob(repo string, digest godigest.Digest) error { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return err - } - - blobPath := is.BlobPath(repo, digest) - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - _, err := os.Stat(blobPath) - if err != nil { - is.log.Debug().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return zerr.ErrBlobNotFound - } - - // first check if this blob is not currently in use - if ok, _ := common.IsBlobReferenced(is, repo, digest, is.log); ok { - return zerr.ErrBlobReferenced - } - - if fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - if err := is.cache.DeleteBlob(digest, blobPath); err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", blobPath). - Msg("unable to remove blob path from cache") - - return err - } - } - - if err := os.Remove(blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") - - return err - } - - return nil -} - -func (is *ImageStoreLocal) GetReferrers(repo string, gdigest godigest.Digest, artifactTypes []string, -) (ispec.Index, error) { - var lockLatency time.Time - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - return common.GetReferrers(is, repo, gdigest, artifactTypes, is.log) -} - -func (is *ImageStoreLocal) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string, -) ([]oras.Descriptor, error) { - var lockLatency time.Time - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - return common.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) -} - -func (is *ImageStoreLocal) writeFile(filename string, data []byte) error { - if !is.commit { - return os.WriteFile(filename, data, storageConstants.DefaultFilePerms) - } - - fhandle, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, storageConstants.DefaultFilePerms) - if err != nil { - return err - } - - _, err = fhandle.Write(data) - - if err1 := inject.Error(fhandle.Sync()); err1 != nil && err == nil { - err = err1 - is.log.Error().Err(err).Str("filename", filename).Msg("unable to sync file") - } - - if err1 := inject.Error(fhandle.Close()); err1 != nil && err == nil { - err = err1 - } - - return err -} - -func ValidateHardLink(rootDir string) error { - if err := os.MkdirAll(rootDir, storageConstants.DefaultDirPerms); err != nil { - return err - } - - err := os.WriteFile(path.Join(rootDir, "hardlinkcheck.txt"), - []byte("check whether hardlinks work on filesystem"), storageConstants.DefaultFilePerms) - if err != nil { - return err - } - - err = os.Link(path.Join(rootDir, "hardlinkcheck.txt"), path.Join(rootDir, "duphardlinkcheck.txt")) - if err != nil { - // Remove hardlinkcheck.txt if hardlink fails - zerr := os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt")) - if zerr != nil { - return zerr - } - - return err - } - - err = os.RemoveAll(path.Join(rootDir, "hardlinkcheck.txt")) - if err != nil { - return err - } - - return os.RemoveAll(path.Join(rootDir, "duphardlinkcheck.txt")) -} - -// utility routines. -func ensureDir(dir string, log zerolog.Logger) error { - if err := os.MkdirAll(dir, storageConstants.DefaultDirPerms); err != nil { - log.Error().Err(err).Str("dir", dir).Msg("unable to create dir") - - return err - } - - return nil -} - -type extendedManifest struct { - ispec.Manifest - - Digest godigest.Digest -} - -func (is *ImageStoreLocal) garbageCollect(dir string, repo string) error { - oci, err := umoci.OpenLayout(dir) - if err := inject.Error(err); err != nil { - return err - } - defer oci.Close() - - // gc untagged manifests and signatures - index, err := oci.GetIndex(context.Background()) - if err != nil { - return err - } - - referencedByImageIndex := []string{} - cosignDescriptors := []ispec.Descriptor{} - notationManifests := []extendedManifest{} - - /* gather manifests references by multiarch images (to skip gc) - gather cosign and notation signatures descriptors */ - for _, desc := range index.Manifests { - switch desc.MediaType { - case ispec.MediaTypeImageIndex: - indexImage, err := common.GetImageIndex(is, repo, desc.Digest, is.log) - if err != nil { - is.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read multiarch(index) image") - - return err - } - - for _, indexDesc := range indexImage.Manifests { - referencedByImageIndex = append(referencedByImageIndex, indexDesc.Digest.String()) - } - case ispec.MediaTypeImageManifest: - tag, ok := desc.Annotations[ispec.AnnotationRefName] - if ok { - // gather cosign references - if strings.HasPrefix(tag, "sha256-") && (strings.HasSuffix(tag, cosignSignatureTagSuffix) || - strings.HasSuffix(tag, SBOMTagSuffix)) { - cosignDescriptors = append(cosignDescriptors, desc) - - continue - } - } - - manifestContent, err := common.GetImageManifest(is, repo, desc.Digest, is.log) - if err != nil { - is.log.Error().Err(err).Str("repo", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read manifest image") - - return err - } - - if zcommon.GetManifestArtifactType(manifestContent) == notreg.ArtifactTypeNotation { - notationManifests = append(notationManifests, extendedManifest{ - Digest: desc.Digest, - Manifest: manifestContent, - }) - - continue - } - } - } - - is.log.Info().Msg("gc: untagged manifests") - - if err := gcUntaggedManifests(is, oci, &index, repo, referencedByImageIndex); err != nil { - return err - } - - is.log.Info().Msg("gc: cosign references") - - if err := gcCosignReferences(is, oci, &index, repo, cosignDescriptors); err != nil { - return err - } - - is.log.Info().Msg("gc: notation signatures") - - if err := gcNotationSignatures(is, oci, &index, repo, notationManifests); err != nil { - return err - } - - is.log.Info().Msg("gc: blobs") - - err = oci.GC(context.Background(), ifOlderThan(is, repo, is.gcDelay)) - if err := inject.Error(err); err != nil { - return err - } - - return nil -} - -func gcUntaggedManifests(imgStore *ImageStoreLocal, oci casext.Engine, index *ispec.Index, repo string, - referencedByImageIndex []string, -) error { - for _, desc := range index.Manifests { - // skip manifests referenced in image indexex - if zcommon.Contains(referencedByImageIndex, desc.Digest.String()) { - continue - } - - // remove untagged images - if desc.MediaType == ispec.MediaTypeImageManifest { - _, ok := desc.Annotations[ispec.AnnotationRefName] - if !ok { - // check if is indeed an image and not an artifact by checking it's config blob - buf, err := imgStore.GetBlobContent(repo, desc.Digest) - if err != nil { - imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: failed to read image manifest") - - return err - } - - manifest := ispec.Manifest{} - - err = json.Unmarshal(buf, &manifest) - if err != nil { - return err - } - - // skip manifests which are not of type image - if manifest.Config.MediaType != ispec.MediaTypeImageConfig { - imgStore.log.Info().Str("config mediaType", manifest.Config.MediaType). - Msg("skipping gc untagged manifest, because config blob is not application/vnd.oci.image.config.v1+json") - - continue - } - - // remove manifest if it's older than gc.delay - canGC, err := isBlobOlderThan(imgStore, repo, desc.Digest, imgStore.gcDelay) - if err != nil { - imgStore.log.Error().Err(err).Str("repository", repo).Str("digest", desc.Digest.String()). - Str("delay", imgStore.gcDelay.String()).Msg("gc: failed to check if blob is older than delay") - - return err - } - - if canGC { - imgStore.log.Info().Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: removing manifest without tag") - - _, err = common.RemoveManifestDescByReference(index, desc.Digest.String(), true) - if errors.Is(err, zerr.ErrManifestConflict) { - imgStore.log.Info().Str("repository", repo).Str("digest", desc.Digest.String()). - Msg("gc: skipping removing manifest due to conflict") - - continue - } - - err := oci.PutIndex(context.Background(), *index) - if err != nil { - return err - } - } - } - } - } - - return nil -} - -func gcCosignReferences(imgStore *ImageStoreLocal, oci casext.Engine, index *ispec.Index, repo string, - cosignDescriptors []ispec.Descriptor, -) error { - for _, cosignDesc := range cosignDescriptors { - foundSubject := false - // check if we can find the manifest which the reference points to - for _, desc := range index.Manifests { - // signature - subject := fmt.Sprintf("sha256-%s.%s", desc.Digest.Encoded(), cosignSignatureTagSuffix) - if subject == cosignDesc.Annotations[ispec.AnnotationRefName] { - foundSubject = true - } - - // sbom - subject = fmt.Sprintf("sha256-%s.%s", desc.Digest.Encoded(), SBOMTagSuffix) - if subject == cosignDesc.Annotations[ispec.AnnotationRefName] { - foundSubject = true - } - } - - if !foundSubject { - // remove manifest - imgStore.log.Info().Str("repository", repo).Str("digest", cosignDesc.Digest.String()). - Msg("gc: removing cosign reference without subject") - - // no need to check for manifest conflict, if one doesn't have a subject, then none with same digest will have - _, _ = common.RemoveManifestDescByReference(index, cosignDesc.Digest.String(), false) - - err := oci.PutIndex(context.Background(), *index) - if err != nil { - return err - } - } - } - - return nil -} - -func gcNotationSignatures(imgStore *ImageStoreLocal, oci casext.Engine, index *ispec.Index, repo string, - notationManifests []extendedManifest, -) error { - for _, notationManifest := range notationManifests { - foundSubject := false - - for _, desc := range index.Manifests { - if desc.Digest == notationManifest.Subject.Digest { - foundSubject = true - } - } - - if !foundSubject { - // remove manifest - imgStore.log.Info().Str("repository", repo).Str("digest", notationManifest.Digest.String()). - Msg("gc: removing notation signature without subject") - - // no need to check for manifest conflict, if one doesn't have a subject, then none with same digest will have - _, _ = common.RemoveManifestDescByReference(index, notationManifest.Digest.String(), false) - - err := oci.PutIndex(context.Background(), *index) - if err != nil { - return err - } - } - } - - return nil -} - -func ifOlderThan(imgStore *ImageStoreLocal, repo string, delay time.Duration) casext.GCPolicy { - return func(ctx context.Context, digest godigest.Digest) (bool, error) { - return isBlobOlderThan(imgStore, repo, digest, delay) - } -} - -func isBlobOlderThan(imgStore *ImageStoreLocal, repo string, digest godigest.Digest, delay time.Duration, -) (bool, error) { - blobPath := imgStore.BlobPath(repo, digest) - - fileInfo, err := os.Stat(blobPath) - if err != nil { - imgStore.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", blobPath). - Msg("gc: failed to stat blob") - - return false, err - } - - if fileInfo.ModTime().Add(delay).After(time.Now()) { - return false, nil - } - - imgStore.log.Info().Str("digest", digest.String()).Str("blobPath", blobPath).Msg("perform GC on blob") - - return true, nil -} - -func (is *ImageStoreLocal) gcRepo(repo string) error { - dir := path.Join(is.RootDir(), repo) - - var lockLatency time.Time - - is.Lock(&lockLatency) - err := is.garbageCollect(dir, repo) - is.Unlock(&lockLatency) - - if err != nil { - return err - } - - return nil -} - -func (is *ImageStoreLocal) RunGCRepo(repo string) error { - is.log.Info().Msg(fmt.Sprintf("executing GC of orphaned blobs for %s", path.Join(is.RootDir(), repo))) - - if err := is.gcRepo(repo); err != nil { - errMessage := fmt.Sprintf("error while running GC for %s", path.Join(is.RootDir(), repo)) - is.log.Error().Err(err).Msg(errMessage) - is.log.Info().Msg(fmt.Sprintf("GC unsuccessfully completed for %s", path.Join(is.RootDir(), repo))) - - return err - } - - is.log.Info().Msg(fmt.Sprintf("GC successfully completed for %s", path.Join(is.RootDir(), repo))) - - return nil -} - -func (is *ImageStoreLocal) RunGCPeriodically(interval time.Duration, sch *scheduler.Scheduler) { - generator := &common.GCTaskGenerator{ - ImgStore: is, - } - sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) -} - -func (is *ImageStoreLocal) GetNextDigestWithBlobPaths(lastDigests []godigest.Digest, -) (godigest.Digest, []string, error) { - var lockLatency time.Time - - dir := is.rootDir - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - var duplicateBlobs []string - - var digest godigest.Digest - - err := filepath.WalkDir(dir, func(path string, info fs.DirEntry, err error) error { - if err != nil { - is.log.Warn().Err(err).Msg("unable to walk dir, skipping it") - // skip files/dirs which can't be walked - return filepath.SkipDir - } - - if info.IsDir() { - return nil - } - - blobDigest := godigest.NewDigestFromEncoded("sha256", info.Name()) - if err := blobDigest.Validate(); err != nil { - return nil //nolint:nilerr // ignore files which are not blobs - } - - if digest == "" && !zcommon.Contains(lastDigests, blobDigest) { - digest = blobDigest - } - - if blobDigest == digest { - duplicateBlobs = append(duplicateBlobs, path) - } - - return nil - }) - - return digest, duplicateBlobs, err -} - -func (is *ImageStoreLocal) dedupeBlobs(digest godigest.Digest, duplicateBlobs []string) error { - if fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { - is.log.Error().Err(zerr.ErrDedupeRebuild).Msg("no cache driver found, can not dedupe blobs") - - return zerr.ErrDedupeRebuild - } - - is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest") - - var originalBlob string - - var originalBlobFi fs.FileInfo - - var err error - // rebuild from dedupe false to true - for _, blobPath := range duplicateBlobs { - /* for local storage, because we use hard links, we can assume that any blob can be original - so we skip the first one and hard link the rest of them with the first*/ - if originalBlob == "" { - originalBlob = blobPath - - originalBlobFi, err = os.Stat(originalBlob) - if err != nil { - is.log.Error().Err(err).Str("path", originalBlob).Msg("rebuild dedupe: failed to stat blob") - - return err - } - - // cache it - if ok := is.cache.HasBlob(digest, blobPath); !ok { - if err := is.cache.PutBlob(digest, blobPath); err != nil { - return err - } - } - - continue - } - - binfo, err := os.Stat(blobPath) - if err != nil { - is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") - - return err - } - - // dedupe blob - if !os.SameFile(originalBlobFi, binfo) { - // we should link to a temp file instead of removing blob and then linking - // to make this more atomic - uuid, err := guuid.NewV4() - if err != nil { - return err - } - - // put temp blob in /.uploads dir - tempLinkBlobDir := path.Join(strings.Replace(blobPath, path.Join("blobs/sha256", binfo.Name()), "", 1), - storageConstants.BlobUploadDir) - - if err := os.MkdirAll(tempLinkBlobDir, storageConstants.DefaultDirPerms); err != nil { - is.log.Error().Err(err).Str("dir", tempLinkBlobDir).Msg("rebuild dedupe: unable to mkdir") - - return err - } - - tempLinkBlobPath := path.Join(tempLinkBlobDir, uuid.String()) - - if err := os.Link(originalBlob, tempLinkBlobPath); err != nil { - is.log.Error().Err(err).Str("src", originalBlob). - Str("dst", tempLinkBlobPath).Msg("rebuild dedupe: unable to hard link") - - return err - } - - if err := os.Rename(tempLinkBlobPath, blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("rebuild dedupe: unable to rename temp link") - - return err - } - - // cache it - if ok := is.cache.HasBlob(digest, blobPath); !ok { - if err := is.cache.PutBlob(digest, blobPath); err != nil { - return err - } - } - } - } - - is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest finished successfully") - - return nil -} - -func (is *ImageStoreLocal) RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error { - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - if dedupe { - return is.dedupeBlobs(digest, duplicateBlobs) - } - - // otherwise noop - return nil -} - -func (is *ImageStoreLocal) RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) { - // for local storage no need to undedupe blobs - if is.dedupe { - generator := &common.DedupeTaskGenerator{ - ImgStore: is, - Dedupe: is.dedupe, - Log: is.log, - } - - sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) - } + return imagestore.NewImageStore( + rootDir, + rootDir, + gc, + gcReferrers, + gcDelay, + untaggedImageRetentionDelay, + dedupe, + commit, + log, + metrics, + linter, + New(commit), + cacheDriver, + ) } diff --git a/pkg/storage/local/local_elevated_test.go b/pkg/storage/local/local_elevated_test.go index 07568921..d7921aa4 100644 --- a/pkg/storage/local/local_elevated_test.go +++ b/pkg/storage/local/local_elevated_test.go @@ -36,8 +36,8 @@ func TestElevatedPrivilegesInvalidDedupe(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, - metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) upload, err := imgStore.NewBlobUpload("dedupe1") So(err, ShouldBeNil) diff --git a/pkg/storage/local/local_test.go b/pkg/storage/local/local_test.go index 58be419b..c504f54a 100644 --- a/pkg/storage/local/local_test.go +++ b/pkg/storage/local/local_test.go @@ -37,7 +37,6 @@ import ( "zotregistry.io/zot/pkg/storage/local" storageTypes "zotregistry.io/zot/pkg/storage/types" "zotregistry.io/zot/pkg/test" - "zotregistry.io/zot/pkg/test/inject" "zotregistry.io/zot/pkg/test/mocks" ) @@ -68,8 +67,9 @@ func TestStorageFSAPIs(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, - true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) Convey("Repo layout", t, func(c C) { Convey("Bad image manifest", func() { @@ -205,7 +205,9 @@ func TestGetOrasReferrers(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) Convey("Get referrers", t, func(c C) { err := test.WriteImageToFileSystem(test.CreateDefaultVulnerableImage(), "zot-test", "0.0.1", storage.StoreController{ @@ -263,8 +265,9 @@ func FuzzNewBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, - cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) _, err := imgStore.NewBlobUpload(data) if err != nil { @@ -289,8 +292,8 @@ func FuzzPutBlobChunk(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) repoName := data uuid, err := imgStore.NewBlobUpload(repoName) @@ -323,8 +326,8 @@ func FuzzPutBlobChunkStreamed(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) repoName := data @@ -356,7 +359,8 @@ func FuzzGetBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) _, err := imgStore.GetBlobUpload(data1, data2) @@ -382,8 +386,8 @@ func FuzzTestPutGetImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) cblob, cdigest := test.GetRandomImageConfig() @@ -434,8 +438,8 @@ func FuzzTestPutDeleteImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) cblob, cdigest := test.GetRandomImageConfig() @@ -493,8 +497,8 @@ func FuzzTestDeleteImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) digest, _, err := newRandomBlobForFuzz(data) if err != nil { @@ -529,8 +533,8 @@ func FuzzInitRepo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) err := imgStore.InitRepo(data) if err != nil { if isKnownErr(err) { @@ -554,8 +558,8 @@ func FuzzInitValidateRepo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) err := imgStore.InitRepo(data) if err != nil { if isKnownErr(err) { @@ -586,8 +590,8 @@ func FuzzGetImageTags(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) _, err := imgStore.GetImageTags(data) if err != nil { if errors.Is(err, zerr.ErrRepoNotFound) || isKnownErr(err) { @@ -611,8 +615,8 @@ func FuzzBlobUploadPath(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) _ = imgStore.BlobUploadPath(repo, uuid) }) @@ -631,8 +635,8 @@ func FuzzBlobUploadInfo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) repo := data _, err := imgStore.BlobUploadInfo(repo, uuid) @@ -657,8 +661,8 @@ func FuzzTestGetImageManifest(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) repoName := data @@ -686,8 +690,8 @@ func FuzzFinishBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) repoName := data @@ -736,8 +740,8 @@ func FuzzFullBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) ldigest, lblob, err := newRandomBlobForFuzz(data) if err != nil { @@ -767,7 +771,9 @@ func TestStorageCacheErrors(t *testing.T) { cblob, cdigest := test.GetRandomImageConfig() getBlobPath := "" - imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, + + imgStore := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, &mocks.CacheMock{ PutBlobFn: func(digest godigest.Digest, path string) error { if strings.Contains(path, dedupedRepo) { @@ -790,7 +796,7 @@ func TestStorageCacheErrors(t *testing.T) { _, _, err = imgStore.FullBlobUpload(originRepo, bytes.NewReader(cblob), cdigest) So(err, ShouldBeNil) - getBlobPath = strings.ReplaceAll(imgStore.BlobPath(originRepo, cdigest), imgStore.RootDir(), "") + getBlobPath = imgStore.BlobPath(originRepo, cdigest) _, _, err = imgStore.FullBlobUpload(dedupedRepo, bytes.NewReader(cblob), cdigest) So(err, ShouldNotBeNil) }) @@ -809,8 +815,8 @@ func FuzzDedupeBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) blobDigest := godigest.FromString(data) @@ -851,8 +857,8 @@ func FuzzDeleteBlobUpload(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) uuid, err := imgStore.NewBlobUpload(repoName) if err != nil { @@ -883,8 +889,8 @@ func FuzzBlobPath(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) digest := godigest.FromString(data) _ = imgStore.BlobPath(repoName, digest) @@ -905,8 +911,8 @@ func FuzzCheckBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -937,8 +943,8 @@ func FuzzGetBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -976,8 +982,8 @@ func FuzzDeleteBlob(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -1012,8 +1018,8 @@ func FuzzGetIndexContent(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -1048,8 +1054,8 @@ func FuzzGetBlobContent(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) digest := godigest.FromString(data) _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader([]byte(data)), digest) @@ -1083,8 +1089,9 @@ func FuzzGetOrasReferrers(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) storageCtlr := storage.StoreController{DefaultStore: imgStore} err := test.WriteImageToFileSystem(test.CreateDefaultVulnerableImage(), "zot-test", "0.0.1", storageCtlr) @@ -1145,8 +1152,8 @@ func FuzzRunGCRepo(f *testing.F) { Name: "cache", UseRelPaths: true, }, *log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, *log, metrics, nil, - cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, *log, metrics, nil, cacheDriver) if err := imgStore.RunGCRepo(data); err != nil { t.Error(err) @@ -1185,11 +1192,11 @@ func TestDedupeLinks(t *testing.T) { var imgStore storageTypes.ImageStore if testCase.dedupe { - imgStore = local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - testCase.dedupe, true, log, metrics, nil, cacheDriver) + imgStore = local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, testCase.dedupe, true, log, metrics, nil, cacheDriver) } else { - imgStore = local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - testCase.dedupe, true, log, metrics, nil, nil) + imgStore = local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, testCase.dedupe, true, log, metrics, nil, nil) } // manifest1 @@ -1338,8 +1345,8 @@ func TestDedupeLinks(t *testing.T) { Convey("test RunDedupeForDigest directly, trigger stat error on original blob", func() { // rebuild with dedupe true - imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) duplicateBlobs := []string{ path.Join(dir, "dedupe1", "blobs", "sha256", blobDigest1), @@ -1358,8 +1365,8 @@ func TestDedupeLinks(t *testing.T) { for i := 0; i < 10; i++ { taskScheduler, cancel := runAndGetScheduler() // rebuild with dedupe true - imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) sleepValue := i * 50 @@ -1371,8 +1378,8 @@ func TestDedupeLinks(t *testing.T) { taskScheduler, cancel := runAndGetScheduler() // rebuild with dedupe true - imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) // wait until rebuild finishes @@ -1391,8 +1398,8 @@ func TestDedupeLinks(t *testing.T) { // switch dedupe to true from false taskScheduler, cancel := runAndGetScheduler() - imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, nil) + imgStore := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, nil) // rebuild with dedupe true imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler) @@ -1414,7 +1421,8 @@ func TestDedupeLinks(t *testing.T) { // switch dedupe to true from false taskScheduler, cancel := runAndGetScheduler() - imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, + imgStore := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, &mocks.CacheMock{ HasBlobFn: func(digest godigest.Digest, path string) bool { return false @@ -1443,7 +1451,8 @@ func TestDedupeLinks(t *testing.T) { // switch dedupe to true from false taskScheduler, cancel := runAndGetScheduler() - imgStore := local.NewImageStore(dir, false, storageConstants.DefaultGCDelay, + imgStore := local.NewImageStore(dir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, &mocks.CacheMock{ HasBlobFn: func(digest godigest.Digest, path string) bool { return false @@ -1469,7 +1478,6 @@ func TestDedupeLinks(t *testing.T) { fi2, err := os.Stat(path.Join(dir, "dedupe2", "blobs", "sha256", blobDigest2)) So(err, ShouldBeNil) - // deduped happened, but didn't cached So(os.SameFile(fi1, fi2), ShouldEqual, true) }) } @@ -1539,7 +1547,9 @@ func TestDedupe(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - il := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, true, log, metrics, nil, cacheDriver) + + il := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) So(il.DedupeBlob("", "", ""), ShouldNotBeNil) }) @@ -1558,7 +1568,9 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - So(local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, + + So(local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver), ShouldNotBeNil) if os.Geteuid() != 0 { cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ @@ -1566,8 +1578,9 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - So(local.NewImageStore("/deadBEEF", true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver), ShouldBeNil) + So(local.NewImageStore("/deadBEEF", true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, + true, log, metrics, nil, cacheDriver), ShouldBeNil) } }) @@ -1581,8 +1594,9 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) err := os.Chmod(dir, 0o000) // remove all perms if err != nil { @@ -1631,8 +1645,9 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, - true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -1719,9 +1734,8 @@ func TestNegativeCases(t *testing.T) { panic(err) } - if os.Geteuid() != 0 { - So(func() { _, _ = imgStore.ValidateRepo("test") }, ShouldPanic) - } + _, err = imgStore.GetRepositories() + So(err, ShouldNotBeNil) err = os.Chmod(dir, 0o755) // add perms if err != nil { @@ -1732,16 +1746,9 @@ func TestNegativeCases(t *testing.T) { if err != nil { panic(err) } - - _, err = imgStore.GetRepositories() - So(err, ShouldNotBeNil) }) Convey("Invalid get image tags", t, func(c C) { - var ilfs local.ImageStoreLocal - _, err := ilfs.GetImageTags("test") - So(err, ShouldNotBeNil) - dir := t.TempDir() log := log.Logger{Logger: zerolog.New(os.Stdout)} @@ -1751,13 +1758,14 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) So(os.Remove(path.Join(dir, "test", "index.json")), ShouldBeNil) - _, err = imgStore.GetImageTags("test") + _, err := imgStore.GetImageTags("test") So(err, ShouldNotBeNil) So(os.RemoveAll(path.Join(dir, "test")), ShouldBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -1767,10 +1775,6 @@ func TestNegativeCases(t *testing.T) { }) Convey("Invalid get image manifest", t, func(c C) { - var ilfs local.ImageStoreLocal - _, _, _, err := ilfs.GetImageManifest("test", "") - So(err, ShouldNotBeNil) - dir := t.TempDir() log := log.Logger{Logger: zerolog.New(os.Stdout)} @@ -1780,13 +1784,14 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, - true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) - err = os.Chmod(path.Join(dir, "test", "index.json"), 0o000) + err := os.Chmod(path.Join(dir, "test", "index.json"), 0o000) if err != nil { panic(err) } @@ -1827,8 +1832,9 @@ func TestNegativeCases(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) So(imgStore, ShouldNotBeNil) So(imgStore.InitRepo("test"), ShouldBeNil) @@ -1988,42 +1994,6 @@ func TestHardLink(t *testing.T) { } func TestInjectWriteFile(t *testing.T) { - Convey("writeFile with commit", t, func() { - dir := t.TempDir() - - log := log.Logger{Logger: zerolog.New(os.Stdout)} - metrics := monitoring.NewMetricsServer(false, log) - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) - - Convey("Failure path1", func() { - injected := inject.InjectFailure(0) - - err := imgStore.InitRepo("repo1") - if injected { - So(err, ShouldNotBeNil) - } else { - So(err, ShouldBeNil) - } - }) - - Convey("Failure path2", func() { - injected := inject.InjectFailure(1) - - err := imgStore.InitRepo("repo2") - if injected { - So(err, ShouldNotBeNil) - } else { - So(err, ShouldBeNil) - } - }) - }) - Convey("writeFile without commit", t, func() { dir := t.TempDir() @@ -2034,8 +2004,9 @@ func TestInjectWriteFile(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, false, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, false, log, metrics, nil, cacheDriver) Convey("Failure path not reached", func() { err := imgStore.InitRepo("repo1") @@ -2044,498 +2015,6 @@ func TestInjectWriteFile(t *testing.T) { }) } -func TestGCInjectFailure(t *testing.T) { - Convey("code coverage: error inside garbageCollect method of img store", t, func() { - dir := t.TempDir() - logFile, _ := os.CreateTemp("", "zot-log*.txt") - - defer os.Remove(logFile.Name()) // clean up - - log := log.NewLogger("debug", logFile.Name()) - metrics := monitoring.NewMetricsServer(false, log) - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) - repoName := "test-gc" - - upload, err := imgStore.NewBlobUpload(repoName) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - content := []byte("test-data1") - buf := bytes.NewBuffer(content) - buflen := buf.Len() - bdigest := godigest.FromBytes(content) - - blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repoName, upload, buf, bdigest) - So(err, ShouldBeNil) - - annotationsMap := make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest := test.GetRandomImageConfig() - _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err := imgStore.CheckBlob(repoName, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest := ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - - manifest.SchemaVersion = 2 - manifestBuf, err := json.Marshal(manifest) - So(err, ShouldBeNil) - - _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) - - // umoci.OpenLayout error - injected := inject.InjectFailure(0) - - err = imgStore.RunGCRepo(repoName) - - if injected { - So(err, ShouldNotBeNil) - } else { - So(err, ShouldBeNil) - } - - // oci.GC - injected = inject.InjectFailure(1) - - err = imgStore.RunGCRepo(repoName) - - if injected { - So(err, ShouldNotBeNil) - } else { - So(err, ShouldBeNil) - } - }) -} - -func TestGarbageCollect(t *testing.T) { - Convey("Repo layout", t, func(c C) { - dir := t.TempDir() - - log := log.Logger{Logger: zerolog.New(os.Stdout)} - metrics := monitoring.NewMetricsServer(false, log) - - Convey("Garbage collect with default/long delay", func() { - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) - repoName := "gc-long" - - upload, err := imgStore.NewBlobUpload(repoName) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - content := []byte("test-data1") - buf := bytes.NewBuffer(content) - buflen := buf.Len() - bdigest := godigest.FromBytes(content) - - blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repoName, upload, buf, bdigest) - So(err, ShouldBeNil) - - annotationsMap := make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest := test.GetRandomImageConfig() - _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err := imgStore.CheckBlob(repoName, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest := ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - - manifest.SchemaVersion = 2 - manifestBuf, err := json.Marshal(manifest) - So(err, ShouldBeNil) - digest := godigest.FromBytes(manifestBuf) - - _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) - - err = imgStore.RunGCRepo(repoName) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - err = imgStore.DeleteImageManifest(repoName, digest.String(), false) - So(err, ShouldBeNil) - - err = imgStore.RunGCRepo(repoName) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - }) - - Convey("Garbage collect with short delay", func() { - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil, cacheDriver) - repoName := "gc-short" - - // upload orphan blob - upload, err := imgStore.NewBlobUpload(repoName) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - content := []byte("test-data1") - buf := bytes.NewBuffer(content) - buflen := buf.Len() - odigest := godigest.FromBytes(content) - - blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repoName, upload, buf, odigest) - So(err, ShouldBeNil) - - // sleep so orphan blob can be GC'ed - time.Sleep(5 * time.Second) - - // upload blob - upload, err = imgStore.NewBlobUpload(repoName) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - content = []byte("test-data2") - buf = bytes.NewBuffer(content) - buflen = buf.Len() - bdigest := godigest.FromBytes(content) - - blob, err = imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repoName, upload, buf, bdigest) - So(err, ShouldBeNil) - - annotationsMap := make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest := test.GetRandomImageConfig() - _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err := imgStore.CheckBlob(repoName, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest := ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - - manifest.SchemaVersion = 2 - manifestBuf, err := json.Marshal(manifest) - So(err, ShouldBeNil) - digest := godigest.FromBytes(manifestBuf) - - _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) - - err = imgStore.RunGCRepo(repoName) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repoName, odigest) - So(err, ShouldNotBeNil) - So(hasBlob, ShouldEqual, false) - - hasBlob, _, err = imgStore.StatBlob(repoName, odigest) - So(err, ShouldNotBeNil) - So(hasBlob, ShouldEqual, false) - - hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - hasBlob, _, err = imgStore.StatBlob(repoName, bdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - // sleep so orphan blob can be GC'ed - time.Sleep(5 * time.Second) - - err = imgStore.DeleteImageManifest(repoName, digest.String(), false) - So(err, ShouldBeNil) - - err = imgStore.RunGCRepo(repoName) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) - So(err, ShouldNotBeNil) - So(hasBlob, ShouldEqual, false) - }) - - Convey("Garbage collect with dedupe", func() { - // garbage-collect is repo-local and dedupe is global and they can interact in strange ways - cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ - RootDir: dir, - Name: "cache", - UseRelPaths: true, - }, log) - imgStore := local.NewImageStore(dir, true, 5*time.Second, true, true, log, metrics, nil, cacheDriver) - - // first upload an image to the first repo and wait for GC timeout - - repo1Name := "gc1" - - // upload blob - upload, err := imgStore.NewBlobUpload(repo1Name) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - content := []byte("test-data") - buf := bytes.NewBuffer(content) - buflen := buf.Len() - bdigest := godigest.FromBytes(content) - tdigest := bdigest - - blob, err := imgStore.PutBlobChunk(repo1Name, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repo1Name, upload, buf, bdigest) - So(err, ShouldBeNil) - - annotationsMap := make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest := test.GetRandomImageConfig() - _, clen, err := imgStore.FullBlobUpload(repo1Name, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err := imgStore.CheckBlob(repo1Name, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest := ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - - manifest.SchemaVersion = 2 - manifestBuf, err := json.Marshal(manifest) - So(err, ShouldBeNil) - - _, _, err = imgStore.PutImageManifest(repo1Name, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - // sleep so past GC timeout - time.Sleep(10 * time.Second) - - hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - // upload another image into a second repo with the same blob contents so dedupe is triggered - - repo2Name := "gc2" - - upload, err = imgStore.NewBlobUpload(repo2Name) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - buf = bytes.NewBuffer(content) - buflen = buf.Len() - - blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest) - So(err, ShouldBeNil) - - annotationsMap = make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest = test.GetRandomImageConfig() - _, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest = ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - - manifest.SchemaVersion = 2 - manifestBuf, err = json.Marshal(manifest) - So(err, ShouldBeNil) - - _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) - - hasBlob, _, err = imgStore.CheckBlob(repo2Name, bdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - // immediately upload any other image to second repo and run GC, but expect layers to persist - - upload, err = imgStore.NewBlobUpload(repo2Name) - So(err, ShouldBeNil) - So(upload, ShouldNotBeEmpty) - - content = []byte("test-data-more") - buf = bytes.NewBuffer(content) - buflen = buf.Len() - bdigest = godigest.FromBytes(content) - - blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf) - So(err, ShouldBeNil) - So(blob, ShouldEqual, buflen) - - err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest) - So(err, ShouldBeNil) - - annotationsMap = make(map[string]string) - annotationsMap[ispec.AnnotationRefName] = tag - - cblob, cdigest = test.GetRandomImageConfig() - _, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest) - So(err, ShouldBeNil) - So(clen, ShouldEqual, len(cblob)) - hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - manifest = ispec.Manifest{ - Config: ispec.Descriptor{ - MediaType: "application/vnd.oci.image.config.v1+json", - Digest: cdigest, - Size: int64(len(cblob)), - }, - Layers: []ispec.Descriptor{ - { - MediaType: "application/vnd.oci.image.layer.v1.tar", - Digest: bdigest, - Size: int64(buflen), - }, - }, - Annotations: annotationsMap, - } - - manifest.SchemaVersion = 2 - manifestBuf, err = json.Marshal(manifest) - So(err, ShouldBeNil) - digest := godigest.FromBytes(manifestBuf) - - _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) - So(err, ShouldBeNil) - - err = imgStore.RunGCRepo(repo2Name) - So(err, ShouldBeNil) - - // original blob should exist - - hasBlob, _, err = imgStore.CheckBlob(repo2Name, tdigest) - So(err, ShouldBeNil) - So(hasBlob, ShouldEqual, true) - - _, _, _, err = imgStore.GetImageManifest(repo2Name, digest.String()) - So(err, ShouldBeNil) - }) - }) -} - func TestGarbageCollectForImageStore(t *testing.T) { Convey("Garbage collect for a specific repo from an ImageStore", t, func(c C) { dir := t.TempDir() @@ -2552,7 +2031,9 @@ func TestGarbageCollectForImageStore(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, 1*time.Second, storageConstants.DefaultUntaggedImgeRetentionDelay, + true, true, log, metrics, nil, cacheDriver) repoName := "gc-all-repos-short" image := test.CreateDefaultVulnerableImage() @@ -2590,7 +2071,9 @@ func TestGarbageCollectForImageStore(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, 1*time.Second, storageConstants.DefaultUntaggedImgeRetentionDelay, + true, true, log, metrics, nil, cacheDriver) repoName := "gc-all-repos-short" image := test.CreateDefaultVulnerableImage() @@ -2625,7 +2108,8 @@ func TestGarbageCollectForImageStore(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, 1*time.Second, true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, 1*time.Second, storageConstants.DefaultUntaggedImgeRetentionDelay, + true, true, log, metrics, nil, cacheDriver) repoName := "gc-sig" storeController := storage.StoreController{DefaultStore: imgStore} @@ -2684,7 +2168,9 @@ func TestGarbageCollectErrors(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, 500*time.Millisecond, true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, 500*time.Millisecond, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) repoName := "gc-index" // create a blob/layer @@ -2919,8 +2405,9 @@ func TestInitRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) err := os.Mkdir(path.Join(dir, "test-dir"), 0o000) So(err, ShouldBeNil) @@ -2941,8 +2428,9 @@ func TestValidateRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) err := os.Mkdir(path.Join(dir, "test-dir"), 0o000) So(err, ShouldBeNil) @@ -2961,8 +2449,9 @@ func TestValidateRepo(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) _, err := imgStore.ValidateRepo(".") So(err, ShouldNotBeNil) @@ -3006,8 +2495,9 @@ func TestGetRepositories(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver, + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver, ) // Create valid directory with permissions @@ -3103,8 +2593,8 @@ func TestGetRepositories(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver, + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver, ) // Root dir does not contain repos @@ -3151,7 +2641,8 @@ func TestGetRepositories(t *testing.T) { UseRelPaths: true, }, log) - imgStore := local.NewImageStore(rootDir, true, storageConstants.DefaultGCDelay, + imgStore := local.NewImageStore(rootDir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver, ) @@ -3194,8 +2685,9 @@ func TestGetNextRepository(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver, + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver, ) firstRepoName := "repo1" secondRepoName := "repo2" @@ -3218,15 +2710,13 @@ func TestGetNextRepository(t *testing.T) { Convey("Return first repository", t, func() { firstRepo, err := imgStore.GetNextRepository("") So(firstRepo, ShouldEqual, firstRepoName) - So(err, ShouldNotBeNil) - So(err, ShouldEqual, io.EOF) + So(err, ShouldBeNil) }) Convey("Return second repository", t, func() { secondRepo, err := imgStore.GetNextRepository(firstRepoName) So(secondRepo, ShouldEqual, secondRepoName) - So(err, ShouldNotBeNil) - So(err, ShouldEqual, io.EOF) + So(err, ShouldBeNil) }) Convey("Return error", t, func() { @@ -3250,8 +2740,9 @@ func TestPutBlobChunkStreamed(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) uuid, err := imgStore.NewBlobUpload("test") So(err, ShouldBeNil) @@ -3279,8 +2770,9 @@ func TestPullRange(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) repoName := "pull-range" upload, err := imgStore.NewBlobUpload(repoName) @@ -3317,6 +2809,100 @@ func TestPullRange(t *testing.T) { }) } +func TestStorageDriverErr(t *testing.T) { + dir := t.TempDir() + + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) + + Convey("Init repo", t, func() { + err := imgStore.InitRepo(repoName) + So(err, ShouldBeNil) + + Convey("New blob upload error", func() { + err := os.Chmod(path.Join(imgStore.RootDir(), repoName, storageConstants.BlobUploadDir), 0o000) + So(err, ShouldBeNil) + + _, err = imgStore.NewBlobUpload(repoName) + So(err, ShouldNotBeNil) + + err = os.Chmod(path.Join(imgStore.RootDir(), repoName, storageConstants.BlobUploadDir), + storageConstants.DefaultDirPerms) + So(err, ShouldBeNil) + + uuid, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + + size, err := imgStore.GetBlobUpload(repoName, uuid) + So(err, ShouldBeNil) + So(size, ShouldEqual, 0) + + content := []byte("test-blob") + buf := bytes.NewBuffer(content) + bufLen := buf.Len() + digest := godigest.FromBytes(content) + + size, err = imgStore.PutBlobChunkStreamed(repoName, uuid, buf) + So(err, ShouldBeNil) + So(size, ShouldEqual, bufLen) + + size, err = imgStore.GetBlobUpload(repoName, uuid) + So(err, ShouldBeNil) + So(size, ShouldEqual, bufLen) + + err = imgStore.DeleteBlobUpload(repoName, uuid) + So(err, ShouldBeNil) + + err = imgStore.DeleteBlobUpload(repoName, uuid) + So(err, ShouldNotBeNil) + + _, err = imgStore.GetBlobUpload(repoName, uuid) + So(err, ShouldNotBeNil) + + // push again + buf = bytes.NewBuffer(content) + + uuid, err = imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + + size, err = imgStore.PutBlobChunkStreamed(repoName, uuid, buf) + So(err, ShouldBeNil) + So(size, ShouldEqual, bufLen) + + // finish blob upload + err = os.Chmod(path.Join(imgStore.BlobUploadPath(repoName, uuid)), 0o000) + So(err, ShouldBeNil) + + err = imgStore.FinishBlobUpload(repoName, uuid, &io.LimitedReader{}, digest) + So(err, ShouldNotBeNil) + + err = os.Chmod(path.Join(imgStore.BlobUploadPath(repoName, uuid)), storageConstants.DefaultFilePerms) + So(err, ShouldBeNil) + + err = imgStore.FinishBlobUpload(repoName, uuid, &io.LimitedReader{}, digest) + So(err, ShouldBeNil) + + err = imgStore.FinishBlobUpload(repoName, uuid, &io.LimitedReader{}, digest) + So(err, ShouldNotBeNil) + + // delete blob + err = imgStore.DeleteBlob(repoName, digest) + So(err, ShouldBeNil) + + err = imgStore.DeleteBlob(repoName, digest) + So(err, ShouldNotBeNil) + }) + }) +} + func NewRandomImgManifest(data []byte, cdigest, ldigest godigest.Digest, cblob, lblob []byte) (*ispec.Manifest, error) { annotationsMap := make(map[string]string) diff --git a/pkg/storage/s3/driver.go b/pkg/storage/s3/driver.go new file mode 100644 index 00000000..19b0c860 --- /dev/null +++ b/pkg/storage/s3/driver.go @@ -0,0 +1,115 @@ +package s3 + +import ( + "context" + "io" + + // Add s3 support. + "github.com/docker/distribution/registry/storage/driver" + _ "github.com/docker/distribution/registry/storage/driver/s3-aws" + + storageConstants "zotregistry.io/zot/pkg/storage/constants" +) + +type Driver struct { + store driver.StorageDriver +} + +func New(storeDriver driver.StorageDriver) *Driver { + return &Driver{store: storeDriver} +} + +func (driver *Driver) Name() string { + return storageConstants.S3StorageDriverName +} + +func (driver *Driver) EnsureDir(path string) error { + return nil +} + +func (driver *Driver) DirExists(path string) bool { + if fi, err := driver.store.Stat(context.Background(), path); err == nil && fi.IsDir() { + return true + } + + return false +} + +func (driver *Driver) Reader(path string, offset int64) (io.ReadCloser, error) { + return driver.store.Reader(context.Background(), path, offset) +} + +func (driver *Driver) ReadFile(path string) ([]byte, error) { + return driver.store.GetContent(context.Background(), path) +} + +func (driver *Driver) Delete(path string) error { + return driver.store.Delete(context.Background(), path) +} + +func (driver *Driver) Stat(path string) (driver.FileInfo, error) { + return driver.store.Stat(context.Background(), path) +} + +func (driver *Driver) Writer(filepath string, append bool) (driver.FileWriter, error) { //nolint:predeclared + return driver.store.Writer(context.Background(), filepath, append) +} + +func (driver *Driver) WriteFile(filepath string, content []byte) (int, error) { + var n int + + if stwr, err := driver.store.Writer(context.Background(), filepath, false); err == nil { + defer stwr.Close() + + if n, err = stwr.Write(content); err != nil { + return -1, err + } + + if err := stwr.Commit(); err != nil { + return -1, err + } + } else { + return -1, err + } + + return n, nil +} + +func (driver *Driver) Walk(path string, f driver.WalkFn) error { + return driver.store.Walk(context.Background(), path, f) +} + +func (driver *Driver) List(fullpath string) ([]string, error) { + return driver.store.List(context.Background(), fullpath) +} + +func (driver *Driver) Move(sourcePath string, destPath string) error { + return driver.store.Move(context.Background(), sourcePath, destPath) +} + +func (driver *Driver) SameFile(path1, path2 string) bool { + fi1, _ := driver.store.Stat(context.Background(), path1) + + fi2, _ := driver.store.Stat(context.Background(), path2) + + if fi1 != nil && fi2 != nil { + if fi1.IsDir() == fi2.IsDir() && + fi1.ModTime() == fi2.ModTime() && + fi1.Path() == fi2.Path() && + fi1.Size() == fi2.Size() { + return true + } + } + + return false +} + +/* + Link put an empty file that will act like a link between the original file and deduped one + +because s3 doesn't support symlinks, wherever the storage will encounter an empty file, it will get the original one +from cache. +*/ +func (driver *Driver) Link(src, dest string) error { + return driver.store.PutContent(context.Background(), dest, []byte{}) +} diff --git a/pkg/storage/s3/s3.go b/pkg/storage/s3/s3.go index b02e5d2d..16f7050b 100644 --- a/pkg/storage/s3/s3.go +++ b/pkg/storage/s3/s3.go @@ -1,1729 +1,41 @@ package s3 import ( - "bytes" - "context" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "path" - "path/filepath" - "sync" "time" // Add s3 support. "github.com/docker/distribution/registry/storage/driver" // Load s3 driver. _ "github.com/docker/distribution/registry/storage/driver/s3-aws" - guuid "github.com/gofrs/uuid" - godigest "github.com/opencontainers/go-digest" - ispec "github.com/opencontainers/image-spec/specs-go/v1" - artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" - "github.com/rs/zerolog" - zerr "zotregistry.io/zot/errors" - zcommon "zotregistry.io/zot/pkg/common" "zotregistry.io/zot/pkg/extensions/monitoring" zlog "zotregistry.io/zot/pkg/log" - zreg "zotregistry.io/zot/pkg/regexp" - "zotregistry.io/zot/pkg/scheduler" "zotregistry.io/zot/pkg/storage/cache" common "zotregistry.io/zot/pkg/storage/common" - storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/storage/imagestore" storageTypes "zotregistry.io/zot/pkg/storage/types" - "zotregistry.io/zot/pkg/test/inject" ) -const ( - CacheDBName = "s3_cache" -) - -// ObjectStorage provides the image storage operations. -type ObjectStorage struct { - rootDir string - store driver.StorageDriver - lock *sync.RWMutex - log zerolog.Logger - metrics monitoring.MetricServer - cache cache.Cache - dedupe bool - linter common.Lint -} - -func (is *ObjectStorage) RootDir() string { - return is.rootDir -} - -func (is *ObjectStorage) DirExists(d string) bool { - if fi, err := is.store.Stat(context.Background(), d); err == nil && fi.IsDir() { - return true - } - - return false -} - // NewObjectStorage returns a new image store backed by cloud storages. // see https://github.com/docker/docker.github.io/tree/master/registry/storage-drivers // Use the last argument to properly set a cache database, or it will default to boltDB local storage. -func NewImageStore(rootDir string, cacheDir string, gc bool, gcDelay time.Duration, dedupe, commit bool, - log zlog.Logger, metrics monitoring.MetricServer, linter common.Lint, - store driver.StorageDriver, cacheDriver cache.Cache, +func NewImageStore(rootDir string, cacheDir string, gc bool, gcReferrers bool, gcDelay time.Duration, + untaggedImageRetentionDelay time.Duration, dedupe, commit bool, log zlog.Logger, metrics monitoring.MetricServer, + linter common.Lint, store driver.StorageDriver, cacheDriver cache.Cache, ) storageTypes.ImageStore { - imgStore := &ObjectStorage{ - rootDir: rootDir, - store: store, - lock: &sync.RWMutex{}, - log: log.With().Caller().Logger(), - metrics: metrics, - dedupe: dedupe, - linter: linter, - } - - imgStore.cache = cacheDriver - - return imgStore -} - -// RLock read-lock. -func (is *ObjectStorage) RLock(lockStart *time.Time) { - *lockStart = time.Now() - - is.lock.RLock() -} - -// RUnlock read-unlock. -func (is *ObjectStorage) RUnlock(lockStart *time.Time) { - is.lock.RUnlock() - - lockEnd := time.Now() - // includes time spent in acquiring and holding a lock - latency := lockEnd.Sub(*lockStart) - monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RLOCK) // histogram -} - -// Lock write-lock. -func (is *ObjectStorage) Lock(lockStart *time.Time) { - *lockStart = time.Now() - - is.lock.Lock() -} - -// Unlock write-unlock. -func (is *ObjectStorage) Unlock(lockStart *time.Time) { - is.lock.Unlock() - - lockEnd := time.Now() - // includes time spent in acquiring and holding a lock - latency := lockEnd.Sub(*lockStart) - monitoring.ObserveStorageLockLatency(is.metrics, latency, is.RootDir(), storageConstants.RWLOCK) // histogram -} - -func (is *ObjectStorage) initRepo(name string) error { - repoDir := path.Join(is.rootDir, name) - - if !zreg.FullNameRegexp.MatchString(name) { - is.log.Error().Str("repository", name).Msg("invalid repository name") - - return zerr.ErrInvalidRepositoryName - } - - // "oci-layout" file - create if it doesn't exist - ilPath := path.Join(repoDir, ispec.ImageLayoutFile) - if _, err := is.store.Stat(context.Background(), ilPath); err != nil { - il := ispec.ImageLayout{Version: ispec.ImageLayoutVersion} - - buf, err := json.Marshal(il) - if err != nil { - is.log.Error().Err(err).Msg("unable to marshal JSON") - - return err - } - - if _, err := writeFile(is.store, ilPath, buf); err != nil { - is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") - - return err - } - } - - // "index.json" file - create if it doesn't exist - indexPath := path.Join(repoDir, "index.json") - if _, err := is.store.Stat(context.Background(), indexPath); err != nil { - index := ispec.Index{} - index.SchemaVersion = 2 - - buf, err := json.Marshal(index) - if err != nil { - is.log.Error().Err(err).Msg("unable to marshal JSON") - - return err - } - - if _, err := writeFile(is.store, indexPath, buf); err != nil { - is.log.Error().Err(err).Str("file", ilPath).Msg("unable to write file") - - return err - } - } - - return nil -} - -// InitRepo creates an image repository under this store. -func (is *ObjectStorage) InitRepo(name string) error { - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - return is.initRepo(name) -} - -// ValidateRepo validates that the repository layout is complaint with the OCI repo layout. -func (is *ObjectStorage) ValidateRepo(name string) (bool, error) { - if !zreg.FullNameRegexp.MatchString(name) { - return false, zerr.ErrInvalidRepositoryName - } - - // https://github.com/opencontainers/image-spec/blob/master/image-layout.md#content - // at least, expect at least 3 entries - ["blobs", "oci-layout", "index.json"] - // and an additional/optional BlobUploadDir in each image store - // for objects storage we can not create empty dirs, so we check only against index.json and oci-layout - dir := path.Join(is.rootDir, name) - if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() { - return false, zerr.ErrRepoNotFound - } - - files, err := is.store.List(context.Background(), dir) - if err != nil { - is.log.Error().Err(err).Str("dir", dir).Msg("unable to read directory") - - return false, zerr.ErrRepoNotFound - } - - //nolint:gomnd - if len(files) < 2 { - return false, zerr.ErrRepoBadVersion - } - - found := map[string]bool{ - ispec.ImageLayoutFile: false, - "index.json": false, - } - - for _, file := range files { - _, err := is.store.Stat(context.Background(), file) - if err != nil { - return false, err - } - - filename, err := filepath.Rel(dir, file) - if err != nil { - return false, err - } - - found[filename] = true - } - - for k, v := range found { - if !v && k != storageConstants.BlobUploadDir { - return false, nil - } - } - - buf, err := is.store.GetContent(context.Background(), path.Join(dir, ispec.ImageLayoutFile)) - if err != nil { - return false, err - } - - var il ispec.ImageLayout - if err := json.Unmarshal(buf, &il); err != nil { - return false, err - } - - if il.Version != ispec.ImageLayoutVersion { - return false, zerr.ErrRepoBadVersion - } - - return true, nil -} - -// GetRepositories returns a list of all the repositories under this store. -func (is *ObjectStorage) GetRepositories() ([]string, error) { - var lockLatency time.Time - - dir := is.rootDir - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - stores := make([]string, 0) - err := is.store.Walk(context.Background(), dir, func(fileInfo driver.FileInfo) error { - if !fileInfo.IsDir() { - return nil - } - - rel, err := filepath.Rel(is.rootDir, fileInfo.Path()) - if err != nil { - return nil //nolint:nilerr // ignore paths that are not under root dir - } - - if ok, err := is.ValidateRepo(rel); !ok || err != nil { - return nil //nolint:nilerr // ignore invalid repos - } - - stores = append(stores, rel) - - return nil - }) - - // if the root directory is not yet created then return an empty slice of repositories - var perr driver.PathNotFoundError - if errors.As(err, &perr) { - return stores, nil - } - - return stores, err -} - -// GetNextRepository returns next repository under this store. -func (is *ObjectStorage) GetNextRepository(repo string) (string, error) { - return "", nil -} - -// GetImageTags returns a list of image tags available in the specified repository. -func (is *ObjectStorage) GetImageTags(repo string) ([]string, error) { - var lockLatency time.Time - - dir := path.Join(is.rootDir, repo) - if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() { - return nil, zerr.ErrRepoNotFound - } - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return nil, err - } - - return common.GetTagsByIndex(index), nil -} - -// GetImageManifest returns the image manifest of an image in the specific repository. -func (is *ObjectStorage) GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) { - dir := path.Join(is.rootDir, repo) - if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() { - return nil, "", "", zerr.ErrRepoNotFound - } - - var lockLatency time.Time - - var err error - - is.RLock(&lockLatency) - defer func() { - is.RUnlock(&lockLatency) - - if err == nil { - monitoring.IncDownloadCounter(is.metrics, repo) - } - }() - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return nil, "", "", zerr.ErrRepoNotFound - } - - manifestDesc, found := common.GetManifestDescByReference(index, reference) - if !found { - return nil, "", "", zerr.ErrManifestNotFound - } - - buf, err := is.GetBlobContent(repo, manifestDesc.Digest) - if err != nil { - if errors.Is(err, zerr.ErrBlobNotFound) { - return nil, "", "", zerr.ErrManifestNotFound - } - - return nil, "", "", err - } - - var manifest ispec.Manifest - if err := json.Unmarshal(buf, &manifest); err != nil { - is.log.Error().Err(err).Str("dir", dir).Msg("invalid JSON") - - return nil, "", "", err - } - - return buf, manifestDesc.Digest, manifestDesc.MediaType, nil -} - -// PutImageManifest adds an image manifest to the repository. -func (is *ObjectStorage) PutImageManifest(repo, reference, mediaType string, //nolint: gocyclo - body []byte, -) (godigest.Digest, godigest.Digest, error) { - if err := is.InitRepo(repo); err != nil { - is.log.Debug().Err(err).Msg("init repo") - - return "", "", err - } - - var lockLatency time.Time - - var err error - - is.Lock(&lockLatency) - defer func() { - is.Unlock(&lockLatency) - - if err == nil { - monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) - monitoring.IncUploadCounter(is.metrics, repo) - } - }() - - refIsDigest := true - - mDigest, err := common.GetAndValidateRequestDigest(body, reference, is.log) - if err != nil { - if errors.Is(err, zerr.ErrBadManifest) { - return mDigest, "", err - } - - refIsDigest = false - } - - dig, err := common.ValidateManifest(is, repo, reference, mediaType, body, is.log) - if err != nil { - return dig, "", err - } - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return "", "", err - } - - // create a new descriptor - desc := ispec.Descriptor{ - MediaType: mediaType, Size: int64(len(body)), Digest: mDigest, - } - - if !refIsDigest { - desc.Annotations = map[string]string{ispec.AnnotationRefName: reference} - } - - var subjectDigest godigest.Digest - - artifactType := "" - - if mediaType == ispec.MediaTypeImageManifest { - var manifest ispec.Manifest - - err := json.Unmarshal(body, &manifest) - if err != nil { - return "", "", err - } - - if manifest.Subject != nil { - subjectDigest = manifest.Subject.Digest - } - - artifactType = zcommon.GetManifestArtifactType(manifest) - } else if mediaType == ispec.MediaTypeImageIndex { - var index ispec.Index - - err := json.Unmarshal(body, &index) - if err != nil { - return "", "", err - } - - if index.Subject != nil { - subjectDigest = index.Subject.Digest - } - - artifactType = zcommon.GetIndexArtifactType(index) - } - - updateIndex, oldDgst, err := common.CheckIfIndexNeedsUpdate(&index, &desc, is.log) - if err != nil { - return "", "", err - } - - if !updateIndex { - return desc.Digest, subjectDigest, nil - } - - // write manifest to "blobs" - dir := path.Join(is.rootDir, repo, "blobs", mDigest.Algorithm().String()) - manifestPath := path.Join(dir, mDigest.Encoded()) - - if err = is.store.PutContent(context.Background(), manifestPath, body); err != nil { - is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write") - - return "", "", err - } - - err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, desc, oldDgst, is.log) - if err != nil { - return "", "", err - } - - // now update "index.json" - index.Manifests = append(index.Manifests, desc) - dir = path.Join(is.rootDir, repo) - indexPath := path.Join(dir, "index.json") - - buf, err := json.Marshal(index) - if err != nil { - is.log.Error().Err(err).Str("file", indexPath).Msg("unable to marshal JSON") - - return "", "", err - } - - // update the descriptors artifact type in order to check for signatures when applying the linter - desc.ArtifactType = artifactType - - // apply linter only on images, not signatures - pass, err := common.ApplyLinter(is, is.linter, repo, desc) - if !pass { - is.log.Error().Err(err).Str("repository", repo).Str("reference", reference).Msg("linter didn't pass") - - return "", "", err - } - - if err = is.store.PutContent(context.Background(), indexPath, buf); err != nil { - is.log.Error().Err(err).Str("file", manifestPath).Msg("unable to write") - - return "", "", err - } - - return desc.Digest, subjectDigest, nil -} - -// DeleteImageManifest deletes the image manifest from the repository. -func (is *ObjectStorage) DeleteImageManifest(repo, reference string, detectCollisions bool) error { - dir := path.Join(is.rootDir, repo) - if fi, err := is.store.Stat(context.Background(), dir); err != nil || !fi.IsDir() { - return zerr.ErrRepoNotFound - } - - var lockLatency time.Time - - var err error - - is.Lock(&lockLatency) - defer func() { - is.Unlock(&lockLatency) - - if err == nil { - monitoring.SetStorageUsage(is.metrics, is.rootDir, repo) - } - }() - - index, err := common.GetIndex(is, repo, is.log) - if err != nil { - return err - } - - manifestDesc, err := common.RemoveManifestDescByReference(&index, reference, detectCollisions) - if err != nil { - return err - } - - err = common.UpdateIndexWithPrunedImageManifests(is, &index, repo, manifestDesc, manifestDesc.Digest, is.log) - if err != nil { - return err - } - - // now update "index.json" - dir = path.Join(is.rootDir, repo) - file := path.Join(dir, "index.json") - - buf, err := json.Marshal(index) - if err != nil { - return err - } - - if _, err := writeFile(is.store, file, buf); err != nil { - is.log.Debug().Str("deleting reference", reference).Msg("") - - return err - } - - // Delete blob only when blob digest not present in manifest entry. - // e.g. 1.0.1 & 1.0.2 have same blob digest so if we delete 1.0.1, blob should not be removed. - toDelete := true - - for _, manifest := range index.Manifests { - if manifestDesc.Digest.String() == manifest.Digest.String() { - toDelete = false - - break - } - } - - if toDelete { - p := path.Join(dir, "blobs", manifestDesc.Digest.Algorithm().String(), manifestDesc.Digest.Encoded()) - - err = is.store.Delete(context.Background(), p) - if err != nil { - return err - } - } - - return nil -} - -// BlobUploadPath returns the upload path for a blob in this store. -func (is *ObjectStorage) BlobUploadPath(repo, uuid string) string { - dir := path.Join(is.rootDir, repo) - blobUploadPath := path.Join(dir, storageConstants.BlobUploadDir, uuid) - - return blobUploadPath -} - -// NewBlobUpload returns the unique ID for an upload in progress. -func (is *ObjectStorage) NewBlobUpload(repo string) (string, error) { - if err := is.InitRepo(repo); err != nil { - is.log.Error().Err(err).Msg("error initializing repo") - - return "", err - } - - uuid, err := guuid.NewV4() - if err != nil { - return "", err - } - - uid := uuid.String() - - blobUploadPath := is.BlobUploadPath(repo, uid) - - // create multipart upload (append false) - _, err = is.store.Writer(context.Background(), blobUploadPath, false) - if err != nil { - return "", err - } - - return uid, nil -} - -// GetBlobUpload returns the current size of a blob upload. -func (is *ObjectStorage) GetBlobUpload(repo, uuid string) (int64, error) { - blobUploadPath := is.BlobUploadPath(repo, uuid) - - writer, err := is.store.Writer(context.Background(), blobUploadPath, true) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - return -1, zerr.ErrUploadNotFound - } - - return -1, err - } - - return writer.Size(), nil -} - -// PutBlobChunkStreamed appends another chunk of data to the specified blob. It returns -// the number of actual bytes to the blob. -func (is *ObjectStorage) PutBlobChunkStreamed(repo, uuid string, body io.Reader) (int64, error) { - if err := is.InitRepo(repo); err != nil { - return -1, err - } - - blobUploadPath := is.BlobUploadPath(repo, uuid) - - file, err := is.store.Writer(context.Background(), blobUploadPath, true) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - return -1, zerr.ErrUploadNotFound - } - - is.log.Error().Err(err).Msg("failed to continue multipart upload") - - return -1, err - } - - defer file.Close() - - buf := new(bytes.Buffer) - - _, err = buf.ReadFrom(body) - if err != nil { - is.log.Error().Err(err).Msg("failed to read blob") - - return -1, err - } - - nbytes, err := file.Write(buf.Bytes()) - if err != nil { - is.log.Error().Err(err).Msg("failed to append to file") - - return -1, err - } - - return int64(nbytes), err -} - -// PutBlobChunk writes another chunk of data to the specified blob. It returns -// the number of actual bytes to the blob. -func (is *ObjectStorage) PutBlobChunk(repo, uuid string, from, to int64, - body io.Reader, -) (int64, error) { - if err := is.InitRepo(repo); err != nil { - return -1, err - } - - blobUploadPath := is.BlobUploadPath(repo, uuid) - - file, err := is.store.Writer(context.Background(), blobUploadPath, true) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - return -1, zerr.ErrUploadNotFound - } - - is.log.Error().Err(err).Msg("failed to continue multipart upload") - - return -1, err - } - - defer file.Close() - - if from != file.Size() { - is.log.Error().Int64("expected", from).Int64("actual", file.Size()). - Msg("invalid range start for blob upload") - - return -1, zerr.ErrBadUploadRange - } - - buf := new(bytes.Buffer) - - _, err = buf.ReadFrom(body) - if err != nil { - is.log.Error().Err(err).Msg("failed to read blob") - - return -1, err - } - - nbytes, err := file.Write(buf.Bytes()) - if err != nil { - is.log.Error().Err(err).Msg("failed to append to file") - - return -1, err - } - - return int64(nbytes), err -} - -// BlobUploadInfo returns the current blob size in bytes. -func (is *ObjectStorage) BlobUploadInfo(repo, uuid string) (int64, error) { - blobUploadPath := is.BlobUploadPath(repo, uuid) - - writer, err := is.store.Writer(context.Background(), blobUploadPath, true) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - return -1, zerr.ErrUploadNotFound - } - - return -1, err - } - - return writer.Size(), nil -} - -// FinishBlobUpload finalizes the blob upload and moves blob the repository. -func (is *ObjectStorage) FinishBlobUpload(repo, uuid string, body io.Reader, dstDigest godigest.Digest) error { - if err := dstDigest.Validate(); err != nil { - return err - } - - src := is.BlobUploadPath(repo, uuid) - - // complete multiUploadPart - fileWriter, err := is.store.Writer(context.Background(), src, true) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") - - return zerr.ErrBadBlobDigest - } - - if err := fileWriter.Commit(); err != nil { - is.log.Error().Err(err).Msg("failed to commit file") - - return err - } - - if err := fileWriter.Close(); err != nil { - is.log.Error().Err(err).Msg("failed to close file") - - return err - } - - fileReader, err := is.store.Reader(context.Background(), src, 0) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to open file") - - return zerr.ErrUploadNotFound - } - - defer fileReader.Close() - - srcDigest, err := godigest.FromReader(fileReader) - if err != nil { - is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") - - return zerr.ErrBadBlobDigest - } - - if srcDigest != dstDigest { - is.log.Error().Str("srcDigest", srcDigest.String()). - Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") - - return zerr.ErrBadBlobDigest - } - - dst := is.BlobPath(repo, dstDigest) - - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - if err := is.DedupeBlob(src, dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to dedupe blob") - - return err - } - } else { - if err := is.store.Move(context.Background(), src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to finish blob") - - return err - } - } - - return nil -} - -// FullBlobUpload handles a full blob upload, and no partial session is created. -func (is *ObjectStorage) FullBlobUpload(repo string, body io.Reader, dstDigest godigest.Digest) (string, int64, error) { - if err := dstDigest.Validate(); err != nil { - return "", -1, err - } - - if err := is.InitRepo(repo); err != nil { - return "", -1, err - } - - u, err := guuid.NewV4() - if err != nil { - return "", -1, err - } - - uuid := u.String() - src := is.BlobUploadPath(repo, uuid) - digester := sha256.New() - buf := new(bytes.Buffer) - - _, err = buf.ReadFrom(body) - if err != nil { - is.log.Error().Err(err).Msg("failed to read blob") - - return "", -1, err - } - - nbytes, err := writeFile(is.store, src, buf.Bytes()) - if err != nil { - is.log.Error().Err(err).Msg("failed to write blob") - - return "", -1, err - } - - _, err = digester.Write(buf.Bytes()) - if err != nil { - is.log.Error().Err(err).Msg("digester failed to write") - - return "", -1, err - } - - srcDigest := godigest.NewDigestFromEncoded(godigest.SHA256, fmt.Sprintf("%x", digester.Sum(nil))) - if srcDigest != dstDigest { - is.log.Error().Str("srcDigest", srcDigest.String()). - Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") - - return "", -1, zerr.ErrBadBlobDigest - } - - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - dst := is.BlobPath(repo, dstDigest) - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - if err := is.DedupeBlob(src, dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to dedupe blob") - - return "", -1, err - } - } else { - if err := is.store.Move(context.Background(), src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dstDigest", dstDigest.String()). - Str("dst", dst).Msg("unable to finish blob") - - return "", -1, err - } - } - - return uuid, int64(nbytes), nil -} - -func (is *ObjectStorage) DedupeBlob(src string, dstDigest godigest.Digest, dst string) error { -retry: - is.log.Debug().Str("src", src).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: enter") - - dstRecord, err := is.cache.GetBlob(dstDigest) - if err := inject.Error(err); err != nil && !errors.Is(err, zerr.ErrCacheMiss) { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to lookup blob record") - - return err - } - - if dstRecord == "" { - // cache record doesn't exist, so first disk and cache entry for this digest - if err := is.cache.PutBlob(dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") - - return err - } - - // move the blob from uploads to final dest - if err := is.store.Move(context.Background(), src, dst); err != nil { - is.log.Error().Err(err).Str("src", src).Str("dst", dst).Msg("dedupe: unable to rename blob") - - return err - } - - is.log.Debug().Str("src", src).Str("dst", dst).Msg("dedupe: rename") - } else { - // cache record exists, but due to GC and upgrades from older versions, - // disk content and cache records may go out of sync - _, err := is.store.Stat(context.Background(), dstRecord) - if err != nil { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat") - // the actual blob on disk may have been removed by GC, so sync the cache - err := is.cache.DeleteBlob(dstDigest, dstRecord) - if err = inject.Error(err); err != nil { - //nolint:lll - is.log.Error().Err(err).Str("dstDigest", dstDigest.String()).Str("dst", dst).Msg("dedupe: unable to delete blob record") - - return err - } - - goto retry - } - - fileInfo, err := is.store.Stat(context.Background(), dst) - if err != nil && !errors.As(err, &driver.PathNotFoundError{}) { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to stat") - - return err - } - - // prevent overwrite original blob - if fileInfo == nil && dstRecord != dst { - // put empty file so that we are compliant with oci layout, this will act as a deduped blob - err = is.store.PutContent(context.Background(), dst, []byte{}) - if err != nil { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to write empty file") - - return err - } - - if err := is.cache.PutBlob(dstDigest, dst); err != nil { - is.log.Error().Err(err).Str("blobPath", dst).Msg("dedupe: unable to insert blob record") - - return err - } - } - - // remove temp blobupload - if err := is.store.Delete(context.Background(), src); err != nil { - is.log.Error().Err(err).Str("src", src).Msg("dedupe: unable to remove blob") - - return err - } - - is.log.Debug().Str("src", src).Msg("dedupe: remove") - } - - return nil -} - -func (is *ObjectStorage) RunGCRepo(repo string) error { - return nil -} - -func (is *ObjectStorage) RunGCPeriodically(interval time.Duration, sch *scheduler.Scheduler) { -} - -// DeleteBlobUpload deletes an existing blob upload that is currently in progress. -func (is *ObjectStorage) DeleteBlobUpload(repo, uuid string) error { - blobUploadPath := is.BlobUploadPath(repo, uuid) - - writer, err := is.store.Writer(context.Background(), blobUploadPath, true) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - return zerr.ErrUploadNotFound - } - - return err - } - - defer writer.Close() - - if err := writer.Cancel(); err != nil { - is.log.Error().Err(err).Str("blobUploadPath", blobUploadPath).Msg("error deleting blob upload") - - return err - } - - return nil -} - -// BlobPath returns the repository path of a blob. -func (is *ObjectStorage) BlobPath(repo string, digest godigest.Digest) string { - return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded()) -} - -/* - CheckBlob verifies a blob and returns true if the blob is correct - -If the blob is not found but it's found in cache then it will be copied over. -*/ -func (is *ObjectStorage) CheckBlob(repo string, digest godigest.Digest) (bool, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return false, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - if is.dedupe && fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - } else { - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - } - - binfo, err := is.store.Stat(context.Background(), blobPath) - if err == nil && binfo.Size() > 0 { - is.log.Debug().Str("blob path", blobPath).Msg("blob path found") - - return true, binfo.Size(), nil - } - // otherwise is a 'deduped' blob (empty file) - - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") - - return false, -1, zerr.ErrBlobNotFound - } - - blobSize, err := is.copyBlob(repo, blobPath, dstRecord) - if err != nil { - return false, -1, zerr.ErrBlobNotFound - } - - // put deduped blob in cache - if err := is.cache.PutBlob(digest, blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("dedupe: unable to insert blob record") - - return false, -1, err - } - - return true, blobSize, nil -} - -// StatBlob verifies if a blob is present inside a repository. The caller function SHOULD lock from outside. -func (is *ObjectStorage) StatBlob(repo string, digest godigest.Digest) (bool, int64, error) { - if err := digest.Validate(); err != nil { - return false, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - binfo, err := is.store.Stat(context.Background(), blobPath) - if err == nil && binfo.Size() > 0 { - is.log.Debug().Str("blob path", blobPath).Msg("blob path found") - - return true, binfo.Size(), nil - } - - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return false, -1, zerr.ErrBlobNotFound - } - - // then it's a 'deduped' blob - - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") - - return false, -1, zerr.ErrBlobNotFound - } - - binfo, err = is.store.Stat(context.Background(), dstRecord) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return false, -1, zerr.ErrBlobNotFound - } - - return true, binfo.Size(), nil -} - -func (is *ObjectStorage) checkCacheBlob(digest godigest.Digest) (string, error) { - if err := digest.Validate(); err != nil { - return "", err - } - - if fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { - return "", zerr.ErrBlobNotFound - } - - dstRecord, err := is.cache.GetBlob(digest) - if err != nil { - return "", err - } - - if _, err := is.store.Stat(context.Background(), dstRecord); err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to stat blob") - - // the actual blob on disk may have been removed by GC, so sync the cache - if err := is.cache.DeleteBlob(digest, dstRecord); err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", dstRecord). - Msg("unable to remove blob path from cache") - - return "", err - } - - return "", zerr.ErrBlobNotFound - } - - is.log.Debug().Str("digest", digest.String()).Str("dstRecord", dstRecord).Msg("cache: found dedupe record") - - return dstRecord, nil -} - -func (is *ObjectStorage) copyBlob(repo string, blobPath, dstRecord string) (int64, error) { - if err := is.initRepo(repo); err != nil { - is.log.Error().Err(err).Str("repository", repo).Msg("unable to initialize an empty repo") - - return -1, err - } - - if err := is.store.PutContent(context.Background(), blobPath, []byte{}); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Str("link", dstRecord).Msg("dedupe: unable to link") - - return -1, zerr.ErrBlobNotFound - } - - // return original blob with content instead of the deduped one (blobPath) - binfo, err := is.store.Stat(context.Background(), dstRecord) - if err == nil { - return binfo.Size(), nil - } - - return -1, zerr.ErrBlobNotFound -} - -// blobStream is using to serve blob range requests. -type blobStream struct { - reader io.Reader - closer io.Closer -} - -func NewBlobStream(readCloser io.ReadCloser, from, to int64) (io.ReadCloser, error) { - return &blobStream{reader: io.LimitReader(readCloser, to-from+1), closer: readCloser}, nil -} - -func (bs *blobStream) Read(buf []byte) (int, error) { - return bs.reader.Read(buf) -} - -func (bs *blobStream) Close() error { - return bs.closer.Close() -} - -// GetBlobPartial returns a partial stream to read the blob. -// blob selector instead of directly downloading the blob. -func (is *ObjectStorage) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, -) (io.ReadCloser, int64, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return nil, -1, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return nil, -1, -1, zerr.ErrBlobNotFound - } - - end := to - - if to < 0 || to >= binfo.Size() { - end = binfo.Size() - 1 - } - - blobHandle, err := is.store.Reader(context.Background(), blobPath, from) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") - - return nil, -1, -1, err - } - - blobReadCloser, err := NewBlobStream(blobHandle, from, end) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob stream") - - return nil, -1, -1, err - } - - // is a 'deduped' blob? - if binfo.Size() == 0 { - defer blobReadCloser.Close() - - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") - - return nil, -1, -1, zerr.ErrBlobNotFound - } - - binfo, err := is.store.Stat(context.Background(), dstRecord) - if err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to stat blob") - - return nil, -1, -1, zerr.ErrBlobNotFound - } - - end := to - - if to < 0 || to >= binfo.Size() { - end = binfo.Size() - 1 - } - - blobHandle, err := is.store.Reader(context.Background(), dstRecord, from) - if err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") - - return nil, -1, -1, err - } - - blobReadCloser, err := NewBlobStream(blobHandle, from, end) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob stream") - - return nil, -1, -1, err - } - - return blobReadCloser, end - from + 1, binfo.Size(), nil - } - - // The caller function is responsible for calling Close() - return blobReadCloser, end - from + 1, binfo.Size(), nil -} - -// GetBlob returns a stream to read the blob. -// blob selector instead of directly downloading the blob. -func (is *ObjectStorage) GetBlob(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return nil, -1, err - } - - blobPath := is.BlobPath(repo, digest) - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return nil, -1, zerr.ErrBlobNotFound - } - - blobReadCloser, err := is.store.Reader(context.Background(), blobPath, 0) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") - - return nil, -1, err - } - - // is a 'deduped' blob? - if binfo.Size() == 0 { - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") - - return nil, -1, zerr.ErrBlobNotFound - } - - binfo, err := is.store.Stat(context.Background(), dstRecord) - if err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to stat blob") - - return nil, -1, zerr.ErrBlobNotFound - } - - blobReadCloser, err := is.store.Reader(context.Background(), dstRecord, 0) - if err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") - - return nil, -1, err - } - - return blobReadCloser, binfo.Size(), nil - } - - // The caller function is responsible for calling Close() - return blobReadCloser, binfo.Size(), nil -} - -// GetBlobContent returns blob contents, the caller function SHOULD lock from outside. -func (is *ObjectStorage) GetBlobContent(repo string, digest godigest.Digest) ([]byte, error) { - if err := digest.Validate(); err != nil { - return []byte{}, err - } - - blobPath := is.BlobPath(repo, digest) - - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return []byte{}, zerr.ErrBlobNotFound - } - - blobBuf, err := is.store.GetContent(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to open blob") - - return nil, err - } - - // is a 'deduped' blob? - if binfo.Size() == 0 { - // Check blobs in cache - dstRecord, err := is.checkCacheBlob(digest) - if err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Msg("cache: not found") - - return nil, zerr.ErrBlobNotFound - } - - blobBuf, err := is.store.GetContent(context.Background(), dstRecord) - if err != nil { - is.log.Error().Err(err).Str("blob", dstRecord).Msg("failed to open blob") - - return nil, err - } - - return blobBuf, nil - } - - return blobBuf, nil -} - -func (is *ObjectStorage) GetReferrers(repo string, gdigest godigest.Digest, artifactTypes []string, -) (ispec.Index, error) { - var lockLatency time.Time - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - return common.GetReferrers(is, repo, gdigest, artifactTypes, is.log) -} - -func (is *ObjectStorage) GetOrasReferrers(repo string, gdigest godigest.Digest, artifactType string, -) ([]artifactspec.Descriptor, error) { - var lockLatency time.Time - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - return common.GetOrasReferrers(is, repo, gdigest, artifactType, is.log) -} - -// GetIndexContent returns index.json contents, the caller function SHOULD lock from outside. -func (is *ObjectStorage) GetIndexContent(repo string) ([]byte, error) { - dir := path.Join(is.rootDir, repo) - - buf, err := is.store.GetContent(context.Background(), path.Join(dir, "index.json")) - if err != nil { - if errors.Is(err, driver.PathNotFoundError{}) { - is.log.Error().Err(err).Str("dir", dir).Msg("index.json doesn't exist") - - return []byte{}, zerr.ErrRepoNotFound - } - - is.log.Error().Err(err).Str("dir", dir).Msg("failed to read index.json") - - return []byte{}, err - } - - return buf, nil -} - -// DeleteBlob removes the blob from the repository. -func (is *ObjectStorage) DeleteBlob(repo string, digest godigest.Digest) error { - var lockLatency time.Time - - if err := digest.Validate(); err != nil { - return err - } - - blobPath := is.BlobPath(repo, digest) - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - _, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("blob", blobPath).Msg("failed to stat blob") - - return zerr.ErrBlobNotFound - } - - // first check if this blob is not currently in use - if ok, _ := common.IsBlobReferenced(is, repo, digest, is.log); ok { - return zerr.ErrBlobReferenced - } - - if fmt.Sprintf("%v", is.cache) != fmt.Sprintf("%v", nil) { - dstRecord, err := is.cache.GetBlob(digest) - if err != nil && !errors.Is(err, zerr.ErrCacheMiss) { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to lookup blob record") - - return err - } - - // remove cache entry and move blob contents to the next candidate if there is any - if ok := is.cache.HasBlob(digest, blobPath); ok { - if err := is.cache.DeleteBlob(digest, blobPath); err != nil { - is.log.Error().Err(err).Str("digest", digest.String()).Str("blobPath", blobPath). - Msg("unable to remove blob path from cache") - - return err - } - } - - // if the deleted blob is one with content - if dstRecord == blobPath { - // get next candidate - dstRecord, err := is.cache.GetBlob(digest) - if err != nil && !errors.Is(err, zerr.ErrCacheMiss) { - is.log.Error().Err(err).Str("blobPath", dstRecord).Msg("dedupe: unable to lookup blob record") - - return err - } - - // if we have a new candidate move the blob content to it - if dstRecord != "" { - if err := is.store.Move(context.Background(), blobPath, dstRecord); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") - - return err - } - - return nil - } - } - } - - if err := is.store.Delete(context.Background(), blobPath); err != nil { - is.log.Error().Err(err).Str("blobPath", blobPath).Msg("unable to remove blob path") - - return err - } - - return nil -} - -// Do not use for multipart upload, buf must not be empty. -// If you want to create an empty file use is.store.PutContent(). -func writeFile(store driver.StorageDriver, filepath string, buf []byte) (int, error) { - var n int - - if stwr, err := store.Writer(context.Background(), filepath, false); err == nil { - defer stwr.Close() - - if n, err = stwr.Write(buf); err != nil { - return -1, err - } - - if err := stwr.Commit(); err != nil { - return -1, err - } - } else { - return -1, err - } - - return n, nil -} - -func (is *ObjectStorage) GetNextDigestWithBlobPaths(lastDigests []godigest.Digest) (godigest.Digest, []string, error) { - var lockLatency time.Time - - dir := is.rootDir - - is.RLock(&lockLatency) - defer is.RUnlock(&lockLatency) - - var duplicateBlobs []string - - var digest godigest.Digest - - err := is.store.Walk(context.Background(), dir, func(fileInfo driver.FileInfo) error { - if fileInfo.IsDir() { - return nil - } - - blobDigest := godigest.NewDigestFromEncoded("sha256", path.Base(fileInfo.Path())) - if err := blobDigest.Validate(); err != nil { - return nil //nolint:nilerr // ignore files which are not blobs - } - - if digest == "" && !zcommon.Contains(lastDigests, blobDigest) { - digest = blobDigest - } - - if blobDigest == digest { - duplicateBlobs = append(duplicateBlobs, fileInfo.Path()) - } - - return nil - }) - - // if the root directory is not yet created - var perr driver.PathNotFoundError - - if errors.As(err, &perr) { - return digest, duplicateBlobs, nil - } - - return digest, duplicateBlobs, err -} - -func (is *ObjectStorage) getOriginalBlobFromDisk(duplicateBlobs []string) (string, error) { - for _, blobPath := range duplicateBlobs { - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") - - return "", zerr.ErrBlobNotFound - } - - if binfo.Size() > 0 { - return blobPath, nil - } - } - - return "", zerr.ErrBlobNotFound -} - -func (is *ObjectStorage) getOriginalBlob(digest godigest.Digest, duplicateBlobs []string) (string, error) { - var originalBlob string - - var err error - - originalBlob, err = is.checkCacheBlob(digest) - if err != nil && !errors.Is(err, zerr.ErrBlobNotFound) && !errors.Is(err, zerr.ErrCacheMiss) { - is.log.Error().Err(err).Msg("rebuild dedupe: unable to find blob in cache") - - return originalBlob, err - } - - // if we still don't have, search it - if originalBlob == "" { - is.log.Warn().Msg("rebuild dedupe: failed to find blob in cache, searching it in s3...") - // a rebuild dedupe was attempted in the past - // get original blob, should be found otherwise exit with error - - originalBlob, err = is.getOriginalBlobFromDisk(duplicateBlobs) - if err != nil { - return originalBlob, err - } - } - - is.log.Info().Str("originalBlob", originalBlob).Msg("rebuild dedupe: found original blob") - - return originalBlob, nil -} - -func (is *ObjectStorage) dedupeBlobs(digest godigest.Digest, duplicateBlobs []string) error { - if fmt.Sprintf("%v", is.cache) == fmt.Sprintf("%v", nil) { - is.log.Error().Err(zerr.ErrDedupeRebuild).Msg("no cache driver found, can not dedupe blobs") - - return zerr.ErrDedupeRebuild - } - - is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest") - - var originalBlob string - - // rebuild from dedupe false to true - for _, blobPath := range duplicateBlobs { - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") - - return err - } - - if binfo.Size() == 0 { - is.log.Warn().Msg("rebuild dedupe: found file without content, trying to find the original blob") - // a rebuild dedupe was attempted in the past - // get original blob, should be found otherwise exit with error - if originalBlob == "" { - originalBlob, err = is.getOriginalBlob(digest, duplicateBlobs) - if err != nil { - is.log.Error().Err(err).Msg("rebuild dedupe: unable to find original blob") - - return zerr.ErrDedupeRebuild - } - - // cache original blob - if ok := is.cache.HasBlob(digest, originalBlob); !ok { - if err := is.cache.PutBlob(digest, originalBlob); err != nil { - return err - } - } - } - - // cache dedupe blob - if ok := is.cache.HasBlob(digest, blobPath); !ok { - if err := is.cache.PutBlob(digest, blobPath); err != nil { - return err - } - } - } else { - // cache it - if ok := is.cache.HasBlob(digest, blobPath); !ok { - if err := is.cache.PutBlob(digest, blobPath); err != nil { - return err - } - } - - // if we have an original blob cached then we can safely dedupe the rest of them - if originalBlob != "" { - if err := is.store.PutContent(context.Background(), blobPath, []byte{}); err != nil { - is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: unable to dedupe blob") - - return err - } - } - - // mark blob as preserved - originalBlob = blobPath - } - } - - is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: deduping blobs for digest finished successfully") - - return nil -} - -func (is *ObjectStorage) restoreDedupedBlobs(digest godigest.Digest, duplicateBlobs []string) error { - is.log.Info().Str("digest", digest.String()).Msg("rebuild dedupe: restoring deduped blobs for digest") - - // first we need to find the original blob, either in cache or by checking each blob size - originalBlob, err := is.getOriginalBlob(digest, duplicateBlobs) - if err != nil { - is.log.Error().Err(err).Msg("rebuild dedupe: unable to find original blob") - - return zerr.ErrDedupeRebuild - } - - for _, blobPath := range duplicateBlobs { - binfo, err := is.store.Stat(context.Background(), blobPath) - if err != nil { - is.log.Error().Err(err).Str("path", blobPath).Msg("rebuild dedupe: failed to stat blob") - - return err - } - - // if we find a deduped blob, then copy original blob content to deduped one - if binfo.Size() == 0 { - // move content from original blob to deduped one - buf, err := is.store.GetContent(context.Background(), originalBlob) - if err != nil { - is.log.Error().Err(err).Str("path", originalBlob).Msg("rebuild dedupe: failed to get original blob content") - - return err - } - - _, err = writeFile(is.store, blobPath, buf) - if err != nil { - return err - } - } - } - - is.log.Info().Str("digest", digest.String()). - Msg("rebuild dedupe: restoring deduped blobs for digest finished successfully") - - return nil -} - -func (is *ObjectStorage) RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error { - var lockLatency time.Time - - is.Lock(&lockLatency) - defer is.Unlock(&lockLatency) - - if dedupe { - return is.dedupeBlobs(digest, duplicateBlobs) - } - - return is.restoreDedupedBlobs(digest, duplicateBlobs) -} - -func (is *ObjectStorage) RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) { - generator := &common.DedupeTaskGenerator{ - ImgStore: is, - Dedupe: is.dedupe, - Log: is.log, - } - - sch.SubmitGenerator(generator, interval, scheduler.MediumPriority) + return imagestore.NewImageStore( + rootDir, + cacheDir, + gc, + gcReferrers, + gcDelay, + untaggedImageRetentionDelay, + dedupe, + commit, + log, + metrics, + linter, + New(store), + cacheDriver, + ) } diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 9430b2cc..6a1036a3 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -77,15 +77,17 @@ func createMockStorage(rootDir string, cacheDir string, dedupe bool, store drive var cacheDriver cache.Cache // from pkg/cli/root.go/applyDefaultValues, s3 magic - if _, err := os.Stat(path.Join(cacheDir, "s3_cache.db")); dedupe || (!dedupe && err == nil) { + if _, err := os.Stat(path.Join(cacheDir, + storageConstants.BoltdbName+storageConstants.DBExtensionName)); dedupe || (!dedupe && err == nil) { cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: cacheDir, - Name: "s3_cache", + Name: "cache", UseRelPaths: false, }, log) } - il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay, - dedupe, false, log, metrics, nil, store, cacheDriver, + + il := s3.NewImageStore(rootDir, cacheDir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, dedupe, false, log, metrics, nil, store, cacheDriver, ) return il @@ -97,8 +99,8 @@ func createMockStorageWithMockCache(rootDir string, dedupe bool, store driver.St log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - il := s3.NewImageStore(rootDir, "", false, storageConstants.DefaultGCDelay, - dedupe, false, log, metrics, nil, store, cacheDriver, + il := s3.NewImageStore(rootDir, "", true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, dedupe, false, log, metrics, nil, store, cacheDriver, ) return il @@ -150,17 +152,17 @@ func createObjectsStore(rootDir string, cacheDir string, dedupe bool) ( var err error // from pkg/cli/root.go/applyDefaultValues, s3 magic - s3CacheDBPath := path.Join(cacheDir, s3.CacheDBName+storageConstants.DBExtensionName) + s3CacheDBPath := path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName) if _, err = os.Stat(s3CacheDBPath); dedupe || (!dedupe && err == nil) { cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: cacheDir, - Name: "s3_cache", + Name: "cache", UseRelPaths: false, }, log) } - il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay, - dedupe, false, log, metrics, nil, store, cacheDriver) + il := s3.NewImageStore(rootDir, cacheDir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, dedupe, false, log, metrics, nil, store, cacheDriver) return store, il, err } @@ -194,8 +196,8 @@ func createObjectsStoreDynamo(rootDir string, cacheDir string, dedupe bool, tabl panic(err) } - il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay, - dedupe, false, log, metrics, nil, store, cacheDriver) + il := s3.NewImageStore(rootDir, cacheDir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, dedupe, false, log, metrics, nil, store, cacheDriver) return store, il, err } @@ -893,7 +895,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { _, _, err = imgStore.CheckBlob(testImage, digest) So(err, ShouldNotBeNil) - _, _, err = imgStore.StatBlob(testImage, digest) + _, _, _, err = imgStore.StatBlob(testImage, digest) So(err, ShouldNotBeNil) }) @@ -1050,7 +1052,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) { return &FileWriterMock{WriteFn: func(b []byte) (int, error) { return 0, errS3 - }}, nil + }}, errS3 }, }) _, err := imgStore.PutBlobChunkStreamed(testImage, "uuid", io.NopCloser(strings.NewReader(""))) @@ -1091,7 +1093,7 @@ func TestNegativeCasesObjectsStorage(t *testing.T) { WriteFn: func(b []byte) (int, error) { return 0, errS3 }, - }, nil + }, errS3 }, }) _, err := imgStore.PutBlobChunk(testImage, "uuid", 12, 100, io.NopCloser(strings.NewReader(""))) @@ -1280,7 +1282,7 @@ func TestS3Dedupe(t *testing.T) { So(checkBlobSize1, ShouldBeGreaterThan, 0) So(err, ShouldBeNil) - ok, checkBlobSize1, err = imgStore.StatBlob("dedupe1", digest) + ok, checkBlobSize1, _, err = imgStore.StatBlob("dedupe1", digest) So(ok, ShouldBeTrue) So(checkBlobSize1, ShouldBeGreaterThan, 0) So(err, ShouldBeNil) @@ -1466,12 +1468,12 @@ func TestS3Dedupe(t *testing.T) { Convey("Check backward compatibility - switch dedupe to false", func() { /* copy cache to the new storage with dedupe false (doing this because we already have a cache object holding the lock on cache db file) */ - input, err := os.ReadFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName)) + input, err := os.ReadFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName)) So(err, ShouldBeNil) tdir = t.TempDir() - err = os.WriteFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName), input, 0o600) + err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600) So(err, ShouldBeNil) storeDriver, imgStore, _ := createObjectsStore(testDir, tdir, false) @@ -3306,6 +3308,7 @@ func TestS3ManifestImageIndex(t *testing.T) { err = imgStore.DeleteImageManifest("index", "test:index1", false) So(err, ShouldBeNil) + _, _, _, err = imgStore.GetImageManifest("index", "test:index1") So(err, ShouldNotBeNil) @@ -3599,7 +3602,7 @@ func TestS3DedupeErr(t *testing.T) { imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{}) - err = os.Remove(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName)) + err = os.Remove(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName)) digest := godigest.NewDigestFromEncoded(godigest.SHA256, "digest") // trigger unable to insert blob record @@ -3640,8 +3643,9 @@ func TestS3DedupeErr(t *testing.T) { err := imgStore.DedupeBlob("", digest, "dst") So(err, ShouldBeNil) + // error will be triggered in driver.SameFile() err = imgStore.DedupeBlob("", digest, "dst2") - So(err, ShouldNotBeNil) + So(err, ShouldBeNil) }) Convey("Test DedupeBlob - error on store.PutContent()", t, func(c C) { @@ -3776,12 +3780,12 @@ func TestS3DedupeErr(t *testing.T) { So(err, ShouldBeNil) // copy cache db to the new imagestore - input, err := os.ReadFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName)) + input, err := os.ReadFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName)) So(err, ShouldBeNil) tdir = t.TempDir() - err = os.WriteFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName), input, 0o600) + err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600) So(err, ShouldBeNil) imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ @@ -3797,12 +3801,14 @@ func TestS3DedupeErr(t *testing.T) { _, _, err = imgStore.GetBlob("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip") So(err, ShouldNotBeNil) + // now it should move content from /repo1/dst1 to /repo2/dst2 _, err = imgStore.GetBlobContent("repo2", digest) - So(err, ShouldNotBeNil) + So(err, ShouldBeNil) - _, _, err = imgStore.StatBlob("repo2", digest) - So(err, ShouldNotBeNil) + _, _, _, err = imgStore.StatBlob("repo2", digest) + So(err, ShouldBeNil) + // it errors out because of bad range, as mock store returns a driver.FileInfo with 0 size _, _, _, err = imgStore.GetBlobPartial("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip", 0, 1) So(err, ShouldNotBeNil) }) @@ -3822,12 +3828,12 @@ func TestS3DedupeErr(t *testing.T) { So(err, ShouldBeNil) // copy cache db to the new imagestore - input, err := os.ReadFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName)) + input, err := os.ReadFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName)) So(err, ShouldBeNil) tdir = t.TempDir() - err = os.WriteFile(path.Join(tdir, s3.CacheDBName+storageConstants.DBExtensionName), input, 0o600) + err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600) So(err, ShouldBeNil) imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{ @@ -3887,7 +3893,7 @@ func TestS3DedupeErr(t *testing.T) { _, err = imgStore.GetBlobContent("repo2", digest) So(err, ShouldNotBeNil) - _, _, err = imgStore.StatBlob("repo2", digest) + _, _, _, err = imgStore.StatBlob("repo2", digest) So(err, ShouldNotBeNil) _, _, _, err = imgStore.GetBlobPartial("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip", 0, 1) diff --git a/pkg/storage/scrub_test.go b/pkg/storage/scrub_test.go index 460695ec..1253784b 100644 --- a/pkg/storage/scrub_test.go +++ b/pkg/storage/scrub_test.go @@ -39,8 +39,8 @@ func TestCheckAllBlobsIntegrity(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore := local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, - true, true, log, metrics, nil, cacheDriver) + imgStore := local.NewImageStore(dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, cacheDriver) Convey("Scrub only one repo", t, func(c C) { // initialize repo @@ -113,7 +113,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) { // verify error message So(actual, ShouldContainSubstring, "test 1.0 affected parse application/vnd.oci.image.manifest.v1+json") - index, err := common.GetIndex(imgStore, repoName, log.With().Caller().Logger()) + index, err := common.GetIndex(imgStore, repoName, log) So(err, ShouldBeNil) So(len(index.Manifests), ShouldEqual, 1) @@ -193,7 +193,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) { err = os.Chmod(layerFile, 0x0200) So(err, ShouldBeNil) - index, err := common.GetIndex(imgStore, repoName, log.With().Caller().Logger()) + index, err := common.GetIndex(imgStore, repoName, log) So(err, ShouldBeNil) So(len(index.Manifests), ShouldEqual, 1) @@ -327,7 +327,7 @@ func TestCheckAllBlobsIntegrity(t *testing.T) { So(actual, ShouldContainSubstring, "test 1.0 affected") So(actual, ShouldContainSubstring, "no such file or directory") - index, err := common.GetIndex(imgStore, repoName, log.With().Caller().Logger()) + index, err := common.GetIndex(imgStore, repoName, log) So(err, ShouldBeNil) So(len(index.Manifests), ShouldEqual, 2) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 7aab199d..4ecc7c09 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -51,8 +51,9 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer if config.Storage.StorageDriver == nil { // false positive lint - linter does not implement Lint method //nolint:typecheck,contextcheck - defaultStore = local.NewImageStore(config.Storage.RootDirectory, - config.Storage.GC, config.Storage.GCDelay, + rootDir := config.Storage.RootDirectory + defaultStore = local.NewImageStore(rootDir, + config.Storage.GC, config.Storage.GCReferrers, config.Storage.GCDelay, config.Storage.UntaggedImageRetentionDelay, config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, CreateCacheDatabaseDriver(config.Storage.StorageConfig, log), ) @@ -80,7 +81,8 @@ func New(config *config.Config, linter common.Lint, metrics monitoring.MetricSer // false positive lint - linter does not implement Lint method //nolint: typecheck,contextcheck defaultStore = s3.NewImageStore(rootDir, config.Storage.RootDirectory, - config.Storage.GC, config.Storage.GCDelay, config.Storage.Dedupe, + config.Storage.GC, config.Storage.GCReferrers, config.Storage.GCDelay, + config.Storage.UntaggedImageRetentionDelay, config.Storage.Dedupe, config.Storage.Commit, log, metrics, linter, store, CreateCacheDatabaseDriver(config.Storage.StorageConfig, log)) } @@ -152,9 +154,13 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig, // add it to uniqueSubFiles // Create a new image store and assign it to imgStoreMap if isUnique { - imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(storageConfig.RootDirectory, - storageConfig.GC, storageConfig.GCDelay, storageConfig.Dedupe, - storageConfig.Commit, log, metrics, linter, CreateCacheDatabaseDriver(storageConfig, log)) + rootDir := storageConfig.RootDirectory + imgStoreMap[storageConfig.RootDirectory] = local.NewImageStore(rootDir, + storageConfig.GC, storageConfig.GCReferrers, storageConfig.GCDelay, + storageConfig.UntaggedImageRetentionDelay, storageConfig.Dedupe, + storageConfig.Commit, log, metrics, linter, + CreateCacheDatabaseDriver(storageConfig, log), + ) subImageStore[route] = imgStoreMap[storageConfig.RootDirectory] } @@ -183,8 +189,9 @@ func getSubStore(cfg *config.Config, subPaths map[string]config.StorageConfig, // false positive lint - linter does not implement Lint method //nolint: typecheck subImageStore[route] = s3.NewImageStore(rootDir, storageConfig.RootDirectory, - storageConfig.GC, storageConfig.GCDelay, - storageConfig.Dedupe, storageConfig.Commit, log, metrics, linter, store, + storageConfig.GC, storageConfig.GCReferrers, storageConfig.GCDelay, + storageConfig.UntaggedImageRetentionDelay, storageConfig.Dedupe, + storageConfig.Commit, log, metrics, linter, store, CreateCacheDatabaseDriver(storageConfig, log), ) } diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 53ba4b0d..ea25f053 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -22,6 +22,7 @@ import ( guuid "github.com/gofrs/uuid" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" + artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" "github.com/rs/zerolog" . "github.com/smartystreets/goconvey/convey" "gopkg.in/resty.v1" @@ -33,6 +34,7 @@ import ( "zotregistry.io/zot/pkg/storage/cache" storageCommon "zotregistry.io/zot/pkg/storage/common" storageConstants "zotregistry.io/zot/pkg/storage/constants" + "zotregistry.io/zot/pkg/storage/imagestore" "zotregistry.io/zot/pkg/storage/local" "zotregistry.io/zot/pkg/storage/s3" storageTypes "zotregistry.io/zot/pkg/storage/types" @@ -52,7 +54,9 @@ func skipIt(t *testing.T) { } } -func createObjectsStore(rootDir string, cacheDir string) (driver.StorageDriver, storageTypes.ImageStore, error) { +func createObjectsStore(rootDir string, cacheDir string, gcDelay time.Duration, imageRetention time.Duration) ( + driver.StorageDriver, storageTypes.ImageStore, error, +) { bucket := "zot-storage-test" endpoint := os.Getenv("S3MOCK_ENDPOINT") storageDriverParams := map[string]interface{}{ @@ -85,11 +89,11 @@ func createObjectsStore(rootDir string, cacheDir string) (driver.StorageDriver, cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: cacheDir, - Name: "s3_cache", + Name: "cache", UseRelPaths: false, }, log) - il := s3.NewImageStore(rootDir, cacheDir, false, storageConstants.DefaultGCDelay, + il := s3.NewImageStore(rootDir, cacheDir, true, true, gcDelay, imageRetention, true, false, log, metrics, nil, store, cacheDriver, ) @@ -103,11 +107,11 @@ var testCases = []struct { }{ { testCaseName: "S3APIs", - storageType: "s3", + storageType: storageConstants.S3StorageDriverName, }, { testCaseName: "FileSystemAPIs", - storageType: "fs", + storageType: storageConstants.LocalStorageDriverName, }, } @@ -116,7 +120,7 @@ func TestStorageAPIs(t *testing.T) { testcase := testcase t.Run(testcase.testCaseName, func(t *testing.T) { var imgStore storageTypes.ImageStore - if testcase.storageType == "s3" { + if testcase.storageType == storageConstants.S3StorageDriverName { skipIt(t) uuid, err := guuid.NewV4() @@ -128,7 +132,8 @@ func TestStorageAPIs(t *testing.T) { tdir := t.TempDir() var store driver.StorageDriver - store, imgStore, _ = createObjectsStore(testDir, tdir) + store, imgStore, _ = createObjectsStore(testDir, tdir, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay) defer cleanupStorage(store, testDir) } else { dir := t.TempDir() @@ -140,8 +145,11 @@ func TestStorageAPIs(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore = local.NewImageStore(dir, true, storageConstants.DefaultGCDelay, true, - true, log, metrics, nil, cacheDriver) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, driver, cacheDriver) } Convey("Repo layout", t, func(c C) { @@ -166,15 +174,15 @@ func TestStorageAPIs(t *testing.T) { }) Convey("Validate repo", func() { - v, err := imgStore.ValidateRepo(repoName) + repos, err := imgStore.ValidateRepo(repoName) So(err, ShouldBeNil) - So(v, ShouldEqual, true) + So(repos, ShouldEqual, true) }) Convey("Get repos", func() { - v, err := imgStore.GetRepositories() + repos, err := imgStore.GetRepositories() So(err, ShouldBeNil) - So(v, ShouldNotBeEmpty) + So(repos, ShouldNotBeEmpty) }) Convey("Get image tags", func() { @@ -261,7 +269,7 @@ func TestStorageAPIs(t *testing.T) { _, _, err = imgStore.CheckBlob("test", digest) So(err, ShouldBeNil) - ok, _, err := imgStore.StatBlob("test", digest) + ok, _, _, err := imgStore.StatBlob("test", digest) So(ok, ShouldBeTrue) So(err, ShouldBeNil) @@ -385,6 +393,11 @@ func TestStorageAPIs(t *testing.T) { So(err, ShouldBeNil) So(len(tags), ShouldEqual, 2) + repos, err := imgStore.GetRepositories() + So(err, ShouldBeNil) + So(len(repos), ShouldEqual, 1) + So(repos[0], ShouldEqual, "test") + // We deleted only one tag, make sure blob should not be removed. hasBlob, _, err = imgStore.CheckBlob("test", digest) So(err, ShouldBeNil) @@ -407,7 +420,7 @@ func TestStorageAPIs(t *testing.T) { So(err, ShouldNotBeNil) So(hasBlob, ShouldEqual, false) - hasBlob, _, err = imgStore.StatBlob("test", digest) + hasBlob, _, _, err = imgStore.StatBlob("test", digest) So(err, ShouldNotBeNil) So(hasBlob, ShouldEqual, false) @@ -464,6 +477,10 @@ func TestStorageAPIs(t *testing.T) { err = imgStore.FinishBlobUpload("test", "inexistent", buf, digest) So(err, ShouldNotBeNil) + // invalid digest + err = imgStore.FinishBlobUpload("test", "inexistent", buf, "sha256:invalid") + So(err, ShouldNotBeNil) + err = imgStore.FinishBlobUpload("test", bupload, buf, digest) So(err, ShouldBeNil) @@ -471,7 +488,7 @@ func TestStorageAPIs(t *testing.T) { So(ok, ShouldBeTrue) So(err, ShouldBeNil) - ok, _, err = imgStore.StatBlob("test", digest) + ok, _, _, err = imgStore.StatBlob("test", digest) So(ok, ShouldBeTrue) So(err, ShouldBeNil) @@ -502,7 +519,7 @@ func TestStorageAPIs(t *testing.T) { _, _, err = imgStore.CheckBlob("test", "inexistent") So(err, ShouldNotBeNil) - _, _, err = imgStore.StatBlob("test", "inexistent") + _, _, _, err = imgStore.StatBlob("test", "inexistent") So(err, ShouldNotBeNil) }) @@ -564,7 +581,7 @@ func TestStorageAPIs(t *testing.T) { indexContent, err := imgStore.GetIndexContent("test") So(err, ShouldBeNil) - if testcase.storageType == "fs" { + if testcase.storageType == storageConstants.LocalStorageDriverName { err = os.Chmod(path.Join(imgStore.RootDir(), "test", "index.json"), 0o000) So(err, ShouldBeNil) _, err = imgStore.GetIndexContent("test") @@ -737,7 +754,7 @@ func TestMandatoryAnnotations(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - if testcase.storageType == "s3" { + if testcase.storageType == storageConstants.S3StorageDriverName { skipIt(t) uuid, err := guuid.NewV4() @@ -748,13 +765,16 @@ func TestMandatoryAnnotations(t *testing.T) { testDir = path.Join("/oci-repo-test", uuid.String()) tdir = t.TempDir() - store, _, _ = createObjectsStore(testDir, tdir) - imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics, + store, _, _ = createObjectsStore(testDir, tdir, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay) + driver := s3.New(store) + imgStore = imagestore.NewImageStore(testDir, tdir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { return false, nil }, - }, store, nil) + }, driver, nil) defer cleanupStorage(store, testDir) } else { @@ -764,12 +784,14 @@ func TestMandatoryAnnotations(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore = local.NewImageStore(tdir, true, storageConstants.DefaultGCDelay, true, + driver := local.New(true) + imgStore = imagestore.NewImageStore(tdir, tdir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { return false, nil }, - }, cacheDriver) + }, driver, cacheDriver) } Convey("Setup manifest", t, func() { @@ -815,27 +837,31 @@ func TestMandatoryAnnotations(t *testing.T) { }) Convey("Error on mandatory annotations", func() { - if testcase.storageType == "s3" { - imgStore = s3.NewImageStore(testDir, tdir, false, 1, false, false, log, metrics, + if testcase.storageType == storageConstants.S3StorageDriverName { + driver := s3.New(store) + imgStore = imagestore.NewImageStore(testDir, tdir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { //nolint: goerr113 return false, errors.New("linter error") }, - }, store, nil) + }, driver, nil) } else { cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ RootDir: tdir, Name: "cache", UseRelPaths: true, }, log) - imgStore = local.NewImageStore(tdir, true, storageConstants.DefaultGCDelay, true, + driver := local.New(true) + imgStore = imagestore.NewImageStore(tdir, tdir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, &mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore storageTypes.ImageStore) (bool, error) { //nolint: goerr113 return false, errors.New("linter error") }, - }, cacheDriver) + }, driver, cacheDriver) } _, _, err = imgStore.PutImageManifest("test", "1.0.0", ispec.MediaTypeImageManifest, manifestBuf) @@ -857,7 +883,7 @@ func TestDeleteBlobsInUse(t *testing.T) { log := log.Logger{Logger: zerolog.New(os.Stdout)} metrics := monitoring.NewMetricsServer(false, log) - if testcase.storageType == "s3" { + if testcase.storageType == storageConstants.S3StorageDriverName { skipIt(t) uuid, err := guuid.NewV4() @@ -868,7 +894,8 @@ func TestDeleteBlobsInUse(t *testing.T) { testDir = path.Join("/oci-repo-test", uuid.String()) tdir = t.TempDir() - store, imgStore, _ = createObjectsStore(testDir, tdir) + store, imgStore, _ = createObjectsStore(testDir, tdir, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay) defer cleanupStorage(store, testDir) } else { @@ -878,8 +905,10 @@ func TestDeleteBlobsInUse(t *testing.T) { Name: "cache", UseRelPaths: true, }, log) - imgStore = local.NewImageStore(tdir, true, storageConstants.DefaultGCDelay, true, - true, log, metrics, nil, cacheDriver) + driver := local.New(true) + imgStore = imagestore.NewImageStore(tdir, tdir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, + true, log, metrics, nil, driver, cacheDriver) } Convey("Setup manifest", t, func() { @@ -961,12 +990,12 @@ func TestDeleteBlobsInUse(t *testing.T) { So(err, ShouldBeNil) }) - if testcase.storageType != "s3" { + if testcase.storageType != storageConstants.S3StorageDriverName { Convey("get image manifest error", func() { err := os.Chmod(path.Join(imgStore.RootDir(), "repo", "blobs", "sha256", manifestDigest.Encoded()), 0o000) So(err, ShouldBeNil) - ok, _ := storageCommon.IsBlobReferenced(imgStore, "repo", unusedDigest, log.Logger) + ok, _ := storageCommon.IsBlobReferenced(imgStore, "repo", unusedDigest, log) So(ok, ShouldBeFalse) err = os.Chmod(path.Join(imgStore.RootDir(), "repo", "blobs", "sha256", manifestDigest.Encoded()), 0o755) @@ -1011,6 +1040,7 @@ func TestDeleteBlobsInUse(t *testing.T) { var cdigest godigest.Digest var cblob []byte + //nolint: dupl for i := 0; i < 4; i++ { // upload image config blob upload, err = imgStore.NewBlobUpload(repoName) @@ -1067,6 +1097,12 @@ func TestDeleteBlobsInUse(t *testing.T) { indexManifestDigest, _, err := imgStore.PutImageManifest(repoName, "index", ispec.MediaTypeImageIndex, indexContent) So(err, ShouldBeNil) + Convey("Try to delete manifest being referenced by image index", func() { + // modifying multi arch images should not be allowed + err := imgStore.DeleteImageManifest(repoName, digest.String(), false) + So(err, ShouldEqual, zerr.ErrManifestReferenced) + }) + Convey("Try to delete blob currently in use", func() { // layer blob err := imgStore.DeleteBlob("test", bdgst1) @@ -1103,13 +1139,13 @@ func TestDeleteBlobsInUse(t *testing.T) { So(err, ShouldBeNil) }) - if testcase.storageType != "s3" { + if testcase.storageType != storageConstants.S3StorageDriverName { Convey("repo not found", func() { // delete repo err := os.RemoveAll(path.Join(imgStore.RootDir(), repoName)) So(err, ShouldBeNil) - ok, err := storageCommon.IsBlobReferenced(imgStore, repoName, bdgst1, log.Logger) + ok, err := storageCommon.IsBlobReferenced(imgStore, repoName, bdgst1, log) So(err, ShouldNotBeNil) So(ok, ShouldBeFalse) }) @@ -1118,7 +1154,7 @@ func TestDeleteBlobsInUse(t *testing.T) { err := os.Remove(path.Join(imgStore.RootDir(), repoName, "index.json")) So(err, ShouldBeNil) - ok, err := storageCommon.IsBlobReferenced(imgStore, repoName, bdgst1, log.Logger) + ok, err := storageCommon.IsBlobReferenced(imgStore, repoName, bdgst1, log) So(err, ShouldNotBeNil) So(ok, ShouldBeFalse) }) @@ -1127,7 +1163,7 @@ func TestDeleteBlobsInUse(t *testing.T) { err := os.Remove(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", indexManifestDigest.Encoded())) So(err, ShouldBeNil) - ok, err := storageCommon.IsBlobReferenced(imgStore, repoName, unusedDigest, log.Logger) + ok, err := storageCommon.IsBlobReferenced(imgStore, repoName, unusedDigest, log) So(err, ShouldNotBeNil) So(ok, ShouldBeFalse) }) @@ -1148,22 +1184,25 @@ func TestStorageHandler(t *testing.T) { var secondRootDir string var thirdRootDir string - if testcase.storageType == "s3" { + if testcase.storageType == storageConstants.S3StorageDriverName { skipIt(t) var firstStorageDriver driver.StorageDriver var secondStorageDriver driver.StorageDriver var thirdStorageDriver driver.StorageDriver firstRootDir = "/util_test1" - firstStorageDriver, firstStore, _ = createObjectsStore(firstRootDir, t.TempDir()) + firstStorageDriver, firstStore, _ = createObjectsStore(firstRootDir, t.TempDir(), + storageConstants.DefaultGCDelay, storageConstants.DefaultUntaggedImgeRetentionDelay) defer cleanupStorage(firstStorageDriver, firstRootDir) secondRootDir = "/util_test2" - secondStorageDriver, secondStore, _ = createObjectsStore(secondRootDir, t.TempDir()) + secondStorageDriver, secondStore, _ = createObjectsStore(secondRootDir, t.TempDir(), + storageConstants.DefaultGCDelay, storageConstants.DefaultUntaggedImgeRetentionDelay) defer cleanupStorage(secondStorageDriver, secondRootDir) thirdRootDir = "/util_test3" - thirdStorageDriver, thirdStore, _ = createObjectsStore(thirdRootDir, t.TempDir()) + thirdStorageDriver, thirdStore, _ = createObjectsStore(thirdRootDir, t.TempDir(), + storageConstants.DefaultGCDelay, storageConstants.DefaultUntaggedImgeRetentionDelay) defer cleanupStorage(thirdStorageDriver, thirdRootDir) } else { // Create temporary directory @@ -1175,15 +1214,17 @@ func TestStorageHandler(t *testing.T) { metrics := monitoring.NewMetricsServer(false, log) + driver := local.New(true) + // Create ImageStore - firstStore = local.NewImageStore(firstRootDir, false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil) + firstStore = imagestore.NewImageStore(firstRootDir, firstRootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, driver, nil) - secondStore = local.NewImageStore(secondRootDir, false, - storageConstants.DefaultGCDelay, false, false, log, metrics, nil, nil) + secondStore = imagestore.NewImageStore(secondRootDir, secondRootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, driver, nil) - thirdStore = local.NewImageStore(thirdRootDir, false, storageConstants.DefaultGCDelay, - false, false, log, metrics, nil, nil) + thirdStore = imagestore.NewImageStore(thirdRootDir, thirdRootDir, false, false, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, false, false, log, metrics, nil, driver, nil) } Convey("Test storage handler", t, func() { @@ -1226,3 +1267,1708 @@ func TestRoutePrefix(t *testing.T) { So(routePrefix, ShouldEqual, "/a") }) } + +func TestGarbageCollectImageManifest(t *testing.T) { + for _, testcase := range testCases { + testcase := testcase + t.Run(testcase.testCaseName, func(t *testing.T) { + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + + Convey("Repo layout", t, func(c C) { + Convey("Garbage collect with default/long delay", func() { + var imgStore storageTypes.ImageStore + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, driver, cacheDriver) + } + + repoName := "gc-long" + + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + bdigest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap := make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob(repoName, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + digest := godigest.FromBytes(manifestBuf) + + _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + // put artifact referencing above image + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: "application/vnd.oci.image.manifest.v1+json", + Digest: digest, + Size: int64(len(manifestBuf)), + }, + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + err = imgStore.DeleteImageManifest(repoName, digest.String(), false) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + }) + + Convey("Garbage collect with short delay", func() { + var imgStore storageTypes.ImageStore + + gcDelay := 1 * time.Second + + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, gcDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, true, gcDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, + true, log, metrics, nil, driver, cacheDriver) + } + + // upload orphan blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + odigest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, odigest) + So(err, ShouldBeNil) + + // sleep so orphan blob can be GC'ed + time.Sleep(5 * time.Second) + + // upload blob + upload, err = imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content = []byte("test-data2") + buf = bytes.NewBuffer(content) + buflen = buf.Len() + bdigest := godigest.FromBytes(content) + + blob, err = imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap := make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob(repoName, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + digest := godigest.FromBytes(manifestBuf) + + _, _, err = imgStore.PutImageManifest(repoName, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + // put artifact referencing above image + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: int64(len(manifestBuf)), + }, + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push artifact manifest pointing to artifact above + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: artifactDigest, + Size: int64(len(artifactManifestBuf)), + } + artifactManifest.ArtifactType = "application/forArtifact" //nolint: goconst + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactOfArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + _, _, err = imgStore.PutImageManifest(repoName, artifactOfArtifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push orphan artifact (missing subject) + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: godigest.FromBytes([]byte("miss")), + Size: int64(30), + } + artifactManifest.ArtifactType = "application/orphan" //nolint: goconst + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + orphanArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + + // push orphan artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, orphanArtifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push oras manifest pointing to manifest + orasArtifactManifest := artifactspec.Manifest{} + orasArtifactManifest.ArtifactType = "signature-example" //nolint: goconst + orasArtifactManifest.Subject = &artifactspec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: int64(len(manifestBuf)), + } + orasArtifactManifest.Blobs = []artifactspec.Descriptor{} + + orasArtifactManifestBuf, err := json.Marshal(orasArtifactManifest) + So(err, ShouldBeNil) + + orasDigest := godigest.FromBytes(orasArtifactManifestBuf) + + // push oras manifest + _, _, err = imgStore.PutImageManifest(repoName, orasDigest.Encoded(), + artifactspec.MediaTypeArtifactManifest, orasArtifactManifestBuf) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // sleep so orphan blob can be GC'ed + time.Sleep(5 * time.Second) + + Convey("Garbage collect blobs after manifest is removed", func() { + err = imgStore.DeleteImageManifest(repoName, digest.String(), false) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check artifacts are gc'ed + _, _, _, err := imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orphanArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + // check it gc'ed repo + exists := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(exists, ShouldBeFalse) + }) + + Convey("Garbage collect - don't gc manifests/blobs which are referenced by another image", func() { + // upload same image with another tag + _, _, err = imgStore.PutImageManifest(repoName, "2.0", ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest(repoName, tag, false) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, digest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // orphan artifact should be deleted + _, _, _, err = imgStore.GetImageManifest(repoName, orphanArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + // check artifacts manifests + _, _, _, err := imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldBeNil) + }) + }) + + Convey("Garbage collect with dedupe", func() { + // garbage-collect is repo-local and dedupe is global and they can interact in strange ways + var imgStore storageTypes.ImageStore + + gcDelay := 5 * time.Second + + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, gcDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, true, gcDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, driver, cacheDriver) + } + + // first upload an image to the first repo and wait for GC timeout + + repo1Name := "gc1" + + // upload blob + upload, err := imgStore.NewBlobUpload(repo1Name) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + bdigest := godigest.FromBytes(content) + tdigest := bdigest + + blob, err := imgStore.PutBlobChunk(repo1Name, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repo1Name, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap := make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest := test.GetRandomImageConfig() + _, clen, err := imgStore.FullBlobUpload(repo1Name, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err := imgStore.CheckBlob(repo1Name, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err := json.Marshal(manifest) + So(err, ShouldBeNil) + + _, _, err = imgStore.PutImageManifest(repo1Name, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // sleep so past GC timeout + time.Sleep(10 * time.Second) + + hasBlob, _, err = imgStore.CheckBlob(repo1Name, tdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // upload another image into a second repo with the same blob contents so dedupe is triggered + + repo2Name := "gc2" + + upload, err = imgStore.NewBlobUpload(repo2Name) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + buf = bytes.NewBuffer(content) + buflen = buf.Len() + + blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap = make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest = test.GetRandomImageConfig() + _, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err = json.Marshal(manifest) + So(err, ShouldBeNil) + + _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repo2Name, bdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // immediately upload any other image to second repo which should invoke GC inline, but expect layers to persist + + upload, err = imgStore.NewBlobUpload(repo2Name) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content = []byte("test-data-more") + buf = bytes.NewBuffer(content) + buflen = buf.Len() + bdigest = godigest.FromBytes(content) + + blob, err = imgStore.PutBlobChunk(repo2Name, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repo2Name, upload, buf, bdigest) + So(err, ShouldBeNil) + + annotationsMap = make(map[string]string) + annotationsMap[ispec.AnnotationRefName] = tag + + cblob, cdigest = test.GetRandomImageConfig() + _, clen, err = imgStore.FullBlobUpload(repo2Name, bytes.NewReader(cblob), cdigest) + So(err, ShouldBeNil) + So(clen, ShouldEqual, len(cblob)) + hasBlob, _, err = imgStore.CheckBlob(repo2Name, cdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + manifest = ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: "application/vnd.oci.image.config.v1+json", + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: "application/vnd.oci.image.layer.v1.tar", + Digest: bdigest, + Size: int64(buflen), + }, + }, + Annotations: annotationsMap, + } + + manifest.SchemaVersion = 2 + manifestBuf, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest := godigest.FromBytes(manifestBuf) + + _, _, err = imgStore.PutImageManifest(repo2Name, tag, ispec.MediaTypeImageManifest, manifestBuf) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repo2Name) + So(err, ShouldBeNil) + + // original blob should exist + hasBlob, _, err = imgStore.CheckBlob(repo2Name, tdigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + _, _, _, err = imgStore.GetImageManifest(repo2Name, digest.String()) + So(err, ShouldBeNil) + }) + }) + }) + } +} + +func TestGarbageCollectImageIndex(t *testing.T) { + for _, testcase := range testCases { + testcase := testcase + t.Run(testcase.testCaseName, func(t *testing.T) { + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + + Convey("Repo layout", t, func(c C) { + Convey("Garbage collect with default/long delay", func() { + var imgStore storageTypes.ImageStore + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, true, storageConstants.DefaultGCDelay, + storageConstants.DefaultUntaggedImgeRetentionDelay, true, true, log, metrics, nil, driver, cacheDriver) + } + + repoName := "gc-long" + + bdgst, digest, indexDigest, indexSize := pushRandomImageIndex(imgStore, repoName) + + // put artifact referencing above image + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: indexDigest, + Size: indexSize, + }, + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest referencing index image + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: indexSize, + } + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest referencing a manifest from index image + _, _, err = imgStore.PutImageManifest(repoName, artifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + hasBlob, _, err := imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + Convey("delete index manifest, layers should be persisted", func() { + err = imgStore.DeleteImageManifest(repoName, indexDigest.String(), false) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + // check last manifest from index image + hasBlob, _, err = imgStore.CheckBlob(repoName, digest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + }) + }) + + Convey("Garbage collect with short delay", func() { + var imgStore storageTypes.ImageStore + + gcDelay := 5 * time.Second + imageRetentionDelay := 5 * time.Second + + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, gcDelay, imageRetentionDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, true, gcDelay, + imageRetentionDelay, true, true, log, metrics, nil, driver, cacheDriver) + } + + // upload orphan blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + odigest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, odigest) + So(err, ShouldBeNil) + + bdgst, digest, indexDigest, indexSize := pushRandomImageIndex(imgStore, repoName) + + // put artifact referencing above image + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + // push artifact manifest pointing to index + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: indexDigest, + Size: indexSize, + }, + ArtifactType: "application/forIndex", + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: int64(len(content)), + } + artifactManifest.ArtifactType = "application/forManifestInIndex" + + artifactManifestIndexBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactManifestIndexDigest := godigest.FromBytes(artifactManifestIndexBuf) + + // push artifact manifest referencing a manifest from index image + _, _, err = imgStore.PutImageManifest(repoName, artifactManifestIndexDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestIndexBuf) + So(err, ShouldBeNil) + + // push artifact manifest pointing to artifact above + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: artifactDigest, + Size: int64(len(artifactManifestBuf)), + } + artifactManifest.ArtifactType = "application/forArtifact" + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactOfArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + _, _, err = imgStore.PutImageManifest(repoName, artifactOfArtifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push orphan artifact (missing subject) + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: godigest.FromBytes([]byte("miss")), + Size: int64(30), + } + artifactManifest.ArtifactType = "application/orphan" + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + orphanArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + + // push orphan artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, orphanArtifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push oras manifest pointing to index image + orasArtifactManifest := artifactspec.Manifest{} + orasArtifactManifest.ArtifactType = "signature-example" + orasArtifactManifest.Subject = &artifactspec.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: indexDigest, + Size: indexSize, + } + orasArtifactManifest.Blobs = []artifactspec.Descriptor{} + + orasArtifactManifestBuf, err := json.Marshal(orasArtifactManifest) + So(err, ShouldBeNil) + + orasDigest := godigest.FromBytes(orasArtifactManifestBuf) + + // push oras manifest + _, _, err = imgStore.PutImageManifest(repoName, orasDigest.Encoded(), + artifactspec.MediaTypeArtifactManifest, orasArtifactManifestBuf) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + hasBlob, _, err := imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + time.Sleep(5 * time.Second) + + Convey("delete inner referenced manifest", func() { + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + // check orphan artifact is gc'ed + _, _, _, err = imgStore.GetImageManifest(repoName, orphanArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest(repoName, artifactDigest.String(), false) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + }) + + Convey("delete index manifest, references should not be persisted", func() { + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + // check orphan artifact is gc'ed + _, _, _, err = imgStore.GetImageManifest(repoName, orphanArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest(repoName, indexDigest.String(), false) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + // isn't yet gced because manifests part of index are removed after gcReferrers, + // so the artifacts pointing to manifest which are part of index are not removed after a first gcRepo + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + + // orphan blob + hasBlob, _, err = imgStore.CheckBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check last manifest from index image + hasBlob, _, err = imgStore.CheckBlob(repoName, digest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check referrer is gc'ed + _, _, _, err := imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldNotBeNil) + + // this will remove refferers of manifests part of image index + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldNotBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check it gc'ed repo + exists := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(exists, ShouldBeFalse) + }) + }) + }) + }) + } +} + +func TestGarbageCollectChainedImageIndexes(t *testing.T) { + for _, testcase := range testCases { + testcase := testcase + t.Run(testcase.testCaseName, func(t *testing.T) { + log := log.Logger{Logger: zerolog.New(os.Stdout)} + metrics := monitoring.NewMetricsServer(false, log) + + Convey("Garbage collect with short delay", t, func() { + var imgStore storageTypes.ImageStore + + gcDelay := 5 * time.Second + imageRetentionDelay := 5 * time.Second + + if testcase.storageType == storageConstants.S3StorageDriverName { + skipIt(t) + + uuid, err := guuid.NewV4() + if err != nil { + panic(err) + } + + testDir := path.Join("/oci-repo-test", uuid.String()) + tdir := t.TempDir() + + var store driver.StorageDriver + store, imgStore, _ = createObjectsStore(testDir, tdir, gcDelay, imageRetentionDelay) + defer cleanupStorage(store, testDir) + } else { + dir := t.TempDir() + + cacheDriver, _ := storage.Create("boltdb", cache.BoltDBDriverParameters{ + RootDir: dir, + Name: "cache", + UseRelPaths: true, + }, log) + + driver := local.New(true) + + imgStore = imagestore.NewImageStore(dir, dir, true, true, gcDelay, + imageRetentionDelay, true, true, log, metrics, nil, driver, cacheDriver) + } + + // upload orphan blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + content := []byte("test-data1") + buf := bytes.NewBuffer(content) + buflen := buf.Len() + odigest := godigest.FromBytes(content) + + blob, err := imgStore.PutBlobChunk(repoName, upload, 0, int64(buflen), buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, odigest) + So(err, ShouldBeNil) + + content = []byte("this is a blob") + bdgst := godigest.FromBytes(content) + So(bdgst, ShouldNotBeNil) + + _, bsize, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), bdgst) + So(err, ShouldBeNil) + So(bsize, ShouldEqual, len(content)) + + artifactBlob := []byte("artifact") + artifactBlobDigest := godigest.FromBytes(artifactBlob) + + // push layer + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(artifactBlob), artifactBlobDigest) + So(err, ShouldBeNil) + + // push config + _, _, err = imgStore.FullBlobUpload(repoName, bytes.NewReader(ispec.DescriptorEmptyJSON.Data), + ispec.DescriptorEmptyJSON.Digest) + So(err, ShouldBeNil) + + var index ispec.Index + index.SchemaVersion = 2 + index.MediaType = ispec.MediaTypeImageIndex + + var digest godigest.Digest + for i := 0; i < 4; i++ { //nolint: dupl + // upload image config blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest := test.GetRandomImageConfig() + buf := bytes.NewBuffer(cblob) + buflen := buf.Len() + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst, + Size: bsize, + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: digest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(content)), + }) + + // for each manifest inside index, push an artifact + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: int64(len(content)), + }, + ArtifactType: "application/forManifestInInnerIndex", + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + } + + // also add a new image index inside this one + var innerIndex ispec.Index + innerIndex.SchemaVersion = 2 + innerIndex.MediaType = ispec.MediaTypeImageIndex + + for i := 0; i < 3; i++ { //nolint: dupl + // upload image config blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest := test.GetRandomImageConfig() + buf := bytes.NewBuffer(cblob) + buflen := buf.Len() + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst, + Size: bsize, + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest := godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + innerIndex.Manifests = append(innerIndex.Manifests, ispec.Descriptor{ + Digest: digest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(content)), + }) + } + + // upload inner index image + innerIndexContent, err := json.Marshal(index) + So(err, ShouldBeNil) + innerIndexDigest := godigest.FromBytes(innerIndexContent) + So(innerIndexDigest, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest(repoName, innerIndexDigest.String(), + ispec.MediaTypeImageIndex, innerIndexContent) + So(err, ShouldBeNil) + + // add inner index into root index + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: innerIndexDigest, + MediaType: ispec.MediaTypeImageIndex, + Size: int64(len(innerIndexContent)), + }) + + // push root index + // upload index image + indexContent, err := json.Marshal(index) + So(err, ShouldBeNil) + indexDigest := godigest.FromBytes(indexContent) + So(indexDigest, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + So(err, ShouldBeNil) + + artifactManifest := ispec.Manifest{ + MediaType: ispec.MediaTypeImageManifest, + Layers: []ispec.Descriptor{ + { + MediaType: "application/octet-stream", + Digest: artifactBlobDigest, + Size: int64(len(artifactBlob)), + }, + }, + Config: ispec.DescriptorEmptyJSON, + Subject: &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: indexDigest, + Size: int64(len(indexContent)), + }, + ArtifactType: "application/forIndex", + } + artifactManifest.SchemaVersion = 2 + + artifactManifestBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactDigest := godigest.FromBytes(artifactManifestBuf) + + // push artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, artifactDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: digest, + Size: int64(len(content)), + } + artifactManifest.ArtifactType = "application/forManifestInIndex" + + artifactManifestIndexBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactManifestIndexDigest := godigest.FromBytes(artifactManifestIndexBuf) + + // push artifact manifest referencing a manifest from index image + _, _, err = imgStore.PutImageManifest(repoName, artifactManifestIndexDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestIndexBuf) + So(err, ShouldBeNil) + + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: innerIndexDigest, + Size: int64(len(innerIndexContent)), + } + artifactManifest.ArtifactType = "application/forInnerIndex" + + artifactManifestInnerIndexBuf, err := json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactManifestInnerIndexDigest := godigest.FromBytes(artifactManifestInnerIndexBuf) + + // push artifact manifest referencing a manifest from index image + _, _, err = imgStore.PutImageManifest(repoName, artifactManifestInnerIndexDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestInnerIndexBuf) + So(err, ShouldBeNil) + + // push artifact manifest pointing to artifact above + + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: artifactDigest, + Size: int64(len(artifactManifestBuf)), + } + artifactManifest.ArtifactType = "application/forArtifact" + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + artifactOfArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + _, _, err = imgStore.PutImageManifest(repoName, artifactOfArtifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push orphan artifact (missing subject) + artifactManifest.Subject = &ispec.Descriptor{ + MediaType: ispec.MediaTypeImageManifest, + Digest: godigest.FromBytes([]byte("miss")), + Size: int64(30), + } + artifactManifest.ArtifactType = "application/orphan" + + artifactManifestBuf, err = json.Marshal(artifactManifest) + So(err, ShouldBeNil) + + orphanArtifactManifestDigest := godigest.FromBytes(artifactManifestBuf) + + // push orphan artifact manifest + _, _, err = imgStore.PutImageManifest(repoName, orphanArtifactManifestDigest.String(), + ispec.MediaTypeImageManifest, artifactManifestBuf) + So(err, ShouldBeNil) + + // push oras manifest pointing to index image + orasArtifactManifest := artifactspec.Manifest{} + orasArtifactManifest.ArtifactType = "signature-example" + orasArtifactManifest.Subject = &artifactspec.Descriptor{ + MediaType: ispec.MediaTypeImageIndex, + Digest: indexDigest, + Size: int64(len(indexContent)), + } + orasArtifactManifest.Blobs = []artifactspec.Descriptor{} + + orasArtifactManifestBuf, err := json.Marshal(orasArtifactManifest) + So(err, ShouldBeNil) + + orasDigest := godigest.FromBytes(orasArtifactManifestBuf) + + // push oras manifest + _, _, err = imgStore.PutImageManifest(repoName, orasDigest.Encoded(), + artifactspec.MediaTypeArtifactManifest, orasArtifactManifestBuf) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + hasBlob, _, err := imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, bdgst) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldBeNil) + So(hasBlob, ShouldEqual, true) + + time.Sleep(5 * time.Second) + + Convey("delete inner referenced manifest", func() { + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + // check orphan artifact is gc'ed + _, _, _, err = imgStore.GetImageManifest(repoName, orphanArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest(repoName, artifactDigest.String(), false) + So(err, ShouldBeNil) + + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + }) + + Convey("delete index manifest, references should not be persisted", func() { + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + // check orphan artifact is gc'ed + _, _, _, err = imgStore.GetImageManifest(repoName, orphanArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldBeNil) + + err = imgStore.DeleteImageManifest(repoName, indexDigest.String(), false) + So(err, ShouldBeNil) + + // this will remove artifacts pointing to root index which was remove + // it will also remove inner index because now although its referenced in index.json it has no tag + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, orasDigest.String()) + So(err, ShouldNotBeNil) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactOfArtifactManifestDigest.String()) + So(err, ShouldNotBeNil) + + // isn't yet gced because manifests part of index are removed after gcReferrers, + // so the artifacts pointing to manifest which are part of index are not removed after a single gcRepo + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldBeNil) + + // orphan blob + hasBlob, _, err = imgStore.CheckBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, _, err = imgStore.StatBlob(repoName, odigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check artifact is gc'ed + _, _, _, err := imgStore.GetImageManifest(repoName, artifactDigest.String()) + So(err, ShouldNotBeNil) + + // this will remove manifests referenced in inner index because even if they are referenced in index.json + // they do not have tags + // it will also remove referrers pointing to inner manifest + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + // check inner index artifact is gc'ed + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestInnerIndexDigest.String()) + So(err, ShouldNotBeNil) + + // this will remove referrers pointing to manifests referenced in inner index + err = imgStore.RunGCRepo(repoName) + So(err, ShouldBeNil) + + // check last manifest from index image + hasBlob, _, err = imgStore.CheckBlob(repoName, digest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + _, _, _, err = imgStore.GetImageManifest(repoName, artifactManifestIndexDigest.String()) + So(err, ShouldNotBeNil) + + hasBlob, _, err = imgStore.CheckBlob(repoName, artifactBlobDigest) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + hasBlob, _, err = imgStore.CheckBlob(repoName, bdgst) + So(err, ShouldNotBeNil) + So(hasBlob, ShouldEqual, false) + + // check it gc'ed repo + exists := imgStore.DirExists(path.Join(imgStore.RootDir(), repoName)) + So(exists, ShouldBeFalse) + }) + }) + }) + } +} + +func pushRandomImageIndex(imgStore storageTypes.ImageStore, repoName string, +) (godigest.Digest, godigest.Digest, godigest.Digest, int64) { + content := []byte("this is a blob") + bdgst := godigest.FromBytes(content) + So(bdgst, ShouldNotBeNil) + + _, bsize, err := imgStore.FullBlobUpload(repoName, bytes.NewReader(content), bdgst) + So(err, ShouldBeNil) + So(bsize, ShouldEqual, len(content)) + + var index ispec.Index + index.SchemaVersion = 2 + index.MediaType = ispec.MediaTypeImageIndex + + var digest godigest.Digest + + for i := 0; i < 4; i++ { //nolint: dupl + // upload image config blob + upload, err := imgStore.NewBlobUpload(repoName) + So(err, ShouldBeNil) + So(upload, ShouldNotBeEmpty) + + cblob, cdigest := test.GetRandomImageConfig() + buf := bytes.NewBuffer(cblob) + buflen := buf.Len() + blob, err := imgStore.PutBlobChunkStreamed(repoName, upload, buf) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + err = imgStore.FinishBlobUpload(repoName, upload, buf, cdigest) + So(err, ShouldBeNil) + So(blob, ShouldEqual, buflen) + + // create a manifest + manifest := ispec.Manifest{ + Config: ispec.Descriptor{ + MediaType: ispec.MediaTypeImageConfig, + Digest: cdigest, + Size: int64(len(cblob)), + }, + Layers: []ispec.Descriptor{ + { + MediaType: ispec.MediaTypeImageLayer, + Digest: bdgst, + Size: bsize, + }, + }, + } + manifest.SchemaVersion = 2 + content, err = json.Marshal(manifest) + So(err, ShouldBeNil) + digest = godigest.FromBytes(content) + So(digest, ShouldNotBeNil) + _, _, err = imgStore.PutImageManifest(repoName, digest.String(), ispec.MediaTypeImageManifest, content) + So(err, ShouldBeNil) + + index.Manifests = append(index.Manifests, ispec.Descriptor{ + Digest: digest, + MediaType: ispec.MediaTypeImageManifest, + Size: int64(len(content)), + }) + } + + // upload index image + indexContent, err := json.Marshal(index) + So(err, ShouldBeNil) + indexDigest := godigest.FromBytes(indexContent) + So(indexDigest, ShouldNotBeNil) + + _, _, err = imgStore.PutImageManifest(repoName, "1.0", ispec.MediaTypeImageIndex, indexContent) + So(err, ShouldBeNil) + + return bdgst, digest, indexDigest, int64(len(indexContent)) +} diff --git a/pkg/storage/types/types.go b/pkg/storage/types/types.go index 1d66d7d3..e2a5a18b 100644 --- a/pkg/storage/types/types.go +++ b/pkg/storage/types/types.go @@ -4,6 +4,7 @@ import ( "io" "time" + storagedriver "github.com/docker/distribution/registry/storage/driver" godigest "github.com/opencontainers/go-digest" ispec "github.com/opencontainers/image-spec/specs-go/v1" artifactspec "github.com/oras-project/artifacts-spec/specs-go/v1" @@ -38,7 +39,7 @@ type ImageStore interface { //nolint:interfacebloat DeleteBlobUpload(repo, uuid string) error BlobPath(repo string, digest godigest.Digest) string CheckBlob(repo string, digest godigest.Digest) (bool, int64, error) - StatBlob(repo string, digest godigest.Digest) (bool, int64, error) + StatBlob(repo string, digest godigest.Digest) (bool, int64, time.Time, error) GetBlob(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, ) (io.ReadCloser, int64, int64, error) @@ -52,4 +53,22 @@ type ImageStore interface { //nolint:interfacebloat RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) RunDedupeForDigest(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error GetNextDigestWithBlobPaths(lastDigests []godigest.Digest) (godigest.Digest, []string, error) + GetAllBlobs(repo string) ([]string, error) +} + +type Driver interface { //nolint:interfacebloat + Name() string + EnsureDir(path string) error + DirExists(path string) bool + Reader(path string, offset int64) (io.ReadCloser, error) + ReadFile(path string) ([]byte, error) + Delete(path string) error + Stat(path string) (storagedriver.FileInfo, error) + Writer(filepath string, append bool) (storagedriver.FileWriter, error) //nolint: predeclared + WriteFile(filepath string, content []byte) (int, error) + Walk(path string, f storagedriver.WalkFn) error + List(fullpath string) ([]string, error) + Move(sourcePath string, destPath string) error + SameFile(path1, path2 string) bool + Link(src, dest string) error } diff --git a/pkg/test/common.go b/pkg/test/common.go index 5525101a..5e99f858 100644 --- a/pkg/test/common.go +++ b/pkg/test/common.go @@ -2068,7 +2068,7 @@ func GetDefaultLayersBlobs() [][]byte { } func GetDefaultImageStore(rootDir string, log zLog.Logger) stypes.ImageStore { - return local.NewImageStore(rootDir, false, time.Hour, false, false, log, + return local.NewImageStore(rootDir, false, false, time.Hour, time.Hour, false, false, log, monitoring.NewMetricsServer(false, log), mocks.MockedLint{ LintFn: func(repo string, manifestDigest godigest.Digest, imageStore stypes.ImageStore) (bool, error) { diff --git a/pkg/test/mocks/cache_mock.go b/pkg/test/mocks/cache_mock.go index 1d4e95e8..ef8dbf16 100644 --- a/pkg/test/mocks/cache_mock.go +++ b/pkg/test/mocks/cache_mock.go @@ -17,6 +17,16 @@ type CacheMock struct { // Delete a blob from the cachedb. DeleteBlobFn func(digest godigest.Digest, path string) error + + UsesRelativePathsFn func() bool +} + +func (cacheMock CacheMock) UsesRelativePaths() bool { + if cacheMock.UsesRelativePathsFn != nil { + return cacheMock.UsesRelativePaths() + } + + return false } func (cacheMock CacheMock) Name() string { diff --git a/pkg/test/mocks/image_store_mock.go b/pkg/test/mocks/image_store_mock.go index 63d05e85..7736d946 100644 --- a/pkg/test/mocks/image_store_mock.go +++ b/pkg/test/mocks/image_store_mock.go @@ -35,7 +35,7 @@ type MockedImageStore struct { DeleteBlobUploadFn func(repo string, uuid string) error BlobPathFn func(repo string, digest godigest.Digest) string CheckBlobFn func(repo string, digest godigest.Digest) (bool, int64, error) - StatBlobFn func(repo string, digest godigest.Digest) (bool, int64, error) + StatBlobFn func(repo string, digest godigest.Digest) (bool, int64, time.Time, error) GetBlobPartialFn func(repo string, digest godigest.Digest, mediaType string, from, to int64, ) (io.ReadCloser, int64, int64, error) GetBlobFn func(repo string, digest godigest.Digest, mediaType string) (io.ReadCloser, int64, error) @@ -51,6 +51,7 @@ type MockedImageStore struct { RunDedupeBlobsFn func(interval time.Duration, sch *scheduler.Scheduler) RunDedupeForDigestFn func(digest godigest.Digest, dedupe bool, duplicateBlobs []string) error GetNextDigestWithBlobPathsFn func(lastDigests []godigest.Digest) (godigest.Digest, []string, error) + GetAllBlobsFn func(repo string) ([]string, error) } func (is MockedImageStore) Lock(t *time.Time) { @@ -142,6 +143,14 @@ func (is MockedImageStore) GetImageTags(name string) ([]string, error) { return []string{}, nil } +func (is MockedImageStore) GetAllBlobs(repo string) ([]string, error) { + if is.GetAllBlobsFn != nil { + return is.GetAllBlobsFn(repo) + } + + return []string{}, nil +} + func (is MockedImageStore) DeleteImageManifest(name string, reference string, detectCollision bool) error { if is.DeleteImageManifestFn != nil { return is.DeleteImageManifestFn(name, reference, detectCollision) @@ -252,12 +261,12 @@ func (is MockedImageStore) CheckBlob(repo string, digest godigest.Digest) (bool, return true, 0, nil } -func (is MockedImageStore) StatBlob(repo string, digest godigest.Digest) (bool, int64, error) { +func (is MockedImageStore) StatBlob(repo string, digest godigest.Digest) (bool, int64, time.Time, error) { if is.StatBlobFn != nil { return is.StatBlobFn(repo, digest) } - return true, 0, nil + return true, 0, time.Time{}, nil } func (is MockedImageStore) GetBlobPartial(repo string, digest godigest.Digest, mediaType string, from, to int64, diff --git a/pkg/test/oci-layout/oci_layout_test.go b/pkg/test/oci-layout/oci_layout_test.go index 227be7b2..14c63fe1 100644 --- a/pkg/test/oci-layout/oci_layout_test.go +++ b/pkg/test/oci-layout/oci_layout_test.go @@ -346,7 +346,7 @@ func TestExtractImageDetails(t *testing.T) { Convey("extractImageDetails good workflow", t, func() { dir := t.TempDir() testLogger := log.NewLogger("debug", "") - imageStore := local.NewImageStore(dir, false, 0, false, false, + imageStore := local.NewImageStore(dir, false, false, 0, 0, false, false, testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil) storeController := storage.StoreController{ @@ -382,7 +382,7 @@ func TestExtractImageDetails(t *testing.T) { Convey("extractImageDetails bad ispec.ImageManifest", t, func() { dir := t.TempDir() testLogger := log.NewLogger("debug", "") - imageStore := local.NewImageStore(dir, false, 0, false, false, + imageStore := local.NewImageStore(dir, false, false, 0, 0, false, false, testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil) storeController := storage.StoreController{ @@ -402,7 +402,7 @@ func TestExtractImageDetails(t *testing.T) { Convey("extractImageDetails bad imageConfig", t, func() { dir := t.TempDir() testLogger := log.NewLogger("debug", "") - imageStore := local.NewImageStore(dir, false, 0, false, false, + imageStore := local.NewImageStore(dir, false, false, 0, 0, false, false, testLogger, monitoring.NewMetricsServer(false, testLogger), nil, nil) storeController := storage.StoreController{ diff --git a/test/blackbox/garbage_collect.bats b/test/blackbox/garbage_collect.bats new file mode 100644 index 00000000..73d5f2a4 --- /dev/null +++ b/test/blackbox/garbage_collect.bats @@ -0,0 +1,159 @@ +load helpers_zot + +function verify_prerequisites { + if [ ! $(command -v curl) ]; then + echo "you need to install curl as a prerequisite to running the tests" >&3 + return 1 + fi + + if [ ! $(command -v jq) ]; then + echo "you need to install jq as a prerequisite to running the tests" >&3 + return 1 + fi + + return 0 +} + +function setup_file() { + # Verify prerequisites are available + if ! $(verify_prerequisites); then + exit 1 + fi + + # Download test data to folder common for the entire suite, not just this file + skopeo --insecure-policy copy --format=oci docker://ghcr.io/project-zot/golang:1.20 oci:${TEST_DATA_DIR}/golang:1.20 + # Setup zot server + local zot_root_dir=${BATS_FILE_TMPDIR}/zot + local zot_config_file=${BATS_FILE_TMPDIR}/zot_config.json + local oci_data_dir=${BATS_FILE_TMPDIR}/oci + mkdir -p ${zot_root_dir} + mkdir -p ${oci_data_dir} + cat > ${zot_config_file}< signature.json + run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --image-spec v1.1-image --artifact-type 'signature/example' ./signature.json:application/json + [ "$status" -eq 0 ] + # attach sbom to image + echo "{\"version\": \"0.0.0.0\", \"artifact\": \"'127.0.0.1:8080/golang:1.20'\", \"contents\": \"good\"}" > sbom.json + run oras attach --plain-http 127.0.0.1:8080/golang:1.20 --image-spec v1.1-image --artifact-type 'sbom/example' ./sbom.json:application/json + [ "$status" -eq 0 ] + + # attach signature to index image + run oras attach --plain-http 127.0.0.1:8080/busybox:latest --image-spec v1.1-image --artifact-type 'signature/example' ./signature.json:application/json + [ "$status" -eq 0 ] + # attach sbom to index image + echo "{\"version\": \"0.0.0.0\", \"artifact\": \"'127.0.0.1:8080/golang:1.20'\", \"contents\": \"good\"}" > sbom.json + run oras attach --plain-http 127.0.0.1:8080/busybox:latest --image-spec v1.1-image --artifact-type 'sbom/example' ./sbom.json:application/json + [ "$status" -eq 0 ] +} + +@test "push OCI artifact with regclient" { + run regctl registry set 127.0.0.1:8080 --tls disabled + [ "$status" -eq 0 ] + + run regctl artifact put --artifact-type application/vnd.example.artifact --subject 127.0.0.1:8080/golang:1.20 <