diff --git a/.github/workflows/gc-stress-test.yaml b/.github/workflows/gc-stress-test.yaml index e6bd0c83..8140a85f 100644 --- a/.github/workflows/gc-stress-test.yaml +++ b/.github/workflows/gc-stress-test.yaml @@ -24,6 +24,7 @@ jobs: go-version: 1.20.x - name: Run zb + id: bench run: | make binary make bench @@ -35,6 +36,13 @@ jobs: # clean zot storage sudo rm -rf /tmp/zot + continue-on-error: true + + - name: Check on failures + if: steps.bench.outcome != 'success' + run: | + cat /tmp/gc-referrers-bench-local.log + exit 1 gc-stress-local: name: GC(without referrers) on filesystem with short interval @@ -48,6 +56,7 @@ jobs: go-version: 1.20.x - name: Run zb + id: bench run: | make binary make bench @@ -59,3 +68,188 @@ jobs: # clean zot storage sudo rm -rf /tmp/zot + continue-on-error: true + + - name: Check on failures + if: steps.bench.outcome != 'success' + run: | + cat /tmp/gc-bench-local.log + exit 1 + + gc-referrers-stress-s3: + name: GC(with referrers) on S3(minio) with short interval + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/clean-runner + - uses: actions/setup-go@v4 + with: + cache: false + go-version: 1.20.x + - name: Setup localstack service + run: | + pip install localstack # Install LocalStack cli + docker pull localstack/localstack:1.3 # Make sure to pull the latest version of the image + localstack start -d # Start LocalStack in the background + + echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container + localstack wait -t 30 # to become ready before timing out + echo "Startup complete" + + # aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}" + aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake + - name: Setup minio service + run: | + docker run -d -p 9000:9000 --name minio \ + -e "MINIO_ACCESS_KEY=minioadmin" \ + -e "MINIO_SECRET_KEY=minioadmin" \ + -v /tmp/data:/data \ + -v /tmp/config:/root/.minio \ + --health-cmd "curl http://localhost:9000/minio/health/live" \ + minio/minio:edge-cicd server /data + - name: Install py minio + run: pip3 install minio + + - name: Wait for minio to come up + run: | + sleep 10 + curl --connect-timeout 5 \ + --max-time 120 \ + --retry 12 \ + --retry-max-time 120 \ + 'http://localhost:9000/minio/health/live' + + - name: Create minio bucket + run: | + python3 - <<'EOF' + from minio import Minio + + try: + minio = Minio( + 'localhost:9000', + access_key='minioadmin', + secret_key='minioadmin', + secure=False + ) + except Exception as ex: + raise + + minio.make_bucket('zot-storage') + print(f'{minio.list_buckets()}') + EOF + + - name: Run zb + id: bench + run: | + make binary + make bench + ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3-minio.json & + sleep 10 + bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 + + killall -r zot-* + + # clean zot storage + sudo rm -rf /tmp/zot + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake + continue-on-error: true + + - name: Check on failures + if: steps.bench.outcome != 'success' + run: | + cat /tmp/gc-referrers-bench-s3.log + exit 1 + + gc-stress-s3: + name: GC(without referrers) on S3(minio) with short interval + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/clean-runner + - uses: actions/setup-go@v4 + with: + cache: false + go-version: 1.20.x + - name: Setup localstack service + run: | + pip install localstack # Install LocalStack cli + docker pull localstack/localstack:1.3 # Make sure to pull the latest version of the image + localstack start -d # Start LocalStack in the background + + echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container + localstack wait -t 30 # to become ready before timing out + echo "Startup complete" + + # aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}" + aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5 + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake + - name: Setup minio service + run: | + docker run -d -p 9000:9000 --name minio \ + -e "MINIO_ACCESS_KEY=minioadmin" \ + -e "MINIO_SECRET_KEY=minioadmin" \ + -v /tmp/data:/data \ + -v /tmp/config:/root/.minio \ + --health-cmd "curl http://localhost:9000/minio/health/live" \ + minio/minio:edge-cicd server /data + - name: Install py minio + run: pip3 install minio + + - name: Wait for minio to come up + run: | + sleep 10 + curl --connect-timeout 5 \ + --max-time 120 \ + --retry 12 \ + --retry-max-time 120 \ + 'http://localhost:9000/minio/health/live' + + - name: Create minio bucket + run: | + python3 - <<'EOF' + from minio import Minio + + try: + minio = Minio( + 'localhost:9000', + access_key='minioadmin', + secret_key='minioadmin', + secure=False + ) + except Exception as ex: + raise + + minio.make_bucket('zot-storage') + print(f'{minio.list_buckets()}') + EOF + + - name: Run zb + id: bench + run: | + make binary + make bench + ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3-minio.json & + sleep 10 + bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 + + killall -r zot-* + + # clean zot storage + sudo rm -rf /tmp/zot + env: + AWS_ACCESS_KEY_ID: fake + AWS_SECRET_ACCESS_KEY: fake + continue-on-error: true + + - name: Check on failures + if: steps.bench.outcome != 'success' + run: | + cat /tmp/gc-bench-s3.log + exit 1 diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml index 23eacf84..674b31bf 100644 --- a/.github/workflows/nightly.yaml +++ b/.github/workflows/nightly.yaml @@ -72,7 +72,7 @@ jobs: make run-blackbox-sync-nightly gc-referrers-stress-s3: - name: GC(with referrers) on S3 with short interval + name: GC(with referrers) on S3(localstack) with short interval runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -96,11 +96,13 @@ jobs: env: AWS_ACCESS_KEY_ID: fake AWS_SECRET_ACCESS_KEY: fake + - name: Run zb + id: bench run: | make binary make bench - ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3.json & + ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3-localstack.json & sleep 10 bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 @@ -111,9 +113,16 @@ jobs: env: AWS_ACCESS_KEY_ID: fake AWS_SECRET_ACCESS_KEY: fake + continue-on-error: true + + - name: Check on failures + if: steps.bench.outcome != 'success' + run: | + cat /tmp/gc-referrers-bench-s3.log + exit 1 gc-stress-s3: - name: GC(without referrers) on S3 with short interval + name: GC(without referrers) on S3(localstack) with short interval runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -137,11 +146,13 @@ jobs: env: AWS_ACCESS_KEY_ID: fake AWS_SECRET_ACCESS_KEY: fake + - name: Run zb + id: bench run: | make binary make bench - ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3.json & + ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3-localstack.json & sleep 10 bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 @@ -152,3 +163,10 @@ jobs: env: AWS_ACCESS_KEY_ID: fake AWS_SECRET_ACCESS_KEY: fake + continue-on-error: true + + - name: Check on failures + if: steps.bench.outcome != 'success' + run: | + cat /tmp/gc-bench-s3.log + exit 1 diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go index b234aa1a..e3dbc712 100644 --- a/pkg/storage/imagestore/imagestore.go +++ b/pkg/storage/imagestore/imagestore.go @@ -672,7 +672,7 @@ func (is *ImageStore) deleteImageManifest(repo, reference string, detectCollisio } if _, err := is.storeDriver.WriteFile(file, buf); err != nil { - is.log.Debug().Str("deleting reference", reference).Msg("") + is.log.Debug().Str("reference", reference).Str("repo", repo).Msg("error while updating index.json") return err } @@ -1660,7 +1660,6 @@ func (is *ImageStore) garbageCollectIndexReferrers(repo string, rootIndex ispec. if gced { count++ } - case ispec.MediaTypeImageManifest, artifactspec.MediaTypeArtifactManifest: image, err := common.GetImageManifest(is, repo, desc.Digest, is.log) if err != nil { @@ -1890,8 +1889,13 @@ func (is *ImageStore) garbageCollectBlobs(imgStore *ImageStore, repo string, } } + blobUploads, err := is.storeDriver.List(path.Join(is.RootDir(), repo, storageConstants.BlobUploadDir)) + if err != nil { + is.log.Debug().Str("repository", repo).Msg("unable to list .uploads/ dir") + } + // if we cleaned all blobs let's also remove the repo so that it won't be returned by catalog - if reaped == len(allBlobs) { + if len(allBlobs) > 0 && reaped == len(allBlobs) && len(blobUploads) == 0 { log.Info().Str("repository", repo).Msg("garbage collected all blobs, cleaning repo...") if err := is.storeDriver.Delete(path.Join(is.rootDir, repo)); err != nil { diff --git a/test/blackbox/restore_s3_blobs.bats b/test/blackbox/restore_s3_blobs.bats index 498aea50..f9a95ccd 100644 --- a/test/blackbox/restore_s3_blobs.bats +++ b/test/blackbox/restore_s3_blobs.bats @@ -17,13 +17,15 @@ function setup_file() { local zot_root_dir=${BATS_FILE_TMPDIR}/zot local zot_config_file_dedupe=${BATS_FILE_TMPDIR}/zot_config_dedupe.json local zot_config_file_nodedupe=${BATS_FILE_TMPDIR}/zot_config_nodedupe.json - local ZOT_LOG_FILE=${zot_root_dir}/zot-log.json + local ZOT_LOG_FILE_DEDUPE=${BATS_FILE_TMPDIR}/zot-log-dedupe.json + local ZOT_LOG_FILE_NODEDUPE=${BATS_FILE_TMPDIR}/zot-log-nodedupe.json + mkdir -p ${zot_root_dir} cat > ${zot_config_file_dedupe}< ${zot_config_file_nodedupe}<