Infrastructure/zot
0
Fork 0
mirror of https://github.com/project-zot/zot.git synced 2024-12-16 21:56:37 -05:00
Code Issues Activity

fix(ci): fix nighlty builds and print zot log on failure (#1799)

Browse source 
now gc stress on s3 storage is using minio for ci/cd builds
gc stress on s3 storage is using localstack for nightly builds

fixed(gc): make sure we don't remove repo if there are blobs
being uploaded or the number of blobs gc'ed is not 0

Signed-off-by: Petu Eusebiu <peusebiu@cisco.com>
This commit is contained in:
peusebiu 2023-09-20 19:25:06 +03:00 committed by GitHub
parent a11fe2d195
commit f164fb9e03
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 331 additions and 39 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

194
.github/workflows/gc-stress-test.yaml vendored
View file

@ -24,6 +24,7 @@ jobs:
go-version: 1.20.x
- name: Run zb
id: bench
run: |
make binary
make bench
@ -35,6 +36,13 @@ jobs:
# clean zot storage
sudo rm -rf /tmp/zot
continue-on-error: true
- name: Check on failures
if: steps.bench.outcome != 'success'
run: |
cat /tmp/gc-referrers-bench-local.log
exit 1
gc-stress-local:
name: GC(without referrers) on filesystem with short interval
@ -48,6 +56,7 @@ jobs:
go-version: 1.20.x
- name: Run zb
id: bench
run: |
make binary
make bench
@ -59,3 +68,188 @@ jobs:
# clean zot storage
sudo rm -rf /tmp/zot
continue-on-error: true
- name: Check on failures
if: steps.bench.outcome != 'success'
run: |
cat /tmp/gc-bench-local.log
exit 1
gc-referrers-stress-s3:
name: GC(with referrers) on S3(minio) with short interval
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/clean-runner
- uses: actions/setup-go@v4
with:
cache: false
go-version: 1.20.x
- name: Setup localstack service
run: |
pip install localstack # Install LocalStack cli
docker pull localstack/localstack:1.3 # Make sure to pull the latest version of the image
localstack start -d # Start LocalStack in the background
echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container
localstack wait -t 30 # to become ready before timing out
echo "Startup complete"
# aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}"
aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
env:
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake
- name: Setup minio service
run: |
docker run -d -p 9000:9000 --name minio \
-e "MINIO_ACCESS_KEY=minioadmin" \
-e "MINIO_SECRET_KEY=minioadmin" \
-v /tmp/data:/data \
-v /tmp/config:/root/.minio \
--health-cmd "curl http://localhost:9000/minio/health/live" \
minio/minio:edge-cicd server /data
- name: Install py minio
run: pip3 install minio
- name: Wait for minio to come up
run: |
sleep 10
curl --connect-timeout 5 \
--max-time 120 \
--retry 12 \
--retry-max-time 120 \
'http://localhost:9000/minio/health/live'
- name: Create minio bucket
run: |
python3 - <<'EOF'
from minio import Minio
try:
minio = Minio(
'localhost:9000',
access_key='minioadmin',
secret_key='minioadmin',
secure=False
)
except Exception as ex:
raise
minio.make_bucket('zot-storage')
print(f'{minio.list_buckets()}')
EOF
- name: Run zb
id: bench
run: |
make binary
make bench
./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3-minio.json &
sleep 10
bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
killall -r zot-*
# clean zot storage
sudo rm -rf /tmp/zot
env:
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake
continue-on-error: true
- name: Check on failures
if: steps.bench.outcome != 'success'
run: |
cat /tmp/gc-referrers-bench-s3.log
exit 1
gc-stress-s3:
name: GC(without referrers) on S3(minio) with short interval
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/clean-runner
- uses: actions/setup-go@v4
with:
cache: false
go-version: 1.20.x
- name: Setup localstack service
run: |
pip install localstack # Install LocalStack cli
docker pull localstack/localstack:1.3 # Make sure to pull the latest version of the image
localstack start -d # Start LocalStack in the background
echo "Waiting for LocalStack startup..." # Wait 30 seconds for the LocalStack container
localstack wait -t 30 # to become ready before timing out
echo "Startup complete"
# aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket zot-storage --region us-east-2 --create-bucket-configuration="{\"LocationConstraint\": \"us-east-2\"}"
aws dynamodb --endpoint-url http://localhost:4566 --region "us-east-2" create-table --table-name BlobTable --attribute-definitions AttributeName=Digest,AttributeType=S --key-schema AttributeName=Digest,KeyType=HASH --provisioned-throughput ReadCapacityUnits=10,WriteCapacityUnits=5
env:
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake
- name: Setup minio service
run: |
docker run -d -p 9000:9000 --name minio \
-e "MINIO_ACCESS_KEY=minioadmin" \
-e "MINIO_SECRET_KEY=minioadmin" \
-v /tmp/data:/data \
-v /tmp/config:/root/.minio \
--health-cmd "curl http://localhost:9000/minio/health/live" \
minio/minio:edge-cicd server /data
- name: Install py minio
run: pip3 install minio
- name: Wait for minio to come up
run: |
sleep 10
curl --connect-timeout 5 \
--max-time 120 \
--retry 12 \
--retry-max-time 120 \
'http://localhost:9000/minio/health/live'
- name: Create minio bucket
run: |
python3 - <<'EOF'
from minio import Minio
try:
minio = Minio(
'localhost:9000',
access_key='minioadmin',
secret_key='minioadmin',
secure=False
)
except Exception as ex:
raise
minio.make_bucket('zot-storage')
print(f'{minio.list_buckets()}')
EOF
- name: Run zb
id: bench
run: |
make binary
make bench
./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3-minio.json &
sleep 10
bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
killall -r zot-*
# clean zot storage
sudo rm -rf /tmp/zot
env:
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake
continue-on-error: true
- name: Check on failures
if: steps.bench.outcome != 'success'
run: |
cat /tmp/gc-bench-s3.log
exit 1

26
.github/workflows/nightly.yaml vendored
View file

@ -72,7 +72,7 @@ jobs:
make run-blackbox-sync-nightly
gc-referrers-stress-s3:
name: GC(with referrers) on S3 with short interval
name: GC(with referrers) on S3(localstack) with short interval
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@ -96,11 +96,13 @@ jobs:
env:
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake
- name: Run zb
id: bench
run: |
make binary
make bench
./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3.json &
./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3-localstack.json &
sleep 10
bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
@ -111,9 +113,16 @@ jobs:
env:
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake
continue-on-error: true
- name: Check on failures
if: steps.bench.outcome != 'success'
run: |
cat /tmp/gc-referrers-bench-s3.log
exit 1
gc-stress-s3:
name: GC(without referrers) on S3 with short interval
name: GC(without referrers) on S3(localstack) with short interval
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
@ -137,11 +146,13 @@ jobs:
env:
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake
- name: Run zb
id: bench
run: |
make binary
make bench
./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3.json &
./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3-localstack.json &
sleep 10
bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080
@ -152,3 +163,10 @@ jobs:
env:
AWS_ACCESS_KEY_ID: fake
AWS_SECRET_ACCESS_KEY: fake
continue-on-error: true
- name: Check on failures
if: steps.bench.outcome != 'success'
run: |
cat /tmp/gc-bench-s3.log
exit 1

10
pkg/storage/imagestore/imagestore.go
View file

@ -672,7 +672,7 @@ func (is *ImageStore) deleteImageManifest(repo, reference string, detectCollisio
}
if _, err := is.storeDriver.WriteFile(file, buf); err != nil {
is.log.Debug().Str("deleting reference", reference).Msg("")
is.log.Debug().Str("reference", reference).Str("repo", repo).Msg("error while updating index.json")
return err
}
@ -1660,7 +1660,6 @@ func (is *ImageStore) garbageCollectIndexReferrers(repo string, rootIndex ispec.
if gced {
count++
}
case ispec.MediaTypeImageManifest, artifactspec.MediaTypeArtifactManifest:
image, err := common.GetImageManifest(is, repo, desc.Digest, is.log)
if err != nil {
@ -1890,8 +1889,13 @@ func (is *ImageStore) garbageCollectBlobs(imgStore *ImageStore, repo string,
}
}
blobUploads, err := is.storeDriver.List(path.Join(is.RootDir(), repo, storageConstants.BlobUploadDir))
if err != nil {
is.log.Debug().Str("repository", repo).Msg("unable to list .uploads/ dir")
}
// if we cleaned all blobs let's also remove the repo so that it won't be returned by catalog
if reaped == len(allBlobs) {
if len(allBlobs) > 0 && reaped == len(allBlobs) && len(blobUploads) == 0 {
log.Info().Str("repository", repo).Msg("garbage collected all blobs, cleaning repo...")
if err := is.storeDriver.Delete(path.Join(is.rootDir, repo)); err != nil {

52
test/blackbox/restore_s3_blobs.bats
View file

@ -17,13 +17,15 @@ function setup_file() {
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
local zot_config_file_dedupe=${BATS_FILE_TMPDIR}/zot_config_dedupe.json
local zot_config_file_nodedupe=${BATS_FILE_TMPDIR}/zot_config_nodedupe.json
local ZOT_LOG_FILE=${zot_root_dir}/zot-log.json
local ZOT_LOG_FILE_DEDUPE=${BATS_FILE_TMPDIR}/zot-log-dedupe.json
local ZOT_LOG_FILE_NODEDUPE=${BATS_FILE_TMPDIR}/zot-log-nodedupe.json
mkdir -p ${zot_root_dir}
cat > ${zot_config_file_dedupe}<<EOF
{
"distSpecVersion": "1.1.0-dev",
"storage": {
"distSpecVersion": "1.1.0-dev",
"storage": {
"rootDirectory": "${zot_root_dir}",
"dedupe": true,
"remoteCache": true,
@ -42,21 +44,22 @@ function setup_file() {
"region": "us-east-2",
"cacheTablename": "BlobTable"
}
},
"http": {
"address": "127.0.0.1",
"port": "8080"
},
"log": {
"level": "debug"
}
},
"http": {
"address": "127.0.0.1",
"port": "8080"
},
"log": {
"level": "debug",
"output": "${ZOT_LOG_FILE_DEDUPE}"
}
}
EOF
cat > ${zot_config_file_nodedupe}<<EOF
{
"distSpecVersion": "1.1.0-dev",
"storage": {
"distSpecVersion": "1.1.0-dev",
"storage": {
"rootDirectory": "${zot_root_dir}",
"dedupe": false,
"storageDriver": {
@ -68,15 +71,15 @@ EOF
"secure": false,
"skipverify": false
}
},
"http": {
"address": "127.0.0.1",
"port": "8080"
},
"log": {
"level": "debug",
"output": "${ZOT_LOG_FILE}"
}
},
"http": {
"address": "127.0.0.1",
"port": "8080"
},
"log": {
"level": "debug",
"output": "${ZOT_LOG_FILE_NODEDUPE}"
}
}
EOF
awslocal s3 --region "us-east-2" mb s3://zot-storage
@ -87,7 +90,8 @@ EOF
function teardown() {
# conditionally printing on failure is possible from teardown but not from from teardown_file
cat ${BATS_FILE_TMPDIR}/zot/zot-log.json
cat ${BATS_FILE_TMPDIR}/zot-log-dedupe.json
cat ${BATS_FILE_TMPDIR}/zot-log-nodedupe.json || true
}
function teardown_file() {
@ -111,7 +115,7 @@ function teardown_file() {
@test "restart zot with dedupe false and wait for restore blobs task to finish" {
local zot_config_file_nodedupe=${BATS_FILE_TMPDIR}/zot_config_nodedupe.json
local zot_root_dir=${BATS_FILE_TMPDIR}/zot
local ZOT_LOG_FILE=${zot_root_dir}/zot-log.json
local ZOT_LOG_FILE=${BATS_FILE_TMPDIR}/zot-log-nodedupe.json
# stop server
zot_stop

2
test/gc-stress/config-gc-bench-local.json
View file

@ -14,6 +14,6 @@
},
"log": {
"level": "debug",
"output": "/dev/null"
"output": "/tmp/gc-bench-local.log"
}
}

6
test/gc-stress/config-gc-bench-s3.json → test/gc-stress/config-gc-bench-s3-localstack.json
View file

@ -4,8 +4,8 @@
"rootDirectory": "/tmp/zot/s3",
"gc": true,
"gcReferrers": false,
"gcDelay": "40m",
"untaggedImageRetentionDelay": "40m",
"gcDelay": "50m",
"untaggedImageRetentionDelay": "50m",
"gcInterval": "2m",
"storageDriver": {
"name": "s3",
@ -29,6 +29,6 @@
},
"log": {
"level": "debug",
"output": "/dev/null"
"output": "/tmp/gc-bench-s3.log"
}
}

36
test/gc-stress/config-gc-bench-s3-minio.json Normal file
View file

@ -0,0 +1,36 @@
{
"distSpecVersion": "1.1.0-dev",
"storage": {
"rootDirectory": "/tmp/zot/s3",
"gc": true,
"gcReferrers": false,
"gcDelay": "3m",
"untaggedImageRetentionDelay": "3m",
"gcInterval": "1s",
"storageDriver": {
"name": "s3",
"rootdirectory": "/zot",
"region": "us-east-2",
"bucket": "zot-storage",
"accesskey": "minioadmin",
"secretkey": "minioadmin",
"regionendpoint": "http://localhost:9000",
"secure": false,
"skipverify": false
},
"cacheDriver": {
"name": "dynamodb",
"endpoint": "http://localhost:4566",
"region": "us-east-2",
"cacheTablename": "BlobTable"
}
},
"http": {
"address": "127.0.0.1",
"port": "8080"
},
"log": {
"level": "debug",
"output": "/tmp/gc-bench-s3.log"
}
}

2
test/gc-stress/config-gc-referrers-bench-local.json
View file

@ -14,6 +14,6 @@
},
"log": {
"level": "debug",
"output": "/dev/null"
"output": "/tmp/gc-referrers-bench-local.log"
}
}

6
test/gc-stress/config-gc-referrers-bench-s3.json → test/gc-stress/config-gc-referrers-bench-s3-localstack.json
View file

@ -4,8 +4,8 @@
"rootDirectory": "/tmp/zot/s3",
"gc": true,
"gcReferrers": true,
"gcDelay": "40m",
"untaggedImageRetentionDelay": "40m",
"gcDelay": "50m",
"untaggedImageRetentionDelay": "50m",
"gcInterval": "2m",
"storageDriver": {
"name": "s3",
@ -29,6 +29,6 @@
},
"log": {
"level": "debug",
"output": "/dev/null"
"output": "/tmp/gc-referrers-bench-s3.log"
}
}

36
test/gc-stress/config-gc-referrers-bench-s3-minio.json Normal file
View file

@ -0,0 +1,36 @@
{
"distSpecVersion": "1.1.0-dev",
"storage": {
"rootDirectory": "/tmp/zot/s3",
"gc": true,
"gcReferrers": true,
"gcDelay": "3m",
"untaggedImageRetentionDelay": "3m",
"gcInterval": "1s",
"storageDriver": {
"name": "s3",
"rootdirectory": "/zot",
"region": "us-east-2",
"bucket": "zot-storage",
"accesskey": "minioadmin",
"secretkey": "minioadmin",
"regionendpoint": "http://localhost:9000",
"secure": false,
"skipverify": false
},
"cacheDriver": {
"name": "dynamodb",
"endpoint": "http://localhost:4566",
"region": "us-east-2",
"cacheTablename": "BlobTable"
}
},
"http": {
"address": "127.0.0.1",
"port": "8080"
},
"log": {
"level": "debug",
"output": "/tmp/gc-referrers-bench-s3.log"
}
}