zot/.github/workflows/nightly.yaml

name: 'Nightly jobs'
on:
  schedule:
    - cron: '30 1 * * *'
  workflow_dispatch:

permissions: read-all

#  Here we are running two tests:
#  1. run zot with local storage and dedupe disabled, push images, restart zot with dedupe enabled
#  task scheduler will start a dedupe all blobs process at zot startup and it shouldn't interfere with clients.
#  2. run zot with s3 storage and dynamodb and dedupe enabled, push images, restart zot with dedupe false and no cache
#  task scheduler will start a restore all blobs process at zot startup, after it finishes all blobs should be restored to their original state (have content)
jobs:
  dedupe:
    name: Dedupe/restore blobs
    runs-on: ubuntu-latest-4-cores
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: 1.22.x
      - name: Install dependencies
        run: |
          cd $GITHUB_WORKSPACE
          go install github.com/swaggo/swag/cmd/swag@v1.16.2
          go mod download
          sudo apt-get update
          sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap
          # install skopeo
          git clone -b v1.12.0 https://github.com/containers/skopeo.git
          cd skopeo
          make bin/skopeo
          sudo cp bin/skopeo /usr/bin
          skopeo -v
      - name: Log in to GitHub Docker Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ github.token }}
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Install localstack
        run: |
          pip install --upgrade pyopenssl
          pip install localstack==3.3.0 awscli-local[ver1] # install LocalStack cli and awslocal
          docker pull ghcr.io/project-zot/ci-images/localstack:3.3.0 # Make sure to pull the latest version of the image
          localstack start -d                       # Start LocalStack in the background

          echo "Waiting for LocalStack startup..."  # Wait 30 seconds for the LocalStack container
          localstack wait -t 30                     # to become ready before timing out
          echo "Startup complete"
      - name: Run blackbox nightly dedupe tests
        run: |
            # test restoring s3 blobs after cache is deleted
            # test deduping filesystem blobs after switching dedupe to enable
            make run-blackbox-dedupe-nightly
        env:
          AWS_ACCESS_KEY_ID: fake
          AWS_SECRET_ACCESS_KEY: fake
      - uses: ./.github/actions/teardown-localstack

  sync:
    name: Sync harness
    runs-on: ubuntu-latest-4-cores
    steps:
      - name: Check out source code
        uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: 1.22.x
      - name: Install dependencies
        run: |
          cd $GITHUB_WORKSPACE
          go install github.com/swaggo/swag/cmd/swag@v1.16.2
          go mod download
      - name: Run sync harness
        run: |
            make run-blackbox-sync-nightly

  gc-referrers-stress-s3:
    name: GC(with referrers) on S3(localstack) with short interval
    runs-on: ubuntu-latest-16-cores
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          cache: false
          go-version: 1.22.x
      - uses: ./.github/actions/setup-localstack

      - name: Run zb
        timeout-minutes: 240
        id: bench
        run: |
            make binary
            make bench
            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3-localstack.json &
            sleep 10
            bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 --skip-cleanup

            killall -r zot-*

            # clean zot storage
            sudo rm -rf /tmp/zot
        env:
          AWS_ACCESS_KEY_ID: fake
          AWS_SECRET_ACCESS_KEY: fake
        continue-on-error: true

      - name: Check on failures
        if: steps.bench.outcome != 'success'
        run: |
          cat /tmp/gc-referrers-bench-s3.log
          exit 1
      - uses: ./.github/actions/teardown-localstack

  gc-stress-s3:
    name: GC(without referrers) on S3(localstack) with short interval
    runs-on: ubuntu-latest-16-cores
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          cache: false
          go-version: 1.22.x
      - uses: ./.github/actions/setup-localstack

      - name: Run zb
        timeout-minutes: 240
        id: bench
        run: |
            make binary
            make bench
            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3-localstack.json &
            sleep 10
            bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 --skip-cleanup

            killall -r zot-*

            # clean zot storage
            sudo rm -rf /tmp/zot
        env:
          AWS_ACCESS_KEY_ID: fake
          AWS_SECRET_ACCESS_KEY: fake
        continue-on-error: true

      - name: Check on failures
        if: steps.bench.outcome != 'success'
        run: |
          cat /tmp/gc-bench-s3.log
          exit 1
      - uses: ./.github/actions/teardown-localstack

  docker-image:
    name: Build docker image (for users still using Docker environments)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Check out source code
        uses: actions/checkout@v4
      - uses: ./.github/actions/clean-runner
      - name: Build image
        run: |
          make docker-image

  kind-setup:
    name: Prometheus setup
    runs-on: ubuntu-latest-8-cores
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: 1.22.x
      - name: Install dependencies
        run: |
          cd $GITHUB_WORKSPACE
          go mod download
          sudo apt-get update
          sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap
          # install skopeo
          git clone -b v1.12.0 https://github.com/containers/skopeo.git
          cd skopeo
          make bin/skopeo
          sudo cp bin/skopeo /usr/bin
          skopeo -v
      - name: Log in to GitHub Docker Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ github.token }}
      - name: Run tests
        run: |
            ./examples/kind/kind-ci.sh