zot/.github/workflows/nightly.yaml

name: 'Nightly jobs'
on:
  schedule:
    - cron: '30 1 * * *'
  workflow_dispatch:

permissions: read-all

#  The following tests are run:
#  1. run zot with local storage and dedupe disabled, push images, restart zot with dedupe enabled
#  task scheduler will start a dedupe all blobs process at zot startup and it shouldn't interfere with clients.
#  2. run zot with s3 storage and dynamodb and dedupe enabled, push images, restart zot with dedupe false and no cache
#  task scheduler will start a restore all blobs process at zot startup, after it finishes all blobs should be restored to their original state (have content)
#  3. run many, many, many instances of zot with shared storage and metadata front-ended by HAProxy. start a long-running zb run with high concurrency and number of requests
#  to achieve a long-running sustained load on the system. The system is expected to perform well without errors and return performance data after the test.
jobs:
  dedupe:
    name: Dedupe/restore blobs
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: 1.23.x
      - name: Install dependencies
        run: |
          cd $GITHUB_WORKSPACE
          go install github.com/swaggo/swag/cmd/swag@v1.16.2
          go mod download
          sudo apt-get update
          sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap
          # install skopeo
          git clone -b v1.12.0 https://github.com/containers/skopeo.git
          cd skopeo
          make bin/skopeo
          sudo cp bin/skopeo /usr/bin
          skopeo -v          
      - name: Log in to GitHub Docker Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ github.token }}
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Install localstack
        run: |
          pip install --upgrade pyopenssl
          pip install localstack==3.3.0 awscli-local[ver1] # install LocalStack cli and awslocal
          docker pull ghcr.io/project-zot/ci-images/localstack:3.3.0 # Make sure to pull the latest version of the image
          localstack start -d                       # Start LocalStack in the background

          echo "Waiting for LocalStack startup..."  # Wait 30 seconds for the LocalStack container
          localstack wait -t 30                     # to become ready before timing out
          echo "Startup complete"          
      - name: Run blackbox nightly dedupe tests
        run: |
            # test restoring s3 blobs after cache is deleted
            # test deduping filesystem blobs after switching dedupe to enable
            make run-blackbox-dedupe-nightly            
        env:
          AWS_ACCESS_KEY_ID: fake
          AWS_SECRET_ACCESS_KEY: fake
      - uses: ./.github/actions/teardown-localstack

  sync:
    name: Sync harness
    runs-on: ubuntu-latest
    steps:
      - name: Check out source code
        uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: 1.23.x
      - name: Install dependencies
        run: |
          cd $GITHUB_WORKSPACE
          go install github.com/swaggo/swag/cmd/swag@v1.16.2
          go mod download          
      - name: Run sync harness
        run: |
            make run-blackbox-sync-nightly            

  gc-referrers-stress-s3:
    name: GC(with referrers) on S3(localstack) with short interval
    runs-on: ubuntu-latest-16-cores
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          cache: false
          go-version: 1.23.x
      - uses: ./.github/actions/setup-localstack

      - name: Run zb
        timeout-minutes: 240
        id: bench
        run: |
            make binary
            make bench
            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-referrers-bench-s3-localstack.json &
            sleep 10
            bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 --skip-cleanup

            killall -r zot-*

            # clean zot storage
            sudo rm -rf /tmp/zot            
        env:
          AWS_ACCESS_KEY_ID: fake
          AWS_SECRET_ACCESS_KEY: fake
        continue-on-error: true

      - name: Check on failures
        if: steps.bench.outcome != 'success'
        run: |
          cat /tmp/gc-referrers-bench-s3.log
          exit 1          
      - uses: ./.github/actions/teardown-localstack

  gc-stress-s3:
    name: GC(without referrers) on S3(localstack) with short interval
    runs-on: ubuntu-latest-16-cores
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          cache: false
          go-version: 1.23.x
      - uses: ./.github/actions/setup-localstack

      - name: Run zb
        timeout-minutes: 240
        id: bench
        run: |
            make binary
            make bench
            ./bin/zot-linux-amd64 serve test/gc-stress/config-gc-bench-s3-localstack.json &
            sleep 10
            bin/zb-linux-amd64 -c 10 -n 100 -o ci-cd http://localhost:8080 --skip-cleanup

            killall -r zot-*

            # clean zot storage
            sudo rm -rf /tmp/zot            
        env:
          AWS_ACCESS_KEY_ID: fake
          AWS_SECRET_ACCESS_KEY: fake
        continue-on-error: true

      - name: Check on failures
        if: steps.bench.outcome != 'success'
        run: |
          cat /tmp/gc-bench-s3.log
          exit 1          
      - uses: ./.github/actions/teardown-localstack

  docker-image:
    name: Build docker image (for users still using Docker environments)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Check out source code
        uses: actions/checkout@v4
      - uses: ./.github/actions/clean-runner
      - name: Build image
        run: |
          make docker-image          

  kind-setup:
    name: Prometheus setup
    runs-on: ubuntu-latest-8-cores
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: 1.23.x
      - name: Install dependencies
        run: |
          cd $GITHUB_WORKSPACE
          go mod download
          sudo apt-get update
          sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap
          # install skopeo
          git clone -b v1.12.0 https://github.com/containers/skopeo.git
          cd skopeo
          make bin/skopeo
          sudo cp bin/skopeo /usr/bin
          skopeo -v          
      - name: Log in to GitHub Docker Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ github.token }}
      - name: Run tests
        run: |
            ./examples/kind/kind-ci.sh            

  cloud-scale-out:
    name: s3+dynamodb scale-out
    runs-on: ubuntu-latest-16-cores
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          cache: false
          go-version: 1.23.x
      - name: Install dependencies
        run: |
          cd $GITHUB_WORKSPACE
          go install github.com/swaggo/swag/cmd/swag@v1.16.2
          go mod download
          sudo apt-get update
          sudo apt-get install libgpgme-dev libassuan-dev libbtrfs-dev libdevmapper-dev pkg-config rpm uidmap haproxy jq
          # install skopeo
          git clone -b v1.12.0 https://github.com/containers/skopeo.git
          cd skopeo
          make bin/skopeo
          sudo cp bin/skopeo /usr/bin
          skopeo -v
          cd $GITHUB_WORKSPACE          
      - name: Log in to GitHub Docker Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ github.token }}
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Install localstack
        run: |
          pip install --upgrade pyopenssl
          pip install localstack==3.3.0 awscli-local[ver1] # install LocalStack cli and awslocal
          docker pull ghcr.io/project-zot/ci-images/localstack:3.3.0  # Make sure to pull a working version of the image
          localstack start -d                       # Start LocalStack in the background

          echo "Waiting for LocalStack startup..."  # Wait 30 seconds for the LocalStack container
          localstack wait -t 30                     # to become ready before timing out
          echo "Startup complete"          
      - name: Run cloud scale-out high scale performance tests
        id: scale
        run: |
          make run-cloud-scale-out-high-scale-tests          
        env:
          AWS_ACCESS_KEY_ID: fake
          AWS_SECRET_ACCESS_KEY: fake
        continue-on-error: true
      - name: print service logs
        run: |
          sudo dmesg
          cat /tmp/zot-logs/*.log          
      - name: multi-hop detection
        id: multihop
        run: |
          if cat /tmp/zot-logs/*.log | grep 'cannot proxy an already proxied request'; then
            echo "detected multi-hop"
            exit 1
          else
            exit 0
          fi          
        continue-on-error: true
      - name: clean up logs
        run: |
          rm -r /tmp/zot-logs          
      - name: fail job if error
        if: ${{ steps.scale.outcome != 'success' || steps.multihop.outcome != 'success' }}
        run: |
          exit 1          
      - name: Upload zb test results zip as build artifact
        if: steps.scale.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: zb-cloud-scale-out-perf-results-${{ github.sha }}
          path: ./zb-results/
      - uses: ./.github/actions/teardown-localstack