ci: add script to build/publish zot multiarch images and modify the publish pipeline to use it (#2214)

Example usage:
   scripts/build_multiarch_image.sh --registry ghcr.io/project-zot --source-tag v2.0.0 --file build/multiarch-zot.json --destination-tags="v2.0.0 latest"

Signed-off-by: Andrei Aaron <aaaron@luxoft.com>

.github/workflows/publish.yaml

@@ -2,17 +2,16 @@ on:
   release:
     types:
       - published
-name: publish
+name: Publish OCI images
 
 permissions: read-all
 
 jobs:
-  push-image:
-    name: Push OCI images to GitHub Packages
+  push-singlearch-image:
+    name: Push single arch OCI images to GitHub Packages
     runs-on: ubuntu-latest
     permissions:
       contents: read
-      security-events: write
       packages: write
     strategy:
       matrix:
@@ -41,38 +40,6 @@ jobs:
           tags: ${{ github.event.release.tag_name }} latest
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Run zot container image with docker
-        run: |
-          if [[ $OS == "linux" && $ARCH == "amd64" ]]; then
-            docker run -d -p 5000:5000 ghcr.io/${{ github.repository_owner }}/zot-${{ matrix.os }}-${{ matrix.arch }}:${{ github.event.release.tag_name }}
-            sleep 2
-            curl --connect-timeout 5 \
-              --max-time 10 \
-              --retry 12 \
-              --retry-max-time 360 \
-              --retry-connrefused \
-              'http://localhost:5000/v2/'
-            docker kill $(docker ps -q)
-          fi
-        env:
-          OS: ${{ matrix.os }}
-          ARCH: ${{ matrix.arch }}
-      - name: Run zot container image with podman
-        run: |
-          if [[ $OS == "linux" && $ARCH == "amd64" ]]; then
-            podman run -d -p 5000:5000 ghcr.io/${{ github.repository_owner }}/zot-${{ matrix.os }}-${{ matrix.arch }}:${{ github.event.release.tag_name }}
-            sleep 2
-            curl --connect-timeout 5 \
-              --max-time 10 \
-              --retry 12 \
-              --retry-max-time 360 \
-              --retry-connrefused \
-              'http://localhost:5000/v2/'
-            podman kill --all
-          fi
-        env:
-          OS: ${{ matrix.os }}
-          ARCH: ${{ matrix.arch }}
       - name: Build and push zot-minimal container image
         uses: project-stacker/stacker-build-push-action@main
         with:
@@ -88,38 +55,6 @@ jobs:
           tags: ${{ github.event.release.tag_name }} latest
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Run zot-minimal container image with docker
-        run: |
-          if [[ $OS == "linux" && $ARCH == "amd64" ]]; then
-            docker run -d -p 5000:5000 ghcr.io/${{ github.repository_owner }}/zot-minimal-${{ matrix.os }}-${{ matrix.arch }}:${{ github.event.release.tag_name }}
-            sleep 2
-            curl --connect-timeout 5 \
-              --max-time 10 \
-              --retry 12 \
-              --retry-max-time 360 \
-              --retry-connrefused \
-              'http://localhost:5000/v2/'
-            docker kill $(docker ps -q)
-          fi
-        env:
-          OS: ${{ matrix.os }}
-          ARCH: ${{ matrix.arch }}
-      - name: Run zot-minimal container image with podman
-        run: |
-          if [[ $OS == "linux" && $ARCH == "amd64" ]]; then
-            podman run -d -p 5000:5000 ghcr.io/${{ github.repository_owner }}/zot-minimal-${{ matrix.os }}-${{ matrix.arch }}:${{ github.event.release.tag_name }}
-            sleep 2
-            curl --connect-timeout 5 \
-              --max-time 10 \
-              --retry 12 \
-              --retry-max-time 360 \
-              --retry-connrefused \
-              'http://localhost:5000/v2/'
-            podman kill --all
-          fi
-        env:
-          OS: ${{ matrix.os }}
-          ARCH: ${{ matrix.arch }}
       - name: Build and push zot-exporter container image
         uses: project-stacker/stacker-build-push-action@main
         with:
@@ -134,38 +69,6 @@ jobs:
           tags: ${{ github.event.release.tag_name }} latest
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Run zot-exporter container image with docker
-        run: |
-          if [[ $OS == "linux" && $ARCH == "amd64" ]]; then
-            docker run -d -p 5001:5001 ghcr.io/${{ github.repository_owner }}/zxp-${{ matrix.os }}-${{ matrix.arch }}:${{ github.event.release.tag_name }}
-            sleep 2
-            curl --connect-timeout 5 \
-              --max-time 10 \
-              --retry 12 \
-              --retry-max-time 360 \
-              --retry-connrefused \
-              'http://localhost:5001/metrics'
-            docker kill $(docker ps -q)
-          fi
-        env:
-          OS: ${{ matrix.os }}
-          ARCH: ${{ matrix.arch }}
-      - name: Run zot-exporter container image with podman
-        run: |
-          if [[ $OS == "linux" && $ARCH == "amd64" ]]; then
-            podman run -d -p 5001:5001 ghcr.io/${{ github.repository_owner }}/zxp-${{ matrix.os }}-${{ matrix.arch }}:${{ github.event.release.tag_name }}
-            sleep 2
-            curl --connect-timeout 5 \
-              --max-time 10 \
-              --retry 12 \
-              --retry-max-time 360 \
-              --retry-connrefused \
-              'http://localhost:5001/metrics'
-            podman kill --all
-          fi
-        env:
-          OS: ${{ matrix.os }}
-          ARCH: ${{ matrix.arch }}
       - name: Build and push zb container image
         uses: project-stacker/stacker-build-push-action@main
         with:
@@ -180,26 +83,145 @@ jobs:
           tags: ${{ github.event.release.tag_name }} latest
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+
+  push-multiarch-image:
+    name: Push multiarch OCI images to GitHub Packages
+    needs: push-singlearch-image
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      matrix:
+        image: [zot, zot-minimal, zxp, zb]
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+      - name: Log in to GitHub Docker Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run build
+        run: |
+          echo "Building multiarch image for ${{ matrix.image }}"
+          cd $GITHUB_WORKSPACE
+          make check-blackbox-prerequisites
+          export PATH=${PATH}:${GITHUB_WORKSPACE}/hack/tools/bin
+          ./scripts/build_multiarch_image.sh --registry ghcr.io/${{ github.repository_owner }} \
+            --source-tag ${{ github.event.release.tag_name }} \
+            --destination-tags "${{ github.event.release.tag_name }} latest" \
+            --file build/multiarch-${{ matrix.image }}.json
+
+  test-image:
+    name: Test OCI images published to GitHub Packages
+    needs: push-multiarch-image
+    runs-on: ubuntu-latest
+    permissions:
+      packages: read
+    steps:
+      - name: Log in to GitHub Docker Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run zot container image with docker
+        run: |
+          docker run -d -p 5000:5000 ghcr.io/${{ github.repository_owner }}/zot:${{ github.event.release.tag_name }}
+          sleep 2
+          curl --connect-timeout 5 \
+            --max-time 10 \
+            --retry 12 \
+            --retry-max-time 360 \
+            --retry-connrefused \
+            'http://localhost:5000/v2/'
+          docker kill $(docker ps -q)
+      - name: Run zot container image with podman
+        run: |
+          podman run -d -p 5000:5000 ghcr.io/${{ github.repository_owner }}/zot:${{ github.event.release.tag_name }}
+          sleep 2
+          curl --connect-timeout 5 \
+            --max-time 10 \
+            --retry 12 \
+            --retry-max-time 360 \
+            --retry-connrefused \
+            'http://localhost:5000/v2/'
+          podman kill --all
+      - name: Run zot-minimal container image with docker
+        run: |
+          docker run -d -p 5000:5000 ghcr.io/${{ github.repository_owner }}/zot-minimal:${{ github.event.release.tag_name }}
+          sleep 2
+          curl --connect-timeout 5 \
+            --max-time 10 \
+            --retry 12 \
+            --retry-max-time 360 \
+            --retry-connrefused \
+            'http://localhost:5000/v2/'
+          docker kill $(docker ps -q)
+      - name: Run zot-minimal container image with podman
+        run: |
+          podman run -d -p 5000:5000 ghcr.io/${{ github.repository_owner }}/zot-minimal:${{ github.event.release.tag_name }}
+          sleep 2
+          curl --connect-timeout 5 \
+            --max-time 10 \
+            --retry 12 \
+            --retry-max-time 360 \
+            --retry-connrefused \
+            'http://localhost:5000/v2/'
+          podman kill --all
+      - name: Run zot-exporter container image with docker
+        run: |
+          docker run -d -p 5001:5001 ghcr.io/${{ github.repository_owner }}/zxp:${{ github.event.release.tag_name }}
+          sleep 2
+          curl --connect-timeout 5 \
+            --max-time 10 \
+            --retry 12 \
+            --retry-max-time 360 \
+            --retry-connrefused \
+            'http://localhost:5001/metrics'
+          docker kill $(docker ps -q)
+      - name: Run zot-exporter container image with podman
+        run: |
+          podman run -d -p 5001:5001 ghcr.io/${{ github.repository_owner }}/zxp:${{ github.event.release.tag_name }}
+          sleep 2
+          curl --connect-timeout 5 \
+            --max-time 10 \
+            --retry 12 \
+            --retry-max-time 360 \
+            --retry-connrefused \
+            'http://localhost:5001/metrics'
+          podman kill --all
       - name: Run zb container image with docker
         run: |
-          if [[ $OS == "linux" && $ARCH == "amd64" ]]; then
-            docker run ghcr.io/${{ github.repository_owner }}/zb-${{ matrix.os }}-${{ matrix.arch }}:${{ github.event.release.tag_name }} --help
-          fi
-        env:
-          OS: ${{ matrix.os }}
-          ARCH: ${{ matrix.arch }}
+          docker run ghcr.io/${{ github.repository_owner }}/zb:${{ github.event.release.tag_name }} --help
       - name: Run zb container image with podman
         run: |
-          if [[ $OS == "linux" && $ARCH == "amd64" ]]; then
-            podman run ghcr.io/${{ github.repository_owner }}/zb-${{ matrix.os }}-${{ matrix.arch }}:${{ github.event.release.tag_name }} --help
-          fi
-        env:
-          OS: ${{ matrix.os }}
-          ARCH: ${{ matrix.arch }}
+          podman run ghcr.io/${{ github.repository_owner }}/zb:${{ github.event.release.tag_name }} --help
+
+  scan-image:
+    name: Run Trivy scan on OCI images published to GitHub Packages
+    needs: push-singlearch-image
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      packages: read
+    strategy:
+      matrix:
+        os: [linux, darwin]
+        arch: [amd64, arm64]
+    steps:
+      - name: Log in to GitHub Docker Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: 'ghcr.io/${{ github.repository }}-${{ matrix.os }}-${{ matrix.arch }}:${{ github.event.release.tag_name }}'
+          image-ref: 'ghcr.io/${{ github.repository_owner }}/zot-${{ matrix.os }}-${{ matrix.arch }}:${{ github.event.release.tag_name }}'
           format: 'sarif'
           output: 'trivy-results.sarif'
         env:
@@ -208,7 +230,44 @@ jobs:
       - name: Run Trivy vulnerability scanner (minimal)
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: 'ghcr.io/${{ github.repository }}-minimal-${{ matrix.os }}-${{ matrix.arch }}:${{ github.event.release.tag_name }}'
+          image-ref: 'ghcr.io/${{ github.repository_owner }}/zot-minimal-${{ matrix.os }}-${{ matrix.arch }}:${{ github.event.release.tag_name }}'
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+        env:
+          TRIVY_USERNAME: ${{ github.actor }}
+          TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@v3.23.2
+        with:
+          sarif_file: 'trivy-results.sarif'
+
+  scan-multiarch-image:
+    name: Run Trivy scan on OCI multiarch images published to GitHub Packages
+    needs: push-multiarch-image
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+      packages: read
+    steps:
+      - name: Log in to GitHub Docker Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run Trivy vulnerability scanner
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: 'ghcr.io/${{ github.repository_owner }}/zot:${{ github.event.release.tag_name }}'
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+        env:
+          TRIVY_USERNAME: ${{ github.actor }}
+          TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+      - name: Run Trivy vulnerability scanner (minimal)
+        uses: aquasecurity/trivy-action@master
+        with:
+          image-ref: 'ghcr.io/${{ github.repository_owner }}/zot-minimal:${{ github.event.release.tag_name }}'
           format: 'sarif'
           output: 'trivy-results.sarif'
         env:
@@ -221,7 +280,7 @@ jobs:
 
   update-helm-chart:
     if: github.event_name == 'release' && github.event.action== 'published'
-    needs: push-image
+    needs: push-multiarch-image
     name: Update Helm Chart
     permissions:
       contents: write

build/multiarch-zb.json

@@ -0,0 +1,9 @@
+{
+    "target_repo": "zb",
+    "source_repos": [
+        "zb-linux-amd64",
+        "zb-linux-arm64",
+        "zb-darwin-amd64",
+        "zb-darwin-arm64"
+    ]
+}

build/multiarch-zot-minimal.json

@@ -0,0 +1,9 @@
+{
+    "target_repo": "zot-minimal",
+    "source_repos": [
+        "zot-minimal-linux-amd64",
+        "zot-minimal-linux-arm64",
+        "zot-minimal-darwin-amd64",
+        "zot-minimal-darwin-arm64"
+    ]
+}

build/multiarch-zot.json

@@ -0,0 +1,9 @@
+{
+    "target_repo": "zot",
+    "source_repos": [
+        "zot-linux-amd64",
+        "zot-linux-arm64",
+        "zot-darwin-amd64",
+        "zot-darwin-arm64"
+    ]
+}

build/multiarch-zxp.json

@@ -0,0 +1,9 @@
+{
+    "target_repo": "zxp",
+    "source_repos": [
+        "zxp-linux-amd64",
+        "zxp-linux-arm64",
+        "zxp-darwin-amd64",
+        "zxp-darwin-arm64"
+    ]
+}

scripts/build_multiarch_image.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+
+input_file=""
+source_tag=""
+destination_tags=""
+registry=""
+debug=0
+
+while (( "$#" )); do
+    case $1 in
+        -r|--registry)
+            if [ -z "$2" ]; then
+                echo "Option registry requires an argument"
+                exit 1
+            fi
+            registry=${2};
+            shift 2
+            ;;
+        --destination-tags)
+            if [ -z "$2" ]; then
+                echo "Option destination-tag requires an argument"
+                exit 1
+            fi
+            destination_tags=${2}
+            shift 2
+            ;;
+        --source-tag)
+            if [ -z "$2" ]; then
+                echo "Option source-tag requires an argument"
+                exit 1
+            fi
+            source_tag=${2}
+            shift 2
+            ;;
+        -f|--file)
+            if [ -z "$2" ]; then
+                echo "Option file requires an argument"
+                exit 1
+            fi
+            input_file=${2}
+            shift 2
+            ;;
+        -d|--debug)
+            debug=1
+            shift 1
+            ;;
+        --)
+            shift 1
+            break
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+if [ -z "${registry}" ]; then
+    echo "Parameter --registry is mandatory"
+    exit 1
+fi
+
+if [ -z "${source_tag}" ]; then
+    echo "Parameter --source-tag is mandatory"
+    exit 1
+fi
+
+if [ -z "${destination_tags}" ]; then
+    destination_tags=${source_tag}
+    echo "Parameter --destination-tags is not provided, will use value of --source-tag: ${destination_tags}"
+fi
+
+if [ -z "${input_file}" ]; then
+    echo "Parameter --file is mandatory"
+    exit 1
+fi
+
+function verify_prerequisites {
+    mkdir -p ${data_dir}
+
+    command -v regctl
+    if [ $? -ne 0 ]; then
+        echo "you need to install regctl as a prerequisite"
+        exit 1
+    fi
+
+    command -v jq
+    if [ $? -ne 0 ]; then
+        echo "you need to install jq as a prerequisite"
+        return 1
+    fi
+
+    return 0
+}
+
+if [ ${debug} -eq 1 ]; then
+    set -x
+    regctl_cmd="${regctl_cmd} -v debug"
+fi
+
+target_repo=$(jq -r -c ".target_repo" ${input_file})
+source_repos=$(jq -r -c '.source_repos[]' ${input_file})
+
+regctl_cmd="regctl"
+
+digest_params=""
+for repo in ${source_repos[@]}; do
+    echo "identifying digest for ${registry}/${repo}:${source_tag}"
+    digest=$(${regctl_cmd} image digest ${registry}/${repo}:${source_tag})
+    echo "identified digest ${digest} for ${registry}/${repo}:${source_tag}"
+
+    ${regctl_cmd} image copy ${registry}/${repo}:${source_tag} ${registry}/${target_repo}@${digest}
+
+    digest_params="--digest ${digest} ${digest_params}"
+done
+
+destination_tags_array=($destination_tags)
+
+echo "creating index ${registry}/${target_repo}:${destination_tags_array[0]}"
+${regctl_cmd} index create ${registry}/${target_repo}:${destination_tags_array[0]} ${digest_params}
+
+for destination_tag in ${destination_tags_array[@]:1}; do
+    echo "tagging ${registry}/${target_repo}:${destination_tags_array[0]} as ${registry}/${target_repo}:${destination_tag}"
+    ${regctl_cmd} image copy ${registry}/${target_repo}:${destination_tags_array[0]} ${registry}/${target_repo}:${destination_tag}
+done