From 425c87bce4f33c6bd2cf9e38ed27933aeba0a5ca Mon Sep 17 00:00:00 2001 From: Zack Pollard Date: Mon, 17 Feb 2025 12:56:28 +0100 Subject: [PATCH] ci: machine learning separate native docker image builds (#16102) --- .github/workflows/docker.yml | 152 +++++++++++++++++++++++++---------- 1 file changed, 109 insertions(+), 43 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index b60e02e806..c31a9a5257 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -95,44 +95,56 @@ jobs: name: Build and Push ML needs: pre-job if: ${{ needs.pre-job.outputs.should_run_ml == 'true' }} - runs-on: ubuntu-latest + runs-on: ${{ matrix.runner }} env: image: immich-machine-learning context: machine-learning file: machine-learning/Dockerfile + GHCR_REPO: ghcr.io/${{ github.repository_owner }}/immich-machine-learning + DOCKER_REPO: altran1502/immich-machine-learning strategy: # Prevent a failure in one image from stopping the other builds fail-fast: false matrix: include: - - platforms: linux/amd64,linux/arm64 + - platform: linux/amd64 + runner: ubuntu-latest device: cpu - - platforms: linux/amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + device: cpu + + - platform: linux/amd64 + runner: ubuntu-latest device: cuda suffix: -cuda - - platforms: linux/amd64 + - platform: linux/amd64 + runner: ubuntu-latest device: openvino suffix: -openvino - - platforms: linux/arm64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm device: armnn suffix: -armnn steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + - name: Checkout uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v3.4.0 - - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3.9.0 - name: Login to Docker Hub # Only push to Docker Hub when making a release - if: ${{ github.event_name == 'release' }} + if: ${{ !github.event.pull_request.head.repo.fork }} uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -147,16 +159,92 @@ jobs: username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push image + id: build + uses: docker/build-push-action@v6.13.0 + with: + context: ${{ env.context }} + file: ${{ env.file }} + platforms: ${{ matrix.platforms }} + # Skip pushing when PR from a fork + push: ${{ !github.event.pull_request.head.repo.fork }} + labels: ${{ steps.metadata.outputs.labels }} + outputs: type=image,"name=${{ env.GHCR_REPO }},${{ env.DOCKER_REPO }}",push-by-digest=true,name-canonical=true,push=true + build-args: | + DEVICE=${{ matrix.device }} + BUILD_ID=${{ github.run_id }} + BUILD_IMAGE=${{ github.event_name == 'release' && github.ref_name || steps.metadata.outputs.tags }} + BUILD_SOURCE_REF=${{ github.ref_name }} + BUILD_SOURCE_COMMIT=${{ github.sha }} + + - name: Export digest + run: | + mkdir -p ${{ runner.temp }}/digests + digest="${{ steps.build.outputs.digest }}" + touch "${{ runner.temp }}/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: ml-digests-${{ matrix.device }}-${{ env.PLATFORM_PAIR }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + + merge_ml: + name: Merge & Push ML + runs-on: ubuntu-latest + if: ${{ needs.pre-job.outputs.should_run_ml == 'true' && !github.event.pull_request.head.repo.fork }} + env: + GHCR_REPO: ghcr.io/${{ github.repository_owner }}/immich-machine-learning + DOCKER_REPO: altran1502/immich-machine-learning + strategy: + matrix: + include: + - device: cpu + - device: cuda + suffix: -cuda + - device: openvino + suffix: -openvino + - device: armnn + suffix: -armnn + needs: + - build_and_push_ml + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: ${{ runner.temp }}/digests + pattern: ml-digests-${{ matrix.device }}-* + merge-multiple: true + + - name: Login to Docker Hub + if: ${{ github.event_name == 'release' }} + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Generate docker image tags - id: metadata + id: meta uses: docker/metadata-action@v5 with: flavor: | # Disable latest tag latest=false images: | - name=ghcr.io/${{ github.repository_owner }}/${{env.image}} - name=altran1502/${{env.image}},enable=${{ github.event_name == 'release' }} + name=${{ env.GHCR_REPO }} + name=${{ env.DOCKER_REPO }},enable=${{ github.event_name == 'release' }} tags: | # Tag with branch name type=ref,event=branch,suffix=${{ matrix.suffix }} @@ -168,35 +256,12 @@ jobs: type=ref,event=tag,suffix=${{ matrix.suffix }} type=raw,value=release,enable=${{ github.event_name == 'release' }},suffix=${{ matrix.suffix }} - - name: Determine build cache output - id: cache-target + - name: Create manifest list and push + working-directory: ${{ runner.temp }}/digests run: | - if [[ "${{ github.event_name }}" == "pull_request" ]]; then - # Essentially just ignore the cache output (PR can't write to registry cache) - echo "cache-to=type=local,dest=/tmp/discard,ignore-error=true" >> $GITHUB_OUTPUT - else - echo "cache-to=type=registry,mode=max,ref=ghcr.io/${{ github.repository_owner }}/immich-build-cache:${{ env.image }}" >> $GITHUB_OUTPUT - fi - - - name: Build and push image - uses: docker/build-push-action@v6.13.0 - with: - context: ${{ env.context }} - file: ${{ env.file }} - platforms: ${{ matrix.platforms }} - # Skip pushing when PR from a fork - push: ${{ !github.event.pull_request.head.repo.fork }} - cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/immich-build-cache:${{env.image}} - cache-to: ${{ steps.cache-target.outputs.cache-to }} - tags: ${{ steps.metadata.outputs.tags }} - labels: ${{ steps.metadata.outputs.labels }} - build-args: | - DEVICE=${{ matrix.device }} - BUILD_ID=${{ github.run_id }} - BUILD_IMAGE=${{ github.event_name == 'release' && github.ref_name || steps.metadata.outputs.tags }} - BUILD_SOURCE_REF=${{ github.ref_name }} - BUILD_SOURCE_COMMIT=${{ github.sha }} - + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.GHCR_REPO }}@sha256:%s ' *) \ + $(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *) build_and_push_server: name: Build and Push Server @@ -272,7 +337,7 @@ jobs: - name: Upload digest uses: actions/upload-artifact@v4 with: - name: digests-${{ env.PLATFORM_PAIR }} + name: server-digests-${{ env.PLATFORM_PAIR }} path: ${{ runner.temp }}/digests/* if-no-files-found: error retention-days: 1 @@ -291,10 +356,11 @@ jobs: uses: actions/download-artifact@v4 with: path: ${{ runner.temp }}/digests - pattern: digests-* + pattern: server-digests-* merge-multiple: true - name: Login to Docker Hub + if: ${{ github.event_name == 'release' }} uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -353,7 +419,7 @@ jobs: success-check-ml: name: Docker Build & Push ML Success - needs: [build_and_push_ml, retag_ml] + needs: [merge_ml, retag_ml] runs-on: ubuntu-latest if: always() steps: