CI: Prefer per-job cache rather than artifact I/O (#3781)

* chore(ci): prefer per-job cache rather than artifact io

* chore: update cache step

* chore: fix cache key

* chore(ci): build on windows as well

* chore(ci): enable remote cache

* chore: run build on all three platforms

* chore: prefer turbo to custom caching logic

* chore: build AFTER install

* chore: build on node 14

* chore: remove build, parallelize tasks

* chore: update cache

* chore: prime build caches

* chore: fix indentation

* chore(ci): only build on ubuntu

* chore(ci): changelog needs build

* chore(ci): ignore updates to md files

* chore(ci): add 20 min timeout for e2e tests

* chore(ci): enable FORCE_COLOR

* chore: run turbo with `--output-logs=new-only`

Co-authored-by: Nate Moore <nate@astro.build>

.github/extract-artifacts.sh

@@ -1,11 +0,0 @@
-cd artifacts
-mkdir -p ../tmp/packages
-mv * ../tmp/packages
-cd ../tmp
-tar -cvzpf artifacts.tar.gz *
-mv artifacts.tar.gz ../artifacts.tar.gz
-cd ..
-tar -xvzpf artifacts.tar.gz
-rm -rf artifacts
-rm -rf tmp
-rm -f artifacts.tar.gz

.github/workflows/ci.yml

@@ -7,6 +7,7 @@ on:
   pull_request:
     paths-ignore:
       - '.vscode/**'
+      - '**/*.md'
 
 # Automatically cancel in-progress actions on the same branch
 concurrency:
@@ -17,8 +18,14 @@ defaults:
   run:
     shell: bash
 
+env:
+  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+  TURBO_TEAM: ${{ secrets.TURBO_TEAM }}
+  TURBO_REMOTE_ONLY: true
+  FORCE_COLOR: true
+  ASTRO_TELEMETRY_DISABLED: true
+
 jobs:
-  # Lint can run in parallel with Build.
   lint:
     name: Lint
     runs-on: ubuntu-latest
@@ -65,13 +72,17 @@ jobs:
       # Checks that the formatter runs successfully on all files
       # In the future, we may have this fail PRs on unformatted code
       - name: Format Check
-        run: yarn format --list
+        run: pnpm run format --list
 
-  # Build installs all devDependencies and runs our full build pipeline.
+  # Build primes out build caches for Turbo
-  # We upload all `dist/` artifacts to GitHub, which can be shared by all dependent jobs.
   build:
-    name: Build Packages
+    name: 'Build: ${{ matrix.os }}'
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        node_version: [14]
+      fail-fast: true
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -79,10 +90,10 @@ jobs:
       - name: Setup PNPM
         uses: pnpm/action-setup@v2.2.1
 
-      - name: Setup Node
+      - name: Setup node@${{ matrix.node_version }}
         uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: ${{ matrix.node_version }}
           cache: 'pnpm'
 
       - name: Install dependencies
@@ -91,23 +102,10 @@ jobs:
       - name: Build Packages
         run: pnpm run build
 
-      - name: Upload Package Artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: artifacts
-          path: |
-            packages/*/dist/**
-            packages/*/*/dist/**
-            packages/webapi/mod.js
-            packages/webapi/mod.js.map
-          if-no-files-found: error
-
-  # Test depends on Build's output, which allows us to skip any build process!
   test:
     name: 'Test: ${{ matrix.os }} (node@${{ matrix.node_version }})'
     runs-on: ${{ matrix.os }}
-    env:
+    needs: build
-      ASTRO_TELEMETRY_DISABLED: true
     strategy:
       matrix:
         os: [ubuntu-latest]
@@ -118,8 +116,6 @@ jobs:
           - os: macos-latest
             node_version: 14
       fail-fast: false
-    needs:
-      - build
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -138,30 +134,25 @@ jobs:
         with:
           deno-version: v1.19.3
 
-      - name: Download Build Artifacts
-        uses: actions/download-artifact@v3
-
-      - name: Extract Artifacts
-        run: ./.github/extract-artifacts.sh
-
       - name: Install dependencies
         run: pnpm install
 
+      - name: Build Packages
+        run: pnpm run build
+
       - name: Test
         run: pnpm run test
 
   e2e:
     name: 'Test (E2E): ${{ matrix.os }} (node@${{ matrix.node_version }})'
     runs-on: ${{ matrix.os }}
-    env:
+    timeout-minutes: 20
-      ASTRO_TELEMETRY_DISABLED: true
+    needs: build
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
         node_version: [14]
       fail-fast: false
-    needs:
-      - build
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -175,27 +166,23 @@ jobs:
           node-version: ${{ matrix.node_version }}
           cache: 'pnpm'
 
-      - name: Download Build Artifacts
-        uses: actions/download-artifact@v3
-
-      - name: Extract Artifacts
-        run: ./.github/extract-artifacts.sh
-
       - name: Install dependencies
         run: pnpm install
 
+      - name: Build Packages
+        run: pnpm run build
+
       - name: Test
         run: pnpm run test:e2e
 
   smoke:
     name: 'Test (Smoke): ${{ matrix.os }} (node@${{ matrix.node_version }})'
     runs-on: ${{ matrix.os }}
+    needs: build
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
         node_version: [14]
-    needs:
-      - build
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -214,26 +201,20 @@ jobs:
           node-version: ${{ matrix.node_version }}
           cache: 'pnpm'
 
-      - name: Download Build Artifacts
-        uses: actions/download-artifact@v3
-
-      - name: Extract Artifacts
-        run: ./.github/extract-artifacts.sh
-
       - name: Install dependencies
         run: pnpm install --no-frozen-lockfile
 
+      - name: Build Packages
+        run: pnpm run build
+
       - name: Test
         run: pnpm run test:smoke
 
-
-  # Changelog can only run _after_ build.
-  # We download all `dist/` artifacts from GitHub to skip the build process.
   changelog:
     name: Changelog PR or Release
     if: ${{ (github.ref_name == 'main' || github.head_ref == 'next') && github.repository_owner == 'withastro' }}
-    needs: [build]
     runs-on: ubuntu-latest
+    needs: build
     steps:
       - uses: actions/checkout@v3
 
@@ -246,15 +227,12 @@ jobs:
           node-version: 16
           cache: 'pnpm'
 
-      - name: Download Build Artifacts
-        uses: actions/download-artifact@v3
-
-      - name: Extract Artifacts
-        run: ./.github/extract-artifacts.sh
-
       - name: Install dependencies
         run: pnpm install
 
+      - name: Build Packages
+        run: pnpm run build
+
       - name: Create Release Pull Request or Publish
         id: changesets
         uses: changesets/action@v1

package.json

@@ -9,19 +9,19 @@
   "scripts": {
     "postinstall": "patch-package",
     "release": "pnpm run build && changeset publish",
-    "build": "turbo run build --no-deps --scope=astro --scope=create-astro --scope=\"@astrojs/*\"",
+    "build": "turbo run build --output-logs=new-only --no-deps --scope=astro --scope=create-astro --scope=\"@astrojs/*\"",
-    "build:ci": "turbo run build:ci --no-deps --scope=astro --scope=create-astro --scope=\"@astrojs/*\"",
+    "build:ci": "turbo run build:ci --output-logs=new-only --no-deps --scope=astro --scope=create-astro --scope=\"@astrojs/*\"",
     "build:examples": "turbo run build --scope=\"@example/*\"",
     "dev": "turbo run dev --no-deps --no-cache --parallel --scope=astro --scope=create-astro --scope=\"@astrojs/*\"",
     "format": "pnpm run format:code",
     "format:ci": "pnpm run format:imports && pnpm run format:code",
     "format:code": "prettier -w . --cache",
     "format:imports": "organize-imports-cli ./packages/*/tsconfig.json ./packages/*/*/tsconfig.json",
-    "test": "turbo run test --concurrency=1",
+    "test": "turbo run test --output-logs=new-only --concurrency=1",
     "test:match": "cd packages/astro && pnpm run test:match",
     "test:templates": "turbo run test --filter=create-astro --concurrency=1",
     "test:smoke": "node scripts/smoke/index.js",
-    "test:vite-ci": "turbo run test --no-deps --scope=astro --concurrency=1",
+    "test:vite-ci": "turbo run test --output-logs=new-only --no-deps --scope=astro --concurrency=1",
     "test:e2e": "cd packages/astro && pnpm playwright install && pnpm run test:e2e",
     "test:e2e:match": "cd packages/astro && pnpm playwright install && pnpm run test:e2e:match",
     "benchmark": "turbo run benchmark --scope=astro",

turbo.json

@@ -1,13 +1,14 @@
 {
+  "$schema": "https://turborepo.org/schema.json",
   "baseBranch": "origin/main",
   "pipeline": {
     "build": {
       "dependsOn": ["^build"],
-      "outputs": ["**/dist/**", "!**/vendor/**"]
+      "outputs": ["dist/**/*", "!vendor/**", "mod.js", "mod.js.map"]
     },
     "build:ci": {
       "dependsOn": ["^build:ci"],
-      "outputs": ["**/dist/**", "!**/vendor/**"]
+      "outputs": ["dist/**/*", "!vendor/**", "mod.js", "mod.js.map"]
     },
     "dev": {
       "cache": false