From f9290b328a605a598e2f488e600a58f57450169c Mon Sep 17 00:00:00 2001 From: Nate Moore Date: Thu, 30 Jun 2022 17:26:37 -0400 Subject: [PATCH] CI: Prefer per-job cache rather than artifact I/O (#3781) * chore(ci): prefer per-job cache rather than artifact io * chore: update cache step * chore: fix cache key * chore(ci): build on windows as well * chore(ci): enable remote cache * chore: run build on all three platforms * chore: prefer turbo to custom caching logic * chore: build AFTER install * chore: build on node 14 * chore: remove build, parallelize tasks * chore: update cache * chore: prime build caches * chore: fix indentation * chore(ci): only build on ubuntu * chore(ci): changelog needs build * chore(ci): ignore updates to md files * chore(ci): add 20 min timeout for e2e tests * chore(ci): enable FORCE_COLOR * chore: run turbo with `--output-logs=new-only` Co-authored-by: Nate Moore --- .github/extract-artifacts.sh | 11 ----- .github/workflows/ci.yml | 96 ++++++++++++++---------------------- package.json | 8 +-- turbo.json | 5 +- 4 files changed, 44 insertions(+), 76 deletions(-) delete mode 100755 .github/extract-artifacts.sh diff --git a/.github/extract-artifacts.sh b/.github/extract-artifacts.sh deleted file mode 100755 index 03f0d19a1a..0000000000 --- a/.github/extract-artifacts.sh +++ /dev/null @@ -1,11 +0,0 @@ -cd artifacts -mkdir -p ../tmp/packages -mv * ../tmp/packages -cd ../tmp -tar -cvzpf artifacts.tar.gz * -mv artifacts.tar.gz ../artifacts.tar.gz -cd .. -tar -xvzpf artifacts.tar.gz -rm -rf artifacts -rm -rf tmp -rm -f artifacts.tar.gz diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd46c3ef60..96aecf6acc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,7 @@ on: pull_request: paths-ignore: - '.vscode/**' + - '**/*.md' # Automatically cancel in-progress actions on the same branch concurrency: @@ -17,8 +18,14 @@ defaults: run: shell: bash +env: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ secrets.TURBO_TEAM }} + TURBO_REMOTE_ONLY: true + FORCE_COLOR: true + ASTRO_TELEMETRY_DISABLED: true + jobs: - # Lint can run in parallel with Build. lint: name: Lint runs-on: ubuntu-latest @@ -65,13 +72,17 @@ jobs: # Checks that the formatter runs successfully on all files # In the future, we may have this fail PRs on unformatted code - name: Format Check - run: yarn format --list + run: pnpm run format --list - # Build installs all devDependencies and runs our full build pipeline. - # We upload all `dist/` artifacts to GitHub, which can be shared by all dependent jobs. + # Build primes out build caches for Turbo build: - name: Build Packages - runs-on: ubuntu-latest + name: 'Build: ${{ matrix.os }}' + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + node_version: [14] + fail-fast: true steps: - name: Checkout uses: actions/checkout@v3 @@ -79,35 +90,22 @@ jobs: - name: Setup PNPM uses: pnpm/action-setup@v2.2.1 - - name: Setup Node + - name: Setup node@${{ matrix.node_version }} uses: actions/setup-node@v3 with: - node-version: 16 + node-version: ${{ matrix.node_version }} cache: 'pnpm' - name: Install dependencies run: pnpm install - + - name: Build Packages run: pnpm run build - - name: Upload Package Artifacts - uses: actions/upload-artifact@v3 - with: - name: artifacts - path: | - packages/*/dist/** - packages/*/*/dist/** - packages/webapi/mod.js - packages/webapi/mod.js.map - if-no-files-found: error - - # Test depends on Build's output, which allows us to skip any build process! test: name: 'Test: ${{ matrix.os }} (node@${{ matrix.node_version }})' runs-on: ${{ matrix.os }} - env: - ASTRO_TELEMETRY_DISABLED: true + needs: build strategy: matrix: os: [ubuntu-latest] @@ -118,8 +116,6 @@ jobs: - os: macos-latest node_version: 14 fail-fast: false - needs: - - build steps: - name: Checkout uses: actions/checkout@v3 @@ -138,30 +134,25 @@ jobs: with: deno-version: v1.19.3 - - name: Download Build Artifacts - uses: actions/download-artifact@v3 - - - name: Extract Artifacts - run: ./.github/extract-artifacts.sh - - name: Install dependencies run: pnpm install + - name: Build Packages + run: pnpm run build + - name: Test run: pnpm run test e2e: name: 'Test (E2E): ${{ matrix.os }} (node@${{ matrix.node_version }})' runs-on: ${{ matrix.os }} - env: - ASTRO_TELEMETRY_DISABLED: true + timeout-minutes: 20 + needs: build strategy: matrix: os: [ubuntu-latest, windows-latest] node_version: [14] fail-fast: false - needs: - - build steps: - name: Checkout uses: actions/checkout@v3 @@ -175,27 +166,23 @@ jobs: node-version: ${{ matrix.node_version }} cache: 'pnpm' - - name: Download Build Artifacts - uses: actions/download-artifact@v3 - - - name: Extract Artifacts - run: ./.github/extract-artifacts.sh - - name: Install dependencies run: pnpm install + - name: Build Packages + run: pnpm run build + - name: Test run: pnpm run test:e2e smoke: name: 'Test (Smoke): ${{ matrix.os }} (node@${{ matrix.node_version }})' runs-on: ${{ matrix.os }} + needs: build strategy: matrix: os: [ubuntu-latest, windows-latest] node_version: [14] - needs: - - build steps: - name: Checkout uses: actions/checkout@v3 @@ -214,26 +201,20 @@ jobs: node-version: ${{ matrix.node_version }} cache: 'pnpm' - - name: Download Build Artifacts - uses: actions/download-artifact@v3 - - - name: Extract Artifacts - run: ./.github/extract-artifacts.sh - - name: Install dependencies run: pnpm install --no-frozen-lockfile + - name: Build Packages + run: pnpm run build + - name: Test run: pnpm run test:smoke - - # Changelog can only run _after_ build. - # We download all `dist/` artifacts from GitHub to skip the build process. changelog: name: Changelog PR or Release if: ${{ (github.ref_name == 'main' || github.head_ref == 'next') && github.repository_owner == 'withastro' }} - needs: [build] runs-on: ubuntu-latest + needs: build steps: - uses: actions/checkout@v3 @@ -246,15 +227,12 @@ jobs: node-version: 16 cache: 'pnpm' - - name: Download Build Artifacts - uses: actions/download-artifact@v3 - - - name: Extract Artifacts - run: ./.github/extract-artifacts.sh - - name: Install dependencies run: pnpm install + - name: Build Packages + run: pnpm run build + - name: Create Release Pull Request or Publish id: changesets uses: changesets/action@v1 diff --git a/package.json b/package.json index d9649eb72f..f5d9ed3214 100644 --- a/package.json +++ b/package.json @@ -9,19 +9,19 @@ "scripts": { "postinstall": "patch-package", "release": "pnpm run build && changeset publish", - "build": "turbo run build --no-deps --scope=astro --scope=create-astro --scope=\"@astrojs/*\"", - "build:ci": "turbo run build:ci --no-deps --scope=astro --scope=create-astro --scope=\"@astrojs/*\"", + "build": "turbo run build --output-logs=new-only --no-deps --scope=astro --scope=create-astro --scope=\"@astrojs/*\"", + "build:ci": "turbo run build:ci --output-logs=new-only --no-deps --scope=astro --scope=create-astro --scope=\"@astrojs/*\"", "build:examples": "turbo run build --scope=\"@example/*\"", "dev": "turbo run dev --no-deps --no-cache --parallel --scope=astro --scope=create-astro --scope=\"@astrojs/*\"", "format": "pnpm run format:code", "format:ci": "pnpm run format:imports && pnpm run format:code", "format:code": "prettier -w . --cache", "format:imports": "organize-imports-cli ./packages/*/tsconfig.json ./packages/*/*/tsconfig.json", - "test": "turbo run test --concurrency=1", + "test": "turbo run test --output-logs=new-only --concurrency=1", "test:match": "cd packages/astro && pnpm run test:match", "test:templates": "turbo run test --filter=create-astro --concurrency=1", "test:smoke": "node scripts/smoke/index.js", - "test:vite-ci": "turbo run test --no-deps --scope=astro --concurrency=1", + "test:vite-ci": "turbo run test --output-logs=new-only --no-deps --scope=astro --concurrency=1", "test:e2e": "cd packages/astro && pnpm playwright install && pnpm run test:e2e", "test:e2e:match": "cd packages/astro && pnpm playwright install && pnpm run test:e2e:match", "benchmark": "turbo run benchmark --scope=astro", diff --git a/turbo.json b/turbo.json index cf84971911..ad76a20a96 100644 --- a/turbo.json +++ b/turbo.json @@ -1,13 +1,14 @@ { + "$schema": "https://turborepo.org/schema.json", "baseBranch": "origin/main", "pipeline": { "build": { "dependsOn": ["^build"], - "outputs": ["**/dist/**", "!**/vendor/**"] + "outputs": ["dist/**/*", "!vendor/**", "mod.js", "mod.js.map"] }, "build:ci": { "dependsOn": ["^build:ci"], - "outputs": ["**/dist/**", "!**/vendor/**"] + "outputs": ["dist/**/*", "!vendor/**", "mod.js", "mod.js.map"] }, "dev": { "cache": false