Improve prefetch behaviour for browsers (#10999)

.changeset/great-turtles-clap.md

@@ -0,0 +1,10 @@
+---
+"astro": patch
+---
+
+The prefetch feature is updated to better support different browsers and different cache headers setup, including:
+
+1. All prefetch strategies will now always try to use `<link rel="prefetch">` if supported, or will fall back to `fetch()`.
+2. The `prefetch()` programmatic API's `with` option is deprecated in favour of an automatic approach that will also try to use `<link rel="prefetch>` if supported, or will fall back to `fetch()`.
+
+This change shouldn't affect most sites and should instead make prefetching more effective.

.gitignore

@@ -9,6 +9,7 @@ scripts/smoke/*-main/
 scripts/memory/project/src/pages/
 benchmark/projects/
 benchmark/results/
+test-results/
 *.log
 package-lock.json
 .turbo/

packages/astro/e2e/prefetch.test.js

@@ -5,48 +5,86 @@ const test = testFactory({
 	root: './fixtures/prefetch/',
 });
 
+// Used to track fetch request urls
+/** @type {string[]} */
+const reqUrls = [];
+test.beforeEach(async ({ page }) => {
+	page.on('request', (req) => {
+		const urlObj = new URL(req.url());
+		reqUrls.push(urlObj.pathname + urlObj.search);
+	});
+});
+test.afterEach(() => {
+	reqUrls.length = 0;
+});
+
+/**
+ * Check if url is prefetched via `link[rel="prefetch"]` or `fetch()` (from `reqUrls`)
+ * @param {string} url
+ * @param {import('@playwright/test').Page} page
+ * @param {number} [count] Also expect that it's prefetched this amount of times
+ */
+async function expectUrlPrefetched(url, page, count) {
+	try {
+		await expect(page.locator(`link[rel="prefetch"][href$="${url}"]`)).toBeAttached();
+	} catch {
+		// If link is not found, check if it was fetched via `fetch()`
+		expect(reqUrls, `${url} is not prefetched via link or fetch`).toContainEqual(url);
+	}
+
+	if (count != null) {
+		const linkCount = await page.locator(`link[rel="prefetch"][href$="${url}"]`).count();
+		try {
+			expect(linkCount).toBe(count);
+		} catch {
+			const fetchCount = reqUrls.filter((u) => u.includes(url)).length;
+			expect(
+				fetchCount,
+				`${url} should be prefetched ${count} time(s), but is prefetch with link ${linkCount} time(s) and with fetch ${fetchCount} time(s)`
+			).toEqual(count);
+		}
+	}
+}
+
+/**
+ * Check if url is not prefetched via `link[rel="prefetch"]` and `fetch()` (from `reqUrls`)
+ * @param {string} url
+ * @param {import('@playwright/test').Page} page
+ */
+async function expectUrlNotPrefetched(url, page) {
+	await expect(page.locator(`link[rel="prefetch"][href$="${url}"]`)).not.toBeAttached();
+	expect(reqUrls).not.toContainEqual(url);
+}
+
 test.describe('Prefetch (default)', () => {
 	let devServer;
-	/** @type {string[]} */
-	const reqUrls = [];
 
 	test.beforeAll(async ({ astro }) => {
 		devServer = await astro.startDevServer();
 	});
 
-	test.beforeEach(async ({ page }) => {
-		page.on('request', (req) => {
-			const urlObj = new URL(req.url());
-			reqUrls.push(urlObj.pathname + urlObj.search);
-		});
-	});
-
-	test.afterEach(() => {
-		reqUrls.length = 0;
-	});
-
 	test.afterAll(async () => {
 		await devServer.stop();
 	});
 
 	test('Link without data-astro-prefetch should not prefetch', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/prefetch-default');
+		await expectUrlNotPrefetched('/prefetch-default', page);
 	});
 
 	test('data-astro-prefetch="false" should not prefetch', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/prefetch-false');
+		await expectUrlNotPrefetched('/prefetch-false', page);
 	});
 
 	test('Link with search param should prefetch', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/?search-param=true');
+		await expectUrlNotPrefetched('/?search-param=true', page);
 		await Promise.all([
 			page.waitForEvent('request'), // wait prefetch request
 			page.locator('#prefetch-search-param').hover(),
 		]);
-		expect(reqUrls).toContainEqual('/?search-param=true');
+		await expectUrlPrefetched('/?search-param=true', page);
 	});
 
 	test('data-astro-prefetch="tap" should prefetch on tap', async ({ page, astro }) => {
@@ -61,52 +99,47 @@ test.describe('Prefetch (default)', () => {
 
 	test('data-astro-prefetch="hover" should prefetch on hover', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/prefetch-hover');
+		await expectUrlNotPrefetched('/prefetch-hover', page);
 		await Promise.all([
 			page.waitForEvent('request'), // wait prefetch request
 			page.locator('#prefetch-hover').hover(),
 		]);
-		expect(reqUrls).toContainEqual('/prefetch-hover');
+		await expectUrlPrefetched('/prefetch-hover', page);
 	});
 
 	test('data-astro-prefetch="viewport" should prefetch on viewport', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/prefetch-viewport');
+		await expectUrlNotPrefetched('/prefetch-viewport', page);
 		// Scroll down to show the element
 		await Promise.all([
 			page.waitForEvent('request'), // wait prefetch request
 			page.locator('#prefetch-viewport').scrollIntoViewIfNeeded(),
 		]);
-		expect(reqUrls).toContainEqual('/prefetch-viewport');
+		await expectUrlPrefetched('/prefetch-viewport', page);
-		expect(page.locator('link[rel="prefetch"][href$="/prefetch-viewport"]')).toBeDefined();
 	});
 
 	test('manual prefetch() works once', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/prefetch-manual');
+		await expectUrlNotPrefetched('/prefetch-manual', page);
 		await Promise.all([
 			page.waitForEvent('request'), // wait prefetch request
 			page.locator('#prefetch-manual').click(),
 		]);
-		expect(reqUrls).toContainEqual('/prefetch-manual');
+		await expectUrlPrefetched('/prefetch-manual', page);
-		expect(page.locator('link[rel="prefetch"][href$="/prefetch-manual"]')).toBeDefined();
 
 		// prefetch again should have no effect
 		await page.locator('#prefetch-manual').click();
-		expect(reqUrls.filter((u) => u.includes('/prefetch-manual')).length).toEqual(1);
+		await expectUrlPrefetched('/prefetch-manual', page, 1);
 	});
 
 	test('data-astro-prefetch="load" should prefetch', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).toContainEqual('/prefetch-load');
+		await expectUrlPrefetched('/prefetch-load', page);
-		expect(page.locator('link[rel="prefetch"][href$="/prefetch-load"]')).toBeDefined();
 	});
 });
 
 test.describe("Prefetch (prefetchAll: true, defaultStrategy: 'tap')", () => {
 	let devServer;
-	/** @type {string[]} */
-	const reqUrls = [];
 
 	test.beforeAll(async ({ astro }) => {
 		devServer = await astro.startDevServer({
@@ -117,89 +150,74 @@ test.describe("Prefetch (prefetchAll: true, defaultStrategy: 'tap')", () => {
 		});
 	});
 
-	test.beforeEach(async ({ page }) => {
-		page.on('request', (req) => {
-			const urlObj = new URL(req.url());
-			reqUrls.push(urlObj.pathname + urlObj.search);
-		});
-	});
-
-	test.afterEach(() => {
-		reqUrls.length = 0;
-	});
-
 	test.afterAll(async () => {
 		await devServer.stop();
 	});
 
 	test('Link without data-astro-prefetch should prefetch', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/prefetch-default');
+		await expectUrlNotPrefetched('/prefetch-default', page);
 		await Promise.all([
 			page.waitForEvent('request'), // wait prefetch request
 			page.locator('#prefetch-default').click(),
 		]);
-		expect(reqUrls).toContainEqual('/prefetch-default');
+		await expectUrlPrefetched('/prefetch-default', page);
 	});
 
 	test('data-astro-prefetch="false" should not prefetch', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/prefetch-false');
+		await expectUrlNotPrefetched('/prefetch-false', page);
 	});
 
 	test('Link with search param should prefetch', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/?search-param=true');
+		await expectUrlNotPrefetched('/?search-param=true', page);
 		await Promise.all([
 			page.waitForEvent('request'), // wait prefetch request
 			page.locator('#prefetch-search-param').hover(),
 		]);
-		expect(reqUrls).toContainEqual('/?search-param=true');
+await expectUrlPrefetched('/?search-param=true', page);
 	});
 
 	test('data-astro-prefetch="tap" should prefetch on tap', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/prefetch-tap');
+		await expectUrlNotPrefetched('/prefetch-tap', page);
 		await Promise.all([
 			page.waitForEvent('request'), // wait prefetch request
 			page.locator('#prefetch-tap').click(),
 		]);
-		expect(reqUrls).toContainEqual('/prefetch-tap');
+await expectUrlPrefetched('/prefetch-tap', page);
 	});
 
 	test('data-astro-prefetch="hover" should prefetch on hover', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/prefetch-hover');
+		await expectUrlNotPrefetched('/prefetch-hover', page);
 		await Promise.all([
 			page.waitForEvent('request'), // wait prefetch request
 			page.locator('#prefetch-hover').hover(),
 		]);
-		expect(reqUrls).toContainEqual('/prefetch-hover');
+		await expectUrlPrefetched('/prefetch-hover', page);
 	});
 
 	test('data-astro-prefetch="viewport" should prefetch on viewport', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/prefetch-viewport');
+		await expectUrlNotPrefetched('/prefetch-viewport', page);
 		// Scroll down to show the element
 		await Promise.all([
 			page.waitForEvent('request'), // wait prefetch request
 			page.locator('#prefetch-viewport').scrollIntoViewIfNeeded(),
 		]);
-		expect(reqUrls).toContainEqual('/prefetch-viewport');
+		await expectUrlPrefetched('/prefetch-viewport', page);
-		expect(page.locator('link[rel="prefetch"][href$="/prefetch-viewport"]')).toBeDefined();
 	});
 
 	test('data-astro-prefetch="load" should prefetch', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).toContainEqual('/prefetch-load');
+		await expectUrlPrefetched('/prefetch-load', page);
-		expect(page.locator('link[rel="prefetch"][href$="/prefetch-load"]')).toBeDefined();
 	});
 });
 
 test.describe("Prefetch (prefetchAll: true, defaultStrategy: 'load')", () => {
 	let devServer;
-	/** @type {string[]} */
-	const reqUrls = [];
 
 	test.beforeAll(async ({ astro }) => {
 		devServer = await astro.startDevServer({
@@ -210,78 +228,64 @@ test.describe("Prefetch (prefetchAll: true, defaultStrategy: 'load')", () => {
 		});
 	});
 
-	test.beforeEach(async ({ page }) => {
-		page.on('request', (req) => {
-			const urlObj = new URL(req.url());
-			reqUrls.push(urlObj.pathname + urlObj.search);
-		});
-	});
-
-	test.afterEach(() => {
-		reqUrls.length = 0;
-	});
-
 	test.afterAll(async () => {
 		await devServer.stop();
 	});
 
 	test('Link without data-astro-prefetch should prefetch', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).toContainEqual('/prefetch-default');
+		await expectUrlPrefetched('/prefetch-default', page);
-		expect(page.locator('link[rel="prefetch"][href$="/prefetch-default"]')).toBeDefined();
 	});
 
 	test('data-astro-prefetch="false" should not prefetch', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/prefetch-false');
+		await expectUrlNotPrefetched('/prefetch-false', page);
 	});
 
 	test('Link with search param should prefetch', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/?search-param=true');
+		await expectUrlNotPrefetched('/?search-param=true', page);
 		await Promise.all([
 			page.waitForEvent('request'), // wait prefetch request
 			page.locator('#prefetch-search-param').hover(),
 		]);
-		expect(reqUrls).toContainEqual('/?search-param=true');
+		await expectUrlPrefetched('/?search-param=true', page);
 	});
 
 	test('data-astro-prefetch="tap" should prefetch on tap', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/prefetch-tap');
+		await expectUrlNotPrefetched('/prefetch-tap', page);
 		await Promise.all([
 			page.waitForEvent('request'), // wait prefetch request
 			page.locator('#prefetch-tap').click(),
 		]);
-		expect(reqUrls).toContainEqual('/prefetch-tap');
+		await expectUrlPrefetched('/prefetch-tap', page);
 	});
 
 	test('data-astro-prefetch="hover" should prefetch on hover', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/prefetch-hover');
+		await expectUrlNotPrefetched('/prefetch-hover', page);
 		await Promise.all([
 			page.waitForEvent('request'), // wait prefetch request
 			page.locator('#prefetch-hover').hover(),
 		]);
-		expect(reqUrls).toContainEqual('/prefetch-hover');
+		await expectUrlPrefetched('/prefetch-hover', page);
 	});
 
 	test('data-astro-prefetch="viewport" should prefetch on viewport', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).not.toContainEqual('/prefetch-viewport');
+		await expectUrlNotPrefetched('/prefetch-viewport', page);
 		// Scroll down to show the element
 		await Promise.all([
 			page.waitForEvent('request'), // wait prefetch request
 			page.locator('#prefetch-viewport').scrollIntoViewIfNeeded(),
 		]);
-		expect(reqUrls).toContainEqual('/prefetch-viewport');
+		await expectUrlPrefetched('/prefetch-viewport', page);
-		expect(page.locator('link[rel="prefetch"][href$="/prefetch-viewport"]')).toBeDefined();
 	});
 
 	test('data-astro-prefetch="load" should prefetch', async ({ page, astro }) => {
 		await page.goto(astro.resolveUrl('/'));
-		expect(reqUrls).toContainEqual('/prefetch-load');
+		await expectUrlPrefetched('/prefetch-load', page);
-		expect(page.locator('link[rel="prefetch"][href$="/prefetch-load"]')).toBeDefined();
 	});
 });
 
@@ -311,7 +315,9 @@ test.describe('Prefetch (default), Experimental ({ clientPrerender: true })', ()
 
 	let devServer;
 
-	test.beforeAll(async ({ astro }) => {
+	test.beforeAll(async ({ astro, browserName }) => {
+		test.skip(browserName !== 'chromium', 'Only Chromium supports clientPrerender')
+		
 		devServer = await astro.startDevServer({
 			experimental: {
 				clientPrerender: true,
@@ -320,7 +326,7 @@ test.describe('Prefetch (default), Experimental ({ clientPrerender: true })', ()
 	});
 
 	test.afterAll(async () => {
-		await devServer.stop();
+		await devServer?.stop();
 	});
 
 	test('Link without data-astro-prefetch should not prefetch', async ({ page, astro }) => {

packages/astro/playwright.firefox.config.js

@@ -5,7 +5,7 @@ process.stdout.isTTY = false;
 
 const config = {
 	// TODO: add more tests like view transitions and audits, and fix them. Some of them are failing.
-	testMatch: ['e2e/css.test.js'],
+	testMatch: ['e2e/css.test.js', 'e2e/prefetch.test.js'],
 	/* Maximum time one test can run for. */
 	timeout: 40 * 1000,
 	expect: {

packages/astro/src/prefetch/index.ts

@@ -59,7 +59,7 @@ function initTapStrategy() {
 			event,
 			(e) => {
 				if (elMatchesStrategy(e.target, 'tap')) {
-					prefetch(e.target.href, { with: 'fetch', ignoreSlowConnection: true });
+					prefetch(e.target.href, { ignoreSlowConnection: true });
 				}
 			},
 			{ passive: true }
@@ -107,7 +107,7 @@ function initHoverStrategy() {
 			clearTimeout(timeout);
 		}
 		timeout = setTimeout(() => {
-			prefetch(href, { with: 'fetch' });
+			prefetch(href);
 		}, 80) as unknown as number;
 	}
 
@@ -158,7 +158,7 @@ function createViewportIntersectionObserver() {
 					setTimeout(() => {
 						observer.unobserve(anchor);
 						timeouts.delete(anchor);
-						prefetch(anchor.href, { with: 'link' });
+						prefetch(anchor.href);
 					}, 300) as unknown as number
 				);
 			} else {
@@ -180,7 +180,7 @@ function initLoadStrategy() {
 		for (const anchor of document.getElementsByTagName('a')) {
 			if (elMatchesStrategy(anchor, 'load')) {
 				// Prefetch every link in this page
-				prefetch(anchor.href, { with: 'link' });
+				prefetch(anchor.href);
 			}
 		}
 	});
@@ -189,8 +189,12 @@ function initLoadStrategy() {
 export interface PrefetchOptions {
 	/**
 	 * How the prefetch should prioritize the URL. (default `'link'`)
-	 * - `'link'`: use `<link rel="prefetch">`, has lower loading priority.
+	 * - `'link'`: use `<link rel="prefetch">`.
-	 * - `'fetch'`: use `fetch()`, has higher loading priority.
+	 * - `'fetch'`: use `fetch()`.
+	 *
+	 * @deprecated It is recommended to not use this option, and let prefetch use `'link'` whenever it's supported,
+	 * or otherwise fall back to `'fetch'`. `'link'` works better if the URL doesn't set an appropriate cache header,
+	 * as the browser will continue to cache it as long as it's used subsequently.
 	 */
 	with?: 'link' | 'fetch';
 	/**
@@ -215,28 +219,27 @@ export function prefetch(url: string, opts?: PrefetchOptions) {
 	if (!canPrefetchUrl(url, ignoreSlowConnection)) return;
 	prefetchedUrls.add(url);
 
-	const priority = opts?.with ?? 'link';
+	// Prefetch with speculationrules if `clientPrerender` is enabled and supported
-	debug?.(`[astro] Prefetching ${url} with ${priority}`);
+	// NOTE: This condition is tree-shaken if `clientPrerender` is false as its a static value
-
+	if (clientPrerender && HTMLScriptElement.supports?.('speculationrules')) {
-	if (
+		debug?.(`[astro] Prefetching ${url} with <script type="speculationrules">`);
-		clientPrerender &&
-		HTMLScriptElement.supports &&
-		HTMLScriptElement.supports('speculationrules')
-	) {
-		// this code is tree-shaken if unused
 		appendSpeculationRules(url);
-	} else if (priority === 'link') {
+	}
+	// Prefetch with link if supported
+	else if (
+		document.createElement('link').relList?.supports?.('prefetch') &&
+		opts?.with !== 'fetch'
+	) {
+		debug?.(`[astro] Prefetching ${url} with <link rel="prefetch">`);
 		const link = document.createElement('link');
 		link.rel = 'prefetch';
 		link.setAttribute('href', url);
 		document.head.append(link);
-	} else {
+	}
-		fetch(url).catch((e) => {
+	// Otherwise, fallback prefetch with fetch
-			// eslint-disable-next-line no-console
+	else {
-			console.log(`[astro] Failed to prefetch ${url}`);
+		debug?.(`[astro] Prefetching ${url} with fetch`);
-			// eslint-disable-next-line no-console
+		fetch(url, { priority: 'low' });
-			console.error(e);
-		});
 	}
 }