Adds a prefetch integration for near-instant page navigations (#3725)

* Adds a basic @astrojs/prefetch integration

* adding tests for custom selectors

* missed in last commit

* Adding a few docs, removing the option for `selectors` to be an element array

* adding an option for the concurrency limit

* fixing test for updated integration options

* Update packages/labs/prefetch/src/client.ts

Co-authored-by: Nate Moore <natemoo-re@users.noreply.github.com>

* nit: removing the NodeJS.Timer type to allow typescript to infer the return

* updating docs for default selector with ~=

* Skip prefetching on 2G connections, or when data saver is enabled

* refactor: moving to packages/integrations, Astro Labs TBD down the road

* README typo fix

Co-authored-by: Nate Moore <natemoo-re@users.noreply.github.com>

packages/integrations/prefetch/@types/network-information.d.ts

@@ -0,0 +1,15 @@
+export { };
+
+declare global {
+	interface NetworkInformation {
+		// http://wicg.github.io/netinfo/#effectiveconnectiontype-enum
+		readonly effectiveType: '2g' | '3g' | '4g' | 'slow-2g';
+		// http://wicg.github.io/netinfo/#savedata-attribute
+		readonly saveData?: boolean;
+	}
+
+	var NetworkInformation: {
+    prototype: NetworkInformation;
+    new(): NetworkInformation;
+	};
+}

packages/integrations/prefetch/README.md

@@ -0,0 +1,127 @@
+# @astrojs/prefetch 🔗
+
+- <strong>[Why Prefetch?](#why-prefetch)</strong>
+- <strong>[Installation](#installation)</strong>
+- <strong>[Usage](#usage)</strong>
+- <strong>[Configuration](#configuration)</strong>
+- <strong>[Examples](#examples)</strong>
+- <strong>[Troubleshooting](#troubleshooting)</strong>
+- <strong>[Contributing](#contributing)</strong>
+- <strong>[Changelog](#changelog)</strong>
+
+## Why Prefetch?
+
+Page load times play a big role in usability and overall enjoyment of a site. This integration brings the benefits of near-instant page navigations to your multi-page application (MPA) by prefetching page links when they are visible on screen.
+
+To further improve the experience, especially on similar pages, stylesheets are also prefetched along with the HTML. This is particularly useful when navigating between tabs on a static site, where most of the page's content and styles don't change.
+
+## Installation
+
+<details>
+  <summary>Quick Install</summary>
+  <br/>
+  
+The experimental `astro add` command-line tool automates the installation for you. Run one of the following commands in a new terminal window. (If you aren't sure which package manager you're using, run the first command.) Then, follow the prompts, and type "y" in the terminal (meaning "yes") for each one.
+  
+  ```sh
+  # Using NPM
+  npx astro add prefetch
+  # Using Yarn
+  yarn astro add prefetch
+  # Using PNPM
+  pnpx astro add prefetch
+  ```
+  
+Then, restart the dev server by typing `CTRL-C` and then `npm run astro dev` in the terminal window that was running Astro.
+  
+Because this command is new, it might not properly set things up. If that happens, [feel free to log an issue on our GitHub](https://github.com/withastro/astro/issues) and try the manual installation steps below.
+</details>
+
+<details>
+  <summary>Manual Install</summary>
+  
+<br/>
+  
+First, install the `@astrojs/prefetch` package using your package manager. If you're using npm or aren't sure, run this in the terminal:
+```sh
+npm install @astrojs/prefetch
+```
+Then, apply this integration to your `astro.config.*` file using the `integrations` property:
+
+__astro.config.mjs__
+
+```js
+import prefetch from '@astrojs/prefetch';
+
+export default {
+  // ...
+  integrations: [prefetch()],
+}
+```
+  
+Then, restart the dev server.
+</details>
+
+## Usage
+
+When you install the integration, the prefetch script is automatically added to every page in the project. Just add `rel="prefetch"` to any `<a />` links on your page and you're ready to go!
+
+## Configuration
+
+The Astro Prefetch integration handles which links on the site are prefetched and it has its own options. Change these in the `astro.config.mjs` file which is where your project's integration settings live.
+
+<details>
+  <summary><strong>config.selector</strong></summary>
+  
+  <br/>
+  
+By default the prefetch script searches the page for any links that include a `rel="prefetch"` attribute, ex: `<a rel="prefetch" />` or `<a rel="nofollow prefetch" />`. This behavior can be changed in your `astro.config.*` file to use a custom query selector when finding prefetch links. 
+  
+  <br/>
+
+```js
+import prefetch from '@astrojs/prefetch';
+
+export default {
+  // ...
+  integrations: [prefetch({
+    // Only prefetch links with an href that begins with `/products`
+    selector: "a[href^='/products']"
+  })],
+}
+```
+</details>
+
+<details>
+  <summary><strong>config.throttle</strong></summary>
+  
+  <br/>
+  
+By default the prefetch script will only prefetch one link at a time. This behavior can be changed in your `astro.config.*` file to increase the limit for concurrent downloads.
+  
+  <br/>
+
+```js
+import prefetch from '@astrojs/prefetch';
+
+export default {
+  // ...
+  integrations: [prefetch({
+    // Allow up to three links to be prefetched concurrently
+    throttle: 3
+  })],
+}
+```
+</details>
+
+## Examples
+
+> Coming soon!
+
+## Troubleshooting
+- If your installation doesn't seem to be working, make sure to restart the dev server.
+- If a link doesn't seem to be prefetching, make sure that the link is pointing to a page on the same domain and matches the integration's `selector` option.
+
+## Contributing
+
+This package is maintained by Astro's Core team. You're welcome to submit an issue or PR!

packages/integrations/prefetch/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "@astrojs/prefetch",
+  "description": "Faster page navigations by prefetching links when the browser is idle.",
+  "version": "0.0.1",
+  "type": "module",
+  "types": "./dist/index.d.ts",
+  "author": "withastro",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/withastro/astro.git",
+    "directory": "packages/astro-prefetch"
+  },
+  "bugs": "https://github.com/withastro/astro/issues",
+  "homepage": "https://astro.build",
+  "exports": {
+    ".": "./dist/index.js",
+    "./client.js": "./dist/client.js",
+    "./package.json": "./package.json"
+  },
+  "scripts": {
+    "build": "astro-scripts build \"src/**/*.ts\" && tsc",
+    "build:ci": "astro-scripts build \"src/**/*.ts\"",
+    "dev": "astro-scripts dev \"src/**/*.ts\"",
+    "test": "playwright test",
+    "test:match": "playwright test -g"
+  },
+  "devDependencies": {
+    "@types/chai": "^4.3.1",
+    "@types/chai-as-promised": "^7.1.5",
+    "@types/mocha": "^9.1.1",
+    "astro": "workspace:*",
+    "astro-scripts": "workspace:*",
+    "playwright": "^1.22.2"
+  },
+  "dependencies": {
+    "throttles": "^1.0.1"
+  }
+}

packages/integrations/prefetch/playwright.config.js

@@ -0,0 +1,41 @@
+import { devices } from '@playwright/test';
+
+const config = {
+	testMatch: 'test/*.test.js',
+	/* Maximum time one test can run for. */
+	timeout: 30 * 1000,
+	expect: {
+		/**
+		 * Maximum time expect() should wait for the condition to be met.
+		 * For example in `await expect(locator).toHaveText();`
+		 */
+		timeout: 5000,
+	},
+	/* Fail the build on CI if you accidentally left test in the source code. */
+	forbidOnly: !!process.env.CI,
+	/* Retry on CI only */
+	retries: process.env.CI ? 5 : 0,
+	/* Opt out of parallel tests on CI. */
+	workers: 1,
+	/* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+	use: {
+		/* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
+		actionTimeout: 0,
+		/* Base URL to use in actions like `await page.goto('/')`. */
+		baseURL: process.env.PLAYWRIGHT_TEST_BASE_URL || 'http://localhost:3000',
+
+		/* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+		trace: 'on-first-retry',
+	},
+	projects: [
+		{
+			name: 'Chrome Stable',
+			use: {
+				browserName: 'chromium',
+				channel: 'chrome',
+			},
+		},
+	],
+};
+
+export default config;

packages/integrations/prefetch/src/client.ts

@@ -0,0 +1,109 @@
+/// <reference path="../@types/network-information.d.ts" />
+import throttles from 'throttles';
+import requestIdleCallback from './requestIdleCallback.js';
+
+const events = ['mouseenter', 'touchstart', 'focus'];
+
+const preloaded = new Set<string>();
+
+function shouldPreload({ href }: { href: string }) {
+	try {
+		const url = new URL(href);
+		return (
+			window.location.origin === url.origin &&
+			window.location.pathname !== url.hash &&
+			!preloaded.has(href)
+		);
+	} catch {}
+
+	return false;
+}
+
+let parser: DOMParser;
+let observer: IntersectionObserver;
+
+function observe(link: HTMLAnchorElement) {
+	preloaded.add(link.href);
+	observer.observe(link);
+	events.map((event) => link.addEventListener(event, onLinkEvent, { once: true }));
+}
+
+function unobserve(link: HTMLAnchorElement) {
+	observer.unobserve(link);
+	events.map((event) => link.removeEventListener(event, onLinkEvent));
+}
+
+function onLinkEvent({ target }: Event) {
+	if (!(target instanceof HTMLAnchorElement)) {
+		return;
+	}
+
+	preloadHref(target);
+}
+
+async function preloadHref(link: HTMLAnchorElement) {
+	unobserve(link);
+
+	const { href } = link;
+
+	try {
+		const contents = await fetch(href).then((res) => res.text());
+		parser = parser || new DOMParser();
+
+		const html = parser.parseFromString(contents, 'text/html');
+		const styles = Array.from(html.querySelectorAll<HTMLLinkElement>('link[rel="stylesheet"]'));
+
+		await Promise.all(styles.map(({ href }) => fetch(href)));
+	} catch {}
+}
+
+export interface PrefetchOptions {
+	/**
+	 * Element selector used to find all links on the page that should be prefetched.
+	 *
+	 * @default 'a[href][rel~="prefetch"]'
+	 */
+	selector?: string;
+	/**
+	 * The number of pages that can be prefetched concurrently.
+	 * 
+	 * @default 1
+	 */
+	throttle?: number;
+}
+
+export default function prefetch({ selector = 'a[href][rel~="prefetch"]', throttle = 1 }: PrefetchOptions) {
+	const conn = navigator.connection;
+
+	if (typeof conn !== 'undefined') {
+    // Don't prefetch if using 2G or if Save-Data is enabled.
+    if (conn.saveData) {
+      return Promise.reject(new Error('Cannot prefetch, Save-Data is enabled'));
+    }
+    if (/2g/.test(conn.effectiveType)) {
+      return Promise.reject(new Error('Cannot prefetch, network conditions are poor'));
+    }
+  }
+
+	const [toAdd, isDone] = throttles(throttle);
+
+	observer =
+		observer ||
+		new IntersectionObserver((entries) => {
+			entries.forEach((entry) => {
+				if (entry.isIntersecting && entry.target instanceof HTMLAnchorElement) {
+					toAdd(() => preloadHref(entry.target as HTMLAnchorElement).finally(isDone));
+				}
+			});
+		});
+
+	requestIdleCallback(() => {
+		const links = Array.from(document.querySelectorAll<HTMLAnchorElement>(selector)).filter(
+			shouldPreload
+		);
+
+		for (const link of links) {
+			observe(link);
+		}
+	});
+}

packages/integrations/prefetch/src/index.ts

@@ -0,0 +1,17 @@
+import type { AstroIntegration } from 'astro';
+import type { PrefetchOptions } from './client.js';
+
+export default function (options: PrefetchOptions = {}): AstroIntegration {
+	return {
+		name: '@astrojs/lit',
+		hooks: {
+			'astro:config:setup': ({ updateConfig, addRenderer, injectScript }) => {
+				// Inject the necessary polyfills on every page (inlined for speed).
+				injectScript(
+					'page',
+					`import prefetch from "@astrojs/prefetch/client.js"; prefetch(${JSON.stringify(options)});`
+				);
+			}
+		}
+	};
+}

packages/integrations/prefetch/src/requestIdleCallback.ts

@@ -0,0 +1,16 @@
+function shim(callback: IdleRequestCallback, options?: IdleRequestOptions) {
+	const timeout = options?.timeout ?? 50;
+	const start = Date.now();
+
+	return setTimeout(function () {
+		callback({
+			didTimeout: false,
+			timeRemaining: function () {
+				return Math.max(0, timeout - (Date.now() - start));
+			},
+		});
+	}, 1);
+}
+
+const requestIdleCallback = window.requestIdleCallback || shim;
+export default requestIdleCallback;

packages/integrations/prefetch/test/basic-prefetch.test.js

@@ -0,0 +1,64 @@
+import { expect } from '@playwright/test';
+import { testFactory } from './test-utils.js';
+
+const test = testFactory({ root: './fixtures/basic-prefetch/' });
+
+test.describe('Basic prefetch', () => {
+	test.describe('dev', () => {
+		let devServer;
+
+		test.beforeEach(async ({ astro }) => {
+			devServer = await astro.startDevServer();
+		});
+
+		test.afterEach(async () => {
+			await devServer.stop();
+		});
+
+		test.describe('prefetches rel="prefetch" links', () => {
+			test('skips /admin', async ({ page, astro }) => {
+				const requests = new Set();
+
+				page.on('request', async (request) => requests.add(request.url()));
+
+				await page.goto(astro.resolveUrl('/'));
+
+				await page.waitForLoadState('networkidle');
+
+				await expect(requests.has(astro.resolveUrl('/about')), '/about was prefetched').toBeTruthy();
+				await expect(requests.has(astro.resolveUrl('/contact')), '/contact was prefetched').toBeTruthy();
+				await expect(requests.has(astro.resolveUrl('/admin')), '/admin was skipped').toBeFalsy();
+			});
+		});
+	});
+
+	test.describe('build', () => {
+		let previewServer;
+
+		test.beforeAll(async ({ astro }) => {
+			await astro.build();
+			previewServer = await astro.preview();
+		});
+
+		// important: close preview server (free up port and connection)
+		test.afterAll(async () => {
+			await previewServer.stop();
+		});
+
+		test.describe('prefetches rel="prefetch" links', () => {
+			test('skips /admin', async ({ page, astro }) => {
+				const requests = new Set();
+
+				page.on('request', async (request) => requests.add(request.url()));
+
+				await page.goto(astro.resolveUrl('/'));
+
+				await page.waitForLoadState('networkidle');
+
+				await expect(requests.has(astro.resolveUrl('/about')), '/about was prefetched').toBeTruthy();
+				await expect(requests.has(astro.resolveUrl('/contact')), '/contact was prefetched').toBeTruthy();
+				await expect(requests.has(astro.resolveUrl('/admin')), '/admin was skipped').toBeFalsy();
+			});
+		});
+	});
+});

packages/integrations/prefetch/test/custom-selectors.test.js

@@ -0,0 +1,72 @@
+import { expect } from '@playwright/test';
+import { testFactory } from './test-utils.js';
+import prefetch from '../dist/index.js';
+
+const test = testFactory({
+	root: './fixtures/basic-prefetch/',
+	integrations: [
+		prefetch({
+			selector: 'a[href="/contact"]'
+		}),
+	]
+});
+
+test.describe('Custom prefetch selectors', () => {
+	test.describe('dev', () => {
+		let devServer;
+
+		test.beforeEach(async ({ astro }) => {
+			devServer = await astro.startDevServer();
+		});
+
+		test.afterEach(async () => {
+			await devServer.stop();
+		});
+
+		test.describe('prefetches links by custom selector', () => {
+			test('only prefetches /contact', async ({ page, astro }) => {
+				const requests = new Set();
+
+				page.on('request', async (request) => requests.add(request.url()));
+
+				await page.goto(astro.resolveUrl('/'));
+
+				await page.waitForLoadState('networkidle');
+
+				await expect(requests.has(astro.resolveUrl('/about')), '/about was skipped').toBeFalsy();
+				await expect(requests.has(astro.resolveUrl('/contact')), '/contact was prefetched').toBeTruthy();
+				await expect(requests.has(astro.resolveUrl('/admin')), '/admin was skipped').toBeFalsy();
+			});
+		});
+	});
+
+	test.describe('build', () => {
+		let previewServer;
+
+		test.beforeAll(async ({ astro }) => {
+			await astro.build();
+			previewServer = await astro.preview();
+		});
+
+		// important: close preview server (free up port and connection)
+		test.afterAll(async () => {
+			await previewServer.stop();
+		});
+
+		test.describe('prefetches links by custom selector', () => {
+			test('only prefetches /contact', async ({ page, astro }) => {
+				const requests = new Set();
+
+				page.on('request', async (request) => requests.add(request.url()));
+
+				await page.goto(astro.resolveUrl('/'));
+
+				await page.waitForLoadState('networkidle');
+
+				await expect(requests.has(astro.resolveUrl('/about')), '/about was skipped').toBeFalsy();
+				await expect(requests.has(astro.resolveUrl('/contact')), '/contact was prefetched').toBeTruthy();
+				await expect(requests.has(astro.resolveUrl('/admin')), '/admin was skipped').toBeFalsy();
+			});
+		});
+	});
+});

packages/integrations/prefetch/test/fixtures/basic-prefetch/astro.config.mjs

@@ -0,0 +1,7 @@
+import { defineConfig } from 'astro/config';
+import prefetch from '@astrojs/prefetch';
+
+// https://astro.build/config
+export default defineConfig({
+	integrations: [prefetch()],
+});

packages/integrations/prefetch/test/fixtures/basic-prefetch/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "@test/astro-prefetch",
+  "version": "0.0.0",
+  "private": true,
+  "dependencies": {
+    "@astrojs/prefetch": "workspace:*",
+    "astro": "workspace:*"
+  }
+}

packages/integrations/prefetch/test/fixtures/basic-prefetch/src/pages/about.astro

@@ -0,0 +1,11 @@
+---
+---
+
+<html>
+<head>
+<title>About Us</title>
+</head>
+<body>
+	<h1>About Us</h1>
+</body>
+</html>

packages/integrations/prefetch/test/fixtures/basic-prefetch/src/pages/admin.astro

@@ -0,0 +1,11 @@
+---
+---
+
+<html>
+<head>
+<title>Admin</title>
+</head>
+<body>
+	<h1>Admin</h1>
+</body>
+</html>

packages/integrations/prefetch/test/fixtures/basic-prefetch/src/pages/contact.astro

@@ -0,0 +1,11 @@
+---
+---
+
+<html>
+<head>
+<title>Contact Us</title>
+</head>
+<body>
+	<h1>Contact Us</h1>
+</body>
+</html>

packages/integrations/prefetch/test/fixtures/basic-prefetch/src/pages/index.astro

@@ -0,0 +1,26 @@
+---
+---
+
+<html>
+<head>
+<title>Home</title>
+</head>
+<body>
+	<h1>Home</h1>
+
+	<nav>
+		<ul>
+			<li>
+				<a href="/about" rel="prefetch">About</a>
+			</li>
+			<li>
+				<a href="/admin">Admin</a>
+			</li>
+		</ul>
+	</nav>
+
+	<footer>
+		<a href="/contact" rel="prefetch">Contact</a>
+	</footer>
+</body>
+</html>

packages/integrations/prefetch/test/test-utils.js

@@ -0,0 +1,31 @@
+import { test as testBase } from '@playwright/test';
+import { loadFixture as baseLoadFixture } from '../../../astro/test/test-utils.js';
+
+export function loadFixture(inlineConfig) {
+	if (!inlineConfig || !inlineConfig.root)
+		throw new Error("Must provide { root: './fixtures/...' }");
+
+	// resolve the relative root (i.e. "./fixtures/tailwindcss") to a full filepath
+	// without this, the main `loadFixture` helper will resolve relative to `packages/astro/test`
+	return baseLoadFixture({
+		...inlineConfig,
+		root: new URL(inlineConfig.root, import.meta.url).toString(),
+	});
+}
+
+export function testFactory(inlineConfig) {
+	let fixture;
+
+	const test = testBase.extend({
+		astro: async ({}, use) => {
+			fixture = await loadFixture(inlineConfig);
+			await use(fixture);
+		},
+	});
+
+	test.afterEach(() => {
+		fixture.resetAllFiles();
+	});
+
+	return test;
+}

packages/integrations/prefetch/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "include": ["src", "@types"],
+  "compilerOptions": {
+    "allowJs": true,
+    "module": "ES2020",
+    "outDir": "./dist",
+    "target": "ES2020",
+  }
+}

pnpm-lock.yaml

@@ -1982,6 +1982,33 @@ importers:
       astro-scripts: link:../../../scripts
       preact: 10.7.3
 
+  packages/integrations/prefetch:
+    specifiers:
+      '@types/chai': ^4.3.1
+      '@types/chai-as-promised': ^7.1.5
+      '@types/mocha': ^9.1.1
+      astro: workspace:*
+      astro-scripts: workspace:*
+      playwright: ^1.22.2
+      throttles: ^1.0.1
+    dependencies:
+      throttles: 1.0.1
+    devDependencies:
+      '@types/chai': 4.3.1
+      '@types/chai-as-promised': 7.1.5
+      '@types/mocha': 9.1.1
+      astro: link:../../astro
+      astro-scripts: link:../../../scripts
+      playwright: 1.22.2
+
+  packages/integrations/prefetch/test/fixtures/basic-prefetch:
+    specifiers:
+      '@astrojs/prefetch': workspace:*
+      astro: workspace:*
+    dependencies:
+      '@astrojs/prefetch': link:../../..
+      astro: link:../../../../../astro
+
   packages/integrations/react:
     specifiers:
       '@babel/plugin-transform-react-jsx': ^7.17.12
@@ -11998,6 +12025,15 @@ packages:
     hasBin: true
     dev: true
 
+  /playwright/1.22.2:
+    resolution: {integrity: sha512-hUTpg7LytIl3/O4t0AQJS1V6hWsaSY5uZ7w1oCC8r3a1AQN5d6otIdCkiB3cbzgQkcMaRxisinjMFMVqZkybdQ==}
+    engines: {node: '>=14'}
+    hasBin: true
+    requiresBuild: true
+    dependencies:
+      playwright-core: 1.22.2
+    dev: true
+
   /postcss-attribute-case-insensitive/5.0.1_postcss@8.4.14:
     resolution: {integrity: sha512-wrt2VndqSLJpyBRNz9OmJcgnhI9MaongeWgapdBuUMu2a/KNJ8SENesG4SdiTnQwGO9b1VKbTWYAfCPeokLqZQ==}
     engines: {node: ^12 || ^14 || >=16}
@@ -13742,6 +13778,11 @@ packages:
     resolution: {integrity: sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=}
     dev: true
 
+  /throttles/1.0.1:
+    resolution: {integrity: sha512-fab7Xg+zELr9KOv4fkaBoe/b3L0GMGLd0IBSCn16GoE/Qx6/OfCr1eGNyEcDU2pUA79qQfZ8kPQWlRuok4YwTw==}
+    engines: {node: '>=6'}
+    dev: false
+
   /tiny-glob/0.2.9:
     resolution: {integrity: sha512-g/55ssRPUjShh+xkfx9UPDXqhckHEsHr4Vd9zX55oSdGZc/MD0m3sferOkwWtp98bv+kcVfEHtRJgBVJzelrzg==}
     dependencies:
@@ -14772,7 +14813,7 @@ packages:
     dev: false
 
   /wrappy/1.0.2:
-    resolution: {integrity: sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=}
+    resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==}
 
   /xregexp/2.0.0:
     resolution: {integrity: sha1-UqY+VsoLhKfzpfPWGHLxJq16WUM=}