Add web-vitals integration (#10883)

.changeset/great-swans-punch.md

@@ -0,0 +1,5 @@
+---
+"@astrojs/web-vitals": minor
+---
+
+Adds a new web-vitals integration powered by Astro DB

packages/integrations/web-vitals/README.md

@@ -0,0 +1,59 @@
+# @astrojs/web-vitals (experimental) ⏱️
+
+This **[Astro integration][astro-integration]** enables tracking real-world website performance and storing the data in [Astro DB][db].
+
+## Pre-requisites
+
+- [Astro DB](https://astro.build/db) — `@astrojs/web-vitals` will store performance data in Astro DB in production
+- [An SSR adapter](https://docs.astro.build/en/guides/server-side-rendering/) — `@astrojs/web-vitals` injects a server endpoint to manage saving data to Astro DB
+
+## Installation
+
+1. Install and configure the Web Vitals integration using `astro add`:
+
+   ```sh
+   npx astro add web-vitals
+   ```
+
+2. Push the tables added by the Web Vitals integration to Astro Studio:
+
+   ```sh
+   npx astro db push
+   ```
+
+3. Redeploy your site.
+
+4. Visit your project dashboard at https://studio.astro.build to see the data collected.
+
+Learn more about [Astro DB](https://docs.astro.build/en/guides/astro-db/) and [deploying with Astro Studio](https://docs.astro.build/en/guides/astro-db/#astro-studio) in the Astro docs.
+
+## Support
+
+- Get help in the [Astro Discord][discord]. Post questions in our `#support` forum, or visit our dedicated `#dev` channel to discuss current development and more!
+
+- Check our [Astro Integration Documentation][astro-integration] for more on integrations.
+
+- Submit bug reports and feature requests as [GitHub issues][issues].
+
+## Contributing
+
+This package is maintained by Astro's Core team. You're welcome to submit an issue or PR! These links will help you get started:
+
+- [Contributor Manual][contributing]
+- [Code of Conduct][coc]
+- [Community Guide][community]
+
+## License
+
+MIT
+
+Copyright (c) 2023–present [Astro][astro]
+
+[astro]: https://astro.build/
+[db]: https://astro.build/db/
+[contributing]: https://github.com/withastro/astro/blob/main/CONTRIBUTING.md
+[coc]: https://github.com/withastro/.github/blob/main/CODE_OF_CONDUCT.md
+[community]: https://github.com/withastro/.github/blob/main/COMMUNITY_GUIDE.md
+[discord]: https://astro.build/chat/
+[issues]: https://github.com/withastro/astro/issues
+[astro-integration]: https://docs.astro.build/en/guides/integrations-guide/

packages/integrations/web-vitals/package.json

@@ -0,0 +1,49 @@
+{
+  "name": "@astrojs/web-vitals",
+  "description": "Track your website’s performance with Astro DB",
+  "version": "0.0.0",
+  "type": "module",
+  "author": "withastro",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/withastro/astro.git",
+    "directory": "packages/integrations/web-vitals"
+  },
+  "keywords": [
+    "withastro",
+    "astro-integration"
+  ],
+  "bugs": "https://github.com/withastro/astro/issues",
+  "exports": {
+    ".": "./dist/index.js",
+    "./middleware": "./dist/middleware.js",
+    "./endpoint": "./dist/endpoint.js",
+    "./client-script": "./dist/client-script.js",
+    "./db-config": "./dist/db-config.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "astro-scripts build \"src/**/*.ts\" && tsc",
+    "build:ci": "astro-scripts build \"src/**/*.ts\"",
+    "dev": "astro-scripts dev \"src/**/*.ts\"",
+    "test": "astro-scripts test --timeout 50000 \"test/**/*.test.js\""
+  },
+	"dependencies": {
+		"web-vitals": "^3.5.2"
+	},
+  "peerDependencies": {
+    "@astrojs/db": "^0.11.0"
+  },
+	"devDependencies": {
+		"@astrojs/db": "workspace:*",
+		"astro": "workspace:*",
+    "astro-scripts": "workspace:*",
+    "linkedom": "^0.16.11"
+	},
+  "publishConfig": {
+    "provenance": true
+  }
+}

packages/integrations/web-vitals/src/client-script.ts

@@ -0,0 +1,36 @@
+import { type Metric, onCLS, onFCP, onFID, onINP, onLCP, onTTFB } from 'web-vitals';
+import { WEB_VITALS_ENDPOINT_PATH } from './constants.js';
+import type { ClientMetric } from './schemas.js';
+
+const pathname = location.pathname.replace(/(?<=.)\/$/, '');
+const route =
+	document
+		.querySelector<HTMLMetaElement>('meta[name="x-astro-vitals-route"]')
+		?.getAttribute('content') || pathname;
+
+const queue = new Set<Metric>();
+const addToQueue = (metric: Metric) => queue.add(metric);
+function flushQueue() {
+	if (!queue.size) return;
+	const rawBody: ClientMetric[] = [...queue].map(({ name, id, value, rating }) => ({
+		pathname,
+		route,
+		name,
+		id,
+		value,
+		rating,
+	}));
+	const body = JSON.stringify(rawBody);
+	if (navigator.sendBeacon) navigator.sendBeacon(WEB_VITALS_ENDPOINT_PATH, body);
+	else fetch(WEB_VITALS_ENDPOINT_PATH, { body, method: 'POST', keepalive: true });
+	queue.clear();
+}
+
+for (const listener of [onCLS, onLCP, onINP, onFID, onFCP, onTTFB]) {
+	listener(addToQueue);
+}
+
+addEventListener('visibilitychange', () => {
+	if (document.visibilityState === 'hidden') flushQueue();
+});
+addEventListener('pagehide', flushQueue);

packages/integrations/web-vitals/src/constants.ts

@@ -0,0 +1 @@
+export const WEB_VITALS_ENDPOINT_PATH = '/_web-vitals'

packages/integrations/web-vitals/src/db-config.ts

@@ -0,0 +1,22 @@
+import { column, defineDb, defineTable } from 'astro:db';
+import { asDrizzleTable } from '@astrojs/db/utils';
+
+const Metric = defineTable({
+	columns: {
+		pathname: column.text(),
+		route: column.text(),
+		name: column.text(),
+		id: column.text({ primaryKey: true }),
+		value: column.number(),
+		rating: column.text(),
+		timestamp: column.date(),
+	},
+});
+
+export const AstrojsWebVitals_Metric = asDrizzleTable('AstrojsWebVitals_Metric', Metric);
+
+export default defineDb({
+	tables: {
+		AstrojsWebVitals_Metric: Metric,
+	},
+});

packages/integrations/web-vitals/src/endpoint.ts

@@ -0,0 +1,23 @@
+import { db, sql } from 'astro:db';
+import type { APIRoute } from 'astro';
+import { AstrojsWebVitals_Metric } from './db-config.js';
+import { ServerMetricSchema } from './schemas.js';
+
+export const prerender = false;
+
+export const ALL: APIRoute = async ({ request }) => {
+	try {
+		const rawBody = await request.json();
+		const body = ServerMetricSchema.array().parse(rawBody);
+		await db
+			.insert(AstrojsWebVitals_Metric)
+			.values(body)
+			.onConflictDoUpdate({
+				target: AstrojsWebVitals_Metric.id,
+				set: { value: sql`excluded.value` },
+			});
+	} catch (error) {
+		console.error(error);
+	}
+	return new Response();
+};

packages/integrations/web-vitals/src/env.d.ts

@@ -0,0 +1 @@
+/// <reference types="@astrojs/db" />

packages/integrations/web-vitals/src/index.ts

@@ -0,0 +1,42 @@
+import { defineDbIntegration } from '@astrojs/db/utils';
+import { AstroError } from 'astro/errors';
+import { WEB_VITALS_ENDPOINT_PATH } from './constants.js';
+
+export default function webVitals() {
+	return defineDbIntegration({
+		name: '@astrojs/web-vitals',
+		hooks: {
+			'astro:db:setup'({ extendDb }) {
+				extendDb({ configEntrypoint: '@astrojs/web-vitals/db-config' });
+			},
+
+			'astro:config:setup'({ addMiddleware, config, injectRoute, injectScript }) {
+				if (!config.integrations.find(({ name }) => name === 'astro:db')) {
+					throw new AstroError(
+						'Astro DB integration not found.',
+						'Run `npx astro add db` to install `@astrojs/db` and add it to your Astro config.'
+					);
+				}
+
+				if (config.output !== 'hybrid' && config.output !== 'server') {
+					throw new AstroError(
+						'No SSR adapter found.',
+						'`@astrojs/web-vitals` requires your site to be built with `hybrid` or `server` output.\n' +
+							'Please add an SSR adapter: https://docs.astro.build/en/guides/server-side-rendering/'
+					);
+				}
+
+				// Middleware that adds a `<meta>` tag to each page.
+				addMiddleware({ entrypoint: '@astrojs/web-vitals/middleware', order: 'post' });
+				// Endpoint that collects metrics and inserts them in Astro DB.
+				injectRoute({
+					entrypoint: '@astrojs/web-vitals/endpoint',
+					pattern: WEB_VITALS_ENDPOINT_PATH,
+					prerender: false,
+				});
+				// Client-side performance measurement script.
+				injectScript('page', `import '@astrojs/web-vitals/client-script';`);
+			},
+		},
+	});
+}

packages/integrations/web-vitals/src/middleware.ts

@@ -0,0 +1,60 @@
+import type { MiddlewareHandler } from 'astro';
+
+/**
+ * Middleware which adds the web vitals `<meta>` tag to each page’s `<head>`.
+ *
+ * @example
+ * <meta name="x-astro-vitals-route" content="/blog/[slug]" />
+ */
+export const onRequest: MiddlewareHandler = async ({ params, url }, next) => {
+	const response = await next();
+	const contentType = response.headers.get('Content-Type');
+	if (contentType !== 'text/html') return response;
+	const webVitalsMetaTag = getMetaTag(url, params);
+	return new Response(
+		response.body
+			?.pipeThrough(new TextDecoderStream())
+			.pipeThrough(HeadInjectionTransformStream(webVitalsMetaTag))
+			.pipeThrough(new TextEncoderStream()),
+		response
+	);
+};
+
+/** TransformStream which injects the passed HTML just before the closing </head> tag.  */
+function HeadInjectionTransformStream(htmlToInject: string) {
+	let hasInjected = false;
+	return new TransformStream({
+		transform: (chunk, controller) => {
+			if (!hasInjected) {
+				const headCloseIndex = chunk.indexOf('</head>');
+				if (headCloseIndex > -1) {
+					chunk = chunk.slice(0, headCloseIndex) + htmlToInject + chunk.slice(headCloseIndex);
+					hasInjected = true;
+				}
+			}
+			controller.enqueue(chunk);
+		},
+	});
+}
+
+/** Get a `<meta>` tag to identify the current Astro route. */
+function getMetaTag(url: URL, params: Record<string, string | undefined>) {
+	let route = url.pathname;
+	for (const [key, value] of Object.entries(params)) {
+		if (value) route = route.replace(value, `[${key}]`);
+	}
+	route = miniEncodeAttribute(stripTrailingSlash(route));
+	return `<meta name="x-astro-vitals-route" content="${route}" />`;
+}
+
+function stripTrailingSlash(str: string) {
+	return str.length > 1 && str.at(-1) === '/' ? str.slice(0, -1) : str;
+}
+
+function miniEncodeAttribute(str: string) {
+	return str
+		.replaceAll('&', '&amp;')
+		.replaceAll('<', '&lt;')
+		.replaceAll('>', '&gt;')
+		.replaceAll('"', '&quot;');
+}

packages/integrations/web-vitals/src/schemas.ts

@@ -0,0 +1,32 @@
+import { z } from 'astro/zod';
+
+export const RatingSchema = z.enum(['good', 'needs-improvement', 'poor']);
+const MetricTypeSchema = z.enum(['CLS', 'INP', 'LCP', 'FCP', 'FID', 'TTFB']);
+
+/** `web-vitals` generated ID, transformed to reduce data resolution. */
+const MetricIdSchema = z
+	.string()
+	// Match https://github.com/GoogleChrome/web-vitals/blob/main/src/lib/generateUniqueID.ts
+	.regex(/^v3-\d{13}-\d{13}$/)
+	// Avoid collecting higher resolution timestamp in ID.
+	// Transforms `'v3-1711484350895-3748043125387'` to `'v3-17114843-3748043125387'`
+	.transform((id) => id.replace(/^(v3-\d{8})\d{5}(-\d{13})$/, '$1$2'));
+
+/** Shape of the data submitted from clients to the collection API. */
+const ClientMetricSchema = z.object({
+	pathname: z.string(),
+	route: z.string(),
+	name: MetricTypeSchema,
+	id: MetricIdSchema,
+	value: z.number().gte(0),
+	rating: RatingSchema,
+});
+
+/** Transformed client data with added timestamp. */
+export const ServerMetricSchema = ClientMetricSchema.transform((metric) => {
+	const timestamp = new Date();
+	timestamp.setMinutes(0, 0, 0);
+	return { ...metric, timestamp };
+});
+
+export type ClientMetric = z.input<typeof ClientMetricSchema>;

packages/integrations/web-vitals/test/basics.test.js

@@ -0,0 +1,118 @@
+// @ts-check
+
+import * as assert from 'node:assert/strict';
+import { after, before, beforeEach, describe, it } from 'node:test';
+import { parseHTML } from 'linkedom';
+import { loadFixture } from './test-utils.js';
+
+/**
+ * @template {Record<K, (...args: any[]) => void>} T
+ * @template {keyof T} K
+ */
+class MockFunction {
+	/** @type {Parameters<T[K]>[]} */
+	calls = [];
+
+	/**
+	 * @param {T} object
+	 * @param {K} property
+	 */
+	constructor(object, property) {
+		this.object = object;
+		this.property = property;
+		this.original = object[property];
+		object[property] = /** @param {Parameters<T[K]>} args */ (...args) => {
+			this.calls.push(args);
+		};
+	}
+	restore() {
+		this.object[this.property] = this.original;
+	}
+	reset() {
+		this.calls = [];
+	}
+}
+
+describe('Web Vitals integration basics', () => {
+	/** @type {import('./test-utils').Fixture} */
+	let fixture;
+	/** @type {import('./test-utils').DevServer} */
+	let devServer;
+	/** @type {MockFunction<Console, 'error'>} */
+	let consoleErrorMock;
+
+	before(async () => {
+		consoleErrorMock = new MockFunction(console, 'error');
+		fixture = await loadFixture({ root: './fixtures/basics/' });
+		devServer = await fixture.startDevServer({});
+	});
+
+	after(async () => {
+		consoleErrorMock.restore();
+		await devServer.stop();
+	});
+
+	beforeEach(() => {
+		consoleErrorMock.reset();
+	});
+
+	it('adds a meta tag to the page', async () => {
+		const html = await fixture.fetch('/', {}).then((res) => res.text());
+		const { document } = parseHTML(html);
+		const meta = document.querySelector('head > meta[name="x-astro-vitals-route"]');
+		assert.ok(meta);
+		assert.equal(meta.getAttribute('content'), '/');
+	});
+
+	it('adds a meta tag using the route pattern to the page', async () => {
+		const html = await fixture.fetch('/test', {}).then((res) => res.text());
+		const { document } = parseHTML(html);
+		const meta = document.querySelector('head > meta[name="x-astro-vitals-route"]');
+		assert.ok(meta);
+		assert.equal(meta.getAttribute('content'), '/[dynamic]');
+	});
+
+	it('returns a 200 response even when bad data is sent to the injected endpoint', async () => {
+		{
+			// bad data
+			const res = await fixture.fetch('/_web-vitals', { method: 'POST', body: 'garbage' });
+			assert.equal(res.status, 200);
+		}
+		{
+			// no data
+			const res = await fixture.fetch('/_web-vitals', { method: 'POST', body: '[]' });
+			assert.equal(res.status, 200);
+		}
+		assert.equal(consoleErrorMock.calls.length, 2);
+	});
+
+	it('validates data sent to the injected endpoint with Zod', async () => {
+		const res = await fixture.fetch('/_web-vitals', { method: 'POST', body: '[{}]' });
+		assert.equal(res.status, 200);
+		const call = consoleErrorMock.calls[0][0];
+		assert.ok(call instanceof Error);
+		assert.equal(call.name, 'ZodError');
+	});
+
+	it('inserts data via the injected endpoint', async () => {
+		const res = await fixture.fetch('/_web-vitals', {
+			method: 'POST',
+			body: JSON.stringify([
+				{
+					pathname: '/',
+					route: '/',
+					name: 'CLS',
+					id: 'v3-1711484350895-3748043125387',
+					value: 0,
+					rating: 'good',
+				},
+			]),
+		});
+		assert.equal(res.status, 200);
+		assert.equal(
+			consoleErrorMock.calls.length,
+			0,
+			'Endpoint logged errors:\n' + consoleErrorMock.calls[0]?.join(' ')
+		);
+	});
+});

packages/integrations/web-vitals/test/fixtures/basics/astro.config.mjs

@@ -0,0 +1,14 @@
+import db from '@astrojs/db';
+import node from '@astrojs/node';
+import webVitals from '@astrojs/web-vitals';
+import { defineConfig } from 'astro/config';
+
+// https://astro.build/config
+export default defineConfig({
+	integrations: [db(), webVitals()],
+	output: 'hybrid',
+	adapter: node({ mode: 'standalone' }),
+	devToolbar: {
+		enabled: false,
+	},
+});

packages/integrations/web-vitals/test/fixtures/basics/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "@test/web-vitals",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "dev": "astro dev",
+    "build": "astro build",
+    "preview": "astro preview"
+  },
+  "dependencies": {
+    "@astrojs/db": "workspace:*",
+    "@astrojs/node": "workspace:*",
+    "@astrojs/web-vitals": "workspace:*",
+    "astro": "workspace:*"
+  }
+}

packages/integrations/web-vitals/test/fixtures/basics/src/pages/[dynamic].astro

@@ -0,0 +1,19 @@
+---
+import type { GetStaticPaths } from "astro";
+export const getStaticPaths = (() => {
+	return [{ params: { dynamic: 'test' } }];
+}) satisfies GetStaticPaths;
+---
+
+<!DOCTYPE html>
+<html lang="en">
+<head>
+	<meta charset="UTF-8">
+	<meta name="viewport" content="width=device-width, initial-scale=1.0">
+	<title>Web Vitals basics — dynamic route test</title>
+</head>
+<body>
+	<h1>Web Vitals basics</h1>
+	<p>Dynamic route test</p>
+</body>
+</html>

packages/integrations/web-vitals/test/fixtures/basics/src/pages/index.astro

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+	<meta charset="UTF-8">
+	<meta name="viewport" content="width=device-width, initial-scale=1.0">
+	<title>Web Vitals basics test</title>
+</head>
+<body>
+	<h1>Web Vitals basics test</h1>
+</body>
+</html>

packages/integrations/web-vitals/test/test-utils.js

@@ -0,0 +1,16 @@
+import { loadFixture as baseLoadFixture } from '../../../astro/test/test-utils.js';
+
+/** @typedef {import('../../../astro/test/test-utils').Fixture} Fixture */
+/** @typedef {import('../../../astro/test/test-utils').DevServer} DevServer */
+
+/** @type {typeof import('../../../astro/test/test-utils.js')['loadFixture']} */
+export function loadFixture(inlineConfig) {
+	if (!inlineConfig?.root) throw new Error("Must provide { root: './fixtures/...' }");
+
+	// resolve the relative root (i.e. "./fixtures/tailwindcss") to a full filepath
+	// without this, the main `loadFixture` helper will resolve relative to `packages/astro/test`
+	return baseLoadFixture({
+		...inlineConfig,
+		root: new URL(inlineConfig.root, import.meta.url).toString(),
+	});
+}

packages/integrations/web-vitals/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../../tsconfig.base.json",
+  "include": ["src"],
+  "compilerOptions": {
+    "outDir": "./dist"
+  }
+}

pnpm-lock.yaml

@@ -5327,6 +5327,40 @@ importers:
         specifier: workspace:*
         version: link:../../../../../astro
 
+  packages/integrations/web-vitals:
+    dependencies:
+      web-vitals:
+        specifier: ^3.5.2
+        version: 3.5.2
+    devDependencies:
+      '@astrojs/db':
+        specifier: workspace:*
+        version: link:../../db
+      astro:
+        specifier: workspace:*
+        version: link:../../astro
+      astro-scripts:
+        specifier: workspace:*
+        version: link:../../../scripts
+      linkedom:
+        specifier: ^0.16.11
+        version: 0.16.11
+
+  packages/integrations/web-vitals/test/fixtures/basics:
+    dependencies:
+      '@astrojs/db':
+        specifier: workspace:*
+        version: link:../../../../../db
+      '@astrojs/node':
+        specifier: workspace:*
+        version: link:../../../../node
+      '@astrojs/web-vitals':
+        specifier: workspace:*
+        version: link:../../..
+      astro:
+        specifier: workspace:*
+        version: link:../../../../../astro
+
   packages/internal-helpers:
     devDependencies:
       astro-scripts: