Improve build perf (#2697)

* improve md perf

* chore: add changesets

Co-authored-by: Nate Moore <nate@skypack.dev>

.changeset/orange-coins-whisper.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/markdown-remark': patch
+---
+
+Improve performance by optimizing calls to `getHighlighter`

.changeset/short-rats-double.md

@@ -0,0 +1,5 @@
+---
+'astro': patch
+---
+
+Improve build performance by processing `ssrPreload` in serial rather than in parallel

packages/astro/src/core/build/page-data.ts

@@ -35,89 +35,87 @@ export async function collectPagesData(opts: CollectPagesDataOptions): Promise<C
 	// NOTE: This enforces that `getStaticPaths()` is only called once per route,
 	// and is then cached across all future SSR builds. In the past, we've had trouble
 	// with parallelized builds without guaranteeing that this is called first.
-	await Promise.all(
+	for (const route of manifest.routes) {
-		manifest.routes.map(async (route) => {
+		// static route:
-			// static route:
+		if (route.pathname) {
-			if (route.pathname) {
-				allPages[route.component] = {
-					route,
-					paths: [route.pathname],
-					preload: await ssrPreload({
-						astroConfig,
-						filePath: new URL(`./${route.component}`, astroConfig.projectRoot),
-						logging,
-						mode: 'production',
-						origin,
-						pathname: route.pathname,
-						route,
-						routeCache,
-						viteServer,
-					})
-						.then((routes) => {
-							const html = `${route.pathname}`.replace(/\/?$/, '/index.html');
-							debug('build', `├── ${colors.bold(colors.green('✔'))} ${route.component} → ${colors.yellow(html)}`);
-							return routes;
-						})
-						.catch((err) => {
-							debug('build', `├── ${colors.bold(colors.red('✘'))} ${route.component}`);
-							throw err;
-						}),
-				};
-				return;
-			}
-			// dynamic route:
-			const result = await getStaticPathsForRoute(opts, route)
-				.then((_result) => {
-					const label = _result.staticPaths.length === 1 ? 'page' : 'pages';
-					debug('build', `├── ${colors.bold(colors.green('✔'))} ${route.component} → ${colors.magenta(`[${_result.staticPaths.length} ${label}]`)}`);
-					return _result;
-				})
-				.catch((err) => {
-					debug('build', `├── ${colors.bold(colors.red('✗'))} ${route.component}`);
-					throw err;
-				});
-			const rssFn = generateRssFunction(astroConfig.buildOptions.site, route);
-			for (const rssCallArg of result.rss) {
-				const rssResult = rssFn(rssCallArg);
-				if (rssResult.xml) {
-					const { url, content } = rssResult.xml;
-					if (content) {
-						const rssFile = new URL(url.replace(/^\/?/, './'), astroConfig.dist);
-						if (assets[fileURLToPath(rssFile)]) {
-							throw new Error(`[getStaticPaths] RSS feed ${url} already exists.\nUse \`rss(data, {url: '...'})\` to choose a unique, custom URL. (${route.component})`);
-						}
-						assets[fileURLToPath(rssFile)] = content;
-					}
-				}
-				if (rssResult.xsl?.content) {
-					const { url, content } = rssResult.xsl;
-					const stylesheetFile = new URL(url.replace(/^\/?/, './'), astroConfig.dist);
-					if (assets[fileURLToPath(stylesheetFile)]) {
-						throw new Error(
-							`[getStaticPaths] RSS feed stylesheet ${url} already exists.\nUse \`rss(data, {stylesheet: '...'})\` to choose a unique, custom URL. (${route.component})`
-						);
-					}
-					assets[fileURLToPath(stylesheetFile)] = content;
-				}
-			}
-			const finalPaths = result.staticPaths.map((staticPath) => staticPath.params && route.generate(staticPath.params)).filter(Boolean);
 			allPages[route.component] = {
 				route,
-				paths: finalPaths,
+				paths: [route.pathname],
 				preload: await ssrPreload({
 					astroConfig,
 					filePath: new URL(`./${route.component}`, astroConfig.projectRoot),
 					logging,
 					mode: 'production',
 					origin,
-					pathname: finalPaths[0],
+					pathname: route.pathname,
 					route,
 					routeCache,
 					viteServer,
-				}),
+				})
+					.then((routes) => {
+						const html = `${route.pathname}`.replace(/\/?$/, '/index.html');
+						debug('build', `├── ${colors.bold(colors.green('✔'))} ${route.component} → ${colors.yellow(html)}`);
+						return routes;
+					})
+					.catch((err) => {
+						debug('build', `├── ${colors.bold(colors.red('✘'))} ${route.component}`);
+						throw err;
+					}),
 			};
-		})
+			continue;
-	);
+		}
+		// dynamic route:
+		const result = await getStaticPathsForRoute(opts, route)
+			.then((_result) => {
+				const label = _result.staticPaths.length === 1 ? 'page' : 'pages';
+				debug('build', `├── ${colors.bold(colors.green('✔'))} ${route.component} → ${colors.magenta(`[${_result.staticPaths.length} ${label}]`)}`);
+				return _result;
+			})
+			.catch((err) => {
+				debug('build', `├── ${colors.bold(colors.red('✗'))} ${route.component}`);
+				throw err;
+			});
+		const rssFn = generateRssFunction(astroConfig.buildOptions.site, route);
+		for (const rssCallArg of result.rss) {
+			const rssResult = rssFn(rssCallArg);
+			if (rssResult.xml) {
+				const { url, content } = rssResult.xml;
+				if (content) {
+					const rssFile = new URL(url.replace(/^\/?/, './'), astroConfig.dist);
+					if (assets[fileURLToPath(rssFile)]) {
+						throw new Error(`[getStaticPaths] RSS feed ${url} already exists.\nUse \`rss(data, {url: '...'})\` to choose a unique, custom URL. (${route.component})`);
+					}
+					assets[fileURLToPath(rssFile)] = content;
+				}
+			}
+			if (rssResult.xsl?.content) {
+				const { url, content } = rssResult.xsl;
+				const stylesheetFile = new URL(url.replace(/^\/?/, './'), astroConfig.dist);
+				if (assets[fileURLToPath(stylesheetFile)]) {
+					throw new Error(
+						`[getStaticPaths] RSS feed stylesheet ${url} already exists.\nUse \`rss(data, {stylesheet: '...'})\` to choose a unique, custom URL. (${route.component})`
+					);
+				}
+				assets[fileURLToPath(stylesheetFile)] = content;
+			}
+		}
+		const finalPaths = result.staticPaths.map((staticPath) => staticPath.params && route.generate(staticPath.params)).filter(Boolean);
+		allPages[route.component] = {
+			route,
+			paths: finalPaths,
+			preload: await ssrPreload({
+				astroConfig,
+				filePath: new URL(`./${route.component}`, astroConfig.projectRoot),
+				logging,
+				mode: 'production',
+				origin,
+				pathname: finalPaths[0],
+				route,
+				routeCache,
+				viteServer,
+			}),
+		};
+	}
 
 	return { assets, allPages };
 }

packages/markdown/remark/src/remark-shiki.ts

@@ -30,16 +30,25 @@ export interface ShikiConfig {
 	wrap?: boolean | null;
 }
 
-const remarkShiki = async ({ langs = [], theme = 'github-dark', wrap = false }: ShikiConfig) => {
+/**
-	const highlighter = await getHighlighter({ theme });
+ * getHighlighter() is the most expensive step of Shiki. Instead of calling it on every page,
+ * cache it here as much as possible. Make sure that your highlighters can be cached, state-free.
+ */
+const highlighterCache = new Map<string, shiki.Highlighter>();
 
+const remarkShiki = async ({ langs = [], theme = 'github-dark', wrap = false }: ShikiConfig) => {
+	const cacheID: string = typeof theme === 'string' ? theme : theme.name;
+	let highlighter = highlighterCache.get(cacheID);
+	if (!highlighter) {
+		highlighter = await getHighlighter({ theme });
+		highlighterCache.set(cacheID, highlighter);
+	}
 	for (const lang of langs) {
 		await highlighter.loadLanguage(lang);
 	}
-
 	return () => (tree: any) => {
 		visit(tree, 'code', (node) => {
-			let html = highlighter.codeToHtml(node.value, { lang: node.lang ?? 'plaintext' });
+			let html = highlighter!.codeToHtml(node.value, { lang: node.lang ?? 'plaintext' });
 
 			// Replace "shiki" class naming with "astro" and add "data-astro-raw".
 			html = html.replace('<pre class="shiki"', '<pre data-astro-raw class="astro-code"');