feat: Rework image generation to improve performance (#8821)

* feat: implement concurrency for asset generation

* add changeset

* fix: count

* feat: rework image generation to reuse image buffer for transforms of the same image

* fix: assetsPrefix nonsense

* feat: add back the counter

* refactor: cleanup my TS nonsense

* nit: reuse type

* nit: apply suggestions

* nit: macOS micro optimization

* Update .changeset/good-mirrors-bake.md

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

---------

Co-authored-by: Matteo Manfredi <matteo@manfredi.io>
Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

.changeset/good-mirrors-bake.md

@@ -0,0 +1,9 @@
+---
+'astro': minor
+---
+
+Improved image optimization performance
+
+Astro will now generate optimized images concurrently at build time, which can significantly speed up build times for sites with many images. Additionally, Astro will now reuse the same buffer for all variants of an image. This should improve performance for websites with many variants of the same image, especially when using remote images.
+
+No code changes are required to take advantage of these improvements.

.eslintrc.cjs

@@ -17,7 +17,12 @@ module.exports = {
     '@typescript-eslint/array-type': ['error', { default: 'array-simple' }],
     '@typescript-eslint/no-unused-vars': [
       'warn',
-      { argsIgnorePattern: '^_', ignoreRestSiblings: true },
+      {
+        argsIgnorePattern: '^_',
+        varsIgnorePattern: '^_',
+        caughtErrorsIgnorePattern: '^_',
+        ignoreRestSiblings: true,
+      },
     ],
     'no-only-tests/no-only-tests': 'error',
     '@typescript-eslint/no-shadow': ['error'],

packages/astro/package.json

@@ -157,6 +157,7 @@
     "mime": "^3.0.0",
     "ora": "^7.0.1",
     "p-limit": "^4.0.0",
+    "p-queue": "^7.4.1",
     "path-to-regexp": "^6.2.1",
     "preferred-pm": "^3.1.2",
     "probe-image-size": "^7.2.3",

packages/astro/src/assets/build/generate.ts

@@ -1,12 +1,18 @@
+import { dim, green } from 'kleur/colors';
 import fs, { readFileSync } from 'node:fs';
 import { basename, join } from 'node:path/posix';
+import PQueue from 'p-queue';
+import type { AstroConfig } from '../../@types/astro.js';
 import type { BuildPipeline } from '../../core/build/buildPipeline.js';
 import { getOutDirWithinCwd } from '../../core/build/common.js';
-import { prependForwardSlash } from '../../core/path.js';
+import { getTimeStat } from '../../core/build/util.js';
+import type { Logger } from '../../core/logger/core.js';
+import { isRemotePath, prependForwardSlash } from '../../core/path.js';
 import { isServerLikeOutput } from '../../prerender/utils.js';
+import type { MapValue } from '../../type-utils.js';
 import { getConfiguredImageService, isESMImportedImage } from '../internal.js';
 import type { LocalImageService } from '../services/service.js';
-import type { ImageMetadata, ImageTransform } from '../types.js';
+import type { AssetsGlobalStaticImagesList, ImageMetadata, ImageTransform } from '../types.js';
 import { loadRemoteImage, type RemoteCacheEntry } from './remote.js';
 
 interface GenerationDataUncached {
@@ -23,15 +29,28 @@ interface GenerationDataCached {
 
 type GenerationData = GenerationDataUncached | GenerationDataCached;
 
-export async function generateImage(
+type AssetEnv = {
+	logger: Logger;
+	count: { total: number; current: number };
+	useCache: boolean;
+	assetsCacheDir: URL;
+	serverRoot: URL;
+	clientRoot: URL;
+	imageConfig: AstroConfig['image'];
+	assetsFolder: AstroConfig['build']['assets'];
+};
+
+type ImageData = { data: Buffer; expires: number };
+
+export async function prepareAssetsGenerationEnv(
 	pipeline: BuildPipeline,
-	options: ImageTransform,
-	filepath: string
-): Promise<GenerationData | undefined> {
+	totalCount: number
+): Promise<AssetEnv> {
 	const config = pipeline.getConfig();
 	const logger = pipeline.getLogger();
 	let useCache = true;
 	const assetsCacheDir = new URL('assets/', config.cacheDir);
+	const count = { total: totalCount, current: 1 };
 
 	// Ensure that the cache directory exists
 	try {
@@ -53,113 +72,182 @@ export async function generateImage(
 		clientRoot = config.outDir;
 	}
 
-	const isLocalImage = isESMImportedImage(options.src);
+	return {
+		logger,
+		count,
+		useCache,
+		assetsCacheDir,
+		serverRoot,
+		clientRoot,
+		imageConfig: config.image,
+		assetsFolder: config.build.assets,
+	};
+}
 
-	const finalFileURL = new URL('.' + filepath, clientRoot);
-	const finalFolderURL = new URL('./', finalFileURL);
+export async function generateImagesForPath(
+	originalFilePath: string,
+	transforms: MapValue<AssetsGlobalStaticImagesList>,
+	env: AssetEnv,
+	queue: PQueue
+) {
+	const originalImageData = await loadImage(originalFilePath, env);
 
-	// For remote images, instead of saving the image directly, we save a JSON file with the image data and expiration date from the server
-	const cacheFile = basename(filepath) + (isLocalImage ? '' : '.json');
-	const cachedFileURL = new URL(cacheFile, assetsCacheDir);
+	for (const [_, transform] of transforms) {
+		queue.add(async () =>
+			generateImage(originalImageData, transform.finalPath, transform.transform)
+		);
+	}
 
-	await fs.promises.mkdir(finalFolderURL, { recursive: true });
+	async function generateImage(
+		originalImage: ImageData,
+		filepath: string,
+		options: ImageTransform
+	) {
+		const timeStart = performance.now();
+		const generationData = await generateImageInternal(originalImage, filepath, options);
 
-	// Check if we have a cached entry first
-	try {
-		if (isLocalImage) {
-			await fs.promises.copyFile(cachedFileURL, finalFileURL);
+		const timeEnd = performance.now();
+		const timeChange = getTimeStat(timeStart, timeEnd);
+		const timeIncrease = `(+${timeChange})`;
+		const statsText = generationData.cached
+			? `(reused cache entry)`
+			: `(before: ${generationData.weight.before}kB, after: ${generationData.weight.after}kB)`;
+		const count = `(${env.count.current}/${env.count.total})`;
+		env.logger.info(
+			null,
+			`  ${green('▶')} ${filepath} ${dim(statsText)} ${dim(timeIncrease)} ${dim(count)}`
+		);
+		env.count.current++;
+	}
 
-			return {
-				cached: true,
-			};
-		} else {
-			const JSONData = JSON.parse(readFileSync(cachedFileURL, 'utf-8')) as RemoteCacheEntry;
+	async function generateImageInternal(
+		originalImage: ImageData,
+		filepath: string,
+		options: ImageTransform
+	): Promise<GenerationData> {
+		const isLocalImage = isESMImportedImage(options.src);
+		const finalFileURL = new URL('.' + filepath, env.clientRoot);
 
-			// If the cache entry is not expired, use it
-			if (JSONData.expires > Date.now()) {
-				await fs.promises.writeFile(finalFileURL, Buffer.from(JSONData.data, 'base64'));
+		// For remote images, instead of saving the image directly, we save a JSON file with the image data and expiration date from the server
+		const cacheFile = basename(filepath) + (isLocalImage ? '' : '.json');
+		const cachedFileURL = new URL(cacheFile, env.assetsCacheDir);
+
+		// Check if we have a cached entry first
+		try {
+			if (isLocalImage) {
+				await fs.promises.copyFile(cachedFileURL, finalFileURL, fs.constants.COPYFILE_FICLONE);
 
 				return {
 					cached: true,
 				};
-			}
-		}
-	} catch (e: any) {
-		if (e.code !== 'ENOENT') {
-			throw new Error(`An error was encountered while reading the cache file. Error: ${e}`);
-		}
-		// If the cache file doesn't exist, just move on, and we'll generate it
-	}
-
-	// The original filepath or URL from the image transform
-	const originalImagePath = isLocalImage
-		? (options.src as ImageMetadata).src
-		: (options.src as string);
-
-	let imageData;
-	let resultData: { data: Buffer | undefined; expires: number | undefined } = {
-		data: undefined,
-		expires: undefined,
-	};
-
-	// If the image is local, we can just read it directly, otherwise we need to download it
-	if (isLocalImage) {
-		imageData = await fs.promises.readFile(
-			new URL(
-				'.' + prependForwardSlash(join(config.build.assets, basename(originalImagePath))),
-				serverRoot
-			)
-		);
-	} else {
-		const remoteImage = await loadRemoteImage(originalImagePath);
-		resultData.expires = remoteImage.expires;
-		imageData = remoteImage.data;
-	}
-
-	const imageService = (await getConfiguredImageService()) as LocalImageService;
-	resultData.data = (
-		await imageService.transform(imageData, { ...options, src: originalImagePath }, config.image)
-	).data;
-
-	try {
-		// Write the cache entry
-		if (useCache) {
-			if (isLocalImage) {
-				await fs.promises.writeFile(cachedFileURL, resultData.data);
 			} else {
-				await fs.promises.writeFile(
-					cachedFileURL,
-					JSON.stringify({
-						data: Buffer.from(resultData.data).toString('base64'),
-						expires: resultData.expires,
-					})
-				);
-			}
-		}
-	} catch (e) {
-		logger.warn(
-			'astro:assets',
-			`An error was encountered while creating the cache directory. Proceeding without caching. Error: ${e}`
-		);
-	} finally {
-		// Write the final file
-		await fs.promises.writeFile(finalFileURL, resultData.data);
-	}
+				const JSONData = JSON.parse(readFileSync(cachedFileURL, 'utf-8')) as RemoteCacheEntry;
 
-	return {
-		cached: false,
-		weight: {
-			// Divide by 1024 to get size in kilobytes
-			before: Math.trunc(imageData.byteLength / 1024),
-			after: Math.trunc(Buffer.from(resultData.data).byteLength / 1024),
-		},
-	};
+				if (!JSONData.data || !JSONData.expires) {
+					await fs.promises.unlink(cachedFileURL);
+
+					throw new Error(
+						`Malformed cache entry for ${filepath}, cache will be regenerated for this file.`
+					);
+				}
+
+				// If the cache entry is not expired, use it
+				if (JSONData.expires > Date.now()) {
+					await fs.promises.writeFile(finalFileURL, Buffer.from(JSONData.data, 'base64'));
+
+					return {
+						cached: true,
+					};
+				} else {
+					await fs.promises.unlink(cachedFileURL);
+				}
+			}
+		} catch (e: any) {
+			if (e.code !== 'ENOENT') {
+				throw new Error(`An error was encountered while reading the cache file. Error: ${e}`);
+			}
+			// If the cache file doesn't exist, just move on, and we'll generate it
+		}
+
+		const finalFolderURL = new URL('./', finalFileURL);
+		await fs.promises.mkdir(finalFolderURL, { recursive: true });
+
+		// The original filepath or URL from the image transform
+		const originalImagePath = isLocalImage
+			? (options.src as ImageMetadata).src
+			: (options.src as string);
+
+		let resultData: Partial<ImageData> = {
+			data: undefined,
+			expires: originalImage.expires,
+		};
+
+		const imageService = (await getConfiguredImageService()) as LocalImageService;
+		resultData.data = (
+			await imageService.transform(
+				originalImage.data,
+				{ ...options, src: originalImagePath },
+				env.imageConfig
+			)
+		).data;
+
+		try {
+			// Write the cache entry
+			if (env.useCache) {
+				if (isLocalImage) {
+					await fs.promises.writeFile(cachedFileURL, resultData.data);
+				} else {
+					await fs.promises.writeFile(
+						cachedFileURL,
+						JSON.stringify({
+							data: Buffer.from(resultData.data).toString('base64'),
+							expires: resultData.expires,
+						})
+					);
+				}
+			}
+		} catch (e) {
+			env.logger.warn(
+				'astro:assets',
+				`An error was encountered while creating the cache directory. Proceeding without caching. Error: ${e}`
+			);
+		} finally {
+			// Write the final file
+			await fs.promises.writeFile(finalFileURL, resultData.data);
+		}
+
+		return {
+			cached: false,
+			weight: {
+				// Divide by 1024 to get size in kilobytes
+				before: Math.trunc(originalImage.data.byteLength / 1024),
+				after: Math.trunc(Buffer.from(resultData.data).byteLength / 1024),
+			},
+		};
+	}
 }
 
-export function getStaticImageList(): Map<string, { path: string; options: ImageTransform }> {
+export function getStaticImageList(): AssetsGlobalStaticImagesList {
 	if (!globalThis?.astroAsset?.staticImages) {
 		return new Map();
 	}
 
 	return globalThis.astroAsset.staticImages;
 }
+
+async function loadImage(path: string, env: AssetEnv): Promise<ImageData> {
+	if (isRemotePath(path)) {
+		const remoteImage = await loadRemoteImage(path);
+		return {
+			data: remoteImage.data,
+			expires: remoteImage.expires,
+		};
+	}
+
+	return {
+		data: await fs.promises.readFile(
+			new URL('.' + prependForwardSlash(join(env.assetsFolder, basename(path))), env.serverRoot)
+		),
+		expires: 0,
+	};
+}

packages/astro/src/assets/types.ts

@@ -8,12 +8,17 @@ export type ImageQuality = ImageQualityPreset | number;
 export type ImageInputFormat = (typeof VALID_INPUT_FORMATS)[number];
 export type ImageOutputFormat = (typeof VALID_OUTPUT_FORMATS)[number] | (string & {});
 
+export type AssetsGlobalStaticImagesList = Map<
+	string,
+	Map<string, { finalPath: string; transform: ImageTransform }>
+>;
+
 declare global {
 	// eslint-disable-next-line no-var
 	var astroAsset: {
 		imageService?: ImageService;
 		addStaticImage?: ((options: ImageTransform) => string) | undefined;
-		staticImages?: Map<string, { path: string; options: ImageTransform }>;
+		staticImages?: AssetsGlobalStaticImagesList;
 	};
 }
 

packages/astro/src/assets/vite-plugin-assets.ts

@@ -12,6 +12,7 @@ import {
 } from '../core/path.js';
 import { isServerLikeOutput } from '../prerender/utils.js';
 import { VALID_INPUT_FORMATS, VIRTUAL_MODULE_ID, VIRTUAL_SERVICE_ID } from './consts.js';
+import { isESMImportedImage } from './internal.js';
 import { emitESMImage } from './utils/emitAsset.js';
 import { hashTransform, propsToFilename } from './utils/transformToPath.js';
 
@@ -80,27 +81,37 @@ export default function assets({
 					if (!globalThis.astroAsset.staticImages) {
 						globalThis.astroAsset.staticImages = new Map<
 							string,
-							{ path: string; options: ImageTransform }
+							Map<string, { finalPath: string; transform: ImageTransform }>
 						>();
 					}
 
+					const originalImagePath = (
+						isESMImportedImage(options.src) ? options.src.src : options.src
+					).replace(settings.config.build.assetsPrefix || '', '');
 					const hash = hashTransform(options, settings.config.image.service.entrypoint);
 
-					let filePath: string;
-					if (globalThis.astroAsset.staticImages.has(hash)) {
-						filePath = globalThis.astroAsset.staticImages.get(hash)!.path;
+					let finalFilePath: string;
+					let transformsForPath = globalThis.astroAsset.staticImages.get(originalImagePath);
+					let transformForHash = transformsForPath?.get(hash);
+					if (transformsForPath && transformForHash) {
+						finalFilePath = transformForHash.finalPath;
 					} else {
-						filePath = prependForwardSlash(
+						finalFilePath = prependForwardSlash(
 							joinPaths(settings.config.build.assets, propsToFilename(options, hash))
 						);
 
-						globalThis.astroAsset.staticImages.set(hash, { path: filePath, options: options });
+						if (!transformsForPath) {
+							globalThis.astroAsset.staticImages.set(originalImagePath, new Map());
+							transformsForPath = globalThis.astroAsset.staticImages.get(originalImagePath)!;
+						}
+
+						transformsForPath.set(hash, { finalPath: finalFilePath, transform: options });
 					}
 
 					if (settings.config.build.assetsPrefix) {
-						return joinPaths(settings.config.build.assetsPrefix, filePath);
+						return joinPaths(settings.config.build.assetsPrefix, finalFilePath);
 					} else {
-						return prependForwardSlash(joinPaths(settings.config.base, filePath));
+						return prependForwardSlash(joinPaths(settings.config.base, finalFilePath));
 					}
 				};
 			},

packages/astro/src/core/build/generate.ts

@@ -1,7 +1,9 @@
 import * as colors from 'kleur/colors';
 import { bgGreen, black, cyan, dim, green, magenta } from 'kleur/colors';
 import fs from 'node:fs';
+import os from 'node:os';
 import { fileURLToPath } from 'node:url';
+import PQueue from 'p-queue';
 import type { OutputAsset, OutputChunk } from 'rollup';
 import type { BufferEncoding } from 'vfile';
 import type {
@@ -9,7 +11,6 @@ import type {
 	AstroSettings,
 	ComponentInstance,
 	GetStaticPathsItem,
-	ImageTransform,
 	MiddlewareEndpointHandler,
 	RouteData,
 	RouteType,
@@ -18,8 +19,9 @@ import type {
 	SSRManifest,
 } from '../../@types/astro.js';
 import {
-	generateImage as generateImageInternal,
+	generateImagesForPath,
 	getStaticImageList,
+	prepareAssetsGenerationEnv,
 } from '../../assets/build/generate.js';
 import { hasPrerenderedPages, type BuildInternals } from '../../core/build/internal.js';
 import {
@@ -196,58 +198,35 @@ export async function generatePages(opts: StaticBuildOptions, internals: BuildIn
 		}
 	}
 
+	logger.info(null, dim(`Completed in ${getTimeStat(timer, performance.now())}.\n`));
+
 	const staticImageList = getStaticImageList();
-
-	if (staticImageList.size)
+	if (staticImageList.size) {
 		logger.info(null, `\n${bgGreen(black(` generating optimized images `))}`);
-	let count = 0;
-	for (const imageData of staticImageList.entries()) {
-		count++;
-		await generateImage(
-			pipeline,
-			imageData[1].options,
-			imageData[1].path,
-			count,
-			staticImageList.size
-		);
-	}
 
-	delete globalThis?.astroAsset?.addStaticImage;
+		const totalCount = Array.from(staticImageList.values())
+			.map((x) => x.size)
+			.reduce((a, b) => a + b, 0);
+		const cpuCount = os.cpus().length;
+		const assetsCreationEnvironment = await prepareAssetsGenerationEnv(pipeline, totalCount);
+		const queue = new PQueue({ concurrency: cpuCount });
+
+		const assetsTimer = performance.now();
+		for (const [originalPath, transforms] of staticImageList) {
+			await generateImagesForPath(originalPath, transforms, assetsCreationEnvironment, queue);
+		}
+
+		await queue.onIdle();
+		const assetsTimeEnd = performance.now();
+		logger.info(null, dim(`Completed in ${getTimeStat(assetsTimer, assetsTimeEnd)}.\n`));
+
+		delete globalThis?.astroAsset?.addStaticImage;
+	}
 
 	await runHookBuildGenerated({
 		config: opts.settings.config,
 		logger: pipeline.getLogger(),
 	});
-
-	logger.info(null, dim(`Completed in ${getTimeStat(timer, performance.now())}.\n`));
-}
-
-async function generateImage(
-	pipeline: BuildPipeline,
-	transform: ImageTransform,
-	path: string,
-	count: number,
-	totalCount: number
-) {
-	const logger = pipeline.getLogger();
-	let timeStart = performance.now();
-	const generationData = await generateImageInternal(pipeline, transform, path);
-
-	if (!generationData) {
-		return;
-	}
-
-	const timeEnd = performance.now();
-	const timeChange = getTimeStat(timeStart, timeEnd);
-	const timeIncrease = `(+${timeChange})`;
-	const statsText = generationData.cached
-		? `(reused cache entry)`
-		: `(before: ${generationData.weight.before}kB, after: ${generationData.weight.after}kB)`;
-	const counter = `(${count}/${totalCount})`;
-	logger.info(
-		null,
-		`  ${green('▶')} ${path} ${dim(statsText)} ${dim(timeIncrease)} ${dim(counter)}`
-	);
 }
 
 async function generatePage(

packages/astro/src/type-utils.ts

@@ -27,3 +27,6 @@ export type KebabKeys<T> = { [K in keyof T as K extends string ? Kebab<K> : K]:
 
 // Similar to `keyof`, gets the type of all the values of an object
 export type ValueOf<T> = T[keyof T];
+
+// Gets the type of the values of a Map
+export type MapValue<T> = T extends Map<any, infer V> ? V : never;

pnpm-lock.yaml

@@ -595,6 +595,9 @@ importers:
       p-limit:
         specifier: ^4.0.0
         version: 4.0.0
+      p-queue:
+        specifier: ^7.4.1
+        version: 7.4.1
       path-to-regexp:
         specifier: ^6.2.1
         version: 6.2.1
@@ -10993,6 +10996,10 @@ packages:
     resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==}
     engines: {node: '>= 0.6'}
 
+  /eventemitter3@5.0.1:
+    resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==}
+    dev: false
+
   /execa@5.1.1:
     resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==}
     engines: {node: '>=10'}
@@ -14125,6 +14132,19 @@ packages:
       aggregate-error: 4.0.1
     dev: true
 
+  /p-queue@7.4.1:
+    resolution: {integrity: sha512-vRpMXmIkYF2/1hLBKisKeVYJZ8S2tZ0zEAmIJgdVKP2nq0nh4qCdf8bgw+ZgKrkh71AOCaqzwbJJk1WtdcF3VA==}
+    engines: {node: '>=12'}
+    dependencies:
+      eventemitter3: 5.0.1
+      p-timeout: 5.1.0
+    dev: false
+
+  /p-timeout@5.1.0:
+    resolution: {integrity: sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==}
+    engines: {node: '>=12'}
+    dev: false
+
   /p-try@2.2.0:
     resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==}
     engines: {node: '>=6'}