feat: Rework image generation to improve performance (#8821)

* feat: implement concurrency for asset generation

* add changeset

* fix: count

* feat: rework image generation to reuse image buffer for transforms of the same image

* fix: assetsPrefix nonsense

* feat: add back the counter

* refactor: cleanup my TS nonsense

* nit: reuse type

* nit: apply suggestions

* nit: macOS micro optimization

* Update .changeset/good-mirrors-bake.md

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

---------

Co-authored-by: Matteo Manfredi <matteo@manfredi.io>
Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

.changeset/good-mirrors-bake.md

@@ -0,0 +1,9 @@
+---
+'astro': minor
+---
+
+Improved image optimization performance
+
+Astro will now generate optimized images concurrently at build time, which can significantly speed up build times for sites with many images. Additionally, Astro will now reuse the same buffer for all variants of an image. This should improve performance for websites with many variants of the same image, especially when using remote images.
+
+No code changes are required to take advantage of these improvements.

.eslintrc.cjs

@@ -17,7 +17,12 @@ module.exports = {
     '@typescript-eslint/array-type': ['error', { default: 'array-simple' }],
     '@typescript-eslint/no-unused-vars': [
       'warn',
-      { argsIgnorePattern: '^_', ignoreRestSiblings: true },
+      {
+        argsIgnorePattern: '^_',
+        varsIgnorePattern: '^_',
+        caughtErrorsIgnorePattern: '^_',
+        ignoreRestSiblings: true,
+      },
     ],
     'no-only-tests/no-only-tests': 'error',
     '@typescript-eslint/no-shadow': ['error'],

packages/astro/package.json

@@ -157,6 +157,7 @@
     "mime": "^3.0.0",
     "ora": "^7.0.1",
     "p-limit": "^4.0.0",
+    "p-queue": "^7.4.1",
     "path-to-regexp": "^6.2.1",
     "preferred-pm": "^3.1.2",
     "probe-image-size": "^7.2.3",

packages/astro/src/assets/build/generate.ts

@@ -1,12 +1,18 @@
+import { dim, green } from 'kleur/colors';
 import fs, { readFileSync } from 'node:fs';
 import { basename, join } from 'node:path/posix';
+import PQueue from 'p-queue';
+import type { AstroConfig } from '../../@types/astro.js';
 import type { BuildPipeline } from '../../core/build/buildPipeline.js';
 import { getOutDirWithinCwd } from '../../core/build/common.js';
-import { prependForwardSlash } from '../../core/path.js';
+import { getTimeStat } from '../../core/build/util.js';
+import type { Logger } from '../../core/logger/core.js';
+import { isRemotePath, prependForwardSlash } from '../../core/path.js';
 import { isServerLikeOutput } from '../../prerender/utils.js';
+import type { MapValue } from '../../type-utils.js';
 import { getConfiguredImageService, isESMImportedImage } from '../internal.js';
 import type { LocalImageService } from '../services/service.js';
-import type { ImageMetadata, ImageTransform } from '../types.js';
+import type { AssetsGlobalStaticImagesList, ImageMetadata, ImageTransform } from '../types.js';
 import { loadRemoteImage, type RemoteCacheEntry } from './remote.js';
 
 interface GenerationDataUncached {
@@ -23,15 +29,28 @@ interface GenerationDataCached {
 
 type GenerationData = GenerationDataUncached | GenerationDataCached;
 
-export async function generateImage(
+type AssetEnv = {
+	logger: Logger;
+	count: { total: number; current: number };
+	useCache: boolean;
+	assetsCacheDir: URL;
+	serverRoot: URL;
+	clientRoot: URL;
+	imageConfig: AstroConfig['image'];
+	assetsFolder: AstroConfig['build']['assets'];
+};
+
+type ImageData = { data: Buffer; expires: number };
+
+export async function prepareAssetsGenerationEnv(
 	pipeline: BuildPipeline,
-	options: ImageTransform,
+	totalCount: number
-	filepath: string
+): Promise<AssetEnv> {
-): Promise<GenerationData | undefined> {
 	const config = pipeline.getConfig();
 	const logger = pipeline.getLogger();
 	let useCache = true;
 	const assetsCacheDir = new URL('assets/', config.cacheDir);
+	const count = { total: totalCount, current: 1 };
 
 	// Ensure that the cache directory exists
 	try {
@@ -53,21 +72,70 @@ export async function generateImage(
 		clientRoot = config.outDir;
 	}
 
-	const isLocalImage = isESMImportedImage(options.src);
+	return {
+		logger,
+		count,
+		useCache,
+		assetsCacheDir,
+		serverRoot,
+		clientRoot,
+		imageConfig: config.image,
+		assetsFolder: config.build.assets,
+	};
+}
 
-	const finalFileURL = new URL('.' + filepath, clientRoot);
+export async function generateImagesForPath(
-	const finalFolderURL = new URL('./', finalFileURL);
+	originalFilePath: string,
+	transforms: MapValue<AssetsGlobalStaticImagesList>,
+	env: AssetEnv,
+	queue: PQueue
+) {
+	const originalImageData = await loadImage(originalFilePath, env);
+
+	for (const [_, transform] of transforms) {
+		queue.add(async () =>
+			generateImage(originalImageData, transform.finalPath, transform.transform)
+		);
+	}
+
+	async function generateImage(
+		originalImage: ImageData,
+		filepath: string,
+		options: ImageTransform
+	) {
+		const timeStart = performance.now();
+		const generationData = await generateImageInternal(originalImage, filepath, options);
+
+		const timeEnd = performance.now();
+		const timeChange = getTimeStat(timeStart, timeEnd);
+		const timeIncrease = `(+${timeChange})`;
+		const statsText = generationData.cached
+			? `(reused cache entry)`
+			: `(before: ${generationData.weight.before}kB, after: ${generationData.weight.after}kB)`;
+		const count = `(${env.count.current}/${env.count.total})`;
+		env.logger.info(
+			null,
+			`  ${green('▶')} ${filepath} ${dim(statsText)} ${dim(timeIncrease)} ${dim(count)}`
+		);
+		env.count.current++;
+	}
+
+	async function generateImageInternal(
+		originalImage: ImageData,
+		filepath: string,
+		options: ImageTransform
+	): Promise<GenerationData> {
+		const isLocalImage = isESMImportedImage(options.src);
+		const finalFileURL = new URL('.' + filepath, env.clientRoot);
 
 		// For remote images, instead of saving the image directly, we save a JSON file with the image data and expiration date from the server
 		const cacheFile = basename(filepath) + (isLocalImage ? '' : '.json');
-	const cachedFileURL = new URL(cacheFile, assetsCacheDir);
+		const cachedFileURL = new URL(cacheFile, env.assetsCacheDir);
-
-	await fs.promises.mkdir(finalFolderURL, { recursive: true });
 
 		// Check if we have a cached entry first
 		try {
 			if (isLocalImage) {
-			await fs.promises.copyFile(cachedFileURL, finalFileURL);
+				await fs.promises.copyFile(cachedFileURL, finalFileURL, fs.constants.COPYFILE_FICLONE);
 
 				return {
 					cached: true,
@@ -75,6 +143,14 @@ export async function generateImage(
 			} else {
 				const JSONData = JSON.parse(readFileSync(cachedFileURL, 'utf-8')) as RemoteCacheEntry;
 
+				if (!JSONData.data || !JSONData.expires) {
+					await fs.promises.unlink(cachedFileURL);
+
+					throw new Error(
+						`Malformed cache entry for ${filepath}, cache will be regenerated for this file.`
+					);
+				}
+
 				// If the cache entry is not expired, use it
 				if (JSONData.expires > Date.now()) {
 					await fs.promises.writeFile(finalFileURL, Buffer.from(JSONData.data, 'base64'));
@@ -82,6 +158,8 @@ export async function generateImage(
 					return {
 						cached: true,
 					};
+				} else {
+					await fs.promises.unlink(cachedFileURL);
 				}
 			}
 		} catch (e: any) {
@@ -91,39 +169,31 @@ export async function generateImage(
 			// If the cache file doesn't exist, just move on, and we'll generate it
 		}
 
+		const finalFolderURL = new URL('./', finalFileURL);
+		await fs.promises.mkdir(finalFolderURL, { recursive: true });
+
 		// The original filepath or URL from the image transform
 		const originalImagePath = isLocalImage
 			? (options.src as ImageMetadata).src
 			: (options.src as string);
 
-	let imageData;
+		let resultData: Partial<ImageData> = {
-	let resultData: { data: Buffer | undefined; expires: number | undefined } = {
 			data: undefined,
-		expires: undefined,
+			expires: originalImage.expires,
 		};
 
-	// If the image is local, we can just read it directly, otherwise we need to download it
-	if (isLocalImage) {
-		imageData = await fs.promises.readFile(
-			new URL(
-				'.' + prependForwardSlash(join(config.build.assets, basename(originalImagePath))),
-				serverRoot
-			)
-		);
-	} else {
-		const remoteImage = await loadRemoteImage(originalImagePath);
-		resultData.expires = remoteImage.expires;
-		imageData = remoteImage.data;
-	}
-
 		const imageService = (await getConfiguredImageService()) as LocalImageService;
 		resultData.data = (
-		await imageService.transform(imageData, { ...options, src: originalImagePath }, config.image)
+			await imageService.transform(
+				originalImage.data,
+				{ ...options, src: originalImagePath },
+				env.imageConfig
+			)
 		).data;
 
 		try {
 			// Write the cache entry
-		if (useCache) {
+			if (env.useCache) {
 				if (isLocalImage) {
 					await fs.promises.writeFile(cachedFileURL, resultData.data);
 				} else {
@@ -137,7 +207,7 @@ export async function generateImage(
 				}
 			}
 		} catch (e) {
-		logger.warn(
+			env.logger.warn(
 				'astro:assets',
 				`An error was encountered while creating the cache directory. Proceeding without caching. Error: ${e}`
 			);
@@ -150,16 +220,34 @@ export async function generateImage(
 			cached: false,
 			weight: {
 				// Divide by 1024 to get size in kilobytes
-			before: Math.trunc(imageData.byteLength / 1024),
+				before: Math.trunc(originalImage.data.byteLength / 1024),
 				after: Math.trunc(Buffer.from(resultData.data).byteLength / 1024),
 			},
 		};
+	}
 }
 
-export function getStaticImageList(): Map<string, { path: string; options: ImageTransform }> {
+export function getStaticImageList(): AssetsGlobalStaticImagesList {
 	if (!globalThis?.astroAsset?.staticImages) {
 		return new Map();
 	}
 
 	return globalThis.astroAsset.staticImages;
 }
+
+async function loadImage(path: string, env: AssetEnv): Promise<ImageData> {
+	if (isRemotePath(path)) {
+		const remoteImage = await loadRemoteImage(path);
+		return {
+			data: remoteImage.data,
+			expires: remoteImage.expires,
+		};
+	}
+
+	return {
+		data: await fs.promises.readFile(
+			new URL('.' + prependForwardSlash(join(env.assetsFolder, basename(path))), env.serverRoot)
+		),
+		expires: 0,
+	};
+}

packages/astro/src/assets/types.ts

@@ -8,12 +8,17 @@ export type ImageQuality = ImageQualityPreset | number;
 export type ImageInputFormat = (typeof VALID_INPUT_FORMATS)[number];
 export type ImageOutputFormat = (typeof VALID_OUTPUT_FORMATS)[number] | (string & {});
 
+export type AssetsGlobalStaticImagesList = Map<
+	string,
+	Map<string, { finalPath: string; transform: ImageTransform }>
+>;
+
 declare global {
 	// eslint-disable-next-line no-var
 	var astroAsset: {
 		imageService?: ImageService;
 		addStaticImage?: ((options: ImageTransform) => string) | undefined;
-		staticImages?: Map<string, { path: string; options: ImageTransform }>;
+		staticImages?: AssetsGlobalStaticImagesList;
 	};
 }
 

packages/astro/src/assets/vite-plugin-assets.ts

@@ -12,6 +12,7 @@ import {
 } from '../core/path.js';
 import { isServerLikeOutput } from '../prerender/utils.js';
 import { VALID_INPUT_FORMATS, VIRTUAL_MODULE_ID, VIRTUAL_SERVICE_ID } from './consts.js';
+import { isESMImportedImage } from './internal.js';
 import { emitESMImage } from './utils/emitAsset.js';
 import { hashTransform, propsToFilename } from './utils/transformToPath.js';
 
@@ -80,27 +81,37 @@ export default function assets({
 					if (!globalThis.astroAsset.staticImages) {
 						globalThis.astroAsset.staticImages = new Map<
 							string,
-							{ path: string; options: ImageTransform }
+							Map<string, { finalPath: string; transform: ImageTransform }>
 						>();
 					}
 
+					const originalImagePath = (
+						isESMImportedImage(options.src) ? options.src.src : options.src
+					).replace(settings.config.build.assetsPrefix || '', '');
 					const hash = hashTransform(options, settings.config.image.service.entrypoint);
 
-					let filePath: string;
+					let finalFilePath: string;
-					if (globalThis.astroAsset.staticImages.has(hash)) {
+					let transformsForPath = globalThis.astroAsset.staticImages.get(originalImagePath);
-						filePath = globalThis.astroAsset.staticImages.get(hash)!.path;
+					let transformForHash = transformsForPath?.get(hash);
+					if (transformsForPath && transformForHash) {
+						finalFilePath = transformForHash.finalPath;
 					} else {
-						filePath = prependForwardSlash(
+						finalFilePath = prependForwardSlash(
 							joinPaths(settings.config.build.assets, propsToFilename(options, hash))
 						);
 
-						globalThis.astroAsset.staticImages.set(hash, { path: filePath, options: options });
+						if (!transformsForPath) {
+							globalThis.astroAsset.staticImages.set(originalImagePath, new Map());
+							transformsForPath = globalThis.astroAsset.staticImages.get(originalImagePath)!;
+						}
+
+						transformsForPath.set(hash, { finalPath: finalFilePath, transform: options });
 					}
 
 					if (settings.config.build.assetsPrefix) {
-						return joinPaths(settings.config.build.assetsPrefix, filePath);
+						return joinPaths(settings.config.build.assetsPrefix, finalFilePath);
 					} else {
-						return prependForwardSlash(joinPaths(settings.config.base, filePath));
+						return prependForwardSlash(joinPaths(settings.config.base, finalFilePath));
 					}
 				};
 			},

packages/astro/src/core/build/generate.ts

@@ -1,7 +1,9 @@
 import * as colors from 'kleur/colors';
 import { bgGreen, black, cyan, dim, green, magenta } from 'kleur/colors';
 import fs from 'node:fs';
+import os from 'node:os';
 import { fileURLToPath } from 'node:url';
+import PQueue from 'p-queue';
 import type { OutputAsset, OutputChunk } from 'rollup';
 import type { BufferEncoding } from 'vfile';
 import type {
@@ -9,7 +11,6 @@ import type {
 	AstroSettings,
 	ComponentInstance,
 	GetStaticPathsItem,
-	ImageTransform,
 	MiddlewareEndpointHandler,
 	RouteData,
 	RouteType,
@@ -18,8 +19,9 @@ import type {
 	SSRManifest,
 } from '../../@types/astro.js';
 import {
-	generateImage as generateImageInternal,
+	generateImagesForPath,
 	getStaticImageList,
+	prepareAssetsGenerationEnv,
 } from '../../assets/build/generate.js';
 import { hasPrerenderedPages, type BuildInternals } from '../../core/build/internal.js';
 import {
@@ -196,58 +198,35 @@ export async function generatePages(opts: StaticBuildOptions, internals: BuildIn
 		}
 	}
 
-	const staticImageList = getStaticImageList();
+	logger.info(null, dim(`Completed in ${getTimeStat(timer, performance.now())}.\n`));
 
-	if (staticImageList.size)
+	const staticImageList = getStaticImageList();
+	if (staticImageList.size) {
 		logger.info(null, `\n${bgGreen(black(` generating optimized images `))}`);
-	let count = 0;
+
-	for (const imageData of staticImageList.entries()) {
+		const totalCount = Array.from(staticImageList.values())
-		count++;
+			.map((x) => x.size)
-		await generateImage(
+			.reduce((a, b) => a + b, 0);
-			pipeline,
+		const cpuCount = os.cpus().length;
-			imageData[1].options,
+		const assetsCreationEnvironment = await prepareAssetsGenerationEnv(pipeline, totalCount);
-			imageData[1].path,
+		const queue = new PQueue({ concurrency: cpuCount });
-			count,
+
-			staticImageList.size
+		const assetsTimer = performance.now();
-		);
+		for (const [originalPath, transforms] of staticImageList) {
+			await generateImagesForPath(originalPath, transforms, assetsCreationEnvironment, queue);
 		}
 
+		await queue.onIdle();
+		const assetsTimeEnd = performance.now();
+		logger.info(null, dim(`Completed in ${getTimeStat(assetsTimer, assetsTimeEnd)}.\n`));
+
 		delete globalThis?.astroAsset?.addStaticImage;
+	}
 
 	await runHookBuildGenerated({
 		config: opts.settings.config,
 		logger: pipeline.getLogger(),
 	});
-
-	logger.info(null, dim(`Completed in ${getTimeStat(timer, performance.now())}.\n`));
-}
-
-async function generateImage(
-	pipeline: BuildPipeline,
-	transform: ImageTransform,
-	path: string,
-	count: number,
-	totalCount: number
-) {
-	const logger = pipeline.getLogger();
-	let timeStart = performance.now();
-	const generationData = await generateImageInternal(pipeline, transform, path);
-
-	if (!generationData) {
-		return;
-	}
-
-	const timeEnd = performance.now();
-	const timeChange = getTimeStat(timeStart, timeEnd);
-	const timeIncrease = `(+${timeChange})`;
-	const statsText = generationData.cached
-		? `(reused cache entry)`
-		: `(before: ${generationData.weight.before}kB, after: ${generationData.weight.after}kB)`;
-	const counter = `(${count}/${totalCount})`;
-	logger.info(
-		null,
-		`  ${green('▶')} ${path} ${dim(statsText)} ${dim(timeIncrease)} ${dim(counter)}`
-	);
 }
 
 async function generatePage(

packages/astro/src/type-utils.ts

@@ -27,3 +27,6 @@ export type KebabKeys<T> = { [K in keyof T as K extends string ? Kebab<K> : K]:
 
 // Similar to `keyof`, gets the type of all the values of an object
 export type ValueOf<T> = T[keyof T];
+
+// Gets the type of the values of a Map
+export type MapValue<T> = T extends Map<any, infer V> ? V : never;

pnpm-lock.yaml

@@ -595,6 +595,9 @@ importers:
       p-limit:
         specifier: ^4.0.0
         version: 4.0.0
+      p-queue:
+        specifier: ^7.4.1
+        version: 7.4.1
       path-to-regexp:
         specifier: ^6.2.1
         version: 6.2.1
@@ -10993,6 +10996,10 @@ packages:
     resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==}
     engines: {node: '>= 0.6'}
 
+  /eventemitter3@5.0.1:
+    resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==}
+    dev: false
+
   /execa@5.1.1:
     resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==}
     engines: {node: '>=10'}
@@ -14125,6 +14132,19 @@ packages:
       aggregate-error: 4.0.1
     dev: true
 
+  /p-queue@7.4.1:
+    resolution: {integrity: sha512-vRpMXmIkYF2/1hLBKisKeVYJZ8S2tZ0zEAmIJgdVKP2nq0nh4qCdf8bgw+ZgKrkh71AOCaqzwbJJk1WtdcF3VA==}
+    engines: {node: '>=12'}
+    dependencies:
+      eventemitter3: 5.0.1
+      p-timeout: 5.1.0
+    dev: false
+
+  /p-timeout@5.1.0:
+    resolution: {integrity: sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==}
+    engines: {node: '>=12'}
+    dev: false
+
   /p-try@2.2.0:
     resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==}
     engines: {node: '>=6'}