Infrastructure/astro
0
Fork 0
mirror of https://github.com/withastro/astro.git synced 2025-01-13 22:11:20 -05:00
Code Issues Activity

feat: Rework image generation to improve performance (#8821)

Browse source 
* feat: implement concurrency for asset generation

* add changeset

* fix: count

* feat: rework image generation to reuse image buffer for transforms of the same image

* fix: assetsPrefix nonsense

* feat: add back the counter

* refactor: cleanup my TS nonsense

* nit: reuse type

* nit: apply suggestions

* nit: macOS micro optimization

* Update .changeset/good-mirrors-bake.md

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

---------

Co-authored-by: Matteo Manfredi <matteo@manfredi.io>
Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>
This commit is contained in:
Erika 2023-10-25 14:21:38 +02:00 • committed by GitHub
parent 6a991012c3
commit 4740d761ae
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 273 additions and 152 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.changeset/good-mirrors-bake.md Normal file
View file

@ -0,0 +1,9 @@
---
'astro': minor
---
Improved image optimization performance
Astro will now generate optimized images concurrently at build time, which can significantly speed up build times for sites with many images. Additionally, Astro will now reuse the same buffer for all variants of an image. This should improve performance for websites with many variants of the same image, especially when using remote images.
No code changes are required to take advantage of these improvements.

7
.eslintrc.cjs
View file

@ -17,7 +17,12 @@ module.exports = {
'@typescript-eslint/array-type': ['error', { default: 'array-simple' }],
'@typescript-eslint/no-unused-vars': [
'warn',
{ argsIgnorePattern: '^_', ignoreRestSiblings: true },
{
argsIgnorePattern: '^_',
varsIgnorePattern: '^_',
caughtErrorsIgnorePattern: '^_',
ignoreRestSiblings: true,
},
],
'no-only-tests/no-only-tests': 'error',
'@typescript-eslint/no-shadow': ['error'],

1
packages/astro/package.json
View file

@ -157,6 +157,7 @@
"mime": "^3.0.0",
"ora": "^7.0.1",
"p-limit": "^4.0.0",
"p-queue": "^7.4.1",
"path-to-regexp": "^6.2.1",
"preferred-pm": "^3.1.2",
"probe-image-size": "^7.2.3",

158
packages/astro/src/assets/build/generate.ts
View file

@ -1,12 +1,18 @@
import { dim, green } from 'kleur/colors';
import fs, { readFileSync } from 'node:fs';
import { basename, join } from 'node:path/posix';
import PQueue from 'p-queue';
import type { AstroConfig } from '../../@types/astro.js';
import type { BuildPipeline } from '../../core/build/buildPipeline.js';
import { getOutDirWithinCwd } from '../../core/build/common.js';
import { prependForwardSlash } from '../../core/path.js';
import { getTimeStat } from '../../core/build/util.js';
import type { Logger } from '../../core/logger/core.js';
import { isRemotePath, prependForwardSlash } from '../../core/path.js';
import { isServerLikeOutput } from '../../prerender/utils.js';
import type { MapValue } from '../../type-utils.js';
import { getConfiguredImageService, isESMImportedImage } from '../internal.js';
import type { LocalImageService } from '../services/service.js';
import type { ImageMetadata, ImageTransform } from '../types.js';
import type { AssetsGlobalStaticImagesList, ImageMetadata, ImageTransform } from '../types.js';
import { loadRemoteImage, type RemoteCacheEntry } from './remote.js';
interface GenerationDataUncached {
@ -23,15 +29,28 @@ interface GenerationDataCached {
type GenerationData = GenerationDataUncached | GenerationDataCached;
export async function generateImage(
type AssetEnv = {
logger: Logger;
count: { total: number; current: number };
useCache: boolean;
assetsCacheDir: URL;
serverRoot: URL;
clientRoot: URL;
imageConfig: AstroConfig['image'];
assetsFolder: AstroConfig['build']['assets'];
};
type ImageData = { data: Buffer; expires: number };
export async function prepareAssetsGenerationEnv(
pipeline: BuildPipeline,
options: ImageTransform,
filepath: string
): Promise<GenerationData | undefined> {
totalCount: number
): Promise<AssetEnv> {
const config = pipeline.getConfig();
const logger = pipeline.getLogger();
let useCache = true;
const assetsCacheDir = new URL('assets/', config.cacheDir);
const count = { total: totalCount, current: 1 };
// Ensure that the cache directory exists
try {
@ -53,21 +72,70 @@ export async function generateImage(
clientRoot = config.outDir;
}
const isLocalImage = isESMImportedImage(options.src);
return {
logger,
count,
useCache,
assetsCacheDir,
serverRoot,
clientRoot,
imageConfig: config.image,
assetsFolder: config.build.assets,
};
}
const finalFileURL = new URL('.' + filepath, clientRoot);
const finalFolderURL = new URL('./', finalFileURL);
export async function generateImagesForPath(
originalFilePath: string,
transforms: MapValue<AssetsGlobalStaticImagesList>,
env: AssetEnv,
queue: PQueue
) {
const originalImageData = await loadImage(originalFilePath, env);
for (const [_, transform] of transforms) {
queue.add(async () =>
generateImage(originalImageData, transform.finalPath, transform.transform)
);
}
async function generateImage(
originalImage: ImageData,
filepath: string,
options: ImageTransform
) {
const timeStart = performance.now();
const generationData = await generateImageInternal(originalImage, filepath, options);
const timeEnd = performance.now();
const timeChange = getTimeStat(timeStart, timeEnd);
const timeIncrease = `(+${timeChange})`;
const statsText = generationData.cached
? `(reused cache entry)`
: `(before: ${generationData.weight.before}kB, after: ${generationData.weight.after}kB)`;
const count = `(${env.count.current}/${env.count.total})`;
env.logger.info(
null,
` ${green('▶')} ${filepath} ${dim(statsText)} ${dim(timeIncrease)} ${dim(count)}`
);
env.count.current++;
}
async function generateImageInternal(
originalImage: ImageData,
filepath: string,
options: ImageTransform
): Promise<GenerationData> {
const isLocalImage = isESMImportedImage(options.src);
const finalFileURL = new URL('.' + filepath, env.clientRoot);
// For remote images, instead of saving the image directly, we save a JSON file with the image data and expiration date from the server
const cacheFile = basename(filepath) + (isLocalImage ? '' : '.json');
const cachedFileURL = new URL(cacheFile, assetsCacheDir);
await fs.promises.mkdir(finalFolderURL, { recursive: true });
const cachedFileURL = new URL(cacheFile, env.assetsCacheDir);
// Check if we have a cached entry first
try {
if (isLocalImage) {
await fs.promises.copyFile(cachedFileURL, finalFileURL);
await fs.promises.copyFile(cachedFileURL, finalFileURL, fs.constants.COPYFILE_FICLONE);
return {
cached: true,
@ -75,6 +143,14 @@ export async function generateImage(
} else {
const JSONData = JSON.parse(readFileSync(cachedFileURL, 'utf-8')) as RemoteCacheEntry;
if (!JSONData.data || !JSONData.expires) {
await fs.promises.unlink(cachedFileURL);
throw new Error(
`Malformed cache entry for ${filepath}, cache will be regenerated for this file.`
);
}
// If the cache entry is not expired, use it
if (JSONData.expires > Date.now()) {
await fs.promises.writeFile(finalFileURL, Buffer.from(JSONData.data, 'base64'));
@ -82,6 +158,8 @@ export async function generateImage(
return {
cached: true,
};
} else {
await fs.promises.unlink(cachedFileURL);
}
}
} catch (e: any) {
@ -91,39 +169,31 @@ export async function generateImage(
// If the cache file doesn't exist, just move on, and we'll generate it
}
const finalFolderURL = new URL('./', finalFileURL);
await fs.promises.mkdir(finalFolderURL, { recursive: true });
// The original filepath or URL from the image transform
const originalImagePath = isLocalImage
? (options.src as ImageMetadata).src
: (options.src as string);
let imageData;
let resultData: { data: Buffer | undefined; expires: number | undefined } = {
let resultData: Partial<ImageData> = {
data: undefined,
expires: undefined,
expires: originalImage.expires,
};
// If the image is local, we can just read it directly, otherwise we need to download it
if (isLocalImage) {
imageData = await fs.promises.readFile(
new URL(
'.' + prependForwardSlash(join(config.build.assets, basename(originalImagePath))),
serverRoot
)
);
} else {
const remoteImage = await loadRemoteImage(originalImagePath);
resultData.expires = remoteImage.expires;
imageData = remoteImage.data;
}
const imageService = (await getConfiguredImageService()) as LocalImageService;
resultData.data = (
await imageService.transform(imageData, { ...options, src: originalImagePath }, config.image)
await imageService.transform(
originalImage.data,
{ ...options, src: originalImagePath },
env.imageConfig
)
).data;
try {
// Write the cache entry
if (useCache) {
if (env.useCache) {
if (isLocalImage) {
await fs.promises.writeFile(cachedFileURL, resultData.data);
} else {
@ -137,7 +207,7 @@ export async function generateImage(
}
}
} catch (e) {
logger.warn(
env.logger.warn(
'astro:assets',
`An error was encountered while creating the cache directory. Proceeding without caching. Error: ${e}`
);
@ -150,16 +220,34 @@ export async function generateImage(
cached: false,
weight: {
// Divide by 1024 to get size in kilobytes
before: Math.trunc(imageData.byteLength / 1024),
before: Math.trunc(originalImage.data.byteLength / 1024),
after: Math.trunc(Buffer.from(resultData.data).byteLength / 1024),
},
};
}
}
export function getStaticImageList(): Map<string, { path: string; options: ImageTransform }> {
export function getStaticImageList(): AssetsGlobalStaticImagesList {
if (!globalThis?.astroAsset?.staticImages) {
return new Map();
}
return globalThis.astroAsset.staticImages;
}
async function loadImage(path: string, env: AssetEnv): Promise<ImageData> {
if (isRemotePath(path)) {
const remoteImage = await loadRemoteImage(path);
return {
data: remoteImage.data,
expires: remoteImage.expires,
};
}
return {
data: await fs.promises.readFile(
new URL('.' + prependForwardSlash(join(env.assetsFolder, basename(path))), env.serverRoot)
),
expires: 0,
};
}

7
packages/astro/src/assets/types.ts
View file

@ -8,12 +8,17 @@ export type ImageQuality = ImageQualityPreset | number;
export type ImageInputFormat = (typeof VALID_INPUT_FORMATS)[number];
export type ImageOutputFormat = (typeof VALID_OUTPUT_FORMATS)[number] | (string & {});
export type AssetsGlobalStaticImagesList = Map<
string,
Map<string, { finalPath: string; transform: ImageTransform }>
>;
declare global {
// eslint-disable-next-line no-var
var astroAsset: {
imageService?: ImageService;
addStaticImage?: ((options: ImageTransform) => string) | undefined;
staticImages?: Map<string, { path: string; options: ImageTransform }>;
staticImages?: AssetsGlobalStaticImagesList;
};
}

27
packages/astro/src/assets/vite-plugin-assets.ts
View file

@ -12,6 +12,7 @@ import {
} from '../core/path.js';
import { isServerLikeOutput } from '../prerender/utils.js';
import { VALID_INPUT_FORMATS, VIRTUAL_MODULE_ID, VIRTUAL_SERVICE_ID } from './consts.js';
import { isESMImportedImage } from './internal.js';
import { emitESMImage } from './utils/emitAsset.js';
import { hashTransform, propsToFilename } from './utils/transformToPath.js';
@ -80,27 +81,37 @@ export default function assets({
if (!globalThis.astroAsset.staticImages) {
globalThis.astroAsset.staticImages = new Map<
string,
{ path: string; options: ImageTransform }
Map<string, { finalPath: string; transform: ImageTransform }>
>();
}
const originalImagePath = (
isESMImportedImage(options.src) ? options.src.src : options.src
).replace(settings.config.build.assetsPrefix || '', '');
const hash = hashTransform(options, settings.config.image.service.entrypoint);
let filePath: string;
if (globalThis.astroAsset.staticImages.has(hash)) {
filePath = globalThis.astroAsset.staticImages.get(hash)!.path;
let finalFilePath: string;
let transformsForPath = globalThis.astroAsset.staticImages.get(originalImagePath);
let transformForHash = transformsForPath?.get(hash);
if (transformsForPath && transformForHash) {
finalFilePath = transformForHash.finalPath;
} else {
filePath = prependForwardSlash(
finalFilePath = prependForwardSlash(
joinPaths(settings.config.build.assets, propsToFilename(options, hash))
);
globalThis.astroAsset.staticImages.set(hash, { path: filePath, options: options });
if (!transformsForPath) {
globalThis.astroAsset.staticImages.set(originalImagePath, new Map());
transformsForPath = globalThis.astroAsset.staticImages.get(originalImagePath)!;
}
transformsForPath.set(hash, { finalPath: finalFilePath, transform: options });
}
if (settings.config.build.assetsPrefix) {
return joinPaths(settings.config.build.assetsPrefix, filePath);
return joinPaths(settings.config.build.assetsPrefix, finalFilePath);
} else {
return prependForwardSlash(joinPaths(settings.config.base, filePath));
return prependForwardSlash(joinPaths(settings.config.base, finalFilePath));
}
};
},

67
packages/astro/src/core/build/generate.ts
View file

@ -1,7 +1,9 @@
import * as colors from 'kleur/colors';
import { bgGreen, black, cyan, dim, green, magenta } from 'kleur/colors';
import fs from 'node:fs';
import os from 'node:os';
import { fileURLToPath } from 'node:url';
import PQueue from 'p-queue';
import type { OutputAsset, OutputChunk } from 'rollup';
import type { BufferEncoding } from 'vfile';
import type {
@ -9,7 +11,6 @@ import type {
AstroSettings,
ComponentInstance,
GetStaticPathsItem,
ImageTransform,
MiddlewareEndpointHandler,
RouteData,
RouteType,
@ -18,8 +19,9 @@ import type {
SSRManifest,
} from '../../@types/astro.js';
import {
generateImage as generateImageInternal,
generateImagesForPath,
getStaticImageList,
prepareAssetsGenerationEnv,
} from '../../assets/build/generate.js';
import { hasPrerenderedPages, type BuildInternals } from '../../core/build/internal.js';
import {
@ -196,58 +198,35 @@ export async function generatePages(opts: StaticBuildOptions, internals: BuildIn
}
}
const staticImageList = getStaticImageList();
logger.info(null, dim(`Completed in ${getTimeStat(timer, performance.now())}.\n`));
if (staticImageList.size)
const staticImageList = getStaticImageList();
if (staticImageList.size) {
logger.info(null, `\n${bgGreen(black(` generating optimized images `))}`);
let count = 0;
for (const imageData of staticImageList.entries()) {
count++;
await generateImage(
pipeline,
imageData[1].options,
imageData[1].path,
count,
staticImageList.size
);
const totalCount = Array.from(staticImageList.values())
.map((x) => x.size)
.reduce((a, b) => a + b, 0);
const cpuCount = os.cpus().length;
const assetsCreationEnvironment = await prepareAssetsGenerationEnv(pipeline, totalCount);
const queue = new PQueue({ concurrency: cpuCount });
const assetsTimer = performance.now();
for (const [originalPath, transforms] of staticImageList) {
await generateImagesForPath(originalPath, transforms, assetsCreationEnvironment, queue);
}
await queue.onIdle();
const assetsTimeEnd = performance.now();
logger.info(null, dim(`Completed in ${getTimeStat(assetsTimer, assetsTimeEnd)}.\n`));
delete globalThis?.astroAsset?.addStaticImage;
}
await runHookBuildGenerated({
config: opts.settings.config,
logger: pipeline.getLogger(),
});
logger.info(null, dim(`Completed in ${getTimeStat(timer, performance.now())}.\n`));
}
async function generateImage(
pipeline: BuildPipeline,
transform: ImageTransform,
path: string,
count: number,
totalCount: number
) {
const logger = pipeline.getLogger();
let timeStart = performance.now();
const generationData = await generateImageInternal(pipeline, transform, path);
if (!generationData) {
return;
}
const timeEnd = performance.now();
const timeChange = getTimeStat(timeStart, timeEnd);
const timeIncrease = `(+${timeChange})`;
const statsText = generationData.cached
? `(reused cache entry)`
: `(before: ${generationData.weight.before}kB, after: ${generationData.weight.after}kB)`;
const counter = `(${count}/${totalCount})`;
logger.info(
null,
` ${green('▶')} ${path} ${dim(statsText)} ${dim(timeIncrease)} ${dim(counter)}`
);
}
async function generatePage(

3
packages/astro/src/type-utils.ts
View file

@ -27,3 +27,6 @@ export type KebabKeys<T> = { [K in keyof T as K extends string ? Kebab<K> : K]:
// Similar to `keyof`, gets the type of all the values of an object
export type ValueOf<T> = T[keyof T];
// Gets the type of the values of a Map
export type MapValue<T> = T extends Map<any, infer V> ? V : never;

20
pnpm-lock.yaml generated
View file

@ -595,6 +595,9 @@ importers:
p-limit:
specifier: ^4.0.0
version: 4.0.0
p-queue:
specifier: ^7.4.1
version: 7.4.1
path-to-regexp:
specifier: ^6.2.1
version: 6.2.1
@ -10993,6 +10996,10 @@ packages:
resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==}
engines: {node: '>= 0.6'}
/eventemitter3@5.0.1:
resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==}
dev: false
/execa@5.1.1:
resolution: {integrity: sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==}
engines: {node: '>=10'}
@ -14125,6 +14132,19 @@ packages:
aggregate-error: 4.0.1
dev: true
/p-queue@7.4.1:
resolution: {integrity: sha512-vRpMXmIkYF2/1hLBKisKeVYJZ8S2tZ0zEAmIJgdVKP2nq0nh4qCdf8bgw+ZgKrkh71AOCaqzwbJJk1WtdcF3VA==}
engines: {node: '>=12'}
dependencies:
eventemitter3: 5.0.1
p-timeout: 5.1.0
dev: false
/p-timeout@5.1.0:
resolution: {integrity: sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==}
engines: {node: '>=12'}
dev: false
/p-try@2.2.0:
resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==}
engines: {node: '>=6'}