From f0031b0a3959b03d1b28e173982c7e1ca60e735f Mon Sep 17 00:00:00 2001 From: Erika <3019731+Princesseuh@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:23:23 +0100 Subject: [PATCH] feat(assets): Delete original assets unused outside of the optimization pipeline (#8954) Co-authored-by: Sarah Rainsberger --- .changeset/nasty-elephants-provide.md | 5 +++ packages/astro/src/assets/build/generate.ts | 31 ++++++++++--- packages/astro/src/assets/internal.ts | 9 ++++ packages/astro/src/assets/types.ts | 8 +++- packages/astro/src/assets/utils/emitAsset.ts | 11 ++++- packages/astro/src/assets/utils/metadata.ts | 2 +- packages/astro/src/assets/utils/proxy.ts | 13 ++++++ .../astro/src/assets/utils/queryParams.ts | 2 +- .../astro/src/assets/vite-plugin-assets.ts | 44 +++++++++++++------ packages/astro/src/content/runtime-assets.ts | 2 +- .../content/vite-plugin-content-imports.ts | 14 ++++-- packages/astro/src/core/build/generate.ts | 2 +- tsconfig.base.json | 3 +- 13 files changed, 116 insertions(+), 30 deletions(-) create mode 100644 .changeset/nasty-elephants-provide.md create mode 100644 packages/astro/src/assets/utils/proxy.ts diff --git a/.changeset/nasty-elephants-provide.md b/.changeset/nasty-elephants-provide.md new file mode 100644 index 0000000000..8a5ab241a3 --- /dev/null +++ b/.changeset/nasty-elephants-provide.md @@ -0,0 +1,5 @@ +--- +'astro': minor +--- + +Updates the Image Services API to now delete original images from the final build that are not used outside of the optimization pipeline. For users with a large number of these images (e.g. thumbnails), this should reduce storage consumption and deployment times. diff --git a/packages/astro/src/assets/build/generate.ts b/packages/astro/src/assets/build/generate.ts index 4447fa0a73..c34e136ddd 100644 --- a/packages/astro/src/assets/build/generate.ts +++ b/packages/astro/src/assets/build/generate.ts @@ -31,6 +31,7 @@ type GenerationData = GenerationDataUncached | GenerationDataCached; type AssetEnv = { logger: Logger; + isSSR: boolean; count: { total: number; current: number }; useCache: boolean; assetsCacheDir: URL; @@ -74,6 +75,7 @@ export async function prepareAssetsGenerationEnv( return { logger, + isSSR: isServerLikeOutput(config), count, useCache, assetsCacheDir, @@ -84,20 +86,41 @@ export async function prepareAssetsGenerationEnv( }; } +function getFullImagePath(originalFilePath: string, env: AssetEnv): URL { + return new URL( + '.' + prependForwardSlash(join(env.assetsFolder, basename(originalFilePath))), + env.serverRoot + ); +} + export async function generateImagesForPath( originalFilePath: string, - transforms: MapValue, + transformsAndPath: MapValue, env: AssetEnv, queue: PQueue ) { const originalImageData = await loadImage(originalFilePath, env); - for (const [_, transform] of transforms) { + for (const [_, transform] of transformsAndPath.transforms) { queue.add(async () => generateImage(originalImageData, transform.finalPath, transform.transform) ); } + // In SSR, we cannot know if an image is referenced in a server-rendered page, so we can't delete anything + // For instance, the same image could be referenced in both a server-rendered page and build-time-rendered page + if ( + !env.isSSR && + !isRemotePath(originalFilePath) && + !globalThis.astroAsset.referencedImages?.has(transformsAndPath.originalSrcPath) + ) { + try { + await fs.promises.unlink(getFullImagePath(originalFilePath, env)); + } catch (e) { + /* No-op, it's okay if we fail to delete one of the file, we're not too picky. */ + } + } + async function generateImage( originalImage: ImageData, filepath: string, @@ -245,9 +268,7 @@ async function loadImage(path: string, env: AssetEnv): Promise { } return { - data: await fs.promises.readFile( - new URL('.' + prependForwardSlash(join(env.assetsFolder, basename(path))), env.serverRoot) - ), + data: await fs.promises.readFile(getFullImagePath(path, env)), expires: 0, }; } diff --git a/packages/astro/src/assets/internal.ts b/packages/astro/src/assets/internal.ts index ef628b69f4..46319ed3bc 100644 --- a/packages/astro/src/assets/internal.ts +++ b/packages/astro/src/assets/internal.ts @@ -91,6 +91,15 @@ export async function getImage( : options.src, }; + // Clone the `src` object if it's an ESM import so that we don't refer to any properties of the original object + // Causing our generate step to think the image is used outside of the image optimization pipeline + const clonedSrc = isESMImportedImage(resolvedOptions.src) + ? // @ts-expect-error - clone is a private, hidden prop + resolvedOptions.src.clone ?? resolvedOptions.src + : resolvedOptions.src; + + resolvedOptions.src = clonedSrc; + const validatedOptions = service.validateOptions ? await service.validateOptions(resolvedOptions, imageConfig) : resolvedOptions; diff --git a/packages/astro/src/assets/types.ts b/packages/astro/src/assets/types.ts index 91d6ba1ff6..34ded257f6 100644 --- a/packages/astro/src/assets/types.ts +++ b/packages/astro/src/assets/types.ts @@ -10,7 +10,10 @@ export type ImageOutputFormat = (typeof VALID_OUTPUT_FORMATS)[number] | (string export type AssetsGlobalStaticImagesList = Map< string, - Map + { + originalSrcPath: string; + transforms: Map; + } >; declare global { @@ -19,6 +22,7 @@ declare global { imageService?: ImageService; addStaticImage?: ((options: ImageTransform, hashProperties: string[]) => string) | undefined; staticImages?: AssetsGlobalStaticImagesList; + referencedImages?: Set; }; } @@ -31,6 +35,8 @@ export interface ImageMetadata { height: number; format: ImageInputFormat; orientation?: number; + /** @internal */ + fsPath: string; } /** diff --git a/packages/astro/src/assets/utils/emitAsset.ts b/packages/astro/src/assets/utils/emitAsset.ts index b9ca146b71..0f996691be 100644 --- a/packages/astro/src/assets/utils/emitAsset.ts +++ b/packages/astro/src/assets/utils/emitAsset.ts @@ -24,11 +24,18 @@ export async function emitESMImage( const fileMetadata = await imageMetadata(fileData, id); - const emittedImage: ImageMetadata = { + const emittedImage: Omit = { src: '', ...fileMetadata, }; + // Private for now, we generally don't want users to rely on filesystem paths, but we need it so that we can maybe remove the original asset from the build if it's unused. + Object.defineProperty(emittedImage, 'fsPath', { + enumerable: false, + writable: false, + value: url, + }); + // Build if (!watchMode) { const pathname = decodeURI(url.pathname); @@ -50,7 +57,7 @@ export async function emitESMImage( emittedImage.src = `/@fs` + prependForwardSlash(fileURLToNormalizedPath(url)); } - return emittedImage; + return emittedImage as ImageMetadata; } function fileURLToNormalizedPath(filePath: URL): string { diff --git a/packages/astro/src/assets/utils/metadata.ts b/packages/astro/src/assets/utils/metadata.ts index fc89ca1ca1..2ee96a7aca 100644 --- a/packages/astro/src/assets/utils/metadata.ts +++ b/packages/astro/src/assets/utils/metadata.ts @@ -5,7 +5,7 @@ import type { ImageInputFormat, ImageMetadata } from '../types.js'; export async function imageMetadata( data: Buffer, src?: string -): Promise> { +): Promise> { const result = probe.sync(data); if (result === null) { diff --git a/packages/astro/src/assets/utils/proxy.ts b/packages/astro/src/assets/utils/proxy.ts new file mode 100644 index 0000000000..0dcaffb822 --- /dev/null +++ b/packages/astro/src/assets/utils/proxy.ts @@ -0,0 +1,13 @@ +export function getProxyCode(options: Record, isSSR: boolean): string { + return ` + new Proxy(${JSON.stringify(options)}, { + get(target, name, receiver) { + if (name === 'clone') { + return structuredClone(target); + } + ${!isSSR ? 'globalThis.astroAsset.referencedImages.add(target.fsPath);' : ''} + return target[name]; + } + }) + `; +} diff --git a/packages/astro/src/assets/utils/queryParams.ts b/packages/astro/src/assets/utils/queryParams.ts index 18acc8876a..56bb4b3ff9 100644 --- a/packages/astro/src/assets/utils/queryParams.ts +++ b/packages/astro/src/assets/utils/queryParams.ts @@ -2,7 +2,7 @@ import type { ImageInputFormat, ImageMetadata } from '../types.js'; export function getOrigQueryParams( params: URLSearchParams -): Omit | undefined { +): Pick | undefined { const width = params.get('origWidth'); const height = params.get('origHeight'); const format = params.get('origFormat'); diff --git a/packages/astro/src/assets/vite-plugin-assets.ts b/packages/astro/src/assets/vite-plugin-assets.ts index 23e2924bae..75ff7d2c43 100644 --- a/packages/astro/src/assets/vite-plugin-assets.ts +++ b/packages/astro/src/assets/vite-plugin-assets.ts @@ -14,6 +14,7 @@ import { isServerLikeOutput } from '../prerender/utils.js'; import { VALID_INPUT_FORMATS, VIRTUAL_MODULE_ID, VIRTUAL_SERVICE_ID } from './consts.js'; import { isESMImportedImage } from './internal.js'; import { emitESMImage } from './utils/emitAsset.js'; +import { getProxyCode } from './utils/proxy.js'; import { hashTransform, propsToFilename } from './utils/transformToPath.js'; const resolvedVirtualModuleId = '\0' + VIRTUAL_MODULE_ID; @@ -26,7 +27,9 @@ export default function assets({ }: AstroPluginOptions & { mode: string }): vite.Plugin[] { let resolvedConfig: vite.ResolvedConfig; - globalThis.astroAsset = {}; + globalThis.astroAsset = { + referencedImages: new Set(), + }; return [ // Expose the components and different utilities from `astro:assets` and handle serving images from `/_image` in dev @@ -81,22 +84,28 @@ export default function assets({ if (!globalThis.astroAsset.staticImages) { globalThis.astroAsset.staticImages = new Map< string, - Map + { + originalSrcPath: string; + transforms: Map; + } >(); } - const originalImagePath = ( + // Rollup will copy the file to the output directory, this refer to this final path, not to the original path + const finalOriginalImagePath = ( isESMImportedImage(options.src) ? options.src.src : options.src ).replace(settings.config.build.assetsPrefix || '', ''); - const hash = hashTransform( - options, - settings.config.image.service.entrypoint, - hashProperties - ); + + // This, however, is the real original path, in `src` and all. + const originalSrcPath = isESMImportedImage(options.src) + ? options.src.fsPath + : options.src; + + const hash = hashTransform(options, settings.config.image.service.entrypoint, hashProperties); let finalFilePath: string; - let transformsForPath = globalThis.astroAsset.staticImages.get(originalImagePath); - let transformForHash = transformsForPath?.get(hash); + let transformsForPath = globalThis.astroAsset.staticImages.get(finalOriginalImagePath); + let transformForHash = transformsForPath?.transforms.get(hash); if (transformsForPath && transformForHash) { finalFilePath = transformForHash.finalPath; } else { @@ -105,11 +114,17 @@ export default function assets({ ); if (!transformsForPath) { - globalThis.astroAsset.staticImages.set(originalImagePath, new Map()); - transformsForPath = globalThis.astroAsset.staticImages.get(originalImagePath)!; + globalThis.astroAsset.staticImages.set(finalOriginalImagePath, { + originalSrcPath: originalSrcPath, + transforms: new Map(), + }); + transformsForPath = globalThis.astroAsset.staticImages.get(finalOriginalImagePath)!; } - transformsForPath.set(hash, { finalPath: finalFilePath, transform: options }); + transformsForPath.transforms.set(hash, { + finalPath: finalFilePath, + transform: options, + }); } if (settings.config.build.assetsPrefix) { @@ -171,7 +186,8 @@ export default function assets({ }); } - return `export default ${JSON.stringify(meta)}`; + return ` + export default ${getProxyCode(meta, isServerLikeOutput(settings.config))}`; } }, }, diff --git a/packages/astro/src/content/runtime-assets.ts b/packages/astro/src/content/runtime-assets.ts index eaca83740d..0709933def 100644 --- a/packages/astro/src/content/runtime-assets.ts +++ b/packages/astro/src/content/runtime-assets.ts @@ -22,7 +22,7 @@ export function createImage(pluginContext: PluginContext, entryFilePath: string) return z.never(); } - return metadata; + return { ...metadata, ASTRO_ASSET: true }; }); }; } diff --git a/packages/astro/src/content/vite-plugin-content-imports.ts b/packages/astro/src/content/vite-plugin-content-imports.ts index 15ca6d956b..052a673f48 100644 --- a/packages/astro/src/content/vite-plugin-content-imports.ts +++ b/packages/astro/src/content/vite-plugin-content-imports.ts @@ -12,8 +12,10 @@ import type { DataEntryModule, DataEntryType, } from '../@types/astro.js'; +import { getProxyCode } from '../assets/utils/proxy.js'; import { AstroError } from '../core/errors/errors.js'; import { AstroErrorData } from '../core/errors/index.js'; +import { isServerLikeOutput } from '../prerender/utils.js'; import { escapeViteEnvReferences } from '../vite-plugin-utils/index.js'; import { CONTENT_FLAG, DATA_FLAG } from './consts.js'; import { @@ -94,7 +96,7 @@ export function astroContentImportPlugin({ const code = escapeViteEnvReferences(` export const id = ${JSON.stringify(id)}; export const collection = ${JSON.stringify(collection)}; -export const data = ${stringifyEntryData(data)}; +export const data = ${stringifyEntryData(data, isServerLikeOutput(settings.config))}; export const _internal = { type: 'data', filePath: ${JSON.stringify(_internal.filePath)}, @@ -118,7 +120,7 @@ export const _internal = { export const collection = ${JSON.stringify(collection)}; export const slug = ${JSON.stringify(slug)}; export const body = ${JSON.stringify(body)}; - export const data = ${stringifyEntryData(data)}; + export const data = ${stringifyEntryData(data, isServerLikeOutput(settings.config))}; export const _internal = { type: 'content', filePath: ${JSON.stringify(_internal.filePath)}, @@ -352,13 +354,19 @@ async function getContentConfigFromGlobal() { } /** Stringify entry `data` at build time to be used as a Vite module */ -function stringifyEntryData(data: Record): string { +function stringifyEntryData(data: Record, isSSR: boolean): string { try { return devalue.uneval(data, (value) => { // Add support for URL objects if (value instanceof URL) { return `new URL(${JSON.stringify(value.href)})`; } + + // For Astro assets, add a proxy to track references + if (typeof value === 'object' && 'ASTRO_ASSET' in value) { + const { ASTRO_ASSET, ...asset } = value; + return getProxyCode(asset, isSSR); + } }); } catch (e) { if (e instanceof Error) { diff --git a/packages/astro/src/core/build/generate.ts b/packages/astro/src/core/build/generate.ts index 258d02038d..4bc1b25ef1 100644 --- a/packages/astro/src/core/build/generate.ts +++ b/packages/astro/src/core/build/generate.ts @@ -205,7 +205,7 @@ export async function generatePages(opts: StaticBuildOptions, internals: BuildIn logger.info(null, `\n${bgGreen(black(` generating optimized images `))}`); const totalCount = Array.from(staticImageList.values()) - .map((x) => x.size) + .map((x) => x.transforms.size) .reduce((a, b) => a + b, 0); const cpuCount = os.cpus().length; const assetsCreationEnvironment = await prepareAssetsGenerationEnv(pipeline, totalCount); diff --git a/tsconfig.base.json b/tsconfig.base.json index 0349af8a15..d800f2f513 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -9,6 +9,7 @@ "module": "Node16", "esModuleInterop": true, "skipLibCheck": true, - "verbatimModuleSyntax": true + "verbatimModuleSyntax": true, + "stripInternal": true } }