0
Fork 0
mirror of https://github.com/withastro/astro.git synced 2025-04-07 23:41:43 -05:00

Invalidate CC cache manifest when lockfile or config changes (#10763)

* Invalidate CC cache manifest when lockfile or config changes

* Close the handle and increment manifest version

* debug info

* Provide a reason for cache busting

* Handle compile metadata missing

* Try it this way

* Copy over cached assets as well

* Only restore chunks when cache is valid

* Better handle invalid caches

* Explain when there is no content manifest

* Add tests

* debugging

* Remove debugging

* Update packages/astro/src/core/build/plugins/plugin-content.ts

Co-authored-by: Bjorn Lu <bjornlu.dev@gmail.com>

* Update packages/astro/src/core/build/plugins/plugin-content.ts

Co-authored-by: Bjorn Lu <bjornlu.dev@gmail.com>

* Review comments

* Add chunks path constant

---------

Co-authored-by: Bjorn Lu <bjornlu.dev@gmail.com>
This commit is contained in:
Matthew Phillips 2024-04-18 11:23:05 -04:00 committed by GitHub
parent 77822a822b
commit 6313277137
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 355 additions and 51 deletions

View file

@ -0,0 +1,5 @@
---
"astro": patch
---
Invalidate CC cache manifest when lockfile or config changes

View file

@ -2779,6 +2779,7 @@ export interface AstroIntegration {
dir: URL;
routes: RouteData[];
logger: AstroIntegrationLogger;
cacheManifest: boolean;
}) => void | Promise<void>;
};
}

View file

@ -0,0 +1 @@
export const CHUNKS_PATH = 'chunks/';

View file

@ -218,6 +218,7 @@ class AstroBuilder {
.flat()
.map((pageData) => pageData.route),
logging: this.logger,
cacheManifest: internals.cacheManifestUsed,
});
if (this.logger.level && levels[this.logger.level()] <= levels['info']) {

View file

@ -89,6 +89,7 @@ export interface BuildInternals {
discoveredScripts: Set<string>;
cachedClientEntries: string[];
cacheManifestUsed: boolean;
propagatedStylesMap: Map<string, Set<StylesheetAsset>>;
propagatedScriptsMap: Map<string, Set<string>>;
@ -140,6 +141,7 @@ export function createBuildInternals(): BuildInternals {
componentMetadata: new Map(),
ssrSplitEntryChunks: new Map(),
entryPoints: new Map(),
cacheManifestUsed: false,
};
}

View file

@ -3,6 +3,7 @@ import fsMod from 'node:fs';
import { fileURLToPath } from 'node:url';
import pLimit from 'p-limit';
import { type Plugin as VitePlugin, normalizePath } from 'vite';
import { configPaths } from '../../config/index.js';
import { CONTENT_RENDER_FLAG, PROPAGATED_ASSET_FLAG } from '../../../content/consts.js';
import { type ContentLookupMap, hasContentFlag } from '../../../content/utils.js';
import {
@ -10,7 +11,7 @@ import {
generateLookupMap,
} from '../../../content/vite-plugin-content-virtual-mod.js';
import { isServerLikeOutput } from '../../../prerender/utils.js';
import { joinPaths, removeFileExtension, removeLeadingForwardSlash } from '../../path.js';
import { joinPaths, removeFileExtension, removeLeadingForwardSlash, appendForwardSlash } from '../../path.js';
import { addRollupInput } from '../add-rollup-input.js';
import { type BuildInternals } from '../internal.js';
import type { AstroBuildPlugin } from '../plugin.js';
@ -18,12 +19,14 @@ import { copyFiles } from '../static-build.js';
import type { StaticBuildOptions } from '../types.js';
import { encodeName } from '../util.js';
import { extendManualChunks } from './util.js';
import { emptyDir } from '../../fs/index.js';
import { CHUNKS_PATH } from '../consts.js';
const CONTENT_CACHE_DIR = './content/';
const CONTENT_MANIFEST_FILE = './manifest.json';
// IMPORTANT: Update this version when making significant changes to the manifest format.
// Only manifests generated with the same version number can be compared.
const CONTENT_MANIFEST_VERSION = 0;
const CONTENT_MANIFEST_VERSION = 1;
interface ContentManifestKey {
collection: string;
@ -39,40 +42,44 @@ interface ContentManifest {
// Tracks components that should be passed to the client build
// When the cache is restored, these might no longer be referenced
clientEntries: string[];
// Hash of the lockfiles, pnpm-lock.yaml, package-lock.json, etc.
// Kept so that installing new packages results in a full rebuild.
lockfiles: string;
// Hash of the Astro config. Changing options results in invalidating the cache.
configs: string;
}
const virtualEmptyModuleId = `virtual:empty-content`;
const resolvedVirtualEmptyModuleId = `\0${virtualEmptyModuleId}`;
const NO_MANIFEST_VERSION = -1 as const;
function createContentManifest(): ContentManifest {
return { version: -1, entries: [], serverEntries: [], clientEntries: [] };
return { version: NO_MANIFEST_VERSION, entries: [], serverEntries: [], clientEntries: [], lockfiles: "", configs: "" };
}
function vitePluginContent(
opts: StaticBuildOptions,
lookupMap: ContentLookupMap,
internals: BuildInternals
internals: BuildInternals,
cachedBuildOutput: Array<{ cached: URL; dist: URL; }>
): VitePlugin {
const { config } = opts.settings;
const { cacheDir } = config;
const distRoot = config.outDir;
const distContentRoot = new URL('./content/', distRoot);
const cachedChunks = new URL('./chunks/', opts.settings.config.cacheDir);
const distChunks = new URL('./chunks/', opts.settings.config.outDir);
const contentCacheDir = new URL(CONTENT_CACHE_DIR, cacheDir);
const contentManifestFile = new URL(CONTENT_MANIFEST_FILE, contentCacheDir);
const cache = contentCacheDir;
const cacheTmp = new URL('./.tmp/', cache);
const cacheTmp = new URL('./.tmp/', contentCacheDir);
let oldManifest = createContentManifest();
let newManifest = createContentManifest();
let entries: ContentEntries;
let injectedEmptyFile = false;
let currentManifestState: ReturnType<typeof manifestState> = 'valid';
if (fsMod.existsSync(contentManifestFile)) {
try {
const data = fsMod.readFileSync(contentManifestFile, { encoding: 'utf8' });
oldManifest = JSON.parse(data);
internals.cachedClientEntries = oldManifest.clientEntries;
} catch {}
}
@ -84,6 +91,32 @@ function vitePluginContent(
newManifest = await generateContentManifest(opts, lookupMap);
entries = getEntriesFromManifests(oldManifest, newManifest);
// If the manifest is valid, use the cached client entries as nothing has changed
currentManifestState = manifestState(oldManifest, newManifest);
if(currentManifestState === 'valid') {
internals.cachedClientEntries = oldManifest.clientEntries;
} else {
let logReason = '';
switch(currentManifestState) {
case 'config-mismatch':
logReason = 'Astro config has changed';
break;
case 'lockfile-mismatch':
logReason = 'Lockfiles have changed';
break;
case 'no-entries':
logReason = 'No content collections entries cached';
break;
case 'version-mismatch':
logReason = 'The cache manifest version has changed';
break;
case 'no-manifest':
logReason = 'No content manifest was found in the cache';
break;
}
opts.logger.info('build', `Cache invalid, rebuilding from source. Reason: ${logReason}.`);
}
// Of the cached entries, these ones need to be rebuilt
for (const { type, entry } of entries.buildFromSource) {
const fileURL = encodeURI(joinPaths(opts.settings.config.root.toString(), entry));
@ -96,10 +129,18 @@ function vitePluginContent(
}
newOptions = addRollupInput(newOptions, inputs);
}
// Restores cached chunks from the previous build
if (fsMod.existsSync(cachedChunks)) {
await copyFiles(cachedChunks, distChunks, true);
// Restores cached chunks and assets from the previous build
// If the manifest state is not valid then it needs to rebuild everything
// so don't do that in this case.
if(currentManifestState === 'valid') {
for(const { cached, dist } of cachedBuildOutput) {
if (fsMod.existsSync(cached)) {
await copyFiles(cached, dist, true);
}
}
}
// If nothing needs to be rebuilt, we inject a fake entrypoint to appease Rollup
if (entries.buildFromSource.length === 0) {
newOptions = addRollupInput(newOptions, [virtualEmptyModuleId]);
@ -199,16 +240,20 @@ function vitePluginContent(
]);
newManifest.serverEntries = Array.from(serverComponents);
newManifest.clientEntries = Array.from(clientComponents);
const cacheExists = fsMod.existsSync(contentCacheDir);
// If the manifest is invalid, empty the cache so that we can create a new one.
if(cacheExists && currentManifestState !== 'valid') {
emptyDir(contentCacheDir);
}
await fsMod.promises.mkdir(contentCacheDir, { recursive: true });
await fsMod.promises.writeFile(contentManifestFile, JSON.stringify(newManifest), {
encoding: 'utf8',
});
const cacheExists = fsMod.existsSync(cache);
fsMod.mkdirSync(cache, { recursive: true });
await fsMod.promises.mkdir(cacheTmp, { recursive: true });
await copyFiles(distContentRoot, cacheTmp, true);
if (cacheExists) {
if (cacheExists && currentManifestState === 'valid') {
await copyFiles(contentCacheDir, distContentRoot, false);
}
await copyFiles(cacheTmp, contentCacheDir);
@ -242,12 +287,12 @@ function getEntriesFromManifests(
oldManifest: ContentManifest,
newManifest: ContentManifest
): ContentEntries {
const { version: oldVersion, entries: oldEntries } = oldManifest;
const { version: newVersion, entries: newEntries } = newManifest;
const { entries: oldEntries } = oldManifest;
const { entries: newEntries } = newManifest;
let entries: ContentEntries = { restoreFromCache: [], buildFromSource: [] };
const newEntryMap = new Map<ContentManifestKey, string>(newEntries);
if (oldVersion !== newVersion || oldEntries.length === 0) {
if (manifestState(oldManifest, newManifest) !== 'valid') {
entries.buildFromSource = Array.from(newEntryMap.keys());
return entries;
}
@ -265,16 +310,37 @@ function getEntriesFromManifests(
return entries;
}
type ManifestState = 'valid' | 'no-manifest' | 'version-mismatch' | 'no-entries' | 'lockfile-mismatch' | 'config-mismatch';
function manifestState(oldManifest: ContentManifest, newManifest: ContentManifest): ManifestState {
// There isn't an existing manifest.
if(oldManifest.version === NO_MANIFEST_VERSION) {
return 'no-manifest';
}
// Version mismatch, always invalid
if (oldManifest.version !== newManifest.version) {
return 'version-mismatch';
}
if(oldManifest.entries.length === 0) {
return 'no-entries';
}
// Lockfiles have changed or there is no lockfile at all.
if((oldManifest.lockfiles !== newManifest.lockfiles) || newManifest.lockfiles === '') {
return 'lockfile-mismatch';
}
// Config has changed.
if(oldManifest.configs !== newManifest.configs) {
return 'config-mismatch';
}
return 'valid';
}
async function generateContentManifest(
opts: StaticBuildOptions,
lookupMap: ContentLookupMap
): Promise<ContentManifest> {
let manifest: ContentManifest = {
version: CONTENT_MANIFEST_VERSION,
entries: [],
serverEntries: [],
clientEntries: [],
};
let manifest = createContentManifest();
manifest.version = CONTENT_MANIFEST_VERSION;
const limit = pLimit(10);
const promises: Promise<void>[] = [];
@ -290,13 +356,63 @@ async function generateContentManifest(
);
}
}
const [lockfiles, configs] = await Promise.all([
lockfilesHash(opts.settings.config.root),
configHash(opts.settings.config.root)
]);
manifest.lockfiles = lockfiles;
manifest.configs = configs;
await Promise.all(promises);
return manifest;
}
function checksum(data: string): string {
return createHash('sha1').update(data).digest('base64');
async function pushBufferInto(fileURL: URL, buffers: Uint8Array[]) {
try {
const handle = await fsMod.promises.open(fileURL, 'r');
const data = await handle.readFile();
buffers.push(data);
await handle.close();
} catch {
// File doesn't exist, ignore
}
}
async function lockfilesHash(root: URL) {
// Order is important so don't change this.
const lockfiles = ['package-lock.json', 'pnpm-lock.yaml', 'yarn.lock', 'bun.lockb'];
const datas: Uint8Array[] = [];
const promises: Promise<void>[] = [];
for(const lockfileName of lockfiles) {
const fileURL = new URL(`./${lockfileName}`, root);
promises.push(pushBufferInto(fileURL, datas));
}
await Promise.all(promises);
return checksum(...datas);
}
async function configHash(root: URL) {
const configFileNames = configPaths;
for(const configPath of configFileNames) {
try {
const fileURL = new URL(`./${configPath}`, root);
const data = await fsMod.promises.readFile(fileURL);
const hash = checksum(data);
return hash;
} catch {
// File doesn't exist
}
}
// No config file, still create a hash since we can compare nothing against nothing.
return checksum(`export default {}`);
}
function checksum(...datas: string[] | Uint8Array[]): string {
const hash = createHash('sha1');
datas.forEach(data => hash.update(data));
return hash.digest('base64');
}
function collectionTypeToFlag(type: 'content' | 'data') {
@ -308,8 +424,15 @@ export function pluginContent(
opts: StaticBuildOptions,
internals: BuildInternals
): AstroBuildPlugin {
const cachedChunks = new URL('./chunks/', opts.settings.config.cacheDir);
const distChunks = new URL('./chunks/', opts.settings.config.outDir);
const { cacheDir, outDir } = opts.settings.config;
const chunksFolder = './' + CHUNKS_PATH;
const assetsFolder = './' + appendForwardSlash(opts.settings.config.build.assets);
// These are build output that is kept in the cache.
const cachedBuildOutput = [
{ cached: new URL(chunksFolder, cacheDir), dist: new URL(chunksFolder, outDir) },
{ cached: new URL(assetsFolder, cacheDir), dist: new URL(assetsFolder, outDir) },
];
return {
targets: ['server'],
@ -321,10 +444,9 @@ export function pluginContent(
if (isServerLikeOutput(opts.settings.config)) {
return { vitePlugin: undefined };
}
const lookupMap = await generateLookupMap({ settings: opts.settings, fs: fsMod });
return {
vitePlugin: vitePluginContent(opts, lookupMap, internals),
vitePlugin: vitePluginContent(opts, lookupMap, internals, cachedBuildOutput),
};
},
@ -335,8 +457,11 @@ export function pluginContent(
if (isServerLikeOutput(opts.settings.config)) {
return;
}
if (fsMod.existsSync(distChunks)) {
await copyFiles(distChunks, cachedChunks, true);
// Cache build output of chunks and assets
for(const { cached, dist } of cachedBuildOutput) {
if (fsMod.existsSync(dist)) {
await copyFiles(dist, cached, true);
}
}
},
},

View file

@ -34,6 +34,7 @@ import { RESOLVED_SPLIT_MODULE_ID, RESOLVED_SSR_VIRTUAL_MODULE_ID } from './plug
import { ASTRO_PAGE_EXTENSION_POST_PATTERN } from './plugins/util.js';
import type { StaticBuildOptions } from './types.js';
import { encodeName, getTimeStat, viteBuildReturnToRollupOutputs } from './util.js';
import { CHUNKS_PATH } from './consts.js';
export async function viteBuild(opts: StaticBuildOptions) {
const { allPages, settings } = opts;
@ -196,7 +197,7 @@ async function ssrBuild(
// We need to keep these separate
chunkFileNames(chunkInfo) {
const { name } = chunkInfo;
let prefix = 'chunks/';
let prefix = CHUNKS_PATH;
let suffix = '_[hash].mjs';
if (isContentCache) {
@ -454,7 +455,7 @@ export async function copyFiles(fromFolder: URL, toFolder: URL, includeDotfiles
dot: includeDotfiles,
});
if (files.length === 0) return;
await Promise.all(
return await Promise.all(
files.map(async function copyFile(filename) {
const from = new URL(filename, fromFolder);
const to = new URL(filename, toFolder);

View file

@ -78,15 +78,19 @@ export function resolveRoot(cwd?: string | URL): string {
return cwd ? path.resolve(cwd) : process.cwd();
}
// Config paths to search for. In order of likely appearance
// to speed up the check.
export const configPaths = Object.freeze([
'astro.config.mjs',
'astro.config.js',
'astro.config.ts',
'astro.config.mts',
'astro.config.cjs',
'astro.config.cts',
]);
async function search(fsMod: typeof fs, root: string) {
const paths = [
'astro.config.mjs',
'astro.config.js',
'astro.config.ts',
'astro.config.mts',
'astro.config.cjs',
'astro.config.cts',
].map((p) => path.join(root, p));
const paths = configPaths.map((p) => path.join(root, p));
for (const file of paths) {
if (fsMod.existsSync(file)) {

View file

@ -1,4 +1,4 @@
export { resolveConfig, resolveConfigPath, resolveFlags, resolveRoot } from './config.js';
export { configPaths, resolveConfig, resolveConfigPath, resolveFlags, resolveRoot } from './config.js';
export { createNodeLogger } from './logging.js';
export { mergeConfig } from './merge.js';
export type { AstroConfigType } from './schema.js';

View file

@ -477,9 +477,10 @@ type RunHookBuildDone = {
pages: string[];
routes: RouteData[];
logging: Logger;
cacheManifest: boolean;
};
export async function runHookBuildDone({ config, pages, routes, logging }: RunHookBuildDone) {
export async function runHookBuildDone({ config, pages, routes, logging, cacheManifest }: RunHookBuildDone) {
const dir = isServerLikeOutput(config) ? config.build.client : config.outDir;
await fs.promises.mkdir(dir, { recursive: true });
@ -495,6 +496,7 @@ export async function runHookBuildDone({ config, pages, routes, logging }: RunHo
dir,
routes,
logger,
cacheManifest,
}),
logger: logging,
});

View file

@ -89,12 +89,22 @@ export default function astro({ settings, logger }: AstroPluginOptions): vite.Pl
// modules are compiled first, then its virtual modules.
const filename = normalizePath(normalizeFilename(parsedId.filename, config.root));
let compileMetadata = astroFileToCompileMetadata.get(filename);
// If `compileMetadata` doesn't exist in dev, that means the virtual module may have been invalidated.
// We try to re-compile the main Astro module (`filename`) first before retrieving the metadata again.
if (!compileMetadata && server) {
const code = await loadId(server.pluginContainer, filename);
// `compile` should re-set `filename` in `astroFileToCompileMetadata`
if (code != null) await compile(code, filename);
if (!compileMetadata) {
// If `compileMetadata` doesn't exist in dev, that means the virtual module may have been invalidated.
// We try to re-compile the main Astro module (`filename`) first before retrieving the metadata again.
if(server) {
const code = await loadId(server.pluginContainer, filename);
// `compile` should re-set `filename` in `astroFileToCompileMetadata`
if (code != null) await compile(code, filename);
}
// When cached we might load client-side scripts during the build
else if(config.experimental.contentCollectionCache) {
await this.load({
id: filename,
resolveDependencies: false,
});
}
compileMetadata = astroFileToCompileMetadata.get(filename);
}
// If the metadata still doesn't exist, that means the virtual modules are somehow compiled first,

View file

@ -0,0 +1,98 @@
import assert from 'node:assert/strict';
import { after, before, describe, it } from 'node:test';
import { loadFixture } from './test-utils.js';
import fs from 'node:fs';
import { copyFiles } from '../dist/core/build/static-build.js';
describe('Experimental Content Collections cache - invalidation', () => {
class CacheBackup {
constructor(root, relCacheDir) {
this.root = new URL(root, import.meta.url);
this.cacheDir = new URL(relCacheDir, this.root);
this.tmpDir = new URL(`./tmp` + relCacheDir.slice(1), this.root);
}
backup() {
this.rmTmp();
copyFiles(this.cacheDir, this.tmpDir);
}
restore() {
fs.rmSync(this.cacheDir, { recursive: true });
copyFiles(this.tmpDir, this.cacheDir);
}
rmTmp() {
fs.rmSync(this.tmpDir, { force: true, recursive: true });
}
}
class ManifestTestPlugin {
used = false;
plugin() {
return {
name: '@test/manifest-used',
hooks: {
'astro:build:done': ({ cacheManifest }) => {
this.used = cacheManifest;
}
}
}
}
}
describe('manifest version', () => {
let fixture, backup,
/** @type {ManifestTestPlugin} */
testPlugin;
before(async () => {
testPlugin = new ManifestTestPlugin();
fixture = await loadFixture({
root: './fixtures/content-collections-cache-invalidation/',
cacheDir: './cache/version-mismatch/',
experimental: { contentCollectionCache: true },
integrations: [
testPlugin.plugin()
]
});
backup = new CacheBackup('./fixtures/content-collections-cache-invalidation/', './cache/version-mismatch/');
backup.backup();
await fixture.build();
});
after(async () => {
backup.restore();
//await fixture.clean();
});
it('Manifest was not used', () => {
assert.equal(testPlugin.used, false, 'manifest not used because of version mismatch');
});
});
describe('lockfiles', () => {
let fixture, backup,
/** @type {ManifestTestPlugin} */
testPlugin;
before(async () => {
testPlugin = new ManifestTestPlugin();
fixture = await loadFixture({
root: './fixtures/content-collections-cache-invalidation/',
cacheDir: './cache/lockfile-mismatch/',
experimental: { contentCollectionCache: true },
integrations: [
testPlugin.plugin()
]
});
backup = new CacheBackup('./fixtures/content-collections-cache-invalidation/', './cache/lockfile-mismatch/');
backup.backup();
await fixture.build();
});
after(async () => {
backup.restore();
//await fixture.clean();
});
it('Manifest was not used', () => {
assert.equal(testPlugin.used, false, 'manifest not used because of lockfile mismatch');
});
});
});

View file

@ -0,0 +1 @@
tmp/

View file

@ -0,0 +1,12 @@
import { defineConfig } from 'astro/config';
// https://astro.build/config
export default defineConfig({
base: '/docs',
compressHTML: false,
vite: {
build: {
assetsInlineLimit: 0,
}
}
});

View file

@ -0,0 +1 @@
{"version":1,"entries":[[{"collection":"blog","type":"content","entry":"/src/content/blog/one.md"},"No8AlxYwy8HK3dH9W3Mj/6SeHMI="]],"serverEntries":[],"clientEntries":[],"lockfiles":"2jmj7l5rSw0yVb/vlWAYkK/YBwk=","configs":"h80ch7FwzpG2BXKQM39ZqFpU3dg="}

View file

@ -0,0 +1 @@
{"version":1111111,"entries":[[{"collection":"blog","type":"content","entry":"/src/content/blog/one.md"},"No8AlxYwy8HK3dH9W3Mj/6SeHMI="]],"serverEntries":[],"clientEntries":[],"lockfiles":"2jmj7l5rSw0yVb/vlWAYkK/YBwk=","configs":"h80ch7FwzpG2BXKQM39ZqFpU3dg="}

View file

@ -0,0 +1,8 @@
{
"name": "@test/content-collections-cache-invalidation",
"version": "0.0.0",
"private": true,
"dependencies": {
"astro": "workspace:*"
}
}

View file

@ -0,0 +1,5 @@
---
title: One
---
Hello world

View file

@ -0,0 +1,10 @@
import { defineCollection, z } from 'astro:content';
const blog = defineCollection({
type: 'collection',
schema: z.object({
title: z.string()
})
});
export const collections = { blog };

View file

@ -0,0 +1,10 @@
---
---
<html>
<head>
<title>Testing</title>
</head>
<body>
<h1>Testing</h1>
</body>
</html>

6
pnpm-lock.yaml generated
View file

@ -2466,6 +2466,12 @@ importers:
specifier: workspace:*
version: link:../../..
packages/astro/test/fixtures/content-collections-cache-invalidation:
dependencies:
astro:
specifier: workspace:*
version: link:../../..
packages/astro/test/fixtures/content-collections-empty-dir:
dependencies:
astro: