From e5743c334cf656cfa2a213a895c82fddfe4148e6 Mon Sep 17 00:00:00 2001 From: Nate Moore Date: Wed, 15 Nov 2023 08:56:29 -0600 Subject: [PATCH] wip(cache): add full module graph implementation --- .../core/build/plugins/cache/module-graph.ts | 177 ++++++++++ .../src/core/build/plugins/plugin-content.ts | 334 +++++++++++++----- packages/astro/src/core/build/static-build.ts | 2 - 3 files changed, 419 insertions(+), 94 deletions(-) create mode 100644 packages/astro/src/core/build/plugins/cache/module-graph.ts diff --git a/packages/astro/src/core/build/plugins/cache/module-graph.ts b/packages/astro/src/core/build/plugins/cache/module-graph.ts new file mode 100644 index 0000000000..faba6d692e --- /dev/null +++ b/packages/astro/src/core/build/plugins/cache/module-graph.ts @@ -0,0 +1,177 @@ +import { slash } from "@astrojs/internal-helpers/path"; +import { builtinModules } from "node:module"; +import { fileURLToPath } from "node:url"; +import fs from 'node:fs'; +import { createHash } from 'node:crypto'; + +export function resolvePackageVersion(specifier: string, require: NodeRequire) { + try { + // External packages will not be resolved + // We try to resolve their package.json version + const pkgInfo = require(`${specifier}/package.json`); + if (pkgInfo.version) { + return pkgInfo.version; + } + } catch {} +} + +export function checksum(path: fs.PathLike): string { + return checksumData(fs.readFileSync(path)); +} + +export function checksumData(data: Buffer | string): string { + return createHash('sha1').update(data).digest('hex'); +} + +export interface SerializedModuleGraph { + /** Track the version of this graph format to invalidate the entire graph in case of format changes */ + version: string; + checksums: Record; + graph: SerializedModuleNode[]; +} + +export class ModuleGraph { + // IMPORTANT: Update this version when making significant changes to the manifest format. + // Only manifests generated with the same version number can be compared. + static currentVersion = '1'; + version = ModuleGraph.currentVersion; + + #nodes = new Map(); + #entrypoints = new Map(); + #root: string; + + checksums = new Map(); + + rehydrate(data: SerializedModuleGraph) { + this.version = data.version; + this.checksums = new Map(Object.entries(data.checksums)); + for (const item of data.graph) { + const node = this.get(item.id); + for (const importer of item.importers) { + node.addImporter(importer); + } + for (const imported of item.imports) { + node.addImport(imported); + } + if (item.isEntry) { + this.addEntrypoint(item.id); + } + } + return this; + } + + constructor(root: URL) { + this.#root = fileURLToPath(root); + } + + get entrypoints() { + return this.#entrypoints.values(); + } + + get components() { + return Array.from(this.#nodes.values()).filter(n => n.id.startsWith('src/components/')); + } + + normalizeId(id: string): string { + if (id.includes('?')) id = id.split('?')[0]; + if (id.startsWith(this.#root)) id = id.slice(this.#root.length); + if (id.includes('\\')) id = slash(id); + if (id.startsWith('/')) id = id.slice(1); + if (id.startsWith('node_modules')) { + return id.split('node_modules').at(-1)?.split('/').filter(x => x).at(0) ?? id; + } + return id; + } + + get(id: string) { + id = this.normalizeId(id); + if (this.#nodes.has(id)) return this.#getNode(id); + return this.#addNode(id); + } + + addEntrypoint(id: string) { + id = this.normalizeId(id); + const node = this.get(id); + node.isEntry = true; + this.#entrypoints.set(node.id, node); + } + + #getNode(id: string) { + id = this.normalizeId(id); + return this.#nodes.get(id)!; + } + + #addNode(id: string) { + id = this.normalizeId(id); + const node = new ModuleNode(this, id); + this.#nodes.set(id, node); + return node; + } + + toJSON(): SerializedModuleGraph { + const checksums = Object.fromEntries(this.checksums.entries()); + const graph = Array.from(this.#nodes.values()).map(node => node.toJSON()); + return { version: this.version, checksums, graph } + } +} + +export class ModuleNode { + #graph: ModuleGraph; + #importers = new Set(); + #imports = new Set(); + isEntry = false; + + constructor(graph: ModuleGraph, public id: string) { + this.#graph = graph; + } + + addImporter(id: string) { + this.#importers.add(id); + } + addImport(id: string) { + this.#imports.add(id); + } + + get importers() { + return new Map(Array.from(this.#importers.values()).map(id => ([id, this.#graph.get(id)]))); + } + get imports() { + return new Map(Array.from(this.#imports.values()).map(id => ([id, this.#graph.get(id)]))); + } + + doesImport(id: string, seen = new WeakSet()): boolean { + seen.add(this); + if (id === this.id) return true; + if (seen.has(this)) return false; + for (const imported of this.imports.values()) { + seen.add(imported); + if (imported.id === id) return true; + if (imported.doesImport(id, seen)) return true; + } + return false; + } + toJSON(): SerializedModuleNode { + const isEntry = this.isEntry || undefined; + return { id: this.id, isEntry, importers: Array.from(this.#importers), imports: Array.from(this.#imports) } + } +} + +export interface SerializedModuleNode { + id: string; + isEntry?: boolean; + importers: string[]; + imports: string[]; +} + +export function isTrackableModule(id: string) { + if (id.startsWith('\0astro:')) return true; + if (id.startsWith('\0') || id.startsWith('virtual:')) return false + if (id.startsWith('node:')) return false + if (builtinModules.includes(id)) return false + return true; +} + +export function isDependency(id: string) { + if (!id.includes('node_modules')) return false + return true; +} diff --git a/packages/astro/src/core/build/plugins/plugin-content.ts b/packages/astro/src/core/build/plugins/plugin-content.ts index c28fa69046..b88831d61b 100644 --- a/packages/astro/src/core/build/plugins/plugin-content.ts +++ b/packages/astro/src/core/build/plugins/plugin-content.ts @@ -1,5 +1,4 @@ -import { createHash } from 'node:crypto'; -import fsMod from 'node:fs'; +import fs from 'node:fs'; import { fileURLToPath } from 'node:url'; import pLimit from 'p-limit'; import { normalizePath, type Plugin as VitePlugin } from 'vite'; @@ -18,9 +17,22 @@ import { copyFiles } from '../static-build.js'; import type { StaticBuildOptions } from '../types.js'; import { encodeName } from '../util.js'; import { extendManualChunks } from './util.js'; +import { isTrackableModule, checksum, ModuleGraph, ModuleNode, resolvePackageVersion } from './cache/module-graph.js'; +import { createRequire } from 'node:module'; -const CONTENT_CACHE_DIR = './content/'; -const CONTENT_MANIFEST_FILE = './manifest.json'; +function getEntrypointsFromModuleNode(input: ModuleNode): ModuleNode[] { + const result = new Set(input.isEntry ? [input] : []); + for (const importer of input.importers.values()) { + if (result.has(importer)) break; + for (const node of getEntrypointsFromModuleNode(importer)) { + result.add(node); + } + } + return Array.from(result); +} + +const BUILD_CACHE_DIR = './build/'; +const BUILD_MANIFEST_FILE = './astro.manifest.json'; // IMPORTANT: Update this version when making significant changes to the manifest format. // Only manifests generated with the same version number can be compared. const CONTENT_MANIFEST_VERSION = 0; @@ -48,6 +60,157 @@ function createContentManifest(): ContentManifest { return { version: -1, entries: [], serverEntries: [], clientEntries: [] }; } +function vitePluginModuleGraph(opts: StaticBuildOptions, lookupMap: ContentLookupMap): VitePlugin { + const graph = new ModuleGraph(opts.settings.config.root); + const rootPath = fileURLToPath(opts.settings.config.root); + const require = createRequire(opts.settings.config.root); + const packageResolutions = new Map(); + + const { config } = opts.settings; + const { cacheDir } = config; + const buildCacheDir = new URL(BUILD_CACHE_DIR, cacheDir); + const buildManifestFile = new URL(BUILD_MANIFEST_FILE, buildCacheDir); + + function resolveId(id: string): string { + // Intentionally using `||` instead of `??` here to catch empty strings + return packageResolutions.get(id) || graph.normalizeId(id); + } + + const content = graph.get('astro:content'); + for (const { entries } of Object.values(lookupMap)) { + for (const entrypoint of Object.values(entries)) { + const node = graph.get(entrypoint); + graph.checksums.set(graph.normalizeId(entrypoint), checksum(new URL(`./${entrypoint}`, opts.settings.config.root))); + graph.addEntrypoint(entrypoint); + node.addImporter('astro:content'); + content.addImport(entrypoint) + } + } + + let oldModuleGraph: ModuleGraph | undefined; + + const plugin: VitePlugin = { + name: 'astro:module-graph', + enforce: 'post', + options(options) { + if (Array.isArray(options.plugins)) { + const index = options.plugins.findIndex(v => typeof v === 'object' && !Array.isArray(v) && v && v.name === 'astro:module-graph'); + options.plugins.splice(index, 1); + options.plugins.splice(0, 0, plugin); + } + + if (fs.existsSync(buildManifestFile)) { + const oldManifest = fs.readFileSync(buildManifestFile, { encoding: 'utf8' }) + oldModuleGraph = new ModuleGraph(opts.settings.config.root); + oldModuleGraph.rehydrate(JSON.parse(oldManifest)); + if (oldModuleGraph.version !== ModuleGraph.currentVersion) { + fs.rmSync(buildManifestFile); + return; + } + let invalidatedNodes = new Set(); + for (let [id, oldChecksum] of oldModuleGraph.checksums) { + if (id.startsWith('dep:')) { + id = id.replace(/^dep\:/, ''); + const newChecksum = resolvePackageVersion(id, require); + if (newChecksum === oldChecksum) continue; + } else { + const fileURL = new URL('./' + id, opts.settings.config.root); + const newChecksum = checksum(fileURL); + if (newChecksum === oldChecksum) continue; + } + invalidatedNodes.add(oldModuleGraph.get(id)); + } + + const invalidatedEntrypoints = new Set(); + console.log(' node', Array.from(invalidatedNodes).map(n => n.id)); + for (const invalidatedNode of invalidatedNodes) { + for (const entrypoint of getEntrypointsFromModuleNode(invalidatedNode)) { + invalidatedEntrypoints.add(entrypoint); + } + } + console.log('entrypoint', Array.from(invalidatedEntrypoints).map(n => n.id)); + } + }, + async resolveId(id, importer) { + const resolution = await this.resolve(id, importer, { skipSelf: true }); + if (resolution?.id) { + const resolvedId = resolution.id; + if (!isTrackableModule(resolvedId)) return; + if (graph.checksums.has(resolvedId) || graph.checksums.has(`dep:${resolvedId}`)) return; + if (resolvedId.startsWith(rootPath) && !resolvedId.includes('node_modules')) { + graph.checksums.set(graph.normalizeId(resolvedId), checksum(graph.normalizeId(resolvedId))); + return; + } + if (packageResolutions.has(resolvedId)) return; + let packageSpecifier = id; + if (id.startsWith('.')) { + if (importer && packageResolutions.has(importer)) { + packageResolutions.set(resolvedId, packageResolutions.get(importer)!); + } + return; + } + if (id.startsWith('@')) { + packageSpecifier = id.split('/').slice(0, 2).join('/'); + } else { + packageSpecifier = id.split('/').at(0)!; + } + packageResolutions.set(resolvedId, packageSpecifier); + if (graph.checksums.has(`dep:${packageSpecifier}`)) { + return; + } + try { + // External packages will not be resolved + // We try to resolve their package.json version + const pkgInfo = require(`${packageSpecifier}/package.json`); + if (pkgInfo.version) { + graph.checksums.set(`dep:${packageSpecifier}`, pkgInfo.version); + } + } catch {} + } + }, + async buildEnd() { + for (let moduleId of this.getModuleIds()) { + let isPackage = false; + const info = this.getModuleInfo(moduleId); + if (!info) continue; + if (!isTrackableModule(info.id)) continue; + const id = graph.normalizeId(resolveId(moduleId)) + if (packageResolutions.has(id)) { + isPackage = true; + } + const node = graph.get(id); + const isEntry = info.dynamicImporters.find(n => n.startsWith('\0@astro-page:')); + if (isEntry) { + graph.addEntrypoint(id); + } + for (const importer of info.importers) { + if (!isTrackableModule(importer)) continue; + node.addImporter(resolveId(importer)); + } + for (const importer of info.dynamicImporters) { + if (!isTrackableModule(importer)) continue; + node.addImporter(resolveId(importer)); + } + if (isPackage) continue; + for (const imported of info.importedIds) { + if (!isTrackableModule(imported)) continue; + node.addImport(resolveId(imported)); + } + for (const imported of info.dynamicallyImportedIds) { + if (!isTrackableModule(imported)) continue; + node.addImport(resolveId(imported)); + } + } + + await fs.promises.mkdir(new URL('./', buildManifestFile), { recursive: true }); + await fs.promises.writeFile(buildManifestFile, JSON.stringify(graph), { + encoding: 'utf8', + }); + } + } + return plugin; +} + function vitePluginContent( opts: StaticBuildOptions, lookupMap: ContentLookupMap, @@ -56,56 +219,67 @@ function vitePluginContent( const { config } = opts.settings; const { cacheDir } = config; const distRoot = config.outDir; - const distContentRoot = new URL('./content/', distRoot); - const cachedChunks = new URL('./chunks/', opts.settings.config.cacheDir); - const distChunks = new URL('./chunks/', opts.settings.config.outDir); - const contentCacheDir = new URL(CONTENT_CACHE_DIR, cacheDir); - const contentManifestFile = new URL(CONTENT_MANIFEST_FILE, contentCacheDir); - const cache = contentCacheDir; - const cacheTmp = new URL('./.tmp/', cache); - let oldManifest = createContentManifest(); + const buildCacheDir = new URL(BUILD_CACHE_DIR, cacheDir); + // const contentManifestFile = new URL(BUILD_MANIFEST_FILE, buildCacheDir); + // let oldManifest = createContentManifest(); let newManifest = createContentManifest(); - let entries: ContentEntries; + // let entries: ContentEntries; let injectedEmptyFile = false; - if (fsMod.existsSync(contentManifestFile)) { - try { - const data = fsMod.readFileSync(contentManifestFile, { encoding: 'utf8' }); - oldManifest = JSON.parse(data); - internals.cachedClientEntries = oldManifest.clientEntries; - } catch {} - } + // if (fsMod.existsSync(contentManifestFile)) { + // try { + // const data = fsMod.readFileSync(contentManifestFile, { encoding: 'utf8' }); + // oldManifest = JSON.parse(data); + // internals.cachedClientEntries = oldManifest.clientEntries; + // } catch {} + // } return { name: '@astro/plugin-build-content', async options(options) { let newOptions = Object.assign({}, options); - newManifest = await generateContentManifest(opts, lookupMap); - entries = getEntriesFromManifests(oldManifest, newManifest); - - // Of the cached entries, these ones need to be rebuilt - for (const { type, entry } of entries.buildFromSource) { - const fileURL = encodeURI(joinPaths(opts.settings.config.root.toString(), entry)); - const input = fileURLToPath(fileURL); - // Adds `/src/content/blog/post-1.md?astroContentCollectionEntry` as a top-level input - const inputs = [`${input}?${collectionTypeToFlag(type)}`]; - if (type === 'content') { - // Content entries also need to include the version with the RENDER flag - inputs.push(`${input}?${CONTENT_RENDER_FLAG}`); + // newManifest = await generateContentManifest(opts, lookupMap); + // entries = getEntriesFromManifests(oldManifest, newManifest); + const inputs: string[] = []; + + // newOptions = addRollupInput(newOptions, inputs); + for (const { type, entries: collectionEntries } of Object.values(lookupMap)) { + for (const entrypoint of Object.values(collectionEntries)) { + const fileURL = encodeURI(joinPaths(opts.settings.config.root.toString(), entrypoint)); + const input = fileURLToPath(fileURL); + inputs.push(`${input}?${collectionTypeToFlag(type)}`); + if (type === 'content') { + // Content entries also need to include the version with the RENDER flag + inputs.push(`${input}?${CONTENT_RENDER_FLAG}`); + } } - newOptions = addRollupInput(newOptions, inputs); - } - // Restores cached chunks from the previous build - if (fsMod.existsSync(cachedChunks)) { - await copyFiles(cachedChunks, distChunks, true); - } - // If nothing needs to be rebuilt, we inject a fake entrypoint to appease Rollup - if (entries.buildFromSource.length === 0) { - newOptions = addRollupInput(newOptions, [virtualEmptyModuleId]); - injectedEmptyFile = true; } + newOptions = addRollupInput(newOptions, inputs); return newOptions; + + // // Of the cached entries, these ones need to be rebuilt + // for (const { type, entry } of entries.buildFromSource) { + // const fileURL = encodeURI(joinPaths(opts.settings.config.root.toString(), entry)); + // const input = fileURLToPath(fileURL); + // // Adds `/src/content/blog/post-1.md?astroContentCollectionEntry` as a top-level input + // const inputs = [`${input}?${collectionTypeToFlag(type)}`]; + // if (type === 'content') { + // // Content entries also need to include the version with the RENDER flag + // inputs.push(`${input}?${CONTENT_RENDER_FLAG}`); + // } + // newOptions = addRollupInput(newOptions, inputs); + // } + // // Restores cached chunks from the previous build + // if (fsMod.existsSync(buildCacheDir)) { + // await copyFiles(buildCacheDir, distRoot, true); + // } + // // If nothing needs to be rebuilt, we inject a fake entrypoint to appease Rollup + // if (entries.buildFromSource.length === 0) { + // newOptions = addRollupInput(newOptions, [virtualEmptyModuleId]); + // injectedEmptyFile = true; + // } + // return newOptions; }, outputOptions(outputOptions) { @@ -160,7 +334,7 @@ function vitePluginContent( async generateBundle(_options, bundle) { const code = await generateContentEntryFile({ settings: opts.settings, - fs: fsMod, + fs: fs, lookupMap, IS_DEV: false, IS_SERVER: false, @@ -185,7 +359,6 @@ function vitePluginContent( // in case they aren't referenced _outside_ of the cached content. // We can use this info in the manifest to run a proper client build again. const clientComponents = new Set([ - ...oldManifest.clientEntries, ...internals.discoveredHydratedComponents.keys(), ...internals.discoveredClientOnlyComponents.keys(), ...internals.discoveredScripts, @@ -193,25 +366,15 @@ function vitePluginContent( // Likewise, these are server modules that might not be referenced // once the cached items are excluded from the build process const serverComponents = new Set([ - ...oldManifest.serverEntries, ...internals.discoveredHydratedComponents.keys(), ]); newManifest.serverEntries = Array.from(serverComponents); newManifest.clientEntries = Array.from(clientComponents); - await fsMod.promises.mkdir(contentCacheDir, { recursive: true }); - await fsMod.promises.writeFile(contentManifestFile, JSON.stringify(newManifest), { - encoding: 'utf8', - }); - - const cacheExists = fsMod.existsSync(cache); - fsMod.mkdirSync(cache, { recursive: true }); - await fsMod.promises.mkdir(cacheTmp, { recursive: true }); - await copyFiles(distContentRoot, cacheTmp, true); - if (cacheExists) { - await copyFiles(contentCacheDir, distContentRoot, false); - } - await copyFiles(cacheTmp, contentCacheDir); - await fsMod.promises.rm(cacheTmp, { recursive: true, force: true }); + // await fsMod.promises.mkdir(new URL('./', contentManifestFile), { recursive: true }); + // await fsMod.promises.writeFile(contentManifestFile, JSON.stringify(newManifest), { + // encoding: 'utf8', + // }); + await copyFiles(distRoot, buildCacheDir); }, }; } @@ -277,27 +440,23 @@ async function generateContentManifest( const limit = pLimit(10); const promises: Promise[] = []; - for (const [collection, { type, entries }] of Object.entries(lookupMap)) { - for (const entry of Object.values(entries)) { - const key: ContentManifestKey = { collection, type, entry }; - const fileURL = new URL(encodeURI(joinPaths(opts.settings.config.root.toString(), entry))); - promises.push( - limit(async () => { - const data = await fsMod.promises.readFile(fileURL, { encoding: 'utf8' }); - manifest.entries.push([key, checksum(data)]); - }) - ); - } - } + // for (const [collection, { type, entries }] of Object.entries(lookupMap)) { + // for (const entry of Object.values(entries)) { + // const key: ContentManifestKey = { collection, type, entry }; + // const fileURL = new URL(encodeURI(joinPaths(opts.settings.config.root.toString(), entry))); + // promises.push( + // limit(async () => { + // const data = await fsMod.promises.readFile(fileURL, { encoding: 'utf8' }); + // // manifest.entries.push([key, checksum(data)]); + // }) + // ); + // } + // } await Promise.all(promises); return manifest; } -function checksum(data: string): string { - return createHash('sha1').update(data).digest('base64'); -} - function collectionTypeToFlag(type: 'content' | 'data') { const name = type[0].toUpperCase() + type.slice(1); return `astro${name}CollectionEntry`; @@ -307,8 +466,8 @@ export function pluginContent( opts: StaticBuildOptions, internals: BuildInternals ): AstroBuildPlugin { - const cachedChunks = new URL('./chunks/', opts.settings.config.cacheDir); - const distChunks = new URL('./chunks/', opts.settings.config.outDir); + const buildCacheDir = new URL(BUILD_CACHE_DIR, opts.settings.config.cacheDir); + const outDir = opts.settings.config.outDir; return { targets: ['server'], @@ -320,24 +479,15 @@ export function pluginContent( if (isServerLikeOutput(opts.settings.config)) { return { vitePlugin: undefined }; } + if (fs.existsSync(buildCacheDir)) { + await copyFiles(buildCacheDir, outDir, true); + } - const lookupMap = await generateLookupMap({ settings: opts.settings, fs: fsMod }); + const lookupMap = await generateLookupMap({ settings: opts.settings, fs: fs }); return { - vitePlugin: vitePluginContent(opts, lookupMap, internals), + vitePlugin: [vitePluginModuleGraph(opts, lookupMap), vitePluginContent(opts, lookupMap, internals)], }; - }, - - async 'build:post'() { - if (!opts.settings.config.experimental.contentCollectionCache) { - return; - } - if (isServerLikeOutput(opts.settings.config)) { - return; - } - if (fsMod.existsSync(distChunks)) { - await copyFiles(distChunks, cachedChunks, true); - } - }, + } }, }; } diff --git a/packages/astro/src/core/build/static-build.ts b/packages/astro/src/core/build/static-build.ts index 81dcdb4a00..5fcd66eff7 100644 --- a/packages/astro/src/core/build/static-build.ts +++ b/packages/astro/src/core/build/static-build.ts @@ -145,7 +145,6 @@ async function ssrBuild( input: Set, container: AstroBuildPluginContainer ) { - const buildID = Date.now().toString(); const { allPages, settings, viteConfig } = opts; const ssr = isServerLikeOutput(settings.config); const out = getOutputDirectory(settings.config); @@ -185,7 +184,6 @@ async function ssrBuild( let suffix = '_[hash].mjs'; if (isContentCache) { - prefix += `${buildID}/`; suffix = '.mjs'; }