wip: content collections refactor

packages/astro/src/content/vite-plugin-content-virtual-mod.ts

@@ -6,7 +6,7 @@ import pLimit from 'p-limit';
 import type { Plugin } from 'vite';
 import type { AstroSettings } from '../@types/astro.js';
 import { AstroError, AstroErrorData } from '../core/errors/index.js';
-import { appendForwardSlash, joinPaths, removeFileExtension, removeLeadingForwardSlash, slash } from '../core/path.js';
+import { appendForwardSlash, removeFileExtension, removeLeadingForwardSlash, slash } from '../core/path.js';
 import { rootRelativePath } from '../core/util.js';
 import { VIRTUAL_MODULE_ID } from './consts.js';
 import {
@@ -70,7 +70,7 @@ export async function generateContentEntryFile({
 	const contentEntryExts = [...contentEntryConfigByExt.keys()];
 	const dataEntryExts = getDataEntryExts(settings);
 
-	const [contentEntryGlobResult, dataEntryGlobResult, renderEntryGlobResult] = await Promise.all([contentEntryExts, dataEntryExts, contentEntryExts].map((exts, i) => getStringifiedGlobResult(settings, exts, i === 2 ? '.render.mjs' : undefined)));
+	const [contentEntryGlobResult, dataEntryGlobResult, renderEntryGlobResult] = await Promise.all([contentEntryExts, dataEntryExts, contentEntryExts].map((exts, i) => getStringifiedGlobResult(settings, exts, i === 2 ? '.entry.mjs' : undefined)));
 
 	const virtualModContents = fs
 		.readFileSync(contentPaths.virtualModTemplate, 'utf-8')

packages/astro/src/core/build/buildPipeline.ts

@@ -26,6 +26,7 @@ export class BuildPipeline extends Pipeline {
 		manifest: SSRManifest
 	) {
 		const ssr = isServerLikeOutput(staticBuildOptions.settings.config);
+		const resolveCache = new Map<string, string>();
 		super(
 			createEnvironment({
 				adapterName: manifest.adapterName,
@@ -35,16 +36,22 @@ export class BuildPipeline extends Pipeline {
 				clientDirectives: manifest.clientDirectives,
 				compressHTML: manifest.compressHTML,
 				async resolve(specifier: string) {
+					if (resolveCache.has(specifier)) {
+						return resolveCache.get(specifier)!;
+					}
 					const hashedFilePath = manifest.entryModules[specifier];
 					if (typeof hashedFilePath !== 'string' || hashedFilePath === '') {
 						// If no "astro:scripts/before-hydration.js" script exists in the build,
 						// then we can assume that no before-hydration scripts are needed.
 						if (specifier === BEFORE_HYDRATION_SCRIPT_ID) {
+							resolveCache.set(specifier, '');
 							return '';
 						}
 						throw new Error(`Cannot find the built path for ${specifier}`);
 					}
-					return createAssetLink(hashedFilePath, manifest.base, manifest.assetsPrefix);
+					const assetLink = createAssetLink(hashedFilePath, manifest.base, manifest.assetsPrefix);
+					resolveCache.set(specifier, assetLink);
+					return assetLink;
 				},
 				routeCache: staticBuildOptions.routeCache,
 				site: manifest.site,

packages/astro/src/core/build/internal.ts

@@ -30,6 +30,10 @@ export interface BuildInternals {
 	// Used to render pages with the correct specifiers.
 	entrySpecifierToBundleMap: Map<string, string>;
 
+	// A mapping of all specifiers referenced by the ssrBuild to their final location.
+	// This allows the contentBuild to externalize any shared assets!
+	serverModulesToOutputFile: Map<string, URL>;
+
 	/**
 	 * A map to get a specific page's bundled output file.
 	 */
@@ -110,6 +114,7 @@ export function createBuildInternals(): BuildInternals {
 		hoistedScriptIdToHoistedMap,
 		hoistedScriptIdToPagesMap,
 		entrySpecifierToBundleMap: new Map<string, string>(),
+		serverModulesToOutputFile: new Map<string, URL>(),
 		pageToBundleMap: new Map<string, string>(),
 		pagesByComponent: new Map(),
 		pageOptionsByPage: new Map(),

packages/astro/src/core/build/plugins/plugin-analyzer.ts

@@ -330,7 +330,7 @@ export function pluginAnalyzer(
 	internals: BuildInternals
 ): AstroBuildPlugin {
 	return {
-		targets: ['server'],
+		targets: ['server', 'content'],
 		hooks: {
 			'build:before': () => {
 				return {

packages/astro/src/core/build/plugins/plugin-content.ts

@@ -9,8 +9,11 @@ import { joinPaths } from '../../path.js';
 import { fileURLToPath } from 'node:url';
 import type { ContentLookupMap } from '../../../content/utils.js';
 import { CONTENT_RENDER_FLAG } from '../../../content/consts.js';
+import glob from 'fast-glob';
+import { dirname } from 'node:path';
+
+function vitePluginContent(opts: StaticBuildOptions, lookupMap: ContentLookupMap, _internals: BuildInternals): VitePlugin {
 
-function vitePluginContent(opts: StaticBuildOptions, lookupMap: ContentLookupMap): VitePlugin {
 	return {
 		name: '@astro/plugin-build-content',
 
@@ -33,10 +36,6 @@ function vitePluginContent(opts: StaticBuildOptions, lookupMap: ContentLookupMap
 			return newOptions;
 		},
 
-		resolveId(id) {
-			console.log(id);
-		},
-
 		async generateBundle() {
 			const content = await generateContentEntryFile({ settings: opts.settings, fs: fsMod, lookupMap });
 			this.emitFile({
@@ -44,6 +43,42 @@ function vitePluginContent(opts: StaticBuildOptions, lookupMap: ContentLookupMap
 				code: content,
 				fileName: 'content/index.mjs'
 			})
+		},
+
+		async writeBundle(options, bundle) {
+			const dist = options.dir!
+			const cache = dist.replace('dist', 'node_modules/.astro/content-cache')
+			const callbacks: (() => void)[] = [];
+			if (fsMod.existsSync(cache)) {
+				const cachedFiles = await glob('**/*', {
+					cwd: cache,
+					onlyFiles: true
+				})
+				for (const file of cachedFiles) {
+					const filePath = joinPaths(dist, file);
+					const cachePath = joinPaths(cache, file);
+					callbacks.push(() => {
+						fsMod.mkdirSync(dirname(filePath), { recursive: true })
+						fsMod.copyFileSync(cachePath, filePath, fsMod.constants.COPYFILE_FICLONE)
+					})
+				}
+			} else {
+				fsMod.mkdirSync(cache, { recursive: true })
+				for (const [file, chunk] of Object.entries(bundle)) {
+					const cachePath = joinPaths(cache, file);
+					callbacks.push(() => {
+						fsMod.mkdirSync(dirname(cachePath), { recursive: true })
+						if (chunk.type === 'chunk') {
+							fsMod.writeFileSync(cachePath, chunk.code, { encoding: 'utf8' })
+						} else {
+							fsMod.writeFileSync(cachePath, chunk.source)
+						}
+					})
+				}
+			}
+			for (const cb of callbacks) {
+				cb();
+			}
 		}
 	};
 }
@@ -53,7 +88,7 @@ function collectionTypeToFlag(type: 'content' | 'data') {
 	return `astro${name}CollectionEntry`
 }
 
-export function pluginContent(opts: StaticBuildOptions, _internals: BuildInternals): AstroBuildPlugin {
+export function pluginContent(opts: StaticBuildOptions, internals: BuildInternals): AstroBuildPlugin {
 	return {
 		targets: ['content'],
 		hooks: {
@@ -62,9 +97,9 @@ export function pluginContent(opts: StaticBuildOptions, _internals: BuildInterna
 				const lookupMap = await generateLookupMap({ settings: opts.settings, fs: fsMod });
 				
 				return {
-					vitePlugin: vitePluginContent(opts, lookupMap),
+					vitePlugin: vitePluginContent(opts, lookupMap, internals),
 				};
-			},
+			}
 		},
 	};
 }

packages/astro/src/core/build/plugins/plugin-external.ts

@@ -14,7 +14,6 @@ export function vitePluginExternalize(): VitePlugin {
 			if (id === MODULE_ID) {
 				return { id: VIRTUAL_MODULE_ID, external: true }
 			}
-            return null;
 		},
 		renderChunk(code, chunk) {
 			if (chunk.imports.find(name => name === VIRTUAL_MODULE_ID)) {
@@ -22,7 +21,7 @@ export function vitePluginExternalize(): VitePlugin {
 				const steps = removeLeadingForwardSlash(slash(chunk.fileName)).split('/').length - 1;
 				const prefix = '../'.repeat(steps) || './';
 				// dist/content/index.mjs is generated by the "content" build
-				return code.replace(VIRTUAL_MODULE_ID, `${prefix}content/index.mjs`);
+				return code.replaceAll(VIRTUAL_MODULE_ID, `${prefix}content/index.mjs`);
 			}
 		}
 	};
@@ -30,7 +29,7 @@ export function vitePluginExternalize(): VitePlugin {
 
 export function pluginExternalize(): AstroBuildPlugin {
 	return {
-		targets: ['server'],
+		targets: ['server', 'content'],
 		hooks: {
 			'build:before': () => {
 				return {

packages/astro/src/core/build/plugins/plugin-internals.ts

@@ -61,7 +61,7 @@ export function vitePluginInternals(input: Set<string>, internals: BuildInternal
 
 export function pluginInternals(internals: BuildInternals): AstroBuildPlugin {
 	return {
-		targets: ['client', 'server'],
+		targets: ['client', 'server', 'content'],
 		hooks: {
 			'build:before': ({ input }) => {
 				return {

packages/astro/src/core/build/static-build.ts

@@ -32,8 +32,7 @@ import { RESOLVED_SPLIT_MODULE_ID, RESOLVED_SSR_VIRTUAL_MODULE_ID } from './plug
 import { ASTRO_PAGE_EXTENSION_POST_PATTERN } from './plugins/util.js';
 import type { PageBuildData, StaticBuildOptions } from './types.js';
 import { getTimeStat } from './util.js';
-import { hasContentFlag } from '../../content/utils.js';
-import { CONTENT_FLAGS, CONTENT_RENDER_FLAG, PROPAGATED_ASSET_FLAG } from '../../content/consts.js';
+import { PROPAGATED_ASSET_FLAG } from '../../content/consts.js';
 
 export async function viteBuild(opts: StaticBuildOptions) {
 	const { allPages, settings } = opts;
@@ -79,25 +78,18 @@ export async function viteBuild(opts: StaticBuildOptions) {
 	const container = createPluginContainer(opts, internals);
 	registerAllPlugins(container);
 
-	let buildContent = async () => {
+	// Build your project (SSR application code, assets, client JS, etc.)
+	const ssrTime = performance.now();
+	opts.logger.info('build', `Building ${settings.config.output} entrypoints...`);
+	const ssrOutput = await ssrBuild(opts, internals, pageInput, container);
+	opts.logger.info('build', dim(`Completed in ${getTimeStat(ssrTime, performance.now())}.`));
+	settings.timer.end('SSR build');
+
+	// Build `astro:content` collections
 	const contentTime = performance.now();
 	opts.logger.info('content', `Building collections...`);
 	await contentBuild(opts, internals, new Set(), container);
 	opts.logger.info('content', dim(`Completed in ${getTimeStat(contentTime, performance.now())}.`));
-	}
-
-	let ssrOutput: any;
-	let buildServer = async () => {
-		// Build your project (SSR application code, assets, client JS, etc.)
-		const ssrTime = performance.now();
-		opts.logger.info('build', `Building ${settings.config.output} entrypoints...`);
-		ssrOutput = await ssrBuild(opts, internals, pageInput, container);
-		opts.logger.info('build', dim(`Completed in ${getTimeStat(ssrTime, performance.now())}.`));
-
-		settings.timer.end('SSR build');	
-	}
-
-	await Promise.all([buildContent(), buildServer()]);
 
 	settings.timer.start('Client build');
 
@@ -185,6 +177,7 @@ async function ssrBuild(
 				...viteConfig.build?.rollupOptions,
 				input: [],
 				output: {
+					hoistTransitiveImports: false,
 					format: 'esm',
 					// Server chunks can't go in the assets (_astro) folder
 					// We need to keep these separate
@@ -267,7 +260,6 @@ async function contentBuild(
 	container: AstroBuildPluginContainer
 ) {
 	const { settings, viteConfig } = opts;
-	const ssr = isServerLikeOutput(settings.config);
 	const out = getOutputDirectory(settings.config);
 	const { lastVitePlugins, vitePlugins } = await container.runBeforeHook('content', input);
 
@@ -285,20 +277,24 @@ async function contentBuild(
 			emptyOutDir: false,
 			manifest: false,
 			outDir: fileURLToPath(out),
-			copyPublicDir: !ssr,
+			copyPublicDir: false,
 			rollupOptions: {
 				...viteConfig.build?.rollupOptions,
 				input: [],
 				output: {
+					hoistTransitiveImports: false,
 					format: 'esm',
 					chunkFileNames(info) {
 						if (info.moduleIds.length === 1) {
+							const moduleId = info.moduleIds[0];
+							if (moduleId.includes('/content/docs/')) {
 								const url = pathToFileURL(info.moduleIds[0]);
 								const distRelative = url.toString().replace(settings.config.srcDir.toString(), '')
-							let entryFileName = removeFileExtension(distRelative);
+								const entryFileName = removeFileExtension(distRelative);
 								return `${entryFileName}.render.mjs`;
 							}
-						return '[name]_[hash].mjs';
+						}
+						return '[name].mjs';
 					},
 					...viteConfig.build?.rollupOptions?.output,
 					entryFileNames(info) {
@@ -309,13 +305,11 @@ async function contentBuild(
 
 						let entryFileName = removeFileExtension(distRelative);
 						if (flags[0] === PROPAGATED_ASSET_FLAG) {
-							entryFileName += `.assets`
-						} else if (flags[0] === CONTENT_RENDER_FLAG) {
-							entryFileName += '.render'
+							entryFileName += `.entry`
 						}
 						return `${entryFileName}.mjs`;
 					},
-					assetFileNames: `${settings.config.build.assets}/[name].[extname]`,
+					assetFileNames: `${settings.config.build.assets}/[name].[hash][extname]`,
 				},
 			},
 			ssr: true,
@@ -596,7 +590,7 @@ export function makeAstroPageEntryPointFileName(
  * 2. We split the file path using the file system separator and attempt to retrieve the last entry
  * 3. The last entry should be the file
  * 4. We prepend the file name with `entry.`
- * 5. We built the file path again, using the new entry built in the previous step
+ * 5. We built the file path again, using the new en3built in the previous step
  *
  * @param facadeModuleId
  * @param opts