Infrastructure/astro
0
Fork 0
mirror of https://github.com/withastro/astro.git synced 2024-12-16 21:46:22 -05:00
Code Issues Activity

wip: content collections refactor

Browse source
This commit is contained in:
Nate Moore 2023-10-17 13:27:19 -05:00
parent 4cc3e37238
commit 6b094e800a
8 changed files with 89 additions and 49 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
packages/astro/src/content/vite-plugin-content-virtual-mod.ts
View file

@ -6,7 +6,7 @@ import pLimit from 'p-limit';
import type { Plugin } from 'vite';
import type { AstroSettings } from '../@types/astro.js';
import { AstroError, AstroErrorData } from '../core/errors/index.js';
import { appendForwardSlash, joinPaths, removeFileExtension, removeLeadingForwardSlash, slash } from '../core/path.js';
import { appendForwardSlash, removeFileExtension, removeLeadingForwardSlash, slash } from '../core/path.js';
import { rootRelativePath } from '../core/util.js';
import { VIRTUAL_MODULE_ID } from './consts.js';
import {
@ -70,7 +70,7 @@ export async function generateContentEntryFile({
const contentEntryExts = [...contentEntryConfigByExt.keys()];
const dataEntryExts = getDataEntryExts(settings);
const [contentEntryGlobResult, dataEntryGlobResult, renderEntryGlobResult] = await Promise.all([contentEntryExts, dataEntryExts, contentEntryExts].map((exts, i) => getStringifiedGlobResult(settings, exts, i === 2 ? '.render.mjs' : undefined)));
const [contentEntryGlobResult, dataEntryGlobResult, renderEntryGlobResult] = await Promise.all([contentEntryExts, dataEntryExts, contentEntryExts].map((exts, i) => getStringifiedGlobResult(settings, exts, i === 2 ? '.entry.mjs' : undefined)));
const virtualModContents = fs
.readFileSync(contentPaths.virtualModTemplate, 'utf-8')

9
packages/astro/src/core/build/buildPipeline.ts
View file

@ -26,6 +26,7 @@ export class BuildPipeline extends Pipeline {
manifest: SSRManifest
) {
const ssr = isServerLikeOutput(staticBuildOptions.settings.config);
const resolveCache = new Map<string, string>();
super(
createEnvironment({
adapterName: manifest.adapterName,
@ -35,16 +36,22 @@ export class BuildPipeline extends Pipeline {
clientDirectives: manifest.clientDirectives,
compressHTML: manifest.compressHTML,
async resolve(specifier: string) {
if (resolveCache.has(specifier)) {
return resolveCache.get(specifier)!;
}
const hashedFilePath = manifest.entryModules[specifier];
if (typeof hashedFilePath !== 'string' || hashedFilePath === '') {
// If no "astro:scripts/before-hydration.js" script exists in the build,
// then we can assume that no before-hydration scripts are needed.
if (specifier === BEFORE_HYDRATION_SCRIPT_ID) {
resolveCache.set(specifier, '');
return '';
}
throw new Error(`Cannot find the built path for ${specifier}`);
}
return createAssetLink(hashedFilePath, manifest.base, manifest.assetsPrefix);
const assetLink = createAssetLink(hashedFilePath, manifest.base, manifest.assetsPrefix);
resolveCache.set(specifier, assetLink);
return assetLink;
},
routeCache: staticBuildOptions.routeCache,
site: manifest.site,

5
packages/astro/src/core/build/internal.ts
View file

@ -30,6 +30,10 @@ export interface BuildInternals {
// Used to render pages with the correct specifiers.
entrySpecifierToBundleMap: Map<string, string>;
// A mapping of all specifiers referenced by the ssrBuild to their final location.
// This allows the contentBuild to externalize any shared assets!
serverModulesToOutputFile: Map<string, URL>;
/**
* A map to get a specific page's bundled output file.
*/
@ -110,6 +114,7 @@ export function createBuildInternals(): BuildInternals {
hoistedScriptIdToHoistedMap,
hoistedScriptIdToPagesMap,
entrySpecifierToBundleMap: new Map<string, string>(),
serverModulesToOutputFile: new Map<string, URL>(),
pageToBundleMap: new Map<string, string>(),
pagesByComponent: new Map(),
pageOptionsByPage: new Map(),

2
packages/astro/src/core/build/plugins/plugin-analyzer.ts
View file

@ -330,7 +330,7 @@ export function pluginAnalyzer(
internals: BuildInternals
): AstroBuildPlugin {
return {
targets: ['server'],
targets: ['server', 'content'],
hooks: {
'build:before': () => {
return {

51
packages/astro/src/core/build/plugins/plugin-content.ts
View file

@ -9,8 +9,11 @@ import { joinPaths } from '../../path.js';
import { fileURLToPath } from 'node:url';
import type { ContentLookupMap } from '../../../content/utils.js';
import { CONTENT_RENDER_FLAG } from '../../../content/consts.js';
import glob from 'fast-glob';
import { dirname } from 'node:path';
function vitePluginContent(opts: StaticBuildOptions, lookupMap: ContentLookupMap, _internals: BuildInternals): VitePlugin {
function vitePluginContent(opts: StaticBuildOptions, lookupMap: ContentLookupMap): VitePlugin {
return {
name: '@astro/plugin-build-content',
@ -33,10 +36,6 @@ function vitePluginContent(opts: StaticBuildOptions, lookupMap: ContentLookupMap
return newOptions;
},
resolveId(id) {
console.log(id);
},
async generateBundle() {
const content = await generateContentEntryFile({ settings: opts.settings, fs: fsMod, lookupMap });
this.emitFile({
@ -44,6 +43,42 @@ function vitePluginContent(opts: StaticBuildOptions, lookupMap: ContentLookupMap
code: content,
fileName: 'content/index.mjs'
})
},
async writeBundle(options, bundle) {
const dist = options.dir!
const cache = dist.replace('dist', 'node_modules/.astro/content-cache')
const callbacks: (() => void)[] = [];
if (fsMod.existsSync(cache)) {
const cachedFiles = await glob('**/*', {
cwd: cache,
onlyFiles: true
})
for (const file of cachedFiles) {
const filePath = joinPaths(dist, file);
const cachePath = joinPaths(cache, file);
callbacks.push(() => {
fsMod.mkdirSync(dirname(filePath), { recursive: true })
fsMod.copyFileSync(cachePath, filePath, fsMod.constants.COPYFILE_FICLONE)
})
}
} else {
fsMod.mkdirSync(cache, { recursive: true })
for (const [file, chunk] of Object.entries(bundle)) {
const cachePath = joinPaths(cache, file);
callbacks.push(() => {
fsMod.mkdirSync(dirname(cachePath), { recursive: true })
if (chunk.type === 'chunk') {
fsMod.writeFileSync(cachePath, chunk.code, { encoding: 'utf8' })
} else {
fsMod.writeFileSync(cachePath, chunk.source)
}
})
}
}
for (const cb of callbacks) {
cb();
}
}
};
}
@ -53,7 +88,7 @@ function collectionTypeToFlag(type: 'content' | 'data') {
return `astro${name}CollectionEntry`
}
export function pluginContent(opts: StaticBuildOptions, _internals: BuildInternals): AstroBuildPlugin {
export function pluginContent(opts: StaticBuildOptions, internals: BuildInternals): AstroBuildPlugin {
return {
targets: ['content'],
hooks: {
@ -62,9 +97,9 @@ export function pluginContent(opts: StaticBuildOptions, _internals: BuildInterna
const lookupMap = await generateLookupMap({ settings: opts.settings, fs: fsMod });
return {
vitePlugin: vitePluginContent(opts, lookupMap),
vitePlugin: vitePluginContent(opts, lookupMap, internals),
};
},
}
},
};
}

7
packages/astro/src/core/build/plugins/plugin-external.ts
View file

@ -13,8 +13,7 @@ export function vitePluginExternalize(): VitePlugin {
// Ensure that `astro:content` is treated as external and rewritten to the final entrypoint
if (id === MODULE_ID) {
return { id: VIRTUAL_MODULE_ID, external: true }
}
return null;
}
},
renderChunk(code, chunk) {
if (chunk.imports.find(name => name === VIRTUAL_MODULE_ID)) {
@ -22,7 +21,7 @@ export function vitePluginExternalize(): VitePlugin {
const steps = removeLeadingForwardSlash(slash(chunk.fileName)).split('/').length - 1;
const prefix = '../'.repeat(steps) || './';
// dist/content/index.mjs is generated by the "content" build
return code.replace(VIRTUAL_MODULE_ID, `${prefix}content/index.mjs`);
return code.replaceAll(VIRTUAL_MODULE_ID, `${prefix}content/index.mjs`);
}
}
};
@ -30,7 +29,7 @@ export function vitePluginExternalize(): VitePlugin {
export function pluginExternalize(): AstroBuildPlugin {
return {
targets: ['server'],
targets: ['server', 'content'],
hooks: {
'build:before': () => {
return {

2
packages/astro/src/core/build/plugins/plugin-internals.ts
View file

@ -61,7 +61,7 @@ export function vitePluginInternals(input: Set<string>, internals: BuildInternal
export function pluginInternals(internals: BuildInternals): AstroBuildPlugin {
return {
targets: ['client', 'server'],
targets: ['client', 'server', 'content'],
hooks: {
'build:before': ({ input }) => {
return {

58
packages/astro/src/core/build/static-build.ts
View file

@ -32,8 +32,7 @@ import { RESOLVED_SPLIT_MODULE_ID, RESOLVED_SSR_VIRTUAL_MODULE_ID } from './plug
import { ASTRO_PAGE_EXTENSION_POST_PATTERN } from './plugins/util.js';
import type { PageBuildData, StaticBuildOptions } from './types.js';
import { getTimeStat } from './util.js';
import { hasContentFlag } from '../../content/utils.js';
import { CONTENT_FLAGS, CONTENT_RENDER_FLAG, PROPAGATED_ASSET_FLAG } from '../../content/consts.js';
import { PROPAGATED_ASSET_FLAG } from '../../content/consts.js';
export async function viteBuild(opts: StaticBuildOptions) {
const { allPages, settings } = opts;
@ -79,25 +78,18 @@ export async function viteBuild(opts: StaticBuildOptions) {
const container = createPluginContainer(opts, internals);
registerAllPlugins(container);
let buildContent = async () => {
const contentTime = performance.now();
opts.logger.info('content', `Building collections...`);
await contentBuild(opts, internals, new Set(), container);
opts.logger.info('content', dim(`Completed in ${getTimeStat(contentTime, performance.now())}.`));
}
// Build your project (SSR application code, assets, client JS, etc.)
const ssrTime = performance.now();
opts.logger.info('build', `Building ${settings.config.output} entrypoints...`);
const ssrOutput = await ssrBuild(opts, internals, pageInput, container);
opts.logger.info('build', dim(`Completed in ${getTimeStat(ssrTime, performance.now())}.`));
settings.timer.end('SSR build');
let ssrOutput: any;
let buildServer = async () => {
// Build your project (SSR application code, assets, client JS, etc.)
const ssrTime = performance.now();
opts.logger.info('build', `Building ${settings.config.output} entrypoints...`);
ssrOutput = await ssrBuild(opts, internals, pageInput, container);
opts.logger.info('build', dim(`Completed in ${getTimeStat(ssrTime, performance.now())}.`));
settings.timer.end('SSR build');
}
await Promise.all([buildContent(), buildServer()]);
// Build `astro:content` collections
const contentTime = performance.now();
opts.logger.info('content', `Building collections...`);
await contentBuild(opts, internals, new Set(), container);
opts.logger.info('content', dim(`Completed in ${getTimeStat(contentTime, performance.now())}.`));
settings.timer.start('Client build');
@ -185,6 +177,7 @@ async function ssrBuild(
...viteConfig.build?.rollupOptions,
input: [],
output: {
hoistTransitiveImports: false,
format: 'esm',
// Server chunks can't go in the assets (_astro) folder
// We need to keep these separate
@ -267,7 +260,6 @@ async function contentBuild(
container: AstroBuildPluginContainer
) {
const { settings, viteConfig } = opts;
const ssr = isServerLikeOutput(settings.config);
const out = getOutputDirectory(settings.config);
const { lastVitePlugins, vitePlugins } = await container.runBeforeHook('content', input);
@ -285,20 +277,24 @@ async function contentBuild(
emptyOutDir: false,
manifest: false,
outDir: fileURLToPath(out),
copyPublicDir: !ssr,
copyPublicDir: false,
rollupOptions: {
...viteConfig.build?.rollupOptions,
input: [],
output: {
hoistTransitiveImports: false,
format: 'esm',
chunkFileNames(info) {
if (info.moduleIds.length === 1) {
const url = pathToFileURL(info.moduleIds[0]);
const distRelative = url.toString().replace(settings.config.srcDir.toString(), '')
let entryFileName = removeFileExtension(distRelative);
return `${entryFileName}.render.mjs`;
const moduleId = info.moduleIds[0];
if (moduleId.includes('/content/docs/')) {
const url = pathToFileURL(info.moduleIds[0]);
const distRelative = url.toString().replace(settings.config.srcDir.toString(), '')
const entryFileName = removeFileExtension(distRelative);
return `${entryFileName}.render.mjs`;
}
}
return '[name]_[hash].mjs';
return '[name].mjs';
},
...viteConfig.build?.rollupOptions?.output,
entryFileNames(info) {
@ -309,13 +305,11 @@ async function contentBuild(
let entryFileName = removeFileExtension(distRelative);
if (flags[0] === PROPAGATED_ASSET_FLAG) {
entryFileName += `.assets`
} else if (flags[0] === CONTENT_RENDER_FLAG) {
entryFileName += '.render'
entryFileName += `.entry`
}
return `${entryFileName}.mjs`;
},
assetFileNames: `${settings.config.build.assets}/[name].[extname]`,
assetFileNames: `${settings.config.build.assets}/[name].[hash][extname]`,
},
},
ssr: true,
@ -596,7 +590,7 @@ export function makeAstroPageEntryPointFileName(
* 2. We split the file path using the file system separator and attempt to retrieve the last entry
* 3. The last entry should be the file
* 4. We prepend the file name with `entry.`
* 5. We built the file path again, using the new entry built in the previous step
* 5. We built the file path again, using the new en3built in the previous step
*
* @param facadeModuleId
* @param opts