From 73a50c9ce217aa392905d968e14bffb12146f399 Mon Sep 17 00:00:00 2001 From: "Fred K. Schott" Date: Sat, 29 Jan 2022 23:15:15 -0800 Subject: [PATCH] wip --- docs/src/pages/[...slug].astro | 27 +++++-- packages/astro/src/@types/astro.ts | 31 ++++++-- packages/astro/src/core/build/page-data.ts | 12 +-- packages/astro/src/core/ssr/index.ts | 15 ++-- packages/astro/src/core/ssr/route-cache.ts | 55 +++++++------- packages/astro/src/core/ssr/routing.ts | 5 +- packages/astro/src/runtime/server/content.ts | 73 +++++++++++++++++++ packages/astro/src/runtime/server/index.ts | 4 +- .../src/vite-plugin-astro-server/index.ts | 27 ++++--- 9 files changed, 182 insertions(+), 67 deletions(-) create mode 100644 packages/astro/src/runtime/server/content.ts diff --git a/docs/src/pages/[...slug].astro b/docs/src/pages/[...slug].astro index 613e2b3d5f..3ad93ba53d 100644 --- a/docs/src/pages/[...slug].astro +++ b/docs/src/pages/[...slug].astro @@ -1,19 +1,30 @@ --- -export async function getStaticPaths() { + +// NOTE: This should be setup(), however the compiler currently only hoists +// getStaticPaths(). So we're using getStaticPaths() for the moment. +export async function getStaticPaths({content, buildStaticPaths, rss}) { + // content() instructions: + // To fetch via glob: + // console.log(await content('./en/**/*.md')); + // To fetch via glob + filter (filtering improves HMR): + // console.log(await content('./en/**/*.md', (f) => f.file.includes('getting-started'))); + // get english pages that moved from `/` to `/en/` - const englishPages = Astro.fetchContent('./en/**/*.md'); + const englishPages = await content('./en/**/*.md'); // add pages that are `*.astro` files as well const otherPages = [{ url: "/en/themes" }]; - return [...englishPages, ...otherPages].map((page) => ({ + buildStaticPaths([...englishPages, ...otherPages].map((page) => ({ params: { - slug: page.url.slice(4), + slug: page.url ? page.url.slice(4) : page.file.replace(/^.*src.pages.en/, ''), }, props: { - englishSlug: page.url, + englishSlug: page.url ? page.url : page.file.replace(/^.*src.pages/, ''), } - })); + }))); } --- - - + diff --git a/packages/astro/src/@types/astro.ts b/packages/astro/src/@types/astro.ts index abe62b13a8..0967b0b04e 100644 --- a/packages/astro/src/@types/astro.ts +++ b/packages/astro/src/@types/astro.ts @@ -173,9 +173,10 @@ export interface ComponentInstance { $$metadata: Metadata; default: AstroComponentFactory; css?: string[]; - getStaticPaths?: (options: GetStaticPathsOptions) => GetStaticPathsResult; + getStaticPaths?: (options: GetStaticPathsOptions) => void | GetStaticPathsResultKeyed; } + /** * Astro.fetchContent() result * Docs: https://docs.astro.build/reference/api-reference/#astrofetchcontent @@ -191,14 +192,28 @@ export type FetchContentResultBase = { url: URL; }; +export type NewFetchContentResult = { + file: string, + data: T, + Content: any, + content: { + headers: string[]; + source: string; + html: string; + }, +}; + export type GetHydrateCallback = () => Promise<(element: Element, innerHTML: string | null) => void>; /** * getStaticPaths() options * Docs: https://docs.astro.build/reference/api-reference/#getstaticpaths - */ export interface GetStaticPathsOptions { - paginate?: PaginateFunction; - rss?: (...args: any[]) => any; + */ +export interface GetStaticPathsOptions { + content(globStr: string, filter?: (data: any) => boolean): Promise[]>; + paginate: PaginateFunction; + buildStaticPaths: (paths: GetStaticPathsResultKeyed) => void; + rss: (...args: any[]) => any; } export type GetStaticPathsItem = { params: Params; props?: Props }; @@ -206,6 +221,12 @@ export type GetStaticPathsResult = GetStaticPathsItem[]; export type GetStaticPathsResultKeyed = GetStaticPathsResult & { keyed: Map; }; +export type GetStaticPathsResultObject = { + filePath: URL; + rss: undefined | RSS; + staticPaths: GetStaticPathsResultKeyed; // TODO: if setup(), this is optional + linkedContent: string[]; +}; export interface HydrateOptions { value?: string; @@ -341,7 +362,7 @@ export interface RouteData { type: 'page'; } -export type RouteCache = Record; +export type RouteCache = Record; export type RuntimeMode = 'development' | 'production'; diff --git a/packages/astro/src/core/build/page-data.ts b/packages/astro/src/core/build/page-data.ts index 43620ac4ef..3e159314f2 100644 --- a/packages/astro/src/core/build/page-data.ts +++ b/packages/astro/src/core/build/page-data.ts @@ -8,9 +8,8 @@ import * as colors from 'kleur/colors'; import { debug } from '../logger.js'; import { preload as ssrPreload } from '../ssr/index.js'; import { validateGetStaticPathsModule, validateGetStaticPathsResult } from '../ssr/routing.js'; -import { generatePaginateFunction } from '../ssr/paginate.js'; import { generateRssFunction } from '../ssr/rss.js'; -import { assignStaticPaths } from '../ssr/route-cache.js'; +import { callGetStaticPaths } from '../ssr/route-cache.js'; export interface CollectPagesDataOptions { astroConfig: AstroConfig; @@ -131,9 +130,12 @@ async function getStaticPathsForRoute(opts: CollectPagesDataOptions, route: Rout const mod = (await viteServer.ssrLoadModule(fileURLToPath(filePath))) as ComponentInstance; validateGetStaticPathsModule(mod); const rss = generateRssFunction(astroConfig.buildOptions.site, route); - await assignStaticPaths(routeCache, route, mod, rss.generator); - const staticPaths = routeCache[route.component]; - validateGetStaticPathsResult(staticPaths, logging); + routeCache[route.component] = routeCache[route.component] || await callGetStaticPaths(filePath, mod, route, (f) => viteServer.ssrLoadModule(f)); + if (routeCache[route.component].rss) { + rss.generator(routeCache[route.component].rss!); + } + validateGetStaticPathsResult(routeCache[route.component], logging); + const staticPaths = routeCache[route.component].staticPaths; return { paths: staticPaths.map((staticPath) => staticPath.params && route.generate(staticPath.params)).filter(Boolean), rss: rss.rss, diff --git a/packages/astro/src/core/ssr/index.ts b/packages/astro/src/core/ssr/index.ts index e7299d20a0..a1395e409e 100644 --- a/packages/astro/src/core/ssr/index.ts +++ b/packages/astro/src/core/ssr/index.ts @@ -27,7 +27,7 @@ import { injectTags } from './html.js'; import { generatePaginateFunction } from './paginate.js'; import { getParams, validateGetStaticPathsModule, validateGetStaticPathsResult } from './routing.js'; import { createResult } from './result.js'; -import { assignStaticPaths, ensureRouteCached, findPathItemByKey } from './route-cache.js'; +import { callGetStaticPaths, findPathItemByKey } from './route-cache.js'; const svelteStylesRE = /svelte\?svelte&type=style/; @@ -45,7 +45,7 @@ interface SSROptions { /** the web request (needed for dynamic routes) */ pathname: string; /** optional, in case we need to render something outside of a dev server */ - route?: RouteData; + route: RouteData; /** pass in route cache because SSR can’t manage cache-busting */ routeCache: RouteCache; /** Vite instance */ @@ -130,12 +130,11 @@ ${err.frame} export type ComponentPreload = [Renderer[], ComponentInstance]; -export async function preload({ astroConfig, filePath, viteServer }: SSROptions): Promise { +export async function preload({ astroConfig, route, routeCache, filePath, viteServer }: SSROptions): Promise { // Important: This needs to happen first, in case a renderer provides polyfills. const renderers = await resolveRenderers(viteServer, astroConfig); // Load the module from the Vite SSR Runtime. const mod = (await viteServer.ssrLoadModule(fileURLToPath(filePath))) as ComponentInstance; - return [renderers, mod]; } @@ -168,13 +167,13 @@ export async function getParamsAndProps({ validateGetStaticPathsModule(mod); } if (!routeCache[route.component]) { - await assignStaticPaths(routeCache, route, mod); + throw new Error(`[${route.component}] Internal error: route cache was empty, but expected to be full.`); } if (validate) { // This validation is expensive so we only want to do it in dev. validateGetStaticPathsResult(routeCache[route.component], logging); } - const staticPaths: GetStaticPathsResultKeyed = routeCache[route.component]; + const staticPaths: GetStaticPathsResultKeyed = routeCache[route.component].staticPaths; const paramsKey = JSON.stringify(params); const matchedStaticPath = findPathItemByKey(staticPaths, paramsKey, logging); if (!matchedStaticPath) { @@ -204,9 +203,9 @@ export async function render(renderers: Renderer[], mod: ComponentInstance, ssrO } } validateGetStaticPathsModule(mod); - await ensureRouteCached(routeCache, route, mod); + routeCache[route.component] = routeCache[route.component] || (await callGetStaticPaths(filePath, mod, route, (f) => viteServer.ssrLoadModule(f))); validateGetStaticPathsResult(routeCache[route.component], logging); - const routePathParams: GetStaticPathsResult = routeCache[route.component]; + const routePathParams: GetStaticPathsResult = routeCache[route.component].staticPaths; const matchedStaticPath = routePathParams.find(({ params: _params }) => JSON.stringify(_params) === JSON.stringify(params)); if (!matchedStaticPath) { throw new Error(`[getStaticPaths] route pattern matched, but no matching static path found. (${pathname})`); diff --git a/packages/astro/src/core/ssr/route-cache.ts b/packages/astro/src/core/ssr/route-cache.ts index 3ebfc4d7ef..04cc8d9633 100644 --- a/packages/astro/src/core/ssr/route-cache.ts +++ b/packages/astro/src/core/ssr/route-cache.ts @@ -1,47 +1,48 @@ -import type { ComponentInstance, GetStaticPathsItem, GetStaticPathsResult, GetStaticPathsResultKeyed, RouteCache, RouteData } from '../../@types/astro'; +import type { ComponentInstance, GetStaticPathsOptions, GetStaticPathsResult, RouteCache, RouteData, GetStaticPathsItem, GetStaticPathsResultKeyed, GetStaticPathsResultObject } from '../../@types/astro'; import type { LogOptions } from '../logger'; import { debug } from '../logger.js'; import { generatePaginateFunction } from '../ssr/paginate.js'; +import { createNewFetchContentFn } from '../../runtime/server/content.js'; -type RSSFn = (...args: any[]) => any; - -export async function callGetStaticPaths(mod: ComponentInstance, route: RouteData, rssFn?: RSSFn): Promise { - const staticPaths: GetStaticPathsResult = await ( +export async function callGetStaticPaths(filePath: URL, mod: ComponentInstance, route: RouteData, loadContent: (filePath: string) => Promise): Promise { + let result: GetStaticPathsResultObject = { + filePath, + rss: undefined, + // @ts-expect-error + staticPaths: undefined, + linkedContent: [], + }; + let staticPaths: GetStaticPathsResult = []; + const newFetchContentFn = createNewFetchContentFn(filePath, mod, loadContent); + await ( await mod.getStaticPaths!({ + content: async (globStr, filter) => { + const [fetchContentResults, linkedContentIds] = await newFetchContentFn(globStr, filter); + result.linkedContent.push(...linkedContentIds); + return fetchContentResults; + }, paginate: generatePaginateFunction(route), - rss: - rssFn || - (() => { - /* noop */ - }), + buildStaticPaths: (result) => { + staticPaths = result; + }, + rss: (fn) => { + result.rss = fn; + }, }) - ).flat(); + ); - const keyedStaticPaths = staticPaths as GetStaticPathsResultKeyed; + const keyedStaticPaths: GetStaticPathsResultKeyed = (staticPaths || []) as any; keyedStaticPaths.keyed = new Map(); for (const sp of keyedStaticPaths) { const paramsKey = JSON.stringify(sp.params); keyedStaticPaths.keyed.set(paramsKey, sp); } + result.staticPaths = keyedStaticPaths; - return keyedStaticPaths; + return result; } -export async function assignStaticPaths(routeCache: RouteCache, route: RouteData, mod: ComponentInstance, rssFn?: RSSFn): Promise { - const staticPaths = await callGetStaticPaths(mod, route, rssFn); - routeCache[route.component] = staticPaths; -} - -export async function ensureRouteCached(routeCache: RouteCache, route: RouteData, mod: ComponentInstance, rssFn?: RSSFn): Promise { - if (!routeCache[route.component]) { - const staticPaths = await callGetStaticPaths(mod, route, rssFn); - routeCache[route.component] = staticPaths; - return staticPaths; - } else { - return routeCache[route.component]; - } -} export function findPathItemByKey(staticPaths: GetStaticPathsResultKeyed, paramsKey: string, logging: LogOptions) { let matchedStaticPath = staticPaths.keyed.get(paramsKey); diff --git a/packages/astro/src/core/ssr/routing.ts b/packages/astro/src/core/ssr/routing.ts index b6a2cf1a4d..375fe8e8d9 100644 --- a/packages/astro/src/core/ssr/routing.ts +++ b/packages/astro/src/core/ssr/routing.ts @@ -1,4 +1,4 @@ -import type { AstroConfig, ComponentInstance, GetStaticPathsResult, ManifestData, Params, RouteData } from '../../@types/astro'; +import type { AstroConfig, ComponentInstance, GetStaticPathsResult, GetStaticPathsResultObject, ManifestData, Params, RouteData } from '../../@types/astro'; import type { LogOptions } from '../logger'; import fs from 'fs'; @@ -45,7 +45,8 @@ export function validateGetStaticPathsModule(mod: ComponentInstance) { } /** Throw error for malformed getStaticPaths() response */ -export function validateGetStaticPathsResult(result: GetStaticPathsResult, logging: LogOptions) { +export function validateGetStaticPathsResult(resultObj: GetStaticPathsResultObject, logging: LogOptions) { + const result = resultObj.staticPaths; if (!Array.isArray(result)) { throw new Error(`[getStaticPaths] invalid return value. Expected an array of path objects, but got \`${JSON.stringify(result)}\`.`); } diff --git a/packages/astro/src/runtime/server/content.ts b/packages/astro/src/runtime/server/content.ts new file mode 100644 index 0000000000..09ce21227b --- /dev/null +++ b/packages/astro/src/runtime/server/content.ts @@ -0,0 +1,73 @@ +import type { AstroComponentMetadata, Renderer, AstroGlobalPartial, SSRResult, SSRElement, GetStaticPathsOptions, ComponentInstance } from '../../@types/astro'; +import glob from 'fast-glob'; +import { pathToFileURL, fileURLToPath } from 'url'; +import { dirname } from 'path'; + +/** Create the Astro.content() runtime function. */ +export function createNewFetchContentFn(fileUrl: URL, mod: ComponentInstance, loadContent: (filePath: string) => Promise): any { + const fetchResults: string[][] = []; + const filePath = fileURLToPath(fileUrl); + const cwd = dirname(filePath); + console.log(filePath, cwd, mod); + return (async (pattern: string, filter?: (data: any) => boolean) => { + const files = await glob(pattern, { + cwd, + absolute: true, + // Ignore node_modules by default unless explicitly indicated in the pattern + ignore: /(^|\/)node_modules\//.test(pattern) ? [] : ['**/node_modules/**'], + }); + + // for each file, import it and pass it to filter + const modules = + (await Promise.all(files.map((f) => loadContent(f)))) + .map((mod, i) => { + // Only return Markdown files for now. + if (!mod.frontmatter) { + return; + } + const filePath = files[i]; + return { + file: filePath, + data: mod.frontmatter, + Content: mod.default, + content: mod.metadata, + // TODO: figure out if we want to do the url property + // We would need to use some Vite resolution logic, I think + // but we may not even want to bring this along + // url: urlSpec.includes('/pages/') ? urlSpec.replace(/^.*\/pages\//, site.pathname).replace(/(\/index)?\.md$/, '') : undefined, + }; + }) + .filter(Boolean) + .filter(filter || (() => true)); + console.log(files, modules); + + return [modules as any[], files]; + }); + + // PREVIOUS CODE - to be deleted before merging + // let allEntries = [...Object.entries(importMetaGlobResult)]; + // if (allEntries.length === 0) { + // throw new Error(`[${url.pathname}] Astro.fetchContent() no matches found.`); + // } + // return allEntries + // .map(([spec, mod]) => { + // // Only return Markdown files for now. + // if (!mod.frontmatter) { + // return; + // } + // const urlSpec = new URL(spec, url).pathname; + // return { + // ...mod.frontmatter, + // Content: mod.default, + // content: mod.metadata, + // file: new URL(spec, url), + // url: urlSpec.includes('/pages/') ? urlSpec.replace(/^.*\/pages\//, site.pathname).replace(/(\/index)?\.md$/, '') : undefined, + // }; + // }) + // .filter(Boolean); + // }; + // // This has to be cast because the type of fetchContent is the type of the function + // // that receives the import.meta.glob result, but the user is using it as + // // another type. + // return fetchContent as unknown as AstroGlobalPartial['fetchContent']; +} diff --git a/packages/astro/src/runtime/server/index.ts b/packages/astro/src/runtime/server/index.ts index 894f9867d8..375f3a4387 100644 --- a/packages/astro/src/runtime/server/index.ts +++ b/packages/astro/src/runtime/server/index.ts @@ -1,5 +1,4 @@ -import type { AstroComponentMetadata, Renderer } from '../../@types/astro'; -import type { AstroGlobalPartial, SSRResult, SSRElement } from '../../@types/astro'; +import type { AstroComponentMetadata, Renderer, AstroGlobalPartial, SSRResult, SSRElement } from '../../@types/astro'; import shorthash from 'shorthash'; import { extractDirectives, generateHydrateScript } from './hydration.js'; @@ -264,6 +263,7 @@ If you're still stuck, please open an issue on GitHub or join us at https://astr /** Create the Astro.fetchContent() runtime function. */ function createFetchContentFn(url: URL, site: URL) { const fetchContent = (importMetaGlobResult: Record) => { + // let allEntries = [...Object.entries(importMetaGlobResult)]; if (allEntries.length === 0) { throw new Error(`[${url.pathname}] Astro.fetchContent() no matches found.`); diff --git a/packages/astro/src/vite-plugin-astro-server/index.ts b/packages/astro/src/vite-plugin-astro-server/index.ts index 701457f937..16c871a969 100644 --- a/packages/astro/src/vite-plugin-astro-server/index.ts +++ b/packages/astro/src/vite-plugin-astro-server/index.ts @@ -128,20 +128,27 @@ export default function createPlugin({ config, logging }: AstroPluginOptions): v const pagesDirectory = fileURLToPath(config.pages); let routeCache: RouteCache = {}; let manifest: ManifestData = createRouteManifest({ config: config }, logging); - /** rebuild the route cache + manifest if the changed file impacts routing. */ - function rebuildManifestIfNeeded(file: string) { - if (file.startsWith(pagesDirectory)) { - routeCache = {}; + /** rebuild the route cache + manifest, as needed. */ + function rebuildManifest(needsManifestRebuild: boolean, file: string) { + for (const [routeComponent, routeCacheEntry] of Object.entries(routeCache)) { + if (fileURLToPath(routeCacheEntry.filePath) === file || routeCacheEntry.linkedContent.includes(file)) { + delete routeCache[routeComponent]; + // Vite doesn't give us a way to trigger a page-specific HMR event manually. + // Instead, just trigger a full page reload, which should be fine for most users. + // The routeCache entry deletion prevents stale pages from reloading. + viteServer.ws.send({ + type: 'full-reload', + }); + } + } + if (needsManifestRebuild) { manifest = createRouteManifest({ config: config }, logging); } } // Rebuild route manifest on file change, if needed. - viteServer.watcher.on('add', rebuildManifestIfNeeded); - viteServer.watcher.on('unlink', rebuildManifestIfNeeded); - // No need to rebuild routes on content-only changes. - // However, we DO want to clear the cache in case - // the change caused a getStaticPaths() return to change. - viteServer.watcher.on('change', () => (routeCache = {})); + viteServer.watcher.on('add', rebuildManifest.bind(null, true)); + viteServer.watcher.on('unlink', rebuildManifest.bind(null, true)); + viteServer.watcher.on('change', rebuildManifest.bind(null, false)); return () => { removeViteHttpMiddleware(viteServer.middlewares); viteServer.middlewares.use(async (req, res) => {