move Astro.fetchContent to runtime API (#652)

.changeset/strange-kids-sing.md

@@ -0,0 +1,5 @@
+---
+'astro': patch
+---
+
+Remove custom Astro.fetchContent() glob implementation, use `import.meta.globEager` internally instead.

package.json

@@ -12,7 +12,7 @@
     "build": "yarn build:core",
     "build:all": "lerna run build",
     "build:one": "lerna run build --scope",
-    "build:core": "lerna run build --scope astro --scope @astrojs/parser --scope @astrojs/markdown-support --scope create-astro",
+    "build:core": "lerna run build --scope astro --scope @astrojs/parser --scope @astrojs/markdown-support",
     "build:vscode": "lerna run build --scope astro-languageserver --scope astro-vscode --scope @astrojs/parser",
     "dev:vscode": "lerna run dev --scope astro-languageserver --scope astro-vscode --scope @astrojs/parser --parallel --stream",
     "format": "prettier -w \"**/*.{js,jsx,ts,tsx,md,json}\"",

packages/astro/src/compiler/codegen/content.ts

@@ -1,58 +0,0 @@
-import path from 'path';
-import glob from 'tiny-glob/sync.js';
-import slash from 'slash';
-
-/**
- * Handling for import.meta.glob and import.meta.globEager
- */
-interface GlobOptions {
-  namespace: string;
-  filename: string;
-}
-
-interface GlobResult {
-  /** Array of import statements to inject */
-  imports: Set<string>;
-  /** Replace original code with */
-  code: string;
-}
-
-/** General glob handling */
-function globSearch(spec: string, { filename }: { filename: string }): string[] {
-  try {
-    const cwd = path.dirname(filename);
-    let found = glob(spec, { cwd, filesOnly: true });
-    if (!found.length) {
-      throw new Error(`No files matched "${spec}" from ${filename}`);
-    }
-    return found.map((f) => slash(f[0] === '.' ? f : `./${f}`));
-  } catch (err) {
-    throw new Error(`No files matched "${spec}" from ${filename}`);
-  }
-}
-
-/** Astro.fetchContent() */
-export function fetchContent(spec: string, { namespace, filename }: GlobOptions): GlobResult {
-  let code = '';
-  const imports = new Set<string>();
-  const importPaths = globSearch(spec, { filename });
-
-  // gather imports
-  importPaths.forEach((importPath, j) => {
-    const id = `${namespace}_${j}`;
-    imports.add(`import { __content as ${id} } from '${importPath}';`);
-
-    // add URL if this appears within the /pages/ directory (probably can be improved)
-    const fullPath = path.resolve(path.dirname(filename), importPath);
-
-    if (fullPath.includes(`${path.sep}pages${path.sep}`)) {
-      const url = importPath.replace(/^\./, '').replace(/\.md$/, '');
-      imports.add(`${id}.url = '${url}';`);
-    }
-  });
-
-  // generate replacement code
-  code += `${namespace} = [${importPaths.map((_, j) => `${namespace}_${j}`).join(',')}];\n`;
-
-  return { imports, code };
-}

packages/astro/src/compiler/codegen/index.ts

@@ -13,8 +13,6 @@ import babelParser from '@babel/parser';
 import { codeFrameColumns } from '@babel/code-frame';
 import * as babelTraverse from '@babel/traverse';
 import { error, warn, parseError } from '../../logger.js';
-import { fetchContent } from './content.js';
-import { isFetchContent } from './utils.js';
 import { yellow } from 'kleur/colors';
 import { isComponentTag, isCustomElementTag, positionAt } from '../utils.js';
 import { renderMarkdown } from '@astrojs/markdown-support';
@@ -331,11 +329,9 @@ function compileModule(ast: Ast, module: Script, state: CodegenState, compileOpt
   const componentImports: ImportDeclaration[] = [];
   const componentProps: VariableDeclarator[] = [];
   const componentExports: ExportNamedDeclaration[] = [];
-  const contentImports = new Map<string, { spec: string; declarator: string }>();
 
   let script = '';
   let propsStatement = '';
-  let contentCode = ''; // code for handling Astro.fetchContent(), if any;
   let createCollection = ''; // function for executing collection
 
   if (module) {
@@ -354,8 +350,41 @@ function compileModule(ast: Ast, module: Script, state: CodegenState, compileOpt
       err.start = err.loc;
       throw err;
     }
-    const program = parseResult.program;
 
+    // Convert Astro.fetchContent() to use import.meta.glob
+    if ((/Astro\s*\.\s*fetchContent/).test(module.content)) {
+      state.importStatements.add(`import {fetchContent} from 'astro/dist/internal/fetch-content.js';\n`);
+      traverse(parseResult, {
+        enter({ node }) {
+          if (
+            node.type !== 'CallExpression' ||
+            node.callee.type !== 'MemberExpression' ||
+            (node.callee.object as any).name !== 'Astro' ||
+            (node.callee.property as any).name !== 'fetchContent'
+          ) {
+            return;
+          }
+          if (node.arguments[0].type !== 'StringLiteral') {
+            throw new Error(`[Astro.fetchContent] Only string literals allowed, ex: \`Astro.fetchContent('./post/*.md')\`\n  ${state.filename}`);
+          }
+          // Replace `Astro.fetchContent(str)` with `Astro.fetchContent(import.meta.globEager(str))`
+          node.arguments = [
+            {
+              type: 'CallExpression',
+              callee: {
+                type: 'MemberExpression',
+                object: { type: 'MetaProperty', meta: { type: 'Identifier', name: 'import' }, property: { type: 'Identifier', name: 'meta' } },
+                property: { type: 'Identifier', name: 'globEager' },
+                computed: false,
+              },
+              arguments: node.arguments
+            },
+          ] as any;
+        },
+      });
+    }
+
+    const program = parseResult.program;
     const { body } = program;
     let i = body.length;
     while (--i >= 0) {
@@ -378,7 +407,7 @@ function compileModule(ast: Ast, module: Script, state: CodegenState, compileOpt
           } else if (node.declaration.type === 'FunctionDeclaration') {
             // case 2: createCollection (export async function)
             if (!node.declaration.id || node.declaration.id.name !== 'createCollection') break;
-            createCollection = module.content.substring(node.start || 0, node.end || 0);
+            createCollection = babelGenerator(node).code;
           }
 
           body.splice(i, 1);
@@ -396,37 +425,11 @@ function compileModule(ast: Ast, module: Script, state: CodegenState, compileOpt
           break;
         }
         case 'VariableDeclaration': {
+          // Support frontmatter-defined components
           for (const declaration of node.declarations) {
-            // only select Astro.fetchContent() calls for more processing,
+            if (declaration.id.type === 'Identifier') {
-            // otherwise just push name to declarations
+              state.declarations.add(declaration.id.name);
-            if (!isFetchContent(declaration)) {
-              if (declaration.id.type === 'Identifier') {
-                state.declarations.add(declaration.id.name);
-              }
-              continue;
             }
-
-            // remove node
-            body.splice(i, 1);
-
-            // a bit of munging
-            let { id, init } = declaration;
-            if (!id || !init || id.type !== 'Identifier') continue;
-            if (init.type === 'AwaitExpression') {
-              init = init.argument;
-              const shortname = path.posix.relative(compileOptions.astroConfig.projectRoot.pathname, state.filename);
-              warn(compileOptions.logging, shortname, yellow('awaiting Astro.fetchContent() not necessary'));
-            }
-            if (init.type !== 'CallExpression') continue;
-
-            // gather data
-            const namespace = id.name;
-
-            if ((init as any).arguments[0].type !== 'StringLiteral') {
-              throw new Error(`[Astro.fetchContent] Only string literals allowed, ex: \`Astro.fetchContent('./post/*.md')\`\n  ${state.filename}`);
-            }
-            const spec = (init as any).arguments[0].value;
-            if (typeof spec === 'string') contentImports.set(namespace, { spec, declarator: node.kind });
           }
           break;
         }
@@ -475,64 +478,7 @@ const { ${props.join(', ')} } = Astro.props;\n`)
       );
     }
 
-    // handle createCollection, if any
+    script = propsStatement + babelGenerator(program).code;
-    if (createCollection) {
-      const ast = babelParser.parse(createCollection, {
-        sourceType: 'module',
-      });
-      traverse(ast, {
-        enter({ node }) {
-          switch (node.type) {
-            case 'VariableDeclaration': {
-              for (const declaration of node.declarations) {
-                // only select Astro.fetchContent() calls here. this utility filters those out for us.
-                if (!isFetchContent(declaration)) continue;
-
-                // a bit of munging
-                let { id, init } = declaration;
-                if (!id || !init || id.type !== 'Identifier') continue;
-                if (init.type === 'AwaitExpression') {
-                  init = init.argument;
-                  const shortname = path.relative(compileOptions.astroConfig.projectRoot.pathname, state.filename);
-                  warn(compileOptions.logging, shortname, yellow('awaiting Astro.fetchContent() not necessary'));
-                }
-                if (init.type !== 'CallExpression') continue;
-
-                // gather data
-                const namespace = id.name;
-
-                if ((init as any).arguments[0].type !== 'StringLiteral') {
-                  throw new Error(`[Astro.fetchContent] Only string literals allowed, ex: \`Astro.fetchContent('./post/*.md')\`\n  ${state.filename}`);
-                }
-                const spec = (init as any).arguments[0].value;
-                if (typeof spec !== 'string') break;
-
-                const globResult = fetchContent(spec, { namespace, filename: state.filename });
-
-                let imports = '';
-                for (const importStatement of globResult.imports) {
-                  imports += importStatement + '\n';
-                }
-
-                createCollection = imports + createCollection.substring(0, declaration.start || 0) + globResult.code + createCollection.substring(declaration.end || 0);
-              }
-              break;
-            }
-          }
-        },
-      });
-    }
-
-    // Astro.fetchContent()
-    for (const [namespace, { spec }] of contentImports.entries()) {
-      const globResult = fetchContent(spec, { namespace, filename: state.filename });
-      for (const importStatement of globResult.imports) {
-        state.importStatements.add(importStatement);
-      }
-      contentCode += globResult.code;
-    }
-
-    script = propsStatement + contentCode + babelGenerator(program).code;
     const location = { start: module.start, end: module.end };
     let transpiledScript = transpileExpressionSafe(script, { state, compileOptions, location });
     if (transpiledScript === null) throw new Error(`Unable to compile script`);

packages/astro/src/compiler/codegen/utils.ts

@@ -2,7 +2,7 @@
  * Codegen utils
  */
 
-import type { VariableDeclarator } from '@babel/types';
+import type { VariableDeclarator, CallExpression } from '@babel/types';
 
 /** Is this an import.meta.* built-in? You can pass an optional 2nd param to see if the name matches as well. */
 export function isImportMetaDeclaration(declaration: VariableDeclarator, metaName?: string): boolean {
@@ -18,22 +18,3 @@ export function isImportMetaDeclaration(declaration: VariableDeclarator, metaNam
   if (metaName && (init.callee.property.type !== 'Identifier' || init.callee.property.name !== metaName)) return false;
   return true;
 }
-
-/** Is this an Astro.fetchContent() call? */
-export function isFetchContent(declaration: VariableDeclarator): boolean {
-  let { init } = declaration;
-  if (!init) return false; // definitely not import.meta
-  // this could be `await import.meta`; if so, evaluate that:
-  if (init.type === 'AwaitExpression') {
-    init = init.argument;
-  }
-  // continue evaluating
-  if (
-    init.type !== 'CallExpression' ||
-    init.callee.type !== 'MemberExpression' ||
-    (init.callee.object as any).name !== 'Astro' ||
-    (init.callee.property as any).name !== 'fetchContent'
-  )
-    return false;
-  return true;
-}

packages/astro/src/compiler/index.ts

@@ -112,9 +112,15 @@ export async function compileComponent(source: string, { compileOptions, filenam
   // return template
   let moduleJavaScript = `
 import fetch from 'node-fetch';
-
-// <script astro></script>
 ${result.imports.join('\n')}
+
+${/* Global Astro Namespace (shadowed & extended by the scoped namespace inside of __render()) */''}
+const __TopLevelAstro = {
+  site: new URL('/', ${JSON.stringify(site)}),
+  fetchContent: (globResult) => fetchContent(globResult, import.meta.url),
+};
+const Astro = __TopLevelAstro;
+
 ${
   result.hasCustomElements
     ? `
@@ -132,11 +138,11 @@ import { h, Fragment } from 'astro/dist/internal/h.js';
 const __astroInternal = Symbol('astro.internal');
 async function __render(props, ...children) {
   const Astro = {
+    ...__TopLevelAstro,
     props,
-    site: new URL('/', ${JSON.stringify(site)}),
     css: props[__astroInternal]?.css || [],
     request: props[__astroInternal]?.request || {},
-    isPage: props[__astroInternal]?.isPage || false
+    isPage: props[__astroInternal]?.isPage || false,
   };
 
   ${result.script}

packages/astro/src/internal/fetch-content.ts

@@ -0,0 +1,23 @@
+/**
+ * Convert the result of an `import.meta.globEager()` call to an array of processed
+ * Markdown content objects. Filter out any non-Markdown files matched in the glob
+ * result, by default.
+ */
+export function fetchContent(importMetaGlobResult: Record<string, any>, url: string) {
+  console.log(importMetaGlobResult);
+  return [...Object.entries(importMetaGlobResult)]
+    .map(([spec, mod]) => {
+      // Only return Markdown files, which export the __content object.
+      if (!mod.__content) {
+        return;
+      }
+      const urlSpec = new URL(spec, url).pathname.replace(/[\\/\\\\]/, '/');
+      if (!urlSpec.includes('/pages/')) {
+        return mod.__content;
+      }
+      return {
+        ...mod.__content,
+        url: urlSpec.replace(/^.*\/pages\//, '/').replace(/\.md$/, ''),
+      };
+    }).filter(Boolean);
+}