Infrastructure/astro
0
Fork 0
mirror of https://github.com/withastro/astro.git synced 2025-04-07 23:41:43 -05:00
Code Issues Activity

Parse frontmatter ourselves (#12075)

Browse source
This commit is contained in:
Bjorn Lu 2024-09-26 14:59:39 +01:00 • committed by GitHub
parent acf264d8c0
commit a19530e377
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 127 additions and 108 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.changeset/sweet-timers-smash.md Normal file
View file

@ -0,0 +1,8 @@
---
'@astrojs/markdoc': patch
'@astrojs/mdx': patch
'@astrojs/markdown-remark': patch
'astro': patch
---
Parses frontmatter ourselves

1
packages/astro/package.json
View file

@ -152,7 +152,6 @@
"fastq": "^1.17.1",
"flattie": "^1.1.1",
"github-slugger": "^2.0.0",
"gray-matter": "^4.0.3",
"html-escaper": "^3.0.3",
"http-cache-semantics": "^4.1.1",
"js-yaml": "^4.1.0",

4
packages/astro/src/content/utils.ts
View file

@ -1,8 +1,8 @@
import fsMod from 'node:fs';
import path from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { parseFrontmatter } from '@astrojs/markdown-remark';
import { slug as githubSlug } from 'github-slugger';
import matter from 'gray-matter';
import type { PluginContext } from 'rollup';
import type { ViteDevServer } from 'vite';
import xxhash from 'xxhash-wasm';
@ -455,7 +455,7 @@ function getYAMLErrorLine(rawData: string | undefined, objectKey: string) {
export function safeParseFrontmatter(source: string, id?: string) {
try {
return matter(source);
return parseFrontmatter(source, { frontmatter: 'empty-with-spaces' });
} catch (err: any) {
const markdownError = new MarkdownError({
name: 'MarkdownError',

8
packages/astro/src/vite-plugin-markdown/content-entry-type.ts
View file

@ -8,10 +8,10 @@ export const markdownContentEntryType: ContentEntryType = {
async getEntryInfo({ contents, fileUrl }: { contents: string; fileUrl: URL }) {
const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl));
return {
data: parsed.data,
body: parsed.content,
slug: parsed.data.slug,
rawData: parsed.matter,
data: parsed.frontmatter,
body: parsed.content.trim(),
slug: parsed.frontmatter.slug,
rawData: parsed.rawFrontmatter,
};
},
// We need to handle propagation for Markdown because they support layouts which will bring in styles.

2
packages/astro/src/vite-plugin-markdown/index.ts
View file

@ -66,7 +66,7 @@ export default function markdown({ settings, logger }: AstroPluginOptions): Plug
const renderResult = await (await processor).render(raw.content, {
// @ts-expect-error passing internal prop
fileURL,
frontmatter: raw.data,
frontmatter: raw.frontmatter,
});
// Improve error message for invalid astro frontmatter

1
packages/integrations/markdoc/package.json
View file

@ -68,7 +68,6 @@
"@markdoc/markdoc": "^0.4.0",
"esbuild": "^0.21.5",
"github-slugger": "^2.0.0",
"gray-matter": "^4.0.3",
"htmlparser2": "^9.1.0"
},
"peerDependencies": {

43
packages/integrations/markdoc/src/content-entry-type.ts
View file

@ -1,11 +1,11 @@
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';
import { parseFrontmatter } from '@astrojs/markdown-remark';
import type { Config as MarkdocConfig, Node } from '@markdoc/markdoc';
import Markdoc from '@markdoc/markdoc';
import type { AstroConfig, ContentEntryType } from 'astro';
import { emitESMImage } from 'astro/assets/utils';
import matter from 'gray-matter';
import type { Rollup, ErrorPayload as ViteErrorPayload } from 'vite';
import type { ComponentConfig } from './config.js';
import { htmlTokenTransform } from './html/transform/html-token-transform.js';
@ -26,12 +26,20 @@ export async function getContentEntryType({
}): Promise<ContentEntryType> {
return {
extensions: ['.mdoc'],
getEntryInfo,
getEntryInfo({ fileUrl, contents }) {
const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl));
return {
data: parsed.frontmatter,
body: parsed.content.trim(),
slug: parsed.frontmatter.slug,
rawData: parsed.rawFrontmatter,
};
},
handlePropagation: true,
async getRenderModule({ contents, fileUrl, viteId }) {
const entry = getEntryInfo({ contents, fileUrl });
const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl));
const tokenizer = getMarkdocTokenizer(options);
let tokens = tokenizer.tokenize(entry.body);
let tokens = tokenizer.tokenize(parsed.content);
if (options?.allowHTML) {
tokens = htmlTokenTransform(tokenizer, tokens);
@ -47,7 +55,6 @@ export async function getContentEntryType({
ast,
/* Raised generics issue with Markdoc core https://github.com/markdoc/markdoc/discussions/400 */
markdocConfig: markdocConfig as MarkdocConfig,
entry,
viteId,
astroConfig,
filePath,
@ -64,7 +71,6 @@ export async function getContentEntryType({
raiseValidationErrors({
ast: partialAst,
markdocConfig: markdocConfig as MarkdocConfig,
entry,
viteId,
astroConfig,
filePath: partialPath,
@ -224,14 +230,12 @@ async function resolvePartials({
function raiseValidationErrors({
ast,
markdocConfig,
entry,
viteId,
astroConfig,
filePath,
}: {
ast: Node;
markdocConfig: MarkdocConfig;
entry: ReturnType<typeof getEntryInfo>;
viteId: string;
astroConfig: AstroConfig;
filePath: string;
@ -250,8 +254,6 @@ function raiseValidationErrors({
});
if (validationErrors.length) {
// Heuristic: take number of newlines for `rawData` and add 2 for the `---` fences
const frontmatterBlockOffset = entry.rawData.split('\n').length + 2;
const rootRelativePath = path.relative(fileURLToPath(astroConfig.root), filePath);
throw new MarkdocError({
message: [
@ -261,7 +263,7 @@ function raiseValidationErrors({
location: {
// Error overlay does not support multi-line or ranges.
// Just point to the first line.
line: frontmatterBlockOffset + validationErrors[0].lines[0],
line: validationErrors[0].lines[0],
file: viteId,
},
});
@ -282,16 +284,6 @@ function getUsedTags(markdocAst: Node) {
return tags;
}
function getEntryInfo({ fileUrl, contents }: { fileUrl: URL; contents: string }) {
const parsed = parseFrontmatter(contents, fileURLToPath(fileUrl));
return {
data: parsed.data,
body: parsed.content,
slug: parsed.data.slug,
rawData: parsed.matter,
};
}
/**
* Emits optimized images, and appends the generated `src` to each AST node
* via the `__optimizedSrc` attribute.
@ -410,12 +402,11 @@ function getStringifiedMap(
* Match YAML exception handling from Astro core errors
* @see 'astro/src/core/errors.ts'
*/
function parseFrontmatter(fileContents: string, filePath: string) {
function safeParseFrontmatter(fileContents: string, filePath: string) {
try {
// `matter` is empty string on cache results
// clear cache to prevent this
(matter as any).clearCache();
return matter(fileContents);
// empty with lines to preserve sourcemap location, but not `empty-with-spaces`
// because markdoc struggles with spaces
return parseFrontmatter(fileContents, { frontmatter: 'empty-with-lines' });
} catch (e: any) {
if (e.name === 'YAMLException') {
const err: Error & ViteErrorPayload['err'] = e;

6
packages/integrations/markdoc/test/content-collections.test.js
View file

@ -83,7 +83,7 @@ const post1Entry = {
schemaWorks: true,
title: 'Post 1',
},
body: '\n## Post 1\n\nThis is the contents of post 1.\n',
body: '## Post 1\n\nThis is the contents of post 1.',
};
const post2Entry = {
@ -94,7 +94,7 @@ const post2Entry = {
schemaWorks: true,
title: 'Post 2',
},
body: '\n## Post 2\n\nThis is the contents of post 2.\n',
body: '## Post 2\n\nThis is the contents of post 2.',
};
const post3Entry = {
@ -105,5 +105,5 @@ const post3Entry = {
schemaWorks: true,
title: 'Post 3',
},
body: '\n## Post 3\n\nThis is the contents of post 3.\n',
body: '## Post 3\n\nThis is the contents of post 3.',
};

1
packages/integrations/mdx/package.json
View file

@ -39,7 +39,6 @@
"acorn": "^8.12.1",
"es-module-lexer": "^1.5.4",
"estree-util-visit": "^2.0.0",
"gray-matter": "^4.0.3",
"hast-util-to-html": "^9.0.2",
"kleur": "^4.1.5",
"rehype-raw": "^7.0.0",

12
packages/integrations/mdx/src/index.ts
View file

@ -11,7 +11,7 @@ import type {
import type { Options as RemarkRehypeOptions } from 'remark-rehype';
import type { PluggableList } from 'unified';
import type { OptimizeOptions } from './rehype-optimize-static.js';
import { ignoreStringPlugins, parseFrontmatter } from './utils.js';
import { ignoreStringPlugins, safeParseFrontmatter } from './utils.js';
import { vitePluginMdxPostprocess } from './vite-plugin-mdx-postprocess.js';
import { vitePluginMdx } from './vite-plugin-mdx.js';
@ -60,12 +60,12 @@ export default function mdx(partialMdxOptions: Partial<MdxOptions> = {}): AstroI
addContentEntryType({
extensions: ['.mdx'],
async getEntryInfo({ fileUrl, contents }: { fileUrl: URL; contents: string }) {
const parsed = parseFrontmatter(contents, fileURLToPath(fileUrl));
const parsed = safeParseFrontmatter(contents, fileURLToPath(fileUrl));
return {
data: parsed.data,
body: parsed.content,
slug: parsed.data.slug,
rawData: parsed.matter,
data: parsed.frontmatter,
body: parsed.content.trim(),
slug: parsed.frontmatter.slug,
rawData: parsed.rawFrontmatter,
};
},
contentModuleTypes: await fs.readFile(

6
packages/integrations/mdx/src/utils.ts
View file

@ -1,7 +1,7 @@
import { parseFrontmatter } from '@astrojs/markdown-remark';
import type { Options as AcornOpts } from 'acorn';
import { parse } from 'acorn';
import type { AstroConfig, AstroIntegrationLogger, SSRError } from 'astro';
import matter from 'gray-matter';
import { bold } from 'kleur/colors';
import type { MdxjsEsm } from 'mdast-util-mdx';
import type { PluggableList } from 'unified';
@ -48,9 +48,9 @@ export function getFileInfo(id: string, config: AstroConfig): FileInfo {
* Match YAML exception handling from Astro core errors
* @see 'astro/src/core/errors.ts'
*/
export function parseFrontmatter(code: string, id: string) {
export function safeParseFrontmatter(code: string, id: string) {
try {
return matter(code);
return parseFrontmatter(code, { frontmatter: 'empty-with-spaces' });
} catch (e: any) {
if (e.name === 'YAMLException') {
const err: SSRError = e;

7
packages/integrations/mdx/src/vite-plugin-mdx.ts
View file

@ -4,7 +4,7 @@ import { VFile } from 'vfile';
import type { Plugin } from 'vite';
import type { MdxOptions } from './index.js';
import { createMdxProcessor } from './plugins.js';
import { parseFrontmatter } from './utils.js';
import { safeParseFrontmatter } from './utils.js';
export function vitePluginMdx(mdxOptions: MdxOptions): Plugin {
let processor: ReturnType<typeof createMdxProcessor> | undefined;
@ -38,11 +38,10 @@ export function vitePluginMdx(mdxOptions: MdxOptions): Plugin {
async transform(code, id) {
if (!id.endsWith('.mdx')) return;
const { data: frontmatter, content: pageContent, matter } = parseFrontmatter(code, id);
const frontmatterLines = matter ? matter.match(/\n/g)?.join('') + '\n\n' : '';
const { frontmatter, content } = safeParseFrontmatter(code, id);
const vfile = new VFile({
value: frontmatterLines + pageContent,
value: content,
path: id,
data: {
astro: {

2
packages/markdown/remark/package.json
View file

@ -37,6 +37,7 @@
"hast-util-from-html": "^2.0.2",
"hast-util-to-text": "^4.0.2",
"import-meta-resolve": "^4.1.0",
"js-yaml": "^4.1.0",
"mdast-util-definitions": "^6.0.0",
"rehype-raw": "^7.0.0",
"rehype-stringify": "^10.0.0",
@ -54,6 +55,7 @@
"devDependencies": {
"@types/estree": "^1.0.5",
"@types/hast": "^3.0.4",
"@types/js-yaml": "^4.0.9",
"@types/mdast": "^4.0.4",
"@types/unist": "^3.0.3",
"astro-scripts": "workspace:*",

65
packages/markdown/remark/src/frontmatter.ts
View file

@ -1,3 +1,5 @@
import yaml from 'js-yaml';
export function isFrontmatterValid(frontmatter: Record<string, any>) {
try {
// ensure frontmatter is JSON-serializable
@ -7,3 +9,66 @@ export function isFrontmatterValid(frontmatter: Record<string, any>) {
}
return typeof frontmatter === 'object' && frontmatter !== null;
}
const frontmatterRE = /^---(.*?)^---/ms;
export function extractFrontmatter(code: string): string | undefined {
return frontmatterRE.exec(code)?.[1];
}
export interface ParseFrontmatterOptions {
/**
* How the frontmatter should be handled in the returned `content` string.
* - `preserve`: Keep the frontmatter.
* - `remove`: Remove the frontmatter.
* - `empty-with-spaces`: Replace the frontmatter with empty spaces. (preserves sourcemap line/col/offset)
* - `empty-with-lines`: Replace the frontmatter with empty line breaks. (preserves sourcemap line/col)
*
* @default 'remove'
*/
frontmatter: 'preserve' | 'remove' | 'empty-with-spaces' | 'empty-with-lines';
}
export interface ParseFrontmatterResult {
frontmatter: Record<string, any>;
rawFrontmatter: string;
content: string;
}
export function parseFrontmatter(
code: string,
options?: ParseFrontmatterOptions,
): ParseFrontmatterResult {
const rawFrontmatter = extractFrontmatter(code);
if (rawFrontmatter == null) {
return { frontmatter: {}, rawFrontmatter: '', content: code };
}
const parsed = yaml.load(rawFrontmatter);
const frontmatter = (parsed && typeof parsed === 'object' ? parsed : {}) as Record<string, any>;
let content: string;
switch (options?.frontmatter ?? 'remove') {
case 'preserve':
content = code;
break;
case 'remove':
content = code.replace(`---${rawFrontmatter}---`, '');
break;
case 'empty-with-spaces':
content = code.replace(
`---${rawFrontmatter}---`,
` ${rawFrontmatter.replace(/[^\r\n]/g, ' ')} `,
);
break;
case 'empty-with-lines':
content = code.replace(`---${rawFrontmatter}---`, rawFrontmatter.replace(/[^\r\n]/g, ''));
break;
}
return {
frontmatter,
rawFrontmatter,
content,
};
}

8
packages/markdown/remark/src/index.ts
View file

@ -20,7 +20,13 @@ export { rehypeHeadingIds } from './rehype-collect-headings.js';
export { remarkCollectImages } from './remark-collect-images.js';
export { rehypePrism } from './rehype-prism.js';
export { rehypeShiki } from './rehype-shiki.js';
export { isFrontmatterValid } from './frontmatter.js';
export {
isFrontmatterValid,
extractFrontmatter,
parseFrontmatter,
type ParseFrontmatterOptions,
type ParseFrontmatterResult,
} from './frontmatter.js';
export {
createShikiHighlighter,
type ShikiHighlighter,

61
pnpm-lock.yaml generated
View file

@ -627,9 +627,6 @@ importers:
github-slugger:
specifier: ^2.0.0
version: 2.0.0
gray-matter:
specifier: ^4.0.3
version: 4.0.3
html-escaper:
specifier: ^3.0.3
version: 3.0.3
@ -4538,9 +4535,6 @@ importers:
github-slugger:
specifier: ^2.0.0
version: 2.0.0
gray-matter:
specifier: ^4.0.3
version: 4.0.3
htmlparser2:
specifier: ^9.1.0
version: 9.1.0
@ -4731,9 +4725,6 @@ importers:
estree-util-visit:
specifier: ^2.0.0
version: 2.0.0
gray-matter:
specifier: ^4.0.3
version: 4.0.3
hast-util-to-html:
specifier: ^9.0.2
version: 9.0.2
@ -5389,6 +5380,9 @@ importers:
import-meta-resolve:
specifier: ^4.1.0
version: 4.1.0
js-yaml:
specifier: ^4.1.0
version: 4.1.0
mdast-util-definitions:
specifier: ^6.0.0
version: 6.0.0
@ -5435,6 +5429,9 @@ importers:
'@types/hast':
specifier: ^3.0.4
version: 3.0.4
'@types/js-yaml':
specifier: ^4.0.9
version: 4.0.9
'@types/mdast':
specifier: ^4.0.4
version: 4.0.4
@ -8215,10 +8212,6 @@ packages:
resolution: {integrity: sha512-uHaC9LYNv6BcW+8SvXcwUUDCrrUxt3GSa61DFvTHj8JC+M0hekMFBwMlCarLQDk5bbpZ2vStpnQPIwRuV98YMw==}
engines: {node: '>=12.0.0'}
extend-shallow@2.0.1:
resolution: {integrity: sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==}
engines: {node: '>=0.10.0'}
extend@3.0.2:
resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==}
@ -8407,10 +8400,6 @@ packages:
graphemer@1.4.0:
resolution: {integrity: sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==}
gray-matter@4.0.3:
resolution: {integrity: sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==}
engines: {node: '>=6.0'}
has-async-hooks@1.0.0:
resolution: {integrity: sha512-YF0VPGjkxr7AyyQQNykX8zK4PvtEDsUJAPqwu06UFz1lb6EvI53sPh5H1kWxg8NXI5LsfRCZ8uX9NkYDZBb/mw==}
@ -8613,10 +8602,6 @@ packages:
engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
hasBin: true
is-extendable@0.1.1:
resolution: {integrity: sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==}
engines: {node: '>=0.10.0'}
is-extglob@2.1.1:
resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==}
engines: {node: '>=0.10.0'}
@ -8761,10 +8746,6 @@ packages:
keyv@4.5.4:
resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==}
kind-of@6.0.3:
resolution: {integrity: sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==}
engines: {node: '>=0.10.0'}
kleur@3.0.3:
resolution: {integrity: sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==}
engines: {node: '>=6'}
@ -10004,10 +9985,6 @@ packages:
resolution: {integrity: sha512-3A6sD0WYP7+QrjbfNA2FN3FsOaGGFoekCVgTyypy53gPxhbkCIjtO6YWgdrfM+n/8sI8JeXZOIxsHjMTNxQ4nQ==}
engines: {node: ^14.0.0 || >=16.0.0}
section-matter@1.0.0:
resolution: {integrity: sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==}
engines: {node: '>=4'}
semver@6.3.1:
resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==}
hasBin: true
@ -10193,10 +10170,6 @@ packages:
resolution: {integrity: sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==}
engines: {node: '>=12'}
strip-bom-string@1.0.0:
resolution: {integrity: sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==}
engines: {node: '>=0.10.0'}
strip-bom@3.0.0:
resolution: {integrity: sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==}
engines: {node: '>=4'}
@ -13865,10 +13838,6 @@ snapshots:
expect-type@0.20.0: {}
extend-shallow@2.0.1:
dependencies:
is-extendable: 0.1.1
extend@3.0.2: {}
extendable-error@0.1.7: {}
@ -14064,13 +14033,6 @@ snapshots:
graphemer@1.4.0: {}
gray-matter@4.0.3:
dependencies:
js-yaml: 3.14.1
kind-of: 6.0.3
section-matter: 1.0.0
strip-bom-string: 1.0.0
has-async-hooks@1.0.0: {}
has-flag@3.0.0: {}
@ -14382,8 +14344,6 @@ snapshots:
is-docker@3.0.0: {}
is-extendable@0.1.1: {}
is-extglob@2.1.1: {}
is-fullwidth-code-point@3.0.0: {}
@ -14515,8 +14475,6 @@ snapshots:
dependencies:
json-buffer: 3.0.1
kind-of@6.0.3: {}
kleur@3.0.3: {}
kleur@4.1.5: {}
@ -16158,11 +16116,6 @@ snapshots:
refa: 0.12.1
regexp-ast-analysis: 0.7.1
section-matter@1.0.0:
dependencies:
extend-shallow: 2.0.1
kind-of: 6.0.3
semver@6.3.1: {}
semver@7.6.3: {}
@ -16379,8 +16332,6 @@ snapshots:
dependencies:
ansi-regex: 6.0.1
strip-bom-string@1.0.0: {}
strip-bom@3.0.0: {}
strip-final-newline@3.0.0: {}