Unset charset=utf-8 content-type for md/mdx pages (#12231)

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

.changeset/dirty-cooks-explode.md

@@ -0,0 +1,9 @@
+---
+'@astrojs/mdx': major
+---
+
+Handles the breaking change in Astro where content pages (including `.mdx` pages located within `src/pages/`) no longer respond with `charset=utf-8` in the `Content-Type` header.
+
+For MDX pages without layouts, `@astrojs/mdx` will automatically add the `<meta charset="utf-8">` tag to the page by default. This reduces the boilerplate needed to write with non-ASCII characters. If your MDX pages have a layout, the layout component should include the `<meta charset="utf-8">` tag.
+
+If you require `charset=utf-8` to render your page correctly, make sure that your layout components have the `<meta charset="utf-8">` tag added.

.changeset/strong-months-grab.md

@@ -0,0 +1,11 @@
+---
+'astro': major
+---
+
+Updates the automatic `charset=utf-8` behavior for Markdown pages, where instead of responding with `charset=utf-8` in the `Content-Type` header, Astro will now automatically add the `<meta charset="utf-8">` tag instead.
+
+This behaviour only applies to Markdown pages (`.md` or similar Markdown files located within `src/pages/`) that do not use Astro's special `layout` frontmatter property. It matches the rendering behaviour of other non-content pages, and retains the minimal boilerplate needed to write with non-ASCII characters when adding individual Markdown pages to your site.
+
+If your Markdown pages use the `layout` frontmatter property, then HTML encoding will be handled by the designated layout component instead, and the `<meta charset="utf-8">` tag will not be added to your page by default.
+
+If you require `charset=utf-8` to render your page correctly, make sure that your layout components contain the `<meta charset="utf-8">` tag. You may need to add this if you have not already done so.

packages/astro/src/core/preview/vite-plugin-astro-preview.ts

@@ -16,7 +16,7 @@ export function vitePluginAstroPreview(settings: AstroSettings): Plugin {
 		const errorPagePath = fileURLToPath(outDir + '/404.html');
 		if (fs.existsSync(errorPagePath)) {
 			res.statusCode = 404;
-			res.setHeader('Content-Type', 'text/html;charset=utf-8');
+			res.setHeader('Content-Type', 'text/html');
 			res.end(fs.readFileSync(errorPagePath));
 		} else {
 			res.statusCode = 404;

packages/astro/src/core/routing/astro-designed-error-pages.ts

@@ -46,7 +46,7 @@ async function default404Page({ pathname }: { pathname: string }) {
 			tabTitle: '404: Not Found',
 			pathname,
 		}),
-		{ status: 404, headers: { 'Content-Type': 'text/html; charset=utf-8' } },
+		{ status: 404, headers: { 'Content-Type': 'text/html' } },
 	);
 }
 // mark the function as an AstroComponentFactory for the rendering internals

packages/astro/src/runtime/server/render/page.ts

@@ -35,7 +35,7 @@ export async function renderPage(
 
 		return new Response(bytes, {
 			headers: new Headers([
-				['Content-Type', 'text/html; charset=utf-8'],
+				['Content-Type', 'text/html'],
 				['Content-Length', bytes.byteLength.toString()],
 			]),
 		});
@@ -80,11 +80,6 @@ export async function renderPage(
 		body = encoder.encode(body);
 		headers.set('Content-Length', body.byteLength.toString());
 	}
-	// TODO: Revisit if user should manually set charset by themselves in Astro 4
-	// This code preserves the existing behaviour for markdown pages since Astro 2
-	if (route?.component.endsWith('.md')) {
-		headers.set('Content-Type', 'text/html; charset=utf-8');
-	}
 	let status = init.status;
 	// Custom 404.astro and 500.astro are particular routes that must return a fixed status code
 	if (route?.route === '/404') {

packages/astro/src/vite-plugin-astro-server/response.ts

@@ -46,7 +46,7 @@ export async function handle500Response(
 
 export function writeHtmlResponse(res: http.ServerResponse, statusCode: number, html: string) {
 	res.writeHead(statusCode, {
-		'Content-Type': 'text/html; charset=utf-8',
+		'Content-Type': 'text/html',
 		'Content-Length': Buffer.byteLength(html, 'utf-8'),
 	});
 	res.write(html);

packages/astro/src/vite-plugin-markdown/index.ts

@@ -10,7 +10,7 @@ import { normalizePath } from 'vite';
 import { safeParseFrontmatter } from '../content/utils.js';
 import { AstroError, AstroErrorData } from '../core/errors/index.js';
 import type { Logger } from '../core/logger/core.js';
-import { isMarkdownFile } from '../core/util.js';
+import { isMarkdownFile, isPage } from '../core/util.js';
 import { shorthash } from '../runtime/server/shorthash.js';
 import type { AstroSettings } from '../types/astro.js';
 import { createDefaultAstroMetadata } from '../vite-plugin-astro/metadata.js';
@@ -77,6 +77,10 @@ export default function markdown({ settings, logger }: AstroPluginOptions): Plug
 				let html = renderResult.code;
 				const { headings, imagePaths: rawImagePaths, frontmatter } = renderResult.metadata;
 
+				// Add default charset for markdown pages
+				const isMarkdownPage = isPage(fileURL, settings);
+				const charset = isMarkdownPage ? '<meta charset="utf-8">' : '';
+
 				// Resolve all the extracted images from the content
 				const imagePaths: MarkdownImagePath[] = [];
 				for (const imagePath of rawImagePaths) {
@@ -141,7 +145,7 @@ export default function markdown({ settings, logger }: AstroPluginOptions): Plug
 							}, {
 								'default': () => render\`\${unescapeHTML(html())}\`
 							})}\`;`
-							: `render\`\${maybeRenderHead(result)}\${unescapeHTML(html())}\`;`
+							: `render\`${charset}\${maybeRenderHead(result)}\${unescapeHTML(html())}\`;`
 					}
 				});
 				export default Content;

packages/astro/test/astro-basic.test.js

@@ -115,12 +115,7 @@ describe('Astro basic build', () => {
 		const html = await fixture.readFile('/chinese-encoding-md/index.html');
 		const $ = cheerio.load(html);
 		assert.equal($('h1').text(), '我的第一篇博客文章');
-	});
-
-	it('renders MDX in utf-8 by default', async () => {
-		const html = await fixture.readFile('/chinese-encoding-mdx/index.html');
-		const $ = cheerio.load(html);
-		assert.equal($('h1').text(), '我的第一篇博客文章');
+		assert.match(html, /<meta charset="utf-8"/);
 	});
 
 	it('Supports void elements whose name is a string (#2062)', async () => {
@@ -207,22 +202,8 @@ describe('Astro basic development', () => {
 		const html = await res.text();
 		const $ = cheerio.load(html);
 		assert.equal($('h1').text(), '我的第一篇博客文章');
-		const isUtf8 =
-			res.headers.get('content-type').includes('charset=utf-8') ||
-			html.includes('<meta charset="utf-8">');
-		assert.ok(isUtf8);
-	});
-
-	it('Renders MDX in utf-8 by default', async () => {
-		const res = await fixture.fetch('/chinese-encoding-mdx');
-		assert.equal(res.status, 200);
-		const html = await res.text();
-		const $ = cheerio.load(html);
-		assert.equal($('h1').text(), '我的第一篇博客文章');
-		const isUtf8 =
-			res.headers.get('content-type').includes('charset=utf-8') ||
-			html.includes('<meta charset="utf-8">');
-		assert.ok(isUtf8);
+		assert.doesNotMatch(res.headers.get('content-type'), /charset=utf-8/);
+		assert.match(html, /<meta charset="utf-8"/);
 	});
 
 	it('Handles importing .astro?raw correctly', async () => {

packages/integrations/mdx/src/index.ts

@@ -13,7 +13,7 @@ import type { PluggableList } from 'unified';
 import type { OptimizeOptions } from './rehype-optimize-static.js';
 import { ignoreStringPlugins, safeParseFrontmatter } from './utils.js';
 import { vitePluginMdxPostprocess } from './vite-plugin-mdx-postprocess.js';
-import { vitePluginMdx } from './vite-plugin-mdx.js';
+import { type VitePluginMdxOptions, vitePluginMdx } from './vite-plugin-mdx.js';
 
 export type MdxOptions = Omit<typeof markdownConfigDefaults, 'remarkPlugins' | 'rehypePlugins'> & {
 	extendMarkdownConfig: boolean;
@@ -43,7 +43,7 @@ export function getContainerRenderer(): ContainerRenderer {
 export default function mdx(partialMdxOptions: Partial<MdxOptions> = {}): AstroIntegration {
 	// @ts-expect-error Temporarily assign an empty object here, which will be re-assigned by the
 	// `astro:config:done` hook later. This is so that `vitePluginMdx` can get hold of a reference earlier.
-	let mdxOptions: MdxOptions = {};
+	let vitePluginMdxOptions: VitePluginMdxOptions = {};
 
 	return {
 		name: '@astrojs/mdx',
@@ -79,7 +79,7 @@ export default function mdx(partialMdxOptions: Partial<MdxOptions> = {}): AstroI
 
 				updateConfig({
 					vite: {
-						plugins: [vitePluginMdx(mdxOptions), vitePluginMdxPostprocess(config)],
+						plugins: [vitePluginMdx(vitePluginMdxOptions), vitePluginMdxPostprocess(config)],
 					},
 				});
 			},
@@ -98,10 +98,13 @@ export default function mdx(partialMdxOptions: Partial<MdxOptions> = {}): AstroI
 				});
 
 				// Mutate `mdxOptions` so that `vitePluginMdx` can reference the actual options
-				Object.assign(mdxOptions, resolvedMdxOptions);
+				Object.assign(vitePluginMdxOptions, {
+					mdxOptions: resolvedMdxOptions,
+					srcDir: config.srcDir,
+				});
 				// @ts-expect-error After we assign, we don't need to reference `mdxOptions` in this context anymore.
 				// Re-assign it so that the garbage can be collected later.
-				mdxOptions = {};
+				vitePluginMdxOptions = {};
 			},
 		},
 	};

packages/integrations/mdx/src/rehype-apply-frontmatter-export.ts

@@ -1,9 +1,23 @@
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
 import { isFrontmatterValid } from '@astrojs/markdown-remark';
+import type { Root, RootContent } from 'hast';
 import type { VFile } from 'vfile';
 import { jsToTreeNode } from './utils.js';
 
+// Passed metadata to help determine adding charset utf8 by default
+declare module 'vfile' {
+	interface DataMap {
+		applyFrontmatterExport?: {
+			srcDir?: URL;
+		};
+	}
+}
+
+const exportConstPartialTrueRe = /export\s+const\s+partial\s*=\s*true/;
+
 export function rehypeApplyFrontmatterExport() {
-	return function (tree: any, vfile: VFile) {
+	return function (tree: Root, vfile: VFile) {
 		const frontmatter = vfile.data.astro?.frontmatter;
 		if (!frontmatter || !isFrontmatterValid(frontmatter))
 			throw new Error(
@@ -11,11 +25,11 @@ export function rehypeApplyFrontmatterExport() {
 				// TODO: find way to import error data from core
 				'[MDX] A remark or rehype plugin attempted to inject invalid frontmatter. Ensure "astro.frontmatter" is set to a valid JSON object that is not `null` or `undefined`.',
 			);
-		const exportNodes = [
+		const extraChildren: RootContent[] = [
 			jsToTreeNode(`export const frontmatter = ${JSON.stringify(frontmatter)};`),
 		];
 		if (frontmatter.layout) {
-			exportNodes.unshift(
+			extraChildren.unshift(
 				jsToTreeNode(
 					// NOTE: Use `__astro_*` import names to prevent conflicts with user code
 					/** @see 'vite-plugin-markdown' for layout props reference */
@@ -39,7 +53,61 @@ export default function ({ children }) {
 };`,
 				),
 			);
+		} else if (shouldAddCharset(tree, vfile)) {
+			extraChildren.unshift({
+				type: 'mdxJsxFlowElement',
+				name: 'meta',
+				attributes: [
+					{
+						type: 'mdxJsxAttribute',
+						name: 'charset',
+						value: 'utf-8',
+					},
+				],
+				children: [],
+			});
 		}
-		tree.children = exportNodes.concat(tree.children);
+		tree.children = extraChildren.concat(tree.children);
 	};
 }
+
+/**
+ * If this is a page (e.g. in src/pages), has no layout frontmatter (handled before calling this function),
+ * has no leading component that looks like a wrapping layout, and `partial` isn't set to true, we default to
+ * adding charset=utf-8 like markdown so that users don't have to worry about it for MDX pages without layouts.
+ */
+function shouldAddCharset(tree: Root, vfile: VFile) {
+	const srcDirUrl = vfile.data.applyFrontmatterExport?.srcDir;
+	if (!srcDirUrl) return false;
+
+	const hasConstPartialTrue = tree.children.some(
+		(node) => node.type === 'mdxjsEsm' && exportConstPartialTrueRe.test(node.value),
+	);
+	if (hasConstPartialTrue) return false;
+
+	// NOTE: the pages directory is a non-configurable Astro behaviour
+	const pagesDir = path.join(fileURLToPath(srcDirUrl), 'pages').replace(/\\/g, '/');
+	// `vfile.path` comes from Vite, which is a normalized path (no backslashes)
+	const filePath = vfile.path;
+	if (!filePath.startsWith(pagesDir)) return false;
+
+	const hasLeadingUnderscoreInPath = filePath
+		.slice(pagesDir.length)
+		.replace(/\\/g, '/')
+		.split('/')
+		.some((part) => part.startsWith('_'));
+	if (hasLeadingUnderscoreInPath) return false;
+
+	// Bail if the first content found is a wrapping layout component
+	for (const child of tree.children) {
+		if (child.type === 'element') break;
+		if (child.type === 'mdxJsxFlowElement') {
+			// If is fragment or lowercase tag name (html tags), skip and assume there's no layout
+			if (child.name == null) break;
+			if (child.name[0] === child.name[0].toLowerCase()) break;
+			return false;
+		}
+	}
+
+	return true;
+}

packages/integrations/mdx/src/vite-plugin-mdx.ts

@@ -6,7 +6,13 @@ import type { MdxOptions } from './index.js';
 import { createMdxProcessor } from './plugins.js';
 import { safeParseFrontmatter } from './utils.js';
 
-export function vitePluginMdx(mdxOptions: MdxOptions): Plugin {
+export interface VitePluginMdxOptions {
+	mdxOptions: MdxOptions;
+	srcDir: URL;
+}
+
+// NOTE: Do not destructure `opts` as we're assigning a reference that will be mutated later
+export function vitePluginMdx(opts: VitePluginMdxOptions): Plugin {
 	let processor: ReturnType<typeof createMdxProcessor> | undefined;
 	let sourcemapEnabled: boolean;
 
@@ -47,12 +53,15 @@ export function vitePluginMdx(mdxOptions: MdxOptions): Plugin {
 					astro: {
 						frontmatter,
 					},
+					applyFrontmatterExport: {
+						srcDir: opts.srcDir,
+					},
 				},
 			});
 
 			// Lazily initialize the MDX processor
 			if (!processor) {
-				processor = createMdxProcessor(mdxOptions, { sourcemap: sourcemapEnabled });
+				processor = createMdxProcessor(opts.mdxOptions, { sourcemap: sourcemapEnabled });
 			}
 
 			try {

packages/integrations/mdx/test/fixtures/mdx-page/src/layouts/EncodingLayout.astro

@@ -0,0 +1 @@
+<slot></slot>

packages/integrations/mdx/test/fixtures/mdx-page/src/pages/chinese-encoding-layout-frontmatter.mdx

@@ -0,0 +1,7 @@
+---
+layout: ../layouts/EncodingLayout.astro
+---
+
+# 我的第一篇博客文章
+
+发表于：2022-07-01

packages/integrations/mdx/test/fixtures/mdx-page/src/pages/chinese-encoding-layout-manual.mdx

@@ -0,0 +1,12 @@
+import EncodingLayout from '../layouts/EncodingLayout.astro'
+
+{/* Ensure random stuff preceding the wrapper layout is ignored when detecting a wrapper layout */}
+export const foo = {}
+
+<EncodingLayout>
+
+# 我的第一篇博客文章
+
+发表于：2022-07-01
+
+</EncodingLayout>

packages/integrations/mdx/test/mdx-page.test.js

@@ -1,5 +1,6 @@
 import * as assert from 'node:assert/strict';
 import { after, before, describe, it } from 'node:test';
+import * as cheerio from 'cheerio';
 import { parseHTML } from 'linkedom';
 import { loadFixture } from '../../../astro/test/test-utils.js';
 
@@ -36,6 +37,23 @@ describe('MDX Page', () => {
 
 			assert.notEqual(stylesheet, null);
 		});
+
+		it('Renders MDX in utf-8 by default', async () => {
+			const html = await fixture.readFile('/chinese-encoding/index.html');
+			const $ = cheerio.load(html);
+			assert.equal($('h1').text(), '我的第一篇博客文章');
+			assert.match(html, /<meta charset="utf-8"/);
+		});
+
+		it('Renders MDX with layout frontmatter without utf-8 by default', async () => {
+			const html = await fixture.readFile('/chinese-encoding-layout-frontmatter/index.html');
+			assert.doesNotMatch(html, /<meta charset="utf-8"/);
+		});
+
+		it('Renders MDX with layout manual import without utf-8 by default', async () => {
+			const html = await fixture.readFile('/chinese-encoding-layout-manual/index.html');
+			assert.doesNotMatch(html, /<meta charset="utf-8"/);
+		});
 	});
 
 	describe('dev', () => {
@@ -61,5 +79,31 @@ describe('MDX Page', () => {
 
 			assert.equal(h1.textContent, 'Hello page!');
 		});
+
+		it('Renders MDX in utf-8 by default', async () => {
+			const res = await fixture.fetch('/chinese-encoding/');
+			assert.equal(res.status, 200);
+			const html = await res.text();
+			const $ = cheerio.load(html);
+			assert.equal($('h1').text(), '我的第一篇博客文章');
+			assert.doesNotMatch(res.headers.get('content-type'), /charset=utf-8/);
+			assert.match(html, /<meta charset="utf-8"/);
+		});
+
+		it('Renders MDX with layout frontmatter without utf-8 by default', async () => {
+			const res = await fixture.fetch('/chinese-encoding-layout-frontmatter/');
+			assert.equal(res.status, 200);
+			const html = await res.text();
+			assert.doesNotMatch(res.headers.get('content-type'), /charset=utf-8/);
+			assert.doesNotMatch(html, /<meta charset="utf-8"/);
+		});
+
+		it('Renders MDX with layout manual import without utf-8 by default', async () => {
+			const res = await fixture.fetch('/chinese-encoding-layout-manual/');
+			assert.equal(res.status, 200);
+			const html = await res.text();
+			assert.doesNotMatch(res.headers.get('content-type'), /charset=utf-8/);
+			assert.doesNotMatch(html, /<meta charset="utf-8"/);
+		});
 	});
 });