diff --git a/.changeset/heavy-pugs-jam.md b/.changeset/heavy-pugs-jam.md
new file mode 100644
index 0000000000..8d200cd913
--- /dev/null
+++ b/.changeset/heavy-pugs-jam.md
@@ -0,0 +1,5 @@
+---
+'astro': patch
+---
+
+Initializes the Markdown processor only when there's `.md` files
diff --git a/.changeset/moody-cougars-taste.md b/.changeset/moody-cougars-taste.md
new file mode 100644
index 0000000000..fd84443460
--- /dev/null
+++ b/.changeset/moody-cougars-taste.md
@@ -0,0 +1,5 @@
+---
+'@astrojs/mdx': patch
+---
+
+Initializes the MDX processor only when there's `.mdx` files
diff --git a/packages/astro/src/vite-plugin-markdown/index.ts b/packages/astro/src/vite-plugin-markdown/index.ts
index 37af2dc50c..23ca78bb16 100644
--- a/packages/astro/src/vite-plugin-markdown/index.ts
+++ b/packages/astro/src/vite-plugin-markdown/index.ts
@@ -31,14 +31,11 @@ const astroErrorModulePath = normalizePath(
 );
 
 export default function markdown({ settings, logger }: AstroPluginOptions): Plugin {
-	let processor: MarkdownProcessor | undefined;
+	let processor: Promise<MarkdownProcessor> | undefined;
 
 	return {
 		enforce: 'pre',
 		name: 'astro:markdown',
-		async buildStart() {
-			processor = await createMarkdownProcessor(settings.config.markdown);
-		},
 		buildEnd() {
 			processor = undefined;
 		},
@@ -61,15 +58,12 @@ export default function markdown({ settings, logger }: AstroPluginOptions): Plug
 
 				const fileURL = pathToFileURL(fileId);
 
-				// `processor` is initialized in `buildStart`, and removed in `buildEnd`. `load`
-				// should be called in between those two lifecycles, so this error should never happen
+				// Lazily initialize the Markdown processor
 				if (!processor) {
-					return this.error(
-						'MDX processor is not initialized. This is an internal error. Please file an issue.',
-					);
+					processor = createMarkdownProcessor(settings.config.markdown);
 				}
 
-				const renderResult = await processor
+				const renderResult = await (await processor)
 					.render(raw.content, {
 						// @ts-expect-error passing internal prop
 						fileURL,
diff --git a/packages/integrations/mdx/src/vite-plugin-mdx.ts b/packages/integrations/mdx/src/vite-plugin-mdx.ts
index 7b37535d8e..5a409d40d5 100644
--- a/packages/integrations/mdx/src/vite-plugin-mdx.ts
+++ b/packages/integrations/mdx/src/vite-plugin-mdx.ts
@@ -9,6 +9,7 @@ import { parseFrontmatter } from './utils.js';
 
 export function vitePluginMdx(mdxOptions: MdxOptions): Plugin {
 	let processor: ReturnType<typeof createMdxProcessor> | undefined;
+	let sourcemapEnabled: boolean;
 
 	return {
 		name: '@mdx-js/rollup',
@@ -17,13 +18,7 @@ export function vitePluginMdx(mdxOptions: MdxOptions): Plugin {
 			processor = undefined;
 		},
 		configResolved(resolved) {
-			// `mdxOptions` should be populated at this point, but `astro sync` doesn't call `astro:config:done` :(
-			// Workaround this for now by skipping here. `astro sync` shouldn't call the `transform()` hook here anyways.
-			if (Object.keys(mdxOptions).length === 0) return;
-
-			processor = createMdxProcessor(mdxOptions, {
-				sourcemap: !!resolved.build.sourcemap,
-			});
+			sourcemapEnabled = !!resolved.build.sourcemap;
 
 			// HACK: Remove the `astro:jsx` plugin if defined as we handle the JSX transformation ourselves
 			const jsxPluginIndex = resolved.plugins.findIndex((p) => p.name === 'astro:jsx');
@@ -51,12 +46,9 @@ export function vitePluginMdx(mdxOptions: MdxOptions): Plugin {
 			// Ensure `data.astro` is available to all remark plugins
 			setVfileFrontmatter(vfile, frontmatter);
 
-			// `processor` is initialized in `configResolved`, and removed in `buildEnd`. `transform`
-			// should be called in between those two lifecycle, so this error should never happen
+			// Lazily initialize the MDX processor
 			if (!processor) {
-				return this.error(
-					'MDX processor is not initialized. This is an internal error. Please file an issue.',
-				);
+				processor = createMdxProcessor(mdxOptions, { sourcemap: sourcemapEnabled });
 			}
 
 			try {