Add Shiki as an alternative to Prism (#2497)

* [ci] yarn format

* Added shiki to markdown-remark

* Upgraded astro shiki

* Added minimal example

* Changed defaults to match <Code />

* Replace `shiki` with `astro` classes

* Added documentation

* Updated Astro code to use new `codeToHtml`

* Added changesets

* Added basic test

* Updated tests a bit

Co-authored-by: JuanM04 <JuanM04@users.noreply.github.com>

.changeset/rotten-planets-love.md

@@ -0,0 +1,5 @@
+---
+'@astrojs/markdown-remark': patch
+---
+
+Add Shiki as an alternative to Prism

.changeset/tiny-owls-dress.md

@@ -0,0 +1,5 @@
+---
+'astro': patch
+---
+
+Bumped Shiki version

docs/src/pages/en/guides/markdown-content.md

@@ -33,7 +33,6 @@ In addition to custom components inside the [`<Markdown>` component](/en/guides/
 - [GitHub-flavored Markdown](https://github.com/remarkjs/remark-gfm)
 - [remark-smartypants](https://github.com/silvenon/remark-smartypants)
 - [rehype-slug](https://github.com/rehypejs/rehype-slug)
-- [Prism](https://prismjs.com/)
 
 Also, Astro supports third-party plugins for Markdown. You can provide your plugins in `astro.config.mjs`.
 
@@ -85,6 +84,27 @@ export default {
 };
 ```
 
+### Syntax Highlighting
+
+Astro comes with built-in support for [Prism](https://prismjs.com/) and [Shiki](https://shiki.matsu.io/). By default, Prism is enabled. You can modify this behavior by updating the `@astrojs/markdown-remark` options:
+
+```js
+// astro.config.mjs
+export default {
+  markdownOptions: {
+    render: [
+      '@astrojs/markdown-remark',
+      {
+        // Pick a syntax highlighter. Can be 'prism' (default), 'shiki' or false to disable any highlighting.
+        syntaxHighlight: 'prism',
+        // If you are using shiki, here you can define a global theme.
+        shikiTheme: 'github-dark',
+      },
+    ],
+  },
+};
+```
+
 ## Markdown Pages
 
 Astro treats any `.md` files inside of the `/src/pages` directory as pages. These files can contain frontmatter, but are otherwise processed as plain markdown files and do not support components. If you're looking to embed rich components in your markdown, take a look at the [Markdown Component](#astros-markdown-component) section.

examples/with-markdown-shiki/.gitignore

@@ -0,0 +1,17 @@
+# build output
+dist
+
+# dependencies
+node_modules/
+
+# logs
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# environment variables
+.env
+.env.production
+
+# macOS-specific files
+.DS_Store

examples/with-markdown-shiki/.npmrc

@@ -0,0 +1,2 @@
+## force pnpm to hoist 
+shamefully-hoist = true

examples/with-markdown-shiki/.stackblitzrc

@@ -0,0 +1,6 @@
+{
+  "startCommand": "npm start",
+  "env": {
+    "ENABLE_CJS_IMPORTS": true
+  }
+}

examples/with-markdown-shiki/README.md

@@ -0,0 +1,12 @@
+# Astro Example: Markdown with Shiki
+
+```
+npm init astro -- --template with-markdown-shiki
+```
+
+[![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/github/withastro/astro/tree/latest/examples/with-markdown)
+
+This example showcases Astro's [built-in Markdown support](../../docs/markdown.md).
+
+- `src/pages/index.astro` uses Astro's `<Markdown>` component.
+- `src/pages/other.md` is a treated as a page entrypoint and uses a `layout`.

examples/with-markdown-shiki/astro.config.mjs

@@ -0,0 +1,22 @@
+// Full Astro Configuration API Documentation:
+// https://docs.astro.build/reference/configuration-reference
+
+// @type-check enabled!
+// VSCode and other TypeScript-enabled text editors will provide auto-completion,
+// helpful tooltips, and warnings if your exported object is invalid.
+// You can disable this by removing "@ts-check" and `@type` comments below.
+import astroRemark from '@astrojs/markdown-remark';
+
+// @ts-check
+export default /** @type {import('astro').AstroUserConfig} */ ({
+	// Enable Custom Markdown options, plugins, etc.
+	markdownOptions: {
+		render: [
+			astroRemark,
+			{
+				syntaxHighlight: 'shiki',
+				shikiTheme: 'dracula',
+			},
+		],
+	},
+});

examples/with-markdown-shiki/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "@example/with-markdown-shiki",
+  "version": "0.0.1",
+  "private": true,
+  "scripts": {
+    "dev": "astro dev",
+    "start": "astro dev",
+    "build": "astro build",
+    "preview": "astro preview"
+  },
+  "devDependencies": {
+    "astro": "^0.22.20"
+  }
+}

examples/with-markdown-shiki/sandbox.config.json

@@ -0,0 +1,11 @@
+{
+  "infiniteLoopProtection": true,
+  "hardReloadOnChange": false,
+  "view": "browser",
+  "template": "node",
+  "container": {
+    "port": 3000,
+    "startScript": "start",
+    "node": "14"
+  }
+}

examples/with-markdown-shiki/src/layouts/main.astro

@@ -0,0 +1,20 @@
+---
+const { content } = Astro.props;
+---
+
+<html lang={content.lang || 'en'}>
+	<head>
+		<meta charset="utf-8" />
+
+		<link rel="icon" type="image/x-icon" href="/favicon.ico" />
+
+		<title>{content.title}</title>
+
+		<style global>
+			@import "../styles/global.css";
+		</style>
+	</head>
+	<body>
+		<slot />
+	</body>
+</html>

examples/with-markdown-shiki/src/pages/index.md

@@ -0,0 +1,14 @@
+---
+title: Shiki demo
+layout: ../layouts/main.astro
+---
+
+# Shiki demo
+
+```js
+var foo = 'bar';
+
+function doSomething() {
+  return foo;
+}
+```

examples/with-markdown-shiki/src/styles/global.css

@@ -0,0 +1,54 @@
+pre,
+code {
+	color: #d4d4d4;
+	font-size: 14px;
+	font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace;
+	line-height: 1.5;
+	direction: ltr;
+	white-space: pre;
+	text-align: left;
+	text-shadow: none;
+	word-break: normal;
+	word-spacing: normal;
+	-moz-tab-size: 4;
+	-o-tab-size: 4;
+	tab-size: 4;
+	-webkit-hyphens: none;
+	-moz-hyphens: none;
+	-ms-hyphens: none;
+	hyphens: none;
+}
+
+pre::selection,
+code::selection {
+	text-shadow: none;
+	background: #b3d4fc;
+}
+
+@media print {
+	pre,
+	code {
+		text-shadow: none;
+	}
+}
+
+pre {
+	margin: 0.5rem 0 16px;
+	padding: 0.8rem 1rem 0.9rem;
+	overflow: auto;
+	background: #282a36;
+	border-radius: 4px;
+}
+
+:not(pre) > code {
+	padding: 0.1em 0.3em;
+	color: #db4c69;
+	background: #f9f2f4;
+	border-radius: 0.3em;
+	white-space: pre-wrap;
+}
+
+body {
+	max-width: 900px;
+	margin: auto;
+}

examples/with-markdown-shiki/tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "compilerOptions": {
+    "moduleResolution": "node"
+  }
+}

packages/astro/components/Code.astro

@@ -43,7 +43,7 @@ function repairShikiTheme(html: string): string {
 }
 
 const highlighter = await shiki.getHighlighter({ theme });
-const _html = highlighter.codeToHtml(code, lang);
+const _html = highlighter.codeToHtml(code, { lang });
 const html = repairShikiTheme(_html);
 ---
 

packages/astro/package.json

@@ -97,7 +97,7 @@
     "sass": "^1.43.4",
     "semver": "^7.3.5",
     "send": "^0.17.1",
-    "shiki": "^0.9.10",
+    "shiki": "^0.10.0",
     "shorthash": "^0.0.2",
     "slash": "^4.0.0",
     "sourcemap-codec": "^1.4.8",

packages/astro/test/astro-markdown-shiki.test.js

@@ -0,0 +1,53 @@
+import { expect } from 'chai';
+import cheerio from 'cheerio';
+import { loadFixture } from './test-utils.js';
+import markdownRemark from '@astrojs/markdown-remark';
+
+describe('Astro Markdown Shiki', () => {
+	let fixture;
+
+	before(async () => {
+		fixture = await loadFixture({
+			projectRoot: './fixtures/astro-markdown-shiki/',
+			markdownOptions: {
+				render: [
+					markdownRemark,
+					{
+						syntaxHighlight: 'shiki',
+						shikiTheme: 'github-light',
+					},
+				],
+			},
+			buildOptions: {
+				sitemap: false,
+			},
+		});
+		await fixture.build();
+	});
+
+	it('Can render markdown with shiki', async () => {
+		const html = await fixture.readFile('/index.html');
+		const $ = cheerio.load(html);
+
+		// There should be no HTML from Prism
+		expect($('.token')).to.have.lengthOf(0);
+
+		expect($('pre')).to.have.lengthOf(1);
+		expect($('pre').hasClass('astro-code')).to.equal(true);
+		expect($('pre').attr().style).to.equal('background-color: #ffffff');
+	});
+
+	it('Can render Astro <Markdown> with shiki', async () => {
+		const html = await fixture.readFile('/astro/index.html');
+		const $ = cheerio.load(html);
+
+		// There should be no HTML from Prism
+		expect($('.token')).to.have.lengthOf(0);
+
+		expect($('pre')).to.have.lengthOf(2);
+
+		expect($('span.line')).to.have.lengthOf(2);
+		expect($('span.line').get(0).children).to.have.lengthOf(1);
+		expect($('span.line').get(1).children).to.have.lengthOf(5);
+	});
+});

packages/astro/test/fixtures/astro-markdown-shiki/src/layouts/content.astro

@@ -0,0 +1,10 @@
+<html>
+  <head>
+    <!-- Head Stuff -->
+  </head>
+  <body>
+    <div class="container">
+      <slot></slot>
+    </div>
+  </body>
+</html>

packages/astro/test/fixtures/astro-markdown-shiki/src/pages/astro.astro

@@ -0,0 +1,18 @@
+---
+import { Markdown } from 'astro/components';
+import Layout from '../layouts/content.astro';
+---
+
+<Layout>
+	<Markdown>
+		# Hello world
+
+		```
+			plaintext
+		```
+
+		```js
+			console.log('JavaScript')
+		```
+	</Markdown>
+</Layout>

packages/astro/test/fixtures/astro-markdown-shiki/src/pages/index.md

@@ -0,0 +1,24 @@
+---
+layout: ../layouts/content.astro
+---
+
+# Hello world
+
+```yaml
+apiVersion: v3
+kind: Pod
+metadata:
+  name: rss-site
+  labels:
+    app: web
+spec:
+  containers:
+    - name: front-end
+      image: nginx
+      ports:
+        - containerPort: 80
+    - name: rss-reader
+      image: nickchase/rss-php-nginx:v1
+      ports:
+        - containerPort: 88
+```

packages/markdown/remark/package.json

@@ -38,6 +38,7 @@
     "remark-parse": "^10.0.1",
     "remark-rehype": "^10.0.1",
     "remark-smartypants": "^2.0.0",
+    "shiki": "^0.10.0",
     "unified": "^10.1.1",
     "unist-util-map": "^3.0.0",
     "unist-util-visit": "^4.1.0"

packages/markdown/remark/src/index.ts

@@ -9,6 +9,7 @@ import { remarkJsx, loadRemarkJsx } from './remark-jsx.js';
 import rehypeJsx from './rehype-jsx.js';
 import rehypeEscape from './rehype-escape.js';
 import remarkPrism from './remark-prism.js';
+import remarkShiki from './remark-shiki.js';
 import remarkUnwrap from './remark-unwrap.js';
 import { loadPlugins } from './load-plugins.js';
 
@@ -37,6 +38,8 @@ export async function renderMarkdown(content: string, opts?: MarkdownRenderingOp
 	let { remarkPlugins = [], rehypePlugins = [] } = opts ?? {};
 	const scopedClassName = opts?.$?.scopedClassName;
 	const mode = opts?.mode ?? 'mdx';
+	const syntaxHighlight = opts?.syntaxHighlight ?? 'prism';
+	const shikiTheme = opts?.shikiTheme ?? 'github-dark';
 	const isMDX = mode === 'mdx';
 	const { headers, rehypeCollectHeaders } = createCollectHeaders();
 
@@ -64,7 +67,12 @@ export async function renderMarkdown(content: string, opts?: MarkdownRenderingOp
 		parser.use([scopedStyles(scopedClassName)]);
 	}
 
-	parser.use([remarkPrism(scopedClassName)]);
+	if (syntaxHighlight === 'prism') {
+		parser.use([remarkPrism(scopedClassName)]);
+	} else if (syntaxHighlight === 'shiki') {
+		parser.use([await remarkShiki(shikiTheme)]);
+	}
+
 	parser.use([[markdownToHtml as any, { allowDangerousHtml: true, passThrough: ['raw', 'mdxTextExpression', 'mdxJsxTextElement', 'mdxJsxFlowElement'] }]]);
 
 	loadedRehypePlugins.forEach(([plugin, opts]) => {

packages/markdown/remark/src/remark-shiki.ts

@@ -0,0 +1,23 @@
+import shiki from 'shiki';
+import { visit } from 'unist-util-visit';
+
+const remarkShiki = async (theme: shiki.Theme) => {
+	const highlighter = await shiki.getHighlighter({ theme });
+
+	return () => (tree: any) => {
+		visit(tree, 'code', (node) => {
+			let html = highlighter.codeToHtml(node.value, { lang: node.lang ?? 'plaintext' });
+
+			// Replace "shiki" class naming with "astro".
+			html = html.replace('<pre class="shiki"', '<pre class="astro-code"');
+			// Replace "shiki" css variable naming with "astro".
+			html = html.replace(/style="(background-)?color: var\(--shiki-/g, 'style="$1color: var(--astro-code-');
+
+			node.type = 'html';
+			node.value = html;
+			node.children = [];
+		});
+	};
+};
+
+export default remarkShiki;

packages/markdown/remark/src/types.ts

@@ -1,10 +1,13 @@
-import * as unified from 'unified';
+import type * as unified from 'unified';
+import type * as shiki from 'shiki';
 
 export type UnifiedPluginImport = Promise<{ default: unified.Plugin }>;
 export type Plugin = string | [string, any] | UnifiedPluginImport | [UnifiedPluginImport, any];
 
 export interface AstroMarkdownOptions {
 	mode?: 'md' | 'mdx';
+	syntaxHighlight?: 'prism' | 'shiki' | false;
+	shikiTheme?: shiki.Theme;
 	remarkPlugins?: Plugin[];
 	rehypePlugins?: Plugin[];
 }

yarn.lock

@@ -8152,10 +8152,10 @@ shell-quote@^1.6.1:
   resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.7.3.tgz#aa40edac170445b9a431e17bb62c0b881b9c4123"
   integrity sha512-Vpfqwm4EnqGdlsBFNmHhxhElJYrdfcxPThu+ryKS5J8L/fhAwLazFZtq+S+TWZ9ANj2piSQLGj6NQg+lKPmxrw==
 
-shiki@^0.9.10:
-  version "0.9.15"
-  resolved "https://registry.yarnpkg.com/shiki/-/shiki-0.9.15.tgz#2481b46155364f236651319d2c18e329ead6fa44"
-  integrity sha512-/Y0z9IzhJ8nD9nbceORCqu6NgT9X6I8Fk8c3SICHI5NbZRLdZYFaB233gwct9sU0vvSypyaL/qaKvzyQGJBZSw==
+shiki@^0.10.0:
+  version "0.10.0"
+  resolved "https://registry.yarnpkg.com/shiki/-/shiki-0.10.0.tgz#85f21ecfa95b377ff64db6c71442c22c220e9540"
+  integrity sha512-iczxaIYeBFHTFrQPb9DVy2SKgYxC4Wo7Iucm7C17cCh2Ge/refnvHscUOxM85u57MfLoNOtjoEFUWt9gBexblA==
   dependencies:
     jsonc-parser "^3.0.0"
     vscode-oniguruma "^1.6.1"