From 5a9528741fa98d017b269c7e4f013058028bdc5d Mon Sep 17 00:00:00 2001 From: Bjorn Lu Date: Fri, 8 Mar 2024 18:53:19 +0800 Subject: [PATCH] Move to shiki and stabilize dual themes support (#10130) * Update to shiki * Stabilize shikiConfig.experimentalThemes * Add changeset * Fix merge issues --- .changeset/curvy-hats-shave.md | 5 ++++ .changeset/thin-chicken-greet.md | 6 ++++ examples/with-mdx/src/pages/index.mdx | 2 +- packages/astro/components/Code.astro | 22 +++++++------- packages/astro/package.json | 4 +-- packages/astro/src/@types/astro.ts | 2 +- packages/astro/src/core/config/schema.ts | 23 +++++++++++---- packages/astro/src/core/errors/dev/vite.ts | 2 +- packages/markdown/remark/package.json | 2 +- packages/markdown/remark/src/index.ts | 2 +- packages/markdown/remark/src/shiki.ts | 9 ++---- packages/markdown/remark/src/types.ts | 8 +++--- packages/markdown/remark/test/shiki.test.js | 2 +- pnpm-lock.yaml | 32 ++++++++++----------- 14 files changed, 70 insertions(+), 51 deletions(-) create mode 100644 .changeset/curvy-hats-shave.md create mode 100644 .changeset/thin-chicken-greet.md diff --git a/.changeset/curvy-hats-shave.md b/.changeset/curvy-hats-shave.md new file mode 100644 index 0000000000..a1488e0786 --- /dev/null +++ b/.changeset/curvy-hats-shave.md @@ -0,0 +1,5 @@ +--- +"astro": minor +--- + +Stabilizes `markdown.shikiConfig.experimentalThemes` as `markdown.shikiConfig.themes`. No behaviour changes are made to this option. diff --git a/.changeset/thin-chicken-greet.md b/.changeset/thin-chicken-greet.md new file mode 100644 index 0000000000..a018fd10f2 --- /dev/null +++ b/.changeset/thin-chicken-greet.md @@ -0,0 +1,6 @@ +--- +"@astrojs/markdown-remark": minor +"astro": minor +--- + +Migrates `shikiji` to `shiki` 1.0 diff --git a/examples/with-mdx/src/pages/index.mdx b/examples/with-mdx/src/pages/index.mdx index 13a929624b..df0dd47c3a 100644 --- a/examples/with-mdx/src/pages/index.mdx +++ b/examples/with-mdx/src/pages/index.mdx @@ -18,7 +18,7 @@ Published on: {new Intl.DateTimeFormat('en', {dateStyle: 'long'}).format(publish ## Syntax highlighting -We also support syntax highlighting in MDX out-of-the-box! This example uses our default [Shiki theme](https://github.com/shikijs/shiki). See the [MDX integration docs](https://docs.astro.build/en/guides/integrations-guide/mdx/#syntax-highlighting) for configuration options. +We also support syntax highlighting in MDX out-of-the-box! This example uses the default [Shiki](https://shiki.style) theme. See the [MDX integration docs](https://docs.astro.build/en/guides/integrations-guide/mdx/#syntax-highlighting) for configuration options. ```astro --- diff --git a/packages/astro/components/Code.astro b/packages/astro/components/Code.astro index ab2d88696c..a7f6f725ae 100644 --- a/packages/astro/components/Code.astro +++ b/packages/astro/components/Code.astro @@ -6,8 +6,8 @@ import type { SpecialLanguage, ThemeRegistration, ThemeRegistrationRaw, -} from 'shikiji'; -import { bundledLanguages } from 'shikiji/langs'; +} from 'shiki'; +import { bundledLanguages } from 'shiki/langs'; import { getCachedHighlighter } from '../dist/core/shiki.js'; interface Props { @@ -15,25 +15,25 @@ interface Props { code: string; /** * The language of your code. - * Supports all languages listed here: https://github.com/shikijs/shiki/blob/main/docs/languages.md#all-languages - * Instructions for loading a custom language: https://github.com/shikijs/shiki/blob/main/docs/languages.md#supporting-your-own-languages-with-shiki + * Supports all languages listed here: https://shiki.style/languages + * Instructions for loading a custom language: https://shiki.style/guide/load-lang * * @default "plaintext" */ lang?: BuiltinLanguage | SpecialLanguage | LanguageRegistration; /** * The styling theme. - * Supports all themes listed here: https://github.com/shikijs/shiki/blob/main/docs/themes.md#all-themes - * Instructions for loading a custom theme: https://github.com/shikijs/shiki/blob/main/docs/themes.md#loading-theme + * Supports all themes listed here: https://shiki.style/themes + * Instructions for loading a custom theme: https://shiki.style/guide/load-theme * * @default "github-dark" */ theme?: ThemePresets | ThemeRegistration | ThemeRegistrationRaw; /** * Multiple themes to style with -- alternative to "theme" option. - * Supports all themes found above; see https://github.com/antfu/shikiji#lightdark-dual-themes for more information. + * Supports all themes found above; see https://shiki.style/guide/dual-themes for more information. */ - experimentalThemes?: Record; + themes?: Record; /** * Enable word wrapping. * - true: enabled. @@ -55,12 +55,12 @@ const { code, lang = 'plaintext', theme = 'github-dark', - experimentalThemes = {}, + themes = {}, wrap = false, inline = false, } = Astro.props; -// shiki -> shikiji compat +// shiki 1.0 compat if (typeof lang === 'object') { // `id` renamed to `name` (always override) if ((lang as any).id) { @@ -81,7 +81,7 @@ const highlighter = await getCachedHighlighter({ : lang, ], theme, - experimentalThemes, + themes, wrap, }); diff --git a/packages/astro/package.json b/packages/astro/package.json index d3bb6acd57..42ee8a7415 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -125,6 +125,7 @@ "@babel/traverse": "^7.23.3", "@babel/types": "^7.23.3", "@medv/finder": "^3.1.0", + "@shikijs/core": "^1.1.2", "@types/babel__core": "^7.20.4", "acorn": "^8.11.2", "aria-query": "^5.3.0", @@ -166,8 +167,7 @@ "rehype": "^13.0.1", "resolve": "^1.22.4", "semver": "^7.5.4", - "shikiji": "^0.9.19", - "shikiji-core": "^0.9.19", + "shiki": "^1.1.2", "string-width": "^7.0.0", "strip-ansi": "^7.1.0", "tsconfck": "^3.0.0", diff --git a/packages/astro/src/@types/astro.ts b/packages/astro/src/@types/astro.ts index 89283ce34a..932d656e7c 100644 --- a/packages/astro/src/@types/astro.ts +++ b/packages/astro/src/@types/astro.ts @@ -1294,7 +1294,7 @@ export interface AstroUserConfig { * @default `shiki` * @description * Which syntax highlighter to use, if any. - * - `shiki` - use the [Shiki](https://github.com/shikijs/shiki) highlighter + * - `shiki` - use the [Shiki](https://shiki.style) highlighter * - `prism` - use the [Prism](https://prismjs.com/) highlighter * - `false` - do not apply syntax highlighting. * diff --git a/packages/astro/src/core/config/schema.ts b/packages/astro/src/core/config/schema.ts index fcfd91639b..4c0276ddf7 100644 --- a/packages/astro/src/core/config/schema.ts +++ b/packages/astro/src/core/config/schema.ts @@ -5,7 +5,7 @@ import type { ShikiConfig, } from '@astrojs/markdown-remark'; import { markdownConfigDefaults } from '@astrojs/markdown-remark'; -import { type BuiltinTheme, bundledThemes } from 'shikiji'; +import { type BuiltinTheme, bundledThemes } from 'shiki'; import type { AstroUserConfig, ViteUserConfig } from '../../@types/astro.js'; import type { OutgoingHttpHeaders } from 'node:http'; @@ -14,10 +14,10 @@ import { pathToFileURL } from 'node:url'; import { z } from 'zod'; import { appendForwardSlash, prependForwardSlash, removeTrailingForwardSlash } from '../path.js'; -// This import is required to appease TypeScript! +// These imports are required to appease TypeScript! // See https://github.com/withastro/astro/pull/8762 +import '@shikijs/core'; import 'mdast-util-to-hast'; -import 'shikiji-core'; type ShikiLangs = NonNullable; type ShikiTheme = NonNullable; @@ -250,7 +250,7 @@ export const AstroConfigSchema = z.object({ .array() .transform((langs) => { for (const lang of langs) { - // shiki -> shikiji compat + // shiki 1.0 compat if (typeof lang === 'object') { // `id` renamed to `name` (always override) if ((lang as any).id) { @@ -269,17 +269,28 @@ export const AstroConfigSchema = z.object({ .enum(Object.keys(bundledThemes) as [BuiltinTheme, ...BuiltinTheme[]]) .or(z.custom()) .default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.theme!), - experimentalThemes: z + themes: z .record( z .enum(Object.keys(bundledThemes) as [BuiltinTheme, ...BuiltinTheme[]]) .or(z.custom()) ) - .default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.experimentalThemes!), + .default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.themes!), wrap: z.boolean().or(z.null()).default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.wrap!), transformers: z .custom() .array() + .transform((transformers) => { + for (const transformer of transformers) { + // When updating shikiji to shiki, the `token` property was renamed to `span`. + // We apply the compat here for now until the next Astro major. + // TODO: Remove this in Astro 5.0 + if ((transformer as any).token && !('span' in transformer)) { + transformer.span = (transformer as any).token; + } + } + return transformers; + }) .default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.transformers!), }) .default({}), diff --git a/packages/astro/src/core/errors/dev/vite.ts b/packages/astro/src/core/errors/dev/vite.ts index bfdd043a75..96e53d41c2 100644 --- a/packages/astro/src/core/errors/dev/vite.ts +++ b/packages/astro/src/core/errors/dev/vite.ts @@ -1,6 +1,6 @@ import * as fs from 'node:fs'; import { fileURLToPath } from 'node:url'; -import { codeToHtml, createCssVariablesTheme } from 'shikiji'; +import { codeToHtml, createCssVariablesTheme } from 'shiki'; import type { ErrorPayload } from 'vite'; import type { ModuleLoader } from '../../module-loader/index.js'; import { FailedToLoadModuleSSR, InvalidGlob, MdxIntegrationMissingError } from '../errors-data.js'; diff --git a/packages/markdown/remark/package.json b/packages/markdown/remark/package.json index 2af49fc088..98a2393013 100644 --- a/packages/markdown/remark/package.json +++ b/packages/markdown/remark/package.json @@ -44,7 +44,7 @@ "remark-parse": "^11.0.0", "remark-rehype": "^11.0.0", "remark-smartypants": "^2.0.0", - "shikiji": "^0.9.19", + "shiki": "^1.1.2", "unified": "^11.0.4", "unist-util-visit": "^5.0.0", "vfile": "^6.0.1" diff --git a/packages/markdown/remark/src/index.ts b/packages/markdown/remark/src/index.ts index 7881614e5d..6ab0e5c676 100644 --- a/packages/markdown/remark/src/index.ts +++ b/packages/markdown/remark/src/index.ts @@ -34,7 +34,7 @@ export const markdownConfigDefaults: Required = { shikiConfig: { langs: [], theme: 'github-dark', - experimentalThemes: {}, + themes: {}, wrap: false, transformers: [], }, diff --git a/packages/markdown/remark/src/shiki.ts b/packages/markdown/remark/src/shiki.ts index 26ee22bfa8..c0fc798e98 100644 --- a/packages/markdown/remark/src/shiki.ts +++ b/packages/markdown/remark/src/shiki.ts @@ -1,5 +1,5 @@ import type { Properties } from 'hast'; -import { bundledLanguages, createCssVariablesTheme, getHighlighter } from 'shikiji'; +import { bundledLanguages, createCssVariablesTheme, getHighlighter } from 'shiki'; import { visit } from 'unist-util-visit'; import type { ShikiConfig } from './types.js'; @@ -25,12 +25,10 @@ const cssVariablesTheme = () => export async function createShikiHighlighter({ langs = [], theme = 'github-dark', - experimentalThemes = {}, + themes = {}, wrap = false, transformers = [], }: ShikiConfig = {}): Promise { - const themes = experimentalThemes; - theme = theme === 'css-variables' ? cssVariablesTheme() : theme; const highlighter = await getHighlighter({ @@ -106,11 +104,10 @@ export async function createShikiHighlighter({ } }, root(node) { - if (Object.values(experimentalThemes).length) { + if (Object.values(themes).length) { return; } - // theme.id for shiki -> shikiji compat const themeName = typeof theme === 'string' ? theme : theme.name; if (themeName === 'css-variables') { // Replace special color tokens to CSS variables diff --git a/packages/markdown/remark/src/types.ts b/packages/markdown/remark/src/types.ts index 69e6d52009..1fb89316d6 100644 --- a/packages/markdown/remark/src/types.ts +++ b/packages/markdown/remark/src/types.ts @@ -4,10 +4,10 @@ import type { Options as RemarkRehypeOptions } from 'remark-rehype'; import type { BuiltinTheme, LanguageRegistration, - ShikijiTransformer, + ShikiTransformer, ThemeRegistration, ThemeRegistrationRaw, -} from 'shikiji'; +} from 'shiki'; import type * as unified from 'unified'; import type { VFile } from 'vfile'; @@ -38,9 +38,9 @@ export type ThemePresets = BuiltinTheme | 'css-variables'; export interface ShikiConfig { langs?: LanguageRegistration[]; theme?: ThemePresets | ThemeRegistration | ThemeRegistrationRaw; - experimentalThemes?: Record; + themes?: Record; wrap?: boolean | null; - transformers?: ShikijiTransformer[]; + transformers?: ShikiTransformer[]; } export interface AstroMarkdownOptions { diff --git a/packages/markdown/remark/test/shiki.test.js b/packages/markdown/remark/test/shiki.test.js index 2fb40c217c..c6044b019d 100644 --- a/packages/markdown/remark/test/shiki.test.js +++ b/packages/markdown/remark/test/shiki.test.js @@ -13,7 +13,7 @@ describe('shiki syntax highlighting', () => { it('supports light/dark themes', async () => { const processor = await createMarkdownProcessor({ shikiConfig: { - experimentalThemes: { + themes: { light: 'github-light', dark: 'github-dark', }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index aa4ceb9e3b..e32f61f2b0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -530,6 +530,9 @@ importers: '@medv/finder': specifier: ^3.1.0 version: 3.1.0 + '@shikijs/core': + specifier: ^1.1.2 + version: 1.1.2 '@types/babel__core': specifier: ^7.20.4 version: 7.20.5 @@ -653,12 +656,9 @@ importers: semver: specifier: ^7.5.4 version: 7.5.4 - shikiji: - specifier: ^0.9.19 - version: 0.9.19 - shikiji-core: - specifier: ^0.9.19 - version: 0.9.19 + shiki: + specifier: ^1.1.2 + version: 1.1.2 string-width: specifier: ^7.0.0 version: 7.0.0 @@ -5236,9 +5236,9 @@ importers: remark-smartypants: specifier: ^2.0.0 version: 2.0.0 - shikiji: - specifier: ^0.9.19 - version: 0.9.19 + shiki: + specifier: ^1.1.2 + version: 1.1.2 unified: specifier: ^11.0.4 version: 11.0.4 @@ -7590,6 +7590,10 @@ packages: requiresBuild: true optional: true + /@shikijs/core@1.1.2: + resolution: {integrity: sha512-ERVzNQz88ZkDqUpWeC57Kp+Kmx5RjqeDBR1M8AGWGom4yrkITiTfXCGmjchlDSw12MhDTuPYR4HVFW8uT61RaQ==} + dev: false + /@sinclair/typebox@0.27.8: resolution: {integrity: sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==} dev: false @@ -15025,14 +15029,10 @@ packages: vscode-textmate: 5.2.0 dev: true - /shikiji-core@0.9.19: - resolution: {integrity: sha512-AFJu/vcNT21t0e6YrfadZ+9q86gvPum6iywRyt1OtIPjPFe25RQnYJyxHQPMLKCCWA992TPxmEmbNcOZCAJclw==} - dev: false - - /shikiji@0.9.19: - resolution: {integrity: sha512-Kw2NHWktdcdypCj1GkKpXH4o6Vxz8B8TykPlPuLHOGSV8VkhoCLcFOH4k19K4LXAQYRQmxg+0X/eM+m2sLhAkg==} + /shiki@1.1.2: + resolution: {integrity: sha512-qNzFwTv5uhEDNUIwp7wHjsrffVeLbmOgWnM5mZZhoiz7G2qAUvqVfUzuWfieD45/YAKipzCtdV9SndacKtABow==} dependencies: - shikiji-core: 0.9.19 + '@shikijs/core': 1.1.2 dev: false /side-channel@1.0.4: