From e9a72d9a91a3741566866bcaab11172cb0dc7d31 Mon Sep 17 00:00:00 2001 From: Degreat Date: Wed, 17 Jan 2024 13:13:05 +0000 Subject: [PATCH] Bump shikiji, use transformers API, expose transformers API (#9643) * Bump shikiji, use transformers API, expose transformers API * update astro config schema * include shikiji-core * Use default import * address css-variables theme * Remove shikiji markdoc * Improve schema transformers handling * Fix tests * Update changeset * bump shikiji version * Update .changeset/six-scissors-worry.md Co-authored-by: Sarah Rainsberger * Update wording Co-authored-by: Sarah Rainsberger --------- Co-authored-by: bluwy Co-authored-by: Emanuele Stoppa Co-authored-by: Sarah Rainsberger --- .changeset/polite-dogs-join.md | 5 + .changeset/six-scissors-worry.md | 8 ++ packages/astro/package.json | 3 +- packages/astro/src/core/config/schema.ts | 6 + packages/astro/src/core/errors/dev/vite.ts | 19 ++- packages/astro/src/core/errors/overlay.ts | 4 +- packages/integrations/markdoc/package.json | 1 - packages/markdown/remark/package.json | 2 +- packages/markdown/remark/src/index.ts | 1 + packages/markdown/remark/src/shiki.ts | 147 +++++++++++---------- packages/markdown/remark/src/types.ts | 8 +- pnpm-lock.yaml | 23 ++-- 12 files changed, 127 insertions(+), 100 deletions(-) create mode 100644 .changeset/polite-dogs-join.md create mode 100644 .changeset/six-scissors-worry.md diff --git a/.changeset/polite-dogs-join.md b/.changeset/polite-dogs-join.md new file mode 100644 index 0000000000..3bb0128d62 --- /dev/null +++ b/.changeset/polite-dogs-join.md @@ -0,0 +1,5 @@ +--- +"@astrojs/markdoc": patch +--- + +Removes unnecessary `shikiji` dependency diff --git a/.changeset/six-scissors-worry.md b/.changeset/six-scissors-worry.md new file mode 100644 index 0000000000..ebba0da66d --- /dev/null +++ b/.changeset/six-scissors-worry.md @@ -0,0 +1,8 @@ +--- +"@astrojs/markdown-remark": minor +"astro": minor +--- + +Adds a new `markdown.shikiConfig.transformers` config option. You can use this option to transform the Shikiji hast (AST format of the generated HTML) to customize the final HTML. Also updates Shikiji to the latest stable version. + +See [Shikiji's documentation](https://shikiji.netlify.app/guide/transformers) for more details about creating your own custom transformers, and [a list of common transformers](https://shikiji.netlify.app/packages/transformers) you can add directly to your project. diff --git a/packages/astro/package.json b/packages/astro/package.json index 7ca4923678..e1743f8f64 100644 --- a/packages/astro/package.json +++ b/packages/astro/package.json @@ -166,7 +166,7 @@ "resolve": "^1.22.4", "semver": "^7.5.4", "server-destroy": "^1.0.1", - "shikiji": "^0.6.13", + "shikiji": "^0.9.18", "string-width": "^7.0.0", "strip-ansi": "^7.1.0", "tsconfck": "^3.0.0", @@ -224,6 +224,7 @@ "remark-code-titles": "^0.1.2", "rollup": "^4.5.0", "sass": "^1.69.5", + "shikiji-core": "^0.9.18", "srcset-parse": "^1.1.0", "unified": "^11.0.4" }, diff --git a/packages/astro/src/core/config/schema.ts b/packages/astro/src/core/config/schema.ts index 7be8043c08..1f021f5680 100644 --- a/packages/astro/src/core/config/schema.ts +++ b/packages/astro/src/core/config/schema.ts @@ -17,9 +17,11 @@ import { appendForwardSlash, prependForwardSlash, removeTrailingForwardSlash } f // This import is required to appease TypeScript! // See https://github.com/withastro/astro/pull/8762 import 'mdast-util-to-hast'; +import 'shikiji-core'; type ShikiLangs = NonNullable; type ShikiTheme = NonNullable; +type ShikiTransformers = NonNullable; const ASTRO_CONFIG_DEFAULTS = { root: '.', @@ -275,6 +277,10 @@ export const AstroConfigSchema = z.object({ ) .default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.experimentalThemes!), wrap: z.boolean().or(z.null()).default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.wrap!), + transformers: z + .custom() + .array() + .default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.transformers!), }) .default({}), remarkPlugins: z diff --git a/packages/astro/src/core/errors/dev/vite.ts b/packages/astro/src/core/errors/dev/vite.ts index 1bd4481451..bfdd043a75 100644 --- a/packages/astro/src/core/errors/dev/vite.ts +++ b/packages/astro/src/core/errors/dev/vite.ts @@ -1,7 +1,6 @@ -import { replaceCssVariables } from '@astrojs/markdown-remark'; import * as fs from 'node:fs'; import { fileURLToPath } from 'node:url'; -import { codeToHtml } from 'shikiji'; +import { codeToHtml, createCssVariablesTheme } from 'shikiji'; import type { ErrorPayload } from 'vite'; import type { ModuleLoader } from '../../module-loader/index.js'; import { FailedToLoadModuleSSR, InvalidGlob, MdxIntegrationMissingError } from '../errors-data.js'; @@ -124,7 +123,11 @@ export interface AstroErrorPayload { // Map these to `.js` during error highlighting. const ALTERNATIVE_JS_EXTS = ['cjs', 'mjs']; const ALTERNATIVE_MD_EXTS = ['mdoc']; -const INLINE_STYLE_SELECTOR_GLOBAL = /style="(.*?)"/g; + +let _cssVariablesTheme: ReturnType; +const cssVariablesTheme = () => + _cssVariablesTheme ?? + (_cssVariablesTheme = createCssVariablesTheme({ variablePrefix: '--astro-code-' })); /** * Generate a payload for Vite's error overlay @@ -147,21 +150,15 @@ export async function getViteErrorPayload(err: ErrorWithMetadata): Promise - replaceCssVariables(m) - ); - } - return { type: 'error', err: { diff --git a/packages/astro/src/core/errors/overlay.ts b/packages/astro/src/core/errors/overlay.ts index 1062680bd9..282c79cd6a 100644 --- a/packages/astro/src/core/errors/overlay.ts +++ b/packages/astro/src/core/errors/overlay.ts @@ -68,7 +68,7 @@ const style = /* css */ ` --toggle-border-color: #C3CADB; /* Syntax Highlighting */ - --astro-code-color-text: #000000; + --astro-code-foreground: #000000; --astro-code-token-constant: #4ca48f; --astro-code-token-string: #9f722a; --astro-code-token-comment: #8490b5; @@ -121,7 +121,7 @@ const style = /* css */ ` --toggle-border-color: #3D4663; /* Syntax Highlighting */ - --astro-code-color-text: #ffffff; + --astro-code-foreground: #ffffff; --astro-code-token-constant: #90f4e3; --astro-code-token-string: #f4cf90; --astro-code-token-comment: #8490b5; diff --git a/packages/integrations/markdoc/package.json b/packages/integrations/markdoc/package.json index d3edca0afc..0c6a997f03 100644 --- a/packages/integrations/markdoc/package.json +++ b/packages/integrations/markdoc/package.json @@ -71,7 +71,6 @@ "gray-matter": "^4.0.3", "htmlparser2": "^9.0.0", "kleur": "^4.1.5", - "shikiji": "^0.6.13", "zod": "^3.22.4" }, "peerDependencies": { diff --git a/packages/markdown/remark/package.json b/packages/markdown/remark/package.json index c65a582e77..8594fcb2fd 100644 --- a/packages/markdown/remark/package.json +++ b/packages/markdown/remark/package.json @@ -38,7 +38,7 @@ "remark-parse": "^11.0.0", "remark-rehype": "^11.0.0", "remark-smartypants": "^2.0.0", - "shikiji": "^0.6.13", + "shikiji": "^0.9.18", "unified": "^11.0.4", "unist-util-visit": "^5.0.0", "vfile": "^6.0.1" diff --git a/packages/markdown/remark/src/index.ts b/packages/markdown/remark/src/index.ts index 1430158479..7881614e5d 100644 --- a/packages/markdown/remark/src/index.ts +++ b/packages/markdown/remark/src/index.ts @@ -36,6 +36,7 @@ export const markdownConfigDefaults: Required = { theme: 'github-dark', experimentalThemes: {}, wrap: false, + transformers: [], }, remarkPlugins: [], rehypePlugins: [], diff --git a/packages/markdown/remark/src/shiki.ts b/packages/markdown/remark/src/shiki.ts index 477ab2184b..04b1bbcf6a 100644 --- a/packages/markdown/remark/src/shiki.ts +++ b/packages/markdown/remark/src/shiki.ts @@ -1,4 +1,4 @@ -import { bundledLanguages, getHighlighter } from 'shikiji'; +import { bundledLanguages, createCssVariablesTheme, getHighlighter } from 'shikiji'; import { visit } from 'unist-util-visit'; import type { ShikiConfig } from './types.js'; @@ -6,32 +6,32 @@ export interface ShikiHighlighter { highlight(code: string, lang?: string, options?: { inline?: boolean }): string; } +// TODO: Remove this special replacement in Astro 5 const ASTRO_COLOR_REPLACEMENTS: Record = { - '#000001': 'var(--astro-code-color-text)', - '#000002': 'var(--astro-code-color-background)', - '#000004': 'var(--astro-code-token-constant)', - '#000005': 'var(--astro-code-token-string)', - '#000006': 'var(--astro-code-token-comment)', - '#000007': 'var(--astro-code-token-keyword)', - '#000008': 'var(--astro-code-token-parameter)', - '#000009': 'var(--astro-code-token-function)', - '#000010': 'var(--astro-code-token-string-expression)', - '#000011': 'var(--astro-code-token-punctuation)', - '#000012': 'var(--astro-code-token-link)', + '--astro-code-foreground': '--astro-code-color-text', + '--astro-code-background': '--astro-code-color-background', }; const COLOR_REPLACEMENT_REGEX = new RegExp( `(${Object.keys(ASTRO_COLOR_REPLACEMENTS).join('|')})`, 'g' ); +let _cssVariablesTheme: ReturnType; +const cssVariablesTheme = () => + _cssVariablesTheme ?? + (_cssVariablesTheme = createCssVariablesTheme({ variablePrefix: '--astro-code-' })); + export async function createShikiHighlighter({ langs = [], theme = 'github-dark', experimentalThemes = {}, wrap = false, + transformers = [], }: ShikiConfig = {}): Promise { const themes = experimentalThemes; + theme = theme === 'css-variables' ? cssVariablesTheme() : theme; + const highlighter = await getHighlighter({ langs: langs.length ? langs : Object.keys(bundledLanguages), themes: Object.values(themes).length ? Object.values(themes) : [theme], @@ -53,74 +53,77 @@ export async function createShikiHighlighter({ return highlighter.codeToHtml(code, { ...themeOptions, lang, - transforms: { - pre(node) { - // Swap to `code` tag if inline - if (inline) { - node.tagName = 'code'; - } + transformers: [ + { + pre(node) { + // Swap to `code` tag if inline + if (inline) { + node.tagName = 'code'; + } - // Cast to string as shikiji will always pass them as strings instead of any other types - const classValue = (node.properties.class as string) ?? ''; - const styleValue = (node.properties.style as string) ?? ''; + // Cast to string as shikiji will always pass them as strings instead of any other types + const classValue = (node.properties.class as string) ?? ''; + const styleValue = (node.properties.style as string) ?? ''; - // Replace "shiki" class naming with "astro-code" - node.properties.class = classValue.replace(/shiki/g, 'astro-code'); + // Replace "shiki" class naming with "astro-code" + node.properties.class = classValue.replace(/shiki/g, 'astro-code'); - // Handle code wrapping - // if wrap=null, do nothing. - if (wrap === false) { - node.properties.style = styleValue + '; overflow-x: auto;'; - } else if (wrap === true) { - node.properties.style = - styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;'; - } - }, - line(node) { - // Add "user-select: none;" for "+"/"-" diff symbols. - // Transform `+ something - // into `+ something` - if (lang === 'diff') { - const innerSpanNode = node.children[0]; - const innerSpanTextNode = - innerSpanNode?.type === 'element' && innerSpanNode.children?.[0]; + // Handle code wrapping + // if wrap=null, do nothing. + if (wrap === false) { + node.properties.style = styleValue + '; overflow-x: auto;'; + } else if (wrap === true) { + node.properties.style = + styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;'; + } + }, + line(node) { + // Add "user-select: none;" for "+"/"-" diff symbols. + // Transform `+ something + // into `+ something` + if (lang === 'diff') { + const innerSpanNode = node.children[0]; + const innerSpanTextNode = + innerSpanNode?.type === 'element' && innerSpanNode.children?.[0]; - if (innerSpanTextNode && innerSpanTextNode.type === 'text') { - const start = innerSpanTextNode.value[0]; - if (start === '+' || start === '-') { - innerSpanTextNode.value = innerSpanTextNode.value.slice(1); - innerSpanNode.children.unshift({ - type: 'element', - tagName: 'span', - properties: { style: 'user-select: none;' }, - children: [{ type: 'text', value: start }], - }); + if (innerSpanTextNode && innerSpanTextNode.type === 'text') { + const start = innerSpanTextNode.value[0]; + if (start === '+' || start === '-') { + innerSpanTextNode.value = innerSpanTextNode.value.slice(1); + innerSpanNode.children.unshift({ + type: 'element', + tagName: 'span', + properties: { style: 'user-select: none;' }, + children: [{ type: 'text', value: start }], + }); + } } } - } - }, - code(node) { - if (inline) { - return node.children[0] as typeof node; - } - }, - root(node) { - if (Object.values(experimentalThemes).length) { - return; - } + }, + code(node) { + if (inline) { + return node.children[0] as typeof node; + } + }, + root(node) { + if (Object.values(experimentalThemes).length) { + return; + } - // theme.id for shiki -> shikiji compat - const themeName = typeof theme === 'string' ? theme : theme.name; - if (themeName === 'css-variables') { - // Replace special color tokens to CSS variables - visit(node as any, 'element', (child) => { - if (child.properties?.style) { - child.properties.style = replaceCssVariables(child.properties.style); - } - }); - } + // theme.id for shiki -> shikiji compat + const themeName = typeof theme === 'string' ? theme : theme.name; + if (themeName === 'css-variables') { + // Replace special color tokens to CSS variables + visit(node as any, 'element', (child) => { + if (child.properties?.style) { + child.properties.style = replaceCssVariables(child.properties.style); + } + }); + } + }, }, - }, + ...transformers, + ], }); }, }; diff --git a/packages/markdown/remark/src/types.ts b/packages/markdown/remark/src/types.ts index ab5af8ed13..69e6d52009 100644 --- a/packages/markdown/remark/src/types.ts +++ b/packages/markdown/remark/src/types.ts @@ -4,6 +4,7 @@ import type { Options as RemarkRehypeOptions } from 'remark-rehype'; import type { BuiltinTheme, LanguageRegistration, + ShikijiTransformer, ThemeRegistration, ThemeRegistrationRaw, } from 'shikiji'; @@ -32,11 +33,14 @@ export type RehypePlugins = (string | [string, any] | RehypePlugin | [RehypePlug export type RemarkRehype = RemarkRehypeOptions; +export type ThemePresets = BuiltinTheme | 'css-variables'; + export interface ShikiConfig { langs?: LanguageRegistration[]; - theme?: BuiltinTheme | ThemeRegistration | ThemeRegistrationRaw; - experimentalThemes?: Record; + theme?: ThemePresets | ThemeRegistration | ThemeRegistrationRaw; + experimentalThemes?: Record; wrap?: boolean | null; + transformers?: ShikijiTransformer[]; } export interface AstroMarkdownOptions { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a27e801548..8027ea4346 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -645,8 +645,8 @@ importers: specifier: ^1.0.1 version: 1.0.1 shikiji: - specifier: ^0.6.13 - version: 0.6.13 + specifier: ^0.9.18 + version: 0.9.18 string-width: specifier: ^7.0.0 version: 7.0.0 @@ -808,6 +808,9 @@ importers: sass: specifier: ^1.69.5 version: 1.69.6 + shikiji-core: + specifier: ^0.9.18 + version: 0.9.18 srcset-parse: specifier: ^1.1.0 version: 1.1.0 @@ -3844,9 +3847,6 @@ importers: kleur: specifier: ^4.1.5 version: 4.1.5 - shikiji: - specifier: ^0.6.13 - version: 0.6.13 zod: specifier: ^3.22.4 version: 3.22.4 @@ -5017,8 +5017,8 @@ importers: specifier: ^2.0.0 version: 2.0.0 shikiji: - specifier: ^0.6.13 - version: 0.6.13 + specifier: ^0.9.18 + version: 0.9.18 unified: specifier: ^11.0.4 version: 11.0.4 @@ -14288,10 +14288,13 @@ packages: vscode-textmate: 5.2.0 dev: true - /shikiji@0.6.13: - resolution: {integrity: sha512-4T7X39csvhT0p7GDnq9vysWddf2b6BeioiN3Ymhnt3xcy9tXmDcnsEFVxX18Z4YcQgEE/w48dLJ4pPPUcG9KkA==} + /shikiji-core@0.9.18: + resolution: {integrity: sha512-PKTXptbrp/WEDjNHV8OFG9KkfhmR0pSd161kzlDDlgQ0HXAnqJYNDSjqsy1CYZMx5bSvLMy42yJj9oFTqmkNTQ==} + + /shikiji@0.9.18: + resolution: {integrity: sha512-/tFMIdV7UQklzN13VjF0/XFzmii6C606Jc878hNezvB8ZR8FG8FW9j0I4J9EJre0owlnPntgLVPpHqy27Gs+DQ==} dependencies: - hast-util-to-html: 9.0.0 + shikiji-core: 0.9.18 dev: false /side-channel@1.0.4: