Infrastructure/astro
0
Fork 0
mirror of https://github.com/withastro/astro.git synced 2025-03-31 23:31:30 -05:00
Code Issues Activity

Bump shikiji, use transformers API, expose transformers API (#9643)

Browse source 
* Bump shikiji, use transformers API, expose transformers API

* update astro config schema

* include shikiji-core

* Use default import

* address css-variables theme

* Remove shikiji markdoc

* Improve schema transformers handling

* Fix tests

* Update changeset

* bump shikiji version

* Update .changeset/six-scissors-worry.md

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

* Update wording

Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>

---------

Co-authored-by: bluwy <bjornlu.dev@gmail.com>
Co-authored-by: Emanuele Stoppa <my.burning@gmail.com>
Co-authored-by: Sarah Rainsberger <sarah@rainsberger.ca>
This commit is contained in:
Degreat 2024-01-17 13:13:05 +00:00 • committed by GitHub
parent 8521ff77fb
commit e9a72d9a91
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 127 additions and 100 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.changeset/polite-dogs-join.md Normal file
View file

@ -0,0 +1,5 @@
---
"@astrojs/markdoc": patch
---
Removes unnecessary `shikiji` dependency

8
.changeset/six-scissors-worry.md Normal file
View file

@ -0,0 +1,8 @@
---
"@astrojs/markdown-remark": minor
"astro": minor
---
Adds a new `markdown.shikiConfig.transformers` config option. You can use this option to transform the Shikiji hast (AST format of the generated HTML) to customize the final HTML. Also updates Shikiji to the latest stable version.
See [Shikiji's documentation](https://shikiji.netlify.app/guide/transformers) for more details about creating your own custom transformers, and [a list of common transformers](https://shikiji.netlify.app/packages/transformers) you can add directly to your project.

3
packages/astro/package.json
View file

@ -166,7 +166,7 @@
"resolve": "^1.22.4",
"semver": "^7.5.4",
"server-destroy": "^1.0.1",
"shikiji": "^0.6.13",
"shikiji": "^0.9.18",
"string-width": "^7.0.0",
"strip-ansi": "^7.1.0",
"tsconfck": "^3.0.0",
@ -224,6 +224,7 @@
"remark-code-titles": "^0.1.2",
"rollup": "^4.5.0",
"sass": "^1.69.5",
"shikiji-core": "^0.9.18",
"srcset-parse": "^1.1.0",
"unified": "^11.0.4"
},

6
packages/astro/src/core/config/schema.ts
View file

@ -17,9 +17,11 @@ import { appendForwardSlash, prependForwardSlash, removeTrailingForwardSlash } f
// This import is required to appease TypeScript!
// See https://github.com/withastro/astro/pull/8762
import 'mdast-util-to-hast';
import 'shikiji-core';
type ShikiLangs = NonNullable<ShikiConfig['langs']>;
type ShikiTheme = NonNullable<ShikiConfig['theme']>;
type ShikiTransformers = NonNullable<ShikiConfig['transformers']>;
const ASTRO_CONFIG_DEFAULTS = {
root: '.',
@ -275,6 +277,10 @@ export const AstroConfigSchema = z.object({
)
.default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.experimentalThemes!),
wrap: z.boolean().or(z.null()).default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.wrap!),
transformers: z
.custom<ShikiTransformers[number]>()
.array()
.default(ASTRO_CONFIG_DEFAULTS.markdown.shikiConfig.transformers!),
})
.default({}),
remarkPlugins: z

19
packages/astro/src/core/errors/dev/vite.ts
View file

@ -1,7 +1,6 @@
import { replaceCssVariables } from '@astrojs/markdown-remark';
import * as fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { codeToHtml } from 'shikiji';
import { codeToHtml, createCssVariablesTheme } from 'shikiji';
import type { ErrorPayload } from 'vite';
import type { ModuleLoader } from '../../module-loader/index.js';
import { FailedToLoadModuleSSR, InvalidGlob, MdxIntegrationMissingError } from '../errors-data.js';
@ -124,7 +123,11 @@ export interface AstroErrorPayload {
// Map these to `.js` during error highlighting.
const ALTERNATIVE_JS_EXTS = ['cjs', 'mjs'];
const ALTERNATIVE_MD_EXTS = ['mdoc'];
const INLINE_STYLE_SELECTOR_GLOBAL = /style="(.*?)"/g;
let _cssVariablesTheme: ReturnType<typeof createCssVariablesTheme>;
const cssVariablesTheme = () =>
_cssVariablesTheme ??
(_cssVariablesTheme = createCssVariablesTheme({ variablePrefix: '--astro-code-' }));
/**
* Generate a payload for Vite's error overlay
@ -147,21 +150,15 @@ export async function getViteErrorPayload(err: ErrorWithMetadata): Promise<Astro
if (ALTERNATIVE_MD_EXTS.includes(highlighterLang ?? '')) {
highlighterLang = 'md';
}
let highlightedCode = err.fullCode
const highlightedCode = err.fullCode
? await codeToHtml(err.fullCode, {
// @ts-expect-error always assume that shiki can accept the lang string
lang: highlighterLang,
theme: 'css-variables',
theme: cssVariablesTheme(),
lineOptions: err.loc?.line ? [{ line: err.loc.line, classes: ['error-line'] }] : undefined,
})
: undefined;
if (highlightedCode) {
highlightedCode = highlightedCode.replace(INLINE_STYLE_SELECTOR_GLOBAL, (m) =>
replaceCssVariables(m)
);
}
return {
type: 'error',
err: {

4
packages/astro/src/core/errors/overlay.ts
View file

@ -68,7 +68,7 @@ const style = /* css */ `
--toggle-border-color: #C3CADB;
/* Syntax Highlighting */
--astro-code-color-text: #000000;
--astro-code-foreground: #000000;
--astro-code-token-constant: #4ca48f;
--astro-code-token-string: #9f722a;
--astro-code-token-comment: #8490b5;
@ -121,7 +121,7 @@ const style = /* css */ `
--toggle-border-color: #3D4663;
/* Syntax Highlighting */
--astro-code-color-text: #ffffff;
--astro-code-foreground: #ffffff;
--astro-code-token-constant: #90f4e3;
--astro-code-token-string: #f4cf90;
--astro-code-token-comment: #8490b5;

1
packages/integrations/markdoc/package.json
View file

@ -71,7 +71,6 @@
"gray-matter": "^4.0.3",
"htmlparser2": "^9.0.0",
"kleur": "^4.1.5",
"shikiji": "^0.6.13",
"zod": "^3.22.4"
},
"peerDependencies": {

2
packages/markdown/remark/package.json
View file

@ -38,7 +38,7 @@
"remark-parse": "^11.0.0",
"remark-rehype": "^11.0.0",
"remark-smartypants": "^2.0.0",
"shikiji": "^0.6.13",
"shikiji": "^0.9.18",
"unified": "^11.0.4",
"unist-util-visit": "^5.0.0",
"vfile": "^6.0.1"

1
packages/markdown/remark/src/index.ts
View file

@ -36,6 +36,7 @@ export const markdownConfigDefaults: Required<AstroMarkdownOptions> = {
theme: 'github-dark',
experimentalThemes: {},
wrap: false,
transformers: [],
},
remarkPlugins: [],
rehypePlugins: [],

147
packages/markdown/remark/src/shiki.ts
View file

@ -1,4 +1,4 @@
import { bundledLanguages, getHighlighter } from 'shikiji';
import { bundledLanguages, createCssVariablesTheme, getHighlighter } from 'shikiji';
import { visit } from 'unist-util-visit';
import type { ShikiConfig } from './types.js';
@ -6,32 +6,32 @@ export interface ShikiHighlighter {
highlight(code: string, lang?: string, options?: { inline?: boolean }): string;
}
// TODO: Remove this special replacement in Astro 5
const ASTRO_COLOR_REPLACEMENTS: Record<string, string> = {
'#000001': 'var(--astro-code-color-text)',
'#000002': 'var(--astro-code-color-background)',
'#000004': 'var(--astro-code-token-constant)',
'#000005': 'var(--astro-code-token-string)',
'#000006': 'var(--astro-code-token-comment)',
'#000007': 'var(--astro-code-token-keyword)',
'#000008': 'var(--astro-code-token-parameter)',
'#000009': 'var(--astro-code-token-function)',
'#000010': 'var(--astro-code-token-string-expression)',
'#000011': 'var(--astro-code-token-punctuation)',
'#000012': 'var(--astro-code-token-link)',
'--astro-code-foreground': '--astro-code-color-text',
'--astro-code-background': '--astro-code-color-background',
};
const COLOR_REPLACEMENT_REGEX = new RegExp(
`(${Object.keys(ASTRO_COLOR_REPLACEMENTS).join('|')})`,
'g'
);
let _cssVariablesTheme: ReturnType<typeof createCssVariablesTheme>;
const cssVariablesTheme = () =>
_cssVariablesTheme ??
(_cssVariablesTheme = createCssVariablesTheme({ variablePrefix: '--astro-code-' }));
export async function createShikiHighlighter({
langs = [],
theme = 'github-dark',
experimentalThemes = {},
wrap = false,
transformers = [],
}: ShikiConfig = {}): Promise<ShikiHighlighter> {
const themes = experimentalThemes;
theme = theme === 'css-variables' ? cssVariablesTheme() : theme;
const highlighter = await getHighlighter({
langs: langs.length ? langs : Object.keys(bundledLanguages),
themes: Object.values(themes).length ? Object.values(themes) : [theme],
@ -53,74 +53,77 @@ export async function createShikiHighlighter({
return highlighter.codeToHtml(code, {
...themeOptions,
lang,
transforms: {
pre(node) {
// Swap to `code` tag if inline
if (inline) {
node.tagName = 'code';
}
transformers: [
{
pre(node) {
// Swap to `code` tag if inline
if (inline) {
node.tagName = 'code';
}
// Cast to string as shikiji will always pass them as strings instead of any other types
const classValue = (node.properties.class as string) ?? '';
const styleValue = (node.properties.style as string) ?? '';
// Cast to string as shikiji will always pass them as strings instead of any other types
const classValue = (node.properties.class as string) ?? '';
const styleValue = (node.properties.style as string) ?? '';
// Replace "shiki" class naming with "astro-code"
node.properties.class = classValue.replace(/shiki/g, 'astro-code');
// Replace "shiki" class naming with "astro-code"
node.properties.class = classValue.replace(/shiki/g, 'astro-code');
// Handle code wrapping
// if wrap=null, do nothing.
if (wrap === false) {
node.properties.style = styleValue + '; overflow-x: auto;';
} else if (wrap === true) {
node.properties.style =
styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
}
},
line(node) {
// Add "user-select: none;" for "+"/"-" diff symbols.
// Transform `<span class="line"><span style="...">+ something</span></span>
// into `<span class="line"><span style="..."><span style="user-select: none;">+</span> something</span></span>`
if (lang === 'diff') {
const innerSpanNode = node.children[0];
const innerSpanTextNode =
innerSpanNode?.type === 'element' && innerSpanNode.children?.[0];
// Handle code wrapping
// if wrap=null, do nothing.
if (wrap === false) {
node.properties.style = styleValue + '; overflow-x: auto;';
} else if (wrap === true) {
node.properties.style =
styleValue + '; overflow-x: auto; white-space: pre-wrap; word-wrap: break-word;';
}
},
line(node) {
// Add "user-select: none;" for "+"/"-" diff symbols.
// Transform `<span class="line"><span style="...">+ something</span></span>
// into `<span class="line"><span style="..."><span style="user-select: none;">+</span> something</span></span>`
if (lang === 'diff') {
const innerSpanNode = node.children[0];
const innerSpanTextNode =
innerSpanNode?.type === 'element' && innerSpanNode.children?.[0];
if (innerSpanTextNode && innerSpanTextNode.type === 'text') {
const start = innerSpanTextNode.value[0];
if (start === '+' || start === '-') {
innerSpanTextNode.value = innerSpanTextNode.value.slice(1);
innerSpanNode.children.unshift({
type: 'element',
tagName: 'span',
properties: { style: 'user-select: none;' },
children: [{ type: 'text', value: start }],
});
if (innerSpanTextNode && innerSpanTextNode.type === 'text') {
const start = innerSpanTextNode.value[0];
if (start === '+' || start === '-') {
innerSpanTextNode.value = innerSpanTextNode.value.slice(1);
innerSpanNode.children.unshift({
type: 'element',
tagName: 'span',
properties: { style: 'user-select: none;' },
children: [{ type: 'text', value: start }],
});
}
}
}
}
},
code(node) {
if (inline) {
return node.children[0] as typeof node;
}
},
root(node) {
if (Object.values(experimentalThemes).length) {
return;
}
},
code(node) {
if (inline) {
return node.children[0] as typeof node;
}
},
root(node) {
if (Object.values(experimentalThemes).length) {
return;
}
// theme.id for shiki -> shikiji compat
const themeName = typeof theme === 'string' ? theme : theme.name;
if (themeName === 'css-variables') {
// Replace special color tokens to CSS variables
visit(node as any, 'element', (child) => {
if (child.properties?.style) {
child.properties.style = replaceCssVariables(child.properties.style);
}
});
}
// theme.id for shiki -> shikiji compat
const themeName = typeof theme === 'string' ? theme : theme.name;
if (themeName === 'css-variables') {
// Replace special color tokens to CSS variables
visit(node as any, 'element', (child) => {
if (child.properties?.style) {
child.properties.style = replaceCssVariables(child.properties.style);
}
});
}
},
},
},
...transformers,
],
});
},
};

8
packages/markdown/remark/src/types.ts
View file

@ -4,6 +4,7 @@ import type { Options as RemarkRehypeOptions } from 'remark-rehype';
import type {
BuiltinTheme,
LanguageRegistration,
ShikijiTransformer,
ThemeRegistration,
ThemeRegistrationRaw,
} from 'shikiji';
@ -32,11 +33,14 @@ export type RehypePlugins = (string | [string, any] | RehypePlugin | [RehypePlug
export type RemarkRehype = RemarkRehypeOptions;
export type ThemePresets = BuiltinTheme | 'css-variables';
export interface ShikiConfig {
langs?: LanguageRegistration[];
theme?: BuiltinTheme | ThemeRegistration | ThemeRegistrationRaw;
experimentalThemes?: Record<string, BuiltinTheme | ThemeRegistration | ThemeRegistrationRaw>;
theme?: ThemePresets | ThemeRegistration | ThemeRegistrationRaw;
experimentalThemes?: Record<string, ThemePresets | ThemeRegistration | ThemeRegistrationRaw>;
wrap?: boolean | null;
transformers?: ShikijiTransformer[];
}
export interface AstroMarkdownOptions {

23
pnpm-lock.yaml generated
View file

@ -645,8 +645,8 @@ importers:
specifier: ^1.0.1
version: 1.0.1
shikiji:
specifier: ^0.6.13
version: 0.6.13
specifier: ^0.9.18
version: 0.9.18
string-width:
specifier: ^7.0.0
version: 7.0.0
@ -808,6 +808,9 @@ importers:
sass:
specifier: ^1.69.5
version: 1.69.6
shikiji-core:
specifier: ^0.9.18
version: 0.9.18
srcset-parse:
specifier: ^1.1.0
version: 1.1.0
@ -3844,9 +3847,6 @@ importers:
kleur:
specifier: ^4.1.5
version: 4.1.5
shikiji:
specifier: ^0.6.13
version: 0.6.13
zod:
specifier: ^3.22.4
version: 3.22.4
@ -5017,8 +5017,8 @@ importers:
specifier: ^2.0.0
version: 2.0.0
shikiji:
specifier: ^0.6.13
version: 0.6.13
specifier: ^0.9.18
version: 0.9.18
unified:
specifier: ^11.0.4
version: 11.0.4
@ -14288,10 +14288,13 @@ packages:
vscode-textmate: 5.2.0
dev: true
/shikiji@0.6.13:
resolution: {integrity: sha512-4T7X39csvhT0p7GDnq9vysWddf2b6BeioiN3Ymhnt3xcy9tXmDcnsEFVxX18Z4YcQgEE/w48dLJ4pPPUcG9KkA==}
/shikiji-core@0.9.18:
resolution: {integrity: sha512-PKTXptbrp/WEDjNHV8OFG9KkfhmR0pSd161kzlDDlgQ0HXAnqJYNDSjqsy1CYZMx5bSvLMy42yJj9oFTqmkNTQ==}
/shikiji@0.9.18:
resolution: {integrity: sha512-/tFMIdV7UQklzN13VjF0/XFzmii6C606Jc878hNezvB8ZR8FG8FW9j0I4J9EJre0owlnPntgLVPpHqy27Gs+DQ==}
dependencies:
hast-util-to-html: 9.0.0
shikiji-core: 0.9.18
dev: false
/side-channel@1.0.4: