diff --git a/packages/core/src/highlight/code-to-tokens-base.ts b/packages/core/src/highlight/code-to-tokens-base.ts index 1419df00..1cf24c9c 100644 --- a/packages/core/src/highlight/code-to-tokens-base.ts +++ b/packages/core/src/highlight/code-to-tokens-base.ts @@ -1,13 +1,12 @@ /* --------------------------------------------------------- * Copyright (C) Microsoft Corporation. All rights reserved. *-------------------------------------------------------- */ -import { INITIAL } from '@shikijs/vscode-textmate' +import { EncodedTokenMetadata, INITIAL } from '@shikijs/vscode-textmate' import type { IGrammar, IRawThemeSetting, StateStack } from '@shikijs/vscode-textmate' import type { CodeToTokensBaseOptions, FontStyle, ShikiInternal, ThemeRegistrationResolved, ThemedToken, ThemedTokenScopeExplanation, TokenizeWithThemeOptions } from '../types' import { applyColorReplacements, isNoneTheme, isPlainLang, resolveColorReplacements, splitLines } from '../utils' import { ShikiError } from '../error' import { GrammarState, getGrammarStack } from '../textmate/grammar-state' -import { StackElementMetadata } from '../textmate/stack-element-metadata' import { tokenizeAnsiWithTheme } from './code-to-tokens-ansi' /** @@ -166,10 +165,10 @@ function _tokenizeWithTheme( const metadata = result.tokens[2 * j + 1] const color = applyColorReplacements( - colorMap[StackElementMetadata.getForeground(metadata)], + colorMap[EncodedTokenMetadata.getForeground(metadata)], colorReplacements, ) - const fontStyle: FontStyle = StackElementMetadata.getFontStyle(metadata) + const fontStyle: FontStyle = EncodedTokenMetadata.getFontStyle(metadata) const token: ThemedToken = { content: line.substring(startIndex, nextStartIndex), diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index b37b4fab..d6bc99c5 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -12,7 +12,6 @@ export { createWasmOnigEngine, loadWasm } from './engines/wasm' export { createJavaScriptRegexEngine } from './engines/javascript' // TextMate Utilities -export { StackElementMetadata } from './textmate/stack-element-metadata' export { normalizeTheme } from './textmate/normalize-theme' // Low-level Highlighting @@ -27,3 +26,11 @@ export { codeToTokensWithThemes } from './highlight/code-to-tokens-themes' export * from './utils' export { transformerDecorations } from './transformer-decorations' export { ShikiError } from './error' + +// Deprecated +export { + /** + * @deprecated Use `EncodedTokenMetadata` from `@shikijs/vscode-textmate` instead. + */ + EncodedTokenMetadata as StackElementMetadata, +} from '@shikijs/vscode-textmate' diff --git a/packages/core/src/textmate/stack-element-metadata.ts b/packages/core/src/textmate/stack-element-metadata.ts deleted file mode 100644 index 8fc87ec0..00000000 --- a/packages/core/src/textmate/stack-element-metadata.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { FontStyle } from '../types' - -/** - * Helpers to manage the "collapsed" metadata of an entire StackElement stack. - * The following assumptions have been made: - * - languageId < 256 => needs 8 bits - * - unique color count < 512 => needs 9 bits - * - * The binary format is: - * - ------------------------------------------- - * 3322 2222 2222 1111 1111 1100 0000 0000 - * 1098 7654 3210 9876 5432 1098 7654 3210 - * - ------------------------------------------- - * xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx - * bbbb bbbb bfff ffff ffFF FTTT LLLL LLLL - * - ------------------------------------------- - * - L = LanguageId (8 bits) - * - T = StandardTokenType (3 bits) - * - F = FontStyle (3 bits) - * - f = foreground color (9 bits) - * - b = background color (9 bits) - */ -const MetadataConsts = { - LANGUAGEID_MASK: 0b00000000000000000000000011111111, - TOKEN_TYPE_MASK: 0b00000000000000000000001100000000, - BALANCED_BRACKETS_MASK: 0b00000000000000000000010000000000, - FONT_STYLE_MASK: 0b00000000000000000011100000000000, - FOREGROUND_MASK: 0b00000000011111111100000000000000, - BACKGROUND_MASK: 0b11111111100000000000000000000000, - - LANGUAGEID_OFFSET: 0, - TOKEN_TYPE_OFFSET: 8, - BALANCED_BRACKETS_OFFSET: 10, - FONT_STYLE_OFFSET: 11, - FOREGROUND_OFFSET: 15, - BACKGROUND_OFFSET: 24, -} - -export const enum TemporaryStandardTokenType { - Other = 0, - Comment = 1, - String = 2, - RegEx = 4, - MetaEmbedded = 8, -} - -export const enum StandardTokenType { - Other = 0, - Comment = 1, - String = 2, - RegEx = 4, -} - -export class StackElementMetadata { - public static toBinaryStr(metadata: number): string { - let r = metadata.toString(2) - while (r.length < 32) - r = `0${r}` - - return r - } - - // public static printMetadata(metadata: number): void { - // const languageId = StackElementMetadata.getLanguageId(metadata) - // const tokenType = StackElementMetadata.getTokenType(metadata) - // const fontStyle = StackElementMetadata.getFontStyle(metadata) - // const foreground = StackElementMetadata.getForeground(metadata) - // const background = StackElementMetadata.getBackground(metadata) - - // console.log({ - // languageId, - // tokenType, - // fontStyle, - // foreground, - // background, - // }) - // } - - public static getLanguageId(metadata: number): number { - return (metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET - } - - public static getTokenType(metadata: number): number { - return (metadata & MetadataConsts.TOKEN_TYPE_MASK) >>> MetadataConsts.TOKEN_TYPE_OFFSET - } - - public static getFontStyle(metadata: number): number { - return (metadata & MetadataConsts.FONT_STYLE_MASK) >>> MetadataConsts.FONT_STYLE_OFFSET - } - - public static getForeground(metadata: number): number { - return (metadata & MetadataConsts.FOREGROUND_MASK) >>> MetadataConsts.FOREGROUND_OFFSET - } - - public static getBackground(metadata: number): number { - return (metadata & MetadataConsts.BACKGROUND_MASK) >>> MetadataConsts.BACKGROUND_OFFSET - } - - public static containsBalancedBrackets(metadata: number): boolean { - return (metadata & MetadataConsts.BALANCED_BRACKETS_MASK) !== 0 - } - - public static set( - metadata: number, - languageId: number, - tokenType: TemporaryStandardTokenType, - fontStyle: FontStyle, - foreground: number, - background: number, - ): number { - let _languageId = StackElementMetadata.getLanguageId(metadata) - let _tokenType = StackElementMetadata.getTokenType(metadata) - let _fontStyle = StackElementMetadata.getFontStyle(metadata) - let _foreground = StackElementMetadata.getForeground(metadata) - let _background = StackElementMetadata.getBackground(metadata) - const _containsBalancedBracketsBit: 0 | 1 = StackElementMetadata.containsBalancedBrackets( - metadata, - ) - ? 1 - : 0 - - if (languageId !== 0) - _languageId = languageId - - if (tokenType !== TemporaryStandardTokenType.Other) { - _tokenType - = tokenType === TemporaryStandardTokenType.MetaEmbedded ? StandardTokenType.Other : tokenType - } - if (fontStyle !== FontStyle.NotSet) - _fontStyle = fontStyle - - if (foreground !== 0) - _foreground = foreground - - if (background !== 0) - _background = background - - return ( - ((_languageId << MetadataConsts.LANGUAGEID_OFFSET) - | (_tokenType << MetadataConsts.TOKEN_TYPE_OFFSET) - | (_fontStyle << MetadataConsts.FONT_STYLE_OFFSET) - | (_containsBalancedBracketsBit << MetadataConsts.BALANCED_BRACKETS_OFFSET) - | (_foreground << MetadataConsts.FOREGROUND_OFFSET) - | (_background << MetadataConsts.BACKGROUND_OFFSET)) - >>> 0 - ) - } -} diff --git a/packages/monaco/src/index.ts b/packages/monaco/src/index.ts index d7d1fdbc..89987b12 100644 --- a/packages/monaco/src/index.ts +++ b/packages/monaco/src/index.ts @@ -1,8 +1,7 @@ import type monacoNs from 'monaco-editor-core' import type { ShikiInternal, ThemeRegistrationResolved } from '@shikijs/core' import type { StateStack } from '@shikijs/vscode-textmate' -import { INITIAL } from '@shikijs/vscode-textmate' -import { StackElementMetadata } from '@shikijs/core' +import { EncodedTokenMetadata, INITIAL } from '@shikijs/vscode-textmate' export interface MonacoTheme extends monacoNs.editor.IStandaloneThemeData {} @@ -132,7 +131,7 @@ export function shikiToMonaco( for (let j = 0; j < tokensLength; j++) { const startIndex = result.tokens[2 * j] const metadata = result.tokens[2 * j + 1] - const color = normalizeColor(colorMap[StackElementMetadata.getForeground(metadata)] || '') + const color = normalizeColor(colorMap[EncodedTokenMetadata.getForeground(metadata)] || '') // Because Monaco only support one scope per token, // we workaround this to use color to trace back the scope const scope = findScopeByColor(color) || '' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 43b6abb1..ddc71ed9 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -46,8 +46,8 @@ catalogs: specifier: ^0.4.4 version: 0.4.4 '@shikijs/vscode-textmate': - specifier: ^9.1.1 - version: 9.1.1 + specifier: ^9.2.0 + version: 9.2.0 '@types/fs-extra': specifier: ^11.0.4 version: 11.0.4 @@ -505,7 +505,7 @@ importers: dependencies: '@shikijs/vscode-textmate': specifier: 'catalog:' - version: 9.1.1 + version: 9.2.0 '@types/hast': specifier: ^3.0.4 version: 3.0.4 @@ -543,7 +543,7 @@ importers: version: link:../core '@shikijs/vscode-textmate': specifier: 'catalog:' - version: 9.1.1 + version: 9.2.0 devDependencies: monaco-editor-core: specifier: 'catalog:' @@ -599,7 +599,7 @@ importers: version: link:../core '@shikijs/vscode-textmate': specifier: 'catalog:' - version: 9.1.1 + version: 9.2.0 '@types/hast': specifier: ^3.0.4 version: 3.0.4 @@ -1771,8 +1771,8 @@ packages: cpu: [x64] os: [win32] - '@shikijs/vscode-textmate@9.1.1': - resolution: {integrity: sha512-K/Xw/+uLXHtEp9bwTRbcwSB05du8J0k+O/n8nU7Z+J84BS9C+x6dWRUh3q2Pn8F19p+wPV1GU1xWSSimHlk9nA==} + '@shikijs/vscode-textmate@9.2.0': + resolution: {integrity: sha512-5FinaOp6Vdh/dl4/yaOTh0ZeKch+rYS8DUb38V3GMKYVkdqzxw53lViRKUYkVILRiVQT7dcPC7VvAKOR73zVtQ==} '@stylistic/eslint-plugin@2.7.1': resolution: {integrity: sha512-JqnHom8CP14oOgPhwTPbn0QgsBJwgNySQSe00V9GQQDlY1tEqZUlK4jM2DIOJ5nE+oXoy51vZWHnHkfZ6rEruw==} @@ -2032,8 +2032,8 @@ packages: '@vitest/eslint-plugin@1.1.0': resolution: {integrity: sha512-Ur80Y27Wbw8gFHJ3cv6vypcjXmrx6QHfw+q435h6Q2L+tf+h4Xf5pJTCL4YU/Jps9EVeggQxS85OcUZU7sdXRw==} peerDependencies: - '@typescript-eslint/utils': '>= 8.0' - eslint: '>= 8.57.0' + '@typescript-eslint/utils': ^8.3.0 + eslint: ^9.9.1 typescript: '>= 5.0.0' vitest: '*' peerDependenciesMeta: @@ -2903,7 +2903,7 @@ packages: eslint-config-flat-gitignore@0.3.0: resolution: {integrity: sha512-0Ndxo4qGhcewjTzw52TK06Mc00aDtHNTdeeW2JfONgDcLkRO/n/BteMRzNVpLQYxdCC/dFEilfM9fjjpGIJ9Og==} peerDependencies: - eslint: ^9.5.0 + eslint: ^9.9.1 eslint-flat-config-utils@0.3.1: resolution: {integrity: sha512-eFT3EaoJN1hlN97xw4FIEX//h0TiFUobgl2l5uLkIwhVN9ahGq95Pbs+i1/B5UACA78LO3rco3JzuvxLdTUOPA==} @@ -2949,7 +2949,7 @@ packages: resolution: {integrity: sha512-dBEM8fACIFNt4H7GoOaRmnH6evJW6JSTJTYYgmRd3vI4geBTjgDM/JyUDKUwIw0HDSyI+u7Vs3vFRXUo/BOAtA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} peerDependencies: - eslint: ^8.57.0 || ^9.0.0 + eslint: ^9.9.1 eslint-plugin-jsdoc@50.2.2: resolution: {integrity: sha512-i0ZMWA199DG7sjxlzXn5AeYZxpRfMJjDPUl7lL9eJJX8TPRoIaxJU4ys/joP5faM5AXE1eqW/dslCj3uj4Nqpg==} @@ -2984,7 +2984,7 @@ packages: engines: {node: ^18.0.0 || >=20.0.0} peerDependencies: astro-eslint-parser: ^1.0.2 - eslint: '>=8.0.0' + eslint: ^9.9.1 svelte: '>=3.0.0' svelte-eslint-parser: ^0.41.0 vue-eslint-parser: '>=9.0.0' @@ -6445,7 +6445,7 @@ snapshots: '@rollup/rollup-win32-x64-msvc@4.21.2': optional: true - '@shikijs/vscode-textmate@9.1.1': {} + '@shikijs/vscode-textmate@9.2.0': {} '@stylistic/eslint-plugin@2.7.1(eslint@9.9.1(jiti@1.21.6))(typescript@5.5.4)': dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index db58c27d..59678461 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -19,7 +19,7 @@ catalog: '@rollup/plugin-node-resolve': ^15.2.3 '@rollup/plugin-replace': ^5.0.7 '@rollup/plugin-terser': ^0.4.4 - '@shikijs/vscode-textmate': ^9.1.1 + '@shikijs/vscode-textmate': ^9.2.0 '@types/fs-extra': ^11.0.4 '@types/hast': ^3.0.4 '@types/markdown-it': ^14.1.2