diff --git a/.changeset/silver-carrots-allow.md b/.changeset/silver-carrots-allow.md new file mode 100644 index 000000000..77f0e40ec --- /dev/null +++ b/.changeset/silver-carrots-allow.md @@ -0,0 +1,7 @@ +--- +'myst-transforms': minor +'myst-common': patch +'myst-cli': patch +--- + +Add new mermaid-conversion transform diff --git a/package-lock.json b/package-lock.json index 454c1cb54..8e5d4d58b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15963,7 +15963,8 @@ "unist-util-select": "^4.0.3", "unist-util-visit": "^4.1.0", "vfile": "^5.0.0", - "vfile-message": "^3.1.2" + "vfile-message": "^3.1.2", + "which": "^3.0.1" }, "devDependencies": { "@types/katex": "^0.14.0" diff --git a/packages/myst-cli/src/build/tex/single.ts b/packages/myst-cli/src/build/tex/single.ts index 24d456eee..a058eb90e 100644 --- a/packages/myst-cli/src/build/tex/single.ts +++ b/packages/myst-cli/src/build/tex/single.ts @@ -153,6 +153,7 @@ export async function localArticleToTexRaw( imageAltOutputFolder: 'files/', imageExtensions: TEX_IMAGE_EXTENSIONS, simplifyFigures: true, + mermaidAsImage: true, }); return mdastToTex(session, mdast, references, frontmatter, null, false); }), @@ -252,6 +253,7 @@ export async function localArticleToTexTemplated( imageAltOutputFolder: 'files/', imageExtensions: TEX_IMAGE_EXTENSIONS, simplifyFigures: true, + mermaidAsImage: true, }); partDefinitions.forEach((def) => { diff --git a/packages/myst-cli/src/process/mdast.ts b/packages/myst-cli/src/process/mdast.ts index da6e7779a..bbc0ab72f 100644 --- a/packages/myst-cli/src/process/mdast.ts +++ b/packages/myst-cli/src/process/mdast.ts @@ -32,6 +32,7 @@ import { inlineMathSimplificationPlugin, checkLinkTextTransform, indexIdentifierPlugin, + mermaidToImageTransform, } from 'myst-transforms'; import { unified } from 'unified'; import { select, selectAll } from 'unist-util-select'; @@ -341,6 +342,7 @@ export async function finalizeMdast( file: string, { imageWriteFolder, + mermaidAsImage, useExistingImages, imageAltOutputFolder, imageExtensions, @@ -350,6 +352,7 @@ export async function finalizeMdast( maxSizeWebp, }: { imageWriteFolder: string; + mermaidAsImage?: boolean; useExistingImages?: boolean; imageAltOutputFolder?: string; imageExtensions?: ImageExtensions[]; @@ -372,6 +375,9 @@ export async function finalizeMdast( vfile, }); if (!useExistingImages) { + if (mermaidAsImage) { + await mermaidToImageTransform(session, mdast, imageWriteFolder, vfile); + } await transformImagesToDisk(session, mdast, file, imageWriteFolder, { altOutputFolder: imageAltOutputFolder, imageExtensions, diff --git a/packages/myst-common/src/ruleids.ts b/packages/myst-common/src/ruleids.ts index d833a0dd0..afc9fc045 100644 --- a/packages/myst-common/src/ruleids.ts +++ b/packages/myst-common/src/ruleids.ts @@ -48,6 +48,7 @@ export enum RuleId { imageFormatConverts = 'image-format-converts', imageCopied = 'image-copied', imageFormatOptimizes = 'image-format-optimizes', + mermaidDiagramConverted = 'mermaid-diagram-converted', // Math rules mathLabelLifted = 'math-label-lifted', mathEquationEnvRemoved = 'math-equation-env-removed', diff --git a/packages/myst-transforms/package.json b/packages/myst-transforms/package.json index ff8e47970..8b12953ea 100644 --- a/packages/myst-transforms/package.json +++ b/packages/myst-transforms/package.json @@ -21,10 +21,10 @@ "dependencies": { "doi-utils": "^2.0.0", "hast-util-from-html": "^2.0.1", + "hast-util-to-mdast": "^8.3.1", "intersphinx": "^1.0.2", "js-yaml": "^4.1.0", "katex": "^0.15.2", - "hast-util-to-mdast": "^8.3.1", "mdast-util-find-and-replace": "^2.1.0", "myst-common": "^1.5.3", "myst-frontmatter": "^1.5.3", @@ -36,13 +36,14 @@ "unified": "^10.0.0", "unist-builder": "^3.0.0", "unist-util-find-after": "^4.0.0", - "unist-util-modify-children": "^3.1.0", "unist-util-map": "^3.0.0", + "unist-util-modify-children": "^3.1.0", "unist-util-remove": "^3.1.0", "unist-util-select": "^4.0.3", "unist-util-visit": "^4.1.0", "vfile": "^5.0.0", - "vfile-message": "^3.1.2" + "vfile-message": "^3.1.2", + "which": "^3.0.1" }, "devDependencies": { "@types/katex": "^0.14.0" diff --git a/packages/myst-transforms/src/index.ts b/packages/myst-transforms/src/index.ts index 0dc78a0db..6d6d3626d 100644 --- a/packages/myst-transforms/src/index.ts +++ b/packages/myst-transforms/src/index.ts @@ -41,6 +41,7 @@ export { } from './code.js'; export { blockquotePlugin, blockquoteTransform } from './blockquote.js'; export { imageAltTextPlugin, imageAltTextTransform } from './images.js'; +export { mermaidToImageTransform } from './mermaid.js'; export { buildIndexTransform, indexIdentifierPlugin, indexIdentifierTransform } from './indices.js'; export { liftMystDirectivesAndRolesPlugin, diff --git a/packages/myst-transforms/src/mermaid.ts b/packages/myst-transforms/src/mermaid.ts new file mode 100644 index 000000000..8c1caaec2 --- /dev/null +++ b/packages/myst-transforms/src/mermaid.ts @@ -0,0 +1,106 @@ +import { selectAll } from 'unist-util-select'; +import type { GenericParent, GenericNode } from 'myst-common'; +import { RuleId } from 'myst-common'; +import which from 'which'; +import type { VFile } from 'vfile'; +import type { LoggerDE } from 'myst-cli-utils'; +import type { ISession } from 'myst-cli'; +import { makeExecutable } from 'myst-cli-utils'; +import { createHash } from 'node:crypto'; +import fs from 'node:fs/promises'; +import { join } from 'node:path'; +import { createTempFolder, addWarningForFile } from 'myst-cli'; + +function isMMDCCommandAvailable() { + return !!which.sync('mmdc', { nothrow: true }); +} +type Literal = { + type: string; + value: string; +}; + +function createMMDCLogger(session: ISession): LoggerDE { + const logger = { + debug(data: string) { + const line = data.trim(); + if (!line) return; + session.log.debug(data); + }, + error(data: string) { + const line = data.trim(); + if (!line) return; + session.log.error(data); + }, + }; + return logger; +} + +async function convertMermaidToSVG( + session: ISession, + writeFolder: string, + data: string, + vfile: VFile, +) { + const hash = createHash('md5').update(data).digest('hex'); + const tempFolder = createTempFolder(session); + + const srcPath = join(tempFolder, `${hash}.mmd`); + await fs.writeFile(srcPath, data); + + await fs.mkdir(writeFolder, { recursive: true }); + const dstPath = join(writeFolder, `${hash}.pdf`); + + const executable = `mmdc -i ${srcPath} -o ${dstPath}`; + const exec = makeExecutable(executable, createMMDCLogger(session)); + try { + await exec(); + } catch (err) { + addWarningForFile( + session, + vfile.path, + `Could not convert Mermaid diagram to svg: ${err}`, + 'error', + { + ruleId: RuleId.mermaidDiagramConverted, + }, + ); + return null; + } + + return dstPath; +} + +/** + * Ensure caption content is nested in a paragraph. + * + * This function is idempotent. + */ +export async function mermaidToImageTransform( + session: ISession, + tree: GenericParent, + writeFolder: string, + vfile: VFile, +) { + if (!isMMDCCommandAvailable()) { + addWarningForFile( + session, + vfile.path, + `Could not find mmdc, required for conversion of Mermaid diagrams\n`, + 'warn', + { ruleId: RuleId.mermaidDiagramConverted }, + ); + return null; + } + + const nodes = selectAll('mermaid', tree) as Literal[]; + await Promise.all( + nodes.map(async (node) => { + const dst = await convertMermaidToSVG(session, writeFolder, node.value, vfile); + if (dst) { + const newNode = node as GenericNode; + newNode.type = 'image'; + newNode.url = dst ?? ''; + } + }), + ); +}