🧹 Upgrade unified/unist stack

package.json

@@ -28,7 +28,7 @@
     "npm-run-all": "^4.1.5",
     "prettier": "latest",
     "rimraf": "^5.0.1",
-    "turbo": "latest",
+    "turbo": "^1.10.0",
     "typescript": "^5.6.2",
     "vitest": "^2.1.2"
   },

packages/jats-to-myst/package.json

@@ -43,12 +43,12 @@
     "myst-spec": "^0.0.5",
     "myst-spec-ext": "^1.7.2",
     "myst-transforms": "^1.3.26",
-    "unified": "^10.0.0",
-    "unist-builder": "^3.0.0",
-    "unist-util-select": "^4.0.3",
-    "unist-util-remove": "^3.1.0",
-    "vfile": "^5.0.0",
-    "vfile-reporter": "^7.0.4"
+    "unified": "^11.0.0",
+    "unist-builder": "^4.0.0",
+    "unist-util-select": "^5.0.0",
+    "unist-util-remove": "^4.0.0",
+    "vfile": "^6.0.0",
+    "vfile-reporter": "^8.0.0"
   },
   "devDependencies": {
     "myst-to-tex": "^1.0.38"

packages/jats-to-myst/src/index.ts

@@ -198,8 +198,8 @@ const handlers: Record<string, Handler> = {
     state.openNode('tabSet');
     node.children?.forEach((n) => {
       state.openNode('tabItem', {
-        title: toText(select('label', n)),
-        sync: toText(select('label', n)),
+        title: toText(select('label', n) as GenericNode),
+        sync: toText(select('label', n) as GenericNode),
       });
       state.renderChildren({ children: [n] });
       state.closeNode();
@@ -451,18 +451,23 @@ export class JatsParser implements IJatsParser {
     return this.pushNode(node);
   }
 }
+declare module 'unified' {
+  interface CompileResultMap {
+    VFile: VFile;
+  }
+}
 
 export const jatsToMystPlugin: Plugin<[Jats, Options?], Root, Root> = function (jats, opts) {
-  this.Compiler = (node: GenericParent, file: VFile) => {
+  this.compiler = (node, file: VFile) => {
     const tree = jats.abstract
       ? {
           type: 'root',
           children: [
             u('block', { part: 'abstract' }, copyNode(jats.abstract).children),
-            ...copyNode(node).children,
+            ...copyNode(node as any).children,
           ],
         }
-      : copyNode(node);
+      : copyNode(node as any);
     // Can do better than this in the future, but for now, just put them at the end!
     const floatsGroup = selectAll('floats-group', jats.tree) as GenericParent[];
     if (floatsGroup.length > 0) {
@@ -479,18 +484,21 @@ export const jatsToMystPlugin: Plugin<[Jats, Options?], Root, Root> = function (
       jats.references.map((bibr) => {
         const id = bibr.id;
         const names = selectAll('name,string-name', bibr)
-          .map((n) => `${toText(select('surname', n))}, ${toText(select('given-names', n))}`)
+          .map(
+            (n) =>
+              `${toText(select('surname', n) as GenericNode)}, ${toText(select('given-names', n) as GenericNode)}`,
+          )
           .join(', ');
-        const year = toText(select('year', bibr));
-        const title = toText(select('article-title', bibr));
-        const source = toText(select('source', bibr));
-        const volume = toText(select('volume', bibr));
-        const fpage = toText(select('fpage', bibr));
-        const lpage = toText(select('lpage', bibr));
+        const year = toText(select('year', bibr) as GenericNode);
+        const title = toText(select('article-title', bibr) as GenericNode);
+        const source = toText(select('source', bibr) as GenericNode);
+        const volume = toText(select('volume', bibr) as GenericNode);
+        const fpage = toText(select('fpage', bibr) as GenericNode);
+        const lpage = toText(select('lpage', bibr) as GenericNode);
         const doiElement = selectAll('ext-link,[pub-id-type=doi]', bibr).find((e) =>
-          doi.validate(toText(e)),
+          doi.validate(toText(e as GenericNode)),
         );
-        const doiString = doiElement ? toText(doiElement) : undefined;
+        const doiString = doiElement ? toText(doiElement as GenericNode) : undefined;
         const doiLink = doiString ? ` <a href=${doi.buildUrl(doiString)}>${doiString}</a>` : '';
         return [
           id,
@@ -520,10 +528,6 @@ export const jatsToMystPlugin: Plugin<[Jats, Options?], Root, Root> = function (
     file.result = result;
     return file;
   };
-
-  return (node: Root) => {
-    return node;
-  };
 };
 
 export function jatsToMystTransform(

packages/markdown-it-myst/package.json

@@ -34,6 +34,6 @@
   "dependencies": {
     "js-yaml": "^4.1.0",
     "markdown-it": "^13.0.1",
-    "vfile": "^5.3.7"
+    "vfile": "^6.0.0"
   }
 }

packages/myst-cli/package.json

@@ -99,13 +99,13 @@
     "simple-validators": "^1.1.0",
     "strip-ansi": "^7.0.1",
     "tex-to-myst": "^1.0.40",
-    "unified": "^10.1.2",
+    "unified": "^11.0.0",
     "unist-util-filter": "^4.0.0",
-    "unist-util-remove": "^3.1.0",
-    "unist-util-select": "^4.0.3",
+    "unist-util-remove": "^4.0.0",
+    "unist-util-select": "^5.0.0",
     "unist-util-visit": "^4.0.0",
     "uuid": "^8.3.2",
-    "vfile": "^5.3.5",
+    "vfile": "^6.0.0",
     "which": "^3.0.1",
     "ws": "^8.9.0",
     "xml-js": "^1.6.11",

packages/myst-cli/src/build/docx/single.ts

@@ -1,6 +1,6 @@
 import fs from 'node:fs';
 import path from 'node:path';
-import type { Content } from 'mdast';
+import type { RootContent } from 'mdast';
 import { createDocFromState, DocxSerializer, writeDocx } from 'myst-to-docx';
 import { tic, writeFileToFolder } from 'myst-cli-utils';
 import {
@@ -12,7 +12,7 @@ import {
 import type { RendererDoc } from 'myst-templates';
 import MystTemplate from 'myst-templates';
 import { htmlTransform } from 'myst-transforms';
-import { fileError, fileWarn, RuleId, TemplateKind } from 'myst-common';
+import { fileError, fileWarn, RuleId, TemplateKind, type GenericNode } from 'myst-common';
 import { selectAll } from 'unist-util-select';
 import { filterKeys } from 'simple-validators';
 import { VFile } from 'vfile';
@@ -40,7 +40,10 @@ function defaultWordRenderer(
   vfile: VFile,
 ) {
   const { mdast, frontmatter, references } = data;
-  const frontmatterNodes = createArticleTitle(frontmatter.title, frontmatter.authors) as Content[];
+  const frontmatterNodes = createArticleTitle(
+    frontmatter.title,
+    frontmatter.authors,
+  ) as RootContent[];
   const serializer = new DocxSerializer(
     vfile,
     {
@@ -54,7 +57,7 @@ function defaultWordRenderer(
     frontmatter,
   );
   frontmatterNodes.forEach((node) => {
-    serializer.render(node);
+    serializer.render(node as GenericNode);
   });
   serializer.renderChildren(mdast);
   const referencesDocStates = Object.values(references.cite?.data ?? {})
@@ -69,7 +72,7 @@ function defaultWordRenderer(
     serializer.renderChildren(referencesRoot);
   }
   selectAll('footnoteDefinition', mdast).forEach((footnote) => {
-    serializer.render(footnote);
+    serializer.render(footnote as GenericNode);
   });
   const logo = path.join(staticPath, 'logo.png');
   const docfooter = fs.existsSync(logo) && !opts.hideFooter ? createFooter(logo) : undefined;

packages/myst-cli/src/build/tex/single.ts

@@ -60,7 +60,7 @@ export function mdastToTex(
     ...frontmatter.settings?.myst_to_tex,
   });
   const result = pipe.runSync(mdast as any);
-  const tex = pipe.stringify(result);
+  const tex = pipe.stringify(result as any);
   logMessagesFromVFile(session, tex);
   return tex.result as LatexResult;
 }

packages/myst-cli/src/build/typst.ts

@@ -78,7 +78,7 @@ export function mdastToTypst(
     // ...frontmatter.settings?.myst_to_tex,
   });
   const result = pipe.runSync(mdast as any);
-  const typ = pipe.stringify(result);
+  const typ = pipe.stringify(result as any);
   logMessagesFromVFile(session, typ);
   return typ.result as TypstResult;
 }

packages/myst-cli/src/executablePlugin.ts

@@ -5,6 +5,7 @@ import {
   type RoleSpec,
   type TransformSpec,
   type GenericNode,
+  type GenericParent,
 } from 'myst-common';
 import { spawn, spawnSync } from 'node:child_process';
 
@@ -117,7 +118,7 @@ function wrapTransform(
   return {
     ...transform,
     plugin: () => {
-      return async (node) => {
+      return async (node: GenericParent) => {
         // Modify the tree in-place
         const result = await executeAsyncParser(session, transform.name, node, path, 'transform');
         Object.assign(node, result);

packages/myst-cli/src/frontmatter.ts

@@ -76,7 +76,7 @@ export function processPageFrontmatter(
   const cache = castSession(session);
   const state = session.store.getState();
   const siteFrontmatter = selectors.selectCurrentSiteConfig(state) ?? {};
-  const projectFrontmatter = path ? selectors.selectLocalProjectConfig(state, path) ?? {} : {};
+  const projectFrontmatter = path ? (selectors.selectLocalProjectConfig(state, path) ?? {}) : {};
 
   const frontmatter = fillPageFrontmatter(pageFrontmatter, projectFrontmatter, validationOpts);
   const siteTemplate = cache.$siteTemplate;

packages/myst-cli/src/process/mdast.ts

@@ -81,7 +81,7 @@ import { rawDirectiveTransform } from '../transforms/raw.js';
 
 const LINKS_SELECTOR = 'link,card,linkBlock';
 
-const pluginUtils: PluginUtils = { select, selectAll };
+const pluginUtils: PluginUtils = { select, selectAll } as any;
 
 const htmlHandlers = {
   comment(h: any, node: any) {

packages/myst-cli/src/process/site.ts

@@ -3,7 +3,7 @@ import { basename, extname, join } from 'node:path';
 import chalk from 'chalk';
 import { Inventory, Domains } from 'intersphinx';
 import { writeFileToFolder, tic, hashAndCopyStaticFile } from 'myst-cli-utils';
-import { RuleId, toText, plural, slugToUrl } from 'myst-common';
+import { RuleId, toText, plural, slugToUrl, type GenericNode } from 'myst-common';
 import type { SiteConfig, SiteProject } from 'myst-config';
 import type { Node } from 'myst-spec';
 import type { SearchRecord, MystSearchIndex } from 'myst-spec-ext';
@@ -110,7 +110,7 @@ function getReferenceTitleAsText(targetNode: Node): string | undefined {
     return toText(targetNode);
   }
   const caption = select('caption > paragraph', targetNode);
-  if (caption) return toText(caption);
+  if (caption) return toText(caption as GenericNode);
 }
 
 /**

packages/myst-cli/src/store/types.ts

@@ -1,4 +1,4 @@
-import type { VFileMessage } from 'vfile-message';
+import type { Position } from 'unist';
 
 export type ExternalLinkResult = {
   url: string;
@@ -15,7 +15,7 @@ export type BuildWarning = {
   kind: WarningKind;
   note?: string | null;
   url?: string | null;
-  position?: VFileMessage['position'];
+  position?: Position;
   ruleId?: string | null;
 };
 

packages/myst-cli/src/transforms/crossReferences.ts

@@ -140,6 +140,7 @@ export async function transformMystXRefs(
       } else {
         const data = await fetchMystXRefData(session, node as CrossReference, vfile);
         if (!data) return;
+        console.debug({ identifier: node.identifier });
         const targets = nodesFromMystXRefData(data, node.identifier, vfile, {
           urlSource: node.urlSource,
           maxNodes: 1,

packages/myst-cli/src/transforms/images.ts

@@ -6,7 +6,7 @@ import { computeHash, hashAndCopyStaticFile, isUrl } from 'myst-cli-utils';
 import { remove } from 'unist-util-remove';
 import { selectAll } from 'unist-util-select';
 import path from 'node:path';
-import type { VFileMessage } from 'vfile-message';
+import type { Position } from 'unist';
 import type { PageFrontmatter } from 'myst-frontmatter';
 import type { Image } from 'myst-spec-ext';
 import { extFromMimeType } from 'nbtx';
@@ -127,7 +127,7 @@ export async function saveImageInStaticFolder(
   urlSource: string,
   sourceFile: string,
   writeFolder: string,
-  opts?: { altOutputFolder?: string; position?: VFileMessage['position'] },
+  opts?: { altOutputFolder?: string; position?: Position },
 ): Promise<{ urlSource: string; url: string } | null> {
   const sourceFileFolder = path.dirname(sourceFile);
   const imageLocalFile = path.join(sourceFileFolder, urlSource);

packages/myst-cli/src/utils/addWarningForFile.ts

@@ -1,5 +1,5 @@
 import chalk from 'chalk';
-import type { VFileMessage } from 'vfile-message';
+import type { Position } from 'unist';
 import type { ISession } from '../session/types.js';
 import { warnings } from '../store/reducers.js';
 import type { WarningKind } from '../store/types.js';
@@ -13,7 +13,7 @@ export function addWarningForFile(
   opts?: {
     note?: string | null;
     url?: string | null;
-    position?: VFileMessage['position'];
+    position?: Position;
     ruleId?: string | null;
     /** This key can be combined with the ruleId to suppress a warning */
     key?: string | null;

packages/myst-cli/src/utils/logging.ts

@@ -11,7 +11,12 @@ export function logMessagesFromVFile(session: ISession, file?: VFile): void {
     const kind: WarningKind =
       message.fatal === null ? 'info' : message.fatal === false ? 'warn' : 'error';
     addWarningForFile(session, file.path, message.message, kind, {
-      position: message.position,
+      // TODO: does this logic make sense, or should we support `place`?
+      position: message.place
+        ? 'start' in message.place
+          ? message.place
+          : { start: message.place, end: message.place }
+        : undefined,
       note: message.note,
       url: message.url,
       ruleId: message.ruleId,

packages/myst-common/package.json

@@ -19,21 +19,23 @@
     "build": "npm-run-all -l clean -p build:esm"
   },
   "dependencies": {
+    "css.escape": "^1.5.1",
     "mdast": "^3.0.0",
     "myst-frontmatter": "^1.7.4",
     "myst-spec": "^0.0.5",
     "nanoid": "^4.0.0",
-    "unified": "^10.1.2",
-    "unist-util-remove": "^3.1.0",
-    "unist-util-select": "^4.0.3",
+    "unified": "^11.0.0",
+    "unist-util-remove": "^4.0.0",
+    "unist-util-select": "^5.0.0",
     "unist-util-visit": "^4.1.2",
-    "vfile": "^5.0.0",
-    "vfile-message": "^3.0.0"
+    "vfile": "^6.0.0",
+    "vfile-message": "^4.0.0"
   },
   "devDependencies": {
-    "myst-spec-ext": "^1.7.4",
-    "unist-builder": "3.0.0",
     "@jupyterlab/nbformat": "^3.5.2",
-    "@lumino/coreutils": "^2.0.0"
+    "@lumino/coreutils": "^2.0.0",
+    "@types/css.escape": "^1.5.2",
+    "myst-spec-ext": "^1.7.4",
+    "unist-builder": "4.0.0"
   }
 }

packages/myst-common/src/extractParts.ts

@@ -33,7 +33,7 @@ function coercePart(part?: string | string[]): string[] {
 export function selectBlockParts(tree: GenericParent, part?: string | string[]): Block[] {
   const parts = coercePart(part);
   if (parts.length === 0) return [];
-  const blockParts = selectAll('block', tree).filter((block) => {
+  const blockParts = (selectAll('block', tree) as Block[]).filter((block) => {
     const blockTags: string[] = (
       block.data?.tags && Array.isArray(block.data.tags) ? block.data.tags : []
     ).map((tag) => tag?.toLowerCase());
@@ -84,14 +84,6 @@ function createPartBlock(
   return block;
 }
 
-function forcedRemove(tree: GenericParent, test: string) {
-  let success = remove(tree, test);
-  if (!success) {
-    success = remove(tree, { cascade: false }, test);
-  }
-  return success;
-}
-
 /**
  * Extract implicit part based on heading name
  *
@@ -150,7 +142,7 @@ export function extractImplicitPart(
   });
   if (blockParts.length === 0) return;
   const partsTree = { type: 'root', children: blockParts } as GenericParent;
-  forcedRemove(tree, '__part_delete__');
+  remove(tree, '__part_delete__');
   return partsTree;
 }
 
@@ -209,6 +201,6 @@ export function extractPart(
   blockParts.forEach((block) => {
     (block as any).type = '__delete__';
   });
-  forcedRemove(tree, '__delete__');
+  remove(tree, '__delete__');
   return partsTree;
 }