feat(mdx-loader): upgrade to MDX v3 + (#9451)

packages/docusaurus-mdx-loader/src/processor.ts

@@ -5,7 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import emoji from 'remark-emoji';
 import headings from './remark/headings';
 import contentTitle from './remark/contentTitle';
 import toc from './remark/toc';
@@ -48,13 +47,10 @@ type SimpleProcessor = {
   }) => Promise<SimpleProcessorResult>;
 };
 
-const DEFAULT_OPTIONS: MDXOptions = {
-  admonitions: true,
-  rehypePlugins: [],
-  remarkPlugins: [headings, emoji, toc],
-  beforeDefaultRemarkPlugins: [],
-  beforeDefaultRehypePlugins: [],
-};
+async function getDefaultRemarkPlugins(): Promise<MDXPlugin[]> {
+  const {default: emoji} = await import('remark-emoji');
+  return [headings, emoji, toc];
+}
 
 export type MDXPlugin = Pluggable;
 
@@ -91,6 +87,8 @@ async function createProcessorFactory() {
   const {default: directive} = await import('remark-directive');
   const {VFile} = await import('vfile');
 
+  const defaultRemarkPlugins = await getDefaultRemarkPlugins();
+
   // /!\ this method is synchronous on purpose
   // Using async code here can create cache entry race conditions!
   function createProcessorSync({
@@ -106,7 +104,7 @@ async function createProcessorFactory() {
       directive,
       [contentTitle, {removeContentTitle: options.removeContentTitle}],
       ...getAdmonitionsPlugins(options.admonitions ?? false),
-      ...DEFAULT_OPTIONS.remarkPlugins,
+      ...defaultRemarkPlugins,
       details,
       head,
       ...(options.markdownConfig.mermaid ? [mermaid] : []),
@@ -136,7 +134,6 @@ async function createProcessorFactory() {
 
     const rehypePlugins: MDXPlugin[] = [
       ...(options.beforeDefaultRehypePlugins ?? []),
-      ...DEFAULT_OPTIONS.rehypePlugins,
       ...(options.rehypePlugins ?? []),
     ];
 

packages/docusaurus-mdx-loader/src/remark/admonitions/index.ts

@@ -4,7 +4,6 @@
  * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
-import visit from 'unist-util-visit';
 // @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
 import type {Transformer, Processor} from 'unified';
 
@@ -67,6 +66,7 @@ function parseDirective(directive: ContainerDirective): {
   contentNodes: DirectiveContent;
 } {
   const hasDirectiveLabel =
+    // @ts-expect-error: fine
     directive.children?.[0]?.data?.directiveLabel === true;
   if (hasDirectiveLabel) {
     const [directiveLabel, ...contentNodes] = directive.children;
@@ -92,6 +92,8 @@ const plugin: Plugin = function plugin(
   const {keywords} = normalizeAdmonitionOptions(optionsInput);
 
   return async (root) => {
+    const {visit} = await import('unist-util-visit');
+
     visit(root, (node) => {
       if (node.type === 'containerDirective') {
         const directive = node as ContainerDirective;

packages/docusaurus-mdx-loader/src/remark/contentTitle/index.ts

@@ -5,7 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import visit, {EXIT} from 'unist-util-visit';
 // @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
 import type {Transformer} from 'unified';
 import type {Heading} from 'mdast';
@@ -33,11 +32,13 @@ const plugin: Plugin = function plugin(
 
   return async (root, vfile) => {
     const {toString} = await import('mdast-util-to-string');
+    const {visit, EXIT} = await import('unist-util-visit');
+
     visit(root, 'heading', (headingNode: Heading, index, parent) => {
       if (headingNode.depth === 1) {
-        vfile.data.compilerName;
         vfile.data.contentTitle = toString(headingNode);
         if (removeContentTitle) {
+          // @ts-expect-error: TODO how to fix?
           parent!.children.splice(index, 1);
         }
         return EXIT; // We only handle the very first heading

packages/docusaurus-mdx-loader/src/remark/details/index.ts

@@ -5,7 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import visit from 'unist-util-visit';
 // @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
 import type {Transformer} from 'unified';
 
@@ -15,7 +14,8 @@ import type {MdxJsxFlowElement} from 'mdast-util-mdx';
 // Transform <details> to <Details>
 // MDX 2 doesn't allow to substitute html elements with the provider anymore
 export default function plugin(): Transformer {
-  return (root) => {
+  return async (root) => {
+    const {visit} = await import('unist-util-visit');
     visit(root, 'mdxJsxFlowElement', (node: MdxJsxFlowElement) => {
       if (node.name === 'details') {
         node.name = 'Details';

packages/docusaurus-mdx-loader/src/remark/head/index.ts

@@ -5,7 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import visit from 'unist-util-visit';
 // @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
 import type {Transformer} from 'unified';
 
@@ -15,7 +14,8 @@ import type {MdxJsxFlowElement} from 'mdast-util-mdx';
 // Transform <head> to <Head>
 // MDX 2 doesn't allow to substitute html elements with the provider anymore
 export default function plugin(): Transformer {
-  return (root) => {
+  return async (root) => {
+    const {visit} = await import('unist-util-visit');
     visit(root, 'mdxJsxFlowElement', (node: MdxJsxFlowElement) => {
       if (node.name === 'head') {
         node.name = 'Head';

packages/docusaurus-mdx-loader/src/remark/headings/__tests__/index.test.ts

@@ -8,9 +8,9 @@
 /* Based on remark-slug (https://github.com/remarkjs/remark-slug) and gatsby-remark-autolink-headers (https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby-remark-autolink-headers) */
 
 import u from 'unist-builder';
-import removePosition from 'unist-util-remove-position';
+import {removePosition} from 'unist-util-remove-position';
 import {toString} from 'mdast-util-to-string';
-import visit from 'unist-util-visit';
+import {visit} from 'unist-util-visit';
 import slug from '../index';
 import type {Plugin} from 'unified';
 import type {Parent} from 'unist';
@@ -18,7 +18,9 @@ import type {Parent} from 'unist';
 async function process(doc: string, plugins: Plugin[] = []) {
   const {remark} = await import('remark');
   const processor = await remark().use({plugins: [...plugins, slug]});
-  return removePosition(await processor.run(processor.parse(doc)), true);
+  const result = await processor.run(processor.parse(doc));
+  removePosition(result, {force: true});
+  return result;
 }
 
 function heading(label: string | null, id: string) {

packages/docusaurus-mdx-loader/src/remark/headings/index.ts

@@ -8,7 +8,6 @@
 /* Based on remark-slug (https://github.com/remarkjs/remark-slug) and gatsby-remark-autolink-headers (https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby-remark-autolink-headers) */
 
 import {parseMarkdownHeadingId, createSlugger} from '@docusaurus/utils';
-import visit from 'unist-util-visit';
 // @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
 import type {Transformer} from 'unified';
 import type {Heading, Text} from 'mdast';
@@ -16,6 +15,7 @@ import type {Heading, Text} from 'mdast';
 export default function plugin(): Transformer {
   return async (root) => {
     const {toString} = await import('mdast-util-to-string');
+    const {visit} = await import('unist-util-visit');
 
     const slugs = createSlugger();
     visit(root, 'heading', (headingNode: Heading) => {

packages/docusaurus-mdx-loader/src/remark/mdx1Compat/codeCompatPlugin.ts

@@ -5,7 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import visit from 'unist-util-visit';
 // @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
 import type {Transformer, Processor} from 'unified';
 import type {Code} from 'mdast';
@@ -18,19 +17,18 @@ import type {Code} from 'mdast';
 // To make theme-classic/src/theme/MDXComponents/Pre work
 // we need to fill two properties that mdx v2 doesn't provide anymore
 export default function codeCompatPlugin(this: Processor): Transformer {
-  return (root) => {
+  return async (root) => {
+    const {visit} = await import('unist-util-visit');
+
     visit(root, 'code', (node: Code) => {
       node.data = node.data || {};
+
       node.data.hProperties = node.data.hProperties || {};
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      (node.data.hProperties as any).metastring = node.meta;
+      node.data.hProperties.metastring = node.meta;
 
       // Retrocompatible support for live codeblock metastring
       // Not really the appropriate place to handle that :s
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      (node.data.hProperties as any).live = node.meta
-        ?.split(' ')
-        .includes('live');
+      node.data.hProperties.live = node.meta?.split(' ').includes('live');
     });
   };
 }

packages/docusaurus-mdx-loader/src/remark/mermaid/index.ts

@@ -5,7 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import visit from 'unist-util-visit';
 import {transformNode} from '../utils';
 
 // @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
@@ -17,7 +16,9 @@ import type {Code} from 'mdast';
 // https://github.com/facebook/docusaurus/issues/6370), this should be provided
 // by theme-mermaid itself
 export default function plugin(): Transformer {
-  return (root) => {
+  return async (root) => {
+    const {visit} = await import('unist-util-visit');
+
     visit(root, 'code', (node: Code) => {
       if (node.lang === 'mermaid') {
         // TODO migrate to mdxJsxFlowElement? cf admonitions

packages/docusaurus-mdx-loader/src/remark/toc/index.ts

@@ -8,13 +8,16 @@
 import {parse, type ParserOptions} from '@babel/parser';
 import traverse from '@babel/traverse';
 import stringifyObject from 'stringify-object';
-import visit from 'unist-util-visit';
 import {toValue} from '../utils';
 import type {Identifier} from '@babel/types';
 import type {Node, Parent} from 'unist';
 import type {Heading, Literal} from 'mdast';
 // @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
 import type {Transformer} from 'unified';
+import type {
+  MdxjsEsm,
+  // @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
+} from 'mdast-util-mdx';
 
 // TODO as of April 2023, no way to import/re-export this ESM type easily :/
 // TODO upgrade to TS 5.3
@@ -88,6 +91,8 @@ const plugin: Plugin = function plugin(
 
   return async (root) => {
     const {toString} = await import('mdast-util-to-string');
+    const {visit} = await import('unist-util-visit');
+
     const headings: TOCItem[] = [];
 
     visit(root, 'heading', (child: Heading) => {
@@ -100,11 +105,12 @@ const plugin: Plugin = function plugin(
 
       headings.push({
         value: toValue(child, toString),
-        id: child.data!.id as string,
+        id: child.data!.id!,
         level: child.depth,
       });
     });
-    const {children} = root as Parent<Literal>;
+
+    const {children} = root as Parent;
     const targetIndex = await getOrCreateExistingTargetIndex(children, name);
 
     if (headings?.length) {
@@ -115,7 +121,7 @@ const plugin: Plugin = function plugin(
 
 export default plugin;
 
-async function createExportNode(name: string, object: any) {
+async function createExportNode(name: string, object: any): Promise<MdxjsEsm> {
   const {valueToEstree} = await import('estree-util-value-to-estree');
 
   return {

packages/docusaurus-mdx-loader/src/remark/transformImage/index.ts

@@ -16,7 +16,6 @@ import {
   getFileLoaderUtils,
   findAsyncSequential,
 } from '@docusaurus/utils';
-import visit from 'unist-util-visit';
 import escapeHtml from 'escape-html';
 import sizeOf from 'image-size';
 import logger from '@docusaurus/logger';
@@ -192,6 +191,8 @@ async function processImageNode(target: Target, context: Context) {
 
 export default function plugin(options: PluginOptions): Transformer {
   return async (root, vfile) => {
+    const {visit} = await import('unist-util-visit');
+
     const promises: Promise<void>[] = [];
     visit(root, 'image', (node: Image, index, parent) => {
       promises.push(

packages/docusaurus-mdx-loader/src/remark/transformLinks/index.ts

@@ -15,7 +15,6 @@ import {
   getFileLoaderUtils,
   findAsyncSequential,
 } from '@docusaurus/utils';
-import visit from 'unist-util-visit';
 import escapeHtml from 'escape-html';
 import {assetRequireAttributeValue, transformNode} from '../utils';
 // @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
@@ -175,6 +174,8 @@ async function processLinkNode(target: Target, context: Context) {
 
 export default function plugin(options: PluginOptions): Transformer {
   return async (root, vfile) => {
+    const {visit} = await import('unist-util-visit');
+
     const promises: Promise<void>[] = [];
     visit(root, 'link', (node: Link, index, parent) => {
       promises.push(

packages/docusaurus-mdx-loader/src/remark/unusedDirectives/index.ts

@@ -6,7 +6,6 @@
  */
 import path from 'path';
 import process from 'process';
-import visit from 'unist-util-visit';
 import logger from '@docusaurus/logger';
 import {posixPath} from '@docusaurus/utils';
 import {transformNode} from '../utils';
@@ -14,7 +13,7 @@ import {transformNode} from '../utils';
 // @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
 import type {Transformer, Processor, Parent} from 'unified';
 import type {
-  Directive,
+  Directives,
   TextDirective,
   // @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
 } from 'mdast-util-directive';
@@ -25,7 +24,7 @@ import type {
 // import type {Plugin} from 'unified';
 type Plugin = any; // TODO fix this asap
 
-type DirectiveType = Directive['type'];
+type DirectiveType = Directives['type'];
 
 const directiveTypes: DirectiveType[] = [
   'containerDirective',
@@ -39,7 +38,7 @@ const directivePrefixMap: {[key in DirectiveType]: string} = {
   containerDirective: ':::',
 };
 
-function formatDirectiveName(directive: Directive) {
+function formatDirectiveName(directive: Directives) {
   const prefix = directivePrefixMap[directive.type];
   if (!prefix) {
     throw new Error(
@@ -50,13 +49,13 @@ function formatDirectiveName(directive: Directive) {
   return `${prefix}${directive.name}`;
 }
 
-function formatDirectivePosition(directive: Directive): string | undefined {
+function formatDirectivePosition(directive: Directives): string | undefined {
   return directive.position?.start
     ? logger.interpolate`number=${directive.position.start.line}:number=${directive.position.start.column}`
     : undefined;
 }
 
-function formatUnusedDirectiveMessage(directive: Directive) {
+function formatUnusedDirectiveMessage(directive: Directives) {
   const name = formatDirectiveName(directive);
   const position = formatDirectivePosition(directive);
 
@@ -67,7 +66,7 @@ function formatUnusedDirectivesMessage({
   directives,
   filePath,
 }: {
-  directives: Directive[];
+  directives: Directives[];
   filePath: string;
 }): string {
   const supportUrl = 'https://github.com/facebook/docusaurus/pull/9394';
@@ -87,7 +86,7 @@ function logUnusedDirectivesWarning({
   directives,
   filePath,
 }: {
-  directives: Directive[];
+  directives: Directives[];
   filePath: string;
 }) {
   if (directives.length > 0) {
@@ -99,13 +98,13 @@ function logUnusedDirectivesWarning({
   }
 }
 
-function isTextDirective(directive: Directive): directive is TextDirective {
+function isTextDirective(directive: Directives): directive is TextDirective {
   return directive.type === 'textDirective';
 }
 
 // A simple text directive is one without any label/props
 function isSimpleTextDirective(
-  directive: Directive,
+  directive: Directives,
 ): directive is TextDirective {
   if (isTextDirective(directive)) {
     // Attributes in MDAST = Directive props
@@ -118,34 +117,40 @@ function isSimpleTextDirective(
   return false;
 }
 
-function transformSimpleTextDirectiveToString(textDirective: Directive) {
+function transformSimpleTextDirectiveToString(textDirective: Directives) {
   transformNode(textDirective, {
     type: 'text',
     value: `:${textDirective.name}`, // We ignore label/props on purpose here
   });
 }
 
-function isUnusedDirective(directive: Directive) {
+function isUnusedDirective(directive: Directives) {
   // If directive data is set (notably hName/hProperties set by admonitions)
   // this usually means the directive has been handled by another plugin
   return !directive.data;
 }
 
 const plugin: Plugin = function plugin(this: Processor): Transformer {
-  return (tree, file) => {
-    const unusedDirectives: Directive[] = [];
+  return async (tree, file) => {
+    const {visit} = await import('unist-util-visit');
 
-    visit<Parent>(tree, directiveTypes, (directive: Directive) => {
-      // If directive data is set (notably hName/hProperties set by admonitions)
-      // this usually means the directive has been handled by another plugin
-      if (isUnusedDirective(directive)) {
-        if (isSimpleTextDirective(directive)) {
-          transformSimpleTextDirectiveToString(directive);
-        } else {
-          unusedDirectives.push(directive);
+    const unusedDirectives: Directives[] = [];
+
+    visit<Parent, DirectiveType[]>(
+      tree,
+      directiveTypes,
+      (directive: Directives) => {
+        // If directive data is set (hName/hProperties set by admonitions)
+        // this usually means the directive has been handled by another plugin
+        if (isUnusedDirective(directive)) {
+          if (isSimpleTextDirective(directive)) {
+            transformSimpleTextDirectiveToString(directive);
+          } else {
+            unusedDirectives.push(directive);
+          }
         }
-      }
-    });
+      },
+    );
 
     // We only enable these warnings for the client compiler
     // This avoids emitting duplicate warnings in prod mode

packages/docusaurus-mdx-loader/src/remark/utils/index.ts

@@ -74,7 +74,7 @@ function mdxJsxTextElementToHtml(
 }
 
 export function toValue(
-  node: PhrasingContent | Heading,
+  node: PhrasingContent | Heading | MdxJsxTextElement,
   toString: (param: unknown) => string, // TODO weird but works
 ): string {
   switch (node.type) {

packages/docusaurus-mdx-loader/src/types.d.mts

@@ -19,3 +19,16 @@ declare module 'vfile' {
     contentTitle?: string;
   }
 }
+
+declare module 'unist' {
+  interface Data {
+    hName?: string;
+    hProperties?: Record<string, unknown>;
+  }
+}
+
+declare module 'mdast' {
+  interface HeadingData {
+    id?: string;
+  }
+}