feat: upgrade to MDX v2 (#8288)

Co-authored-by: Armano <armano2@users.noreply.github.com>

packages/docusaurus-mdx-loader/src/loader.ts

@@ -6,6 +6,7 @@
  */
 
 import fs from 'fs-extra';
+import path from 'path';
 import logger from '@docusaurus/logger';
 import {
   parseFrontMatter,
@@ -13,46 +14,73 @@ import {
   escapePath,
   getFileLoaderUtils,
 } from '@docusaurus/utils';
-import {createCompiler} from '@mdx-js/mdx';
 import emoji from 'remark-emoji';
 import stringifyObject from 'stringify-object';
-
+import preprocessor from './preprocessor';
 import headings from './remark/headings';
 import toc from './remark/toc';
-import unwrapMdxCodeBlocks from './remark/unwrapMdxCodeBlocks';
 import transformImage from './remark/transformImage';
 import transformLinks from './remark/transformLinks';
+import details from './remark/details';
+import head from './remark/head';
 import mermaid from './remark/mermaid';
-
 import transformAdmonitions from './remark/admonitions';
+import codeCompatPlugin from './remark/mdx1Compat/codeCompatPlugin';
+import {validateMDXFrontMatter} from './frontMatter';
+
 import type {MarkdownConfig} from '@docusaurus/types';
 import type {LoaderContext} from 'webpack';
-import type {Processor, Plugin} from 'unified';
+
+// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
+import type {Processor} from 'unified';
 import type {AdmonitionOptions} from './remark/admonitions';
 
+// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
+import type {ProcessorOptions} from '@mdx-js/mdx';
+
+// TODO as of April 2023, no way to import/re-export this ESM type easily :/
+// This might change soon, likely after TS 5.2
+// See https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1517839391
+type Pluggable = any; // TODO fix this asap
+
+// Copied from https://mdxjs.com/packages/mdx/#optionsmdextensions
+// Although we are likely to only use .md / .mdx anyway...
+const mdFormatExtensions = [
+  '.md',
+  '.markdown',
+  '.mdown',
+  '.mkdn',
+  '.mkd',
+  '.mdwn',
+  '.mkdown',
+  '.ron',
+];
+
+function isMDFormat(filepath: string) {
+  return mdFormatExtensions.includes(path.extname(filepath));
+}
+
 const {
   loaders: {inlineMarkdownImageFileLoader},
 } = getFileLoaderUtils();
 
-const pragma = `
-/* @jsxRuntime classic */
-/* @jsx mdx */
-/* @jsxFrag React.Fragment */
-`;
-
 const DEFAULT_OPTIONS: MDXOptions = {
   admonitions: true,
   rehypePlugins: [],
-  remarkPlugins: [unwrapMdxCodeBlocks, emoji, headings, toc],
+  remarkPlugins: [emoji, headings, toc],
   beforeDefaultRemarkPlugins: [],
   beforeDefaultRehypePlugins: [],
 };
 
-const compilerCache = new Map<string | Options, [Processor, Options]>();
+type CompilerCacheEntry = {
+  mdCompiler: Processor;
+  mdxCompiler: Processor;
+  options: Options;
+};
 
-export type MDXPlugin =
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  [Plugin<any[]>, any] | Plugin<any[]>;
+const compilerCache = new Map<string | Options, CompilerCacheEntry>();
+
+export type MDXPlugin = Pluggable;
 
 export type MDXOptions = {
   admonitions: boolean | Partial<AdmonitionOptions>;
@@ -149,9 +177,21 @@ function getAdmonitionsPlugins(
         : [transformAdmonitions, admonitionsOption];
     return [plugin];
   }
+
   return [];
 }
 
+// TODO temporary, remove this after v3.1?
+// Some plugin authors use our mdx-loader, despite it not being public API
+// see https://github.com/facebook/docusaurus/issues/8298
+function ensureMarkdownConfig(reqOptions: Options) {
+  if (!reqOptions.markdownConfig) {
+    throw new Error(
+      'Docusaurus v3+ requires MDX loader options.markdownConfig - plugin authors using the MDX loader should make sure to provide that option',
+    );
+  }
+}
+
 export async function mdxLoader(
   this: LoaderContext<Options>,
   fileString: string,
@@ -159,21 +199,47 @@ export async function mdxLoader(
   const callback = this.async();
   const filePath = this.resourcePath;
   const reqOptions = this.getOptions();
+  ensureMarkdownConfig(reqOptions);
+
+  const {createProcessor} = await import('@mdx-js/mdx');
+  const {default: gfm} = await import('remark-gfm');
+  const {default: comment} = await import('remark-comment');
+  const {default: directives} = await import('remark-directive');
 
   const {frontMatter, content: contentWithTitle} = parseFrontMatter(fileString);
+  const mdxFrontMatter = validateMDXFrontMatter(frontMatter.mdx);
 
-  const {content, contentTitle} = parseMarkdownContentTitle(contentWithTitle, {
-    removeContentTitle: reqOptions.removeContentTitle,
+  const {content: contentUnprocessed, contentTitle} = parseMarkdownContentTitle(
+    contentWithTitle,
+    {
+      removeContentTitle: reqOptions.removeContentTitle,
+    },
+  );
+
+  const content = preprocessor({
+    fileContent: contentUnprocessed,
+    filePath,
+    admonitions: reqOptions.admonitions,
+    markdownConfig: reqOptions.markdownConfig,
   });
 
   const hasFrontMatter = Object.keys(frontMatter).length > 0;
 
   if (!compilerCache.has(this.query)) {
+    /*
+    /!\ DO NOT PUT ANY ASYNC / AWAIT / DYNAMIC IMPORTS HERE
+    This creates cache creation race conditions
+    TODO extract this in a synchronous method
+     */
+
     const remarkPlugins: MDXPlugin[] = [
       ...(reqOptions.beforeDefaultRemarkPlugins ?? []),
+      directives,
       ...getAdmonitionsPlugins(reqOptions.admonitions ?? false),
       ...DEFAULT_OPTIONS.remarkPlugins,
-      ...(reqOptions.markdownConfig?.mermaid ? [mermaid] : []),
+      details,
+      head,
+      ...(reqOptions.markdownConfig.mermaid ? [mermaid] : []),
       [
         transformImage,
         {
@@ -188,8 +254,14 @@ export async function mdxLoader(
           siteDir: reqOptions.siteDir,
         },
       ],
+      gfm,
+      reqOptions.markdownConfig.mdx1Compat.comments ? comment : null,
       ...(reqOptions.remarkPlugins ?? []),
-    ];
+    ].filter((plugin): plugin is MDXPlugin => Boolean(plugin));
+
+    // codeCompatPlugin needs to be applied last after user-provided plugins
+    // (after npm2yarn for example)
+    remarkPlugins.push(codeCompatPlugin);
 
     const rehypePlugins: MDXPlugin[] = [
       ...(reqOptions.beforeDefaultRehypePlugins ?? []),
@@ -197,26 +269,60 @@ export async function mdxLoader(
       ...(reqOptions.rehypePlugins ?? []),
     ];
 
-    const options: Options = {
+    const options: ProcessorOptions & Options = {
       ...reqOptions,
       remarkPlugins,
       rehypePlugins,
+      providerImportSource: '@mdx-js/react',
     };
-    compilerCache.set(this.query, [createCompiler(options), options]);
+
+    const compilerCacheEntry: CompilerCacheEntry = {
+      mdCompiler: createProcessor({
+        ...options,
+        format: 'md',
+      }),
+      mdxCompiler: createProcessor({
+        ...options,
+        format: 'mdx',
+      }),
+      options,
+    };
+
+    compilerCache.set(this.query, compilerCacheEntry);
   }
 
-  const [compiler, options] = compilerCache.get(this.query)!;
+  const {mdCompiler, mdxCompiler, options} = compilerCache.get(this.query)!;
+
+  function getCompiler() {
+    const format =
+      mdxFrontMatter.format === 'detect'
+        ? isMDFormat(filePath)
+          ? 'md'
+          : 'mdx'
+        : mdxFrontMatter.format;
+
+    return format === 'md' ? mdCompiler : mdxCompiler;
+  }
 
   let result: string;
   try {
-    result = await compiler
+    result = await getCompiler()
       .process({
-        contents: content,
-        path: this.resourcePath,
+        value: content,
+        path: filePath,
       })
       .then((res) => res.toString());
-  } catch (err) {
-    return callback(err as Error);
+  } catch (errorUnknown) {
+    const error = errorUnknown as Error;
+    return callback(
+      new Error(
+        `MDX compilation failed for file ${logger.path(filePath)}\nCause: ${
+          error.message
+        }\nDetails:\n${JSON.stringify(error, null, 2)}`,
+        // TODO error cause doesn't seem to be used by Webpack stats.errors :s
+        {cause: error},
+      ),
+    );
   }
 
   // MDX partials are MDX files starting with _ or in a folder starting with _
@@ -265,6 +371,8 @@ ${JSON.stringify(frontMatter, null, 2)}`;
       ? reqOptions.createAssets({frontMatter, metadata})
       : undefined;
 
+  // TODO use remark plugins to insert extra exports instead of string concat?
+  // cf how the toc is exported
   const exportsCode = `
 export const frontMatter = ${stringifyObject(frontMatter)};
 export const contentTitle = ${stringifyObject(contentTitle)};
@@ -273,10 +381,6 @@ ${assets ? `export const assets = ${createAssetsExportCode(assets)};` : ''}
 `;
 
   const code = `
-${pragma}
-import React from 'react';
-import { mdx } from '@mdx-js/react';
-
 ${exportsCode}
 ${result}
 `;