fix: allow html syntax in MDX v2 with format md (#8960)

* attempt to support html embeds in mdx with format md

* refactor mdx loader + support embedding html in commonmark thanks to rehype-raw

* extract processor code

* refactor processor code

* extract format + unit test

* try to refactor processor

* try to refactor processor

* adjust md page

* do not apply rehype-raw when format is mdx

* fix lint issue
This commit is contained in:
Sébastien Lorber 2023-05-12 11:36:42 +02:00 committed by GitHub
parent af9a4f2a2e
commit 07ad635b69
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 491 additions and 173 deletions

View file

@ -0,0 +1,52 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
import {getFormat} from '../format';
describe('getFormat', () => {
it('uses frontMatter format over anything else', () => {
expect(getFormat({frontMatterFormat: 'md', filePath: 'xyz.md'})).toBe('md');
expect(getFormat({frontMatterFormat: 'md', filePath: 'xyz.mdx'})).toBe(
'md',
);
expect(getFormat({frontMatterFormat: 'mdx', filePath: 'xyz.md'})).toBe(
'mdx',
);
expect(getFormat({frontMatterFormat: 'mdx', filePath: 'xyz.mdx'})).toBe(
'mdx',
);
});
it('detects appropriate format from file extension', () => {
expect(getFormat({frontMatterFormat: 'detect', filePath: 'xyz.md'})).toBe(
'md',
);
expect(
getFormat({frontMatterFormat: 'detect', filePath: 'xyz.markdown'}),
).toBe('md');
expect(
getFormat({frontMatterFormat: 'detect', filePath: 'folder/xyz.md'}),
).toBe('md');
expect(
getFormat({frontMatterFormat: 'detect', filePath: 'folder/xyz.markdown'}),
).toBe('md');
expect(getFormat({frontMatterFormat: 'detect', filePath: 'xyz.mdx'})).toBe(
'mdx',
);
expect(
getFormat({frontMatterFormat: 'detect', filePath: 'folder/xyz.mdx'}),
).toBe('mdx');
expect(
getFormat({frontMatterFormat: 'detect', filePath: 'xyz.unknown'}),
).toBe('mdx');
expect(
getFormat({frontMatterFormat: 'detect', filePath: 'folder/xyz.unknown'}),
).toBe('mdx');
});
});

View file

@ -0,0 +1,33 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
// import {createProcessor} from '../processor';
// import type {Options} from '../loader';
/*
async function testProcess({
format,
options,
}: {
format: 'md' | 'mdx';
options: Options;
}) {
return async (content: string) => {
const processor = await createProcessor({format, options});
return processor.process(content);
};
}
*/
describe('md processor', () => {
it('parses simple commonmark', async () => {
// TODO no tests for now, wait until ESM support
// Jest does not support well ESM modules
// It would require to vendor too much Unified modules as CJS
// See https://mdxjs.com/docs/troubleshooting-mdx/#esm
});
});

View file

@ -0,0 +1,40 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
import path from 'path';
import type {MDXFrontMatter} from './frontMatter';
// Copied from https://mdxjs.com/packages/mdx/#optionsmdextensions
// Although we are likely to only use .md / .mdx anyway...
const mdFormatExtensions = [
'.md',
'.markdown',
'.mdown',
'.mkdn',
'.mkd',
'.mdwn',
'.mkdown',
'.ron',
];
function isMDFormat(filepath: string) {
return mdFormatExtensions.includes(path.extname(filepath));
}
export function getFormat({
filePath,
frontMatterFormat,
}: {
filePath: string;
frontMatterFormat: MDXFrontMatter['format'];
}): 'md' | 'mdx' {
if (frontMatterFormat !== 'detect') {
return frontMatterFormat;
}
// Bias toward mdx if unknown extension
return isMDFormat(filePath) ? 'md' : 'mdx';
}

View file

@ -30,4 +30,6 @@ export type LoadedMDXContent<FrontMatter, Metadata, Assets = undefined> = {
readonly assets: Assets;
(): JSX.Element;
};
export type {Options, MDXPlugin, MDXOptions} from './loader';
export type {Options, MDXPlugin} from './loader';
export type {MDXOptions} from './processor';

View file

@ -6,7 +6,6 @@
*/
import fs from 'fs-extra';
import path from 'path';
import logger from '@docusaurus/logger';
import {
parseFrontMatter,
@ -14,82 +13,26 @@ import {
escapePath,
getFileLoaderUtils,
} from '@docusaurus/utils';
import emoji from 'remark-emoji';
import stringifyObject from 'stringify-object';
import preprocessor from './preprocessor';
import headings from './remark/headings';
import toc from './remark/toc';
import transformImage from './remark/transformImage';
import transformLinks from './remark/transformLinks';
import details from './remark/details';
import head from './remark/head';
import mermaid from './remark/mermaid';
import transformAdmonitions from './remark/admonitions';
import codeCompatPlugin from './remark/mdx1Compat/codeCompatPlugin';
import {validateMDXFrontMatter} from './frontMatter';
import {createProcessorCached} from './processor';
import type {MDXOptions} from './processor';
import type {MarkdownConfig} from '@docusaurus/types';
import type {LoaderContext} from 'webpack';
// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
import type {Processor} from 'unified';
import type {AdmonitionOptions} from './remark/admonitions';
// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
import type {ProcessorOptions} from '@mdx-js/mdx';
// TODO as of April 2023, no way to import/re-export this ESM type easily :/
// This might change soon, likely after TS 5.2
// See https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1517839391
type Pluggable = any; // TODO fix this asap
// Copied from https://mdxjs.com/packages/mdx/#optionsmdextensions
// Although we are likely to only use .md / .mdx anyway...
const mdFormatExtensions = [
'.md',
'.markdown',
'.mdown',
'.mkdn',
'.mkd',
'.mdwn',
'.mkdown',
'.ron',
];
function isMDFormat(filepath: string) {
return mdFormatExtensions.includes(path.extname(filepath));
}
const {
loaders: {inlineMarkdownImageFileLoader},
} = getFileLoaderUtils();
const DEFAULT_OPTIONS: MDXOptions = {
admonitions: true,
rehypePlugins: [],
remarkPlugins: [emoji, headings, toc],
beforeDefaultRemarkPlugins: [],
beforeDefaultRehypePlugins: [],
};
type CompilerCacheEntry = {
mdCompiler: Processor;
mdxCompiler: Processor;
options: Options;
};
const compilerCache = new Map<string | Options, CompilerCacheEntry>();
export type MDXPlugin = Pluggable;
export type MDXOptions = {
admonitions: boolean | Partial<AdmonitionOptions>;
remarkPlugins: MDXPlugin[];
rehypePlugins: MDXPlugin[];
beforeDefaultRemarkPlugins: MDXPlugin[];
beforeDefaultRehypePlugins: MDXPlugin[];
};
export type Options = Partial<MDXOptions> & {
markdownConfig: MarkdownConfig;
staticDirs: string[];
@ -167,20 +110,6 @@ function createAssetsExportCode(assets: unknown) {
return `{\n${codeLines.join('\n')}\n}`;
}
function getAdmonitionsPlugins(
admonitionsOption: MDXOptions['admonitions'],
): MDXPlugin[] {
if (admonitionsOption) {
const plugin: MDXPlugin =
admonitionsOption === true
? transformAdmonitions
: [transformAdmonitions, admonitionsOption];
return [plugin];
}
return [];
}
// TODO temporary, remove this after v3.1?
// Some plugin authors use our mdx-loader, despite it not being public API
// see https://github.com/facebook/docusaurus/issues/8298
@ -198,14 +127,10 @@ export async function mdxLoader(
): Promise<void> {
const callback = this.async();
const filePath = this.resourcePath;
const reqOptions = this.getOptions();
const reqOptions: Options = this.getOptions();
const {query} = this;
ensureMarkdownConfig(reqOptions);
const {createProcessor} = await import('@mdx-js/mdx');
const {default: gfm} = await import('remark-gfm');
const {default: comment} = await import('remark-comment');
const {default: directives} = await import('remark-directive');
const {frontMatter, content: contentWithTitle} = parseFrontMatter(fileString);
const mdxFrontMatter = validateMDXFrontMatter(frontMatter.mdx);
@ -225,93 +150,16 @@ export async function mdxLoader(
const hasFrontMatter = Object.keys(frontMatter).length > 0;
if (!compilerCache.has(this.query)) {
/*
/!\ DO NOT PUT ANY ASYNC / AWAIT / DYNAMIC IMPORTS HERE
This creates cache creation race conditions
TODO extract this in a synchronous method
*/
const remarkPlugins: MDXPlugin[] = [
...(reqOptions.beforeDefaultRemarkPlugins ?? []),
directives,
...getAdmonitionsPlugins(reqOptions.admonitions ?? false),
...DEFAULT_OPTIONS.remarkPlugins,
details,
head,
...(reqOptions.markdownConfig.mermaid ? [mermaid] : []),
[
transformImage,
{
staticDirs: reqOptions.staticDirs,
siteDir: reqOptions.siteDir,
},
],
[
transformLinks,
{
staticDirs: reqOptions.staticDirs,
siteDir: reqOptions.siteDir,
},
],
gfm,
reqOptions.markdownConfig.mdx1Compat.comments ? comment : null,
...(reqOptions.remarkPlugins ?? []),
].filter((plugin): plugin is MDXPlugin => Boolean(plugin));
// codeCompatPlugin needs to be applied last after user-provided plugins
// (after npm2yarn for example)
remarkPlugins.push(codeCompatPlugin);
const rehypePlugins: MDXPlugin[] = [
...(reqOptions.beforeDefaultRehypePlugins ?? []),
...DEFAULT_OPTIONS.rehypePlugins,
...(reqOptions.rehypePlugins ?? []),
];
const options: ProcessorOptions & Options = {
...reqOptions,
remarkPlugins,
rehypePlugins,
providerImportSource: '@mdx-js/react',
};
const compilerCacheEntry: CompilerCacheEntry = {
mdCompiler: createProcessor({
...options,
format: 'md',
}),
mdxCompiler: createProcessor({
...options,
format: 'mdx',
}),
options,
};
compilerCache.set(this.query, compilerCacheEntry);
}
const {mdCompiler, mdxCompiler, options} = compilerCache.get(this.query)!;
function getCompiler() {
const format =
mdxFrontMatter.format === 'detect'
? isMDFormat(filePath)
? 'md'
: 'mdx'
: mdxFrontMatter.format;
return format === 'md' ? mdCompiler : mdxCompiler;
}
const processor = await createProcessorCached({
filePath,
reqOptions,
query,
mdxFrontMatter,
});
let result: string;
try {
result = await getCompiler()
.process({
value: content,
path: filePath,
})
.then((res) => res.toString());
result = await processor.process({content, filePath});
} catch (errorUnknown) {
const error = errorUnknown as Error;
return callback(
@ -327,14 +175,14 @@ export async function mdxLoader(
// MDX partials are MDX files starting with _ or in a folder starting with _
// Partial are not expected to have associated metadata files or front matter
const isMDXPartial = options.isMDXPartial?.(filePath);
const isMDXPartial = reqOptions.isMDXPartial?.(filePath);
if (isMDXPartial && hasFrontMatter) {
const errorMessage = `Docusaurus MDX partial files should not contain front matter.
Those partial files use the _ prefix as a convention by default, but this is configurable.
File at ${filePath} contains front matter that will be ignored:
${JSON.stringify(frontMatter, null, 2)}`;
if (!options.isMDXPartialFrontMatterWarningDisabled) {
if (!reqOptions.isMDXPartialFrontMatterWarningDisabled) {
const shouldError = process.env.NODE_ENV === 'test' || process.env.CI;
if (shouldError) {
return callback(new Error(errorMessage));
@ -346,8 +194,11 @@ ${JSON.stringify(frontMatter, null, 2)}`;
function getMetadataPath(): string | undefined {
if (!isMDXPartial) {
// Read metadata for this MDX and export it.
if (options.metadataPath && typeof options.metadataPath === 'function') {
return options.metadataPath(filePath);
if (
reqOptions.metadataPath &&
typeof reqOptions.metadataPath === 'function'
) {
return reqOptions.metadataPath(filePath);
}
}
return undefined;

View file

@ -0,0 +1,241 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
import emoji from 'remark-emoji';
import headings from './remark/headings';
import toc from './remark/toc';
import transformImage from './remark/transformImage';
import transformLinks from './remark/transformLinks';
import details from './remark/details';
import head from './remark/head';
import mermaid from './remark/mermaid';
import transformAdmonitions from './remark/admonitions';
import codeCompatPlugin from './remark/mdx1Compat/codeCompatPlugin';
import {getFormat} from './format';
import type {MDXFrontMatter} from './frontMatter';
import type {Options} from './loader';
import type {AdmonitionOptions} from './remark/admonitions';
// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
import type {ProcessorOptions} from '@mdx-js/mdx';
// TODO as of April 2023, no way to import/re-export this ESM type easily :/
// This might change soon, likely after TS 5.2
// See https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1517839391
type Pluggable = any; // TODO fix this asap
// TODO alt interface because impossible to import type Processor (ESM + TS :/)
type SimpleProcessor = {
process: ({
content,
filePath,
}: {
content: string;
filePath: string;
}) => Promise<string>;
};
const DEFAULT_OPTIONS: MDXOptions = {
admonitions: true,
rehypePlugins: [],
remarkPlugins: [emoji, headings, toc],
beforeDefaultRemarkPlugins: [],
beforeDefaultRehypePlugins: [],
};
export type MDXPlugin = Pluggable;
export type MDXOptions = {
admonitions: boolean | Partial<AdmonitionOptions>;
remarkPlugins: MDXPlugin[];
rehypePlugins: MDXPlugin[];
beforeDefaultRemarkPlugins: MDXPlugin[];
beforeDefaultRehypePlugins: MDXPlugin[];
};
function getAdmonitionsPlugins(
admonitionsOption: MDXOptions['admonitions'],
): MDXPlugin[] {
if (admonitionsOption) {
const plugin: MDXPlugin =
admonitionsOption === true
? transformAdmonitions
: [transformAdmonitions, admonitionsOption];
return [plugin];
}
return [];
}
// Need to be async due to ESM dynamic imports...
async function createProcessorFactory() {
const {createProcessor: createMdxProcessor} = await import('@mdx-js/mdx');
const {default: rehypeRaw} = await import('rehype-raw');
const {default: gfm} = await import('remark-gfm');
const {default: comment} = await import('remark-comment');
const {default: directive} = await import('remark-directive');
// /!\ this method is synchronous on purpose
// Using async code here can create cache entry race conditions!
function createProcessorSync({
options,
format,
}: {
options: Options;
format: 'md' | 'mdx';
}): SimpleProcessor {
const remarkPlugins: MDXPlugin[] = [
...(options.beforeDefaultRemarkPlugins ?? []),
directive,
...getAdmonitionsPlugins(options.admonitions ?? false),
...DEFAULT_OPTIONS.remarkPlugins,
details,
head,
...(options.markdownConfig.mermaid ? [mermaid] : []),
[
transformImage,
{
staticDirs: options.staticDirs,
siteDir: options.siteDir,
},
],
[
transformLinks,
{
staticDirs: options.staticDirs,
siteDir: options.siteDir,
},
],
gfm,
options.markdownConfig.mdx1Compat.comments ? comment : null,
...(options.remarkPlugins ?? []),
].filter((plugin): plugin is MDXPlugin => Boolean(plugin));
// codeCompatPlugin needs to be applied last after user-provided plugins
// (after npm2yarn for example)
remarkPlugins.push(codeCompatPlugin);
const rehypePlugins: MDXPlugin[] = [
...(options.beforeDefaultRehypePlugins ?? []),
...DEFAULT_OPTIONS.rehypePlugins,
...(options.rehypePlugins ?? []),
];
if (format === 'md') {
// This is what permits to embed HTML elements with format 'md'
// See https://github.com/facebook/docusaurus/pull/8960
// See https://github.com/mdx-js/mdx/pull/2295#issuecomment-1540085960
const rehypeRawPlugin: MDXPlugin = [
rehypeRaw,
{
passThrough: [
'mdxFlowExpression',
'mdxJsxFlowElement',
'mdxJsxTextElement',
'mdxTextExpression',
'mdxjsEsm',
],
},
];
rehypePlugins.unshift(rehypeRawPlugin);
}
const processorOptions: ProcessorOptions & Options = {
...options,
remarkPlugins,
rehypePlugins,
providerImportSource: '@mdx-js/react',
};
const mdxProcessor = createMdxProcessor({
...processorOptions,
format,
});
return {
process: async ({content, filePath}) =>
mdxProcessor
.process({
value: content,
path: filePath,
})
.then((res) => res.toString()),
};
}
return {createProcessorSync};
}
// Will be useful for tests
export async function createProcessorUncached(parameters: {
options: Options;
format: 'md' | 'mdx';
}): Promise<SimpleProcessor> {
const {createProcessorSync} = await createProcessorFactory();
return createProcessorSync(parameters);
}
// We use different compilers depending on the file type (md vs mdx)
type ProcessorsCacheEntry = {
mdProcessor: SimpleProcessor;
mdxProcessor: SimpleProcessor;
};
// Compilers are cached so that Remark/Rehype plugins can run
// expensive code during initialization
const ProcessorsCache = new Map<string | Options, ProcessorsCacheEntry>();
async function createProcessorsCacheEntry({
query,
reqOptions,
}: {
query: string | Options;
reqOptions: Options;
}): Promise<ProcessorsCacheEntry> {
const {createProcessorSync} = await createProcessorFactory();
const compilers = ProcessorsCache.get(query);
if (compilers) {
return compilers;
}
const compilerCacheEntry: ProcessorsCacheEntry = {
mdProcessor: createProcessorSync({
options: reqOptions,
format: 'md',
}),
mdxProcessor: createProcessorSync({
options: reqOptions,
format: 'mdx',
}),
};
ProcessorsCache.set(query, compilerCacheEntry);
return compilerCacheEntry;
}
export async function createProcessorCached({
filePath,
mdxFrontMatter,
query,
reqOptions,
}: {
filePath: string;
mdxFrontMatter: MDXFrontMatter;
query: string | Options;
reqOptions: Options;
}): Promise<SimpleProcessor> {
const compilers = await createProcessorsCacheEntry({query, reqOptions});
const format = getFormat({
filePath,
frontMatterFormat: mdxFrontMatter.format,
});
return format === 'md' ? compilers.mdProcessor : compilers.mdxProcessor;
}