feat(core, mdx-loader): deduplicate MDX compilation - siteConfig.future.experimental_faster.mdxCrossCompilerCache (#10479)

This commit is contained in:
Sébastien Lorber 2024-09-06 16:07:09 +02:00 committed by GitHub
parent 897ebbe3ca
commit 5bab0b5432
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 333 additions and 151 deletions

View file

@ -6,30 +6,49 @@
*/
import {createProcessors} from './processor';
import type {Options} from './loader';
import type {Options} from './options';
import type {RuleSetRule, RuleSetUseItem} from 'webpack';
async function enhancedOptions(options: Options): Promise<Options> {
type CreateOptions = {
useCrossCompilerCache?: boolean;
};
async function normalizeOptions(
optionsInput: Options & CreateOptions,
): Promise<Options> {
// Because Jest doesn't like ESM / createProcessors()
if (process.env.N0DE_ENV === 'test' || process.env.JEST_WORKER_ID) {
return options;
return optionsInput;
}
let options = optionsInput;
// We create the processor earlier here, to avoid the lazy processor creating
// Lazy creation messes-up with Rsdoctor ability to measure mdx-loader perf
const newOptions: Options = options.processors
? options
: {...options, processors: await createProcessors({options})};
if (!options.processors) {
options = {...options, processors: await createProcessors({options})};
}
return newOptions;
// Cross-compiler cache permits to compile client/server MDX only once
// We don't want to cache in dev mode (docusaurus start)
// We only have multiple compilers in production mode (docusaurus build)
// TODO wrong but good enough for now (example: "docusaurus build --dev")
if (options.useCrossCompilerCache && process.env.NODE_ENV === 'production') {
options = {
...options,
crossCompilerCache: new Map(),
};
}
return options;
}
export async function createMDXLoaderItem(
options: Options,
options: Options & CreateOptions,
): Promise<RuleSetUseItem> {
return {
loader: require.resolve('@docusaurus/mdx-loader'),
options: await enhancedOptions(options),
options: await normalizeOptions(options),
};
}
@ -38,7 +57,7 @@ export async function createMDXLoaderRule({
options,
}: {
include: RuleSetRule['include'];
options: Options;
options: Options & CreateOptions;
}): Promise<RuleSetRule> {
return {
test: /\.mdx?$/i,

View file

@ -37,5 +37,6 @@ export type LoadedMDXContent<FrontMatter, Metadata, Assets = undefined> = {
(): JSX.Element;
};
export type {Options, MDXPlugin} from './loader';
export type {MDXPlugin} from './loader';
export type {MDXOptions} from './processor';
export type {Options} from './options';

View file

@ -18,14 +18,8 @@ import {
createAssetsExportCode,
extractContentTitleData,
} from './utils';
import type {
SimpleProcessors,
MDXOptions,
SimpleProcessorResult,
} from './processor';
import type {ResolveMarkdownLink} from './remark/resolveMarkdownLinks';
import type {MarkdownConfig} from '@docusaurus/types';
import type {WebpackCompilerName} from '@docusaurus/utils';
import type {Options} from './options';
import type {LoaderContext} from 'webpack';
// TODO as of April 2023, no way to import/re-export this ESM type easily :/
@ -35,33 +29,17 @@ type Pluggable = any; // TODO fix this asap
export type MDXPlugin = Pluggable;
export type Options = Partial<MDXOptions> & {
markdownConfig: MarkdownConfig;
staticDirs: string[];
siteDir: string;
isMDXPartial?: (filePath: string) => boolean;
isMDXPartialFrontMatterWarningDisabled?: boolean;
removeContentTitle?: boolean;
metadataPath?: (filePath: string) => string;
createAssets?: (metadata: {
filePath: string;
frontMatter: {[key: string]: unknown};
}) => {[key: string]: unknown};
resolveMarkdownLink?: ResolveMarkdownLink;
// Will usually be created by "createMDXLoaderItem"
processors?: SimpleProcessors;
};
export async function mdxLoader(
this: LoaderContext<Options>,
fileContent: string,
): Promise<void> {
const compilerName = getWebpackLoaderCompilerName(this);
const callback = this.async();
const filePath = this.resourcePath;
const options: Options = this.getOptions();
async function loadMDX({
fileContent,
filePath,
options,
compilerName,
}: {
fileContent: string;
filePath: string;
options: Options;
compilerName: WebpackCompilerName;
}): Promise<string> {
const {frontMatter} = await options.markdownConfig.parseFrontMatter({
filePath,
fileContent,
@ -70,18 +48,13 @@ export async function mdxLoader(
const hasFrontMatter = Object.keys(frontMatter).length > 0;
let result: SimpleProcessorResult;
try {
result = await compileToJSX({
fileContent,
filePath,
frontMatter,
options,
compilerName,
});
} catch (error) {
return callback(error as Error);
}
const result = await compileToJSX({
fileContent,
filePath,
frontMatter,
options,
compilerName,
});
const contentTitle = extractContentTitleData(result.data);
@ -97,7 +70,7 @@ ${JSON.stringify(frontMatter, null, 2)}`;
if (!options.isMDXPartialFrontMatterWarningDisabled) {
const shouldError = process.env.NODE_ENV === 'test' || process.env.CI;
if (shouldError) {
return callback(new Error(errorMessage));
throw new Error(errorMessage);
}
logger.warn(errorMessage);
}
@ -146,5 +119,68 @@ ${exportsCode}
${result.content}
`;
return callback(null, code);
return code;
}
// Note: we cache promises instead of strings
// This is because client/server compilations might be triggered in parallel
// When this happens for the same file, we don't want to compile it twice
async function loadMDXWithCaching({
resource,
fileContent,
filePath,
options,
compilerName,
}: {
resource: string; // path?query#hash
filePath: string; // path
fileContent: string;
options: Options;
compilerName: WebpackCompilerName;
}): Promise<string> {
// Note we "resource" as cache key, not "filePath" nor "fileContent"
// This is because:
// - the same file can be compiled in different variants (blog.mdx?truncated)
// - the same content can be processed differently (versioned docs links)
const cacheKey = resource;
const cachedPromise = options.crossCompilerCache?.get(cacheKey);
if (cachedPromise) {
// We can clean up the cache and free memory here
// We know there are only 2 compilations for the same file
// Note: once we introduce RSCs we'll probably have 3 compilations
// Note: we can't use string keys in WeakMap
// But we could eventually use WeakRef for the values
options.crossCompilerCache?.delete(cacheKey);
return cachedPromise;
}
const promise = loadMDX({
fileContent,
filePath,
options,
compilerName,
});
options.crossCompilerCache?.set(cacheKey, promise);
return promise;
}
export async function mdxLoader(
this: LoaderContext<Options>,
fileContent: string,
): Promise<void> {
const compilerName = getWebpackLoaderCompilerName(this);
const callback = this.async();
const options: Options = this.getOptions();
try {
const result = await loadMDXWithCaching({
resource: this.resource,
filePath: this.resourcePath,
fileContent,
options,
compilerName,
});
return callback(null, result);
} catch (error) {
return callback(error as Error);
}
}

View file

@ -0,0 +1,29 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*/
import type {MDXOptions, SimpleProcessors} from './processor';
import type {MarkdownConfig} from '@docusaurus/types';
import type {ResolveMarkdownLink} from './remark/resolveMarkdownLinks';
export type Options = Partial<MDXOptions> & {
markdownConfig: MarkdownConfig;
staticDirs: string[];
siteDir: string;
isMDXPartial?: (filePath: string) => boolean;
isMDXPartialFrontMatterWarningDisabled?: boolean;
removeContentTitle?: boolean;
metadataPath?: (filePath: string) => string;
createAssets?: (metadata: {
filePath: string;
frontMatter: {[key: string]: unknown};
}) => {[key: string]: unknown};
resolveMarkdownLink?: ResolveMarkdownLink;
// Will usually be created by "createMDXLoaderItem"
processors?: SimpleProcessors;
crossCompilerCache?: Map<string, Promise<string>>; // MDX => Promise<JSX> cache
};

View file

@ -11,7 +11,7 @@ import {
admonitionTitleToDirectiveLabel,
} from '@docusaurus/utils';
import {normalizeAdmonitionOptions} from './remark/admonitions';
import type {Options} from './loader';
import type {Options} from './options';
/**
* Preprocess the string before passing it to MDX

View file

@ -20,7 +20,7 @@ import codeCompatPlugin from './remark/mdx1Compat/codeCompatPlugin';
import {getFormat} from './format';
import type {WebpackCompilerName} from '@docusaurus/utils';
import type {MDXFrontMatter} from './frontMatter';
import type {Options} from './loader';
import type {Options} from './options';
import type {AdmonitionOptions} from './remark/admonitions';
// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721

View file

@ -10,7 +10,7 @@ import {escapePath, type WebpackCompilerName} from '@docusaurus/utils';
import {getProcessor, type SimpleProcessorResult} from './processor';
import {validateMDXFrontMatter} from './frontMatter';
import preprocessor from './preprocessor';
import type {Options} from './loader';
import type {Options} from './options';
/**
* Converts assets an object with Webpack require calls code.