mirror of
https://github.com/facebook/docusaurus.git
synced 2025-07-23 11:38:48 +02:00
feat(core, mdx-loader): deduplicate MDX compilation - siteConfig.future.experimental_faster.mdxCrossCompilerCache
(#10479)
This commit is contained in:
parent
897ebbe3ca
commit
5bab0b5432
18 changed files with 333 additions and 151 deletions
|
@ -6,30 +6,49 @@
|
|||
*/
|
||||
|
||||
import {createProcessors} from './processor';
|
||||
import type {Options} from './loader';
|
||||
import type {Options} from './options';
|
||||
import type {RuleSetRule, RuleSetUseItem} from 'webpack';
|
||||
|
||||
async function enhancedOptions(options: Options): Promise<Options> {
|
||||
type CreateOptions = {
|
||||
useCrossCompilerCache?: boolean;
|
||||
};
|
||||
|
||||
async function normalizeOptions(
|
||||
optionsInput: Options & CreateOptions,
|
||||
): Promise<Options> {
|
||||
// Because Jest doesn't like ESM / createProcessors()
|
||||
if (process.env.N0DE_ENV === 'test' || process.env.JEST_WORKER_ID) {
|
||||
return options;
|
||||
return optionsInput;
|
||||
}
|
||||
|
||||
let options = optionsInput;
|
||||
|
||||
// We create the processor earlier here, to avoid the lazy processor creating
|
||||
// Lazy creation messes-up with Rsdoctor ability to measure mdx-loader perf
|
||||
const newOptions: Options = options.processors
|
||||
? options
|
||||
: {...options, processors: await createProcessors({options})};
|
||||
if (!options.processors) {
|
||||
options = {...options, processors: await createProcessors({options})};
|
||||
}
|
||||
|
||||
return newOptions;
|
||||
// Cross-compiler cache permits to compile client/server MDX only once
|
||||
// We don't want to cache in dev mode (docusaurus start)
|
||||
// We only have multiple compilers in production mode (docusaurus build)
|
||||
// TODO wrong but good enough for now (example: "docusaurus build --dev")
|
||||
if (options.useCrossCompilerCache && process.env.NODE_ENV === 'production') {
|
||||
options = {
|
||||
...options,
|
||||
crossCompilerCache: new Map(),
|
||||
};
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
export async function createMDXLoaderItem(
|
||||
options: Options,
|
||||
options: Options & CreateOptions,
|
||||
): Promise<RuleSetUseItem> {
|
||||
return {
|
||||
loader: require.resolve('@docusaurus/mdx-loader'),
|
||||
options: await enhancedOptions(options),
|
||||
options: await normalizeOptions(options),
|
||||
};
|
||||
}
|
||||
|
||||
|
@ -38,7 +57,7 @@ export async function createMDXLoaderRule({
|
|||
options,
|
||||
}: {
|
||||
include: RuleSetRule['include'];
|
||||
options: Options;
|
||||
options: Options & CreateOptions;
|
||||
}): Promise<RuleSetRule> {
|
||||
return {
|
||||
test: /\.mdx?$/i,
|
||||
|
|
|
@ -37,5 +37,6 @@ export type LoadedMDXContent<FrontMatter, Metadata, Assets = undefined> = {
|
|||
(): JSX.Element;
|
||||
};
|
||||
|
||||
export type {Options, MDXPlugin} from './loader';
|
||||
export type {MDXPlugin} from './loader';
|
||||
export type {MDXOptions} from './processor';
|
||||
export type {Options} from './options';
|
||||
|
|
|
@ -18,14 +18,8 @@ import {
|
|||
createAssetsExportCode,
|
||||
extractContentTitleData,
|
||||
} from './utils';
|
||||
import type {
|
||||
SimpleProcessors,
|
||||
MDXOptions,
|
||||
SimpleProcessorResult,
|
||||
} from './processor';
|
||||
import type {ResolveMarkdownLink} from './remark/resolveMarkdownLinks';
|
||||
|
||||
import type {MarkdownConfig} from '@docusaurus/types';
|
||||
import type {WebpackCompilerName} from '@docusaurus/utils';
|
||||
import type {Options} from './options';
|
||||
import type {LoaderContext} from 'webpack';
|
||||
|
||||
// TODO as of April 2023, no way to import/re-export this ESM type easily :/
|
||||
|
@ -35,33 +29,17 @@ type Pluggable = any; // TODO fix this asap
|
|||
|
||||
export type MDXPlugin = Pluggable;
|
||||
|
||||
export type Options = Partial<MDXOptions> & {
|
||||
markdownConfig: MarkdownConfig;
|
||||
staticDirs: string[];
|
||||
siteDir: string;
|
||||
isMDXPartial?: (filePath: string) => boolean;
|
||||
isMDXPartialFrontMatterWarningDisabled?: boolean;
|
||||
removeContentTitle?: boolean;
|
||||
metadataPath?: (filePath: string) => string;
|
||||
createAssets?: (metadata: {
|
||||
filePath: string;
|
||||
frontMatter: {[key: string]: unknown};
|
||||
}) => {[key: string]: unknown};
|
||||
resolveMarkdownLink?: ResolveMarkdownLink;
|
||||
|
||||
// Will usually be created by "createMDXLoaderItem"
|
||||
processors?: SimpleProcessors;
|
||||
};
|
||||
|
||||
export async function mdxLoader(
|
||||
this: LoaderContext<Options>,
|
||||
fileContent: string,
|
||||
): Promise<void> {
|
||||
const compilerName = getWebpackLoaderCompilerName(this);
|
||||
const callback = this.async();
|
||||
const filePath = this.resourcePath;
|
||||
const options: Options = this.getOptions();
|
||||
|
||||
async function loadMDX({
|
||||
fileContent,
|
||||
filePath,
|
||||
options,
|
||||
compilerName,
|
||||
}: {
|
||||
fileContent: string;
|
||||
filePath: string;
|
||||
options: Options;
|
||||
compilerName: WebpackCompilerName;
|
||||
}): Promise<string> {
|
||||
const {frontMatter} = await options.markdownConfig.parseFrontMatter({
|
||||
filePath,
|
||||
fileContent,
|
||||
|
@ -70,18 +48,13 @@ export async function mdxLoader(
|
|||
|
||||
const hasFrontMatter = Object.keys(frontMatter).length > 0;
|
||||
|
||||
let result: SimpleProcessorResult;
|
||||
try {
|
||||
result = await compileToJSX({
|
||||
fileContent,
|
||||
filePath,
|
||||
frontMatter,
|
||||
options,
|
||||
compilerName,
|
||||
});
|
||||
} catch (error) {
|
||||
return callback(error as Error);
|
||||
}
|
||||
const result = await compileToJSX({
|
||||
fileContent,
|
||||
filePath,
|
||||
frontMatter,
|
||||
options,
|
||||
compilerName,
|
||||
});
|
||||
|
||||
const contentTitle = extractContentTitleData(result.data);
|
||||
|
||||
|
@ -97,7 +70,7 @@ ${JSON.stringify(frontMatter, null, 2)}`;
|
|||
if (!options.isMDXPartialFrontMatterWarningDisabled) {
|
||||
const shouldError = process.env.NODE_ENV === 'test' || process.env.CI;
|
||||
if (shouldError) {
|
||||
return callback(new Error(errorMessage));
|
||||
throw new Error(errorMessage);
|
||||
}
|
||||
logger.warn(errorMessage);
|
||||
}
|
||||
|
@ -146,5 +119,68 @@ ${exportsCode}
|
|||
${result.content}
|
||||
`;
|
||||
|
||||
return callback(null, code);
|
||||
return code;
|
||||
}
|
||||
|
||||
// Note: we cache promises instead of strings
|
||||
// This is because client/server compilations might be triggered in parallel
|
||||
// When this happens for the same file, we don't want to compile it twice
|
||||
async function loadMDXWithCaching({
|
||||
resource,
|
||||
fileContent,
|
||||
filePath,
|
||||
options,
|
||||
compilerName,
|
||||
}: {
|
||||
resource: string; // path?query#hash
|
||||
filePath: string; // path
|
||||
fileContent: string;
|
||||
options: Options;
|
||||
compilerName: WebpackCompilerName;
|
||||
}): Promise<string> {
|
||||
// Note we "resource" as cache key, not "filePath" nor "fileContent"
|
||||
// This is because:
|
||||
// - the same file can be compiled in different variants (blog.mdx?truncated)
|
||||
// - the same content can be processed differently (versioned docs links)
|
||||
const cacheKey = resource;
|
||||
|
||||
const cachedPromise = options.crossCompilerCache?.get(cacheKey);
|
||||
if (cachedPromise) {
|
||||
// We can clean up the cache and free memory here
|
||||
// We know there are only 2 compilations for the same file
|
||||
// Note: once we introduce RSCs we'll probably have 3 compilations
|
||||
// Note: we can't use string keys in WeakMap
|
||||
// But we could eventually use WeakRef for the values
|
||||
options.crossCompilerCache?.delete(cacheKey);
|
||||
return cachedPromise;
|
||||
}
|
||||
const promise = loadMDX({
|
||||
fileContent,
|
||||
filePath,
|
||||
options,
|
||||
compilerName,
|
||||
});
|
||||
options.crossCompilerCache?.set(cacheKey, promise);
|
||||
return promise;
|
||||
}
|
||||
|
||||
export async function mdxLoader(
|
||||
this: LoaderContext<Options>,
|
||||
fileContent: string,
|
||||
): Promise<void> {
|
||||
const compilerName = getWebpackLoaderCompilerName(this);
|
||||
const callback = this.async();
|
||||
const options: Options = this.getOptions();
|
||||
try {
|
||||
const result = await loadMDXWithCaching({
|
||||
resource: this.resource,
|
||||
filePath: this.resourcePath,
|
||||
fileContent,
|
||||
options,
|
||||
compilerName,
|
||||
});
|
||||
return callback(null, result);
|
||||
} catch (error) {
|
||||
return callback(error as Error);
|
||||
}
|
||||
}
|
||||
|
|
29
packages/docusaurus-mdx-loader/src/options.ts
Normal file
29
packages/docusaurus-mdx-loader/src/options.ts
Normal file
|
@ -0,0 +1,29 @@
|
|||
/**
|
||||
* Copyright (c) Facebook, Inc. and its affiliates.
|
||||
*
|
||||
* This source code is licensed under the MIT license found in the
|
||||
* LICENSE file in the root directory of this source tree.
|
||||
*/
|
||||
|
||||
import type {MDXOptions, SimpleProcessors} from './processor';
|
||||
import type {MarkdownConfig} from '@docusaurus/types';
|
||||
import type {ResolveMarkdownLink} from './remark/resolveMarkdownLinks';
|
||||
|
||||
export type Options = Partial<MDXOptions> & {
|
||||
markdownConfig: MarkdownConfig;
|
||||
staticDirs: string[];
|
||||
siteDir: string;
|
||||
isMDXPartial?: (filePath: string) => boolean;
|
||||
isMDXPartialFrontMatterWarningDisabled?: boolean;
|
||||
removeContentTitle?: boolean;
|
||||
metadataPath?: (filePath: string) => string;
|
||||
createAssets?: (metadata: {
|
||||
filePath: string;
|
||||
frontMatter: {[key: string]: unknown};
|
||||
}) => {[key: string]: unknown};
|
||||
resolveMarkdownLink?: ResolveMarkdownLink;
|
||||
|
||||
// Will usually be created by "createMDXLoaderItem"
|
||||
processors?: SimpleProcessors;
|
||||
crossCompilerCache?: Map<string, Promise<string>>; // MDX => Promise<JSX> cache
|
||||
};
|
|
@ -11,7 +11,7 @@ import {
|
|||
admonitionTitleToDirectiveLabel,
|
||||
} from '@docusaurus/utils';
|
||||
import {normalizeAdmonitionOptions} from './remark/admonitions';
|
||||
import type {Options} from './loader';
|
||||
import type {Options} from './options';
|
||||
|
||||
/**
|
||||
* Preprocess the string before passing it to MDX
|
||||
|
|
|
@ -20,7 +20,7 @@ import codeCompatPlugin from './remark/mdx1Compat/codeCompatPlugin';
|
|||
import {getFormat} from './format';
|
||||
import type {WebpackCompilerName} from '@docusaurus/utils';
|
||||
import type {MDXFrontMatter} from './frontMatter';
|
||||
import type {Options} from './loader';
|
||||
import type {Options} from './options';
|
||||
import type {AdmonitionOptions} from './remark/admonitions';
|
||||
|
||||
// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
|
||||
|
|
|
@ -10,7 +10,7 @@ import {escapePath, type WebpackCompilerName} from '@docusaurus/utils';
|
|||
import {getProcessor, type SimpleProcessorResult} from './processor';
|
||||
import {validateMDXFrontMatter} from './frontMatter';
|
||||
import preprocessor from './preprocessor';
|
||||
import type {Options} from './loader';
|
||||
import type {Options} from './options';
|
||||
|
||||
/**
|
||||
* Converts assets an object with Webpack require calls code.
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue