feat(core): allow sourcing from multiple static directories (#4095)

* [WIP] Implementaion of multiple directory static sourcing

* Move default to validation

* Update test

* Refactor

* Port to MDX loader

* Fix

* Move dogfooding assets

* Doc writeup

* Restore assets

* Support absolute paths

* Dogfood absolute path

* Fix

* More tests

* Fix snapshots

Co-authored-by: Joshua Chen <sidachen2003@gmail.com>

packages/docusaurus-mdx-loader/src/index.ts

@@ -36,7 +36,8 @@ const DEFAULT_OPTIONS: RemarkAndRehypePluginOptions = {
 };
 
 type Options = RemarkAndRehypePluginOptions & {
-  staticDir?: string;
+  staticDirs: string[];
+  siteDir: string;
   isMDXPartial?: (filePath: string) => boolean;
   isMDXPartialFrontMatterWarningDisabled?: boolean;
   removeContentTitle?: boolean;
@@ -123,8 +124,15 @@ export default async function mdxLoader(
     remarkPlugins: [
       ...(reqOptions.beforeDefaultRemarkPlugins || []),
       ...DEFAULT_OPTIONS.remarkPlugins,
-      [transformImage, {staticDir: reqOptions.staticDir, filePath}],
-      [transformLinks, {staticDir: reqOptions.staticDir, filePath}],
+      [transformImage, {staticDirs: reqOptions.staticDirs, filePath}],
+      [
+        transformLinks,
+        {
+          staticDirs: reqOptions.staticDirs,
+          filePath,
+          siteDir: reqOptions.siteDir,
+        },
+      ],
       ...(reqOptions.remarkPlugins || []),
     ],
     rehypePlugins: [

packages/docusaurus-mdx-loader/src/remark/transformImage/__tests__/__fixtures__/img.md

@@ -0,0 +1,19 @@
+![img](https://example.com/img.png)
+
+![](./static/img.png)
+
+![img](./static/img.png)
+
+![img from second static folder](/img2.png)
+
+![img from second static folder](./static2/img2.png)
+
+![img](./static/img.png 'Title') ![img](/img.png)
+
+![img with "quotes"](./static/img.png ''Quoted' title')
+
+## Heading
+
+```md
+![img](./static/img.png)
+```

packages/docusaurus-mdx-loader/src/remark/transformImage/__tests__/__snapshots__/index.test.ts.snap

@@ -1,8 +1,8 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`transformImage plugin fail if image does not exist 1`] = `"Image packages/docusaurus-mdx-loader/src/remark/transformImage/__tests__/fixtures/img/doesNotExist.png used in packages/docusaurus-mdx-loader/src/remark/transformImage/__tests__/fixtures/fail.md not found."`;
+exports[`transformImage plugin fail if image does not exist 1`] = `"Image packages/docusaurus-mdx-loader/src/remark/transformImage/__tests__/__fixtures__/static/img/doesNotExist.png or packages/docusaurus-mdx-loader/src/remark/transformImage/__tests__/__fixtures__/static2/img/doesNotExist.png used in packages/docusaurus-mdx-loader/src/remark/transformImage/__tests__/__fixtures__/fail.md not found."`;
 
-exports[`transformImage plugin fail if image url is absent 1`] = `"Markdown image URL is mandatory in \\"packages/docusaurus-mdx-loader/src/remark/transformImage/__tests__/fixtures/noUrl.md\\" file"`;
+exports[`transformImage plugin fail if image url is absent 1`] = `"Markdown image URL is mandatory in \\"packages/docusaurus-mdx-loader/src/remark/transformImage/__tests__/__fixtures__/noUrl.md\\" file"`;
 
 exports[`transformImage plugin pathname protocol 1`] = `
 "![img](/img/unchecked.png)
@@ -12,20 +12,22 @@ exports[`transformImage plugin pathname protocol 1`] = `
 exports[`transformImage plugin transform md images to <img /> 1`] = `
 "![img](https://example.com/img.png)
 
-<img src={require(\\"!url-loader?limit=10000&name=assets/images/[name]-[hash].[ext]&fallback=file-loader!./img.png\\").default} />
+<img src={require(\\"!url-loader?limit=10000&name=assets/images/[name]-[hash].[ext]&fallback=file-loader!./static/img.png\\").default} />
 
-<img alt={\\"img\\"} src={require(\\"!url-loader?limit=10000&name=assets/images/[name]-[hash].[ext]&fallback=file-loader!./img.png\\").default} />
+<img alt={\\"img\\"} src={require(\\"!url-loader?limit=10000&name=assets/images/[name]-[hash].[ext]&fallback=file-loader!./static/img.png\\").default} />
 
-<img alt={\\"img\\"} src={require(\\"!url-loader?limit=10000&name=assets/images/[name]-[hash].[ext]&fallback=file-loader!./img.png\\").default} title=\\"Title\\" /> <img alt={\\"img\\"} src={require(\\"!url-loader?limit=10000&name=assets/images/[name]-[hash].[ext]&fallback=file-loader!packages/docusaurus-mdx-loader/src/remark/transformImage/__tests__/fixtures/img.png\\").default} />
+<img alt={\\"img from second static folder\\"} src={require(\\"!url-loader?limit=10000&name=assets/images/[name]-[hash].[ext]&fallback=file-loader!packages/docusaurus-mdx-loader/src/remark/transformImage/__tests__/__fixtures__/static2/img2.png\\").default} />
 
-<img alt={\\"img with &quot;quotes&quot;\\"} src={require(\\"!url-loader?limit=10000&name=assets/images/[name]-[hash].[ext]&fallback=file-loader!./img.png\\").default} title=\\"&#39;Quoted&#39; title\\" />
+<img alt={\\"img from second static folder\\"} src={require(\\"!url-loader?limit=10000&name=assets/images/[name]-[hash].[ext]&fallback=file-loader!./static2/img2.png\\").default} />
+
+<img alt={\\"img\\"} src={require(\\"!url-loader?limit=10000&name=assets/images/[name]-[hash].[ext]&fallback=file-loader!./static/img.png\\").default} title=\\"Title\\" /> <img alt={\\"img\\"} src={require(\\"!url-loader?limit=10000&name=assets/images/[name]-[hash].[ext]&fallback=file-loader!packages/docusaurus-mdx-loader/src/remark/transformImage/__tests__/__fixtures__/static/img.png\\").default} />
+
+<img alt={\\"img with &quot;quotes&quot;\\"} src={require(\\"!url-loader?limit=10000&name=assets/images/[name]-[hash].[ext]&fallback=file-loader!./static/img.png\\").default} title=\\"&#39;Quoted&#39; title\\" />
 
 ## Heading
 
 \`\`\`md
-![img](./img.png)
+![img](./static/img.png)
 \`\`\`
-
-<img alt={\\"img\\"} src={require(\\"!url-loader?limit=10000&name=assets/images/[name]-[hash].[ext]&fallback=file-loader!./img.png\\").default} />
 "
 `;

packages/docusaurus-mdx-loader/src/remark/transformImage/__tests__/fixtures/img.md

@@ -1,17 +0,0 @@
-![img](https://example.com/img.png)
-
-![](./img.png)
-
-![img](./img.png)
-
-![img](./img.png 'Title') ![img](/img.png)
-
-![img with "quotes"](./img.png ''Quoted' title')
-
-## Heading
-
-```md
-![img](./img.png)
-```
-
-![img](img.png)

packages/docusaurus-mdx-loader/src/remark/transformImage/__tests__/index.test.ts

@@ -5,7 +5,7 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import {join, relative} from 'path';
+import path from 'path';
 import remark from 'remark';
 import mdx from 'remark-mdx';
 import vfile from 'to-vfile';
@@ -13,47 +13,48 @@ import plugin from '../index';
 import headings from '../../headings/index';
 
 const processFixture = async (name, options) => {
-  const path = join(__dirname, 'fixtures', `${name}.md`);
-  const file = await vfile.read(path);
+  const filePath = path.join(__dirname, `__fixtures__/${name}.md`);
+  const file = await vfile.read(filePath);
   const result = await remark()
     .use(headings)
     .use(mdx)
-    .use(plugin, {...options, filePath: path})
+    .use(plugin, {...options, filePath})
     .process(file);
 
   return result.toString();
 };
 
-// avoid hardcoding absolute
-const staticDir = `./${relative(process.cwd(), join(__dirname, 'fixtures'))}`;
+const staticDirs = [
+  // avoid hardcoding absolute in the snapshot
+  `./${path.relative(
+    process.cwd(),
+    path.join(__dirname, '__fixtures__/static'),
+  )}`,
+  `./${path.relative(
+    process.cwd(),
+    path.join(__dirname, '__fixtures__/static2'),
+  )}`,
+];
 
 describe('transformImage plugin', () => {
   test('fail if image does not exist', async () => {
     await expect(
-      processFixture('fail', {
-        staticDir,
-      }),
+      processFixture('fail', {staticDirs}),
     ).rejects.toThrowErrorMatchingSnapshot();
   });
   test('fail if image url is absent', async () => {
     await expect(
-      processFixture('noUrl', {
-        staticDir,
-      }),
+      processFixture('noUrl', {staticDirs}),
     ).rejects.toThrowErrorMatchingSnapshot();
   });
 
   test('transform md images to <img />', async () => {
-    const result = await processFixture('img', {
-      staticDir,
-    });
+    const result = await processFixture('img', {staticDirs});
     expect(result).toMatchSnapshot();
   });
 
   test('pathname protocol', async () => {
-    const result = await processFixture('pathname', {
-      staticDir,
-    });
+    const result = await processFixture('pathname', {staticDirs});
     expect(result).toMatchSnapshot();
   });
 });

packages/docusaurus-mdx-loader/src/remark/transformImage/index.ts

@@ -25,7 +25,7 @@ const {
 
 interface PluginOptions {
   filePath: string;
-  staticDir: string;
+  staticDirs: string[];
 }
 
 const createJSX = (node: Image, pathUrl: string) => {
@@ -63,9 +63,25 @@ async function ensureImageFileExist(imagePath: string, sourceFilePath: string) {
   }
 }
 
+async function findImage(possiblePaths: string[], sourceFilePath: string) {
+  // eslint-disable-next-line no-restricted-syntax
+  for (const possiblePath of possiblePaths) {
+    if (await fs.pathExists(possiblePath)) {
+      return possiblePath;
+    }
+  }
+  throw new Error(
+    `Image ${possiblePaths
+      .map((p) => toMessageRelativeFilePath(p))
+      .join(' or ')} used in ${toMessageRelativeFilePath(
+      sourceFilePath,
+    )} not found.`,
+  );
+}
+
 async function processImageNode(
   node: Image,
-  {filePath, staticDir}: PluginOptions,
+  {filePath, staticDirs}: PluginOptions,
 ) {
   if (!node.url) {
     throw new Error(
@@ -88,9 +104,11 @@ async function processImageNode(
   // images without protocol
   else if (path.isAbsolute(node.url)) {
     // absolute paths are expected to exist in the static folder
-    const expectedImagePath = path.join(staticDir, node.url);
-    await ensureImageFileExist(expectedImagePath, filePath);
-    createJSX(node, posixPath(expectedImagePath));
+    const possibleImagePaths = staticDirs.map((dir) =>
+      path.join(dir, node.url),
+    );
+    const imagePath = await findImage(possibleImagePaths, filePath);
+    createJSX(node, posixPath(imagePath));
   }
   // We try to convert image urls without protocol to images with require calls
   // going through webpack ensures that image assets exist at build time

packages/docusaurus-mdx-loader/src/remark/transformLinks/__tests__/__fixtures__/asset.md

@@ -20,6 +20,8 @@
 
 [asset](asset.pdf)
 
+[asset2](/asset2.pdf)
+
 [staticAsset.pdf](/staticAsset.pdf)
 
 [@site/static/staticAsset.pdf](@site/static/staticAsset.pdf)

packages/docusaurus-mdx-loader/src/remark/transformLinks/__tests__/__snapshots__/index.test.ts.snap

@@ -1,6 +1,6 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`transformAsset plugin fail if asset url is absent 1`] = `"Markdown link URL is mandatory in \\"packages/docusaurus-mdx-loader/src/remark/transformLinks/__tests__/fixtures/noUrl.md\\" file (title: asset, line: 1)."`;
+exports[`transformAsset plugin fail if asset url is absent 1`] = `"Markdown link URL is mandatory in \\"packages/docusaurus-mdx-loader/src/remark/transformLinks/__tests__/__fixtures__/noUrl.md\\" file (title: asset, line: 1)."`;
 
 exports[`transformAsset plugin pathname protocol 1`] = `
 "[asset](pathname:///asset/unchecked.pdf)
@@ -30,6 +30,8 @@ exports[`transformAsset plugin transform md links to <a /> 1`] = `
 
 <a target=\\"_blank\\" href={require('!file-loader?name=assets/files/[name]-[hash].[ext]!./asset.pdf').default}>asset</a>
 
+<a target=\\"_blank\\" href={require('!file-loader?name=assets/files/[name]-[hash].[ext]!./static2/asset2.pdf').default}>asset2</a>
+
 <a target=\\"_blank\\" href={require('!file-loader?name=assets/files/[name]-[hash].[ext]!./static/staticAsset.pdf').default}>staticAsset.pdf</a>
 
 <a target=\\"_blank\\" href={require('!file-loader?name=assets/files/[name]-[hash].[ext]!./static/staticAsset.pdf').default}>@site/static/staticAsset.pdf</a>

packages/docusaurus-mdx-loader/src/remark/transformLinks/__tests__/index.test.ts

@@ -5,21 +5,29 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import {join} from 'path';
+import path from 'path';
 import remark from 'remark';
 import mdx from 'remark-mdx';
 import vfile from 'to-vfile';
 import plugin from '..';
 import transformImage from '../../transformImage';
 
-const processFixture = async (name, options) => {
-  const path = join(__dirname, 'fixtures', `${name}.md`);
-  const staticDir = join(__dirname, 'fixtures', 'static');
-  const file = await vfile.read(path);
+const processFixture = async (name: string, options?) => {
+  const filePath = path.join(__dirname, `__fixtures__/${name}.md`);
+  const staticDirs = [
+    path.join(__dirname, '__fixtures__/static'),
+    path.join(__dirname, '__fixtures__/static2'),
+  ];
+  const file = await vfile.read(filePath);
   const result = await remark()
     .use(mdx)
-    .use(transformImage, {...options, filePath: path, staticDir})
-    .use(plugin, {...options, filePath: path, staticDir})
+    .use(transformImage, {...options, filePath, staticDirs})
+    .use(plugin, {
+      ...options,
+      filePath,
+      staticDirs,
+      siteDir: path.join(__dirname, '__fixtures__'),
+    })
     .process(file);
 
   return result.toString();

packages/docusaurus-mdx-loader/src/remark/transformLinks/index.ts

@@ -27,7 +27,8 @@ const hashRegex = /#.*$/;
 
 interface PluginOptions {
   filePath: string;
-  staticDir: string;
+  staticDirs: string[];
+  siteDir: string;
 }
 
 async function ensureAssetFileExist(
@@ -81,11 +82,10 @@ function toAssetRequireNode({
 // If the link looks like an asset link, we'll link to the asset,
 // and use a require("assetUrl") (using webpack url-loader/file-loader)
 // instead of navigating to such link
-async function convertToAssetLinkIfNeeded({
-  node,
-  staticDir,
-  filePath,
-}: {node: Link} & PluginOptions) {
+async function convertToAssetLinkIfNeeded(
+  node: Link,
+  {filePath, siteDir, staticDirs}: PluginOptions,
+) {
   const assetPath = node.url.replace(hashRegex, '');
 
   const hasSiteAlias = assetPath.startsWith('@site/');
@@ -107,7 +107,6 @@ async function convertToAssetLinkIfNeeded({
   }
 
   if (assetPath.startsWith('@site/')) {
-    const siteDir = path.join(staticDir, '..');
     const fileSystemAssetPath = path.join(
       siteDir,
       assetPath.replace('@site/', ''),
@@ -115,9 +114,13 @@ async function convertToAssetLinkIfNeeded({
     await ensureAssetFileExist(fileSystemAssetPath, filePath);
     toAssetLinkNode(fileSystemAssetPath);
   } else if (path.isAbsolute(assetPath)) {
-    const fileSystemAssetPath = path.join(staticDir, assetPath);
-    if (await fs.pathExists(fileSystemAssetPath)) {
-      toAssetLinkNode(fileSystemAssetPath);
+    // eslint-disable-next-line no-restricted-syntax
+    for (const staticDir of staticDirs) {
+      const fileSystemAssetPath = path.join(staticDir, assetPath);
+      if (await fs.pathExists(fileSystemAssetPath)) {
+        toAssetLinkNode(fileSystemAssetPath);
+        return;
+      }
     }
   } else {
     const fileSystemAssetPath = path.join(path.dirname(filePath), assetPath);
@@ -127,11 +130,7 @@ async function convertToAssetLinkIfNeeded({
   }
 }
 
-async function processLinkNode({
-  node,
-  filePath,
-  staticDir,
-}: {node: Link} & PluginOptions) {
+async function processLinkNode(node: Link, options: PluginOptions) {
   if (!node.url) {
     // try to improve error feedback
     // see https://github.com/facebook/docusaurus/issues/3309#issuecomment-690371675
@@ -139,7 +138,7 @@ async function processLinkNode({
     const line = node?.position?.start?.line || '?';
     throw new Error(
       `Markdown link URL is mandatory in "${toMessageRelativeFilePath(
-        filePath,
+        options.filePath,
       )}" file (title: ${title}, line: ${line}).`,
     );
   }
@@ -149,14 +148,14 @@ async function processLinkNode({
     return;
   }
 
-  await convertToAssetLinkIfNeeded({node, staticDir, filePath});
+  await convertToAssetLinkIfNeeded(node, options);
 }
 
 const plugin: Plugin<[PluginOptions]> = (options) => {
   const transformer: Transformer = async (root) => {
     const promises: Promise<void>[] = [];
     visit(root, 'link', (node: Link) => {
-      promises.push(processLinkNode({node, ...options}));
+      promises.push(processLinkNode(node, options));
     });
     await Promise.all(promises);
   };