feat(v2): configureWebpack merge strategy + use file-loader for common asset types (#2994)

* Add some default asset loaders
Add webpack merge strategy feature to enable plugins to prepend some webpack configuration (like the ideal image plugin that should override the default image loader)

* Add documentation for using assets from markdown

* add path prefix for webpack file loader

* renaming

* document Merge strategies

* rename mergeStrategies -> mergeStrategy

packages/docusaurus-plugin-ideal-image/src/index.ts

@@ -25,6 +25,9 @@ export default function (
 
     configureWebpack(_config: Configuration, isServer: boolean) {
       return {
+        mergeStrategy: {
+          'module.rules': 'prepend',
+        },
         module: {
           rules: [
             {

packages/docusaurus-types/package.json

@@ -11,6 +11,7 @@
   "dependencies": {
     "@types/webpack": "^4.41.0",
     "commander": "^4.0.1",
-    "querystring": "0.2.0"
+    "querystring": "0.2.0",
+    "webpack-merge": "^4.2.2"
   }
 }

packages/docusaurus-types/src/index.d.ts

@@ -8,6 +8,7 @@
 import {Loader, Configuration} from 'webpack';
 import {Command} from 'commander';
 import {ParsedUrlQueryInput} from 'querystring';
+import {MergeStrategy} from 'webpack-merge';
 
 export interface DocusaurusConfig {
   baseUrl: string;
@@ -118,7 +119,7 @@ export interface Plugin<T, U = unknown> {
     config: Configuration,
     isServer: boolean,
     utils: ConfigureWebpackUtils,
-  ): Configuration;
+  ): Configuration & {mergeStrategy?: ConfigureWebpackFnMergeStrategy};
   getThemePath?(): string;
   getTypeScriptThemePath?(): string;
   getPathsToWatch?(): string[];
@@ -131,6 +132,9 @@ export interface Plugin<T, U = unknown> {
   };
 }
 
+export type ConfigureWebpackFn = Plugin<unknown>['configureWebpack'];
+export type ConfigureWebpackFnMergeStrategy = Record<string, MergeStrategy>;
+
 export type PluginConfig =
   | [string, Record<string, unknown>]
   | [string]

packages/docusaurus/package.json

@@ -61,6 +61,7 @@
     "detect-port": "^1.3.0",
     "eta": "^1.1.1",
     "express": "^4.17.1",
+    "file-loader": "^6.0.0",
     "fs-extra": "^8.1.0",
     "globby": "^10.0.1",
     "html-minifier-terser": "^5.0.5",
@@ -90,6 +91,7 @@
     "shelljs": "^0.8.4",
     "std-env": "^2.2.1",
     "terser-webpack-plugin": "^2.3.5",
+    "url-loader": "^4.1.0",
     "wait-file": "^1.0.5",
     "webpack": "^4.41.2",
     "webpack-bundle-analyzer": "^3.6.1",

packages/docusaurus/src/webpack/__tests__/utils.test.ts

@@ -5,23 +5,33 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import {validate} from 'webpack';
+import {
+  // @ts-expect-error: seems it's not in the typedefs???
+  validate,
+  Configuration,
+} from 'webpack';
 import path from 'path';
 
 import {applyConfigureWebpack} from '../utils';
+import {
+  ConfigureWebpackFn,
+  ConfigureWebpackFnMergeStrategy,
+} from '@docusaurus/types';
 
 describe('extending generated webpack config', () => {
   test('direct mutation on generated webpack config object', async () => {
     // fake generated webpack config
-    let config = {
+    let config: Configuration = {
       output: {
         path: __dirname,
         filename: 'bundle.js',
       },
     };
 
-    /* eslint-disable */
+    const configureWebpack: ConfigureWebpackFn = (
-    const configureWebpack = (generatedConfig, isServer) => {
+      generatedConfig,
+      isServer,
+    ) => {
       if (!isServer) {
         generatedConfig.entry = 'entry.js';
         generatedConfig.output = {
@@ -29,8 +39,8 @@ describe('extending generated webpack config', () => {
           filename: 'new.bundle.js',
         };
       }
+      return {};
     };
-    /* eslint-enable */
 
     config = applyConfigureWebpack(configureWebpack, config, false);
     expect(config).toEqual({
@@ -45,23 +55,20 @@ describe('extending generated webpack config', () => {
   });
 
   test('webpack-merge with user webpack config object', async () => {
-    // fake generated webpack config
+    let config: Configuration = {
-    let config = {
       output: {
         path: __dirname,
         filename: 'bundle.js',
       },
     };
 
-    /* eslint-disable */
+    const configureWebpack: ConfigureWebpackFn = () => ({
-    const configureWebpack = {
       entry: 'entry.js',
       output: {
         path: path.join(__dirname, 'dist'),
         filename: 'new.bundle.js',
       },
-    };
+    });
-    /* eslint-enable */
 
     config = applyConfigureWebpack(configureWebpack, config, false);
     expect(config).toEqual({
@@ -74,4 +81,54 @@ describe('extending generated webpack config', () => {
     const errors = validate(config);
     expect(errors.length).toBe(0);
   });
+
+  test('webpack-merge with custom strategy', async () => {
+    const config: Configuration = {
+      module: {
+        rules: [{use: 'xxx'}, {use: 'yyy'}],
+      },
+    };
+
+    const createConfigureWebpack: (
+      mergeStrategy?: ConfigureWebpackFnMergeStrategy,
+    ) => ConfigureWebpackFn = (mergeStrategy) => () => ({
+      module: {
+        rules: [{use: 'zzz'}],
+      },
+      mergeStrategy,
+    });
+
+    const defaultStrategyMergeConfig = applyConfigureWebpack(
+      createConfigureWebpack(),
+      config,
+      false,
+    );
+    expect(defaultStrategyMergeConfig).toEqual({
+      module: {
+        rules: [{use: 'xxx'}, {use: 'yyy'}, {use: 'zzz'}],
+      },
+    });
+
+    const prependRulesStrategyConfig = applyConfigureWebpack(
+      createConfigureWebpack({'module.rules': 'prepend'}),
+      config,
+      false,
+    );
+    expect(prependRulesStrategyConfig).toEqual({
+      module: {
+        rules: [{use: 'zzz'}, {use: 'xxx'}, {use: 'yyy'}],
+      },
+    });
+
+    const uselessMergeStrategyConfig = applyConfigureWebpack(
+      createConfigureWebpack({uselessAttributeName: 'append'}),
+      config,
+      false,
+    );
+    expect(uselessMergeStrategyConfig).toEqual({
+      module: {
+        rules: [{use: 'xxx'}, {use: 'yyy'}, {use: 'zzz'}],
+      },
+    });
+  });
 });

packages/docusaurus/src/webpack/base.ts

@@ -14,7 +14,12 @@ import TerserPlugin from 'terser-webpack-plugin';
 import {Configuration, Loader} from 'webpack';
 
 import {Props} from '@docusaurus/types';
-import {getBabelLoader, getCacheLoader, getStyleLoaders} from './utils';
+import {
+  getBabelLoader,
+  getCacheLoader,
+  getStyleLoaders,
+  getFileLoaderUtils,
+} from './utils';
 import {BABEL_CONFIG_FILE_NAME} from '../constants';
 
 const CSS_REGEX = /\.css$/;
@@ -48,6 +53,8 @@ export function createBaseConfig(
     BABEL_CONFIG_FILE_NAME,
   );
 
+  const fileLoaderUtils = getFileLoaderUtils();
+
   return {
     mode: isProd ? 'production' : 'development',
     output: {
@@ -158,6 +165,9 @@ export function createBaseConfig(
     },
     module: {
       rules: [
+        fileLoaderUtils.rules.images(),
+        fileLoaderUtils.rules.media(),
+        fileLoaderUtils.rules.otherAssets(),
         {
           test: /\.(j|t)sx?$/,
           exclude: excludeJS,

packages/docusaurus/src/webpack/utils.ts

@@ -8,10 +8,9 @@
 import MiniCssExtractPlugin from 'mini-css-extract-plugin';
 import env from 'std-env';
 import merge from 'webpack-merge';
-import {Configuration, Loader} from 'webpack';
+import {Configuration, Loader, RuleSetRule} from 'webpack';
 import {TransformOptions} from '@babel/core';
-import {ConfigureWebpackUtils} from '@docusaurus/types';
+import {ConfigureWebpackFn} from '@docusaurus/types';
-
 import {version as cacheLoaderVersion} from 'cache-loader/package.json';
 
 // Utility method to get style loaders
@@ -120,20 +119,10 @@ export function getBabelLoader(
  * @returns final/ modified webpack config
  */
 export function applyConfigureWebpack(
-  configureWebpack:
+  configureWebpack: ConfigureWebpackFn,
-    | Configuration
-    | ((
-        config: Configuration,
-        isServer: boolean,
-        utils: ConfigureWebpackUtils,
-      ) => Configuration),
   config: Configuration,
   isServer: boolean,
 ): Configuration {
-  if (typeof configureWebpack === 'object') {
-    return merge(config, configureWebpack);
-  }
-
   // Export some utility functions
   const utils = {
     getStyleLoaders,
@@ -141,10 +130,71 @@ export function applyConfigureWebpack(
     getBabelLoader,
   };
   if (typeof configureWebpack === 'function') {
-    const res = configureWebpack(config, isServer, utils);
+    const {mergeStrategy, ...res} = configureWebpack(config, isServer, utils);
     if (res && typeof res === 'object') {
-      return merge(config, res);
+      return merge.strategy(mergeStrategy ?? {})(config, res);
     }
   }
   return config;
 }
+
+// Inspired by https://github.com/gatsbyjs/gatsby/blob/8e6e021014da310b9cc7d02e58c9b3efe938c665/packages/gatsby/src/utils/webpack-utils.ts#L447
+export function getFileLoaderUtils() {
+  const assetsRelativeRoot = 'assets/';
+
+  const loaders = {
+    file: (options = {}) => {
+      return {
+        loader: require.resolve(`file-loader`),
+        options: {
+          name: `${assetsRelativeRoot}[name]-[hash].[ext]`,
+          ...options,
+        },
+      };
+    },
+    url: (options = {}) => {
+      return {
+        loader: require.resolve(`url-loader`),
+        options: {
+          limit: 10000,
+          name: `${assetsRelativeRoot}[name]-[hash].[ext]`,
+          fallback: require.resolve(`file-loader`),
+          ...options,
+        },
+      };
+    },
+  };
+
+  const rules = {
+    /**
+     * Loads image assets, inlines images via a data URI if they are below
+     * the size threshold
+     */
+    images: (): RuleSetRule => {
+      return {
+        use: [loaders.url()],
+        test: /\.(ico|svg|jpg|jpeg|png|gif|webp)(\?.*)?$/,
+      };
+    },
+
+    /**
+     * Loads audio and video and inlines them via a data URI if they are below
+     * the size threshold
+     */
+    media: (): RuleSetRule => {
+      return {
+        use: [loaders.url()],
+        test: /\.(mp4|webm|ogv|wav|mp3|m4a|aac|oga|flac)$/,
+      };
+    },
+
+    otherAssets: (): RuleSetRule => {
+      return {
+        use: [loaders.file()],
+        test: /\.(pdf|doc|docx|xls|xlsx|zip|rar)$/,
+      };
+    },
+  };
+
+  return {loaders, rules};
+}

website/docs/lifecycle-apis.md

@@ -275,6 +275,28 @@ module.exports = function (context, options) {
 };
 ```
 
+### Merge strategy
+
+We merge the Webpack configuration parts of plugins into the global Webpack config using [webpack-merge](https://github.com/survivejs/webpack-merge).
+
+It is possible to specify the merge strategy. For example, if you want a webpack rule to be prepended instead of appended:
+
+```js {4-11} title="docusaurus-plugin/src/index.js"
+module.exports = function (context, options) {
+  return {
+    name: 'custom-docusaurus-plugin',
+    configureWebpack(config, isServer, utils) {
+      return {
+        mergeStrategy: {'module.rules': 'prepend'},
+        module: {rules: [myRuleToPrepend]},
+      };
+    },
+  };
+};
+```
+
+Read the [webpack-merge strategy doc](https://github.com/survivejs/webpack-merge#merging-with-strategies) for more details.
+
 ## `postBuild(props)`
 
 Called when a (production) build finishes.

website/docs/markdown-features.mdx

@@ -578,16 +578,16 @@ It will produce `prism-include-languages.js` in your `src/theme` folder. You can
 
 ```js {8} title="src/theme/prism-include-languages.js"
 const prismIncludeLanguages = (Prism) => {
-    // ...
+  // ...
 
-    additionalLanguages.forEach((lang) => {
+  additionalLanguages.forEach((lang) => {
-      require(`prismjs/components/prism-${lang}`); // eslint-disable-line
+    require(`prismjs/components/prism-${lang}`); // eslint-disable-line
-    });
+  });
 
-    require('/path/to/your/prism-language-definition');
+  require('/path/to/your/prism-language-definition');
 
-    // ...
+  // ...
-}
+};
 ```
 
 You can refer to [Prism's official language definitions](https://github.com/PrismJS/prism/tree/master/components) when you are writing your own language definitions.
@@ -932,3 +932,96 @@ class HelloWorld {
 You may want to implement your own `<MultiLanguageCode />` abstraction if you find the above approach too verbose. We might just implement one in future for convenience.
 
 If you have multiple of these multi-language code tabs, and you want to sync the selection across the tab instances, refer to the [Syncing tab choices section](#syncing-tab-choices).
+
+## Assets
+
+Sometimes you want to link to static assets directly from markdown files, and it is convenient to co-locate the asset next to the markdown file using it.
+
+We have setup Webpack loaders to handle most common file types, so that when you import a file, you get its url, and the asset is automatically copied to the output folder.
+
+Let's imagine the following file structure:
+
+```
+# Your doc
+/website/docs/myFeature.mdx
+
+# Some assets you want to use
+/website/docs/assets/docusaurus-asset-example-banner.png
+/website/docs/assets/docusaurus-asset-example-pdf.pdf
+/website/docs/assets/docusaurus-asset-example.xyz
+```
+
+### Image assets:
+
+You can use images by requiring them and using an image tag through MDX:
+
+```mdx
+# My markdown page
+
+<img src={require('./assets/docusaurus-asset-example-banner.png').default} />
+```
+
+The ES imports syntax also works:
+
+```mdx
+# My markdown page
+
+import myImageUrl from './assets/docusaurus-asset-example-banner.png';
+
+<img src={myImageUrl)}/>
+```
+
+This results in displaying the image:
+
+<img
+  src={
+    require('!file-loader!./assets/docusaurus-asset-example-banner.png').default
+  }
+  style={{maxWidth: 300}}
+/>
+
+:::note
+
+If you are using [@docusaurus/plugin-ideal-image](./using-plugins.md#docusaurusplugin-ideal-image), you need to use the dedicated image component, as documented.
+
+:::
+
+### Common assets
+
+In the same way, you can link to existing assets by requiring them and using the returned url in videos, links etc...
+
+```mdx
+# My markdown page
+
+<a
+  target="_blank"
+  href={require('./assets/docusaurus-asset-example-pdf.pdf').default}>
+  Download this PDF !!!
+</a>
+```
+
+<a
+  target="_blank"
+  href={require('./assets/docusaurus-asset-example-pdf.pdf').default}>
+  Download this PDF !!!
+</a>
+
+### Unknown assets
+
+This require behavior is not supported for all file extensions, but as an escape hatch you can use the special Webpack syntax to force the `file-loader` to kick-in:
+
+```mdx
+# My markdown page
+
+<a
+  target="_blank"
+  href={require('!file-loader!./assets/docusaurus-asset-example.xyz').default}>
+  Download this unknown file !!!
+</a>
+```
+
+<a
+  target="_blank"
+  href={require('!file-loader!./assets/docusaurus-asset-example.xyz').default}>
+  Download this unknown file !!!
+</a>

yarn.lock

@@ -7999,6 +7999,14 @@ file-entry-cache@^5.0.1:
   dependencies:
     flat-cache "^2.0.1"
 
+file-loader@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/file-loader/-/file-loader-6.0.0.tgz#97bbfaab7a2460c07bcbd72d3a6922407f67649f"
+  integrity sha512-/aMOAYEFXDdjG0wytpTL5YQLfZnnTmLNjn+AIrJ/6HVnTfDqLsVKUUwkDf4I4kgex36BvjuXEn/TX9B/1ESyqQ==
+  dependencies:
+    loader-utils "^2.0.0"
+    schema-utils "^2.6.5"
+
 file-type@5.2.0, file-type@^5.2.0:
   version "5.2.0"
   resolved "https://registry.yarnpkg.com/file-type/-/file-type-5.2.0.tgz#2ddbea7c73ffe36368dfae49dc338c058c2b8ad6"
@@ -12071,7 +12079,7 @@ mime-db@1.43.0:
   resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.43.0.tgz#0a12e0502650e473d735535050e7c8f4eb4fae58"
   integrity sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==
 
-"mime-db@>= 1.43.0 < 2":
+mime-db@1.44.0, "mime-db@>= 1.43.0 < 2":
   version "1.44.0"
   resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92"
   integrity sha512-/NOTfLrsPBVeH7YtFPgsVWveuL+4SjjYxaQ1xtM1KMFj7HdxlBlxeyNLzhyJVx7r4rZGJAZ/6lkKCitSc/Nmpg==
@@ -12095,6 +12103,13 @@ mime-types@^2.1.12, mime-types@~2.1.19:
   dependencies:
     mime-db "1.43.0"
 
+mime-types@^2.1.26:
+  version "2.1.27"
+  resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.27.tgz#47949f98e279ea53119f5722e0f34e529bec009f"
+  integrity sha512-JIhqnCasI9yD+SsmkquHBxTSEuZdQX5BuQnS2Vc7puQQQ+8yiP5AY5uWhpdv4YL4VM5c6iliiYWPgJ/nJQLp7w==
+  dependencies:
+    mime-db "1.44.0"
+
 mime-types@~2.1.17, mime-types@~2.1.24:
   version "2.1.25"
   resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.25.tgz#39772d46621f93e2a80a856c53b86a62156a6437"
@@ -18294,6 +18309,15 @@ urix@^0.1.0:
   resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72"
   integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI=
 
+url-loader@^4.1.0:
+  version "4.1.0"
+  resolved "https://registry.yarnpkg.com/url-loader/-/url-loader-4.1.0.tgz#c7d6b0d6b0fccd51ab3ffc58a78d32b8d89a7be2"
+  integrity sha512-IzgAAIC8wRrg6NYkFIJY09vtktQcsvU8V6HhtQj9PTefbYImzLB1hufqo4m+RyM5N3mLx5BqJKccgxJS+W3kqw==
+  dependencies:
+    loader-utils "^2.0.0"
+    mime-types "^2.1.26"
+    schema-utils "^2.6.5"
+
 url-parse-lax@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/url-parse-lax/-/url-parse-lax-1.0.0.tgz#7af8f303645e9bd79a272e7a14ac68bc0609da73"