feat(v2): docs plugin continued (#1337)

* refactor(v2): shift markdown loader into plugin

* fix(v2): build command configure webpack

* temporary fix for failing test

packages/docusaurus-plugin-content-blog/index.js

@@ -155,6 +155,9 @@ class DocusaurusPluginContentBlog {
       });
     });
   }
+
+  // TODO: Add configureWebpack plugin to read Markdown. Currently it's using
+  // the docs plugin's markdown loader.
 }
 
 module.exports = DocusaurusPluginContentBlog;

packages/docusaurus-plugin-content-docs/package.json

@@ -7,11 +7,16 @@
   "dependencies": {
     "@babel/polyfill": "^7.4.0",
     "@docusaurus/utils": "^1.0.0",
+    "@mapbox/rehype-prism": "^0.3.1",
+    "@mdx-js/mdx": "^0.20.3",
+    "front-matter": "^3.0.1",
     "fs-extra": "^7.0.1",
     "globby": "^9.1.0",
-    "import-fresh": "^3.0.0"
+    "import-fresh": "^3.0.0",
+    "loader-utils": "^1.2.3"
   },
   "peerDependencies": {
-    "@docusaurus/core": "^2.0.0"
+    "@docusaurus/core": "^2.0.0",
+    "@mdx-js/tag": "^0.20.3"
   }
 }

packages/docusaurus-plugin-content-docs/src/__tests__/index.test.js

@@ -59,9 +59,7 @@ describe('loadDocs', () => {
   });
 
   test('versioned website', async () => {
-    const {env, siteDir, siteConfig, versionedDir} = await loadSetup(
+    const {env, siteDir, siteConfig} = await loadSetup('versioned');
-      'versioned',
-    );
     const sidebarPath = path.join(siteDir, 'sidebars.json');
     const plugin = new DocusaurusPluginContentDocs(
       {
@@ -74,7 +72,7 @@ describe('loadDocs', () => {
         siteConfig,
       },
     );
-    const {docs: docsMetadata} = await plugin.loadContent();
+    const {docs: docsMetadata, versionedDir} = await plugin.loadContent();
     const docsDir = plugin.contentPath;
 
     expect(docsMetadata['version-1.0.0-foo/bar']).toEqual({
@@ -108,13 +106,7 @@ describe('loadDocs', () => {
   });
 
   test('versioned & translated website', async () => {
-    const {
+    const {env, siteDir, siteConfig} = await loadSetup('transversioned');
-      env,
-      siteDir,
-      siteConfig,
-      translatedDir,
-      versionedDir,
-    } = await loadSetup('transversioned');
     const sidebarPath = path.join(siteDir, 'sidebars.json');
     const plugin = new DocusaurusPluginContentDocs(
       {
@@ -127,7 +119,11 @@ describe('loadDocs', () => {
         siteConfig,
       },
     );
-    const {docs: docsMetadata} = await plugin.loadContent();
+    const {
+      docs: docsMetadata,
+      translatedDir,
+      versionedDir,
+    } = await plugin.loadContent();
     const docsDir = plugin.contentPath;
 
     expect(docsMetadata['ko-version-1.0.0-foo/bar']).toEqual({
@@ -178,9 +174,7 @@ describe('loadDocs', () => {
   });
 
   test('translated website', async () => {
-    const {env, siteDir, siteConfig, translatedDir} = await loadSetup(
+    const {env, siteDir, siteConfig} = await loadSetup('translated');
-      'translated',
-    );
     const sidebarPath = path.join(siteDir, 'sidebars.json');
     const plugin = new DocusaurusPluginContentDocs(
       {
@@ -193,7 +187,7 @@ describe('loadDocs', () => {
         siteConfig,
       },
     );
-    const {docs: docsMetadata} = await plugin.loadContent();
+    const {docs: docsMetadata, translatedDir} = await plugin.loadContent();
     const docsDir = plugin.contentPath;
 
     expect(docsMetadata['ko-foo/baz']).toEqual({
@@ -230,9 +224,7 @@ describe('loadDocs', () => {
   });
 
   test('versioned website with skip next release', async () => {
-    const {env, siteDir, siteConfig, versionedDir} = await loadSetup(
+    const {env, siteDir, siteConfig} = await loadSetup('versioned');
-      'versioned',
-    );
     const sidebarPath = path.join(siteDir, 'sidebars.json');
     const plugin = new DocusaurusPluginContentDocs(
       {
@@ -246,7 +238,7 @@ describe('loadDocs', () => {
         siteConfig,
       },
     );
-    const {docs: docsMetadata} = await plugin.loadContent();
+    const {docs: docsMetadata, versionedDir} = await plugin.loadContent();
 
     expect(docsMetadata['version-1.0.0-foo/bar']).toEqual({
       category: 'Test',

packages/docusaurus-plugin-content-docs/src/__tests__/metadata.test.js

@@ -13,7 +13,8 @@ import loadSetup from '../../../docusaurus/test/loadSetup';
 describe('processMetadata', () => {
   test('normal docs', async () => {
     const props = await loadSetup('simple');
-    const {docsDir, env, siteConfig} = props;
+    const {siteDir, env, siteConfig} = props;
+    const docsDir = path.resolve(siteDir, '..', 'docs');
     const sourceA = path.join('foo', 'bar.md');
     const sourceB = path.join('hello.md');
     const dataA = await processMetadata(
@@ -54,7 +55,8 @@ describe('processMetadata', () => {
 
   test('docs with custom permalink', async () => {
     const props = await loadSetup('simple');
-    const {docsDir, env, siteConfig} = props;
+    const {siteDir, env, siteConfig} = props;
+    const docsDir = path.resolve(siteDir, '..', 'docs');
     const source = path.join('permalink.md');
     const data = await processMetadata(
       source,
@@ -77,7 +79,8 @@ describe('processMetadata', () => {
 
   test('versioned docs (without translation)', async () => {
     const props = await loadSetup('versioned');
-    const {siteDir, docsDir, env, siteConfig} = props;
+    const {siteDir, env, siteConfig} = props;
+    const docsDir = path.resolve(siteDir, '..', 'docs');
     const versionedDir = path.join(siteDir, 'versioned_docs');
     const sourceA = path.join('version-1.0.0', 'foo', 'bar.md');
     const sourceB = path.join('version-1.0.0', 'hello.md');
@@ -155,7 +158,10 @@ describe('processMetadata', () => {
 
   test('translated versioned docs', async () => {
     const props = await loadSetup('transversioned');
-    const {docsDir, translatedDir, versionedDir, env, siteConfig} = props;
+    const {siteDir, env, siteConfig} = props;
+    const docsDir = path.resolve(siteDir, '..', 'docs');
+    const versionedDir = path.join(siteDir, 'versioned_docs');
+    const translatedDir = path.join(siteDir, 'translated_docs');
     const sourceA = path.join('ko', 'version-1.0.0', 'foo', 'bar.md');
     const sourceB = path.join('ko', 'version-1.0.0', 'hello.md');
     const sourceC = path.join('ko', 'version-1.0.1', 'foo', 'bar.md');
@@ -304,7 +310,9 @@ describe('processMetadata', () => {
 
   test('translated docs only', async () => {
     const props = await loadSetup('translated');
-    const {docsDir, translatedDir, env, siteConfig} = props;
+    const {siteDir, env, siteConfig} = props;
+    const docsDir = path.resolve(siteDir, '..', 'docs');
+    const translatedDir = path.join(siteDir, 'translated_docs');
     const sourceA = path.join('ko', 'foo', 'bar.md');
     const sourceB = path.join('ko', 'hello.md');
     const sourceC = path.join('foo', 'bar.md');

packages/docusaurus-plugin-content-docs/src/index.js

@@ -9,6 +9,7 @@ const globby = require('globby');
 const importFresh = require('import-fresh');
 const path = require('path');
 const {getSubFolder, idx, normalizeUrl} = require('@docusaurus/utils');
+const rehypePrism = require('@mapbox/rehype-prism');
 
 const createOrder = require('./order');
 const loadSidebars = require('./sidebars');
@@ -33,6 +34,7 @@ class DocusaurusPluginContentDocs {
     this.options = {...DEFAULT_OPTIONS, ...opts};
     this.context = context;
     this.contentPath = path.resolve(this.context.siteDir, this.options.path);
+    this.content = {};
   }
 
   getName() {
@@ -104,8 +106,9 @@ class DocusaurusPluginContentDocs {
     }
 
     // Metadata for non-default-language docs.
+    let translatedDir = null;
     if (translationEnabled) {
-      const translatedDir = path.join(siteDir, 'translated_docs');
+      translatedDir = path.join(siteDir, 'translated_docs');
       const translatedFiles = await globby(include, {
         cwd: translatedDir,
       });
@@ -141,8 +144,9 @@ class DocusaurusPluginContentDocs {
     }
 
     // Metadata for versioned docs.
+    let versionedDir = null;
     if (versioningEnabled) {
-      const versionedDir = path.join(siteDir, 'versioned_docs');
+      versionedDir = path.join(siteDir, 'versioned_docs');
       const versionedFiles = await globby(include, {
         cwd: versionedDir,
       });
@@ -185,12 +189,16 @@ class DocusaurusPluginContentDocs {
       };
     });
 
-    return {
+    this.content = {
       docs,
       docsDir,
       docsSidebars,
       sourceToMetadata,
+      translatedDir,
+      versionedDir,
     };
+
+    return this.content;
   }
 
   async contentLoaded({content, actions}) {
@@ -208,6 +216,53 @@ class DocusaurusPluginContentDocs {
       })),
     });
   }
+
+  configureWebpack(config, isServer) {
+    const versionedDir = path.join(this.context.siteDir, 'versioned_docs');
+    const translatedDir = path.join(this.context.siteDir, 'translated_docs');
+
+    return {
+      module: {
+        rules: [
+          {
+            test: /(\.mdx?)$/, // TODO: Read only this plugin's markdown files.
+            use: [
+              // TODO: Add back cache loader and read babel loader from existing config
+              // instead of duplicating it.
+              {
+                loader: 'babel-loader',
+                options: {
+                  // ignore local project babel config (.babelrc)
+                  babelrc: false,
+                  // ignore local project babel config (babel.config.js)
+                  configFile: false,
+                  presets: ['@babel/env', '@babel/react'],
+                  plugins: [
+                    'react-hot-loader/babel', // To enable react-hot-loader
+                    isServer
+                      ? 'dynamic-import-node'
+                      : '@babel/syntax-dynamic-import',
+                    'react-loadable/babel',
+                  ],
+                },
+              },
+              {
+                loader: path.resolve(__dirname, './markdown/index.js'),
+                options: {
+                  siteConfig: this.context.siteConfig,
+                  versionedDir,
+                  translatedDir,
+                  docsDir: this.content.docsDir,
+                  sourceToMetadata: this.content.sourceToMetadata,
+                  hastPlugins: [[rehypePrism, {ignoreMissing: true}]],
+                },
+              },
+            ],
+          },
+        ],
+      },
+    };
+  }
 }
 
 module.exports = DocusaurusPluginContentDocs;

packages/docusaurus/lib/commands/build.js

@@ -60,12 +60,21 @@ module.exports = async function build(siteDir, cliOptions = {}) {
   let serverConfig = createServerConfig(props).toConfig();
 
   // Plugin lifecycle - configureWebpack
-  plugins.forEach(({configureWebpack}) => {
+  plugins.forEach(plugin => {
+    const {configureWebpack} = plugin;
     if (!configureWebpack) {
       return;
     }
-    clientConfig = applyConfigureWebpack(configureWebpack, clientConfig, false);
+    clientConfig = applyConfigureWebpack(
-    serverConfig = applyConfigureWebpack(configureWebpack, serverConfig, true);
+      configureWebpack.bind(plugin), // The plugin lifecycle may reference `this`.
+      clientConfig,
+      false,
+    );
+    serverConfig = applyConfigureWebpack(
+      configureWebpack.bind(plugin), // The plugin lifecycle may reference `this`.
+      serverConfig,
+      true,
+    );
   });
 
   // Build the client bundles first.

packages/docusaurus/lib/commands/start.js

@@ -94,11 +94,17 @@ module.exports = async function start(siteDir, cliOptions = {}) {
   config = config.toConfig();
 
   // Plugin lifecycle - configureWebpack
-  plugins.forEach(({configureWebpack}) => {
+  plugins.forEach(plugin => {
+    const {configureWebpack} = plugin;
     if (!configureWebpack) {
       return;
     }
-    config = applyConfigureWebpack(configureWebpack, config, false);
+
+    config = applyConfigureWebpack(
+      configureWebpack.bind(plugin), // The plugin lifecycle may reference `this`.
+      config,
+      false,
+    );
   });
 
   // https://webpack.js.org/configuration/dev-server

packages/docusaurus/lib/load/index.js

@@ -59,16 +59,6 @@ module.exports = async function load(siteDir, cliOptions = {}) {
   const themePath = loadTheme(siteDir);
 
   const {baseUrl} = siteConfig;
-  const versionedDir = path.join(siteDir, 'versioned_docs');
-  const translatedDir = path.join(siteDir, 'translated_docs');
-
-  // TODO: Make doc dependents use the plugin's content instead
-  // of passing in via props.
-  const {
-    docsDir,
-    docs: docsMetadata,
-    sourceToMetadata,
-  } = pluginsLoadedContent[0].content;
 
   // Generate React Router Config.
   const {routesConfig, routesPaths} = await loadRoutes(pluginsRouteConfigs);
@@ -101,15 +91,10 @@ module.exports = async function load(siteDir, cliOptions = {}) {
   const props = {
     siteConfig,
     siteDir,
-    docsDir,
-    docsMetadata,
     env,
     outDir,
     themePath,
     baseUrl,
-    sourceToMetadata,
-    versionedDir,
-    translatedDir,
     generatedFilesDir,
     routesPaths,
     plugins,

packages/docusaurus/lib/load/plugins.js

@@ -20,8 +20,8 @@ module.exports = async function loadPlugins({pluginConfigs = [], context}) {
       try {
         // eslint-disable-next-line
         Plugin = require(name);
-      } catch (e) {
+      } catch (ex) {
-        throw new Error(`'${name}' plugin cannot be found.`);
+        throw new Error(`Error loading '${name}' plugin.`);
       }
     }
     return new Plugin(options, context);

packages/docusaurus/lib/webpack/base.js

@@ -5,47 +5,22 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-const Config = require('webpack-chain');
 const CSSExtractPlugin = require('mini-css-extract-plugin');
+const Config = require('webpack-chain');
 const cacheLoaderVersion = require('cache-loader/package.json').version;
-const rehypePrism = require('@mapbox/rehype-prism');
 const TerserPlugin = require('terser-webpack-plugin');
 const path = require('path');
 const isWsl = require('is-wsl');
-
+const {applyBabel, applyCacheLoader, applyStyle} = require('./utils');
-const mdLoader = require.resolve('./loaders/markdown');
 
 const CSS_REGEX = /\.css$/;
 const CSS_MODULE_REGEX = /\.module\.css$/;
 
-// Utility method to add styling-related rule to Webpack config.
-function applyStyle(styleRule, {cssOptions, isServer, isProd}) {
-  if (!isServer) {
-    if (isProd) {
-      styleRule.use('extract-css-loader').loader(CSSExtractPlugin.loader);
-    } else {
-      styleRule.use('style-loader').loader('style-loader');
-    }
-  }
-
-  styleRule
-    .use('css-loader')
-    .loader(isServer ? 'css-loader/locals' : 'css-loader')
-    .options(cssOptions);
-
-  return styleRule;
-}
-
 module.exports = function createBaseConfig(props, isServer) {
   const {
-    siteConfig,
     outDir,
     themePath,
-    docsDir,
     siteDir,
-    sourceToMetadata,
-    versionedDir,
-    translatedDir,
     baseUrl,
     generatedFilesDir,
     cliOptions: {cacheLoader},
@@ -69,9 +44,6 @@ module.exports = function createBaseConfig(props, isServer) {
     .set('symlinks', true)
     .alias.set('@theme', themePath)
     .set('@site', siteDir)
-    .set('@versioned_docs', versionedDir)
-    .set('@translated_docs', translatedDir)
-    .set('@docs', docsDir)
     .set('@build', outDir)
     .set('@generated', generatedFilesDir)
     .set('@core', path.resolve(__dirname, '../core'))
@@ -82,36 +54,6 @@ module.exports = function createBaseConfig(props, isServer) {
     .add(path.resolve(process.cwd(), 'node_modules'))
     .add('node_modules');
 
-  function applyCacheLoader(rule) {
-    if (cacheLoader) {
-      rule
-        .use('cache-loader')
-        .loader('cache-loader')
-        .options({
-          cacheDirectory: path.resolve(siteDir, '.cache-loader'),
-          cacheIdentifier: `cache-loader:${cacheLoaderVersion}${isServer}`,
-        });
-    }
-  }
-
-  function applyBabel(rule) {
-    rule
-      .use('babel')
-      .loader('babel-loader')
-      .options({
-        // ignore local project babel config (.babelrc)
-        babelrc: false,
-        // ignore local project babel config (babel.config.js)
-        configFile: false,
-        presets: ['@babel/env', '@babel/react'],
-        plugins: [
-          'react-hot-loader/babel', // To enable react-hot-loader
-          isServer ? 'dynamic-import-node' : '@babel/syntax-dynamic-import',
-          'react-loadable/babel',
-        ],
-      });
-  }
-
   const jsRule = config.module
     .rule('js')
     .test(/\.jsx?$/)
@@ -124,23 +66,13 @@ module.exports = function createBaseConfig(props, isServer) {
       return /node_modules/.test(filepath);
     })
     .end();
-  applyCacheLoader(jsRule);
+  applyCacheLoader(jsRule, {
-  applyBabel(jsRule);
+    cacheLoader,
-
+    siteDir,
-  const mdRule = config.module.rule('markdown').test(/(\.mdx?)$/);
+    cacheLoaderVersion,
-  applyCacheLoader(mdRule);
+    isServer,
-  applyBabel(mdRule);
-  mdRule
-    .use('@docusaurus/mdx-loader')
-    .loader(mdLoader)
-    .options({
-      siteConfig,
-      versionedDir,
-      translatedDir,
-      docsDir,
-      sourceToMetadata,
-      hastPlugins: [[rehypePrism, {ignoreMissing: true}]],
   });
+  applyBabel(jsRule, {isServer});
 
   applyStyle(config.module.rule('css'), {
     cssOptions: {

packages/docusaurus/lib/webpack/utils.js

@@ -5,6 +5,8 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+const CSSExtractPlugin = require('mini-css-extract-plugin');
+const path = require('path');
 const merge = require('webpack-merge');
 
 // Modify the generated webpack config with normal webpack config.
@@ -28,7 +30,61 @@ function applyChainWebpack(userChainWebpack, config, isServer) {
   }
 }
 
+// Utility method to add styling-related rule to Webpack config.
+function applyStyle(styleRule, {cssOptions, isServer, isProd}) {
+  if (!isServer) {
+    if (isProd) {
+      styleRule.use('extract-css-loader').loader(CSSExtractPlugin.loader);
+    } else {
+      styleRule.use('style-loader').loader('style-loader');
+    }
+  }
+
+  styleRule
+    .use('css-loader')
+    .loader(isServer ? 'css-loader/locals' : 'css-loader')
+    .options(cssOptions);
+
+  return styleRule;
+}
+
+function applyCacheLoader(
+  rule,
+  {cacheLoader, siteDir, cacheLoaderVersion, isServer},
+) {
+  if (cacheLoader) {
+    rule
+      .use('cache-loader')
+      .loader('cache-loader')
+      .options({
+        cacheDirectory: path.resolve(siteDir, '.cache-loader'),
+        cacheIdentifier: `cache-loader:${cacheLoaderVersion}${isServer}`,
+      });
+  }
+}
+
+function applyBabel(rule, {isServer}) {
+  rule
+    .use('babel')
+    .loader('babel-loader')
+    .options({
+      // ignore local project babel config (.babelrc)
+      babelrc: false,
+      // ignore local project babel config (babel.config.js)
+      configFile: false,
+      presets: ['@babel/env', '@babel/react'],
+      plugins: [
+        'react-hot-loader/babel', // To enable react-hot-loader
+        isServer ? 'dynamic-import-node' : '@babel/syntax-dynamic-import',
+        'react-loadable/babel',
+      ],
+    });
+}
+
 module.exports = {
+  applyBabel,
+  applyCacheLoader,
   applyConfigureWebpack,
   applyChainWebpack,
+  applyStyle,
 };

packages/docusaurus/package.json

@@ -33,8 +33,6 @@
     "@babel/preset-react": "^7.0.0",
     "@docusaurus/utils": "^1.0.0",
     "@hot-loader/react-dom": "^16.8.4",
-    "@mapbox/rehype-prism": "^0.3.1",
-    "@mdx-js/mdx": "^0.20.3",
     "@mdx-js/tag": "^0.20.3",
     "babel-jest": "^24.1.0",
     "babel-loader": "^8.0.0",

yarn.lock

@@ -8297,7 +8297,7 @@ loader-runner@^2.3.0:
   resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-2.4.0.tgz#ed47066bfe534d7e84c4c7b9998c2a75607d9357"
   integrity sha512-Jsmr89RcXGIwivFY21FcRrisYZfvLMTWx5kOLc+JTxtpBOG6xML0vzbc6SEQG2FO9/4Fc3wW4LVcB5DmGflaRw==
 
-loader-utils@1.2.3, loader-utils@^1.0.2, loader-utils@^1.1.0:
+loader-utils@1.2.3, loader-utils@^1.0.2, loader-utils@^1.1.0, loader-utils@^1.2.3:
   version "1.2.3"
   resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-1.2.3.tgz#1ff5dc6911c9f0a062531a4c04b609406108c2c7"
   integrity sha512-fkpz8ejdnEMG3s37wGL07iSBDg99O9D5yflE9RGNH3hRdx9SOwYfnGYdZOUIZitN8E+E2vkq3MUMYMvPYl5ZZA==