feat(v2): docs plugin continued (#1337)

* refactor(v2): shift markdown loader into plugin

* fix(v2): build command configure webpack

* temporary fix for failing test

packages/docusaurus-plugin-content-docs/package.json

@@ -7,11 +7,16 @@
   "dependencies": {
     "@babel/polyfill": "^7.4.0",
     "@docusaurus/utils": "^1.0.0",
+    "@mapbox/rehype-prism": "^0.3.1",
+    "@mdx-js/mdx": "^0.20.3",
+    "front-matter": "^3.0.1",
     "fs-extra": "^7.0.1",
     "globby": "^9.1.0",
-    "import-fresh": "^3.0.0"
+    "import-fresh": "^3.0.0",
+    "loader-utils": "^1.2.3"
   },
   "peerDependencies": {
-    "@docusaurus/core": "^2.0.0"
+    "@docusaurus/core": "^2.0.0",
+    "@mdx-js/tag": "^0.20.3"
   }
 }

packages/docusaurus-plugin-content-docs/src/__tests__/index.test.js

@@ -59,9 +59,7 @@ describe('loadDocs', () => {
   });
 
   test('versioned website', async () => {
-    const {env, siteDir, siteConfig, versionedDir} = await loadSetup(
-      'versioned',
-    );
+    const {env, siteDir, siteConfig} = await loadSetup('versioned');
     const sidebarPath = path.join(siteDir, 'sidebars.json');
     const plugin = new DocusaurusPluginContentDocs(
       {
@@ -74,7 +72,7 @@ describe('loadDocs', () => {
         siteConfig,
       },
     );
-    const {docs: docsMetadata} = await plugin.loadContent();
+    const {docs: docsMetadata, versionedDir} = await plugin.loadContent();
     const docsDir = plugin.contentPath;
 
     expect(docsMetadata['version-1.0.0-foo/bar']).toEqual({
@@ -108,13 +106,7 @@ describe('loadDocs', () => {
   });
 
   test('versioned & translated website', async () => {
-    const {
-      env,
-      siteDir,
-      siteConfig,
-      translatedDir,
-      versionedDir,
-    } = await loadSetup('transversioned');
+    const {env, siteDir, siteConfig} = await loadSetup('transversioned');
     const sidebarPath = path.join(siteDir, 'sidebars.json');
     const plugin = new DocusaurusPluginContentDocs(
       {
@@ -127,7 +119,11 @@ describe('loadDocs', () => {
         siteConfig,
       },
     );
-    const {docs: docsMetadata} = await plugin.loadContent();
+    const {
+      docs: docsMetadata,
+      translatedDir,
+      versionedDir,
+    } = await plugin.loadContent();
     const docsDir = plugin.contentPath;
 
     expect(docsMetadata['ko-version-1.0.0-foo/bar']).toEqual({
@@ -178,9 +174,7 @@ describe('loadDocs', () => {
   });
 
   test('translated website', async () => {
-    const {env, siteDir, siteConfig, translatedDir} = await loadSetup(
-      'translated',
-    );
+    const {env, siteDir, siteConfig} = await loadSetup('translated');
     const sidebarPath = path.join(siteDir, 'sidebars.json');
     const plugin = new DocusaurusPluginContentDocs(
       {
@@ -193,7 +187,7 @@ describe('loadDocs', () => {
         siteConfig,
       },
     );
-    const {docs: docsMetadata} = await plugin.loadContent();
+    const {docs: docsMetadata, translatedDir} = await plugin.loadContent();
     const docsDir = plugin.contentPath;
 
     expect(docsMetadata['ko-foo/baz']).toEqual({
@@ -230,9 +224,7 @@ describe('loadDocs', () => {
   });
 
   test('versioned website with skip next release', async () => {
-    const {env, siteDir, siteConfig, versionedDir} = await loadSetup(
-      'versioned',
-    );
+    const {env, siteDir, siteConfig} = await loadSetup('versioned');
     const sidebarPath = path.join(siteDir, 'sidebars.json');
     const plugin = new DocusaurusPluginContentDocs(
       {
@@ -246,7 +238,7 @@ describe('loadDocs', () => {
         siteConfig,
       },
     );
-    const {docs: docsMetadata} = await plugin.loadContent();
+    const {docs: docsMetadata, versionedDir} = await plugin.loadContent();
 
     expect(docsMetadata['version-1.0.0-foo/bar']).toEqual({
       category: 'Test',

packages/docusaurus-plugin-content-docs/src/__tests__/metadata.test.js

@@ -13,7 +13,8 @@ import loadSetup from '../../../docusaurus/test/loadSetup';
 describe('processMetadata', () => {
   test('normal docs', async () => {
     const props = await loadSetup('simple');
-    const {docsDir, env, siteConfig} = props;
+    const {siteDir, env, siteConfig} = props;
+    const docsDir = path.resolve(siteDir, '..', 'docs');
     const sourceA = path.join('foo', 'bar.md');
     const sourceB = path.join('hello.md');
     const dataA = await processMetadata(
@@ -54,7 +55,8 @@ describe('processMetadata', () => {
 
   test('docs with custom permalink', async () => {
     const props = await loadSetup('simple');
-    const {docsDir, env, siteConfig} = props;
+    const {siteDir, env, siteConfig} = props;
+    const docsDir = path.resolve(siteDir, '..', 'docs');
     const source = path.join('permalink.md');
     const data = await processMetadata(
       source,
@@ -77,7 +79,8 @@ describe('processMetadata', () => {
 
   test('versioned docs (without translation)', async () => {
     const props = await loadSetup('versioned');
-    const {siteDir, docsDir, env, siteConfig} = props;
+    const {siteDir, env, siteConfig} = props;
+    const docsDir = path.resolve(siteDir, '..', 'docs');
     const versionedDir = path.join(siteDir, 'versioned_docs');
     const sourceA = path.join('version-1.0.0', 'foo', 'bar.md');
     const sourceB = path.join('version-1.0.0', 'hello.md');
@@ -155,7 +158,10 @@ describe('processMetadata', () => {
 
   test('translated versioned docs', async () => {
     const props = await loadSetup('transversioned');
-    const {docsDir, translatedDir, versionedDir, env, siteConfig} = props;
+    const {siteDir, env, siteConfig} = props;
+    const docsDir = path.resolve(siteDir, '..', 'docs');
+    const versionedDir = path.join(siteDir, 'versioned_docs');
+    const translatedDir = path.join(siteDir, 'translated_docs');
     const sourceA = path.join('ko', 'version-1.0.0', 'foo', 'bar.md');
     const sourceB = path.join('ko', 'version-1.0.0', 'hello.md');
     const sourceC = path.join('ko', 'version-1.0.1', 'foo', 'bar.md');
@@ -304,7 +310,9 @@ describe('processMetadata', () => {
 
   test('translated docs only', async () => {
     const props = await loadSetup('translated');
-    const {docsDir, translatedDir, env, siteConfig} = props;
+    const {siteDir, env, siteConfig} = props;
+    const docsDir = path.resolve(siteDir, '..', 'docs');
+    const translatedDir = path.join(siteDir, 'translated_docs');
     const sourceA = path.join('ko', 'foo', 'bar.md');
     const sourceB = path.join('ko', 'hello.md');
     const sourceC = path.join('foo', 'bar.md');

packages/docusaurus-plugin-content-docs/src/index.js

@@ -9,6 +9,7 @@ const globby = require('globby');
 const importFresh = require('import-fresh');
 const path = require('path');
 const {getSubFolder, idx, normalizeUrl} = require('@docusaurus/utils');
+const rehypePrism = require('@mapbox/rehype-prism');
 
 const createOrder = require('./order');
 const loadSidebars = require('./sidebars');
@@ -33,6 +34,7 @@ class DocusaurusPluginContentDocs {
     this.options = {...DEFAULT_OPTIONS, ...opts};
     this.context = context;
     this.contentPath = path.resolve(this.context.siteDir, this.options.path);
+    this.content = {};
   }
 
   getName() {
@@ -104,8 +106,9 @@ class DocusaurusPluginContentDocs {
     }
 
     // Metadata for non-default-language docs.
+    let translatedDir = null;
     if (translationEnabled) {
-      const translatedDir = path.join(siteDir, 'translated_docs');
+      translatedDir = path.join(siteDir, 'translated_docs');
       const translatedFiles = await globby(include, {
         cwd: translatedDir,
       });
@@ -141,8 +144,9 @@ class DocusaurusPluginContentDocs {
     }
 
     // Metadata for versioned docs.
+    let versionedDir = null;
     if (versioningEnabled) {
-      const versionedDir = path.join(siteDir, 'versioned_docs');
+      versionedDir = path.join(siteDir, 'versioned_docs');
       const versionedFiles = await globby(include, {
         cwd: versionedDir,
       });
@@ -185,12 +189,16 @@ class DocusaurusPluginContentDocs {
       };
     });
 
-    return {
+    this.content = {
       docs,
       docsDir,
       docsSidebars,
       sourceToMetadata,
+      translatedDir,
+      versionedDir,
     };
+
+    return this.content;
   }
 
   async contentLoaded({content, actions}) {
@@ -208,6 +216,53 @@ class DocusaurusPluginContentDocs {
       })),
     });
   }
+
+  configureWebpack(config, isServer) {
+    const versionedDir = path.join(this.context.siteDir, 'versioned_docs');
+    const translatedDir = path.join(this.context.siteDir, 'translated_docs');
+
+    return {
+      module: {
+        rules: [
+          {
+            test: /(\.mdx?)$/, // TODO: Read only this plugin's markdown files.
+            use: [
+              // TODO: Add back cache loader and read babel loader from existing config
+              // instead of duplicating it.
+              {
+                loader: 'babel-loader',
+                options: {
+                  // ignore local project babel config (.babelrc)
+                  babelrc: false,
+                  // ignore local project babel config (babel.config.js)
+                  configFile: false,
+                  presets: ['@babel/env', '@babel/react'],
+                  plugins: [
+                    'react-hot-loader/babel', // To enable react-hot-loader
+                    isServer
+                      ? 'dynamic-import-node'
+                      : '@babel/syntax-dynamic-import',
+                    'react-loadable/babel',
+                  ],
+                },
+              },
+              {
+                loader: path.resolve(__dirname, './markdown/index.js'),
+                options: {
+                  siteConfig: this.context.siteConfig,
+                  versionedDir,
+                  translatedDir,
+                  docsDir: this.content.docsDir,
+                  sourceToMetadata: this.content.sourceToMetadata,
+                  hastPlugins: [[rehypePrism, {ignoreMissing: true}]],
+                },
+              },
+            ],
+          },
+        ],
+      },
+    };
+  }
 }
 
 module.exports = DocusaurusPluginContentDocs;

packages/docusaurus-plugin-content-docs/src/markdown/index.js

@@ -0,0 +1,98 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+const fm = require('front-matter');
+const mdx = require('@mdx-js/mdx');
+const {getOptions} = require('loader-utils');
+const path = require('path');
+const {resolve} = require('url');
+
+module.exports = async function(fileString) {
+  const callback = this.async();
+  const options = Object.assign({}, getOptions(this), {
+    filepath: this.resourcePath,
+  });
+  const {versionedDir, docsDir, translatedDir, sourceToMetadata} = options;
+
+  // Extract content of markdown (without frontmatter).
+  const {body} = fm(fileString);
+
+  // Determine the source dir. e.g: /docs, /website/versioned_docs/version-1.0.0
+  let sourceDir;
+  const thisSource = this.resourcePath;
+  if (thisSource.startsWith(translatedDir)) {
+    const {language, version} = sourceToMetadata[thisSource] || {};
+    if (language && version && version !== 'next') {
+      sourceDir = path.join(translatedDir, language, `version-${version}`);
+    } else if (language && (!version || version === 'next')) {
+      sourceDir = path.join(translatedDir, language);
+    }
+  } else if (thisSource.startsWith(versionedDir)) {
+    const {version} = sourceToMetadata[thisSource] || {};
+    if (version) {
+      sourceDir = path.join(versionedDir, `version-${version}`);
+    }
+  } else if (thisSource.startsWith(docsDir)) {
+    sourceDir = docsDir;
+  }
+
+  // Replace internal markdown linking (except in fenced blocks).
+  let content = body;
+  if (sourceDir) {
+    let fencedBlock = false;
+    const lines = body.split('\n').map(line => {
+      if (line.trim().startsWith('```')) {
+        fencedBlock = !fencedBlock;
+      }
+      if (fencedBlock) return line;
+
+      let modifiedLine = line;
+      // Replace inline-style links or reference-style links e.g:
+      // This is [Document 1](doc1.md) -> we replace this doc1.md with correct link
+      // [doc1]: doc1.md -> we replace this doc1.md with correct link
+      const mdRegex = /(?:(?:\]\()|(?:\]:\s?))(?!https)([^'")\]\s>]+\.md)/g;
+      let mdMatch = mdRegex.exec(modifiedLine);
+      while (mdMatch !== null) {
+        // Replace it to correct html link.
+        const mdLink = mdMatch[1];
+        const targetSource = `${sourceDir}/${mdLink}`;
+        const {permalink} =
+          sourceToMetadata[resolve(thisSource, mdLink)] ||
+          sourceToMetadata[targetSource] ||
+          {};
+        if (permalink) {
+          modifiedLine = modifiedLine.replace(mdLink, permalink);
+        }
+        mdMatch = mdRegex.exec(modifiedLine);
+      }
+      return modifiedLine;
+    });
+    content = lines.join('\n');
+  }
+
+  let result;
+
+  try {
+    result = await mdx(content, options);
+  } catch (err) {
+    return callback(err);
+  }
+
+  // TODO: Allow choosing prismjs theme
+  // prismjs/themes/XXXXXX.css https://github.com/PrismJS/prism/tree/master/themes
+  // prism-themes/themes/XXXXXX.css https://github.com/PrismJS/prism-themes/tree/master/themes
+  const prismThemeImport = 'prism-themes/themes/prism-atom-dark.css';
+
+  const code = `
+  import React from 'react';
+  import { MDXTag } from '@mdx-js/tag';
+  import '${prismThemeImport}';
+  ${result}
+  `;
+
+  return callback(null, code);
+};