Refactor + add more tests (Part 2) (#854)

* Refactor doc in server.js & generate.js

* Refactor finding metadata with find() & avoid param reassign

* Refactor replacing markdown link to html + add test

* Minify func to getFile & add test

* use ./ for md link as well

* nits & better test

* better mdToHtmlify

* babel-register on versions & write-translations to transpile ES6 in server/env.js

* better docs test & move out metadata to fixtures

* Update .babelrc

.babelrc

@@ -2,7 +2,8 @@
   "env": {
     "test": {
       "presets": [
-        "env"
+        "env", 
+        "react"
       ],
       "plugins": [
         "transform-class-properties",

lib/server/__tests__/__fixtures__/doc1.md

@@ -0,0 +1,6 @@
+---
+id: doc1
+title: Document 1
+---
+
+Docusaurus is the best :)

lib/server/__tests__/__fixtures__/doc2.md

@@ -0,0 +1,18 @@
+---
+id: doc2
+title: Document 2
+---
+
+### Existing Docs
+
+- [doc1](doc1.md)
+- [doc2](./doc2.md)
+
+### Non-existing Docs
+
+- [hahaha](hahaha.md)
+
+## Repeating Docs
+
+- [doc1](doc1.md)
+- [doc2](./doc2.md)

lib/server/__tests__/__fixtures__/metadata.js

@@ -0,0 +1,116 @@
+module.exports = {
+  'en-doc1': {
+    id: 'en-doc1',
+    title: 'Document 1',
+    source: 'doc1.md',
+    version: 'next',
+    permalink: 'docs/en/next/doc1.html',
+    localized_id: 'doc1',
+    language: 'en',
+    sidebar: 'docs',
+    category: 'Test',
+    next_id: 'doc2',
+    next: 'en-doc2',
+    next_title: 'Document 2',
+  },
+  'en-doc2': {
+    id: 'en-doc2',
+    title: 'Document 2',
+    source: 'doc2.md',
+    version: 'next',
+    permalink: 'docs/en/next/doc2.html',
+    localized_id: 'doc2',
+    language: 'en',
+    sidebar: 'docs',
+    category: 'Test',
+    previous_id: 'doc1',
+    previous: 'en-doc1',
+    previous_title: 'Document 1',
+  },
+  'ko-doc1': {
+    id: 'ko-doc1',
+    title: '문서 1',
+    source: 'doc1.md',
+    version: 'next',
+    permalink: 'docs/ko/next/doc1.html',
+    localized_id: 'doc1',
+    language: 'ko',
+    sidebar: 'docs',
+    category: 'Test',
+    next_id: 'doc2',
+    next: 'ko-doc2',
+    next_title: '문서 2',
+  },
+  'ko-doc2': {
+    id: 'ko-doc2',
+    title: '문서 2',
+    source: 'doc2.md',
+    version: 'next',
+    permalink: 'docs/ko/next/doc2.html',
+    localized_id: 'doc2',
+    language: 'ko',
+    sidebar: 'docs',
+    category: 'Test',
+    previous_id: 'doc1',
+    previous: 'ko-doc1',
+    previous_title: '문서 1',
+  },
+  'en-version-1.0.0-doc1': {
+    id: 'en-version-1.0.0-doc1',
+    original_id: 'doc1',
+    title: 'Document 1',
+    source: 'version-1.0.0/doc1.md',
+    version: '1.0.0',
+    permalink: 'docs/en/doc1.html',
+    localized_id: 'version-1.0.0-doc1',
+    language: 'en',
+    sidebar: 'version-1.0.0-docs',
+    category: 'Test',
+    next_id: 'doc2',
+    next: 'en-version-1.0.0-doc2',
+    next_title: 'Document 2',
+  },
+  'en-version-1.0.0-doc2': {
+    id: 'en-version-1.0.0-doc2',
+    original_id: 'doc2',
+    title: 'Document 2',
+    source: 'version-1.0.0/doc2.md',
+    version: '1.0.0',
+    permalink: 'docs/en/doc2.html',
+    localized_id: 'version-1.0.0-doc2',
+    language: 'en',
+    sidebar: 'version-1.0.0-docs',
+    category: 'Test',
+    previous_id: 'doc1',
+    previous: 'en-version-1.0.0-doc1',
+    previous_title: 'Document 1',
+  },
+  'ko-version-1.0.0-doc1': {
+    id: 'ko-version-1.0.0-doc1',
+    title: '문서 1',
+    source: 'version-1.0.0/doc1.md',
+    version: '1.0.0',
+    permalink: 'docs/ko/doc1.html',
+    localized_id: 'version-1.0.0-doc1',
+    language: 'ko',
+    sidebar: 'version-1.0.0-docs',
+    category: 'Test',
+    next_id: 'doc2',
+    next: 'ko-version-1.0.0-doc2',
+    next_title: '문서 2',
+  },
+  'ko-version-1.0.0-doc2': {
+    id: 'ko-version-1.0.0-doc2',
+    title: '문서 2',
+    source: 'version-1.0.0/doc2.md',
+    version: '1.0.0',
+    permalink: 'docs/ko/doc2.html',
+    localized_id: 'version-1.0.0-doc2',
+    language: 'ko',
+    sidebar: 'version-1.0.0-docs',
+    category: 'Test',
+    previous_id: 'doc1',
+    previous: 'ko-version-1.0.0-doc1',
+    previous_title: '문서 1',
+  },
+};

lib/server/__tests__/__snapshots__/docs.test.js.snap

@@ -0,0 +1,23 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`mdToHtmlify transform nothing 1`] = `
+"
+Docusaurus is the best :)"
+`;
+
+exports[`mdToHtmlify transform to correct link 1`] = `
+"
+### Existing Docs
+
+- [doc1](/docs/en/next/doc1)
+- [doc2](/docs/en/next/doc2)
+
+### Non-existing Docs
+
+- [hahaha](hahaha.md)
+
+## Repeating Docs
+
+- [doc1](/docs/en/next/doc1)
+- [doc2](/docs/en/next/doc2)"
+`;

lib/server/__tests__/docs.test.js

@@ -0,0 +1,134 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// simulate cwd to website so all require (CWD+'/siteConfig.js') will work
+const originalCwd = process.cwd();
+if (!/website$/.test(originalCwd)) {
+  process.chdir(process.cwd() + '/website');
+}
+const path = require('path');
+const fs = require('fs-extra');
+const docs = require('../docs');
+const metadataUtils = require('../metadataUtils');
+
+jest.mock('../env', () => ({
+  translation: {
+    enabled: true,
+    enabledLanguages: () => [
+      {
+        enabled: true,
+        name: 'English',
+        tag: 'en',
+      },
+      {
+        enabled: true,
+        name: '한국어',
+        tag: 'ko',
+      },
+    ],
+  },
+  versioning: {
+    enabled: true,
+    defaultVersion: '1.0.0',
+  },
+}));
+
+const Metadata = require(path.join(__dirname, '__fixtures__', 'metadata.js'));
+
+const doc1 = fs.readFileSync(
+  path.join(__dirname, '__fixtures__', 'doc1.md'),
+  'utf8'
+);
+
+const doc2 = fs.readFileSync(
+  path.join(__dirname, '__fixtures__', 'doc2.md'),
+  'utf8'
+);
+
+describe('mdToHtmlify', () => {
+  const rawContent1 = metadataUtils.extractMetadata(doc1).rawContent;
+  const rawContent2 = metadataUtils.extractMetadata(doc2).rawContent;
+  const mdToHtml = metadataUtils.mdToHtml(Metadata, '/');
+
+  test('transform nothing', () => {
+    const content1 = docs.mdToHtmlify(
+      rawContent1,
+      mdToHtml,
+      Metadata['en-doc1']
+    );
+    expect(content1).not.toContain('/docs/en/next/');
+    expect(content1).toMatchSnapshot();
+    expect(content1).toEqual(rawContent1);
+  });
+
+  test('transform to correct link', () => {
+    const content2 = docs.mdToHtmlify(
+      rawContent2,
+      mdToHtml,
+      Metadata['en-doc2']
+    );
+    expect(content2).toContain('/docs/en/next/');
+    expect(content2).toMatchSnapshot();
+    expect(content2).not.toEqual(rawContent2);
+  });
+});
+
+describe('getFile', () => {
+  const fakeContent = {
+    'website/translated_docs/ko/doc1.md': '이건 가짜 야',
+    'website/versioned_docs/version-1.0.0/doc2.md': 'Document 2 is not good',
+    'website/translated_docs/ko/version-1.0.0/doc1.md':
+      '이것은 오래된 가짜입니다.',
+    'docs/doc1.md': 'Just another document',
+  };
+  fs.existsSync = jest.fn().mockReturnValue(true);
+  fs.readFileSync = jest.fn().mockImplementation(file => {
+    const fakePath = file.replace(process.cwd().replace(/website$/, ''), '');
+    return fakeContent[fakePath];
+  });
+
+  test('docs does not exist', () => {
+    const metadata = Metadata['en-doc1'];
+    fs.existsSync.mockReturnValueOnce(null);
+    expect(docs.getFile(metadata)).toBeNull();
+  });
+
+  test('null/undefined metadata', () => {
+    expect(docs.getFile(null)).toBeNull();
+    expect(docs.getFile(undefined)).toBeNull();
+  });
+
+  test('translated docs', () => {
+    const metadata = Metadata['ko-doc1'];
+    expect(docs.getFile(metadata)).toEqual(
+      fakeContent['website/translated_docs/ko/doc1.md']
+    );
+  });
+
+  test('versioned docs', () => {
+    const metadata = Metadata['en-version-1.0.0-doc2'];
+    expect(docs.getFile(metadata)).toEqual(
+      fakeContent['website/versioned_docs/version-1.0.0/doc2.md']
+    );
+  });
+
+  test('translated & versioned docs', () => {
+    const metadata = Metadata['ko-version-1.0.0-doc1'];
+    expect(docs.getFile(metadata)).toEqual(
+      fakeContent['website/translated_docs/ko/version-1.0.0/doc1.md']
+    );
+  });
+
+  test('normal docs', () => {
+    const metadata = Metadata['en-doc1'];
+    expect(docs.getFile(metadata)).toEqual(fakeContent['docs/doc1.md']);
+  });
+});
+
+afterAll(() => {
+  process.chdir(originalCwd);
+});

lib/server/__tests__/routing.test.js

@@ -4,7 +4,6 @@
  * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
-
 const routing = require('../routing');
 
 describe('Blog routing', () => {

lib/server/docs.js

@@ -0,0 +1,100 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+const CWD = process.cwd();
+const siteConfig = require(`${CWD}/siteConfig.js`);
+const {join} = require('path');
+const fs = require('fs-extra');
+const React = require('react');
+const env = require('./env.js');
+const readMetadata = require('./readMetadata.js');
+const {insertTOC} = require('../core/toc.js');
+const {getPath} = require('../core/utils.js');
+
+function getFile(metadata) {
+  if (!metadata) {
+    return null;
+  }
+  let file;
+  if (env.versioning.enabled && metadata.original_id) {
+    if (env.translation.enabled && metadata.language !== 'en') {
+      file = join(CWD, 'translated_docs', metadata.language, metadata.source);
+    } else {
+      file = join(CWD, 'versioned_docs', metadata.source);
+    }
+  } else if (env.translation.enabled && metadata.language !== 'en') {
+    file = join(CWD, 'translated_docs', metadata.language, metadata.source);
+  } else {
+    file = join(CWD, '..', readMetadata.getDocsPath(), metadata.source);
+  }
+  if (!fs.existsSync(file)) {
+    return null;
+  }
+  return fs.readFileSync(file, 'utf8');
+}
+
+function mdToHtmlify(oldContent, mdToHtml, metadata) {
+  let content = oldContent;
+  const mdLinks = [];
+
+  // find any links to markdown files
+  const regex = /(?:\]\()(?:\.\/)?([^'")\]\s>]+\.md)/g;
+  let match = regex.exec(content);
+  while (match !== null) {
+    mdLinks.push(match[1]);
+    match = regex.exec(content);
+  }
+
+  // replace to their website html links
+  new Set(mdLinks).forEach(mdLink => {
+    let htmlLink = mdToHtml[mdLink];
+    if (htmlLink) {
+      htmlLink = getPath(htmlLink, siteConfig.cleanUrl);
+      htmlLink = htmlLink.replace('/en/', `/${metadata.language}/`);
+      htmlLink = htmlLink.replace(
+        '/VERSION/',
+        metadata.version && metadata.version !== env.versioning.defaultVersion
+          ? `/${metadata.version}/`
+          : '/'
+      );
+      content = content.replace(
+        new RegExp(`\\]\\((\\./)?${mdLink}`, 'g'),
+        `](${htmlLink}`
+      );
+    }
+  });
+  return content;
+}
+
+function getComponent(rawContent, mdToHtml, metadata) {
+  // generate table of contents
+  let content = insertTOC(rawContent);
+
+  // replace any links to markdown files to their website html links
+  content = mdToHtmlify(content, mdToHtml, metadata);
+
+  // replace any relative links to static assets to absolute links
+  content = content.replace(
+    /\]\(assets\//g,
+    `](${siteConfig.baseUrl}docs/assets/`
+  );
+
+  const DocsLayout = require('../core/DocsLayout.js');
+  return (
+    <DocsLayout
+      metadata={metadata}
+      language={metadata.language}
+      config={siteConfig}>
+      {content}
+    </DocsLayout>
+  );
+}
+
+module.exports = {
+  getComponent,
+  getFile,
+  mdToHtmlify,
+};

lib/server/generate.js

@@ -9,12 +9,12 @@ async function execute() {
   require('../write-translations.js');
 
   const metadataUtils = require('./metadataUtils');
+  const docs = require('./docs');
 
   const CWD = process.cwd();
   const fs = require('fs-extra');
   const readMetadata = require('./readMetadata.js');
   const path = require('path');
-  const {insertTOC} = require('../core/toc');
   const {getPath} = require('../core/utils.js');
   const {minifyCss, isSeparateCss} = require('./utils');
   const React = require('react');
@@ -75,7 +75,6 @@ async function execute() {
 
   const mdToHtml = metadataUtils.mdToHtml(Metadata, siteConfig.baseUrl);
 
-  const DocsLayout = require('../core/DocsLayout.js');
   const Redirect = require('../core/Redirect.js');
 
   fs.removeSync(join(CWD, 'build'));
@@ -83,66 +82,13 @@ async function execute() {
   // create html files for all docs by going through all doc ids
   Object.keys(Metadata).forEach(id => {
     const metadata = Metadata[id];
-    // determine what file to use according to its id
-    let file;
-    if (metadata.original_id) {
-      if (env.translation.enabled && metadata.language !== 'en') {
-        file = join(CWD, 'translated_docs', metadata.language, metadata.source);
-      } else {
-        file = join(CWD, 'versioned_docs', metadata.source);
-      }
-    } else if (metadata.language === 'en') {
-      file = join(CWD, '..', readMetadata.getDocsPath(), metadata.source);
-    } else {
-      file = join(CWD, 'translated_docs', metadata.language, metadata.source);
-    }
-
-    if (!fs.existsSync(file)) {
+    const file = docs.getFile(metadata);
+    if (!file) {
       return;
     }
-
-    let rawContent = metadataUtils.extractMetadata(
-      fs.readFileSync(file, 'utf8')
-    ).rawContent;
-
-    const language = metadata.language;
-
-    // generate table of contents if appropriate
-    rawContent = insertTOC(rawContent);
-
-    const defaultVersion = env.versioning.defaultVersion;
-
-    // replace any links to markdown files to their website html links
-    Object.keys(mdToHtml).forEach(key => {
-      let link = mdToHtml[key];
-      link = getPath(link, siteConfig.cleanUrl);
-      link = link.replace('/en/', `/${language}/`);
-      link = link.replace(
-        '/VERSION/',
-        metadata.version && metadata.version !== defaultVersion
-          ? `/${metadata.version}/`
-          : '/'
-      );
-      // replace relative links with & without "./"
-      rawContent = rawContent.replace(
-        new RegExp(`\\]\\((${key}|\\./${key})`, 'g'),
-        `](${link}`
-      );
-    });
-
-    // replace any relative links to static assets to absolute links
-    rawContent = rawContent.replace(
-      /\]\(assets\//g,
-      `](${siteConfig.baseUrl}docs/assets/`
-    );
-
-    const docComp = (
-      <DocsLayout metadata={metadata} language={language} config={siteConfig}>
-        {rawContent}
-      </DocsLayout>
-    );
+    const rawContent = metadataUtils.extractMetadata(file).rawContent;
+    const docComp = docs.getComponent(rawContent, mdToHtml, metadata);
     const str = renderToStaticMarkupWithDoctype(docComp);
-
     const targetFile = join(buildDir, metadata.permalink);
     writeFileAndCreateFolder(targetFile, str);
 
@@ -155,7 +101,7 @@ async function execute() {
       const redirectComp = (
         <Redirect
           metadata={metadata}
-          language={language}
+          language={metadata.language}
           config={siteConfig}
           redirect={siteConfig.baseUrl + redirectlink}
         />

lib/server/server.js

@@ -11,6 +11,7 @@ function execute(port, options) {
   const extractTranslations = require('../write-translations');
 
   const metadataUtils = require('./metadataUtils');
+  const docs = require('./docs');
 
   const env = require('./env.js');
   const express = require('express');
@@ -18,8 +19,6 @@ function execute(port, options) {
   const request = require('request');
   const fs = require('fs-extra');
   const path = require('path');
-  const {insertTOC} = require('../core/toc');
-  const {getPath} = require('../core/utils');
   const {isSeparateCss} = require('./utils');
   const mkdirp = require('mkdirp');
   const glob = require('glob');
@@ -152,100 +151,19 @@ function execute(port, options) {
   // handle all requests for document pages
   app.get(routing.docs(siteConfig.baseUrl), (req, res, next) => {
     const url = req.path.toString().replace(siteConfig.baseUrl, '');
-
-    // links is a map from a permalink to an id for each document
-    const links = {};
-    Object.keys(Metadata).forEach(id => {
-      const metadata = Metadata[id];
-      links[metadata.permalink] = id;
-    });
-
-    const mdToHtml = metadataUtils.mdToHtml(Metadata, siteConfig.baseUrl);
-
-    const metadata = Metadata[links[url]];
-    if (!metadata) {
+    const metadata =
+      Metadata[
+        Object.keys(Metadata).find(id => Metadata[id].permalink === url)
+      ];
+    const file = docs.getFile(metadata);
+    if (!file) {
       next();
       return;
     }
-    const language = metadata.language;
-
-    // determine what file to use according to its id
-    let file;
-    if (metadata.original_id) {
-      if (env.translation.enabled && metadata.language !== 'en') {
-        file = join(CWD, 'translated_docs', metadata.language, metadata.source);
-      } else {
-        file = join(CWD, 'versioned_docs', metadata.source);
-      }
-    } else if (!env.translation.enabled || metadata.language === 'en') {
-      file = join(CWD, '..', readMetadata.getDocsPath(), metadata.source);
-    } else {
-      file = join(CWD, 'translated_docs', metadata.language, metadata.source);
-    }
-
-    if (!fs.existsSync(file)) {
-      next();
-      return;
-    }
-
-    let rawContent = metadataUtils.extractMetadata(
-      fs.readFileSync(file, 'utf8')
-    ).rawContent;
-
-    // generate table of contents if appropriate
-    rawContent = insertTOC(rawContent);
-
-    const defaultVersion = env.versioning.defaultVersion;
-
-    // replace any links to markdown files to their website html links
-    Object.keys(mdToHtml).forEach(key => {
-      let link = mdToHtml[key];
-      link = getPath(link, siteConfig.cleanUrl);
-      link = link.replace('/en/', `/${language}/`);
-      link = link.replace(
-        '/VERSION/',
-        metadata.version && metadata.version !== defaultVersion
-          ? `/${metadata.version}/`
-          : '/'
-      );
-      // replace relative links with & without "./"
-      rawContent = rawContent.replace(
-        new RegExp(`\\]\\((${key}|\\./${key})`, 'g'),
-        `](${link}`
-      );
-    });
-
-    // replace any relative links to static assets to absolute links
-    rawContent = rawContent.replace(
-      /\]\(assets\//g,
-      `](${siteConfig.baseUrl}docs/assets/`
-    );
-
+    const rawContent = metadataUtils.extractMetadata(file).rawContent;
     removeModuleAndChildrenFromCache('../core/DocsLayout.js');
-    const DocsLayout = require('../core/DocsLayout.js');
-
-    let Doc;
-    if (
-      metadata.layout &&
-      siteConfig.layouts &&
-      siteConfig.layouts[metadata.layout]
-    ) {
-      Doc = siteConfig.layouts[metadata.layout]({
-        React,
-        MarkdownBlock: require('../core/MarkdownBlock.js'),
-      });
-    }
-
-    const docComp = (
-      <DocsLayout
-        metadata={metadata}
-        language={language}
-        config={siteConfig}
-        Doc={Doc}>
-        {rawContent}
-      </DocsLayout>
-    );
-
+    const mdToHtml = metadataUtils.mdToHtml(Metadata, siteConfig.baseUrl);
+    const docComp = docs.getComponent(rawContent, mdToHtml, metadata);
     res.send(renderToStaticMarkupWithDoctype(docComp));
   });
 

lib/version.js

@@ -7,6 +7,17 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+require('babel-register')({
+  babelrc: false,
+  only: [__dirname, `${process.cwd()}/core`],
+  plugins: [
+    require('./server/translate-plugin.js'),
+    'transform-class-properties',
+    'transform-object-rest-spread',
+  ],
+  presets: ['react', 'env'],
+});
+
 const program = require('commander');
 const chalk = require('chalk');
 const glob = require('glob');

lib/write-translations.js

@@ -11,6 +11,7 @@
 
 require('babel-register')({
   babelrc: false,
+  only: [__dirname, `${process.cwd()}/core`],
   plugins: [
     require('./server/translate-plugin.js'),
     'transform-class-properties',

package.json

@@ -39,6 +39,9 @@
       ]
     }
   },
+  "jest": {
+    "testPathIgnorePatterns": ["/node_modules/", "__fixtures__"]
+  },
   "bin": {
     "docusaurus-start": "./lib/start-server.js",
     "docusaurus-build": "./lib/build-files.js",