Refactor(server): share logic between server.js & generate.js (#856)

* nits too many comments

* Refactor to blog.getPages

* Refactor to getPost, fileToUrl, urlToFile

* Refactor redirectcomponent generation for docs

* nits & fix typo

* Refactor to blog.getMetadata

* Add test for getMetadata, fileToSUrl and urlToSource

* use includes() and add 'markup' naming for function

lib/server/__tests__/__fixtures__/2018-08-17-docusaurus.md

@@ -0,0 +1,11 @@
+---
+title: Docusaurus
+author: Endilie
+authorURL: https://github.com/endiliey
+authorFBID: 100000251103620
+authorTwitter: endiliey
+---
+
+![Docusaurus](/img/slash-introducing.png)
+
+We are very happy to introduce [Docusaurus](https://github.com/facebook/Docusaurus) to help you manage one or many open source websites.

lib/server/__tests__/__snapshots__/blog.test.js.snap

@@ -0,0 +1,17 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`getMetadata blog file 1`] = `
+Object {
+  "author": "Endilie",
+  "authorFBID": 100000251103620,
+  "authorTwitter": "endiliey",
+  "authorURL": "https://github.com/endiliey",
+  "content": "
+![Docusaurus](/img/slash-introducing.png)
+
+We are very happy to introduce [Docusaurus](https://github.com/facebook/Docusaurus) to help you manage one or many open source websites.",
+  "id": "Docusaurus",
+  "path": "2018/08/17/docusaurus.html",
+  "title": "Docusaurus",
+}
+`;

lib/server/__tests__/blog.test.js

@@ -0,0 +1,68 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+const path = require('path');
+const fs = require('fs-extra');
+const blog = require('../blog');
+
+const testFile = path.join(
+  __dirname,
+  '__fixtures__',
+  '2018-08-17-docusaurus.md'
+);
+
+fs.existsSync = jest.fn().mockReturnValue(true);
+
+describe('getMetadata', () => {
+  test('file does not exist', () => {
+    fs.existsSync.mockReturnValueOnce(null);
+    expect(blog.getMetadata('/this/path/does-not-exist/')).toBeNull();
+  });
+
+  test('null/undefined', () => {
+    expect(blog.getMetadata(null)).toBeNull();
+    expect(blog.getMetadata(undefined)).toBeNull();
+  });
+
+  test('blog file', () => {
+    const metadata = blog.getMetadata(testFile);
+    expect(metadata).toMatchSnapshot();
+    expect(metadata).not.toBeNull();
+    expect(metadata).toHaveProperty('id');
+    expect(metadata).toHaveProperty('path');
+    expect(metadata).toHaveProperty('content');
+  });
+});
+
+describe('fileToUrl', () => {
+  test('invalid file path', () => {
+    expect(blog.fileToUrl(null)).toBeNull();
+    expect(blog.fileToUrl(undefined)).toBeNull();
+    expect(blog.fileToUrl(true)).toBeNull();
+    fs.existsSync.mockReturnValueOnce(null);
+    expect(blog.fileToUrl('2018-03-02-this-does-not-exist.md')).toBeNull();
+  });
+
+  test('valid filepath', () => {
+    expect(blog.fileToUrl(testFile)).toEqual('2018/08/17/docusaurus.html');
+  });
+});
+
+describe('urlToSource', () => {
+  test('invalid url path', () => {
+    expect(blog.urlToSource(null)).toBeNull();
+    expect(blog.urlToSource(undefined)).toBeNull();
+    expect(blog.urlToSource(true)).toBeNull();
+  });
+  test('valid url path', () => {
+    expect(blog.urlToSource(`${blog.fileToUrl(testFile)}`)).toEqual(
+      '2018-08-17-docusaurus.md'
+    );
+    expect(blog.urlToSource('2018/03/04/test-name-lol.html')).toEqual(
+      '2018-03-04-test-name-lol.md'
+    );
+  });
+});

lib/server/blog.js

@@ -0,0 +1,86 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+const React = require('react');
+const path = require('path');
+const fs = require('fs-extra');
+const {renderToStaticMarkupWithDoctype} = require('./renderUtils');
+const metadataUtils = require('./metadataUtils');
+
+function urlToSource(url) {
+  if (!url || typeof url !== 'string') {
+    return null;
+  }
+  return url
+    .replace(/\/index.html$/, '.md')
+    .replace(/\.html$/, '.md')
+    .replace(new RegExp('/', 'g'), '-');
+}
+
+function fileToUrl(file) {
+  if (!file || !fs.existsSync(file) || typeof file !== 'string') {
+    return null;
+  }
+  return path
+    .basename(file)
+    .replace('-', '/')
+    .replace('-', '/')
+    .replace('-', '/')
+    .replace(/\.md$/, '.html');
+}
+
+function getPagesMarkup(numOfBlog, config) {
+  const BlogPageLayout = require('../core/BlogPageLayout.js');
+  const blogPages = {};
+  const perPage = 10;
+  for (let page = 0; page < Math.ceil(numOfBlog / perPage); page++) {
+    const metadata = {page, perPage};
+    const blogPageComp = (
+      <BlogPageLayout metadata={metadata} language="en" config={config} />
+    );
+    const str = renderToStaticMarkupWithDoctype(blogPageComp);
+    const pagePath = `${page > 0 ? `page${page + 1}` : ''}/index.html`;
+    blogPages[pagePath] = str;
+  }
+  return blogPages;
+}
+
+function getMetadata(file) {
+  if (!file || !fs.existsSync(file)) {
+    return null;
+  }
+  const result = metadataUtils.extractMetadata(
+    fs.readFileSync(file, {encoding: 'utf8'})
+  );
+  const metadata = Object.assign(
+    {path: fileToUrl(file), content: result.rawContent},
+    result.metadata
+  );
+  metadata.id = metadata.title;
+  return metadata;
+}
+
+function getPostMarkup(file, config) {
+  const metadata = getMetadata(file);
+  if (!metadata) {
+    return null;
+  }
+  const BlogPostLayout = require('../core/BlogPostLayout.js');
+  const blogPostComp = (
+    <BlogPostLayout metadata={metadata} language="en" config={config}>
+      {metadata.content}
+    </BlogPostLayout>
+  );
+  return renderToStaticMarkupWithDoctype(blogPostComp);
+}
+
+module.exports = {
+  fileToUrl,
+  getMetadata,
+  getPagesMarkup,
+  getPostMarkup,
+  urlToSource,
+};

lib/server/docs.js

@@ -10,6 +10,7 @@ const {join} = require('path');
 const fs = require('fs-extra');
 const React = require('react');
 const env = require('./env.js');
+const {renderToStaticMarkupWithDoctype} = require('./renderUtils');
 const readMetadata = require('./readMetadata.js');
 const {insertTOC} = require('../core/toc.js');
 const {getPath} = require('../core/utils.js');
@@ -82,7 +83,7 @@ function replaceAssetsLink(oldContent) {
   return lines.join('\n');
 }
 
-function getComponent(rawContent, mdToHtml, metadata) {
+function getMarkup(rawContent, mdToHtml, metadata) {
   // generate table of contents
   let content = insertTOC(rawContent);
 
@@ -93,7 +94,7 @@ function getComponent(rawContent, mdToHtml, metadata) {
   content = replaceAssetsLink(content);
 
   const DocsLayout = require('../core/DocsLayout.js');
-  return (
+  return renderToStaticMarkupWithDoctype(
     <DocsLayout
       metadata={metadata}
       language={metadata.language}
@@ -103,9 +104,26 @@ function getComponent(rawContent, mdToHtml, metadata) {
   );
 }
 
+function getRedirectMarkup(metadata) {
+  if (!env.translation.enabled || !metadata.permalink.includes('docs/en')) {
+    return null;
+  }
+  const Redirect = require('../core/Redirect.js');
+  const redirectlink = getPath(metadata.permalink, siteConfig.cleanUrl);
+  return renderToStaticMarkupWithDoctype(
+    <Redirect
+      metadata={metadata}
+      language={metadata.language}
+      config={siteConfig}
+      redirect={siteConfig.baseUrl + redirectlink}
+    />
+  );
+}
+
 module.exports = {
-  getComponent,
+  getMarkup,
   getFile,
+  getRedirectMarkup,
   mdToHtmlify,
   replaceAssetsLink,
 };

lib/server/generate.js

@@ -7,15 +7,13 @@
 
 async function execute() {
   require('../write-translations.js');
-
   const metadataUtils = require('./metadataUtils');
+  const blog = require('./blog');
   const docs = require('./docs');
-
   const CWD = process.cwd();
   const fs = require('fs-extra');
   const readMetadata = require('./readMetadata.js');
   const path = require('path');
-  const {getPath} = require('../core/utils.js');
   const {minifyCss, isSeparateCss, autoPrefixCss} = require('./utils');
   const React = require('react');
   const mkdirp = require('mkdirp');
@@ -55,11 +53,6 @@ async function execute() {
 
   console.log('generate.js triggered...');
 
-  // array of tags of enabled languages
-  const enabledLanguages = env.translation
-    .enabledLanguages()
-    .map(lang => lang.tag);
-
   readMetadata.generateMetadataDocs();
   const Metadata = require('../core/metadata.js');
 
@@ -72,14 +65,10 @@ async function execute() {
   // needed when the project's a GitHub org page
 
   const buildDir = join(CWD, 'build', siteConfig.projectName);
-
-  const mdToHtml = metadataUtils.mdToHtml(Metadata, siteConfig.baseUrl);
-
-  const Redirect = require('../core/Redirect.js');
-
   fs.removeSync(join(CWD, 'build'));
 
   // create html files for all docs by going through all doc ids
+  const mdToHtml = metadataUtils.mdToHtml(Metadata, siteConfig.baseUrl);
   Object.keys(Metadata).forEach(id => {
     const metadata = Metadata[id];
     const file = docs.getFile(metadata);
@@ -87,34 +76,20 @@ async function execute() {
       return;
     }
     const rawContent = metadataUtils.extractMetadata(file).rawContent;
-    const docComp = docs.getComponent(rawContent, mdToHtml, metadata);
+    const str = docs.getMarkup(rawContent, mdToHtml, metadata);
-    const str = renderToStaticMarkupWithDoctype(docComp);
     const targetFile = join(buildDir, metadata.permalink);
     writeFileAndCreateFolder(targetFile, str);
 
     // generate english page redirects when languages are enabled
-    if (
+    const redirectMarkup = docs.getRedirectMarkup(metadata);
-      env.translation.enabled &&
+    if (!redirectMarkup) {
-      metadata.permalink.indexOf('docs/en') !== -1
+      return;
-    ) {
-      const redirectlink = getPath(metadata.permalink, siteConfig.cleanUrl);
-      const redirectComp = (
-        <Redirect
-          metadata={metadata}
-          language={metadata.language}
-          config={siteConfig}
-          redirect={siteConfig.baseUrl + redirectlink}
-        />
-      );
-      const redirectStr = renderToStaticMarkupWithDoctype(redirectComp);
-
-      // create a redirects page for doc files
-      const redirectFile = join(
-        buildDir,
-        metadata.permalink.replace('docs/en', 'docs')
-      );
-      writeFileAndCreateFolder(redirectFile, redirectStr);
     }
+    const redirectFile = join(
+      buildDir,
+      metadata.permalink.replace('docs/en', 'docs')
+    );
+    writeFileAndCreateFolder(redirectFile, redirectMarkup);
   });
 
   // copy docs assets if they exist
@@ -131,7 +106,6 @@ async function execute() {
   }
   readMetadata.generateMetadataBlog();
   const MetadataBlog = require('../core/MetadataBlog.js');
-  const BlogPostLayout = require('../core/BlogPostLayout.js');
 
   let files = glob.sync(join(CWD, 'blog', '**', '*.*'));
   files
@@ -145,61 +119,22 @@ async function execute() {
       if (extension !== '.md' && extension !== '.markdown') {
         return;
       }
-
+      const urlPath = blog.fileToUrl(normalizedFile);
-      // convert filename to use slashes
+      const blogPost = blog.getPostMarkup(normalizedFile, siteConfig);
-      const filePath = path
+      if (!blogPost) {
-        .basename(normalizedFile)
+        return;
-        .replace('-', '/')
+      }
-        .replace('-', '/')
+      const targetFile = join(buildDir, 'blog', urlPath);
-        .replace('-', '/')
+      writeFileAndCreateFolder(targetFile, blogPost);
-        .replace(/\.md$/, '.html');
-      const result = metadataUtils.extractMetadata(
-        fs.readFileSync(normalizedFile, {encoding: 'utf8'})
-      );
-      const rawContent = result.rawContent;
-      const metadata = Object.assign(
-        {path: filePath, content: rawContent},
-        result.metadata
-      );
-      metadata.id = metadata.title;
-
-      const language = 'en';
-      const blogPostComp = (
-        <BlogPostLayout
-          metadata={metadata}
-          language={language}
-          config={siteConfig}>
-          {rawContent}
-        </BlogPostLayout>
-      );
-      const str = renderToStaticMarkupWithDoctype(blogPostComp);
-
-      const targetFile = join(buildDir, 'blog', filePath);
-      writeFileAndCreateFolder(targetFile, str);
     });
-  // create html files for all blog pages (collections of article previews)
-  const BlogPageLayout = require('../core/BlogPageLayout.js');
-  const perPage = 10;
-  for (let page = 0; page < Math.ceil(MetadataBlog.length / perPage); page++) {
-    const language = 'en';
-    const metadata = {page, perPage};
-    const blogPageComp = (
-      <BlogPageLayout
-        metadata={metadata}
-        language={language}
-        config={siteConfig}
-      />
-    );
-    const str = renderToStaticMarkupWithDoctype(blogPageComp);
 
-    const targetFile = join(
+  // create html files for all blog pages (collections of article previews)
-      buildDir,
+  const blogPages = blog.getPagesMarkup(MetadataBlog.length, siteConfig);
-      'blog',
+  Object.keys(blogPages).forEach(pagePath => {
-      page > 0 ? `page${page + 1}` : '',
+    const targetFile = join(buildDir, 'blog', pagePath);
-      'index.html'
+    writeFileAndCreateFolder(targetFile, blogPages[pagePath]);
-    );
+  });
-    writeFileAndCreateFolder(targetFile, str);
+
-  }
   // create rss files for all blog pages, if there are any blog files
   if (MetadataBlog.length > 0) {
     let targetFile = join(buildDir, 'blog', 'feed.xml');
@@ -344,6 +279,9 @@ async function execute() {
   fs.writeFileSync(mainCss, css);
 
   // compile/copy pages from user
+  const enabledLanguages = env.translation
+    .enabledLanguages()
+    .map(lang => lang.tag);
   files = glob.sync(join(CWD, 'pages', '**'));
   files.forEach(file => {
     // Why normalize? In case we are on Windows.

lib/server/readMetadata.js

@@ -14,6 +14,7 @@ const glob = require('glob');
 const metadataUtils = require('./metadataUtils');
 
 const env = require('./env.js');
+const blog = require('./blog.js');
 
 const siteConfig = require(`${CWD}/siteConfig.js`);
 const versionFallback = require('./versionFallback.js');
@@ -313,27 +314,7 @@ function generateMetadataBlog() {
       if (extension !== '.md' && extension !== '.markdown') {
         return;
       }
-      // Transform
+      const metadata = blog.getMetadata(file);
-      //   2015-08-13-blog-post-name-0.5.md
-      // into
-      //   2015/08/13/blog-post-name-0-5.html
-      const filePath = path
-        .basename(file)
-        .replace('-', '/')
-        .replace('-', '/')
-        .replace('-', '/')
-        .replace(/\.md$/, '.html');
-      const result = metadataUtils.extractMetadata(
-        fs.readFileSync(file, {encoding: 'utf8'})
-      );
-      const rawContent = result.rawContent;
-      const metadata = Object.assign(
-        {path: filePath, content: rawContent},
-        result.metadata
-      );
-
-      metadata.id = metadata.title;
-
       // Extract, YYYY, MM, DD from the file name
       const filePathDateArr = path
         .basename(file)

lib/server/server.js

@@ -9,10 +9,9 @@
 
 function execute(port, options) {
   const extractTranslations = require('../write-translations');
-
   const metadataUtils = require('./metadataUtils');
+  const blog = require('./blog');
   const docs = require('./docs');
-
   const env = require('./env.js');
   const express = require('express');
   const React = require('react');
@@ -25,17 +24,13 @@ function execute(port, options) {
   const chalk = require('chalk');
   const gaze = require('gaze');
   const tinylr = require('tiny-lr');
-
   const constants = require('../core/constants');
   const translate = require('./translate');
   const {renderToStaticMarkupWithDoctype} = require('./renderUtils');
-
   const feed = require('./feed');
   const sitemap = require('./sitemap');
   const routing = require('./routing');
-
   const CWD = process.cwd();
-
   const join = path.join;
   const sep = path.sep;
 
@@ -46,7 +41,6 @@ function execute(port, options) {
         delete module.constructor._pathCache[cacheKey];
       }
     });
-    /* eslint-enable no-underscore-dangle */
   }
 
   // Remove a module and child modules from require cache, so server
@@ -112,7 +106,6 @@ function execute(port, options) {
   }
 
   function startLiveReload() {
-    // Start LiveReload server.
     process.env.NODE_ENV = 'development';
     const server = tinylr();
     server.listen(constants.LIVE_RELOAD_PORT, () => {
@@ -122,19 +115,10 @@ function execute(port, options) {
       );
     });
 
-    // gaze watches some specified dirs and triggers a callback when they change.
     gaze(
-      [
+      [`../${readMetadata.getDocsPath()}/**/*`, '**/*', '!node_modules/**/*'],
-        `../${readMetadata.getDocsPath()}/**/*`, // docs
-        '**/*', // website
-        '!node_modules/**/*', // node_modules
-      ],
       function() {
-        // Listen for all kinds of file changes - modified/added/deleted.
         this.on('all', () => {
-          // Notify LiveReload clients that there's a change.
-          // Typically, LiveReload will only refresh the changed paths,
-          // so we use / here to force a full-page reload.
           server.notifyClients(['/']);
         });
       }
@@ -148,7 +132,6 @@ function execute(port, options) {
 
   const app = express();
 
-  // handle all requests for document pages
   app.get(routing.docs(siteConfig.baseUrl), (req, res, next) => {
     const url = req.path.toString().replace(siteConfig.baseUrl, '');
     const metadata =
@@ -163,8 +146,7 @@ function execute(port, options) {
     const rawContent = metadataUtils.extractMetadata(file).rawContent;
     removeModuleAndChildrenFromCache('../core/DocsLayout.js');
     const mdToHtml = metadataUtils.mdToHtml(Metadata, siteConfig.baseUrl);
-    const docComp = docs.getComponent(rawContent, mdToHtml, metadata);
+    res.send(docs.getMarkup(rawContent, mdToHtml, metadata));
-    res.send(renderToStaticMarkupWithDoctype(docComp));
   });
 
   app.get(routing.sitemap(siteConfig.baseUrl), (req, res) => {
@@ -192,97 +174,32 @@ function execute(port, options) {
     next();
   });
 
-  // Handle all requests for blog pages and posts.
   app.get(routing.blog(siteConfig.baseUrl), (req, res, next) => {
     // Regenerate the blog metadata in case it has changed. Consider improving
     // this to regenerate on file save rather than on page request.
     reloadMetadataBlog();
-    // Generate all of the blog pages.
     removeModuleAndChildrenFromCache(join('..', 'core', 'BlogPageLayout.js'));
-    const BlogPageLayout = require(join('..', 'core', 'BlogPageLayout.js'));
+    const blogPages = blog.getPagesMarkup(MetadataBlog.length, siteConfig);
-    const blogPages = {};
+    const urlPath = req.path.toString().split('blog/')[1];
-    // Make blog pages with 10 posts per page.
-    const perPage = 10;
-    for (
-      let page = 0;
-      page < Math.ceil(MetadataBlog.length / perPage);
-      page++
-    ) {
-      const language = 'en';
-      const metadata = {page, perPage};
-      const blogPageComp = (
-        <BlogPageLayout
-          metadata={metadata}
-          language={language}
-          config={siteConfig}
-        />
-      );
-      const str = renderToStaticMarkupWithDoctype(blogPageComp);
 
-      const pagePath = `${page > 0 ? `page${page + 1}` : ''}/index.html`;
+    if (urlPath === 'index.html') {
-      blogPages[pagePath] = str;
-    }
-
-    const parts = req.path.toString().split('blog/');
-    // send corresponding blog page if appropriate
-    if (parts[1] === 'index.html') {
       res.send(blogPages['/index.html']);
-    } else if (parts[1].endsWith('/index.html') && blogPages[parts[1]]) {
+    } else if (urlPath.endsWith('/index.html') && blogPages[urlPath]) {
-      res.send(blogPages[parts[1]]);
+      res.send(blogPages[urlPath]);
-    } else if (parts[1].match(/page([0-9]+)/)) {
+    } else if (urlPath.match(/page([0-9]+)/)) {
-      if (parts[1].endsWith('/')) {
+      res.send(blogPages[`${urlPath.replace(/\/$/, '')}/index.html`]);
-        res.send(blogPages[`${parts[1]}index.html`]);
-      } else {
-        res.send(blogPages[`${parts[1]}/index.html`]);
-      }
     } else {
-      // send corresponding blog post. Ex: request to "blog/test/index.html" or
+      const file = join(CWD, 'blog', blog.urlToSource(urlPath));
-      // "blog/test.html" will return html rendered version of "blog/test.md"
+      removeModuleAndChildrenFromCache(join('..', 'core', 'BlogPostLayout.js'));
-      let file = parts[1];
+      const blogPost = blog.getPostMarkup(file, siteConfig);
-      if (file.endsWith('/index.html')) {
+      if (!blogPost) {
-        file = file.replace(/\/index.html$/, '.md');
-      } else {
-        file = file.replace(/\.html$/, '.md');
-      }
-      file = file.replace(new RegExp('/', 'g'), '-');
-      file = join(CWD, 'blog', file);
-
-      if (!fs.existsSync(file)) {
         next();
         return;
       }
-
+      res.send(blogPost);
-      const result = metadataUtils.extractMetadata(
-        fs.readFileSync(file, {encoding: 'utf8'})
-      );
-      let rawContent = result.rawContent;
-      rawContent = rawContent.replace(
-        /\]\(assets\//g,
-        `](${siteConfig.baseUrl}blog/assets/`
-      );
-      const metadata = Object.assign(
-        {path: req.path.toString().split('blog/')[1], content: rawContent},
-        result.metadata
-      );
-      metadata.id = metadata.title;
-
-      const language = 'en';
-      removeModuleAndChildrenFromCache(join('..', 'core', 'BlogPostLayout.js'));
-      const BlogPostLayout = require(join('..', 'core', 'BlogPostLayout.js'));
-
-      const blogPostComp = (
-        <BlogPostLayout
-          metadata={metadata}
-          language={language}
-          config={siteConfig}>
-          {rawContent}
-        </BlogPostLayout>
-      );
-      res.send(renderToStaticMarkupWithDoctype(blogPostComp));
     }
   });
 
-  // handle all other main pages
   app.get(routing.page(siteConfig.baseUrl), (req, res, next) => {
     // Look for user-provided HTML file first.
     let htmlFile = req.path.toString().replace(siteConfig.baseUrl, '');
@@ -390,7 +307,6 @@ function execute(port, options) {
     }
   });
 
-  // generate the main.css file by concatenating user provided css to the end
   app.get(/main\.css$/, (req, res) => {
     const mainCssPath = join(
       __dirname,
@@ -473,7 +389,7 @@ function execute(port, options) {
   });
 
   // handle special cleanUrl case like '/blog/1.2.3' & '/blog.robots.hai'
-  // where we should try to serve 'blog/1.2.3.html' & '/blog.robots.hai.html'
+  // where we should try to serve '/blog/1.2.3.html' & '/blog.robots.hai.html'
   app.get(routing.dotfiles(), (req, res, next) => {
     if (!siteConfig.cleanUrl) {
       next();